From c5c8e5b7bdc7c8f50f2a5ad31219bb287bd6dd1c Mon Sep 17 00:00:00 2001 From: Adam Shannon Date: Wed, 8 Jan 2025 17:36:21 -0600 Subject: [PATCH 1/9] cmd/ui: initial concept for a cross-platform (and web) v2 UI --- cmd/ui/main.go | 71 +++ go.mod | 27 ++ go.sum | 439 ++++++++++++++++++ internal/ast/variables.go | 85 ++++ internal/search/api_search.go | 10 +- internal/search/service.go | 22 +- internal/search/service_test.go | 3 + internal/ui/search.go | 178 +++++++ internal/ui/ui.go | 51 ++ pkg/search/client.go | 55 +++ .../search/model_searched_entity.go | 6 +- .../search/model_searched_entity_test.go | 4 +- pkg/search/models.go | 21 +- 13 files changed, 941 insertions(+), 31 deletions(-) create mode 100644 cmd/ui/main.go create mode 100644 internal/ast/variables.go create mode 100644 internal/ui/search.go create mode 100644 internal/ui/ui.go create mode 100644 pkg/search/client.go rename {internal => pkg}/search/model_searched_entity.go (54%) rename {internal => pkg}/search/model_searched_entity_test.go (92%) diff --git a/cmd/ui/main.go b/cmd/ui/main.go new file mode 100644 index 00000000..6ebccab4 --- /dev/null +++ b/cmd/ui/main.go @@ -0,0 +1,71 @@ +package main + +import ( + "cmp" + "fmt" + "net/http" + "os" + + "github.com/moov-io/watchman/internal/ui" + "github.com/moov-io/watchman/pkg/search" + + "github.com/urfave/cli/v2" +) + +var ( + flagBaseAddress = &cli.StringFlag{ + Name: "address", + Value: cmp.Or(os.Getenv("WATCHMAN_ADDRESS"), "http://localhost:8084"), + Usage: "Address to connect with Watchman", + } + flagVerbose = &cli.BoolFlag{ + Name: "verbose", + Value: false, + Usage: "Output verbose logging", + } +) + +func main() { + app := &cli.App{ + Name: "watchman-ui", + // UsageText: "watchman-ui [global options] command [command options]", + Description: "Watchman GUI", + Authors: []*cli.Author{ + {Name: "Moov OSS", Email: "oss@moov.io"}, + }, + Flags: []cli.Flag{ + // Common Flags + flagBaseAddress, flagVerbose, + }, + Commands: []*cli.Command{ + // commandFind, + }, + Action: func(ctx *cli.Context) error { + env := ui.Environment{ + Client: createWatchmanClient(ctx.String(flagBaseAddress.Name)), + } + + // cli.ShowAppHelp(ctx) + + return showUI(ctx, env) + }, + } + + if err := app.Run(os.Args); err != nil { + fmt.Printf("ERROR running command: %v\n", err) + os.Exit(127) + } +} + +func showUI(ctx *cli.Context, env ui.Environment) error { + app := ui.New(env) + app.Run() + + return nil +} + +func createWatchmanClient(baseAddress string) search.Client { + var httpClient *http.Client = nil + + return search.NewClient(httpClient, baseAddress) +} diff --git a/go.mod b/go.mod index c44ab107..848cbb65 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/moov-io/watchman go 1.20 require ( + fyne.io/fyne v1.4.3 + fyne.io/fyne/v2 v2.5.3 github.com/abadojack/whatlanggo v1.0.1 github.com/antchfx/htmlquery v1.3.3 github.com/antihax/optional v1.0.0 @@ -16,6 +18,7 @@ require ( github.com/pariz/gountries v0.1.6 github.com/prometheus/client_golang v1.17.0 github.com/stretchr/testify v1.8.4 + github.com/urfave/cli/v2 v2.4.0 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 go4.org v0.0.0-20230225012048-214862532bf5 golang.org/x/oauth2 v0.14.0 @@ -24,20 +27,44 @@ require ( ) require ( + fyne.io/systray v1.11.0 // indirect + github.com/BurntSushi/toml v1.4.0 // indirect github.com/antchfx/xpath v1.3.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/fredbi/uri v1.1.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fyne-io/gl-js v0.0.0-20220119005834-d2da28d9ccfe // indirect + github.com/fyne-io/glfw-js v0.0.0-20241126112943-313d8a0fe1d0 // indirect + github.com/fyne-io/image v0.0.0-20220602074514-4956b0afb3d2 // indirect + github.com/go-gl/gl v0.0.0-20211210172815-726fda9656d6 // indirect + github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect + github.com/go-text/render v0.2.0 // indirect + github.com/go-text/typesetting v0.2.0 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/gopherjs/gopherjs v1.17.2 // indirect + github.com/jeandeaual/go-locale v0.0.0-20240223122105-ce5225dcaa49 // indirect + github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/nicksnyder/go-i18n/v2 v2.4.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/rickar/cal/v2 v2.1.13 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/rymdport/portal v0.3.0 // indirect + github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c // indirect + github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef // indirect + github.com/yuin/goldmark v1.7.1 // indirect + golang.org/x/image v0.18.0 // indirect + golang.org/x/mobile v0.0.0-20231127183840-76ac6878050a // indirect golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/go.sum b/go.sum index 7cfc28ca..7e35fa66 100644 --- a/go.sum +++ b/go.sum @@ -6,30 +6,67 @@ cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxK cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +fyne.io/fyne v1.4.3 h1:356CnXCiYrrfaLGsB7qLK3c6ktzyh8WR05v/2RBu51I= +fyne.io/fyne v1.4.3/go.mod h1:8kiPBNSDmuplxs9WnKCkaWYqbcXFy0DeAzwa6PBO9Z8= +fyne.io/fyne/v2 v2.5.3 h1:k6LjZx6EzRZhClsuzy6vucLZBstdH2USDGHSGWq8ly8= +fyne.io/fyne/v2 v2.5.3/go.mod h1:0GOXKqyvNwk3DLmsFu9v0oYM0ZcD1ysGnlHCerKoAmo= +fyne.io/systray v1.11.0 h1:D9HISlxSkx+jHSniMBR6fCFOUjk1x/OOOJLa9lJYAKg= +fyne.io/systray v1.11.0/go.mod h1:RVwqP9nYMo7h5zViCBHri2FgjXF7H2cub7MAq4NSoLs= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Kodeworks/golang-image-ico v0.0.0-20141118225523-73f0f4cfade9/go.mod h1:7uhhqiBaR4CpN0k9rMjOtjpcfGd6DG2m04zQxKnWQ0I= github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4= github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc= +github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/antchfx/htmlquery v1.3.3 h1:x6tVzrRhVNfECDaVxnZi1mEGrQg3mjE/rxbH2Pe6dNE= github.com/antchfx/htmlquery v1.3.3/go.mod h1:WeU3N7/rL6mb6dCwtE30dURBnBieKDC/fR8t6X+cKjU= github.com/antchfx/xpath v1.3.2 h1:LNjzlsSjinu3bQpw9hWMY9ocB80oLOWuQqFvO6xt51U= github.com/antchfx/xpath v1.3.2/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/bbalet/stopwords v1.0.0 h1:0TnGycCtY0zZi4ltKoOGRFIlZHv0WqpoIGUsObjztfo= github.com/bbalet/stopwords v1.0.0/go.mod h1:sAWrQoDMfqARGIn4s6dp7OW7ISrshUD8IP2q3KoqPjc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -37,19 +74,67 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= +github.com/fredbi/uri v1.1.0 h1:OqLpTXtyRg9ABReqvDGdJPqZUxs8cyBDOMXBbskCaB8= +github.com/fredbi/uri v1.1.0/go.mod h1:aYTUoAXBOq7BLfVJ8GnKmfcuURosB1xyHDIfWeC/iW4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fyne-io/gl-js v0.0.0-20220119005834-d2da28d9ccfe h1:A/wiwvQ0CAjPkuJytaD+SsXkPU0asQ+guQEIg1BJGX4= +github.com/fyne-io/gl-js v0.0.0-20220119005834-d2da28d9ccfe/go.mod h1:d4clgH0/GrRwWjRzJJQXxT/h1TyuNSfF/X64zb/3Ggg= +github.com/fyne-io/glfw-js v0.0.0-20241126112943-313d8a0fe1d0 h1:/1YRWFv9bAWkoo3SuxpFfzpXH0D/bQnTjNXyF4ih7Os= +github.com/fyne-io/glfw-js v0.0.0-20241126112943-313d8a0fe1d0/go.mod h1:gsGA2dotD4v0SR6PmPCYvS9JuOeMwAtmfvDE7mbYXMY= +github.com/fyne-io/image v0.0.0-20220602074514-4956b0afb3d2 h1:hnLq+55b7Zh7/2IRzWCpiTcAvjv/P8ERF+N7+xXbZhk= +github.com/fyne-io/image v0.0.0-20220602074514-4956b0afb3d2/go.mod h1:eO7W361vmlPOrykIg+Rsh1SZ3tQBaOsfzZhsIOb/Lm0= +github.com/fyne-io/mobile v0.1.2/go.mod h1:/kOrWrZB6sasLbEy2JIvr4arEzQTXBTZGb3Y96yWbHY= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/gl v0.0.0-20190320180904-bf2b1f2f34d7/go.mod h1:482civXOzJJCPzJ4ZOX/pwvXBWSnzD4OKMdH4ClKGbk= +github.com/go-gl/gl v0.0.0-20211210172815-726fda9656d6 h1:zDw5v7qm4yH7N8C8uWd+8Ii9rROdgWxQuGoJ9WDXxfk= +github.com/go-gl/gl v0.0.0-20211210172815-726fda9656d6/go.mod h1:9YTyiznxEY1fVinfM7RvRcjRHbw2xLBJ3AAGIT0I4Nw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200625191551-73d3c3675aa3/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a h1:vxnBhFDDT+xzxf1jTJKMKZw3H0swfWk9RpWbBbDK5+0= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.13.0 h1:OoneCcHKHQ03LfBpoQCUfCluwd2Vt3ohz+kvbJneZAU= github.com/go-kit/kit v0.13.0/go.mod h1:phqEHMMUbyrCFCTgH48JueqrM3md2HcAZ8N3XE4FKDg= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-text/render v0.2.0 h1:LBYoTmp5jYiJ4NPqDc2pz17MLmA3wHw1dZSVGcOdeAc= +github.com/go-text/render v0.2.0/go.mod h1:CkiqfukRGKJA5vZZISkjSYrcdtgKQWRa2HIzvwNN5SU= +github.com/go-text/typesetting v0.2.0 h1:fbzsgbmk04KiWtE+c3ZD4W2nmCRzBqrqQOvYlwAOdho= +github.com/go-text/typesetting v0.2.0/go.mod h1:2+owI/sxa73XA581LAzVuEBZ3WEEV2pXeDswCH/3i1I= +github.com/go-text/typesetting-utils v0.0.0-20240317173224-1986cbe96c66 h1:GUrm65PQPlhFSKjLPGOZNPNxLCybjzjYBzjfoBGaDUY= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/goki/freetype v0.0.0-20181231101311-fa8a33aabaff h1:W71vTCKoxtdXgnm1ECDFkfQnpdqAO00zzGXLA5yaEX8= +github.com/goki/freetype v0.0.0-20181231101311-fa8a33aabaff/go.mod h1:wfqRWLHRBsRgkp5dmbG56SA0DmVtwrF5N3oPdI8t+Aw= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -60,11 +145,26 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= @@ -74,44 +174,130 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20211219123610-ec9572f70e60/go.mod h1:cz9oNYuRUWGdHmLF2IodMLkAhcPtXeULvcBNagUrxTI= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/goxjs/gl v0.0.0-20210104184919-e3fafc6f8f2a/go.mod h1:dy/f2gjY09hwVfIyATps4G2ai7/hLwLkc5TrPqONuXY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jackmordaunt/icns v0.0.0-20181231085925-4f16af745526/go.mod h1:UQkeMHVoNcyXYq9otUupF7/h/2tmHlhrS2zw7ZVvUqc= github.com/jaswdr/faker v1.19.1 h1:xBoz8/O6r0QAR8eEvKJZMdofxiRH+F0M/7MU9eNKhsM= github.com/jaswdr/faker v1.19.1/go.mod h1:x7ZlyB1AZqwqKZgyQlnqEG8FDptmHlncA5u2zY/yi6w= +github.com/jeandeaual/go-locale v0.0.0-20240223122105-ce5225dcaa49 h1:Po+wkNdMmN+Zj1tDsJQy7mJlPlwGNQd9JZoPjObagf8= +github.com/jeandeaual/go-locale v0.0.0-20240223122105-ce5225dcaa49/go.mod h1:YiutDnxPRLk5DLUFj6Rw4pRBBURZY07GFr54NdV9mQg= +github.com/josephspurrier/goversioninfo v0.0.0-20200309025242-14b0ab84c6ca/go.mod h1:eJTEwMjXb7kZ633hO3Ln9mBUCOjX2+FlTljvpl9SYdE= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e h1:LvL4XsI70QxOGHed6yhQtAU34Kx3Qq2wwBzGFKY8zKk= +github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/knieriem/odf v0.1.0 h1:9nas0pxrk9EfhD7PouL9RawIaPfETwCnxKCqMjwsjHA= github.com/knieriem/odf v0.1.0/go.mod h1:jRlg9+5Aya1ajQBX2ltU//o50Kn+cApfrsnkLCBjzJA= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lucor/goinfo v0.0.0-20200401173949-526b5363a13a/go.mod h1:ORP3/rB5IsulLEBwQZCJyyV6niqmI7P4EWSmkug+1Ng= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/moov-io/base v0.48.2 h1:BPSNgmwokOVaVzAMJg71L48LCrDYelMfVXJEiZb2zOY= github.com/moov-io/base v0.48.2/go.mod h1:u1/WC3quR6otC9NrM1TtXSwNti1A/m7MR49RIXY1ee4= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= +github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/nicksnyder/go-i18n/v2 v2.4.0 h1:3IcvPOAvnCKwNm0TB0dLDTuawWEj+ax/RERNC+diLMM= +github.com/nicksnyder/go-i18n/v2 v2.4.0/go.mod h1:nxYSZE9M0bf3Y70gPQjN9ha7XNHX7gMc814+6wVyEI4= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/openvenues/gopostal v0.0.0-20240426055609-4fe3a773f519 h1:xZ0ZhxCnrs2zaBBvGIHQqzoeXjzctJP61r+aX3QjXhQ= github.com/openvenues/gopostal v0.0.0-20240426055609-4fe3a773f519/go.mod h1:Ycrd7XnwQdumHzpB/6WEa85B4WNdbLC6Wz4FAQNkaV0= github.com/pariz/gountries v0.1.6 h1:Cu8sBSvD6HvAtzinKJ7Yw8q4wAF2dD7oXjA5yDJQt1I= github.com/pariz/gountries v0.1.6/go.mod h1:Et5QWMc75++5nUKSYKNtz/uc+2LHl4LKhNd6zwdTu+0= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -123,37 +309,101 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rickar/cal/v2 v2.1.13 h1:FENBPXxDPyL1OWGf9ZdpWGcEiGoSjt0UZED8VOxvK0c= github.com/rickar/cal/v2 v2.1.13/go.mod h1:/fdlMcx7GjPlIBibMzOM9gMvDBsrK+mOtRXdTzUqV/A= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/rymdport/portal v0.3.0 h1:QRHcwKwx3kY5JTQcsVhmhC3TGqGQb9LFghVNUy8AdB8= +github.com/rymdport/portal v0.3.0/go.mod h1:kFF4jslnJ8pD5uCi17brj/ODlfIidOxlgUDTO5ncnC4= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/srwiley/oksvg v0.0.0-20200311192757-870daf9aa564/go.mod h1:afMbS0qvv1m5tfENCwnOdZGOF8RGR/FsZ7bvBxQGZG4= +github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c h1:km8GpoQut05eY3GiYWEedbTT0qnSxrCjsVbb7yKY1KE= +github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c/go.mod h1:cNQ3dwVJtS5Hmnjxy6AgTPd0Inb3pW05ftPSX7NZO7Q= +github.com/srwiley/rasterx v0.0.0-20200120212402-85cb7272f5e9/go.mod h1:mvWM0+15UqyrFKqdRjY6LuAVJR0HOVhJlEgZ5JWtSWU= +github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef h1:Ch6Q+AZUxDBCVqdkI8FSpFyZDtCVBc2VmejdNrm5rRQ= +github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef/go.mod h1:nXTWP6+gD5+LUJ8krVhhoeHjvHTutPxMYl5SvkcnJNE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/urfave/cli/v2 v2.4.0 h1:m2pxjjDFgDxSPtO8WSdbndj17Wu2y8vOT86wE/tjr+I= +github.com/urfave/cli/v2 v2.4.0/go.mod h1:NX9W0zmTvedE5oDoOMs2RTC8RvdK98NTYZE5LbaEYPg= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.7.1 h1:3bajkSilaCbjdKVsKdZjZCLBNPL9pYzrCakKaf4U49U= +github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= +golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -163,15 +413,28 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mobile v0.0.0-20211207041440-4e6c2922fdee/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= +golang.org/x/mobile v0.0.0-20231127183840-76ac6878050a h1:sYbmY3FwUWCBTodZL1S3JUuOvaW6kM2o+clDzzDNBWg= +golang.org/x/mobile v0.0.0-20231127183840-76ac6878050a/go.mod h1:Ede7gF0KGoHlj822RtphAHK1jLdrcuRBZg0sF1Q+SPc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -180,10 +443,31 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= @@ -193,6 +477,13 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.14.0 h1:P0Vrf/2538nmC0H+pEQ3MNFRRnVR7RlqyVw+bvm26z0= golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -201,10 +492,17 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -213,11 +511,41 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -231,6 +559,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= @@ -238,34 +569,67 @@ golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190808195139-e713427fea3f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200328031815-3db5fc6bac03/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -274,11 +638,27 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -293,25 +673,82 @@ google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -319,6 +756,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/ast/variables.go b/internal/ast/variables.go new file mode 100644 index 00000000..3fa77a42 --- /dev/null +++ b/internal/ast/variables.go @@ -0,0 +1,85 @@ +package ast + +import ( + "cmp" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "slices" + "strconv" +) + +// ExtractVariablesOfType parses a Go source file and finds all variables of the specified type. +func ExtractVariablesOfType(path, typeName string) ([]string, error) { + src, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read file: %w", err) + } + + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, path, src, parser.AllErrors) + if err != nil { + return nil, fmt.Errorf("failed to parse file: %w", err) + } + + var values []string + + // Walk the AST to find variables of the specified type + ast.Inspect(node, func(n ast.Node) bool { + // Look for variable declarations + decl, ok := n.(*ast.GenDecl) + if !ok || decl.Tok != token.VAR { + return true + } + + // Process each variable in the declaration + for _, spec := range decl.Specs { + vspec, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + + // Check the type of the variable + if vspec.Type != nil { + ident, ok := vspec.Type.(*ast.Ident) + if ok && ident.Name == typeName { + // Add all variable names in this declaration to the result + for i := range vspec.Names { + if i < len(vspec.Values) { + values = append(values, formatNode(vspec.Values[i])) + } + } + } + } + } + return true + }) + + slices.Sort(values) + + return values, nil +} + +func formatNode(expr ast.Expr) string { + if expr == nil { + return "" + } + + switch v := expr.(type) { + case *ast.BasicLit: // Literal values like numbers or strings + value, _ := strconv.Unquote(v.Value) + return cmp.Or(value, v.Value) + + case *ast.Ident: // Identifiers (e.g., constants or variables) + value, _ := strconv.Unquote(v.Name) + return cmp.Or(value, v.Name) + + case *ast.CompositeLit: // Composite literals like structs or arrays + return "composite literal" + + default: + return "complex expression" + } +} diff --git a/internal/search/api_search.go b/internal/search/api_search.go index 76c30ed5..40f0de3e 100644 --- a/internal/search/api_search.go +++ b/internal/search/api_search.go @@ -44,7 +44,7 @@ func (c *controller) AppendRoutes(router *mux.Router) *mux.Router { } type searchResponse struct { - Entities []SearchedEntity[search.Value] `json:"entities"` + Entities []search.SearchedEntity[search.Value] `json:"entities"` } type errorResponse struct { @@ -66,9 +66,12 @@ func (c *controller) search(w http.ResponseWriter, r *http.Request) { return } + q := r.URL.Query() opts := SearchOpts{ - Limit: extractSearchLimit(r), - MinMatch: extractSearchMinMatch(r), + Limit: extractSearchLimit(r), + MinMatch: extractSearchMinMatch(r), + RequestID: q.Get("requestID"), + DebugSourceIDs: strings.Split(q.Get("debugSourceIDs"), ","), } entities, err := c.service.Search(r.Context(), req, opts) @@ -120,7 +123,6 @@ func readSearchRequest(r *http.Request) (search.Entity[search.Value], error) { req.Name = strings.TrimSpace(q.Get("name")) req.Type = search.EntityType(strings.TrimSpace(strings.ToLower(q.Get("type")))) req.Source = search.SourceAPIRequest - req.SourceID = strings.TrimSpace(q.Get("requestID")) switch req.Type { case search.EntityPerson: diff --git a/internal/search/service.go b/internal/search/service.go index dd52e07a..10963bca 100644 --- a/internal/search/service.go +++ b/internal/search/service.go @@ -3,6 +3,7 @@ package search import ( "context" "fmt" + "slices" "sync" "github.com/moov-io/watchman/internal/largest" @@ -13,7 +14,7 @@ import ( ) type Service interface { - Search(ctx context.Context, query search.Entity[search.Value], opts SearchOpts) ([]SearchedEntity[search.Value], error) + Search(ctx context.Context, query search.Entity[search.Value], opts SearchOpts) ([]search.SearchedEntity[search.Value], error) } func NewService(logger log.Logger, entities []search.Entity[search.Value]) Service { @@ -36,7 +37,7 @@ type service struct { *syncutil.Gate // limits concurrent processing } -func (s *service) Search(ctx context.Context, query search.Entity[search.Value], opts SearchOpts) ([]SearchedEntity[search.Value], error) { +func (s *service) Search(ctx context.Context, query search.Entity[search.Value], opts SearchOpts) ([]search.SearchedEntity[search.Value], error) { // Grab a read-lock over our data s.RLock() defer s.RUnlock() @@ -55,9 +56,12 @@ func (s *service) Search(ctx context.Context, query search.Entity[search.Value], type SearchOpts struct { Limit int MinMatch float64 + + RequestID string + DebugSourceIDs []string } -func (s *service) performSearch(ctx context.Context, query search.Entity[search.Value], opts SearchOpts) ([]SearchedEntity[search.Value], error) { +func (s *service) performSearch(ctx context.Context, query search.Entity[search.Value], opts SearchOpts) ([]search.SearchedEntity[search.Value], error) { items := largest.NewItems(opts.Limit, opts.MinMatch) indices := makeIndices(len(s.entities), opts.Limit/3) // limit goroutines @@ -79,13 +83,13 @@ func (s *service) performSearch(ctx context.Context, query search.Entity[search. go func() { defer wg.Done() - performSubSearch(items, query, s.entities[indices[start]:end]) + performSubSearch(items, query, s.entities[indices[start]:end], opts) }() } wg.Wait() results := items.Items() - var out []SearchedEntity[search.Value] + var out []search.SearchedEntity[search.Value] for _, entity := range results { if entity == nil || entity.Value == nil { @@ -96,8 +100,8 @@ func (s *service) performSearch(ctx context.Context, query search.Entity[search. continue } - out = append(out, SearchedEntity[search.Value]{ - Entity: entity.Value.(search.Entity[search.Value]), + out = append(out, search.SearchedEntity[search.Value]{ + Entity: entity.Value.(search.Entity[search.Value]), // TODO(adam): Match: entity.Weight, }) } @@ -105,11 +109,11 @@ func (s *service) performSearch(ctx context.Context, query search.Entity[search. return out, nil } -func performSubSearch(items *largest.Items, query search.Entity[search.Value], entities []search.Entity[search.Value]) { +func performSubSearch(items *largest.Items, query search.Entity[search.Value], entities []search.Entity[search.Value], opts SearchOpts) { for _, entity := range entities { score := search.DebugSimilarity(nil, query, entity) - if entity.Name == "HYDRA MARKET" { + if slices.Contains(opts.DebugSourceIDs, entity.SourceID) { fmt.Printf("%#v\n", entity) fmt.Println("") fmt.Printf("%#v\n", entity.SourceData) diff --git a/internal/search/service_test.go b/internal/search/service_test.go index 091e9719..7a1f2f34 100644 --- a/internal/search/service_test.go +++ b/internal/search/service_test.go @@ -63,6 +63,9 @@ func TestService_Search(t *testing.T) { t.Logf("got %d results", len(results)) t.Logf("") t.Logf("%#v", results[0]) + + res := results[0] + require.InDelta(t, 1.00, res.Match, 0.001) }) } diff --git a/internal/ui/search.go b/internal/ui/search.go new file mode 100644 index 00000000..96617474 --- /dev/null +++ b/internal/ui/search.go @@ -0,0 +1,178 @@ +package ui + +import ( + "fmt" + "path/filepath" + + "github.com/moov-io/watchman/internal/ast" + "github.com/moov-io/watchman/pkg/search" + + "fyne.io/fyne/v2" + "fyne.io/fyne/v2/container" + "fyne.io/fyne/v2/layout" + "fyne.io/fyne/v2/widget" +) + +func SearchContainer(env Environment) fyne.CanvasObject { + wrapper := fyne.NewContainerWithLayout(layout.NewVBoxLayout()) + + warning := container.NewVBox() + warning.Hide() + + results := container.NewVBox() + results.Hide() + + form := searchForm(env, warning, results) + wrapper.Add(form) + wrapper.Add(results) + + return wrapper +} + +func searchForm(env Environment, warning *fyne.Container, results *fyne.Container) *widget.Form { + warning.Hide() + + blankSpace := widget.NewLabel(" ") + + items := []*widget.FormItem{ + {Text: "Name", Widget: newInput()}, + {Text: "EntityType", Widget: newSelect("EntityType")}, + { + Text: "SourceList", + HintText: "Original list the entity appeared on", + Widget: newSelect("SourceList"), + }, + + // Person + {Text: "People", Widget: blankSpace}, + + // Business + + // Organization + + // Aircraft + + // Vessel + + // Other Fields + {Text: "Other Fields", Widget: blankSpace}, + {Text: "CryptoAddresses", Widget: newMultilineInput(2)}, + } + + form := &widget.Form{ + Items: items, + OnSubmit: func() { + populatedItems := collectPopulatedItems(items) + fmt.Printf("%#v\n", populatedItems) + + var entities []search.SearchedEntity[search.Value] + err := showResults(env, results, entities) + if err != nil { + showWarning(env, warning, err) + } + }, + } + + return form +} + +func showResults(env Environment, results *fyne.Container, entities []search.SearchedEntity[search.Value]) error { + results.RemoveAll() + defer results.Show() + + header := widget.NewLabelWithStyle("Results", fyne.TextAlignLeading, fyne.TextStyle{ + Bold: true, + }) + results.Add(header) + + var data = [][]string{ + []string{"top left", "top right"}, + []string{"bottom left", "bottom right"}, + } + + for _, row := range data { + elm := container.NewHBox() + + for _, cell := range row { + c := widget.NewLabel(cell) + elm.Add(c) + } + + results.Add(elm) + } + + results.Refresh() + + return nil +} + +func showWarning(env Environment, warning *fyne.Container, err error) { + warning.RemoveAll() + defer warning.Show() + + header := widget.NewLabelWithStyle("Problem", fyne.TextAlignLeading, fyne.TextStyle{ + Bold: true, + }) + warning.Add(header) + + msg := widget.NewLabel(err.Error()) + warning.Add(msg) + + warning.Refresh() +} + +func newInput() *widget.Entry { + e := widget.NewEntry() + e.Validator = func(input string) error { + return nil + } + return e +} + +func newMultilineInput(visibleRows int) *widget.Entry { + e := widget.NewMultiLineEntry() + e.SetMinRowsVisible(visibleRows) + return e +} + +var ( + modelsPath = filepath.Join("pkg", "search", "models.go") // TODO(adam): relative to Watchman root dir +) + +func newSelect(modelName string) *widget.Select { + values, err := ast.ExtractVariablesOfType(modelsPath, modelName) + if err != nil { + panic(fmt.Sprintf("reading %s values: %w", modelName, err)) //nolint:forbidigo + } + + selectWidget := widget.NewSelect(values, func(_ string) {}) + + return selectWidget +} + +type item struct { + name, value string +} + +func collectPopulatedItems(formItems []*widget.FormItem) []item { + var out []item + for i := range formItems { + switch w := formItems[i].Widget.(type) { + case *widget.Entry: + if w.Text != "" { + out = append(out, item{name: formItems[i].Text, value: w.Text}) + } + case *widget.Select: + if w.Selected != "" { + out = append(out, item{name: formItems[i].Text, value: w.Selected}) + } + + case *widget.Label: + // ignore + + default: + fmt.Printf("%T\n", w) + } + } + return out +} diff --git a/internal/ui/ui.go b/internal/ui/ui.go new file mode 100644 index 00000000..495a5a44 --- /dev/null +++ b/internal/ui/ui.go @@ -0,0 +1,51 @@ +package ui + +import ( + "fmt" + + "github.com/moov-io/watchman/pkg/search" + + "fyne.io/fyne/v2" + "fyne.io/fyne/v2/app" + "fyne.io/fyne/v2/container" + "fyne.io/fyne/v2/widget" +) + +type Environment struct { + Client search.Client + + Width, Height float32 +} + +func New(env Environment) fyne.App { + a := app.New() + + device := fyne.CurrentDevice() + fmt.Printf("mobile=%v browser=%v keyboard=%v\n", + device.IsMobile(), device.IsBrowser(), device.HasKeyboard()) + + w := a.NewWindow("Hello World") + w.SetTitle("Watchman") + + // w.IsMobile() bool + // w.IsBrowser() bool + // w.SetFullScreen(bool) + + // Center the overall window and make it a reasonable size + env.Width = 800.0 + env.Height = 500.0 + w.Resize(fyne.NewSize(env.Width, env.Height)) + w.Show() + w.CenterOnScreen() + + // Set app tabs along the top + tabs := container.NewAppTabs( + container.NewTabItem("Search", SearchContainer(env)), + container.NewTabItem("Admin", widget.NewLabel("TODO - admin operations")), + ) + tabs.SetTabLocation(container.TabLocationTop) + + w.SetContent(tabs) + + return a +} diff --git a/pkg/search/client.go b/pkg/search/client.go new file mode 100644 index 00000000..fca61f55 --- /dev/null +++ b/pkg/search/client.go @@ -0,0 +1,55 @@ +package search + +import ( + "cmp" + "context" + "encoding/json" + "fmt" + "net/http" + "time" +) + +type Client interface { + SearchByEntity(ctx context.Context, entity Entity[Value]) ([]SearchedEntity[Value], error) +} + +func NewClient(httpClient *http.Client, baseAddress string) Client { + httpClient = cmp.Or(httpClient, &http.Client{ + Timeout: 5 * time.Second, + }) + + return &client{ + httpClient: httpClient, + baseAddress: baseAddress, + } +} + +type client struct { + httpClient *http.Client + baseAddress string +} + +func (c *client) SearchByEntity(ctx context.Context, entity Entity[Value]) ([]SearchedEntity[Value], error) { + addr := c.baseAddress + "/v2/search" + // addr += + + req, err := http.NewRequest("GET", addr, nil) + if err != nil { + return nil, fmt.Errorf("creating search request: %w", err) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("search by entity: %w", err) + } + if resp != nil && resp.Body != nil { + defer resp.Body.Close() + } + + var out []SearchedEntity[Value] + err = json.NewDecoder(resp.Body).Decode(&out) + if err != nil { + return nil, fmt.Errorf("decoding search by entity response: %w", err) + } + return out, nil +} diff --git a/internal/search/model_searched_entity.go b/pkg/search/model_searched_entity.go similarity index 54% rename from internal/search/model_searched_entity.go rename to pkg/search/model_searched_entity.go index 59b3ba0d..401f3d39 100644 --- a/internal/search/model_searched_entity.go +++ b/pkg/search/model_searched_entity.go @@ -1,11 +1,7 @@ package search -import ( - "github.com/moov-io/watchman/pkg/search" -) - type SearchedEntity[T any] struct { - search.Entity[T] + Entity[T] Match float64 `json:"match"` } diff --git a/internal/search/model_searched_entity_test.go b/pkg/search/model_searched_entity_test.go similarity index 92% rename from internal/search/model_searched_entity_test.go rename to pkg/search/model_searched_entity_test.go index 123c99c4..e386bc42 100644 --- a/internal/search/model_searched_entity_test.go +++ b/pkg/search/model_searched_entity_test.go @@ -5,8 +5,6 @@ import ( "strings" "testing" - "github.com/moov-io/watchman/pkg/search" - "github.com/stretchr/testify/require" ) @@ -16,7 +14,7 @@ func TestSearchedEntityJSON(t *testing.T) { } bs, err := json.MarshalIndent(SearchedEntity[SDN]{ - Entity: search.Entity[SDN]{ + Entity: Entity[SDN]{ SourceData: SDN{ EntityID: "12345", }, diff --git a/pkg/search/models.go b/pkg/search/models.go index 47c5ff94..8bafa76b 100644 --- a/pkg/search/models.go +++ b/pkg/search/models.go @@ -7,10 +7,12 @@ import ( type Value interface{} type Entity[T Value] struct { - Name string `json:"name"` - Type EntityType `json:"entityType"` - Source SourceList `json:"sourceList"` - SourceID string `json:"sourceID"` // TODO(adam): + Name string `json:"name"` + Type EntityType `json:"entityType"` + Source SourceList `json:"sourceList"` + + // SourceID is the source data's identifier. + SourceID string `json:"sourceID"` // TODO(adam): What has opensanctions done to normalize and join this data // Review https://www.opensanctions.org/reference/ @@ -37,12 +39,11 @@ type Entity[T Value] struct { type EntityType string var ( - EntityPerson EntityType = "person" - EntityBusiness EntityType = "business" - EntityOrganization EntityType = "organization" - EntityAircraft EntityType = "aircraft" - EntityVessel EntityType = "vessel" - EntityCryptoAddress EntityType = "crypto-address" // TODO(adam): Does this make sense? + EntityPerson EntityType = "person" + EntityBusiness EntityType = "business" + EntityOrganization EntityType = "organization" + EntityAircraft EntityType = "aircraft" + EntityVessel EntityType = "vessel" ) type SourceList string From fc3ab84f21d4698f66ae2629516b6c2b85501f75 Mon Sep 17 00:00:00 2001 From: Adam Shannon Date: Thu, 9 Jan 2025 12:13:16 -0600 Subject: [PATCH 2/9] fix: slightly better OFAC -> generic preprocessing, read v2 response/error --- cmd/server/main.go | 56 ++++++++++++++++++++--------------- internal/ui/search.go | 24 ++++++++++++--- internal/ui/search_mapping.go | 22 ++++++++++++++ pkg/ofac/mapper.go | 34 +++++++++++++++++---- pkg/search/client.go | 19 +++++++----- 5 files changed, 114 insertions(+), 41 deletions(-) create mode 100644 internal/ui/search_mapping.go diff --git a/cmd/server/main.go b/cmd/server/main.go index 62f8bdef..a98c8520 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -187,8 +187,8 @@ func main() { var genericEntities []pubsearch.Entity[pubsearch.Value] - genericSDNs := generalizeOFACSDNs(searcher.SDNs, searcher.Addresses) - genericEntities = append(genericEntities, genericSDNs...) + genericOFACEntities := groupOFACRecords(searcher) + genericEntities = append(genericEntities, genericOFACEntities...) v2SearchService := searchv2.NewService(logger, genericEntities) addSearchV2Routes(logger, router, v2SearchService) @@ -275,34 +275,42 @@ func handleDownloadStats(updates chan *DownloadStats, handle func(stats *Downloa } } -func generalizeOFACSDNs(input []*SDN, ofacAddresses []*Address) []pubsearch.Entity[pubsearch.Value] { - var out []pubsearch.Entity[pubsearch.Value] - for _, sdn := range input { - if sdn.SDN == nil { - continue - } +func addSearchV2Routes(logger log.Logger, r *mux.Router, service searchv2.Service) { + searchv2.NewController(logger, service).AppendRoutes(r) +} - var addresses []ofac.Address - for _, ofacAddr := range ofacAddresses { - if ofacAddr.Address == nil { - continue - } +func groupOFACRecords(searcher *searcher) []pubsearch.Entity[pubsearch.Value] { // TODO(adam): remove (refactor) + var sdns []ofac.SDN + var addrs []ofac.Address + var alts []ofac.AlternateIdentity + var comments []ofac.SDNComments - if sdn.EntityID == ofacAddr.Address.EntityID { - addresses = append(addresses, *ofacAddr.Address) - } + for _, sdn := range searcher.SDNs { + if sdn == nil || sdn.SDN == nil { + continue } - - entity := ofac.ToEntity(*sdn.SDN, addresses, nil, nil) - if len(entity.Addresses) > 0 && entity.Addresses[0].Line1 != "" { - out = append(out, entity) + sdns = append(sdns, *sdn.SDN) + } + for _, addr := range searcher.Addresses { + if addr == nil || addr.Address == nil { + continue } + addrs = append(addrs, *addr.Address) + } + for _, alt := range searcher.Alts { + if alt == nil || alt.AlternateIdentity == nil { + continue + } + alts = append(alts, *alt.AlternateIdentity) + } + for _, comment := range searcher.SDNComments { + if comment == nil { + continue + } + comments = append(comments, *comment) } - return out -} -func addSearchV2Routes(logger log.Logger, r *mux.Router, service searchv2.Service) { - searchv2.NewController(logger, service).AppendRoutes(r) + return ofac.GroupIntoEntities(sdns, addrs, comments, alts) } func readInt(override string, value int) int { diff --git a/internal/ui/search.go b/internal/ui/search.go index 96617474..80b209e7 100644 --- a/internal/ui/search.go +++ b/internal/ui/search.go @@ -1,6 +1,7 @@ package ui import ( + "context" "fmt" "path/filepath" @@ -23,7 +24,9 @@ func SearchContainer(env Environment) fyne.CanvasObject { results.Hide() form := searchForm(env, warning, results) + wrapper.Add(form) + wrapper.Add(warning) wrapper.Add(results) return wrapper @@ -35,7 +38,7 @@ func searchForm(env Environment, warning *fyne.Container, results *fyne.Containe blankSpace := widget.NewLabel(" ") items := []*widget.FormItem{ - {Text: "Name", Widget: newInput()}, + {Text: searchName, Widget: newInput()}, {Text: "EntityType", Widget: newSelect("EntityType")}, { Text: "SourceList", @@ -62,13 +65,26 @@ func searchForm(env Environment, warning *fyne.Container, results *fyne.Containe form := &widget.Form{ Items: items, OnSubmit: func() { + warning.Hide() + results.Hide() + populatedItems := collectPopulatedItems(items) - fmt.Printf("%#v\n", populatedItems) + fmt.Printf("searching with %d fields\n", len(populatedItems)) + + ctx := context.Background() // TODO(adam): + query := buildQueryEntity(populatedItems) + resp, err := env.Client.SearchByEntity(ctx, query) + if err != nil { + fmt.Printf("ERROR performing search: %v\n", err) + showWarning(env, warning, err) + return + } - var entities []search.SearchedEntity[search.Value] - err := showResults(env, results, entities) + err = showResults(env, results, resp.Entities) if err != nil { + fmt.Printf("ERROR showing results: %v\n", err) showWarning(env, warning, err) + return } }, } diff --git a/internal/ui/search_mapping.go b/internal/ui/search_mapping.go new file mode 100644 index 00000000..809bfd4d --- /dev/null +++ b/internal/ui/search_mapping.go @@ -0,0 +1,22 @@ +package ui + +import ( + "github.com/moov-io/watchman/pkg/search" +) + +var ( + searchName = "Name" +) + +func buildQueryEntity(populatedItems []item) search.Entity[search.Value] { + var out search.Entity[search.Value] + + for _, qry := range populatedItems { + switch qry.name { + case searchName: + out.Name = qry.value + } + } + + return out +} diff --git a/pkg/ofac/mapper.go b/pkg/ofac/mapper.go index ccb81f76..2250fd07 100644 --- a/pkg/ofac/mapper.go +++ b/pkg/ofac/mapper.go @@ -122,14 +122,36 @@ func extractCountry(remark string) string { return "" } -func ToEntities(sdns []SDN, addresses []Address, comments []SDNComments, altIds []AlternateIdentity) []search.Entity[search.Value] { - // TODO(adam): replace generalizeOFACSDNs with this - // TODO(adam): include []Address, []SDNComments, []AlternateIdentity - +func GroupIntoEntities(sdns []SDN, addresses []Address, comments []SDNComments, altIds []AlternateIdentity) []search.Entity[search.Value] { out := make([]search.Entity[search.Value], len(sdns)) - for i := range sdns { - out[i] = ToEntity(sdns[i], nil, nil, nil) // TODO(adam): fill out + + for idx, sdn := range sdns { + // find addresses, comments, and altIDs which match + + var addrs []Address + for _, addr := range addresses { + if sdn.EntityID == addr.EntityID { + addrs = append(addrs, addr) + } + } + + var cmts []SDNComments + for _, comment := range comments { + if sdn.EntityID == comment.EntityID { + cmts = append(cmts, comment) + } + } + + var alts []AlternateIdentity + for _, alt := range altIds { + if sdn.EntityID == alt.EntityID { + alts = append(alts, alt) + } + } + + out[idx] = ToEntity(sdn, addrs, cmts, alts) } + return out } diff --git a/pkg/search/client.go b/pkg/search/client.go index fca61f55..db1f3356 100644 --- a/pkg/search/client.go +++ b/pkg/search/client.go @@ -10,7 +10,7 @@ import ( ) type Client interface { - SearchByEntity(ctx context.Context, entity Entity[Value]) ([]SearchedEntity[Value], error) + SearchByEntity(ctx context.Context, entity Entity[Value]) (SearchResponse, error) } func NewClient(httpClient *http.Client, baseAddress string) Client { @@ -29,27 +29,32 @@ type client struct { baseAddress string } -func (c *client) SearchByEntity(ctx context.Context, entity Entity[Value]) ([]SearchedEntity[Value], error) { +type SearchResponse struct { + Entities []SearchedEntity[Value] `json:"entities"` +} + +func (c *client) SearchByEntity(ctx context.Context, entity Entity[Value]) (SearchResponse, error) { addr := c.baseAddress + "/v2/search" - // addr += + addr += "?name=" + entity.Name // TODO(adam): escape, use proper setters + + var out SearchResponse req, err := http.NewRequest("GET", addr, nil) if err != nil { - return nil, fmt.Errorf("creating search request: %w", err) + return out, fmt.Errorf("creating search request: %w", err) } resp, err := c.httpClient.Do(req) if err != nil { - return nil, fmt.Errorf("search by entity: %w", err) + return out, fmt.Errorf("search by entity: %w", err) } if resp != nil && resp.Body != nil { defer resp.Body.Close() } - var out []SearchedEntity[Value] err = json.NewDecoder(resp.Body).Decode(&out) if err != nil { - return nil, fmt.Errorf("decoding search by entity response: %w", err) + return out, fmt.Errorf("decoding search by entity response: %w", err) } return out, nil } From ce5ca42ea865adb2c113127e80b215c6b28781da Mon Sep 17 00:00:00 2001 From: Adam Shannon Date: Thu, 9 Jan 2025 15:49:48 -0600 Subject: [PATCH 3/9] cmd/server: massive refactor, remove useless pointers and move towards /v2/search --- cmd/internal/client.go | 54 -- cmd/internal/client_test.go | 28 - cmd/server/config.go | 42 + cmd/server/debug_sdn.go | 49 -- cmd/server/debug_sdn_test.go | 54 -- cmd/server/download.go | 478 ++--------- cmd/server/download_handler.go | 56 -- cmd/server/download_test.go | 264 +++---- cmd/server/download_webhook.go | 37 - cmd/server/filter.go | 82 -- cmd/server/filter_test.go | 195 ----- cmd/server/http.go | 52 -- cmd/server/http_test.go | 25 - cmd/server/issue115_test.go | 52 -- cmd/server/issue326_test.go | 35 - cmd/server/largest.go | 58 -- cmd/server/largest_test.go | 138 ---- cmd/server/main.go | 319 ++------ cmd/server/main_test.go | 42 +- cmd/server/sdn.go | 110 --- cmd/server/sdn_test.go | 127 --- cmd/server/search.go | 674 ---------------- cmd/server/search_benchmark_test.go | 147 ---- cmd/server/search_crypto.go | 74 -- cmd/server/search_crypto_test.go | 98 --- cmd/server/search_eu_csl.go | 49 -- cmd/server/search_eu_csl_test.go | 43 - cmd/server/search_generic.go | 111 --- cmd/server/search_handlers.go | 490 ------------ cmd/server/search_handlers_bench_test.go | 51 -- cmd/server/search_handlers_test.go | 376 --------- cmd/server/search_test.go | 741 ------------------ cmd/server/search_uk_csl.go | 60 -- cmd/server/search_uk_csl_test.go | 66 -- cmd/server/search_us_csl.go | 222 ------ cmd/server/search_us_csl_test.go | 187 ----- cmd/server/values.go | 98 --- cmd/server/values_test.go | 104 --- cmd/server/webhook.go | 101 --- cmd/server/webhook_test.go | 152 ---- configs/config.default.yml | 9 + go.mod | 23 +- go.sum | 81 +- internal/download/download.go | 200 +++++ internal/download/models.go | 23 + internal/prepare/pipeline.go | 250 ------ .../prepare/pipeline_company_name_cleanup.go | 16 +- .../pipeline_company_name_cleanup_test.go | 24 +- internal/prepare/pipeline_normalize.go | 7 - internal/prepare/pipeline_normalize_test.go | 20 +- internal/prepare/pipeline_reorder.go | 18 +- internal/prepare/pipeline_reorder_test.go | 31 +- internal/prepare/pipeline_stopwords.go | 33 +- internal/prepare/pipeline_stopwords_test.go | 71 +- internal/prepare/pipeline_test.go | 140 ++-- internal/search/api_search.go | 5 + internal/search/service.go | 28 +- internal/search/service_test.go | 19 +- internal/ui/search.go | 2 +- package.go | 8 + pkg/csl_eu/download_eu.go | 5 +- pkg/csl_eu/download_eu_test.go | 5 +- pkg/csl_eu/reader_eu.go | 6 +- pkg/csl_uk/download_uk.go | 15 +- pkg/csl_uk/download_uk_test.go | 11 +- pkg/csl_uk/reader_uk.go | 16 +- pkg/csl_us/csl.go | 22 +- pkg/csl_us/csl_test.go | 4 +- pkg/csl_us/download.go | 5 +- pkg/csl_us/download_test.go | 5 +- pkg/csl_us/reader.go | 54 +- pkg/csl_us/reader_test.go | 67 +- pkg/download/client.go | 9 +- pkg/dpl/download.go | 33 - pkg/dpl/download_test.go | 74 -- pkg/dpl/dpl.go | 33 - pkg/dpl/reader.go | 65 -- pkg/dpl/reader_test.go | 38 - pkg/ofac/download.go | 5 +- pkg/ofac/download_test.go | 5 +- pkg/ofac/mapper_person_test.go | 2 +- pkg/ofac/mapper_vehicles_test.go | 4 +- pkg/ofac/reader.go | 26 +- 83 files changed, 897 insertions(+), 6761 deletions(-) delete mode 100644 cmd/internal/client.go delete mode 100644 cmd/internal/client_test.go create mode 100644 cmd/server/config.go delete mode 100644 cmd/server/debug_sdn.go delete mode 100644 cmd/server/debug_sdn_test.go delete mode 100644 cmd/server/download_handler.go delete mode 100644 cmd/server/download_webhook.go delete mode 100644 cmd/server/filter.go delete mode 100644 cmd/server/filter_test.go delete mode 100644 cmd/server/http.go delete mode 100644 cmd/server/http_test.go delete mode 100644 cmd/server/issue115_test.go delete mode 100644 cmd/server/issue326_test.go delete mode 100644 cmd/server/largest.go delete mode 100644 cmd/server/largest_test.go delete mode 100644 cmd/server/sdn.go delete mode 100644 cmd/server/sdn_test.go delete mode 100644 cmd/server/search.go delete mode 100644 cmd/server/search_benchmark_test.go delete mode 100644 cmd/server/search_crypto.go delete mode 100644 cmd/server/search_crypto_test.go delete mode 100644 cmd/server/search_eu_csl.go delete mode 100644 cmd/server/search_eu_csl_test.go delete mode 100644 cmd/server/search_generic.go delete mode 100644 cmd/server/search_handlers.go delete mode 100644 cmd/server/search_handlers_bench_test.go delete mode 100644 cmd/server/search_handlers_test.go delete mode 100644 cmd/server/search_test.go delete mode 100644 cmd/server/search_uk_csl.go delete mode 100644 cmd/server/search_uk_csl_test.go delete mode 100644 cmd/server/search_us_csl.go delete mode 100644 cmd/server/search_us_csl_test.go delete mode 100644 cmd/server/values.go delete mode 100644 cmd/server/values_test.go delete mode 100644 cmd/server/webhook.go delete mode 100644 cmd/server/webhook_test.go create mode 100644 configs/config.default.yml create mode 100644 internal/download/download.go create mode 100644 internal/download/models.go delete mode 100644 internal/prepare/pipeline.go create mode 100644 package.go delete mode 100644 pkg/dpl/download.go delete mode 100644 pkg/dpl/download_test.go delete mode 100644 pkg/dpl/dpl.go delete mode 100644 pkg/dpl/reader.go delete mode 100644 pkg/dpl/reader_test.go diff --git a/cmd/internal/client.go b/cmd/internal/client.go deleted file mode 100644 index 958f9a67..00000000 --- a/cmd/internal/client.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package internal - -import ( - "fmt" - "net/http" - "strings" - "time" - - "github.com/moov-io/base/http/bind" - "github.com/moov-io/watchman" - moov "github.com/moov-io/watchman/client" -) - -const ( - DefaultApiAddress = "https://api.moov.io" -) - -// addr reads flagApiAddress and flagLocal to compute the HTTP address used for connecting with Sanctions Search. -func addr(address string, local bool) string { - if local { - // If '-local and -address ' use - if address != DefaultApiAddress { - return strings.TrimSuffix(address, "/") - } else { - return "http://localhost" + bind.HTTP("watchman") - } - } else { - address = strings.TrimSuffix(address, "/") - // -address isn't changed, so assume Moov's API (needs extra path added) - if address == DefaultApiAddress { - return address + "/v1/watchman" - } - return address - } -} - -func Config(address string, local bool) *moov.Configuration { - conf := moov.NewConfiguration() - conf.BasePath = addr(address, local) - - conf.UserAgent = fmt.Sprintf("moov/watchman:%s", watchman.Version) - conf.HTTPClient = &http.Client{ - Timeout: 10 * time.Second, - Transport: &http.Transport{ - IdleConnTimeout: 1 * time.Minute, - }, - } - - return conf -} diff --git a/cmd/internal/client_test.go b/cmd/internal/client_test.go deleted file mode 100644 index 77847af9..00000000 --- a/cmd/internal/client_test.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package internal - -import ( - "testing" -) - -func TestWatchman_addr(t *testing.T) { - cases := []struct { - addr string - local bool - expected string - }{ - {"http://localhost:8084", false, "http://localhost:8084"}, - {"http://localhost:8084/", false, "http://localhost:8084"}, - {DefaultApiAddress, true, "http://localhost:8084"}, - {DefaultApiAddress, false, DefaultApiAddress + "/v1/watchman"}, - } - for i := range cases { - got := addr(cases[i].addr, cases[i].local) - if got != cases[i].expected { - t.Errorf("idx=%d got=%q expected=%q", i, got, cases[i].expected) - } - } -} diff --git a/cmd/server/config.go b/cmd/server/config.go new file mode 100644 index 00000000..5ff2e8fe --- /dev/null +++ b/cmd/server/config.go @@ -0,0 +1,42 @@ +// Copyright The Moov Authors +// Use of this source code is governed by an Apache License +// license that can be found in the LICENSE file. + +package main + +import ( + watchman "github.com/moov-io/watchman" + "github.com/moov-io/watchman/internal/download" + + "github.com/moov-io/base/config" + "github.com/moov-io/base/log" +) + +type GlobalConfig struct { + Watchman Config +} + +type Config struct { + Download download.Config + + Servers ServerConfig +} + +type ServerConfig struct { + BindAddress string + AdminAddress string + + TLSCertFile string + TLSKeyFile string +} + +func LoadConfig(logger log.Logger) (*Config, error) { + configService := config.NewService(logger) + + global := &GlobalConfig{} + if err := configService.LoadFromFS(global, watchman.ConfigDefaults); err != nil { + return nil, err + } + + return &global.Watchman, nil +} diff --git a/cmd/server/debug_sdn.go b/cmd/server/debug_sdn.go deleted file mode 100644 index 6ab6d740..00000000 --- a/cmd/server/debug_sdn.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "net/http" - - moovhttp "github.com/moov-io/base/http" - "github.com/moov-io/base/log" -) - -const ( - debugSDNPath = "/debug/sdn/{sdnId}" -) - -func debugSDNHandler(logger log.Logger, searcher *searcher) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - sdnID := getSDNId(w, r) - if sdnID == "" { - return - } - - if requestID := moovhttp.GetRequestID(r); requestID != "" { - logger.Info().With(log.Fields{ - "requestID": log.String(requestID), - }).Logf("debug route for SDN=%s", sdnID) - } - - var response struct { - SDN *SDN `json:"SDN"` - Debug struct { - IndexedName string `json:"indexedName"` - ParsedRemarksID string `json:"parsedRemarksId"` - } `json:"debug"` - } - response.SDN = searcher.debugSDN(sdnID) - if response.SDN != nil { - response.Debug.IndexedName = response.SDN.name - response.Debug.ParsedRemarksID = response.SDN.id - } - - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(response) - } -} diff --git a/cmd/server/debug_sdn_test.go b/cmd/server/debug_sdn_test.go deleted file mode 100644 index 820ee142..00000000 --- a/cmd/server/debug_sdn_test.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "io" - "net/http" - "testing" - - "github.com/moov-io/base" - "github.com/moov-io/base/admin" - "github.com/moov-io/base/log" - - "github.com/stretchr/testify/require" -) - -func TestDebug__SDN(t *testing.T) { - svc, err := admin.New(admin.Opts{ - Addr: ":0", - }) - require.NoError(t, err) - - go svc.Listen() - defer svc.Shutdown() - - svc.AddHandler(debugSDNPath, debugSDNHandler(log.NewNopLogger(), idSearcher)) - - req, _ := http.NewRequest("GET", "http://"+svc.BindAddr()+"/debug/sdn/22790", nil) - req.Header.Set("x-request-id", base.ID()) - - resp, err := http.DefaultClient.Do(req) - require.NoError(t, err) - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - bs, _ := io.ReadAll(resp.Body) - t.Fatalf("bogus status code: %s: %s", resp.Status, string(bs)) - } - - var response struct { - Debug struct { - IndexedName string `json:"indexedName"` - ParsedRemarksID string `json:"parsedRemarksId"` - } `json:"debug"` - } - err = json.NewDecoder(resp.Body).Decode(&response) - require.NoError(t, err) - - require.Equal(t, "nicolas maduro moros", response.Debug.IndexedName) - require.Equal(t, "5892464", response.Debug.ParsedRemarksID) -} diff --git a/cmd/server/download.go b/cmd/server/download.go index 01bed343..5a9ba929 100644 --- a/cmd/server/download.go +++ b/cmd/server/download.go @@ -5,454 +5,68 @@ package main import ( - "bytes" - "cmp" - "encoding/json" - "errors" - "fmt" + "context" "os" "strings" "time" - "github.com/moov-io/base/log" - "github.com/moov-io/base/strx" - "github.com/moov-io/watchman/pkg/csl_eu" - "github.com/moov-io/watchman/pkg/csl_uk" - "github.com/moov-io/watchman/pkg/csl_us" - "github.com/moov-io/watchman/pkg/dpl" - "github.com/moov-io/watchman/pkg/ofac" - - "github.com/prometheus/client_golang/prometheus" -) - -var ( - lastDataRefreshSuccess = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "last_data_refresh_success", - Help: "Unix timestamp of when data was last refreshed successfully", - }, nil) + "github.com/moov-io/watchman/internal/download" + "github.com/moov-io/watchman/internal/search" - lastDataRefreshFailure = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "last_data_refresh_failure", - Help: "Unix timestamp of the most recent failure to refresh data", - }, []string{"source"}) - - lastDataRefreshCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "last_data_refresh_count", - Help: "Count of records for a given sanction or entity list", - }, []string{"source"}) + "github.com/moov-io/base/log" ) -func init() { - prometheus.MustRegister(lastDataRefreshSuccess) - prometheus.MustRegister(lastDataRefreshCount) - prometheus.MustRegister(lastDataRefreshFailure) -} - -// DownloadStats holds counts for each type of list data parsed from files and a -// timestamp of when the download happened. -type DownloadStats struct { - // US Office of Foreign Assets Control (OFAC) - SDNs int `json:"SDNs"` - Alts int `json:"altNames"` - Addresses int `json:"addresses"` - - // US Bureau of Industry and Security (BIS) - DeniedPersons int `json:"deniedPersons"` - - // Consolidated Screening List (CSL) - BISEntities int `json:"bisEntities"` - MilitaryEndUsers int `json:"militaryEndUsers"` - SectoralSanctions int `json:"sectoralSanctions"` - Unverified int `json:"unverifiedCSL"` - NonProliferationSanctions int `json:"nonProliferationSanctions"` - ForeignSanctionsEvaders int `json:"foreignSanctionsEvaders"` - PalestinianLegislativeCouncil int `json:"palestinianLegislativeCouncil"` - CAPTA int `json:"CAPTA"` - ITARDebarred int `json:"ITARDebarred"` - ChineseMilitaryIndustrialComplex int `json:"chineseMilitaryIndustrialComplex"` - NonSDNMenuBasedSanctions int `json:"nonSDNMenuBasedSanctions"` - - // EU Consolidated Sanctions List - EUCSL int `json:"europeanSanctionsList"` - - // UK Consolidated Sanctions List - UKCSL int `json:"ukConsolidatedSanctionsList"` - - // UK Sanctions List - UKSanctionsList int `json:"ukSanctionsList"` - - Errors []error `json:"-"` - RefreshedAt time.Time `json:"timestamp"` -} - -func (ss *DownloadStats) Error() string { - var buf bytes.Buffer - for i := range ss.Errors { - buf.WriteString(ss.Errors[i].Error() + "\n") - } - return buf.String() -} - -func (ss *DownloadStats) MarshalJSON() ([]byte, error) { - type Aux struct { - DownloadStats - Errors []string `json:"errors"` - } - errors := make([]string, 0, len(ss.Errors)) - for i := range ss.Errors { - errors = append(errors, ss.Errors[i].Error()) - } - return json.Marshal(Aux{ - DownloadStats: *ss, - Errors: errors, - }) -} - -// periodicDataRefresh will forever block for interval's duration and then download and reparse the data. -// Download stats are recorded as part of a successful re-download and parse. -func (s *searcher) periodicDataRefresh(interval time.Duration, updates chan *DownloadStats) { - if interval == 0*time.Second { - s.logger.Logf("not scheduling periodic refreshing duration=%v", interval) - return - } - for { - time.Sleep(interval) - stats, err := s.refreshData("") - if err != nil { - if s.logger != nil { - s.logger.Info().Logf("ERROR: refreshing data: %v", err) - } - } else { - if s.logger != nil { - s.logger.Info().With(log.Fields{ - // OFAC - "SDNs": log.Int(stats.SDNs), - "AltNames": log.Int(stats.Alts), - "Addresses": log.Int(stats.Addresses), - - // BIS - "DPL": log.Int(stats.DeniedPersons), - - // CSL - "BISEntities": log.Int(stats.BISEntities), - "MilitaryEndUsers": log.Int(stats.MilitaryEndUsers), - "SSI": log.Int(stats.SectoralSanctions), - "UVL": log.Int(stats.Unverified), - "ISN": log.Int(stats.NonProliferationSanctions), - "FSE": log.Int(stats.ForeignSanctionsEvaders), - "PLC": log.Int(stats.PalestinianLegislativeCouncil), - "CAP": log.Int(stats.CAPTA), - "DTC": log.Int(stats.ITARDebarred), - "CMIC": log.Int(stats.ChineseMilitaryIndustrialComplex), - "NS_MBS": log.Int(stats.NonSDNMenuBasedSanctions), - "EU_CSL": log.Int(stats.EUCSL), - "UK_CSL": log.Int(stats.UKCSL), - }).Logf("data refreshed %v ago", time.Since(stats.RefreshedAt)) +func setupPeriodicRefreshing(ctx context.Context, logger log.Logger, conf download.Config, downloader download.Downloader, searchService search.Service) error { + err := refreshAllSources(ctx, logger, downloader, searchService) + if err != nil { + return err + } + + // Setup periodic refreshing + ticker := time.NewTicker(getRefreshInterval(conf)) + defer ticker.Stop() + + errs := make(chan error, 1) + go func() { + for { + select { + case <-ctx.Done(): + errs <- nil + return + + case <-ticker.C: + err := refreshAllSources(ctx, logger, downloader, searchService) + if err != nil { + errs <- err + } } - updates <- stats // send stats back } - } -} - -func ofacRecords(logger log.Logger, initialDir string) (*ofac.Results, error) { - files, err := ofac.Download(logger, initialDir) - if err != nil { - return nil, fmt.Errorf("download: %v", err) - } - if len(files) == 0 { - return nil, errors.New("no OFAC Results") - } - res, err := ofac.Read(files) - if err != nil { - return nil, err - } - - return res, nil -} - -func dplRecords(logger log.Logger, initialDir string) ([]*dpl.DPL, error) { - file, err := dpl.Download(logger, initialDir) - if err != nil { - return nil, err - } - - return dpl.Read(file["dpl.txt"]) -} - -func cslUSRecords(logger log.Logger, initialDir string) (*csl_us.CSL, error) { - file, err := csl_us.Download(logger, initialDir) - if err != nil { - logger.Warn().Logf("skipping CSL US download: %v", err) - return &csl_us.CSL{}, nil - } - cslRecords, err := csl_us.ReadFile(file["csl.csv"]) - if err != nil { - return nil, err - } - return cslRecords, nil -} - -func euCSLRecords(logger log.Logger, initialDir string) ([]*csl_eu.CSLRecord, error) { - file, err := csl_eu.DownloadEU(logger, initialDir) - if err != nil { - logger.Warn().Logf("skipping EU CSL download: %v", err) - // no error to return because we skip the download - return nil, nil - } - - cslRecords, _, err := csl_eu.ParseEU(file["eu_csl.csv"]) - if err != nil { - return nil, err - } - return cslRecords, err - -} - -func ukCSLRecords(logger log.Logger, initialDir string) ([]*csl_uk.CSLRecord, error) { - file, err := csl_uk.DownloadCSL(logger, initialDir) - if err != nil { - logger.Warn().Logf("skipping UK CSL download: %v", err) - // no error to return because we skip the download - return nil, nil - } - cslRecords, _, err := csl_uk.ReadCSLFile(file["ConList.csv"]) - if err != nil { - return nil, err - } - return cslRecords, err + }() + return <-errs } -func ukSanctionsListRecords(logger log.Logger, initialDir string) ([]*csl_uk.SanctionsListRecord, error) { - file, err := csl_uk.DownloadSanctionsList(logger, initialDir) - if file == nil || err != nil { - logger.Warn().Logf("skipping UK Sanctions List download: %v", err) - // no error to return because we skip the download - return nil, nil - } - - records, _, err := csl_uk.ReadSanctionsListFile(file["UK_Sanctions_List.ods"]) - if err != nil { - return nil, err - } - return records, err -} - -// refreshData reaches out to the various websites to download the latest -// files, runs each list's parser, and index data for searches. -func (s *searcher) refreshData(initialDir string) (*DownloadStats, error) { - if s.logger != nil { - s.logger.Log("Starting refresh of data") - - if initialDir != "" { - s.logger.Logf("reading files from %s", initialDir) - } - } - - stats := &DownloadStats{ - RefreshedAt: lastRefresh(initialDir), - } - - var err error - lastDataRefreshFailure.WithLabelValues("SDNs").Set(float64(time.Now().Unix())) - - var ofacResults *ofac.Results - withOFACList := cmp.Or(os.Getenv("WITH_OFAC_LIST"), "true") - if strx.Yes(withOFACList) { - ofacResults, err = ofacRecords(s.logger, initialDir) - if err != nil { - lastDataRefreshFailure.WithLabelValues("SDNs").Set(float64(time.Now().Unix())) - stats.Errors = append(stats.Errors, fmt.Errorf("OFAC: %v", err)) - } - } - if ofacResults == nil { - ofacResults = &ofac.Results{} - } - - sdns := precomputeSDNs(ofacResults.SDNs, ofacResults.Addresses, s.pipe) - adds := precomputeAddresses(ofacResults.Addresses) - alts := precomputeAlts(ofacResults.AlternateIdentities, s.pipe) - sdnComments := ofacResults.SDNComments - - var deniedPersons []*dpl.DPL - withDPLList := cmp.Or(os.Getenv("WITH_US_DPL_LIST"), "true") - if strx.Yes(withDPLList) { - deniedPersons, err = dplRecords(s.logger, initialDir) - if err != nil { - lastDataRefreshFailure.WithLabelValues("DPs").Set(float64(time.Now().Unix())) - stats.Errors = append(stats.Errors, fmt.Errorf("DPL: %v", err)) - } - } - dps := precomputeDPs(deniedPersons, s.pipe) - - var euCSLs []*Result[csl_eu.CSLRecord] - withEUScreeningList := cmp.Or(os.Getenv("WITH_EU_SCREENING_LIST"), "true") - if strx.Yes(withEUScreeningList) { - euConsolidatedList, err := euCSLRecords(s.logger, initialDir) - if err != nil { - lastDataRefreshFailure.WithLabelValues("EUCSL").Set(float64(time.Now().Unix())) - stats.Errors = append(stats.Errors, fmt.Errorf("EUCSL: %v", err)) +func getRefreshInterval(conf download.Config) time.Duration { + override := strings.TrimSpace(os.Getenv("DATA_REFRESH_INTERVAL")) + if override != "" { + dur, err := time.ParseDuration(override) + if err == nil { + return dur } - euCSLs = precomputeCSLEntities[csl_eu.CSLRecord](euConsolidatedList, s.pipe) } - - var ukCSLs []*Result[csl_uk.CSLRecord] - withUKCSLSanctionsList := cmp.Or(os.Getenv("WITH_UK_CSL_SANCTIONS_LIST"), "true") - if strx.Yes(withUKCSLSanctionsList) { - ukConsolidatedList, err := ukCSLRecords(s.logger, initialDir) - if err != nil { - lastDataRefreshFailure.WithLabelValues("UKCSL").Set(float64(time.Now().Unix())) - stats.Errors = append(stats.Errors, fmt.Errorf("UKCSL: %v", err)) - } - ukCSLs = precomputeCSLEntities[csl_uk.CSLRecord](ukConsolidatedList, s.pipe) - } - - var ukSLs []*Result[csl_uk.SanctionsListRecord] - withUKSanctionsList := os.Getenv("WITH_UK_SANCTIONS_LIST") - if strings.ToLower(withUKSanctionsList) == "true" { - ukSanctionsList, err := ukSanctionsListRecords(s.logger, initialDir) - if err != nil { - lastDataRefreshFailure.WithLabelValues("UKSanctionsList").Set(float64(time.Now().Unix())) - stats.Errors = append(stats.Errors, fmt.Errorf("UKSanctionsList: %v", err)) - } - ukSLs = precomputeCSLEntities[csl_uk.SanctionsListRecord](ukSanctionsList, s.pipe) - - stats.UKSanctionsList = len(ukSLs) - lastDataRefreshCount.WithLabelValues("UKSL").Set(float64(len(ukSLs))) - } - - // csl records from US downloaded here - var usConsolidatedLists *csl_us.CSL - withUSConsolidatedLists := cmp.Or(os.Getenv("WITH_US_CSL_SANCTIONS_LIST"), "true") - if strx.Yes(withUSConsolidatedLists) { - usConsolidatedLists, err = cslUSRecords(s.logger, initialDir) - if err != nil { - lastDataRefreshFailure.WithLabelValues("CSL").Set(float64(time.Now().Unix())) - stats.Errors = append(stats.Errors, fmt.Errorf("US CSL: %v", err)) - } - } - if usConsolidatedLists == nil { - usConsolidatedLists = new(csl_us.CSL) - } - - els := precomputeCSLEntities[csl_us.EL](usConsolidatedLists.ELs, s.pipe) - meus := precomputeCSLEntities[csl_us.MEU](usConsolidatedLists.MEUs, s.pipe) - ssis := precomputeCSLEntities[csl_us.SSI](usConsolidatedLists.SSIs, s.pipe) - uvls := precomputeCSLEntities[csl_us.UVL](usConsolidatedLists.UVLs, s.pipe) - isns := precomputeCSLEntities[csl_us.ISN](usConsolidatedLists.ISNs, s.pipe) - fses := precomputeCSLEntities[csl_us.FSE](usConsolidatedLists.FSEs, s.pipe) - plcs := precomputeCSLEntities[csl_us.PLC](usConsolidatedLists.PLCs, s.pipe) - caps := precomputeCSLEntities[csl_us.CAP](usConsolidatedLists.CAPs, s.pipe) - dtcs := precomputeCSLEntities[csl_us.DTC](usConsolidatedLists.DTCs, s.pipe) - cmics := precomputeCSLEntities[csl_us.CMIC](usConsolidatedLists.CMICs, s.pipe) - ns_mbss := precomputeCSLEntities[csl_us.NS_MBS](usConsolidatedLists.NS_MBSs, s.pipe) - - // OFAC - stats.SDNs = len(sdns) - stats.Alts = len(alts) - stats.Addresses = len(adds) - // BIS - stats.DeniedPersons = len(dps) - // CSL - stats.BISEntities = len(els) - stats.MilitaryEndUsers = len(meus) - stats.SectoralSanctions = len(ssis) - stats.Unverified = len(uvls) - stats.NonProliferationSanctions = len(isns) - stats.ForeignSanctionsEvaders = len(fses) - stats.PalestinianLegislativeCouncil = len(plcs) - stats.CAPTA = len(caps) - stats.ITARDebarred = len(dtcs) - stats.ChineseMilitaryIndustrialComplex = len(cmics) - stats.NonSDNMenuBasedSanctions = len(ns_mbss) - // EU - CSL - stats.EUCSL = len(euCSLs) - - // UK - CSL - stats.UKCSL = len(ukCSLs) - - // record prometheus metrics - lastDataRefreshCount.WithLabelValues("SDNs").Set(float64(len(sdns))) - lastDataRefreshCount.WithLabelValues("SSIs").Set(float64(len(ssis))) - lastDataRefreshCount.WithLabelValues("BISEntities").Set(float64(len(els))) - lastDataRefreshCount.WithLabelValues("MilitaryEndUsers").Set(float64(len(meus))) - lastDataRefreshCount.WithLabelValues("DPs").Set(float64(len(dps))) - lastDataRefreshCount.WithLabelValues("UVLs").Set(float64(len(uvls))) - lastDataRefreshCount.WithLabelValues("ISNs").Set(float64(len(isns))) - lastDataRefreshCount.WithLabelValues("FSEs").Set(float64(len(fses))) - lastDataRefreshCount.WithLabelValues("PLCs").Set(float64(len(plcs))) - lastDataRefreshCount.WithLabelValues("CAPs").Set(float64(len(caps))) - lastDataRefreshCount.WithLabelValues("DTCs").Set(float64(len(dtcs))) - lastDataRefreshCount.WithLabelValues("CMICs").Set(float64(len(cmics))) - lastDataRefreshCount.WithLabelValues("NS_MBSs").Set(float64(len(ns_mbss))) - // EU CSL - lastDataRefreshCount.WithLabelValues("EUCSL").Set(float64(len(euCSLs))) - // UK CSL - lastDataRefreshCount.WithLabelValues("UKCSL").Set(float64(len(ukCSLs))) - - if len(stats.Errors) > 0 { - return stats, stats - } - - // Set new records after precomputation (to minimize lock contention) - s.Lock() - // OFAC - s.SDNs = sdns - s.Addresses = adds - s.Alts = alts - s.SDNComments = sdnComments - // BIS - s.DPs = dps - // CSL - s.BISEntities = els - s.MilitaryEndUsers = meus - s.SSIs = ssis - s.UVLs = uvls - s.ISNs = isns - s.FSEs = fses - s.PLCs = plcs - s.CAPs = caps - s.DTCs = dtcs - s.CMICs = cmics - s.NS_MBSs = ns_mbss - //EUCSL - s.EUCSL = euCSLs - //UKCSL - s.UKCSL = ukCSLs - s.UKSanctionsList = ukSLs - // metadata - s.lastRefreshedAt = stats.RefreshedAt - s.Unlock() - - if s.logger != nil { - s.logger.Log("Finished refresh of data") - } - - // record successful data refresh - lastDataRefreshSuccess.WithLabelValues().Set(float64(time.Now().Unix())) - - return stats, nil + return conf.RefreshInterval } -// lastRefresh returns a time.Time for the oldest file in dir or the current time if empty. -func lastRefresh(dir string) time.Time { - if dir == "" { - return time.Now().In(time.UTC) +func refreshAllSources(ctx context.Context, logger log.Logger, downloader download.Downloader, searchService search.Service) error { + // Initial data load + stats, err := downloader.RefreshAll(ctx) + if err != nil { + return err } + logger.Info().Logf("data refreshed - %v entities from %v lists took %v", + len(stats.Entities), len(stats.Lists), stats.EndedAt.Sub(stats.StartedAt)) - fds, err := os.ReadDir(dir) - if len(fds) == 0 || err != nil { - return time.Time{} // zero time because there's no initial data - } + // Replace in-mem entities for search.Service + searchService.UpdateEntities(stats.Entities) - oldest := time.Now().In(time.UTC) - for i := range fds { - info, err := fds[i].Info() - if err != nil { - continue - } - if t := info.ModTime(); t.Before(oldest) { - oldest = t - } - } - return oldest.In(time.UTC) + return nil } diff --git a/cmd/server/download_handler.go b/cmd/server/download_handler.go deleted file mode 100644 index d9a3a63f..00000000 --- a/cmd/server/download_handler.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "net/http" - "time" - - "github.com/moov-io/base/log" -) - -const ( - manualRefreshPath = "/data/refresh" -) - -// manualRefreshHandler will register an endpoint on the admin server data refresh endpoint -func manualRefreshHandler(logger log.Logger, searcher *searcher, updates chan *DownloadStats) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - logger.Log("admin: refreshing data") - - if stats, err := searcher.refreshData(""); err != nil { - logger.LogErrorf("ERROR: admin: problem refreshing data: %v", err) - w.WriteHeader(http.StatusInternalServerError) - } else { - - go func() { - updates <- stats - }() - - logger.Info().With(log.Fields{ - "SDNs": log.Int(stats.SDNs), - "AltNames": log.Int(stats.Alts), - "Addresses": log.Int(stats.Addresses), - "SSI": log.Int(stats.SectoralSanctions), - "DPL": log.Int(stats.DeniedPersons), - "BISEntities": log.Int(stats.BISEntities), - "UVL": log.Int(stats.Unverified), - "ISN": log.Int(stats.NonProliferationSanctions), - "FSE": log.Int(stats.ForeignSanctionsEvaders), - "PLC": log.Int(stats.PalestinianLegislativeCouncil), - "CAP": log.Int(stats.CAPTA), - "DTC": log.Int(stats.ITARDebarred), - "CMIC": log.Int(stats.ChineseMilitaryIndustrialComplex), - "NS_MBS": log.Int(stats.NonSDNMenuBasedSanctions), - "EUCSL": log.Int(stats.EUCSL), - "UKCSL": log.Int(stats.UKCSL), - "UKSanctionsList": log.Int(stats.UKSanctionsList), - }).Logf("admin: finished data refresh %v ago", time.Since(stats.RefreshedAt)) - - json.NewEncoder(w).Encode(stats) - } - } -} diff --git a/cmd/server/download_test.go b/cmd/server/download_test.go index 57461d9d..831bf483 100644 --- a/cmd/server/download_test.go +++ b/cmd/server/download_test.go @@ -4,135 +4,135 @@ package main -import ( - "bytes" - "encoding/json" - "errors" - "os" - "path/filepath" - "slices" - "testing" - "time" - - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/pkg/ofac" - - "github.com/stretchr/testify/require" -) - -func TestDownloadStats(t *testing.T) { - when := time.Date(2022, time.May, 21, 9, 4, 0, 0, time.UTC) - bs, err := json.Marshal(&DownloadStats{ - SDNs: 1, - Errors: []error{ - errors.New("bad thing"), - }, - RefreshedAt: when, - }) - require.NoError(t, err) - - var wrapper struct { - SDNs int - Errors []string - Timestamp time.Time - } - err = json.NewDecoder(bytes.NewReader(bs)).Decode(&wrapper) - require.NoError(t, err) - - require.Equal(t, 1, wrapper.SDNs) - require.Len(t, wrapper.Errors, 1) - require.Equal(t, when, wrapper.Timestamp) -} - -func TestSearcher__refreshInterval(t *testing.T) { - if v := getDataRefreshInterval(log.NewNopLogger(), ""); v.String() != "12h0m0s" { - t.Errorf("Got %v", v) - } - if v := getDataRefreshInterval(log.NewNopLogger(), "60s"); v.String() != "1m0s" { - t.Errorf("Got %v", v) - } - if v := getDataRefreshInterval(log.NewNopLogger(), "off"); v != 0*time.Second { - t.Errorf("got %v", v) - } - - // cover another branch - s := newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - s.periodicDataRefresh(0*time.Second, nil) -} - -func TestSearcher__refreshData(t *testing.T) { - s := createTestSearcher(t) // TODO(adam): initial setup - stats := testSearcherStats - - if len(s.Addresses) == 0 || stats.Addresses == 0 { - t.Errorf("empty Addresses=%d stats.Addresses=%d", len(s.Addresses), stats.Addresses) - } - if len(s.Alts) == 0 || stats.Alts == 0 { - t.Errorf("empty Alts=%d or stats.Alts=%d", len(s.Alts), stats.Alts) - } - if len(s.SDNs) == 0 || stats.SDNs == 0 { - t.Errorf("empty SDNs=%d or stats.SDNs=%d", len(s.SDNs), stats.SDNs) - } - if len(s.DPs) == 0 || stats.DeniedPersons == 0 { - t.Errorf("empty DPs=%d or stats.DeniedPersons=%d", len(s.DPs), stats.DeniedPersons) - } - if len(s.SSIs) == 0 || stats.SectoralSanctions == 0 { - t.Errorf("empty SSIs=%d or stats.SectoralSanctions=%d", len(s.SSIs), stats.SectoralSanctions) - } - if len(s.BISEntities) == 0 || stats.BISEntities == 0 { - t.Errorf("empty searcher.BISEntities=%d or stats.BISEntities=%d", len(s.BISEntities), stats.BISEntities) - } -} - -func TestDownload__lastRefresh(t *testing.T) { - start := time.Now() - time.Sleep(5 * time.Millisecond) // force start to be before our calls - - if when := lastRefresh(""); when.Before(start) { - t.Errorf("expected time.Now()") - } - - // make a temp dir (initially with nothing in it) - dir, err := os.MkdirTemp("", "lastRefresh") - if err != nil { - t.Fatal(err) - } - - if when := lastRefresh(dir); !when.IsZero() { - t.Errorf("expected zero time: %v", t) - } - - // add a file and get it's mtime - path := filepath.Join(dir, "out.txt") - if err := os.WriteFile(path, []byte("hello, world"), 0600); err != nil { - t.Fatal(err) - } - if info, err := os.Stat(path); err != nil { - t.Fatal(err) - } else { - if when := lastRefresh(dir); !when.Equal(info.ModTime()) { - t.Errorf("t=%v", when) - } - } -} - -func TestDownload__OFAC_Spillover(t *testing.T) { - logger := log.NewTestLogger() - initialDir := filepath.Join("..", "..", "test", "testdata", "static") - - res, err := ofacRecords(logger, initialDir) - require.NoError(t, err) - - var sdn *ofac.SDN - idx := slices.IndexFunc(res.SDNs, func(s *ofac.SDN) bool { - return s.EntityID == "12300" - }) - if idx >= 0 { - sdn = res.SDNs[idx] - } - require.NotNil(t, sdn) - - //nolint:misspell - expected := `DOB 13 May 1965; alt. DOB 13 Apr 1968; alt. DOB 07 Jul 1964; POB Medellin, Colombia; alt. POB Marinilla, Antioquia, Colombia; alt. POB Ciudad Victoria, Tamaulipas, Mexico; Cedula No. 7548733 (Colombia); alt. Cedula No. 70163752 (Colombia); alt. Cedula No. 172489729-1 (Ecuador); Passport AL720622 (Colombia); R.F.C. CIVJ650513LJA (Mexico); alt. R.F.C. OUSV-640707 (Mexico); C.U.R.P. CIVJ650513HNEFLR06 (Mexico); alt. C.U.R.P. OUVS640707HTSSLR07 (Mexico); Matricula Mercantil No 181301-1 Cali (Colombia); alt. Matricula Mercantil No 405885 Bogota (Colombia); Linked To: BIO FORESTAL S.A.S.; Linked To: CUBI CAFE CLICK CUBE MEXICO, S.A. DE C.V.; Linked To: DOLPHIN DIVE SCHOOL S.A.; Linked To: GANADERIA LA SORGUITA S.A.S.; Linked To: GESTORES DEL ECUADOR GESTORUM S.A.; Linked To: INVERPUNTO DEL VALLE S.A.; Linked To: INVERSIONES CIFUENTES Y CIA. S. EN C.; Linked To: LE CLAUDE, S.A. DE C.V.; Linked To: OPERADORA NUEVA GRANADA, S.A. DE C.V.; Linked To: PARQUES TEMATICOS S.A.S.; Linked To: PROMO RAIZ S.A.S.; Linked To: RED MUNDIAL INMOBILIARIA, S.A. DE C.V.; Linked To: FUNDACION PARA EL BIENESTAR Y EL PORVENIR; Linked To: C.I. METALURGIA EXTRACTIVA DE COLOMBIA S.A.S.; Linked To: GRUPO MUNDO MARINO, S.A.; Linked To: C.I. DISERCOM S.A.S.; Linked To: C.I. OKCOFFEE COLOMBIA S.A.S.; Linked To: C.I. OKCOFFEE INTERNATIONAL S.A.S.; Linked To: FUNDACION OKCOFFEE COLOMBIA; Linked To: CUBICAFE S.A.S.; Linked To: HOTELES Y BIENES S.A.; Linked To: FUNDACION SALVA LA SELVA; Linked To: LINEA AEREA PUEBLOS AMAZONICOS S.A.S.; Linked To: DESARROLLO MINERO RESPONSABLE C.I. S.A.S.; Linked To: R D I S.A.` - require.Equal(t, expected, sdn.Remarks) -} +// import ( +// "bytes" +// "encoding/json" +// "errors" +// "os" +// "path/filepath" +// "slices" +// "testing" +// "time" + +// "github.com/moov-io/base/log" +// "github.com/moov-io/watchman/pkg/ofac" + +// "github.com/stretchr/testify/require" +// ) + +// func TestDownloadStats(t *testing.T) { +// when := time.Date(2022, time.May, 21, 9, 4, 0, 0, time.UTC) +// bs, err := json.Marshal(&DownloadStats{ +// SDNs: 1, +// Errors: []error{ +// errors.New("bad thing"), +// }, +// RefreshedAt: when, +// }) +// require.NoError(t, err) + +// var wrapper struct { +// SDNs int +// Errors []string +// Timestamp time.Time +// } +// err = json.NewDecoder(bytes.NewReader(bs)).Decode(&wrapper) +// require.NoError(t, err) + +// require.Equal(t, 1, wrapper.SDNs) +// require.Len(t, wrapper.Errors, 1) +// require.Equal(t, when, wrapper.Timestamp) +// } + +// func TestSearcher__refreshInterval(t *testing.T) { +// if v := getDataRefreshInterval(log.NewNopLogger(), ""); v.String() != "12h0m0s" { +// t.Errorf("Got %v", v) +// } +// if v := getDataRefreshInterval(log.NewNopLogger(), "60s"); v.String() != "1m0s" { +// t.Errorf("Got %v", v) +// } +// if v := getDataRefreshInterval(log.NewNopLogger(), "off"); v != 0*time.Second { +// t.Errorf("got %v", v) +// } + +// // cover another branch +// s := newSearcher(log.NewNopLogger(), noLogPipeliner, 1) +// s.periodicDataRefresh(0*time.Second, nil) +// } + +// func TestSearcher__refreshData(t *testing.T) { +// s := createTestSearcher(t) // TODO(adam): initial setup +// stats := testSearcherStats + +// if len(s.Addresses) == 0 || stats.Addresses == 0 { +// t.Errorf("empty Addresses=%d stats.Addresses=%d", len(s.Addresses), stats.Addresses) +// } +// if len(s.Alts) == 0 || stats.Alts == 0 { +// t.Errorf("empty Alts=%d or stats.Alts=%d", len(s.Alts), stats.Alts) +// } +// if len(s.SDNs) == 0 || stats.SDNs == 0 { +// t.Errorf("empty SDNs=%d or stats.SDNs=%d", len(s.SDNs), stats.SDNs) +// } +// if len(s.DPs) == 0 || stats.DeniedPersons == 0 { +// t.Errorf("empty DPs=%d or stats.DeniedPersons=%d", len(s.DPs), stats.DeniedPersons) +// } +// if len(s.SSIs) == 0 || stats.SectoralSanctions == 0 { +// t.Errorf("empty SSIs=%d or stats.SectoralSanctions=%d", len(s.SSIs), stats.SectoralSanctions) +// } +// if len(s.BISEntities) == 0 || stats.BISEntities == 0 { +// t.Errorf("empty searcher.BISEntities=%d or stats.BISEntities=%d", len(s.BISEntities), stats.BISEntities) +// } +// } + +// func TestDownload__lastRefresh(t *testing.T) { +// start := time.Now() +// time.Sleep(5 * time.Millisecond) // force start to be before our calls + +// if when := lastRefresh(""); when.Before(start) { +// t.Errorf("expected time.Now()") +// } + +// // make a temp dir (initially with nothing in it) +// dir, err := os.MkdirTemp("", "lastRefresh") +// if err != nil { +// t.Fatal(err) +// } + +// if when := lastRefresh(dir); !when.IsZero() { +// t.Errorf("expected zero time: %v", t) +// } + +// // add a file and get it's mtime +// path := filepath.Join(dir, "out.txt") +// if err := os.WriteFile(path, []byte("hello, world"), 0600); err != nil { +// t.Fatal(err) +// } +// if info, err := os.Stat(path); err != nil { +// t.Fatal(err) +// } else { +// if when := lastRefresh(dir); !when.Equal(info.ModTime()) { +// t.Errorf("t=%v", when) +// } +// } +// } + +// func TestDownload__OFAC_Spillover(t *testing.T) { +// logger := log.NewTestLogger() +// initialDir := filepath.Join("..", "..", "test", "testdata", "static") + +// res, err := ofacRecords(logger, initialDir) +// require.NoError(t, err) + +// var sdn *ofac.SDN +// idx := slices.IndexFunc(res.SDNs, func(s *ofac.SDN) bool { +// return s.EntityID == "12300" +// }) +// if idx >= 0 { +// sdn = res.SDNs[idx] +// } +// require.NotNil(t, sdn) + +// //nolint:misspell +// expected := `DOB 13 May 1965; alt. DOB 13 Apr 1968; alt. DOB 07 Jul 1964; POB Medellin, Colombia; alt. POB Marinilla, Antioquia, Colombia; alt. POB Ciudad Victoria, Tamaulipas, Mexico; Cedula No. 7548733 (Colombia); alt. Cedula No. 70163752 (Colombia); alt. Cedula No. 172489729-1 (Ecuador); Passport AL720622 (Colombia); R.F.C. CIVJ650513LJA (Mexico); alt. R.F.C. OUSV-640707 (Mexico); C.U.R.P. CIVJ650513HNEFLR06 (Mexico); alt. C.U.R.P. OUVS640707HTSSLR07 (Mexico); Matricula Mercantil No 181301-1 Cali (Colombia); alt. Matricula Mercantil No 405885 Bogota (Colombia); Linked To: BIO FORESTAL S.A.S.; Linked To: CUBI CAFE CLICK CUBE MEXICO, S.A. DE C.V.; Linked To: DOLPHIN DIVE SCHOOL S.A.; Linked To: GANADERIA LA SORGUITA S.A.S.; Linked To: GESTORES DEL ECUADOR GESTORUM S.A.; Linked To: INVERPUNTO DEL VALLE S.A.; Linked To: INVERSIONES CIFUENTES Y CIA. S. EN C.; Linked To: LE CLAUDE, S.A. DE C.V.; Linked To: OPERADORA NUEVA GRANADA, S.A. DE C.V.; Linked To: PARQUES TEMATICOS S.A.S.; Linked To: PROMO RAIZ S.A.S.; Linked To: RED MUNDIAL INMOBILIARIA, S.A. DE C.V.; Linked To: FUNDACION PARA EL BIENESTAR Y EL PORVENIR; Linked To: C.I. METALURGIA EXTRACTIVA DE COLOMBIA S.A.S.; Linked To: GRUPO MUNDO MARINO, S.A.; Linked To: C.I. DISERCOM S.A.S.; Linked To: C.I. OKCOFFEE COLOMBIA S.A.S.; Linked To: C.I. OKCOFFEE INTERNATIONAL S.A.S.; Linked To: FUNDACION OKCOFFEE COLOMBIA; Linked To: CUBICAFE S.A.S.; Linked To: HOTELES Y BIENES S.A.; Linked To: FUNDACION SALVA LA SELVA; Linked To: LINEA AEREA PUEBLOS AMAZONICOS S.A.S.; Linked To: DESARROLLO MINERO RESPONSABLE C.I. S.A.S.; Linked To: R D I S.A.` +// require.Equal(t, expected, sdn.Remarks) +// } diff --git a/cmd/server/download_webhook.go b/cmd/server/download_webhook.go deleted file mode 100644 index 39bb9650..00000000 --- a/cmd/server/download_webhook.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "os" - "strings" - - "github.com/moov-io/base/log" -) - -func callDownloadWebook(logger log.Logger, stats *DownloadStats) error { - webhookURL := strings.TrimSpace(os.Getenv("DOWNLOAD_WEBHOOK_URL")) - webhookAuthToken := strings.TrimSpace(os.Getenv("DOWNLOAD_WEBHOOK_AUTH_TOKEN")) - - if webhookURL == "" { - return nil - } - logger.Info().Log("sending stats to download webhook url") - - var body bytes.Buffer - json.NewEncoder(&body).Encode(stats) - - statusCode, err := callWebhook(&body, webhookURL, webhookAuthToken) - if err != nil { - err = fmt.Errorf("problem calling download webhook: %w", err) - return logger.Error().LogError(err).Err() - } else { - logger.Info().Logf("http status code %d from download webhook", statusCode) - } - return nil -} diff --git a/cmd/server/filter.go b/cmd/server/filter.go deleted file mode 100644 index aac2d311..00000000 --- a/cmd/server/filter.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "net/url" - "strings" -) - -type filterRequest struct { - sdnType string - ofacProgram string -} - -func (req filterRequest) empty() bool { - return req.sdnType == "" && req.ofacProgram == "" -} - -func buildFilterRequest(u *url.URL) filterRequest { - return filterRequest{ - sdnType: u.Query().Get("sdnType"), - ofacProgram: u.Query().Get("ofacProgram"), - } -} - -func filterSDNs(sdns []*SDN, req filterRequest) []*SDN { - if req.empty() { - // short-circuit and return if we have no filters - return sdns - } - - keeper := keepSDN(req) - - var out []*SDN - for i := range sdns { - if keeper(sdns[i]) { - out = append(out, sdns[i]) - } - } - return out -} - -func keepSDN(req filterRequest) func(*SDN) bool { - return func(sdn *SDN) bool { - if req.empty() { - return true // short-circuit if we have no filters - } - // by default exclude the result (as at least one filter is non-empty) - keep := false - - // Look at all our filters - // If the filter is non-empty AND matches the SDN's field then keep it - // - // NOTE: If we add more filters don't forget to also add them in values.go - if req.sdnType != "" { - if sdn.SDNType != "" { - if strings.EqualFold(sdn.SDNType, req.sdnType) { - keep = true - } - } else { - // 'entity' is a special case value for ?sdnType in that it refers to a company or organization - // and not an individual, however OFAC's data files do not contain this value and we must infer - // it instead. - if sdn.SDNType == "" && strings.EqualFold(req.sdnType, "entity") { - keep = true - } else { - return false // skip this SDN as the filter didn't match - } - } - } - if req.ofacProgram != "" { - for j := range sdn.Programs { - if strings.EqualFold(sdn.Programs[j], req.ofacProgram) { - keep = true - } - } - } - return keep - } -} diff --git a/cmd/server/filter_test.go b/cmd/server/filter_test.go deleted file mode 100644 index 8ec49d38..00000000 --- a/cmd/server/filter_test.go +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "net/url" - "testing" - - "github.com/moov-io/watchman/pkg/ofac" -) - -func TestFilter__buildFilterRequest(t *testing.T) { - u, _ := url.Parse("/search?q=jane+doe&sdnType=individual&ofacProgram=SDGT") - req := buildFilterRequest(u) - if req.empty() { - t.Error("filterRequest is not empty") - } - if req.sdnType != "individual" { - t.Errorf("req.sdnType=%s", req.sdnType) - } - if req.ofacProgram != "SDGT" { - t.Errorf("req.ofacProgram=%s", req.ofacProgram) - } - - // just the sdnType filter - u, _ = url.Parse("/search?q=jane+doe&sdnType=individual") - req = buildFilterRequest(u) - if req.empty() { - t.Error("filterRequest is not empty") - } - if req.sdnType != "individual" { - t.Errorf("req.sdnType=%s", req.sdnType) - } - if req.ofacProgram != "" { - t.Errorf("req.ofacProgram=%s", req.ofacProgram) - } - - // empty request - u, _ = url.Parse("/search?q=jane+doe") - req = buildFilterRequest(u) - if !req.empty() { - t.Error("filterRequest is empty") - } - if req.sdnType != "" || req.ofacProgram != "" { - t.Errorf("req.sdnType=%s req.ofacProgram=%s", req.sdnType, req.ofacProgram) - } -} - -var ( - filterableSDNs = []*SDN{ - { - SDN: &ofac.SDN{ - EntityID: "12", - SDNName: "Jane Doe", - SDNType: "individual", - Programs: []string{"other", "barfoo"}, - }, - }, - { - SDN: &ofac.SDN{ - EntityID: "13", - SDNName: "EP-1111", - SDNType: "aircraft", - Programs: []string{"SDGT", "IRAN"}, - }, - }, - } - terrorGroupSDN = &SDN{ - SDN: &ofac.SDN{ - EntityID: "13", - SDNName: "Terror Group", - SDNType: "", - Programs: []string{"SDGT"}, - }, - } - oneEmptySDNType = []*SDN{ - { - SDN: &ofac.SDN{ - EntityID: "12", - SDNName: "Jane Doe", - SDNType: "individual", - Programs: []string{"other"}, - }, - }, - terrorGroupSDN, - } - missingSDNType = []*SDN{ - { - SDN: &ofac.SDN{ - EntityID: "14", - SDNName: "missing sdnType", - Programs: []string{"SDGT"}, - }, - }, - } - missingProgram = []*SDN{ - { - SDN: &ofac.SDN{ - EntityID: "15", - SDNName: "missing program", - SDNType: "individual", - }, - }, - } -) - -func TestFilter__sdnType(t *testing.T) { - sdns := filterSDNs(append(filterableSDNs, terrorGroupSDN), filterRequest{sdnType: "individual"}) - if len(sdns) != 1 { - t.Fatalf("got: %#v", sdns) - } - if sdns[0].EntityID != "12" { - t.Errorf("sdns[0].EntityID=%s", sdns[0].EntityID) - } - - sdns = filterSDNs(filterableSDNs, filterRequest{}) - if len(sdns) != 2 { - t.Errorf("got %#v", sdns) - } - - sdns = filterSDNs(filterableSDNs, filterRequest{sdnType: "other"}) - if len(sdns) != 0 { - t.Errorf("got %#v", sdns) - } -} - -func TestFilter__sdnTypeEntity(t *testing.T) { - sdns := filterSDNs(oneEmptySDNType, filterRequest{sdnType: "entity"}) - if len(sdns) != 1 { - t.Fatalf("got: %#v", sdns) - } - if sdns[0].EntityID != "13" { - t.Errorf("sdns[0].EntityID=%s", sdns[0].EntityID) - } -} - -func TestFilter__program(t *testing.T) { - sdns := filterSDNs(filterableSDNs, filterRequest{ofacProgram: "SDGT"}) - if len(sdns) != 1 { - t.Fatalf("got: %#v", sdns) - } - if sdns[0].EntityID != "13" { - t.Errorf("sdns[0].EntityID=%s", sdns[0].EntityID) - } - - sdns = filterSDNs(filterableSDNs, filterRequest{}) - if len(sdns) != 2 { - t.Errorf("got %#v", sdns) - } - - sdns = filterSDNs(filterableSDNs, filterRequest{ofacProgram: "unknown"}) - if len(sdns) != 0 { - t.Errorf("got %#v", sdns) - } -} - -func TestFilter__multiple(t *testing.T) { - sdns := filterSDNs(filterableSDNs, filterRequest{sdnType: "aircraft", ofacProgram: "SDGT"}) - if len(sdns) != 1 { - t.Errorf("got: %#v", sdns) - } - if sdns[0].EntityID != "13" { - t.Errorf("sdns[0].EntityID=%s", sdns[0].EntityID) - } - - sdns = filterSDNs(filterableSDNs, filterRequest{}) - if len(sdns) != 2 { - t.Errorf("got %#v", sdns) - } - - sdns = filterSDNs(filterableSDNs, filterRequest{sdnType: "other", ofacProgram: "unknown"}) - if len(sdns) != 0 { - t.Errorf("got %#v", sdns) - } -} - -func TestFilter__missing(t *testing.T) { - if len(missingSDNType) != 1 { - t.Fatalf("%#v", missingSDNType) - } - sdns := filterSDNs(append(filterableSDNs, missingSDNType...), filterRequest{sdnType: "individual"}) - if len(sdns) != 1 || sdns[0].EntityID != "12" { - t.Errorf("sdns=%#v", sdns) - } - - if len(missingProgram) != 1 { - t.Fatalf("%#v", missingProgram) - } - sdns = filterSDNs(append(filterableSDNs, missingProgram...), filterRequest{ofacProgram: "SDGT"}) - if len(sdns) != 1 || sdns[0].EntityID != "13" { - t.Errorf("sdns=%#v", sdns) - } -} diff --git a/cmd/server/http.go b/cmd/server/http.go deleted file mode 100644 index 1a94a708..00000000 --- a/cmd/server/http.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "net/http" - "regexp" - "strconv" - "strings" - - moovhttp "github.com/moov-io/base/http" - "github.com/moov-io/base/log" - - "github.com/go-kit/kit/metrics/prometheus" - stdprometheus "github.com/prometheus/client_golang/prometheus" -) - -var ( - routeHistogram = prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{ - Name: "http_response_duration_seconds", - Help: "Histogram representing the http response durations", - }, []string{"route"}) -) - -func wrapResponseWriter(logger log.Logger, w http.ResponseWriter, r *http.Request) http.ResponseWriter { - route := fmt.Sprintf("%s-%s", strings.ToLower(r.Method), cleanMetricsPath(r.URL.Path)) - return moovhttp.Wrap(logger, routeHistogram.With("route", route), w, r) -} - -var baseIDRegex = regexp.MustCompile(`([a-f0-9]{40})`) - -// cleanMetricsPath takes a URL path and formats it for Prometheus metrics -// -// This method replaces /'s with -'s and clean out OFAC ID's (which are numeric). -// This method also strips out moov/base.ID() values from URL path slugs. -func cleanMetricsPath(path string) string { - parts := strings.Split(path, "/") - var out []string - for i := range parts { - if n, _ := strconv.Atoi(parts[i]); n > 0 || parts[i] == "" { - continue // numeric ID - } - if baseIDRegex.MatchString(parts[i]) { - continue // assume it's a moov/base.ID() value - } - out = append(out, parts[i]) - } - return strings.Join(out, "-") -} diff --git a/cmd/server/http_test.go b/cmd/server/http_test.go deleted file mode 100644 index 49ef4169..00000000 --- a/cmd/server/http_test.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "testing" -) - -func TestHTTP__cleanMetricsPath(t *testing.T) { - if v := cleanMetricsPath("/v1/watchman/companies/1234"); v != "v1-watchman-companies" { - t.Errorf("got %q", v) - } - if v := cleanMetricsPath("/v1/watchman/ping"); v != "v1-watchman-ping" { - t.Errorf("got %q", v) - } - if v := cleanMetricsPath("/v1/watchman/customers/19636f90bc95779e2488b0f7a45c4b68958a2ddd"); v != "v1-watchman-customers" { - t.Errorf("got %q", v) - } - // A value which looks like moov/base.ID, but is off by one character (last letter) - if v := cleanMetricsPath("/v1/watchman/customers/19636f90bc95779e2488b0f7a45c4b68958a2ddz"); v != "v1-watchman-customers-19636f90bc95779e2488b0f7a45c4b68958a2ddz" { - t.Errorf("got %q", v) - } -} diff --git a/cmd/server/issue115_test.go b/cmd/server/issue115_test.go deleted file mode 100644 index c8bcd8dc..00000000 --- a/cmd/server/issue115_test.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "testing" - - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/internal/stringscore" - "github.com/moov-io/watchman/pkg/ofac" -) - -func TestIssue115__TopSDNs(t *testing.T) { - score := stringscore.JaroWinkler("georgehabbash", "georgebush") - eql(t, "george bush stringscore.JaroWinkler", score, 0.896) - - score = stringscore.JaroWinkler("g", "geoergebush") - eql(t, "g vs geoergebush", score, 0.070) - - pipe := noLogPipeliner - s := newSearcher(log.NewNopLogger(), pipe, 1) - keeper := keepSDN(filterRequest{}) - - // Issue 115 (https://github.com/moov-io/watchman/issues/115) talks about how "george bush" is a false positive (90%) match against - // several other "George ..." records. This is too sensitive and so we need to tone that down. - - // was 89.6% match - s.SDNs = precomputeSDNs([]*ofac.SDN{{EntityID: "2680", SDNName: "HABBASH, George", SDNType: "INDIVIDUAL"}}, nil, pipe) - - out := s.TopSDNs(1, 0.00, "george bush", keeper) - eql(t, "issue115: top SDN 2680", out[0].match, 0.687) - - // was 88.3% match - s.SDNs = precomputeSDNs([]*ofac.SDN{{EntityID: "9432", SDNName: "CHIWESHE, George", SDNType: "INDIVIDUAL"}}, nil, pipe) - - out = s.TopSDNs(1, 0.00, "george bush", keeper) - eql(t, "issue115: top SDN 18996", out[0].match, 0.686) - - // another example - s.SDNs = precomputeSDNs([]*ofac.SDN{{EntityID: "0", SDNName: "Bush, George W", SDNType: "INDIVIDUAL"}}, nil, pipe) - if s.SDNs[0].name != "george w bush" { - t.Errorf("s.SDNs[0].name=%s", s.SDNs[0].name) - } - - out = s.TopSDNs(1, 0.00, "george w bush", keeper) - eql(t, "issue115: top SDN 0", out[0].match, 1.0) - - out = s.TopSDNs(1, 0.00, "george bush", keeper) - eql(t, "issue115: top SDN 0", out[0].match, 0.986) -} diff --git a/cmd/server/issue326_test.go b/cmd/server/issue326_test.go deleted file mode 100644 index 0c51181d..00000000 --- a/cmd/server/issue326_test.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "testing" - - "github.com/moov-io/watchman/internal/prepare" - "github.com/moov-io/watchman/internal/stringscore" - - "github.com/stretchr/testify/assert" -) - -func TestIssue326(t *testing.T) { - india := prepare.LowerAndRemovePunctuation("Huawei Technologies India Private Limited") - investment := prepare.LowerAndRemovePunctuation("Huawei Technologies Investment Co. Ltd.") - - // Cuba - score := stringscore.JaroWinkler(prepare.LowerAndRemovePunctuation("Huawei Cuba"), prepare.LowerAndRemovePunctuation("Huawei")) - assert.Equal(t, 0.7444444444444445, score) - - // India - score = stringscore.JaroWinkler(india, prepare.LowerAndRemovePunctuation("Huawei")) - assert.Equal(t, 0.4846031746031746, score) - score = stringscore.JaroWinkler(india, prepare.LowerAndRemovePunctuation("Huawei Technologies")) - assert.Equal(t, 0.6084415584415584, score) - - // Investment - score = stringscore.JaroWinkler(investment, prepare.LowerAndRemovePunctuation("Huawei")) - assert.Equal(t, 0.3788888888888889, score) - score = stringscore.JaroWinkler(investment, prepare.LowerAndRemovePunctuation("Huawei Technologies")) - assert.Equal(t, 0.5419191919191919, score) -} diff --git a/cmd/server/largest.go b/cmd/server/largest.go deleted file mode 100644 index 38a3b066..00000000 --- a/cmd/server/largest.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "slices" - "sync" -) - -// item represents an arbitrary value with an associated weight -type item struct { - matched string - value interface{} - weight float64 -} - -// newLargest returns a `largest` instance which can be used to track items with the highest weights -func newLargest(capacity int, minMatch float64) *largest { - return &largest{ - items: make([]*item, capacity), - capacity: capacity, - minMatch: minMatch, - } -} - -// largest keeps track of a set of items with the lowest weights. This is used to -// find the largest weighted values out of a much larger set. -type largest struct { - items []*item - capacity int - minMatch float64 - mu sync.Mutex -} - -func (xs *largest) add(it *item) { - if it.weight < xs.minMatch { - return // skip item as it's below our threshold - } - - xs.mu.Lock() - defer xs.mu.Unlock() - - for i := range xs.items { - if xs.items[i] == nil { - xs.items[i] = it // insert if we found empty slot - break - } - if xs.items[i].weight < it.weight { - xs.items = slices.Insert(xs.items, i, it) - break - } - } - if len(xs.items) > xs.capacity { - xs.items = xs.items[:xs.capacity] - } -} diff --git a/cmd/server/largest_test.go b/cmd/server/largest_test.go deleted file mode 100644 index 8b996b3f..00000000 --- a/cmd/server/largest_test.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "crypto/rand" - "fmt" - "math" - "math/big" - "testing" - - "github.com/stretchr/testify/require" - "golang.org/x/sync/errgroup" -) - -func randomWeight() float64 { - n, _ := rand.Int(rand.Reader, big.NewInt(1000)) - return float64(n.Int64()) / 100.0 -} - -func TestLargest(t *testing.T) { - xs := newLargest(10, 0.0) - - min := 10000.0 - for i := 0; i < 1000; i++ { - it := &item{ - value: i, - weight: randomWeight(), - } - xs.add(it) - min = math.Min(min, it.weight) - } - - // Check we didn't overflow - require.Equal(t, len(xs.items), xs.capacity) - - for i := range xs.items { - if i+1 > len(xs.items)-1 { - continue // don't hit index out of bounds - } - if xs.items[i].weight < 0.0001 { - t.Fatalf("weight of %.2f is too low", xs.items[i].weight) - } - if xs.items[i].weight < xs.items[i+1].weight { - t.Errorf("xs.items[%d].weight=%.2f < xs.items[%d].weight=%.2f", i, xs.items[i].weight, i+1, xs.items[i+1].weight) - } - } -} - -// TestLargest_MaxOrdering will test the ordering of 1.0 values to see -// if they hold their insert ordering. -func TestLargest_MaxOrdering(t *testing.T) { - xs := newLargest(10, 0.0) - - xs.add(&item{value: "A", weight: 0.99}) - xs.add(&item{value: "B", weight: 1.0}) - xs.add(&item{value: "C", weight: 1.0}) - xs.add(&item{value: "D", weight: 1.0}) - xs.add(&item{value: "E", weight: 0.97}) - - if n := len(xs.items); n != 10 { - t.Fatalf("found %d items: %#v", n, xs.items) - } - - if s, ok := xs.items[0].value.(string); !ok || s != "B" { - t.Errorf("xs.items[0]=%#v", xs.items[0]) - } - if s, ok := xs.items[1].value.(string); !ok || s != "C" { - t.Errorf("xs.items[1]=%#v", xs.items[1]) - } - if s, ok := xs.items[2].value.(string); !ok || s != "D" { - t.Errorf("xs.items[2]=%#v", xs.items[2]) - } - if s, ok := xs.items[3].value.(string); !ok || s != "A" { - t.Errorf("xs.items[3]=%#v", xs.items[3]) - } - if s, ok := xs.items[4].value.(string); !ok || s != "E" { - t.Errorf("xs.items[4]=%#v", xs.items[4]) - } - for i := 5; i < 10; i++ { - if xs.items[i] != nil { - t.Errorf("#%d was non-nil: %#v", i, xs.items[i]) - } - } -} - -func TestLargest__MinMatch(t *testing.T) { - xs := newLargest(2, 0.96) - - xs.add(&item{value: "A", weight: 0.94}) - xs.add(&item{value: "B", weight: 1.0}) - xs.add(&item{value: "C", weight: 0.95}) - xs.add(&item{value: "D", weight: 0.09}) - - require.Equal(t, "B", xs.items[0].value) - require.Nil(t, xs.items[1]) -} - -func BenchmarkLargest(b *testing.B) { - size := b.N * 500_000 - - scores := make([]float64, size) - for i := 0; i < b.N; i++ { - n, err := rand.Int(rand.Reader, big.NewInt(100)) - if err != nil { - b.Fatal(err) - } - scores[i] = float64(n.Int64()) / 100.0 - } - - limit := 20 - matches := []float64{0.1, 0.25, 0.5, 0.75, 0.9, 0.99} - for i := range matches { - b.Run(fmt.Sprintf("%.2f%%", matches[i]*100), func(b *testing.B) { - // accumulate scores - xs := newLargest(limit, matches[i]) - - g := &errgroup.Group{} - for i := range scores { - score := scores[i] - g.Go(func() error { - xs.add(&item{ - value: SDN{ - name: fmt.Sprintf("%.2f", score), - }, - weight: score, - }) - return nil - }) - } - require.NoError(b, g.Wait()) - require.Len(b, xs.items, limit) - require.Equal(b, limit, cap(xs.items)) - }) - } -} diff --git a/cmd/server/main.go b/cmd/server/main.go index a98c8520..94f7fdd9 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -5,116 +5,72 @@ package main import ( + "cmp" "context" "crypto/tls" - "flag" "fmt" "net/http" "os" "os/signal" - "path/filepath" - "runtime" - "strconv" - "strings" "syscall" "time" - "github.com/moov-io/base/admin" - moovhttp "github.com/moov-io/base/http" - "github.com/moov-io/base/http/bind" - "github.com/moov-io/base/log" "github.com/moov-io/watchman" - "github.com/moov-io/watchman/internal/prepare" - searchv2 "github.com/moov-io/watchman/internal/search" - "github.com/moov-io/watchman/pkg/ofac" - pubsearch "github.com/moov-io/watchman/pkg/search" + "github.com/moov-io/watchman/internal/download" + "github.com/moov-io/watchman/internal/search" "github.com/gorilla/mux" -) - -var ( - httpAddr = flag.String("http.addr", bind.HTTP("ofac"), "HTTP listen address") - adminAddr = flag.String("admin.addr", bind.Admin("ofac"), "Admin HTTP listen address") - - flagBasePath = flag.String("base-path", "/", "Base path to serve HTTP routes and webui from") - flagLogFormat = flag.String("log.format", "", "Format for log lines (Options: json, plain") - flagMaxProcs = flag.Int("max-procs", runtime.NumCPU(), "Maximum number of CPUs used for search and endpoints") - flagWorkers = flag.Int("workers", 1024, "Maximum number of goroutines used for search") - - dataRefreshInterval = 12 * time.Hour + "github.com/moov-io/base/admin" + "github.com/moov-io/base/log" ) func main() { - flag.Parse() - - runtime.GOMAXPROCS(*flagMaxProcs) + logger := log.NewDefaultLogger().With(log.Fields{ + "app": log.String("watchman"), + "version": log.String(watchman.Version), + }) + logger.Log("Starting watchman server") - var logger log.Logger - if v := os.Getenv("LOG_FORMAT"); v != "" { - *flagLogFormat = v - } - if strings.ToLower(*flagLogFormat) == "json" { - logger = log.NewJSONLogger() - } else { - logger = log.NewDefaultLogger() + config, err := LoadConfig(logger) + if err != nil { + logger.Fatal().LogErrorf("problem loading config: %v", err) + os.Exit(1) } - logger.Logf("Starting watchman server version %s", watchman.Version) + downloader, err := download.NewDownloader(logger, config.Download) + if err != nil { + logger.Fatal().LogErrorf("problem setting up downloader: %v", err) + os.Exit(1) + } - // Channel for errors - errs := make(chan error) + // Setup signal listener + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() - go func() { - c := make(chan os.Signal, 1) - signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) - errs <- fmt.Errorf("signal: %v", <-c) - }() + // Set up a channel to listen for system interrupt signals + ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) + defer stop() - // Setup business HTTP routes - if v := os.Getenv("BASE_PATH"); v != "" { - *flagBasePath = v + // Setup search service and endpoints + searchService := search.NewService(logger) + err = setupPeriodicRefreshing(ctx, logger, config.Download, downloader, searchService) + if err != nil { + logger.Fatal().LogErrorf("problem during initial download: %v", err) + os.Exit(1) } - router := mux.NewRouter().PathPrefix(*flagBasePath).Subrouter() - moovhttp.AddCORSHandler(router) - addPingRoute(router) - - // Start business HTTP server - readTimeout, _ := time.ParseDuration("30s") - writTimeout, _ := time.ParseDuration("30s") - idleTimeout, _ := time.ParseDuration("60s") - // Check to see if our -http.addr flag has been overridden - if v := os.Getenv("HTTP_BIND_ADDRESS"); v != "" { - *httpAddr = v - } + router := mux.NewRouter() + addPingRoute(router) - serve := &http.Server{ - Addr: *httpAddr, - Handler: router, - TLSConfig: &tls.Config{ - InsecureSkipVerify: false, - PreferServerCipherSuites: true, - MinVersion: tls.VersionTLS12, - }, - ReadTimeout: readTimeout, - ReadHeaderTimeout: readTimeout, - WriteTimeout: writTimeout, - IdleTimeout: idleTimeout, - } - shutdownServer := func() { - if err := serve.Shutdown(context.TODO()); err != nil { - logger.LogError(err) - } - } + searchController := search.NewController(logger, searchService) + searchController.AppendRoutes(router) - // Check to see if our -admin.addr flag has been overridden - if v := os.Getenv("HTTP_ADMIN_BIND_ADDRESS"); v != "" { - *adminAddr = v - } + // Listen for errors + errs := make(chan error) // Start Admin server (with Prometheus metrics) adminServer, err := admin.New(admin.Opts{ - Addr: *adminAddr, + Addr: config.Servers.AdminAddress, }) if err != nil { errs <- fmt.Errorf("problem starting admin server: %v", err) @@ -122,92 +78,43 @@ func main() { adminServer.AddVersionHandler(watchman.Version) // Setup 'GET /version' go func() { logger.Logf("listening on %s", adminServer.BindAddr()) + if err := adminServer.Listen(); err != nil { - err = fmt.Errorf("problem starting admin http: %v", err) - logger.LogError(err) - errs <- fmt.Errorf("admin shutdown: %v", err) + errs <- logger.Error().LogErrorf("admin server shutdown: %v", err).Err() } }() defer adminServer.Shutdown() - var pipeline *prepare.Pipeliner - if debug, err := strconv.ParseBool(os.Getenv("DEBUG_NAME_PIPELINE")); debug && err == nil { - pipeline = prepare.NewPipeliner(logger, true) - } else { - pipeline = prepare.NewPipeliner(log.NewNopLogger(), false) + // Setup HTTP server + defaultTimeout := 20 * time.Second + serve := &http.Server{ + Addr: config.Servers.BindAddress, + Handler: router, + TLSConfig: &tls.Config{ + InsecureSkipVerify: false, + PreferServerCipherSuites: true, + MinVersion: tls.VersionTLS12, + }, + ReadTimeout: defaultTimeout, + ReadHeaderTimeout: defaultTimeout, + WriteTimeout: defaultTimeout, + IdleTimeout: defaultTimeout, } - - searchWorkers := readInt(os.Getenv("SEARCH_MAX_WORKERS"), *flagWorkers) - searcher := newSearcher(logger, pipeline, searchWorkers) - - // Add debug routes - adminServer.AddHandler(debugSDNPath, debugSDNHandler(logger, searcher)) - - // Initial download of data - if stats, err := searcher.refreshData(os.Getenv("INITIAL_DATA_DIRECTORY")); err != nil { - logger.LogErrorf("ERROR: failed to download/parse initial data: %v", err) - os.Exit(1) - } else { - logger.Info().With(log.Fields{ - "SDNs": log.Int(stats.SDNs), - "AltNames": log.Int(stats.Alts), - "Addresses": log.Int(stats.Addresses), - "SSI": log.Int(stats.SectoralSanctions), - "DPL": log.Int(stats.DeniedPersons), - "BISEntities": log.Int(stats.BISEntities), - "UVL": log.Int(stats.Unverified), - "ISN": log.Int(stats.NonProliferationSanctions), - "FSE": log.Int(stats.ForeignSanctionsEvaders), - "PLC": log.Int(stats.PalestinianLegislativeCouncil), - "CAP": log.Int(stats.CAPTA), - "DTC": log.Int(stats.ITARDebarred), - "CMIC": log.Int(stats.ChineseMilitaryIndustrialComplex), - "NS_MBS": log.Int(stats.NonSDNMenuBasedSanctions), - "EU_CSL": log.Int(stats.EUCSL), - "UK_CSL": log.Int(stats.UKCSL), - "UK_SanctionsList": log.Int(stats.UKSanctionsList), - }).Logf("data refreshed %v ago", time.Since(stats.RefreshedAt)) + shutdownServer := func() { + serve.Shutdown(context.TODO()) } - // Setup periodic download and re-search - updates := make(chan *DownloadStats) - dataRefreshInterval = getDataRefreshInterval(logger, os.Getenv("DATA_REFRESH_INTERVAL")) - go searcher.periodicDataRefresh(dataRefreshInterval, updates) - go handleDownloadStats(updates, func(stats *DownloadStats) { - callDownloadWebook(logger, stats) - }) - - // Add manual data refresh endpoint - adminServer.AddHandler(manualRefreshPath, manualRefreshHandler(logger, searcher, updates)) - - // Add searcher for HTTP routes - addSDNRoutes(logger, router, searcher) - addSearchRoutes(logger, router, searcher) - addValuesRoutes(logger, router, searcher) - - var genericEntities []pubsearch.Entity[pubsearch.Value] - - genericOFACEntities := groupOFACRecords(searcher) - genericEntities = append(genericEntities, genericOFACEntities...) - - v2SearchService := searchv2.NewService(logger, genericEntities) - addSearchV2Routes(logger, router, v2SearchService) - - // Setup our web UI to be served as well - setupWebui(logger, router, *flagBasePath) - // Start business logic HTTP server go func() { - if certFile, keyFile := os.Getenv("HTTPS_CERT_FILE"), os.Getenv("HTTPS_KEY_FILE"); certFile != "" && keyFile != "" { - logger.Logf("binding to %s for secure HTTP server", *httpAddr) - if err := serve.ListenAndServeTLS(certFile, keyFile); err != nil { - logger.LogErrorf("https shutdown: %v", err) - } + certFile := cmp.Or(os.Getenv("HTTPS_CERT_FILE"), config.Servers.TLSCertFile) + keyFile := cmp.Or(os.Getenv("HTTPS_KEY_FILE"), config.Servers.TLSKeyFile) + + if certFile != "" && keyFile != "" { + logger.Logf("binding to %s for secure HTTP server", config.Servers.BindAddress) + errs <- serve.ListenAndServeTLS(certFile, keyFile) } else { - logger.Logf("binding to %s for HTTP server", *httpAddr) - if err := serve.ListenAndServe(); err != nil { - logger.LogErrorf("http shutdown: %v", err) - } + logger.Logf("binding to %s for HTTP server", config.Servers.BindAddress) + errs <- serve.ListenAndServe() } }() @@ -220,106 +127,8 @@ func main() { func addPingRoute(r *mux.Router) { r.Methods("GET").Path("/ping").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - moovhttp.SetAccessControlAllowHeaders(w, r.Header.Get("Origin")) w.Header().Set("Content-Type", "text/plain") w.WriteHeader(http.StatusOK) w.Write([]byte("PONG")) }) } - -// getDataRefreshInterval returns a time.Duration for how often OFAC should refresh data -// -// env is the value from an environmental variable -func getDataRefreshInterval(logger log.Logger, env string) time.Duration { - if env != "" { - if strings.EqualFold(env, "off") { - return 0 * time.Second - } - if dur, _ := time.ParseDuration(env); dur > 0 { - logger.Logf("Setting data refresh interval to %v", dur) - return dur - } - } - logger.Logf("Setting data refresh interval to %v (default)", dataRefreshInterval) - return dataRefreshInterval -} - -func setupWebui(logger log.Logger, r *mux.Router, basePath string) { - var disableWebUI bool - if val, err := strconv.ParseBool(os.Getenv("DISABLE_WEB_UI")); err == nil { - disableWebUI = val - } - - if disableWebUI { - logger.Log("Disabling webui") - return - } - - dir := os.Getenv("WEB_ROOT") - if dir == "" { - dir = filepath.Join("webui", "build") - } - if _, err := os.Stat(dir); err != nil { - logger.Logf("problem with webui=%s: %v", dir, err) - os.Exit(1) - } - r.PathPrefix("/").Handler(http.StripPrefix(basePath, http.FileServer(http.Dir(dir)))) -} - -func handleDownloadStats(updates chan *DownloadStats, handle func(stats *DownloadStats)) { - for { - stats := <-updates - if stats != nil { - handle(stats) - } - } -} - -func addSearchV2Routes(logger log.Logger, r *mux.Router, service searchv2.Service) { - searchv2.NewController(logger, service).AppendRoutes(r) -} - -func groupOFACRecords(searcher *searcher) []pubsearch.Entity[pubsearch.Value] { // TODO(adam): remove (refactor) - var sdns []ofac.SDN - var addrs []ofac.Address - var alts []ofac.AlternateIdentity - var comments []ofac.SDNComments - - for _, sdn := range searcher.SDNs { - if sdn == nil || sdn.SDN == nil { - continue - } - sdns = append(sdns, *sdn.SDN) - } - for _, addr := range searcher.Addresses { - if addr == nil || addr.Address == nil { - continue - } - addrs = append(addrs, *addr.Address) - } - for _, alt := range searcher.Alts { - if alt == nil || alt.AlternateIdentity == nil { - continue - } - alts = append(alts, *alt.AlternateIdentity) - } - for _, comment := range searcher.SDNComments { - if comment == nil { - continue - } - comments = append(comments, *comment) - } - - return ofac.GroupIntoEntities(sdns, addrs, comments, alts) -} - -func readInt(override string, value int) int { - if override != "" { - n, err := strconv.ParseInt(override, 10, 32) - if err != nil { - panic(fmt.Errorf("unable to parse %q as int", override)) //nolint:forbidigo - } - return int(n) - } - return value -} diff --git a/cmd/server/main_test.go b/cmd/server/main_test.go index b9cae44e..a024a5ab 100644 --- a/cmd/server/main_test.go +++ b/cmd/server/main_test.go @@ -4,30 +4,30 @@ package main -import ( - "sync" - "testing" +// import ( +// "sync" +// "testing" - "github.com/stretchr/testify/require" -) +// "github.com/stretchr/testify/require" +// ) -func TestHandleDownloadStats(t *testing.T) { - updates := make(chan *DownloadStats) +// func TestHandleDownloadStats(t *testing.T) { +// updates := make(chan *DownloadStats) - var wg sync.WaitGroup // make the race detector happy +// var wg sync.WaitGroup // make the race detector happy - var received *DownloadStats - go handleDownloadStats(updates, func(stats *DownloadStats) { - received = stats - wg.Done() - }) +// var received *DownloadStats +// go handleDownloadStats(updates, func(stats *DownloadStats) { +// received = stats +// wg.Done() +// }) - wg.Add(1) - updates <- &DownloadStats{ - SDNs: 123, - } - wg.Wait() +// wg.Add(1) +// updates <- &DownloadStats{ +// SDNs: 123, +// } +// wg.Wait() - require.NotNil(t, received) - require.Equal(t, 123, received.SDNs) -} +// require.NotNil(t, received) +// require.Equal(t, 123, received.SDNs) +// } diff --git a/cmd/server/sdn.go b/cmd/server/sdn.go deleted file mode 100644 index c63da024..00000000 --- a/cmd/server/sdn.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "errors" - "net/http" - - moovhttp "github.com/moov-io/base/http" - "github.com/moov-io/base/log" - - "github.com/gorilla/mux" -) - -var ( - errNoSDNId = errors.New("no SDN Id provided") -) - -func addSDNRoutes(logger log.Logger, r *mux.Router, searcher *searcher) { - r.Methods("GET").Path("/ofac/sdn/{sdnId}/addresses").HandlerFunc(getSDNAddresses(logger, searcher)) - r.Methods("GET").Path("/ofac/sdn/{sdnId}/alts").HandlerFunc(getSDNAltNames(logger, searcher)) - r.Methods("GET").Path("/ofac/sdn/{sdnId}").HandlerFunc(getSDN(logger, searcher)) -} - -func getSDNId(w http.ResponseWriter, r *http.Request) string { - v, ok := mux.Vars(r)["sdnId"] - if !ok || v == "" { - moovhttp.Problem(w, errNoSDNId) - return "" - } - return v -} - -func getSDNAddresses(logger log.Logger, searcher *searcher) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - w = wrapResponseWriter(logger, w, r) - - id, limit := getSDNId(w, r), extractSearchLimit(r) - if id == "" { - return - } - - addresses := searcher.FindAddresses(limit, id) - - logger.Info().With(log.Fields{ - "requestID": log.String(moovhttp.GetRequestID(r)), - }).Logf("get sdn=%s addresses", id) - - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(http.StatusOK) - if err := json.NewEncoder(w).Encode(addresses); err != nil { - moovhttp.Problem(w, err) - return - } - } -} - -func getSDNAltNames(logger log.Logger, searcher *searcher) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - w = wrapResponseWriter(logger, w, r) - - id, limit := getSDNId(w, r), extractSearchLimit(r) - if id == "" { - return - } - - alts := searcher.FindAlts(limit, id) - - logger.Info().With(log.Fields{ - "requestID": log.String(moovhttp.GetRequestID(r)), - }).Logf("get sdn=%s alt names", id) - - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(http.StatusOK) - if err := json.NewEncoder(w).Encode(alts); err != nil { - moovhttp.Problem(w, err) - return - } - } -} - -func getSDN(logger log.Logger, searcher *searcher) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - w = wrapResponseWriter(logger, w, r) - - id := getSDNId(w, r) - if id == "" { - return - } - sdn := searcher.FindSDN(id) - if sdn == nil { - w.WriteHeader(http.StatusNotFound) - return - } - - logger.Info().With(log.Fields{ - "requestID": log.String(moovhttp.GetRequestID(r)), - }).Logf("get sdn=%s", id) - - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(http.StatusOK) - if err := json.NewEncoder(w).Encode(sdn); err != nil { - moovhttp.Problem(w, err) - return - } - } -} diff --git a/cmd/server/sdn_test.go b/cmd/server/sdn_test.go deleted file mode 100644 index 076771b8..00000000 --- a/cmd/server/sdn_test.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/pkg/ofac" - - "github.com/gorilla/mux" -) - -func TestSDN__id(t *testing.T) { - router := mux.NewRouter() - - // Happy path - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/ofac/sdn/random-sdn-id", nil) - router.Methods("GET").Path("/sdn/{sdnId}").HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { - if v := getSDNId(w, r); v != "random-sdn-id" { - t.Errorf("got %s", v) - } - if w.Code != http.StatusOK { - t.Errorf("got status code %d", w.Code) - } - }) - router.ServeHTTP(w, req) - - // Unhappy case - w = httptest.NewRecorder() - req = httptest.NewRequest("GET", "/sdn", nil) - router.Methods("GET").Path("/sdn/{sdnId}").HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { - if v := getSDNId(w, req); v != "" { - t.Errorf("didn't expect SDN, got %s", v) - } - if w.Code != http.StatusBadRequest { - t.Errorf("got status code %d", w.Code) - } - }) - router.ServeHTTP(w, req) - - // Don't pass req through mux so mux.Vars finds nothing - if v := getSDNId(w, req); v != "" { - t.Errorf("expected empty, but got %q", v) - } -} - -func TestSDN__Address(t *testing.T) { - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/ofac/sdn/173/addresses", nil) - req.Header.Set("x-user-id", "test") - - router := mux.NewRouter() - addSDNRoutes(log.NewNopLogger(), router, addressSearcher) - router.ServeHTTP(w, req) - w.Flush() - - if w.Code != http.StatusOK { - t.Errorf("bogus status code: %d", w.Code) - } - - var addresses []*ofac.Address - if err := json.NewDecoder(w.Body).Decode(&addresses); err != nil { - t.Fatal(err) - } - if len(addresses) != 1 { - t.Fatalf("got %#v", addresses) - } - if addresses[0].EntityID != "173" { - t.Errorf("got %s", addresses[0].EntityID) - } -} - -func TestSDN__AltNames(t *testing.T) { - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/ofac/sdn/4691/alts", nil) - req.Header.Set("x-user-id", "test") - - router := mux.NewRouter() - addSDNRoutes(log.NewNopLogger(), router, altSearcher) - router.ServeHTTP(w, req) - w.Flush() - - if w.Code != http.StatusOK { - t.Errorf("bogus status code: %d", w.Code) - } - - var alts []*ofac.AlternateIdentity - if err := json.NewDecoder(w.Body).Decode(&alts); err != nil { - t.Fatal(err) - } - if len(alts) != 1 { - t.Fatalf("got %#v", alts) - } - if alts[0].EntityID != "4691" { - t.Errorf("got %s", alts[0].EntityID) - } -} - -func TestSDN__Get(t *testing.T) { - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/ofac/sdn/2681", nil) - req.Header.Set("x-user-id", "test") - - router := mux.NewRouter() - addSDNRoutes(log.NewNopLogger(), router, sdnSearcher) - router.ServeHTTP(w, req) - w.Flush() - - if w.Code != http.StatusOK { - t.Errorf("bogus status code: %d", w.Code) - } - - var sdn *ofac.SDN - if err := json.NewDecoder(w.Body).Decode(&sdn); err != nil { - t.Fatal(err) - } - if sdn == nil || sdn.EntityID != "2681" { - t.Errorf("got %#v", sdn) - } -} diff --git a/cmd/server/search.go b/cmd/server/search.go deleted file mode 100644 index 6c103562..00000000 --- a/cmd/server/search.go +++ /dev/null @@ -1,674 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "encoding/json" - "errors" - "strconv" - "strings" - "sync" - "time" - - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/internal/prepare" - "github.com/moov-io/watchman/internal/stringscore" - "github.com/moov-io/watchman/pkg/csl_eu" - "github.com/moov-io/watchman/pkg/csl_uk" - "github.com/moov-io/watchman/pkg/csl_us" - "github.com/moov-io/watchman/pkg/dpl" - "github.com/moov-io/watchman/pkg/ofac" - - "go4.org/syncutil" -) - -var ( - errNoSearchParams = errors.New("missing search parameter(s)") - - softResultsLimit, hardResultsLimit = 10, 100 -) - -// searcher holds prepare.LowerAndRemovePunctuationd data for each object available to search against. -// This data comes from various US and EU Federal agencies -type searcher struct { - // OFAC - SDNs []*SDN - Addresses []*Address - Alts []*Alt - SDNComments []*ofac.SDNComments - - // BIS - DPs []*DP - - // TODO: this could be refactored into sub structs that have us/eu (and eventually others) - - // US Consolidated Screening List - BISEntities []*Result[csl_us.EL] - MilitaryEndUsers []*Result[csl_us.MEU] - SSIs []*Result[csl_us.SSI] - UVLs []*Result[csl_us.UVL] - ISNs []*Result[csl_us.ISN] - FSEs []*Result[csl_us.FSE] - PLCs []*Result[csl_us.PLC] - CAPs []*Result[csl_us.CAP] - DTCs []*Result[csl_us.DTC] - CMICs []*Result[csl_us.CMIC] - NS_MBSs []*Result[csl_us.NS_MBS] - - // EU Consolidated List of Sactions - EUCSL []*Result[csl_eu.CSLRecord] - - // UK Consolidated List of Sactions - OFSI - UKCSL []*Result[csl_uk.CSLRecord] - - // UK Sanctions List - UKSanctionsList []*Result[csl_uk.SanctionsListRecord] - - // metadata - lastRefreshedAt time.Time - sync.RWMutex // protects all above fields - *syncutil.Gate // limits concurrent processing - - pipe *prepare.Pipeliner - - logger log.Logger -} - -func newSearcher(logger log.Logger, pipeline *prepare.Pipeliner, workers int) *searcher { - logger.Logf("allowing only %d workers for search", workers) - return &searcher{ - logger: logger.With(log.Fields{ - "component": log.String("pipeline"), - }), - pipe: pipeline, - Gate: syncutil.NewGate(workers), - } -} - -func (s *searcher) FindAddresses(limit int, id string) []*ofac.Address { - s.RLock() - defer s.RUnlock() - - var out []*ofac.Address - for i := range s.Addresses { - if len(out) > limit { - break - } - if s.Addresses[i].Address.EntityID == id { - out = append(out, s.Addresses[i].Address) - } - } - return out -} - -func (s *searcher) TopAddresses(limit int, minMatch float64, reqAddress string) []Address { - s.RLock() - defer s.RUnlock() - - return TopAddressesFn(limit, minMatch, s.Addresses, topAddressesAddress(reqAddress)) -} - -var ( - // topAddressesAddress is a compare method for TopAddressesFn to extract and rank .Address - topAddressesAddress = func(needleAddr string) func(*Address) *item { - return func(add *Address) *item { - return &item{ - value: add, - weight: stringscore.JaroWinkler(add.address, prepare.LowerAndRemovePunctuation(needleAddr)), - } - } - } - - // topAddressesCityState is a compare method for TopAddressesFn to extract and rank - // .City, .State, .Providence, and .Zip to return the average match between non-empty - // search criteria. - topAddressesCityState = func(needleCityState string) func(*Address) *item { - return func(add *Address) *item { - return &item{ - value: add, - weight: stringscore.JaroWinkler(add.citystate, prepare.LowerAndRemovePunctuation(needleCityState)), - } - } - } - - // topAddressesCountry is a compare method for TopAddressesFn to extract and rank .Country - topAddressesCountry = func(needleCountry string) func(*Address) *item { - return func(add *Address) *item { - return &item{ - value: add, - weight: stringscore.JaroWinkler(add.country, prepare.LowerAndRemovePunctuation(needleCountry)), - } - } - } - - // multiAddressCompare is a compare method for taking N higher-order compare methods - // and returning an average weight after computing them all. - multiAddressCompare = func(cmps ...func(*Address) *item) func(*Address) *item { - return func(add *Address) *item { - weight := 0.00 - for i := range cmps { - weight += cmps[i](add).weight - } - return &item{ - value: add, - weight: weight / float64(len(cmps)), - } - } - } -) - -// FilterCountries returns Addresses that match a given country name. -// -// If name is blank all Addresses are returned. -// -// This filtering ignore case differences, but does require the name matches -// to the underlying data. -func (s *searcher) FilterCountries(name string) []*Address { - s.RLock() - defer s.RUnlock() - - if len(s.Addresses) == 0 { - return nil - } - - if name == "" { - out := make([]*Address, len(s.Addresses)) - copy(out, s.Addresses) - return out - } - var out []*Address - for i := range s.Addresses { - if strings.EqualFold(s.Addresses[i].country, name) { - out = append(out, s.Addresses[i]) - } - } - return out -} - -// TopAddressesFn performs a ranked search over an arbitrary set of Address fields. -// -// compare takes an Address (from s.Addresses) and is expected to extract some property to be compared -// against a captured parameter (in a closure calling compare) to return an *item for final sorting. -// See searchByAddress in search_handlers.go for an example -func TopAddressesFn(limit int, minMatch float64, addresses []*Address, compare func(*Address) *item) []Address { - if len(addresses) == 0 { - return nil - } - xs := newLargest(limit, minMatch) - - var wg sync.WaitGroup - wg.Add(len(addresses)) - - for i := range addresses { - go func(i int) { - defer wg.Done() - xs.add(compare(addresses[i])) - }(i) - } - - wg.Wait() - - return largestToAddresses(xs) -} - -func largestToAddresses(xs *largest) []Address { - out := make([]Address, 0, xs.capacity) - for i := range xs.items { - if v := xs.items[i]; v != nil { - aa, ok := v.value.(*Address) - if !ok { - continue - } - address := *aa - address.match = v.weight - out = append(out, address) - } - } - return out -} - -func (s *searcher) FindAlts(limit int, id string) []*ofac.AlternateIdentity { - s.RLock() - defer s.RUnlock() - - var out []*ofac.AlternateIdentity - for i := range s.Alts { - if len(out) > limit { - break - } - if s.Alts[i].AlternateIdentity.EntityID == id { - out = append(out, s.Alts[i].AlternateIdentity) - } - } - return out -} - -func (s *searcher) TopAltNames(limit int, minMatch float64, alt string) []Alt { - alt = prepare.LowerAndRemovePunctuation(alt) - altTokens := strings.Fields(alt) - - s.RLock() - defer s.RUnlock() - - if len(s.Alts) == 0 { - return nil - } - xs := newLargest(limit, minMatch) - - var wg sync.WaitGroup - wg.Add(len(s.Alts)) - - for i := range s.Alts { - s.Gate.Start() - go func(i int) { - defer wg.Done() - defer s.Gate.Done() - xs.add(&item{ - matched: s.Alts[i].name, - value: s.Alts[i], - weight: stringscore.BestPairsJaroWinkler(altTokens, s.Alts[i].name), - }) - }(i) - } - wg.Wait() - - out := make([]Alt, 0, limit) - for i := range xs.items { - if v := xs.items[i]; v != nil { - aa, ok := v.value.(*Alt) - if !ok { - continue - } - alt := *aa - alt.match = v.weight - alt.matchedName = v.matched - out = append(out, alt) - } - } - return out -} - -func (s *searcher) FindSDN(entityID string) *ofac.SDN { - if sdn := s.debugSDN(entityID); sdn != nil { - return sdn.SDN - } - return nil -} - -func (s *searcher) debugSDN(entityID string) *SDN { - s.RLock() - defer s.RUnlock() - - return s.findSDNWithoutLock(entityID) -} - -func (s *searcher) findSDNWithoutLock(entityID string) *SDN { - for i := range s.SDNs { - if s.SDNs[i].EntityID == entityID { - return s.SDNs[i] - } - } - return nil -} - -// FindSDNsByRemarksID looks for SDN's whose remarks property contains an ID matching -// what is provided to this function. It's typically used with values assigned by a local -// government. (National ID, Drivers License, etc) -func (s *searcher) FindSDNsByRemarksID(limit int, id string) []*SDN { - if id == "" { - return nil - } - - var out []*SDN - for i := range s.SDNs { - // If the SDN's remarks ID contains a space then we need to ensure "all the numeric - // parts have to exactly match" between our query and the parsed ID. - if strings.Contains(s.SDNs[i].id, " ") { - qParts := strings.Fields(id) - sdnParts := strings.Fields(s.SDNs[i].id) - - matched, expected := 0, 0 - for j := range sdnParts { - if n, _ := strconv.ParseInt(sdnParts[j], 10, 64); n > 0 { - // This part of the SDN's remarks is a number so it must exactly - // match to a query's part - expected += 1 - - for k := range qParts { - if sdnParts[j] == qParts[k] { - matched += 1 - } - } - } - } - - // If all the numeric parts match between query and SDN return the match - if matched == expected { - sdn := *s.SDNs[i] - sdn.match = 1.0 - out = append(out, &sdn) - } - } else { - // The query and remarks ID must exactly match - if s.SDNs[i].id == id { - sdn := *s.SDNs[i] - sdn.match = 1.0 - out = append(out, &sdn) - } - } - - // quit if we're at our max result size - if len(out) >= limit { - return out - } - } - return out -} - -func (s *searcher) TopSDNs(limit int, minMatch float64, name string, keepSDN func(*SDN) bool) []*SDN { - name = prepare.LowerAndRemovePunctuation(name) - nameTokens := strings.Fields(name) - - s.RLock() - defer s.RUnlock() - - if len(s.SDNs) == 0 { - return nil - } - xs := newLargest(limit, minMatch) - - var wg sync.WaitGroup - wg.Add(len(s.SDNs)) - - for i := range s.SDNs { - if !keepSDN(s.SDNs[i]) { - wg.Done() - continue - } - s.Gate.Start() - go func(i int) { - defer wg.Done() - defer s.Gate.Done() - xs.add(&item{ - matched: s.SDNs[i].name, - value: s.SDNs[i], - weight: stringscore.BestPairsJaroWinkler(nameTokens, s.SDNs[i].name), - }) - }(i) - } - wg.Wait() - - out := make([]*SDN, 0, limit) - for i := range xs.items { - if v := xs.items[i]; v != nil { - ss, ok := v.value.(*SDN) - if !ok { - continue - } - sdn := *ss // deref for a copy - sdn.match = v.weight - sdn.matchedName = v.matched - out = append(out, &sdn) - } - } - return out -} - -func (s *searcher) TopDPs(limit int, minMatch float64, name string) []DP { - name = prepare.LowerAndRemovePunctuation(name) - nameTokens := strings.Fields(name) - - s.RLock() - defer s.RUnlock() - - if len(s.DPs) == 0 { - return nil - } - xs := newLargest(limit, minMatch) - - var wg sync.WaitGroup - wg.Add(len(s.DPs)) - - for i := range s.DPs { - s.Gate.Start() - go func(i int) { - defer wg.Done() - defer s.Gate.Done() - xs.add(&item{ - matched: s.DPs[i].name, - value: s.DPs[i], - weight: stringscore.BestPairsJaroWinkler(nameTokens, s.DPs[i].name), - }) - }(i) - } - wg.Wait() - - out := make([]DP, 0, limit) - for _, thisItem := range xs.items { - if v := thisItem; v != nil { - ss, ok := v.value.(*DP) - if !ok { - continue - } - dp := *ss - dp.match = v.weight - dp.matchedName = v.matched - out = append(out, dp) - } - } - return out -} - -// SDN is ofac.SDN wrapped with prepare.LowerAndRemovePunctuationd search metadata -type SDN struct { - *ofac.SDN - - // match holds the match ratio for an SDN in search results - match float64 - - // matchedName holds the highest scoring term from the search query - matchedName string - - // name is prepare.LowerAndRemovePunctuationd for speed - name string - - // id is the parseed ID value from an SDN's remarks field. Often this - // is a National ID, Drivers License, or similar government value - // ueed to uniquely identify an entiy. - // - // Typically the form of this is 'No. NNNNN' where NNNNN is alphanumeric. - id string -} - -// MarshalJSON is a custom method for marshaling a SDN search result -func (s SDN) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - *ofac.SDN - Match float64 `json:"match"` - MatchedName string `json:"matchedName"` - }{ - s.SDN, - s.match, - s.matchedName, - }) -} - -func findAddresses(entityID string, addrs []*ofac.Address) []*ofac.Address { - var out []*ofac.Address - for i := range addrs { - if entityID == addrs[i].EntityID { - out = append(out, addrs[i]) - } - } - return out -} - -func precomputeSDNs(sdns []*ofac.SDN, addrs []*ofac.Address, pipe *prepare.Pipeliner) []*SDN { - out := make([]*SDN, len(sdns)) - for i := range sdns { - nn := prepare.SdnName(sdns[i], findAddresses(sdns[i].EntityID, addrs)) - - if err := pipe.Do(nn); err != nil { - continue - } - - out[i] = &SDN{ - SDN: sdns[i], - name: nn.Processed, - id: extractIDFromRemark(strings.TrimSpace(sdns[i].Remarks)), - } - } - return out -} - -// Address is ofac.Address wrapped with prepare.LowerAndRemovePunctuationd search metadata -type Address struct { - Address *ofac.Address - - match float64 // match % - - // prepare.LowerAndRemovePunctuationd fields for speed - address, citystate, country string -} - -// MarshalJSON is a custom method for marshaling a SDN Address search result -func (a Address) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - *ofac.Address - Match float64 `json:"match"` - }{ - a.Address, - a.match, - }) -} - -func precomputeAddresses(adds []*ofac.Address) []*Address { - out := make([]*Address, len(adds)) - for i := range adds { - out[i] = &Address{ - Address: adds[i], - address: prepare.LowerAndRemovePunctuation(adds[i].Address), - citystate: prepare.LowerAndRemovePunctuation(adds[i].CityStateProvincePostalCode), - country: prepare.LowerAndRemovePunctuation(adds[i].Country), - } - } - return out -} - -// Alt is an ofac.AlternateIdentity wrapped with prepare.LowerAndRemovePunctuationd search metadata -type Alt struct { - AlternateIdentity *ofac.AlternateIdentity - - // match holds the match ratio for an Alt in search results - match float64 - - // matchedName holds the highest scoring term from the search query - matchedName string - - // name is prepare.LowerAndRemovePunctuationd for speed - name string -} - -// MarshalJSON is a custom method for marshaling a SDN Alternate Identity search result -func (a Alt) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - *ofac.AlternateIdentity - Match float64 `json:"match"` - MatchedName string `json:"matchedName"` - }{ - a.AlternateIdentity, - a.match, - a.matchedName, - }) -} - -func precomputeAlts(alts []*ofac.AlternateIdentity, pipe *prepare.Pipeliner) []*Alt { - out := make([]*Alt, len(alts)) - for i := range alts { - an := prepare.AltName(alts[i]) - - if err := pipe.Do(an); err != nil { - continue - } - - out[i] = &Alt{ - AlternateIdentity: alts[i], - name: an.Processed, - } - } - return out -} - -// DP is a BIS Denied Person wrapped with prepare.LowerAndRemovePunctuationd search metadata -type DP struct { - DeniedPerson *dpl.DPL - match float64 - matchedName string - name string -} - -// MarshalJSON is a custom method for marshaling a BIS Denied Person (DP) -func (d DP) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - *dpl.DPL - Match float64 `json:"match"` - MatchedName string `json:"matchedName"` - }{ - d.DeniedPerson, - d.match, - d.matchedName, - }) -} - -func precomputeDPs(persons []*dpl.DPL, pipe *prepare.Pipeliner) []*DP { - out := make([]*DP, len(persons)) - for i := range persons { - nn := prepare.DPName(persons[i]) - if err := pipe.Do(nn); err != nil { - continue - } - out[i] = &DP{ - DeniedPerson: persons[i], - name: nn.Processed, - } - } - return out -} - -// extractIDFromRemark attempts to parse out a National ID or similar governmental ID value -// from an SDN's remarks property. -// -// Typically the form of this is 'No. NNNNN' where NNNNN is alphanumeric. -func extractIDFromRemark(remarks string) string { - if remarks == "" { - return "" - } - - var out bytes.Buffer - parts := strings.Fields(remarks) - for i := range parts { - if parts[i] == "No." { - trimmed := strings.TrimSuffix(strings.TrimSuffix(parts[i+1], "."), ";") - - // Always take the next part - if strings.HasSuffix(parts[i+1], ".") || strings.HasSuffix(parts[i+1], ";") { - return trimmed - } else { - out.WriteString(trimmed) - } - // possibly take additional parts - for j := i + 2; j < len(parts); j++ { - if strings.HasPrefix(parts[j], "(") { - return out.String() - } - if _, err := strconv.ParseInt(parts[j], 10, 32); err == nil { - out.WriteString(" " + parts[j]) - } - } - } - } - return out.String() -} diff --git a/cmd/server/search_benchmark_test.go b/cmd/server/search_benchmark_test.go deleted file mode 100644 index df59b35c..00000000 --- a/cmd/server/search_benchmark_test.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "testing" - - "github.com/jaswdr/faker" - "github.com/stretchr/testify/require" -) - -var ( - fake = faker.New() -) - -func BenchmarkSearch__All(b *testing.B) { - searcher := createBenchmarkSearcher(b) - b.ResetTimer() - - var filters filterRequest - - for i := 0; i < b.N; i++ { - b.StopTimer() - name := fake.Person().Name() - b.StartTimer() - - buildFullSearchResponse(searcher, filters, 10, 0.0, name) - } -} - -func BenchmarkSearch__Addresses(b *testing.B) { - searcher := createBenchmarkSearcher(b) - b.ResetTimer() - - for i := 0; i < b.N; i++ { - searcher.TopAddresses(10, 0.0, fake.Person().Name()) - } -} - -func BenchmarkSearch__BISEntities(b *testing.B) { - searcher := createBenchmarkSearcher(b) - b.ResetTimer() - - for i := 0; i < b.N; i++ { - searcher.TopBISEntities(10, 0.0, fake.Person().Name()) - } -} - -func BenchmarkSearch__DPs(b *testing.B) { - searcher := createBenchmarkSearcher(b) - b.ResetTimer() - - for i := 0; i < b.N; i++ { - searcher.TopDPs(10, 0.0, fake.Person().Name()) - } -} - -func BenchmarkSearch__SDNsBasic(b *testing.B) { - searcher := createBenchmarkSearcher(b) - keeper := keepSDN(filterRequest{}) - b.ResetTimer() - - for i := 0; i < b.N; i++ { - searcher.TopSDNs(10, 0.0, fake.Person().Name(), keeper) - } -} - -func BenchmarkSearch__SDNsMinMatch50(b *testing.B) { - minMatch := 0.50 - searcher := createBenchmarkSearcher(b) - keeper := keepSDN(filterRequest{}) - b.ResetTimer() - - for i := 0; i < b.N; i++ { - searcher.TopSDNs(10, minMatch, fake.Person().Name(), keeper) - } -} - -func BenchmarkSearch__SDNsMinMatch95(b *testing.B) { - minMatch := 0.95 - searcher := createBenchmarkSearcher(b) - keeper := keepSDN(filterRequest{}) - b.ResetTimer() - - for i := 0; i < b.N; i++ { - searcher.TopSDNs(10, minMatch, fake.Person().Name(), keeper) - } -} - -func BenchmarkSearch__SDNsEntity(b *testing.B) { - searcher := createBenchmarkSearcher(b) - keeper := keepSDN(filterRequest{ - sdnType: "entity", - }) - b.ResetTimer() - - for i := 0; i < b.N; i++ { - searcher.TopSDNs(10, 0.0, fake.Person().Name(), keeper) - } -} - -func BenchmarkSearch__SDNsComplex(b *testing.B) { - minMatch := 0.95 - searcher := createBenchmarkSearcher(b) - keeper := keepSDN(filterRequest{ - sdnType: "entity", - }) - b.ResetTimer() - - for i := 0; i < b.N; i++ { - searcher.TopSDNs(10, minMatch, fake.Person().Name(), keeper) - } -} - -func BenchmarkSearch__SSIs(b *testing.B) { - searcher := createBenchmarkSearcher(b) - b.ResetTimer() - - for i := 0; i < b.N; i++ { - searcher.TopSSIs(10, 0.0, fake.Person().Name()) - } -} - -func BenchmarkSearch__CSL(b *testing.B) { - searcher := createBenchmarkSearcher(b) - b.ResetTimer() - - for i := 0; i < b.N; i++ { - resp := buildFullSearchResponseWith(searcher, cslGatherings, filterRequest{}, 10, 0.0, fake.Person().Name()) - - b.StopTimer() - require.Greater(b, len(resp.BISEntities), 1) - require.Greater(b, len(resp.MilitaryEndUsers), 1) - require.Greater(b, len(resp.SectoralSanctions), 1) - require.Greater(b, len(resp.Unverified), 1) - require.Greater(b, len(resp.NonproliferationSanctions), 1) - require.Greater(b, len(resp.ForeignSanctionsEvaders), 1) - require.Greater(b, len(resp.PalestinianLegislativeCouncil), 1) - require.Greater(b, len(resp.CaptaList), 1) - require.Greater(b, len(resp.ITARDebarred), 1) - require.Greater(b, len(resp.NonSDNChineseMilitaryIndustrialComplex), 1) - require.Greater(b, len(resp.NonSDNMenuBasedSanctionsList), 1) - b.StartTimer() - } -} diff --git a/cmd/server/search_crypto.go b/cmd/server/search_crypto.go deleted file mode 100644 index ba81a9b2..00000000 --- a/cmd/server/search_crypto.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "net/http" - "strings" - - moovhttp "github.com/moov-io/base/http" - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/pkg/ofac" -) - -type cryptoAddressSearchResult struct { - OFAC []SDNWithDigitalCurrencyAddress `json:"ofac"` -} - -func searchByCryptoAddress(logger log.Logger, searcher *searcher) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - cryptoAddress := strings.TrimSpace(r.URL.Query().Get("address")) - cryptoName := strings.TrimSpace(r.URL.Query().Get("name")) - if cryptoAddress == "" { - moovhttp.Problem(w, errNoSearchParams) - return - } - - limit := extractSearchLimit(r) - - // Find SDNs with a crypto address that exactly matches - resp := cryptoAddressSearchResult{ - OFAC: searcher.FindSDNCryptoAddresses(limit, cryptoName, cryptoAddress), - } - - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(resp) - } -} - -type SDNWithDigitalCurrencyAddress struct { - SDN *ofac.SDN `json:"sdn"` - - DigitalCurrencyAddresses []ofac.DigitalCurrencyAddress `json:"digitalCurrencyAddresses"` -} - -func (s *searcher) FindSDNCryptoAddresses(limit int, name, needle string) []SDNWithDigitalCurrencyAddress { - s.RLock() - defer s.RUnlock() - - var out []SDNWithDigitalCurrencyAddress - for i := range s.SDNComments { - addresses := s.SDNComments[i].DigitalCurrencyAddresses - for j := range addresses { - // Skip addresses of a different coin - if name != "" && addresses[j].Currency != name { - continue - } - if addresses[j].Address == needle { - // Find SDN - sdn := s.findSDNWithoutLock(s.SDNComments[i].EntityID) - if sdn != nil { - out = append(out, SDNWithDigitalCurrencyAddress{ - SDN: sdn.SDN, - DigitalCurrencyAddresses: addresses, - }) - } - } - } - } - return out -} diff --git a/cmd/server/search_crypto_test.go b/cmd/server/search_crypto_test.go deleted file mode 100644 index 962cab96..00000000 --- a/cmd/server/search_crypto_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "fmt" - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "testing" - - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/pkg/ofac" - - "github.com/gorilla/mux" - "github.com/stretchr/testify/require" -) - -var ( - cryptoSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) -) - -func init() { - // Set SDN Comments - fd, err := os.Open(filepath.Join("..", "..", "test", "testdata", "sdn_comments.csv")) - if err != nil { - panic(fmt.Sprintf("%v", err)) - } - ofacResults, err := ofac.Read(map[string]io.ReadCloser{"sdn_comments.csv": fd}) - if err != nil { - panic(fmt.Sprintf("ERROR reading sdn_comments.csv: %v", err)) - } - - cryptoSearcher.SDNComments = ofacResults.SDNComments - cryptoSearcher.SDNs = precomputeSDNs([]*ofac.SDN{ - { - EntityID: "39796", // matches TestSearchCrypto - SDNName: "Person A", - SDNType: "individual", - Title: "Guy or Girl doing crypto stuff", - }, - }, nil, noLogPipeliner) -} - -func TestSearchCryptoSetup(t *testing.T) { - require.Len(t, cryptoSearcher.SDNComments, 13) - require.Len(t, cryptoSearcher.SDNs, 1) -} - -type expectedCryptoAddressSearchResult struct { - OFAC []SDNWithDigitalCurrencyAddress `json:"ofac"` -} - -func TestSearchCrypto(t *testing.T) { - router := mux.NewRouter() - addSearchRoutes(log.NewNopLogger(), router, cryptoSearcher) - - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/crypto?address=0x242654336ca2205714071898f67E254EB49ACdCe", nil) - router.ServeHTTP(w, req) - w.Flush() - require.Equal(t, http.StatusOK, w.Code) - - var response expectedCryptoAddressSearchResult - err := json.NewDecoder(w.Body).Decode(&response) - require.NoError(t, err) - - require.Len(t, response.OFAC, 1) - require.Equal(t, "39796", response.OFAC[0].SDN.EntityID) - - // Now with cryptocurrency name specified - req = httptest.NewRequest("GET", "/crypto?name=ETH&address=0x242654336ca2205714071898f67E254EB49ACdCe", nil) - router.ServeHTTP(w, req) - w.Flush() - require.Equal(t, http.StatusOK, w.Code) - - err = json.NewDecoder(w.Body).Decode(&response) - require.NoError(t, err) - - require.Len(t, response.OFAC, 1) - require.Equal(t, "39796", response.OFAC[0].SDN.EntityID) - - // With wrong cryptocurrency name - req = httptest.NewRequest("GET", "/crypto?name=QRR&address=0x242654336ca2205714071898f67E254EB49ACdCe", nil) - router.ServeHTTP(w, req) - w.Flush() - require.Equal(t, http.StatusOK, w.Code) - - err = json.NewDecoder(w.Body).Decode(&response) - require.NoError(t, err) - - require.Len(t, response.OFAC, 0) -} diff --git a/cmd/server/search_eu_csl.go b/cmd/server/search_eu_csl.go deleted file mode 100644 index 2997fee2..00000000 --- a/cmd/server/search_eu_csl.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "net/http" - - moovhttp "github.com/moov-io/base/http" - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/pkg/csl_eu" -) - -// search EUCLS -func searchEUCSL(logger log.Logger, searcher *searcher) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - w = wrapResponseWriter(logger, w, r) - requestID := moovhttp.GetRequestID(r) - - limit := extractSearchLimit(r) - filters := buildFilterRequest(r.URL) - minMatch := extractSearchMinMatch(r) - - name := r.URL.Query().Get("name") - resp := buildFullSearchResponseWith(searcher, euGatherings, filters, limit, minMatch, name) - - logger.Info().With(log.Fields{ - "name": log.String(name), - "requestID": log.String(requestID), - }).Log("performing EU-CSL search") - - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(resp) - } -} - -// TopEUCSL searches the EU Sanctions list by Name and Alias -func (s *searcher) TopEUCSL(limit int, minMatch float64, name string) []*Result[csl_eu.CSLRecord] { - s.RLock() - defer s.RUnlock() - - s.Gate.Start() - defer s.Gate.Done() - - return topResults[csl_eu.CSLRecord](limit, minMatch, name, s.EUCSL) -} diff --git a/cmd/server/search_eu_csl_test.go b/cmd/server/search_eu_csl_test.go deleted file mode 100644 index 8c4eea9e..00000000 --- a/cmd/server/search_eu_csl_test.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/pkg/csl_eu" - - "github.com/gorilla/mux" - "github.com/stretchr/testify/require" -) - -func TestSearch__EU_CSL(t *testing.T) { - w := httptest.NewRecorder() - // misspelled on purpose to also check jaro winkler is picking up the right records - req := httptest.NewRequest("GET", "/search/eu-csl?name=Saddam%20Hussien", nil) - - router := mux.NewRouter() - addSearchRoutes(log.NewNopLogger(), router, eu_cslSearcher) - router.ServeHTTP(w, req) - w.Flush() - - require.Equal(t, http.StatusOK, w.Code) - require.Contains(t, w.Body.String(), `"match":0.92419`) - require.Contains(t, w.Body.String(), `"matchedName":"saddam hussein al tikriti"`) - - var wrapper struct { - EUConsolidatedSanctionsList []csl_eu.CSLRecord `json:"euConsolidatedSanctionsList"` - } - err := json.NewDecoder(w.Body).Decode(&wrapper) - require.NoError(t, err) - - require.Len(t, wrapper.EUConsolidatedSanctionsList, 1) - prolif := wrapper.EUConsolidatedSanctionsList[0] - require.Equal(t, 13, prolif.EntityLogicalID) -} diff --git a/cmd/server/search_generic.go b/cmd/server/search_generic.go deleted file mode 100644 index 4359df83..00000000 --- a/cmd/server/search_generic.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "reflect" - "strings" - "sync" - - "github.com/moov-io/watchman/internal/prepare" - "github.com/moov-io/watchman/internal/stringscore" -) - -type Result[T any] struct { - Data T - - match float64 - matchedName string - - precomputedName string - precomputedAlts []string -} - -func (e Result[T]) MarshalJSON() ([]byte, error) { - // Due to a problem with embedding type parameters we have to dig into - // the parameterized type fields and include them in one object. - // - // Helpful Tips: - // https://stackoverflow.com/a/64420452 - // https://github.com/golang/go/issues/41563 - - v := reflect.ValueOf(e.Data) - - result := make(map[string]interface{}) - for i := 0; i < v.NumField(); i++ { - key := v.Type().Field(i) - value := v.Field(i) - - if key.IsExported() { - result[key.Name] = value.Interface() - } - } - - result["match"] = e.match - result["matchedName"] = e.matchedName - - return json.Marshal(result) -} - -func topResults[T any](limit int, minMatch float64, name string, data []*Result[T]) []*Result[T] { - if len(data) == 0 { - return nil - } - - name = prepare.LowerAndRemovePunctuation(name) - nameTokens := strings.Fields(name) - - xs := newLargest(limit, minMatch) - - var wg sync.WaitGroup - wg.Add(len(data)) - - for i := range data { - go func(i int) { - defer wg.Done() - - it := &item{ - matched: data[i].precomputedName, - value: data[i], - weight: stringscore.BestPairsJaroWinkler(nameTokens, data[i].precomputedName), - } - - for _, alt := range data[i].precomputedAlts { - if alt == "" { - continue - } - - score := stringscore.BestPairsJaroWinkler(nameTokens, alt) - if score > it.weight { - it.matched = alt - it.weight = score - } - } - - xs.add(it) - }(i) - } - wg.Wait() - - out := make([]*Result[T], 0) - for _, thisItem := range xs.items { - if v := thisItem; v != nil { - vv, ok := v.value.(*Result[T]) - if !ok { - continue - } - res := &Result[T]{ - Data: vv.Data, - match: v.weight, - matchedName: v.matched, - precomputedName: vv.precomputedName, - precomputedAlts: vv.precomputedAlts, - } - out = append(out, res) - } - } - return out -} diff --git a/cmd/server/search_handlers.go b/cmd/server/search_handlers.go deleted file mode 100644 index 2a88190e..00000000 --- a/cmd/server/search_handlers.go +++ /dev/null @@ -1,490 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "math" - "net/http" - "net/url" - "strconv" - "strings" - "sync" - "time" - - moovhttp "github.com/moov-io/base/http" - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/pkg/csl_eu" - "github.com/moov-io/watchman/pkg/csl_uk" - "github.com/moov-io/watchman/pkg/csl_us" - - "github.com/go-kit/kit/metrics/prometheus" - "github.com/gorilla/mux" - stdprometheus "github.com/prometheus/client_golang/prometheus" -) - -var ( - matchHist = prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{ - Name: "match_percentages", - Help: "Histogram representing the match percent of search routes", - Buckets: []float64{0.0, 0.5, 0.8, 0.9, 0.99}, - }, []string{"type"}) -) - -// TODO: modify existing search endpoint with additional eu info and add an eu only endpoint -func addSearchRoutes(logger log.Logger, r *mux.Router, searcher *searcher) { - r.Methods("GET").Path("/crypto").HandlerFunc(searchByCryptoAddress(logger, searcher)) - r.Methods("GET").Path("/search").HandlerFunc(search(logger, searcher)) - r.Methods("GET").Path("/search/us-csl").HandlerFunc(searchUSCSL(logger, searcher)) - r.Methods("GET").Path("/search/eu-csl").HandlerFunc(searchEUCSL(logger, searcher)) - r.Methods("GET").Path("/search/uk-csl").HandlerFunc(searchUKCSL(logger, searcher)) -} - -func extractSearchLimit(r *http.Request) int { - limit := softResultsLimit - if v := r.URL.Query().Get("limit"); v != "" { - n, _ := strconv.Atoi(v) - if n > 0 { - limit = n - } - } - if limit > hardResultsLimit { - limit = hardResultsLimit - } - return limit -} - -func extractSearchMinMatch(r *http.Request) float64 { - if v := r.URL.Query().Get("minMatch"); v != "" { - n, _ := strconv.ParseFloat(v, 64) - return n - } - return 0.00 -} - -type addressSearchRequest struct { - Address string `json:"address"` - City string `json:"city"` - State string `json:"state"` - Providence string `json:"providence"` - Zip string `json:"zip"` - Country string `json:"country"` -} - -func (req addressSearchRequest) empty() bool { - return req.Address == "" && req.City == "" && req.State == "" && - req.Providence == "" && req.Zip == "" && req.Country == "" -} - -func readAddressSearchRequest(u *url.URL) addressSearchRequest { - return addressSearchRequest{ - Address: strings.ToLower(strings.TrimSpace(u.Query().Get("address"))), - City: strings.ToLower(strings.TrimSpace(u.Query().Get("city"))), - State: strings.ToLower(strings.TrimSpace(u.Query().Get("state"))), - Providence: strings.ToLower(strings.TrimSpace(u.Query().Get("providence"))), - Zip: strings.ToLower(strings.TrimSpace(u.Query().Get("zip"))), - Country: strings.ToLower(strings.TrimSpace(u.Query().Get("country"))), - } -} - -func search(logger log.Logger, searcher *searcher) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - w = wrapResponseWriter(logger, w, r) - - // Search over all fields - if q := strings.TrimSpace(r.URL.Query().Get("q")); q != "" { - searchViaQ(searcher, q)(w, r) - return - } - - // Search by ID (found in an SDN's Remarks property) - if id := strings.TrimSpace(r.URL.Query().Get("id")); id != "" { - searchByRemarksID(searcher, id)(w, r) - return - } - - // Search by Name - if name := strings.TrimSpace(r.URL.Query().Get("name")); name != "" { - if req := readAddressSearchRequest(r.URL); !req.empty() { - searchViaAddressAndName(searcher, name, req)(w, r) - } else { - searchByName(searcher, name)(w, r) - } - return - } - - // Search by Alt Name - if alt := strings.TrimSpace(r.URL.Query().Get("altName")); alt != "" { - searchByAltName(searcher, alt)(w, r) - return - } - - // Search Addresses - if req := readAddressSearchRequest(r.URL); !req.empty() { - searchByAddress(searcher, req)(w, r) - return - } - - // Fallback if no search params were found - moovhttp.Problem(w, errNoSearchParams) - } -} - -type searchResponse struct { - // OFAC - SDNs []*SDN `json:"SDNs"` - AltNames []Alt `json:"altNames"` - Addresses []Address `json:"addresses"` - - // BIS - DeniedPersons []DP `json:"deniedPersons"` - - // Consolidated Screening List - BISEntities []*Result[csl_us.EL] `json:"bisEntities"` - MilitaryEndUsers []*Result[csl_us.MEU] `json:"militaryEndUsers"` - SectoralSanctions []*Result[csl_us.SSI] `json:"sectoralSanctions"` - Unverified []*Result[csl_us.UVL] `json:"unverifiedCSL"` - NonproliferationSanctions []*Result[csl_us.ISN] `json:"nonproliferationSanctions"` - ForeignSanctionsEvaders []*Result[csl_us.FSE] `json:"foreignSanctionsEvaders"` - PalestinianLegislativeCouncil []*Result[csl_us.PLC] `json:"palestinianLegislativeCouncil"` - CaptaList []*Result[csl_us.CAP] `json:"captaList"` - ITARDebarred []*Result[csl_us.DTC] `json:"itarDebarred"` - NonSDNChineseMilitaryIndustrialComplex []*Result[csl_us.CMIC] `json:"nonSDNChineseMilitaryIndustrialComplex"` - NonSDNMenuBasedSanctionsList []*Result[csl_us.NS_MBS] `json:"nonSDNMenuBasedSanctionsList"` - - // EU - Consolidated Sanctions List - EUCSL []*Result[csl_eu.CSLRecord] `json:"euConsolidatedSanctionsList"` - - // UK - Consolidated Sanctions List - UKCSL []*Result[csl_uk.CSLRecord] `json:"ukConsolidatedSanctionsList"` - - // UK Sanctions List - UKSanctionsList []*Result[csl_uk.SanctionsListRecord] `json:"ukSanctionsList"` - - // Metadata - RefreshedAt time.Time `json:"refreshedAt"` -} - -func buildAddressCompares(req addressSearchRequest) []func(*Address) *item { - var compares []func(*Address) *item - if req.Address != "" { - compares = append(compares, topAddressesAddress(req.Address)) - } - if req.City != "" { - compares = append(compares, topAddressesCityState(req.City)) - } - if req.State != "" { - compares = append(compares, topAddressesCityState(req.State)) - } - if req.Providence != "" { - compares = append(compares, topAddressesCityState(req.Providence)) - } - if req.Zip != "" { - compares = append(compares, topAddressesCityState(req.Zip)) - } - if req.Country != "" { - compares = append(compares, topAddressesCountry(req.Country)) - } - return compares -} - -func searchByAddress(searcher *searcher, req addressSearchRequest) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if req.empty() { - w.WriteHeader(http.StatusBadRequest) - return - } - - resp := searchResponse{ - RefreshedAt: searcher.lastRefreshedAt, - } - limit := extractSearchLimit(r) - minMatch := extractSearchMinMatch(r) - - // Perform our ranking across all accumulated compare functions - // - // TODO(adam): Is there something in the (SDN?) files which signal to block an entire country? (i.e. Needing to block Iran all together) - // https://www.treasury.gov/resource-center/sanctions/CivPen/Documents/20190327_decker_settlement.pdf - compares := buildAddressCompares(req) - - filtered := searcher.FilterCountries(req.Country) - resp.Addresses = TopAddressesFn(limit, minMatch, filtered, multiAddressCompare(compares...)) - - // record Prometheus metrics - if len(resp.Addresses) > 0 { - matchHist.With("type", "address").Observe(resp.Addresses[0].match) - } else { - matchHist.With("type", "address").Observe(0.0) - } - - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(resp) - } -} - -func searchViaQ(searcher *searcher, name string) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - name = strings.TrimSpace(name) - if name == "" { - moovhttp.Problem(w, errNoSearchParams) - return - } - limit := extractSearchLimit(r) - minMatch := extractSearchMinMatch(r) - - // Perform multiple searches over the set of SDNs - resp := buildFullSearchResponse(searcher, buildFilterRequest(r.URL), limit, minMatch, name) - - // record Prometheus metrics - if len(resp.SDNs) > 0 { - matchHist.With("type", "q").Observe(resp.SDNs[0].match) - } else { - matchHist.With("type", "q").Observe(0.0) - } - - // Build our big response object - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(resp) - } -} - -// searchGather performs an inmem search with *searcher and mutates *searchResponse by setting a specific field -type searchGather func(searcher *searcher, filters filterRequest, limit int, minMatch float64, name string, resp *searchResponse) - -var ( - baseGatherings = []searchGather{ - // OFAC SDN Search - func(s *searcher, filters filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - sdns := s.FindSDNsByRemarksID(limit, name) - if len(sdns) == 0 { - sdns = s.TopSDNs(limit, minMatch, name, keepSDN(filters)) - } - resp.SDNs = filterSDNs(sdns, filters) - }, - // OFAC SDN Alt Names - func(s *searcher, _ filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - resp.AltNames = s.TopAltNames(limit, minMatch, name) - }, - // OFAC Addresses - func(s *searcher, _ filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - resp.Addresses = s.TopAddresses(limit, minMatch, name) - }, - - // BIS Denied Persons - func(s *searcher, _ filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - resp.DeniedPersons = s.TopDPs(limit, minMatch, name) - }, - } - - // Consolidated Screening List Results - cslGatherings = []searchGather{ - func(s *searcher, _ filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - resp.BISEntities = s.TopBISEntities(limit, minMatch, name) - }, - func(s *searcher, _ filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - resp.MilitaryEndUsers = s.TopMEUs(limit, minMatch, name) - }, - func(s *searcher, _ filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - resp.SectoralSanctions = s.TopSSIs(limit, minMatch, name) - }, - func(s *searcher, _ filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - resp.Unverified = s.TopUVLs(limit, minMatch, name) - }, - func(s *searcher, _ filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - resp.NonproliferationSanctions = s.TopISNs(limit, minMatch, name) - }, - func(s *searcher, _ filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - resp.ForeignSanctionsEvaders = s.TopFSEs(limit, minMatch, name) - }, - func(s *searcher, _ filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - resp.PalestinianLegislativeCouncil = s.TopPLCs(limit, minMatch, name) - }, - func(s *searcher, _ filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - resp.CaptaList = s.TopCAPs(limit, minMatch, name) - }, - func(s *searcher, _ filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - resp.ITARDebarred = s.TopDTCs(limit, minMatch, name) - }, - func(s *searcher, _ filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - resp.NonSDNChineseMilitaryIndustrialComplex = s.TopCMICs(limit, minMatch, name) - }, - func(s *searcher, _ filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - resp.NonSDNMenuBasedSanctionsList = s.TopNS_MBS(limit, minMatch, name) - }, - } - - // eu - consolidated sanctions list - euGatherings = []searchGather{ - func(s *searcher, _ filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - resp.EUCSL = s.TopEUCSL(limit, minMatch, name) - }, - } - - // uk - consolidated sanctions list - ukGatherings = []searchGather{ - func(s *searcher, _ filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - resp.UKCSL = s.TopUKCSL(limit, minMatch, name) - }, - func(s *searcher, _ filterRequest, limit int, minMatch float64, name string, resp *searchResponse) { - resp.UKSanctionsList = s.TopUKSanctionsList(limit, minMatch, name) - }, - } - - allGatherings = append(append(append(baseGatherings, cslGatherings...), euGatherings...), ukGatherings...) -) - -func buildFullSearchResponse(searcher *searcher, filters filterRequest, limit int, minMatch float64, name string) *searchResponse { - return buildFullSearchResponseWith(searcher, allGatherings, filters, limit, minMatch, name) -} - -func buildFullSearchResponseWith(searcher *searcher, searchGatherings []searchGather, filters filterRequest, limit int, minMatch float64, name string) *searchResponse { - resp := searchResponse{ - RefreshedAt: searcher.lastRefreshedAt, - } - var wg sync.WaitGroup - wg.Add(len(searchGatherings)) - for i := range searchGatherings { - go func(i int) { - searchGatherings[i](searcher, filters, limit, minMatch, name, &resp) - wg.Done() - }(i) - } - wg.Wait() - return &resp -} - -func searchViaAddressAndName(searcher *searcher, name string, req addressSearchRequest) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - name = strings.TrimSpace(name) - if name == "" || req.empty() { - moovhttp.Problem(w, errNoSearchParams) - return - } - - limit, minMatch := extractSearchLimit(r), extractSearchMinMatch(r) - - resp := &searchResponse{ - RefreshedAt: searcher.lastRefreshedAt, - } - - resp.SDNs = searcher.TopSDNs(limit, minMatch, name, keepSDN(buildFilterRequest(r.URL))) - - compares := buildAddressCompares(req) - filtered := searcher.FilterCountries(req.Country) - resp.Addresses = TopAddressesFn(limit, minMatch, filtered, multiAddressCompare(compares...)) - - // record Prometheus metrics - if len(resp.SDNs) > 0 && len(resp.Addresses) > 0 { - matchHist.With("type", "addressname").Observe(math.Max(resp.SDNs[0].match, resp.Addresses[0].match)) - } else { - matchHist.With("type", "addressname").Observe(0.0) - } - - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(resp) - } -} - -func searchByRemarksID(searcher *searcher, id string) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if id == "" { - moovhttp.Problem(w, errNoSearchParams) - return - } - - limit := extractSearchLimit(r) - - sdns := searcher.FindSDNsByRemarksID(limit, id) - sdns = filterSDNs(sdns, buildFilterRequest(r.URL)) - - // record Prometheus metrics - if len(sdns) > 0 { - matchHist.With("type", "remarksID").Observe(sdns[0].match) - } else { - matchHist.With("type", "remarksID").Observe(0.0) - } - - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(&searchResponse{ - SDNs: sdns, - RefreshedAt: searcher.lastRefreshedAt, - }) - } -} - -func searchByName(searcher *searcher, nameSlug string) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - nameSlug = strings.TrimSpace(nameSlug) - if nameSlug == "" { - moovhttp.Problem(w, errNoSearchParams) - return - } - - limit := extractSearchLimit(r) - minMatch := extractSearchMinMatch(r) - - // Grab the SDN's and then filter any out based on query params - sdns := searcher.TopSDNs(limit, minMatch, nameSlug, keepSDN(buildFilterRequest(r.URL))) - - // record Prometheus metrics - if len(sdns) > 0 { - matchHist.With("type", "name").Observe(sdns[0].match) - } else { - matchHist.With("type", "name").Observe(0.0) - } - - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(&searchResponse{ - // OFAC - SDNs: sdns, - AltNames: searcher.TopAltNames(limit, minMatch, nameSlug), - SectoralSanctions: searcher.TopSSIs(limit, minMatch, nameSlug), - // BIS - DeniedPersons: searcher.TopDPs(limit, minMatch, nameSlug), - BISEntities: searcher.TopBISEntities(limit, minMatch, nameSlug), - // EUCSL - EUCSL: searcher.TopEUCSL(limit, minMatch, nameSlug), - // UKCSL - UKCSL: searcher.TopUKCSL(limit, minMatch, nameSlug), - // UKSanctionsList - UKSanctionsList: searcher.TopUKSanctionsList(limit, minMatch, nameSlug), - // Metadata - RefreshedAt: searcher.lastRefreshedAt, - }) - } -} - -func searchByAltName(searcher *searcher, altSlug string) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - altSlug = strings.TrimSpace(altSlug) - if altSlug == "" { - moovhttp.Problem(w, errNoSearchParams) - return - } - - limit := extractSearchLimit(r) - minMatch := extractSearchMinMatch(r) - alts := searcher.TopAltNames(limit, minMatch, altSlug) - - // record Prometheus metrics - if len(alts) > 0 { - matchHist.With("type", "altName").Observe(alts[0].match) - } else { - matchHist.With("type", "altName").Observe(0.0) - } - - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(&searchResponse{ - AltNames: alts, - RefreshedAt: searcher.lastRefreshedAt, - }) - } -} diff --git a/cmd/server/search_handlers_bench_test.go b/cmd/server/search_handlers_bench_test.go deleted file mode 100644 index 5db3348b..00000000 --- a/cmd/server/search_handlers_bench_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "net/http" - "net/http/httptest" - "net/url" - "testing" - - "github.com/moov-io/base/log" - - "github.com/gorilla/mux" - "github.com/stretchr/testify/require" - "golang.org/x/sync/errgroup" -) - -func BenchmarkSearchHandler(b *testing.B) { - searcher := createTestSearcher(b) // Uses live data - b.ResetTimer() - - router := mux.NewRouter() - addSearchRoutes(log.NewNopLogger(), router, searcher) - - g := &errgroup.Group{} - g.SetLimit(10) - - for i := 0; i < b.N; i++ { - g.Go(func() error { - name := fake.Person().Name() - - v := make(url.Values, 0) - v.Set("name", name) - v.Set("limit", "10") - v.Set("minMatch", "0.70") - - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", fmt.Sprintf("/search?%s", v.Encode()), nil) - router.ServeHTTP(w, req) - - if w.Code != http.StatusOK { - return fmt.Errorf("unexpected status: %v", w.Code) - } - return nil - }) - } - require.NoError(b, g.Wait()) -} diff --git a/cmd/server/search_handlers_test.go b/cmd/server/search_handlers_test.go deleted file mode 100644 index 10c6c512..00000000 --- a/cmd/server/search_handlers_test.go +++ /dev/null @@ -1,376 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "net/url" - "strings" - "testing" - - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/pkg/csl_us" - "github.com/moov-io/watchman/pkg/dpl" - "github.com/moov-io/watchman/pkg/ofac" - - "github.com/gorilla/mux" - "github.com/stretchr/testify/require" -) - -func TestSearch__Address(t *testing.T) { - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/search?address=ibex+house+minories&limit=1", nil) - - router := mux.NewRouter() - addSearchRoutes(log.NewNopLogger(), router, addressSearcher) - router.ServeHTTP(w, req) - w.Flush() - - require.Equal(t, http.StatusOK, w.Code) - require.Contains(t, w.Body.String(), `"match":0.88194`) - - var wrapper struct { - Addresses []*ofac.Address `json:"addresses"` - } - if err := json.NewDecoder(w.Body).Decode(&wrapper); err != nil { - t.Fatal(err) - } - if len(wrapper.Addresses) == 0 { - t.Fatal("found no addresses") - } - if wrapper.Addresses[0].EntityID != "173" { - t.Errorf("%#v", wrapper.Addresses[0]) - } - - // send an empty body and get an error - w = httptest.NewRecorder() - req = httptest.NewRequest("GET", "/search?limit=1", nil) - router.ServeHTTP(w, req) - w.Flush() - - if w.Code != http.StatusBadRequest { - t.Errorf("bogus status code: %d", w.Code) - } -} - -func TestSearch__AddressCountry(t *testing.T) { - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/search?country=united+kingdom&limit=1", nil) - - router := mux.NewRouter() - addSearchRoutes(log.NewNopLogger(), router, addressSearcher) - router.ServeHTTP(w, req) - w.Flush() - - require.Equal(t, http.StatusOK, w.Code) - require.Contains(t, w.Body.String(), `"match":1`) - -} - -func TestSearch__AddressMulti(t *testing.T) { - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/search?address=ibex+house&country=united+kingdom&limit=1", nil) - - router := mux.NewRouter() - addSearchRoutes(log.NewNopLogger(), router, addressSearcher) - router.ServeHTTP(w, req) - w.Flush() - - require.Equal(t, http.StatusOK, w.Code) - require.Contains(t, w.Body.String(), `"match":0.8847`) - -} - -func TestSearch__AddressProvidence(t *testing.T) { - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/search?address=ibex+house&country=united+kingdom&providence=london+ec3n+1DY&limit=1", nil) - - router := mux.NewRouter() - addSearchRoutes(log.NewNopLogger(), router, addressSearcher) - router.ServeHTTP(w, req) - w.Flush() - - require.Equal(t, http.StatusOK, w.Code) - require.Contains(t, w.Body.String(), `"match":0.923`) - -} - -func TestSearch__AddressCity(t *testing.T) { - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/search?address=ibex+house&country=united+kingdom&city=london+ec3n+1DY&limit=1", nil) - - router := mux.NewRouter() - addSearchRoutes(log.NewNopLogger(), router, addressSearcher) - router.ServeHTTP(w, req) - w.Flush() - - require.Equal(t, http.StatusOK, w.Code) - require.Contains(t, w.Body.String(), `"match":0.923`) - -} - -func TestSearch__AddressState(t *testing.T) { - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/search?address=ibex+house&country=united+kingdom&state=london+ec3n+1DY&limit=1", nil) - - router := mux.NewRouter() - addSearchRoutes(log.NewNopLogger(), router, addressSearcher) - router.ServeHTTP(w, req) - w.Flush() - - require.Equal(t, http.StatusOK, w.Code) - require.Contains(t, w.Body.String(), `"match":0.923`) - -} - -func TestSearch__NameAndAddress(t *testing.T) { - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/search?name=midco&address=rue+de+rhone&limit=1", nil) - - pipe := noLogPipeliner - s := newSearcher(log.NewNopLogger(), pipe, 1) - s.Addresses = precomputeAddresses([]*ofac.Address{ - { - EntityID: "2831", - AddressID: "1965", - Address: "57 Rue du Rhone", - CityStateProvincePostalCode: "Geneva CH-1204", - Country: "Switzerland", - }, - { - EntityID: "173", - AddressID: "129", - Address: "Ibex House, The Minories", - CityStateProvincePostalCode: "London EC3N 1DY", - Country: "United Kingdom", - }, - }) - s.SDNs = precomputeSDNs([]*ofac.SDN{ - { - EntityID: "2831", - SDNName: "MIDCO FINANCE S.A.", - SDNType: "individual", - Programs: []string{"IRAQ2"}, - Remarks: "US FEIN CH-660-0-469-982-0 (United States); Switzerland.", - }, - }, nil, noLogPipeliner) - - router := mux.NewRouter() - addSearchRoutes(log.NewNopLogger(), router, s) - router.ServeHTTP(w, req) - w.Flush() - - if w.Code != http.StatusOK { - t.Errorf("bogus status code: %d", w.Code) - } - var wrapper struct { - SDNs []*ofac.SDN `json:"SDNs"` - Addresses []*ofac.Address `json:"addresses"` - } - if err := json.NewDecoder(w.Body).Decode(&wrapper); err != nil { - t.Fatal(err) - } - - if len(wrapper.SDNs) != 1 || len(wrapper.Addresses) != 1 { - t.Fatalf("sdns=%#v addresses=%#v", wrapper.SDNs[0], wrapper.Addresses[0]) - } - - if wrapper.SDNs[0].EntityID != "2831" || wrapper.Addresses[0].EntityID != "2831" { - t.Errorf("SDNs[0].EntityID=%s Addresses[0].EntityID=%s", wrapper.SDNs[0].EntityID, wrapper.Addresses[0].EntityID) - } - - // request with no results - req = httptest.NewRequest("GET", "/search?name=midco&country=United+Kingdom&limit=1", nil) - - router.ServeHTTP(w, req) - w.Flush() - - if w.Code != http.StatusOK { - t.Errorf("bogus status code: %d", w.Code) - } - if err := json.NewDecoder(w.Body).Decode(&wrapper); err != nil { - t.Fatal(err) - } - if len(wrapper.SDNs) != 1 || len(wrapper.Addresses) != 1 { - t.Errorf("sdns=%#v", wrapper.SDNs[0]) - t.Fatalf("addresses=%#v", wrapper.Addresses[0]) - } -} - -func TestSearch__NameAndAltName(t *testing.T) { - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/search?limit=1&q=nayif", nil) - - s := newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - // OFAC - s.SDNs = sdnSearcher.SDNs - s.Alts = altSearcher.Alts - s.Addresses = addressSearcher.Addresses - s.SSIs = ssiSearcher.SSIs - // BIS - s.DPs = dplSearcher.DPs - s.BISEntities = bisEntitySearcher.BISEntities - - router := mux.NewRouter() - addSearchRoutes(log.NewNopLogger(), router, s) - router.ServeHTTP(w, req) - w.Flush() - - if w.Code != http.StatusOK { - t.Errorf("bogus status code: %d", w.Code) - } - - // read response body - var wrapper struct { - // OFAC - SDNs []*ofac.SDN `json:"SDNs"` - AltNames []*ofac.AlternateIdentity `json:"altNames"` - Addresses []*ofac.Address `json:"addresses"` - SectoralSanctions []*csl_us.SSI `json:"sectoralSanctions"` - // BIS - DeniedPersons []*dpl.DPL `json:"deniedPersons"` - BISEntities []*csl_us.EL `json:"bisEntities"` - } - if err := json.NewDecoder(w.Body).Decode(&wrapper); err != nil { - t.Fatal(err) - } - - // OFAC - require.Equal(t, "2681", wrapper.SDNs[0].EntityID) - require.Equal(t, "HAWATMA, Nayif", wrapper.SDNs[0].SDNName) - require.Equal(t, "559", wrapper.AltNames[0].EntityID) - require.Equal(t, "735", wrapper.Addresses[0].EntityID) - require.Equal(t, "18736", wrapper.SectoralSanctions[0].EntityID) - - // BIS - require.Equal(t, "P.O. BOX 28360", wrapper.DeniedPersons[0].StreetAddress) - require.Equal(t, "Mohammad Jan Khan Mangal", wrapper.BISEntities[0].Name) -} - -func TestSearch__Name(t *testing.T) { - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/search?name=Dr+AL+ZAWAHIRI&limit=1", nil) - - router := mux.NewRouter() - combinedSearcher := newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - // OFAC - combinedSearcher.SDNs = sdnSearcher.SDNs - combinedSearcher.Alts = altSearcher.Alts - combinedSearcher.SSIs = ssiSearcher.SSIs - // BIS - combinedSearcher.DPs = dplSearcher.DPs - combinedSearcher.BISEntities = bisEntitySearcher.BISEntities - - addSearchRoutes(log.NewNopLogger(), router, combinedSearcher) - router.ServeHTTP(w, req) - w.Flush() - - require.Equal(t, http.StatusOK, w.Code) - require.Contains(t, w.Body.String(), `"match":0.95588`) - require.Contains(t, w.Body.String(), `"matchedName":"dr ayman al zawahiri"`) - - var wrapper struct { - // OFAC - SDNs []*ofac.SDN `json:"SDNs"` - Alts []*ofac.AlternateIdentity `json:"altNames"` - SSIs []*csl_us.SSI `json:"sectoralSanctions"` - // BIS - DPs []*dpl.DPL `json:"deniedPersons"` - ELs []*csl_us.EL `json:"bisEntities"` - } - if err := json.NewDecoder(w.Body).Decode(&wrapper); err != nil { - t.Fatal(err) - } - if len(wrapper.SDNs) != 1 || len(wrapper.Alts) != 1 || len(wrapper.SSIs) != 1 || len(wrapper.DPs) != 1 || len(wrapper.ELs) != 1 { - t.Fatalf("SDNs=%d Alts=%d SSIs=%d DPs=%d ELs=%d", - len(wrapper.SDNs), len(wrapper.Alts), len(wrapper.SSIs), len(wrapper.DPs), len(wrapper.ELs)) - } - - require.Equal(t, "2676", wrapper.SDNs[0].EntityID) - require.Equal(t, "4691", wrapper.Alts[0].EntityID) - - require.Equal(t, "18736", wrapper.SSIs[0].EntityID) - require.Equal(t, "AL NASER WINGS AIRLINES", wrapper.DPs[0].Name) - require.Equal(t, "Luqman Yasin Yunus Shgragi", wrapper.ELs[0].Name) -} - -func TestSearch__AltName(t *testing.T) { - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/search?altName=SOGO+KENKYUSHO&limit=1", nil) - - router := mux.NewRouter() - addSearchRoutes(log.NewNopLogger(), router, altSearcher) - router.ServeHTTP(w, req) - w.Flush() - - if w.Code != http.StatusOK { - t.Errorf("bogus status code: %d", w.Code) - } - - require.Equal(t, http.StatusOK, w.Code) - require.Contains(t, w.Body.String(), `"match":0.98`) - require.Contains(t, w.Body.String(), `"matchedName":"i c sogo kenkyusho"`) - - var wrapper struct { - Alts []*ofac.AlternateIdentity `json:"altNames"` - } - if err := json.NewDecoder(w.Body).Decode(&wrapper); err != nil { - t.Fatal(err) - } - if wrapper.Alts[0].EntityID != "4691" { - t.Errorf("%#v", wrapper.Alts[0]) - } -} - -func TestSearch__ID(t *testing.T) { - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/search?id=5892464&limit=2", nil) - - router := mux.NewRouter() - addSearchRoutes(log.NewNopLogger(), router, idSearcher) - router.ServeHTTP(w, req) - w.Flush() - - if w.Code != http.StatusOK { - t.Errorf("bogus status code: %d", w.Code) - } - - if v := w.Body.String(); !strings.Contains(v, `"match":1`) { - t.Error(v) - } - - var wrapper struct { - SDNs []*ofac.SDN `json:"SDNs"` - } - if err := json.NewDecoder(w.Body).Decode(&wrapper); err != nil { - t.Fatal(err) - } - if wrapper.SDNs[0].EntityID != "22790" { - t.Errorf("%#v", wrapper.SDNs[0]) - } -} - -func TestSearch__EscapeQuery(t *testing.T) { - req, err := http.NewRequest("GET", "/search?name=John%2BDoe", nil) - require.NoError(t, err) - - name := req.URL.Query().Get("name") - require.Equal(t, "John+Doe", name) - - name, _ = url.QueryUnescape(name) - require.Equal(t, "John Doe", name) - - req, err = http.NewRequest("GET", "/search?name=John+Doe", nil) - if err != nil { - t.Fatal(err) - } - name = req.URL.Query().Get("name") - require.Equal(t, "John Doe", name) - - name, _ = url.QueryUnescape(name) - require.Equal(t, "John Doe", name) -} diff --git a/cmd/server/search_test.go b/cmd/server/search_test.go deleted file mode 100644 index 24f9db68..00000000 --- a/cmd/server/search_test.go +++ /dev/null @@ -1,741 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "math" - "net/http/httptest" - "net/url" - "path/filepath" - "sync" - "testing" - - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/internal/prepare" - "github.com/moov-io/watchman/pkg/csl_eu" - "github.com/moov-io/watchman/pkg/csl_uk" - "github.com/moov-io/watchman/pkg/csl_us" - "github.com/moov-io/watchman/pkg/dpl" - "github.com/moov-io/watchman/pkg/ofac" - - "github.com/stretchr/testify/require" -) - -var ( - // Live Searcher - testLiveSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - testSearcherStats *DownloadStats - testSearcherOnce sync.Once - - noLogPipeliner = prepare.NewPipeliner(log.NewNopLogger(), false) - - // Mock Searchers - addressSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - altSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - sdnSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - idSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - dplSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - - // CSL Searchers - bisEntitySearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - meuSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - ssiSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - isnSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - uvlSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - fseSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - plcSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - capSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - dtcSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - cmicSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - ns_mbsSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - - eu_cslSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - uk_cslSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - uk_sanctionsListSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1) -) - -func init() { - addressSearcher.Addresses = precomputeAddresses([]*ofac.Address{ - { - EntityID: "173", - AddressID: "129", - Address: "Ibex House, The Minories", - CityStateProvincePostalCode: "London EC3N 1DY", - Country: "United Kingdom", - }, - { - EntityID: "735", - AddressID: "447", - Address: "Piarco Airport", - CityStateProvincePostalCode: "Port au Prince", - Country: "Haiti", - }, - }) - altSearcher.Alts = precomputeAlts([]*ofac.AlternateIdentity{ - { // Real OFAC entry - EntityID: "559", - AlternateID: "481", - AlternateType: "aka", - AlternateName: "CIMEX", - }, - { - EntityID: "4691", - AlternateID: "3887", - AlternateType: "aka", - AlternateName: "A.I.C. SOGO KENKYUSHO", - }, - }, noLogPipeliner) - sdnSearcher.SDNs = precomputeSDNs([]*ofac.SDN{ - { - EntityID: "2676", - SDNName: "AL ZAWAHIRI, Dr. Ayman", - SDNType: "individual", - Programs: []string{"SDGT", "SDT"}, - Title: "Operational and Military Leader of JIHAD GROUP", - Remarks: "DOB 19 Jun 1951; POB Giza, Egypt; Passport 1084010 (Egypt); alt. Passport 19820215; Operational and Military Leader of JIHAD GROUP.", - }, - { - EntityID: "2681", - SDNName: "HAWATMA, Nayif", - SDNType: "individual", - Programs: []string{"SDT"}, - Title: "Secretary General of DEMOCRATIC FRONT FOR THE LIBERATION OF PALESTINE - HAWATMEH FACTION", - Remarks: "DOB 1933; Secretary General of DEMOCRATIC FRONT FOR THE LIBERATION OF PALESTINE - HAWATMEH FACTION.", - }, - }, nil, noLogPipeliner) - idSearcher.SDNs = precomputeSDNs([]*ofac.SDN{ - { - EntityID: "22790", - SDNName: "MADURO MOROS, Nicolas", - SDNType: "individual", - Programs: []string{"VENEZUELA"}, - Title: "President of the Bolivarian Republic of Venezuela", - Remarks: "DOB 23 Nov 1962; POB Caracas, Venezuela; citizen Venezuela; Gender Male; Cedula No. 5892464 (Venezuela); President of the Bolivarian Republic of Venezuela.", - }, - }, nil, noLogPipeliner) - dplSearcher.DPs = precomputeDPs([]*dpl.DPL{ - { - Name: "AL NASER WINGS AIRLINES", - StreetAddress: "P.O. BOX 28360", - City: "DUBAI", - State: "", - Country: "AE", - PostalCode: "", - EffectiveDate: "06/05/2019", - ExpirationDate: "12/03/2019", - StandardOrder: "Y", - LastUpdate: "2019-06-12", - Action: "FR NOTICE ADDED, TDO RENEWAL, F.R. NOTICE ADDED, TDO RENEWAL ADDED, TDO RENEWAL ADDED, F.R. NOTICE ADDED", - FRCitation: "82 F.R. 61745 12/29/2017, 83F.R. 28801 6/21/2018, 84 F.R. 27233 6/12/2019", - }, - { - Name: "PRESTON JOHN ENGEBRETSON", - StreetAddress: "12725 ROYAL DRIVE", - City: "STAFFORD", - State: "TX", - Country: "US", - PostalCode: "77477", - EffectiveDate: "01/24/2002", - ExpirationDate: "01/24/2027", - StandardOrder: "Y", - LastUpdate: "2002-01-28", - Action: "STANDARD ORDER", - FRCitation: "67 F.R. 7354 2/19/02 66 F.R. 48998 9/25/01 62 F.R. 26471 5/14/97 62 F.R. 34688 6/27/97 62 F.R. 60063 11/6/97 63 F.R. 25817 5/11/98 63 F.R. 58707 11/2/98 64 F.R. 23049 4/29/99", - }, - }, noLogPipeliner) - ssiSearcher.SSIs = precomputeCSLEntities[csl_us.SSI]([]*csl_us.SSI{ - { - EntityID: "18782", - Type: "Entity", - Programs: []string{"SYRIA", "UKRAINE-EO13662"}, - Name: "ROSOBORONEKSPORT OAO", - Addresses: []string{"27 Stromynka ul., Moscow, 107076, RU"}, - Remarks: []string{"For more information on directives, please visit the following link: http://www.treasury.gov/resource-center/sanctions/Programs/Pages/ukraine.aspx#directives", "(Linked To: ROSTEC)"}, - AlternateNames: []string{"RUSSIAN DEFENSE EXPORT ROSOBORONEXPORT", "KENKYUSHO", "ROSOBORONEXPORT JSC", "ROSOBORONEKSPORT OJSC", "OJSC ROSOBORONEXPORT", "ROSOBORONEXPORT"}, - IDsOnRecord: []string{"1117746521452, Registration ID", "56467052, Government Gazette Number", "7718852163, Tax ID No.", "Subject to Directive 3, Executive Order 13662 Directive Determination -", "www.roe.ru, Website"}, - SourceListURL: "http://bit.ly/1QWTIfE", - SourceInfoURL: "http://bit.ly/1MLgou0", - }, - { - EntityID: "18736", - Type: "Entity", - Programs: []string{"UKRAINE-EO13662"}, - Name: "VTB SPECIALIZED DEPOSITORY, CJSC", - Addresses: []string{"35 Myasnitskaya Street, Moscow, 101000, RU"}, - Remarks: []string{"For more information on directives, please visit the following link: http://www.treasury.gov/resource-center/sanctions/Programs/Pages/ukraine.aspx#directives", "(Linked To: ROSTEC)"}, - AlternateNames: []string{"CJS VTB SPECIALIZED DEPOSITORY"}, - IDsOnRecord: []string{"1117746521452, Registration ID", "56467052, Government Gazette Number", "7718852163, Tax ID No.", "Subject to Directive 3, Executive Order 13662 Directive Determination -", "www.roe.ru, Website"}, - SourceListURL: "http://bit.ly/1QWTIfE", - SourceInfoURL: "http://bit.ly/1MLgou0", - }, - }, noLogPipeliner) - meuSearcher.MilitaryEndUsers = precomputeCSLEntities[csl_us.MEU]([]*csl_us.MEU{ - { - EntityID: "26744194bd9b5cbec49db6ee29a4b53c697c7420", - Name: "AECC Aviation Power Co. Ltd.", - Addresses: "Xiujia Bay, Weiyong Dt, Xian, 710021, CN", - FRNotice: "85 FR 83799", - StartDate: "2020-12-23", - EndDate: "", - }, - { - EntityID: "d54346ef81802673c1b1daeb2ca8bd5d13755abd", - Name: "AECC China Gas Turbine Establishment", - Addresses: "No. 1 Hangkong Road, Mianyang, Sichuan, CN", - FRNotice: "85 FR 83799", - StartDate: "2020-12-23", - EndDate: "", - }, - }, noLogPipeliner) - bisEntitySearcher.BISEntities = precomputeCSLEntities[csl_us.EL]([]*csl_us.EL{ - { - Name: "Mohammad Jan Khan Mangal", - AlternateNames: []string{"Air I"}, - Addresses: []string{"Kolola Pushta, Charahi Gul-e-Surkh, Kabul, AF", "Maidan Sahr, Hetefaq Market, Paktiya, AF"}, - StartDate: "11/13/19", - LicenseRequirement: "For all items subject to the EAR (See ¬ß744.11 of the EAR). ", - LicensePolicy: "Presumption of denial.", - FRNotice: "81 FR 57451", - SourceListURL: "http://bit.ly/1L47xrV", - SourceInfoURL: "http://bit.ly/1L47xrV", - }, - { - Name: "Luqman Yasin Yunus Shgragi", - AlternateNames: []string{"Lkemanasel Yosef", "Luqman Sehreci."}, - Addresses: []string{"Savcili Mahalesi Turkmenler Caddesi No:2, Sahinbey, Gaziantep, TR", "Sanayi Mahalesi 60214 Nolu Caddesi No 11, SehitKamil, Gaziantep, TR"}, - StartDate: "8/23/16", - LicenseRequirement: "For all items subject to the EAR. (See ¬ß744.11 of the EAR)", - LicensePolicy: "Presumption of denial.", - FRNotice: "81 FR 57451", - SourceListURL: "http://bit.ly/1L47xrV", - SourceInfoURL: "http://bit.ly/1L47xrV", - }, - }, noLogPipeliner) - isnSearcher.ISNs = precomputeCSLEntities[csl_us.ISN]([]*csl_us.ISN{ - { - EntityID: "2d2db09c686e4829d0ef1b0b04145eec3d42cd88", - Programs: []string{"E.O. 13382", "Export-Import Bank Act", "Nuclear Proliferation Prevention Act"}, - Name: "Abdul Qadeer Khan", - FederalRegisterNotice: "Vol. 74, No. 11, 01/16/09", - StartDate: "2009-01-09", - Remarks: []string{"Associated with the A.Q. Khan Network"}, - SourceListURL: "http://bit.ly/1NuVFxV", - AlternateNames: []string{"ZAMAN", "Haydar"}, - SourceInfoURL: "http://bit.ly/1NuVFxV", - }, - }, noLogPipeliner) - uvlSearcher.UVLs = precomputeCSLEntities[csl_us.UVL]([]*csl_us.UVL{ - { - EntityID: "f15fa805ff4ac5e09026f5e78011a1bb6b26dec2", - Name: "Atlas Sanatgaran", - Addresses: []string{"Komitas 26/114, Yerevan, Armenia, AM"}, - SourceListURL: "http://bit.ly/1iwwTSJ", - SourceInfoURL: "http://bit.ly/1Qi4R7Z", - }, - }, noLogPipeliner) - fseSearcher.FSEs = precomputeCSLEntities[csl_us.FSE]([]*csl_us.FSE{ - { - EntityID: "17526", - EntityNumber: "17526", - Type: "Individual", - Programs: []string{"SYRIA", "FSE-SY"}, - Name: "BEKTAS, Halis", - Addresses: nil, - SourceListURL: "https://bit.ly/1QWTIfE", - Citizenships: "CH", - DatesOfBirth: "1966-02-13", - SourceInfoURL: "http://bit.ly/1N1docf", - IDs: []string{"CH, X0906223, Passport"}, - }, - }, noLogPipeliner) - plcSearcher.PLCs = precomputeCSLEntities[csl_us.PLC]([]*csl_us.PLC{ - { - EntityID: "9702", - EntityNumber: "9702", - Type: "Individual", - Programs: []string{"NS-PLC", "Office of Misinformation"}, - Name: "SALAMEH, Salem", - Addresses: []string{"123 Dunbar Street, Testerville, TX, Palestine"}, - Remarks: "HAMAS - Der al-Balah", - SourceListURL: "https://bit.ly/1QWTIfE", - AlternateNames: []string{"SALAMEH, Salem Ahmad Abdel Hadi"}, - DatesOfBirth: "1951", - PlacesOfBirth: "", - SourceInfoURL: "http://bit.ly/2tjOLpx", - }, - }, noLogPipeliner) - capSearcher.CAPs = precomputeCSLEntities[csl_us.CAP]([]*csl_us.CAP{ - { - EntityID: "20002", - EntityNumber: "20002", - Type: "Entity", - Programs: []string{"UKRAINE-EO13662", "RUSSIA-EO14024"}, - Name: "BM BANK PUBLIC JOINT STOCK COMPANY", - Addresses: []string{"Bld 3 8/15, Rozhdestvenka St., Moscow, 107996, RU"}, - Remarks: []string{"All offices worldwide", "for more information on directives, please visit the following link: https://www.treasury.gov/resource-center/sanctions/Programs/Pages/ukraine.aspx#directives", "(Linked To: VTB BANK PUBLIC JOINT STOCK COMPANY)"}, - SourceListURL: "", - AlternateNames: []string{"BM BANK JSC", "BM BANK AO", "AKTSIONERNOE OBSHCHESTVO BM BANK", - "PAO BM BANK", "BANK MOSKVY PAO", "BANK OF MOSCOW", - "AKTSIONERNY KOMMERCHESKI BANK BANK MOSKVY OTKRYTOE AKTSIONERNOE OBSCHCHESTVO", - "JOINT STOCK COMMERCIAL BANK - BANK OF MOSCOW OPEN JOINT STOCK COMPANY"}, - SourceInfoURL: "http://bit.ly/2PqohAD", - IDs: []string{"RU, 1027700159497, Registration Number", - "RU, 29292940, Government Gazette Number", - "MOSWRUMM, SWIFT/BIC", - "www.bm.ru, Website", - "Subject to Directive 1, Executive Order 13662 Directive Determination -", - "044525219, BIK (RU)", - "Financial Institution, Target Type"}, - }, - }, noLogPipeliner) - dtcSearcher.DTCs = precomputeCSLEntities[csl_us.DTC]([]*csl_us.DTC{ - { - EntityID: "d44d88d0265d93927b9ff1c13bbbb7c7db64142c", - Name: "Yasmin Ahmed", - FederalRegisterNotice: "69 FR 17468", - SourceListURL: "http://bit.ly/307FuRQ", - AlternateNames: []string{"Yasmin Tariq", "Fatimah Mohammad"}, - SourceInfoURL: "http://bit.ly/307FuRQ", - }, - }, noLogPipeliner) - cmicSearcher.CMICs = precomputeCSLEntities[csl_us.CMIC]([]*csl_us.CMIC{ - { - EntityID: "32091", - EntityNumber: "32091", - Type: "Entity", - Programs: []string{"CMIC-EO13959"}, - Name: "PROVEN HONOUR CAPITAL LIMITED", - Addresses: []string{"C/O Vistra Corporate Services Centre, Wickhams Cay II, Road Town, VG1110, VG"}, - Remarks: []string{"(Linked To: HUAWEI INVESTMENT & HOLDING CO., LTD.)"}, - SourceListURL: "https://bit.ly/1QWTIfE", - AlternateNames: []string{"PROVEN HONOUR CAPITAL LTD", "PROVEN HONOUR"}, - SourceInfoURL: "https://bit.ly/3zsMQ4n", - IDs: []string{"Proven Honour Capital Ltd, Issuer Name", "Proven Honour Capital Limited, Issuer Name", "XS1233275194, ISIN", - "HK0000216777, ISIN", "Private Company, Target Type", "XS1401816761, ISIN", "HK0000111952, ISIN", "03 Jun 2021, Listing Date (CMIC)", - "02 Aug 2021, Effective Date (CMIC)", "03 Jun 2022, Purchase/Sales For Divestment Date (CMIC)"}, - }, - }, noLogPipeliner) - ns_mbsSearcher.NS_MBSs = precomputeCSLEntities[csl_us.NS_MBS]([]*csl_us.NS_MBS{ - { - EntityID: "17016", - EntityNumber: "17016", - Type: "Entity", - Programs: []string{"UKRAINE-EO13662", "MBS"}, - Name: "GAZPROMBANK JOINT STOCK COMPANY", - Addresses: []string{"16 Nametkina Street, Bldg. 1, Moscow, 117420, RU"}, - Remarks: []string{"For more information on directives, please visit the following link: http://www.treasury.gov/resource-center/sanctions/Programs/Pages/ukraine.aspx#directives."}, - AlternateNames: []string{"GAZPROMBANK OPEN JOINT STOCK COMPANY", "BANK GPB JSC", "GAZPROMBANK AO", "JOINT STOCK BANK OF THE GAS INDUSTRY GAZPROMBANK"}, - SourceInfoURL: "https://bit.ly/2MbsybU", - IDs: []string{"RU, 1027700167110, Registration Number", "RU, 09807684, Government Gazette Number", "RU, 7744001497, Tax ID No.", - "www.gazprombank.ru, Website", "GAZPRUMM, SWIFT/BIC", "Subject to Directive 1, Executive Order 13662 Directive Determination -", - "Subject to Directive 3 - All transactions in, provision of financing for, and other dealings in new debt of longer than 14 days maturity or new equity where such new debt or new equity is issued on or after the 'Effective Date (EO 14024 Directive)' associated with this name are prohibited., Executive Order 14024 Directive Information", - "31 Jul 1990, Organization Established Date", "24 Feb 2022, Listing Date (EO 14024 Directive 3):", "26 Mar 2022, Effective Date (EO 14024 Directive 3):", - "For more information on directives, please visit the following link: https://home.treasury.gov/policy-issues/financial-sanctions/sanctions-programs-and-country-information/russian-harmful-foreign-activities-sanctions#directives, Executive Order 14024 Directive Information -"}, - }, - }, noLogPipeliner) - - eu_cslSearcher.EUCSL = precomputeCSLEntities[csl_eu.CSLRecord]([]*csl_eu.CSLRecord{{ - FileGenerationDate: "28/10/2022", - EntityLogicalID: 13, - EntityRemark: "(UNSC RESOLUTION 1483)", - EntitySubjectType: "person", - EntityPublicationURL: "http://eur-lex.europa.eu/LexUriServ/LexUriServ.do?uri=OJ:L:2003:169:0006:0023:EN:PDF", - EntityReferenceNumber: "", - NameAliasWholeNames: []string{"Saddam Hussein Al-Tikriti", "Abu Ali", "Abou Ali"}, - AddressCities: []string{"test city"}, - AddressStreets: []string{"test street"}, - AddressPoBoxes: []string{"test po box"}, - AddressZipCodes: []string{"test zip"}, - AddressCountryDescriptions: []string{"test country"}, - BirthDates: []string{"1937-04-28"}, - BirthCities: []string{"al-Awja, near Tikrit"}, - BirthCountries: []string{"IRAQ"}, - ValidFromTo: map[string]string{"2022": "2030"}, - }}, noLogPipeliner) - - uk_cslSearcher.UKCSL = precomputeCSLEntities([]*csl_uk.CSLRecord{{ - Names: []string{"'ABD AL-NASIR"}, - Addresses: []string{"Tall 'Afar"}, - GroupType: "Individual", - GroupID: 13720, - }}, noLogPipeliner) - - uk_sanctionsListSearcher.UKSanctionsList = precomputeCSLEntities([]*csl_uk.SanctionsListRecord{{ - Names: []string{"HAJI KHAIRULLAH HAJI SATTAR MONEY EXCHANGE"}, - Addresses: []string{"Branch Office 2, Peshawar, Khyber Paktunkhwa Province, Pakistan"}, - UniqueID: "AFG0001", - }}, noLogPipeliner) -} - -func createTestSearcher(t testing.TB) *searcher { - t.Helper() - t.Setenv("WITH_UK_SANCTIONS_LIST", "false") - - if testing.Short() { - t.Skip("-short enabled") - } - - testSearcherOnce.Do(func() { - stats, err := testLiveSearcher.refreshData("") - if err != nil { - t.Fatal(err) - } - testSearcherStats = stats - }) - - return testLiveSearcher -} - -func createBenchmarkSearcher(b *testing.B) *searcher { - b.Helper() - testSearcherOnce.Do(func() { - stats, err := testLiveSearcher.refreshData(filepath.Join("..", "..", "test", "testdata", "bench")) - if err != nil { - b.Fatal(err) - } - testSearcherStats = stats - }) - verifyDownloadStats(b) - return testLiveSearcher -} - -func verifyDownloadStats(b *testing.B) { - b.Helper() - - // OFAC - require.Greater(b, testSearcherStats.SDNs, 1) - require.Greater(b, testSearcherStats.Alts, 1) - require.Greater(b, testSearcherStats.Addresses, 1) - - // BIS - require.Greater(b, testSearcherStats.DeniedPersons, 1) - - // CSL - require.Greater(b, testSearcherStats.BISEntities, 1) - require.Greater(b, testSearcherStats.MilitaryEndUsers, 1) - require.Greater(b, testSearcherStats.SectoralSanctions, 1) - require.Greater(b, testSearcherStats.Unverified, 1) - require.Greater(b, testSearcherStats.NonProliferationSanctions, 1) - require.Greater(b, testSearcherStats.ForeignSanctionsEvaders, 1) - require.Greater(b, testSearcherStats.PalestinianLegislativeCouncil, 1) - require.Greater(b, testSearcherStats.CAPTA, 1) - require.Greater(b, testSearcherStats.ITARDebarred, 1) - require.Greater(b, testSearcherStats.ChineseMilitaryIndustrialComplex, 1) - require.Greater(b, testSearcherStats.NonSDNMenuBasedSanctions, 1) - - // EU - CSL - require.Greater(b, testSearcherStats.EUCSL, 1) - // UK - CSL - require.Greater(b, testSearcherStats.UKCSL, 1) - // UK - SanctionsList - require.Equal(b, 0, testSearcherStats.UKSanctionsList) -} - -// TestSearch_liveData will download the real data and run searches against the corpus. -// This test is designed to tweak match percents and results. -func TestSearch_liveData(t *testing.T) { - searcher := createTestSearcher(t) - cases := []struct { - name string - match float64 // top match % - }{ - {"Nicolas MADURO", 0.958}, - {"nicolas maduro", 0.958}, - {"NICOLAS maduro", 0.958}, - } - - keeper := keepSDN(filterRequest{}) - for i := range cases { - sdns := searcher.TopSDNs(1, 0.00, cases[i].name, keeper) - if len(sdns) == 0 { - t.Errorf("name=%q got no results", cases[i].name) - } - eql(t, fmt.Sprintf("%q (SDN=%s) matches %q ", cases[i].name, sdns[0].EntityID, sdns[0].name), sdns[0].match, cases[i].match) - } -} - -func TestSearch__topAddressesAddress(t *testing.T) { - it := topAddressesAddress("needle")(&Address{address: "needleee"}) - - eql(t, "topAddressesAddress", it.weight, 0.950) - if add, ok := it.value.(*Address); !ok || add.address != "needleee" { - t.Errorf("got %#v", add) - } -} - -func TestSearch__topAddressesCountry(t *testing.T) { - it := topAddressesAddress("needle")(&Address{address: "needleee"}) - - eql(t, "topAddressesCountry", it.weight, 0.950) - if add, ok := it.value.(*Address); !ok || add.address != "needleee" { - t.Errorf("got %#v", add) - } -} - -func TestSearch__multiAddressCompare(t *testing.T) { - it := multiAddressCompare( - topAddressesAddress("needle"), - topAddressesCountry("other"), - )(&Address{address: "needlee", country: "other"}) - - eql(t, "multiAddressCompare", it.weight, 0.986) - if add, ok := it.value.(*Address); !ok || add.address != "needlee" || add.country != "other" { - t.Errorf("got %#v", add) - } -} - -func TestSearch__extractSearchLimit(t *testing.T) { - // Too high, fallback to hard max - req := httptest.NewRequest("GET", "/?limit=1000", nil) - if limit := extractSearchLimit(req); limit != hardResultsLimit { - t.Errorf("got limit of %d", limit) - } - - // No limit, use default - req = httptest.NewRequest("GET", "/", nil) - if limit := extractSearchLimit(req); limit != softResultsLimit { - t.Errorf("got limit of %d", limit) - } - - // Between soft and hard max - req = httptest.NewRequest("GET", "/?limit=25", nil) - if limit := extractSearchLimit(req); limit != 25 { - t.Errorf("got limit of %d", limit) - } - - // Lower than soft max - req = httptest.NewRequest("GET", "/?limit=1", nil) - if limit := extractSearchLimit(req); limit != 1 { - t.Errorf("got limit of %d", limit) - } -} - -func TestSearch__addressSearchRequest(t *testing.T) { - u, _ := url.Parse("https://moov.io/search?address=add&city=new+york&state=ny&providence=prov&zip=44433&country=usa") - req := readAddressSearchRequest(u) - if req.Address != "add" { - t.Errorf("req.Address=%s", req.Address) - } - if req.City != "new york" { - t.Errorf("req.City=%s", req.City) - } - if req.State != "ny" { - t.Errorf("req.State=%s", req.State) - } - if req.Providence != "prov" { - t.Errorf("req.Providence=%s", req.Providence) - } - if req.Zip != "44433" { - t.Errorf("req.Zip=%s", req.Zip) - } - if req.Country != "usa" { - t.Errorf("req.Country=%s", req.Country) - } - if req.empty() { - t.Error("req is not empty") - } - - req = addressSearchRequest{} - if !req.empty() { - t.Error("req is empty now") - } - req.Address = "1600 1st St" - if req.empty() { - t.Error("req is not empty now") - } -} - -func TestSearch__FindAddresses(t *testing.T) { - addresses := addressSearcher.FindAddresses(1, "173") - if v := len(addresses); v != 1 { - t.Fatalf("len(addresses)=%d", v) - } - if addresses[0].EntityID != "173" { - t.Errorf("got %#v", addresses[0]) - } -} - -func TestSearch__TopAddresses(t *testing.T) { - addresses := addressSearcher.TopAddresses(1, 0.00, "Piarco Air") - if len(addresses) == 0 { - t.Fatal("empty Addresses") - } - if addresses[0].Address.EntityID != "735" { - t.Errorf("%#v", addresses[0].Address) - } -} - -func TestSearch__TopAddressFn(t *testing.T) { - addresses := TopAddressesFn(1, 0.00, addressSearcher.Addresses, topAddressesCountry("United Kingdom")) - if len(addresses) == 0 { - t.Fatal("empty Addresses") - } - if addresses[0].Address.EntityID != "173" { - t.Errorf("%#v", addresses[0].Address) - } -} - -func TestSearch__FindAlts(t *testing.T) { - alts := altSearcher.FindAlts(1, "559") - if v := len(alts); v != 1 { - t.Fatalf("len(alts)=%d", v) - } - if alts[0].EntityID != "559" { - t.Errorf("got %#v", alts[0]) - } -} - -func TestSearch__TopAlts(t *testing.T) { - alts := altSearcher.TopAltNames(1, 0.00, "SOGO KENKYUSHO") - if len(alts) == 0 { - t.Fatal("empty AltNames") - } - if alts[0].AlternateIdentity.EntityID != "4691" { - t.Errorf("%#v", alts[0].AlternateIdentity) - } -} - -func TestSearch__FindSDN(t *testing.T) { - sdn := sdnSearcher.FindSDN("2676") - if sdn == nil { - t.Fatal("nil SDN") - } - if sdn.EntityID != "2676" { - t.Errorf("got %#v", sdn) - } -} - -func TestSearch__TopSDNs(t *testing.T) { - keeper := keepSDN(filterRequest{}) - sdns := sdnSearcher.TopSDNs(1, 0.00, "Ayman ZAWAHIRI", keeper) - if len(sdns) == 0 { - t.Fatal("empty SDNs") - } - require.Equal(t, "2676", sdns[0].EntityID) -} - -func TestSearch__TopDPs(t *testing.T) { - dps := dplSearcher.TopDPs(1, 0.00, "NASER AIRLINES") - if len(dps) == 0 { - t.Fatal("empty DPs") - } - // DPL doesn't have any entity IDs. Comparing expected address components instead - if dps[0].DeniedPerson.StreetAddress != "P.O. BOX 28360" || dps[0].DeniedPerson.City != "DUBAI" { - t.Errorf("%#v", dps[0].DeniedPerson) - } -} - -func TestSearch__extractIDFromRemark(t *testing.T) { - cases := []struct { - input, expected string - }{ - {"Cedula No. 10517860 (Venezuela);", "10517860"}, - {"National ID No. 22095919778 (Norway).", "22095919778"}, - {"Driver's License No. 180839 (Mexico);", "180839"}, - {"Immigration No. A38839964 (United States).", "A38839964"}, - {"C.R. No. 79190 (United Arab Emirates).", "79190"}, - {"Electoral Registry No. RZZVAL62051010M200 (Mexico).", "RZZVAL62051010M200"}, - {"Trade License No. GE0426505 (Italy).", "GE0426505"}, - {"Public Security and Immigration No. 98.805", "98.805"}, - {"Folio Mercantil No. 578349 (Panama).", "578349"}, - {"Trade License No. C 37422 (Malta).", "C 37422"}, - {"Moroccan Personal ID No. E 427689 (Morocco) issued 20 Mar 2001.", "E 427689"}, - {"National ID No. 5-5715-00025-50-6 (Thailand);", "5-5715-00025-50-6"}, - {"Trade License No. HRB94311.", "HRB94311"}, - {"Registered Charity No. 1040094.", "1040094"}, - {"Bosnian Personal ID No. 1005967953038;", "1005967953038"}, - {"Telephone No. 009613679153;", "009613679153"}, - {"Tax ID No. AABA 670850 Y.", "AABA 670850"}, - {"Phone No. 263-4-486946; Fax No. 263-4-487261.", "263-4-486946"}, - {"D-U-N-S Number 56-558-7594; V.A.T. Number MT15388917 (Malta); Trade License No. C 24129 (Malta); Company Number 4220856; Linked To: DEBONO, Darren.", "C 24129"}, // SDN 23410 - } - for i := range cases { - result := extractIDFromRemark(cases[i].input) - if cases[i].expected != result { - t.Errorf("input=%s expected=%s result=%s", cases[i].input, cases[i].expected, result) - } - } -} - -func TestSearch__FindSDNsByRemarksID(t *testing.T) { - s := newSearcher(log.NewNopLogger(), noLogPipeliner, 1) - s.SDNs = []*SDN{ - { - SDN: &ofac.SDN{ - EntityID: "22790", - }, - id: "Cedula No. C 5892464 (Venezuela);", - }, - { - SDN: &ofac.SDN{ - EntityID: "99999", - }, - id: "Other", - }, - } - - sdns := s.FindSDNsByRemarksID(1, "5892464") - if len(sdns) != 1 { - t.Fatalf("sdns=%#v", sdns) - } - if sdns[0].EntityID != "22790" { - t.Errorf("sdns[0].EntityID=%v", sdns[0].EntityID) - } - - // successful multi-part match - s.SDNs[0].id = "2456 7890" - sdns = s.FindSDNsByRemarksID(1, "2456 7890") - if len(sdns) != 1 { - t.Fatalf("sdns=%#v", sdns) - } - if sdns[0].EntityID != "22790" { - t.Errorf("sdns[0].EntityID=%v", sdns[0].EntityID) - } - - // incomplete query (not enough numerical query parts) - sdns = s.FindSDNsByRemarksID(1, "2456") - if len(sdns) != 0 { - t.Fatalf("sdns=%#v", sdns) - } - sdns = s.FindSDNsByRemarksID(1, "7890") - if len(sdns) != 0 { - t.Fatalf("sdns=%#v", sdns) - } - - // query doesn't match - sdns = s.FindSDNsByRemarksID(1, "12456") - if len(sdns) != 0 { - t.Fatalf("sdns=%#v", sdns) - } - - // empty SDN remarks ID - s.SDNs[0].id = "" - sdns = s.FindSDNsByRemarksID(1, "12456") - if len(sdns) != 0 { - t.Fatalf("sdns=%#v", sdns) - } - - // empty query - sdns = s.FindSDNsByRemarksID(1, "") - if len(sdns) != 0 { - t.Fatalf("sdns=%#v", sdns) - } -} - -func eql(t *testing.T, desc string, x, y float64) { - t.Helper() - if math.IsNaN(x) || math.IsNaN(y) { - t.Fatalf("%s: x=%.2f y=%.2f", desc, x, y) - } - if math.Abs(x-y) > 0.01 { - t.Errorf("%s: %.3f != %.3f", desc, x, y) - } -} - -func TestEql(t *testing.T) { - eql(t, "", 0.1, 0.1) - eql(t, "", 0.0001, 0.00002) -} diff --git a/cmd/server/search_uk_csl.go b/cmd/server/search_uk_csl.go deleted file mode 100644 index 08712eab..00000000 --- a/cmd/server/search_uk_csl.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "net/http" - - moovhttp "github.com/moov-io/base/http" - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/pkg/csl_uk" -) - -// search UKCLS -func searchUKCSL(logger log.Logger, searcher *searcher) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - w = wrapResponseWriter(logger, w, r) - requestID := moovhttp.GetRequestID(r) - - limit := extractSearchLimit(r) - filters := buildFilterRequest(r.URL) - minMatch := extractSearchMinMatch(r) - - name := r.URL.Query().Get("name") - resp := buildFullSearchResponseWith(searcher, ukGatherings, filters, limit, minMatch, name) - - logger.Info().With(log.Fields{ - "name": log.String(name), - "requestID": log.String(requestID), - }).Log("performing UK-CSL search") - - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(resp) - } -} - -// TopCSL searches the UK Sanctions list by Name and Alias -func (s *searcher) TopUKCSL(limit int, minMatch float64, name string) []*Result[csl_uk.CSLRecord] { - s.RLock() - defer s.RUnlock() - - s.Gate.Start() - defer s.Gate.Done() - - return topResults[csl_uk.CSLRecord](limit, minMatch, name, s.UKCSL) -} - -// TopUKSanctionsList searches the UK Sanctions list by Name and Alias -func (s *searcher) TopUKSanctionsList(limit int, minMatch float64, name string) []*Result[csl_uk.SanctionsListRecord] { - s.RLock() - defer s.RUnlock() - - s.Gate.Start() - defer s.Gate.Done() - - return topResults[csl_uk.SanctionsListRecord](limit, minMatch, name, s.UKSanctionsList) -} diff --git a/cmd/server/search_uk_csl_test.go b/cmd/server/search_uk_csl_test.go deleted file mode 100644 index 1cb91d94..00000000 --- a/cmd/server/search_uk_csl_test.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/pkg/csl_uk" - - "github.com/gorilla/mux" - "github.com/stretchr/testify/require" -) - -func TestSearch_UK_CSL(t *testing.T) { - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/search/uk-csl?name=%27ABD%20AL-NASIR", nil) - - router := mux.NewRouter() - addSearchRoutes(log.NewNopLogger(), router, uk_cslSearcher) - router.ServeHTTP(w, req) - w.Flush() - - require.Equal(t, http.StatusOK, w.Code) - require.Contains(t, w.Body.String(), `"match":1`) - require.Contains(t, w.Body.String(), `"matchedName":"'abd al nasir"`) - - var wrapper struct { - UKCSL []csl_uk.CSLRecord `json:"ukConsolidatedSanctionsList"` - } - err := json.NewDecoder(w.Body).Decode(&wrapper) - require.NoError(t, err) - - require.Greater(t, len(wrapper.UKCSL), 0) - - require.Equal(t, int(13720), wrapper.UKCSL[0].GroupID) -} - -func TestSearch_UK_SanctionsList(t *testing.T) { - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/search/uk-csl?name=HAJI%20KHAIRULLAH%20HAJI%20SATTAR%20MONEY%20EXCHANGE", nil) - - router := mux.NewRouter() - addSearchRoutes(log.NewNopLogger(), router, uk_sanctionsListSearcher) - router.ServeHTTP(w, req) - w.Flush() - - require.Equal(t, http.StatusOK, w.Code) - require.Contains(t, w.Body.String(), `"match":1`) - require.Contains(t, w.Body.String(), `"matchedName":"haji khairullah haji sattar money exchange"`) - - var wrapper struct { - UKSanctionsList []csl_uk.SanctionsListRecord `json:"ukSanctionsList"` - } - err := json.NewDecoder(w.Body).Decode(&wrapper) - require.NoError(t, err) - - require.Greater(t, len(wrapper.UKSanctionsList), 0) - - require.Equal(t, "AFG0001", wrapper.UKSanctionsList[0].UniqueID) -} diff --git a/cmd/server/search_us_csl.go b/cmd/server/search_us_csl.go deleted file mode 100644 index f6ef91fe..00000000 --- a/cmd/server/search_us_csl.go +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "net/http" - "reflect" - - moovhttp "github.com/moov-io/base/http" - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/internal/prepare" - "github.com/moov-io/watchman/pkg/csl_us" -) - -func searchUSCSL(logger log.Logger, searcher *searcher) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - w = wrapResponseWriter(logger, w, r) - requestID := moovhttp.GetRequestID(r) - - limit := extractSearchLimit(r) - filters := buildFilterRequest(r.URL) - minMatch := extractSearchMinMatch(r) - - name := r.URL.Query().Get("name") - resp := buildFullSearchResponseWith(searcher, cslGatherings, filters, limit, minMatch, name) - - logger.Info().With(log.Fields{ - "name": log.String(name), - "requestID": log.String(requestID), - }).Log("performing US CSL search") - - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(resp) - } -} - -func precomputeCSLEntities[T any](items []*T, pipe *prepare.Pipeliner) []*Result[T] { - out := make([]*Result[T], len(items)) - if items == nil { - return out - } - - for i, item := range items { - name := prepare.CSLName(item) - if err := pipe.Do(name); err != nil { - continue - } - - var altNames []string - - elm := reflect.ValueOf(item).Elem() - for i := 0; i < elm.NumField(); i++ { - name := elm.Type().Field(i).Name - _type := elm.Type().Field(i).Type.String() - - if name == "AlternateNames" && _type == "[]string" { - alts, ok := elm.Field(i).Interface().([]string) - if !ok { - continue - } - for j := range alts { - alt := &prepare.Name{Processed: alts[j]} - pipe.Do(alt) - altNames = append(altNames, alt.Processed) - } - } else if name == "NameAliasWholesNames" && _type == "[]string" { - alts, ok := elm.Field(i).Interface().([]string) - if !ok { - continue - } - for j := range alts { - alt := &prepare.Name{Processed: alts[j]} - pipe.Do(alt) - altNames = append(altNames, alt.Processed) - } - } else if name == "Names" && _type == "[]string" { - alts, ok := elm.Field(i).Interface().([]string) - if !ok { - continue - } - for j := range alts { - alt := &prepare.Name{Processed: alts[j]} - pipe.Do(alt) - altNames = append(altNames, alt.Processed) - } - } - } - - out[i] = &Result[T]{ - Data: *item, - precomputedName: name.Processed, - precomputedAlts: altNames, - } - } - - return out -} - -// TopBISEntities searches BIS Entity List records by name and alias -func (s *searcher) TopBISEntities(limit int, minMatch float64, name string) []*Result[csl_us.EL] { - s.RLock() - defer s.RUnlock() - - s.Gate.Start() // TODO(adam): This used to be on a pre-record gate, so this may have different perf metrics - defer s.Gate.Done() - - return topResults[csl_us.EL](limit, minMatch, name, s.BISEntities) -} - -// TopMEUs searches Military End User records by name and alias -func (s *searcher) TopMEUs(limit int, minMatch float64, name string) []*Result[csl_us.MEU] { - s.RLock() - defer s.RUnlock() - - s.Gate.Start() - defer s.Gate.Done() - - return topResults[csl_us.MEU](limit, minMatch, name, s.MilitaryEndUsers) -} - -// TopSSIs searches Sectoral Sanctions records by Name and Alias -func (s *searcher) TopSSIs(limit int, minMatch float64, name string) []*Result[csl_us.SSI] { - s.RLock() - defer s.RUnlock() - - s.Gate.Start() - defer s.Gate.Done() - - return topResults[csl_us.SSI](limit, minMatch, name, s.SSIs) -} - -// TopUVLs search Unverified Lists records by Name and Alias -func (s *searcher) TopUVLs(limit int, minMatch float64, name string) []*Result[csl_us.UVL] { - s.RLock() - defer s.RUnlock() - - s.Gate.Start() - defer s.Gate.Done() - - return topResults[csl_us.UVL](limit, minMatch, name, s.UVLs) -} - -// TopISNs searches Nonproliferation Sanctions records by Name and Alias -func (s *searcher) TopISNs(limit int, minMatch float64, name string) []*Result[csl_us.ISN] { - s.RLock() - defer s.RUnlock() - - s.Gate.Start() - defer s.Gate.Done() - - return topResults[csl_us.ISN](limit, minMatch, name, s.ISNs) -} - -// TopFSEs searches Foreign Sanctions Evaders records by Name and Alias -func (s *searcher) TopFSEs(limit int, minMatch float64, name string) []*Result[csl_us.FSE] { - s.RLock() - defer s.RUnlock() - - s.Gate.Start() - defer s.Gate.Done() - - return topResults[csl_us.FSE](limit, minMatch, name, s.FSEs) -} - -// TopPLCs searches Palestinian Legislative Council records by Name and Alias -func (s *searcher) TopPLCs(limit int, minMatch float64, name string) []*Result[csl_us.PLC] { - s.RLock() - defer s.RUnlock() - - s.Gate.Start() - defer s.Gate.Done() - - return topResults[csl_us.PLC](limit, minMatch, name, s.PLCs) -} - -// TopCAPs searches the CAPTA list by Name and Alias -func (s *searcher) TopCAPs(limit int, minMatch float64, name string) []*Result[csl_us.CAP] { - s.RLock() - defer s.RUnlock() - - s.Gate.Start() - defer s.Gate.Done() - - return topResults[csl_us.CAP](limit, minMatch, name, s.CAPs) -} - -// TopDTCs searches the ITAR Debarred list by Name and Alias -func (s *searcher) TopDTCs(limit int, minMatch float64, name string) []*Result[csl_us.DTC] { - s.RLock() - defer s.RUnlock() - - s.Gate.Start() - defer s.Gate.Done() - - return topResults[csl_us.DTC](limit, minMatch, name, s.DTCs) -} - -// TopCMICs searches the Non-SDN Chinese Military Industrial Complex list by Name and Alias -func (s *searcher) TopCMICs(limit int, minMatch float64, name string) []*Result[csl_us.CMIC] { - s.RLock() - defer s.RUnlock() - - s.Gate.Start() - defer s.Gate.Done() - - return topResults[csl_us.CMIC](limit, minMatch, name, s.CMICs) -} - -// TopNS_MBS searches the Non-SDN Menu Based Sanctions list by Name and Alias -func (s *searcher) TopNS_MBS(limit int, minMatch float64, name string) []*Result[csl_us.NS_MBS] { - s.RLock() - defer s.RUnlock() - - s.Gate.Start() - defer s.Gate.Done() - - return topResults[csl_us.NS_MBS](limit, minMatch, name, s.NS_MBSs) -} diff --git a/cmd/server/search_us_csl_test.go b/cmd/server/search_us_csl_test.go deleted file mode 100644 index a74d3e88..00000000 --- a/cmd/server/search_us_csl_test.go +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "fmt" - "math" - "net/http" - "net/http/httptest" - "strconv" - "testing" - - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/pkg/csl_us" - - "github.com/gorilla/mux" - "github.com/stretchr/testify/require" -) - -func TestSearch_US_CSL(t *testing.T) { - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/search/us-csl?name=Khan&limit=1", nil) - - router := mux.NewRouter() - addSearchRoutes(log.NewNopLogger(), router, isnSearcher) - router.ServeHTTP(w, req) - w.Flush() - - require.Equal(t, http.StatusOK, w.Code) - require.Contains(t, w.Body.String(), `"match":0.89`) - require.Contains(t, w.Body.String(), `"matchedName":"abdul qadeer khan"`) - - var wrapper struct { - NonProliferationSanctions []csl_us.ISN `json:"nonProliferationSanctions"` - } - err := json.NewDecoder(w.Body).Decode(&wrapper) - require.NoError(t, err) - - require.Len(t, wrapper.NonProliferationSanctions, 1) - prolif := wrapper.NonProliferationSanctions[0] - require.Equal(t, "2d2db09c686e4829d0ef1b0b04145eec3d42cd88", prolif.EntityID) -} - -func TestSearcher_TopBISEntities(t *testing.T) { - els := bisEntitySearcher.TopBISEntities(1, 0.00, "Khan") - if len(els) == 0 { - t.Fatal("empty ELs") - } - if els[0].Data.Name != "Mohammad Jan Khan Mangal" { - t.Errorf("%#v", els[0].Data) - } - - // Verify AlternateNames are passed through - require.Len(t, els[0].Data.AlternateNames, 1) - require.Equal(t, "Air I", els[0].Data.AlternateNames[0]) -} - -func TestSearcher_TopBISEntities_AltName(t *testing.T) { - els := bisEntitySearcher.TopBISEntities(1, 0.00, "Luqman Sehreci.") - if len(els) == 0 { - t.Fatal("empty ELs") - } - if els[0].Data.Name != "Luqman Yasin Yunus Shgragi" { - t.Errorf("%#v", els[0].Data) - } - if math.Abs(1.0-els[0].match) > 0.001 { - t.Errorf("Expected match=1.0 for alt names: %f - %#v", els[0].match, els[0].Data) - } -} - -func TestSearcher_TopMEUs(t *testing.T) { - meus := meuSearcher.TopMEUs(1, 0.00, "China Gas") - require.Len(t, meus, 1) - - require.Equal(t, "d54346ef81802673c1b1daeb2ca8bd5d13755abd", meus[0].Data.EntityID) - require.Equal(t, "0.88750", fmt.Sprintf("%.5f", meus[0].match)) -} - -func TestSearcher_TopSSIs(t *testing.T) { - ssis := ssiSearcher.TopSSIs(1, 0.00, "ROSOBORONEKSPORT") - if len(ssis) == 0 { - t.Fatal("empty SSIs") - } - if ssis[0].Data.EntityID != "18782" { - t.Errorf("%#v", ssis[0].Data) - } - - // Verify AlternateNames are passed through - require.Len(t, ssis[0].Data.AlternateNames, 6) - require.Equal(t, "RUSSIAN DEFENSE EXPORT ROSOBORONEXPORT", ssis[0].Data.AlternateNames[0]) -} - -func TestSearcher_TopSSIs_limit(t *testing.T) { - ssis := ssiSearcher.TopSSIs(2, 0.00, "SPECIALIZED DEPOSITORY") - if len(ssis) != 2 { - t.Fatalf("Expected 2 results, found %d", len(ssis)) - } - require.Equal(t, "18736", ssis[0].Data.EntityID) -} - -func TestSearcher_TopSSIs_reportAltNameWeight(t *testing.T) { - ssis := ssiSearcher.TopSSIs(1, 0.00, "KENKYUSHO") - if len(ssis) == 0 { - t.Fatal("empty SSIs") - } - if ssis[0].Data.EntityID != "18782" { - t.Errorf("%f - %#v", ssis[0].match, ssis[0].Data) - } - if math.Abs(1.0-ssis[0].match) > 0.001 { - t.Errorf("Expected match=1.0 for alt names: %f - %#v", ssis[0].match, ssis[0].Data) - } -} - -func TestSearcher_TopISNs(t *testing.T) { - isns := isnSearcher.TopISNs(1, 0.00, "Abdul Qadeer K") - require.Len(t, isns, 1) - - isn := isns[0] - require.Equal(t, "2d2db09c686e4829d0ef1b0b04145eec3d42cd88", isn.Data.EntityID) - require.Equal(t, "0.93", fmt.Sprintf("%.2f", isn.match)) -} - -func TestSearcher_TopUVLs(t *testing.T) { - uvls := uvlSearcher.TopUVLs(1, 0.00, "Atlas Sanatgaran") - require.Len(t, uvls, 1) - - uvl := uvls[0] - require.Equal(t, "f15fa805ff4ac5e09026f5e78011a1bb6b26dec2", uvl.Data.EntityID) - require.Equal(t, "1", strconv.Itoa(int(uvl.match))) -} - -func TestSearcher_TopFSEs(t *testing.T) { - fses := fseSearcher.TopFSEs(1, 0.00, "BEKTAS, Halis") - require.Len(t, fses, 1) - - fse := fses[0] - require.Equal(t, "17526", fse.Data.EntityID) - require.Equal(t, "1", strconv.Itoa(int(fse.match))) -} - -func TestSearcher_TopPLCs(t *testing.T) { - plcs := plcSearcher.TopPLCs(1, 0.00, "SALAMEH, Salem") - require.Len(t, plcs, 1) - - plc := plcs[0] - require.Equal(t, "9702", plc.Data.EntityID) - require.Equal(t, "1", strconv.Itoa(int(plc.match))) -} - -func TestSearcher_TopCAPs(t *testing.T) { - caps := capSearcher.TopCAPs(1, 0.00, "BM BANK PUBLIC JOINT STOCK COMPANY") - require.Len(t, caps, 1) - - cap := caps[0] - require.Equal(t, "20002", cap.Data.EntityID) - require.Equal(t, "1", strconv.Itoa(int(cap.match))) -} - -func TestSearcher_TopDTCs(t *testing.T) { - dtcs := dtcSearcher.TopDTCs(1, 0.00, "Yasmin Ahmed") - require.Len(t, dtcs, 1) - - dtc := dtcs[0] - require.Equal(t, "d44d88d0265d93927b9ff1c13bbbb7c7db64142c", dtc.Data.EntityID) - require.Equal(t, "1", strconv.Itoa(int(dtc.match))) -} - -func TestSearcher_TopCMICs(t *testing.T) { - cmics := cmicSearcher.TopCMICs(1, 0.00, "PROVEN HONOUR CAPITAL LIMITED") - require.Len(t, cmics, 1) - - cmic := cmics[0] - require.Equal(t, "32091", cmic.Data.EntityID) - require.Equal(t, "1", strconv.Itoa(int(cmic.match))) -} - -func TestSearcher_TopNSMBSs(t *testing.T) { - ns_mbss := ns_mbsSearcher.TopNS_MBS(1, 0.00, "GAZPROMBANK JOINT STOCK COMPANY") - require.Len(t, ns_mbss, 1) - - ns_mbs := ns_mbss[0] - require.Equal(t, "17016", ns_mbs.Data.EntityID) - require.Equal(t, "1", strconv.Itoa(int(ns_mbs.match))) -} diff --git a/cmd/server/values.go b/cmd/server/values.go deleted file mode 100644 index 48aff427..00000000 --- a/cmd/server/values.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "fmt" - "net/http" - "sort" - "strings" - - moovhttp "github.com/moov-io/base/http" - "github.com/moov-io/base/log" - - "github.com/gorilla/mux" -) - -// accumulator is a case-insensitive collector for string values. -// -// getValues() will return an orderd distinct array of accumulated strings -// where each string is the first seen instance. -type accumulator struct { - limit int - values map[string]string -} - -func newAccumulator(limit int) accumulator { - return accumulator{ - limit: limit, - values: make(map[string]string), - } -} - -func (acc accumulator) add(value string) { - if len(acc.values) >= acc.limit { - return - } - - norm := strings.ToLower(strings.TrimSpace(value)) - if norm == "" { - return - } - if _, exists := acc.values[norm]; !exists { - acc.values[norm] = value - } -} - -func (acc accumulator) getValues() []string { - out := make([]string, 0, len(acc.values)) - for _, v := range acc.values { - out = append(out, v) - } - sort.Strings(out) - return out -} - -func addValuesRoutes(logger log.Logger, r *mux.Router, searcher *searcher) { - r.Methods("GET").Path("/ui/values/{key}").HandlerFunc(getValues(logger, searcher)) -} - -func getKey(r *http.Request) string { - return strings.ToLower(mux.Vars(r)["key"]) -} - -func getValues(logger log.Logger, searcher *searcher) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - w = wrapResponseWriter(logger, w, r) - w.Header().Set("Content-Type", "application/json; charset=utf-8") - - acc := newAccumulator(extractSearchLimit(r)) - - key := getKey(r) - if strings.EqualFold(key, "sdnType") { - acc.add("entity") - } - - for i := range searcher.SDNs { - // If we add support for other filters (CallSign, Tonnage) - // then we should add those keys here. - switch key { - case "sdntype": - acc.add(searcher.SDNs[i].SDNType) - case "ofacprogram": - for j := range searcher.SDNs[i].Programs { - acc.add(searcher.SDNs[i].Programs[j]) - } - default: - moovhttp.Problem(w, fmt.Errorf("unknown key: %s", key)) - return - } - } - - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(acc.getValues()) - } -} diff --git a/cmd/server/values_test.go b/cmd/server/values_test.go deleted file mode 100644 index 1702f2e2..00000000 --- a/cmd/server/values_test.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - "github.com/moov-io/base/log" - - "github.com/gorilla/mux" -) - -func TestValues__accumulator(t *testing.T) { - acc := newAccumulator(2) - acc.add("v2") // out of alphanumeric order - acc.add("") // empty value, ignored - acc.add("v1") - acc.add("v1") // duplicate - - xs := acc.getValues() - if len(xs) != 2 { - t.Errorf("got values: %v", xs) - } - if xs[0] != "v1" || xs[1] != "v2" { - t.Errorf("values: %v", xs) - } - - // add another past the limit and expect it to be excluded - acc.add("v3") - if len(xs) != 2 { - t.Errorf("got values: %v", xs) - } -} - -func TestValues__getValues(t *testing.T) { - router := mux.NewRouter() - addValuesRoutes(log.NewNopLogger(), router, sdnSearcher) - - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/ui/values/sdnType", nil) - router.ServeHTTP(w, req) - w.Flush() - - if w.Code != http.StatusOK { - t.Errorf("bogus HTTP status: %d", w.Code) - } - - var values []string - if err := json.NewDecoder(w.Body).Decode(&values); err != nil { - t.Error(err) - } - if len(values) != 2 { - t.Errorf("values: %v", values) - } - for i := range values { - switch values[i] { - case "individual", "entity": - continue - default: - t.Errorf("values[%d]=%s", i, values[i]) - } - } -} - -func TestValues__getValuesLimit(t *testing.T) { - router := mux.NewRouter() - addValuesRoutes(log.NewNopLogger(), router, sdnSearcher) - - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/ui/values/ofacProgram?limit=1", nil) - router.ServeHTTP(w, req) - w.Flush() - - if w.Code != http.StatusOK { - t.Errorf("bogus HTTP status: %d", w.Code) - } - - var values []string - if err := json.NewDecoder(w.Body).Decode(&values); err != nil { - t.Error(err) - } - if len(values) != 1 { - t.Errorf("values: %v", values) - } -} - -func TestValues__getValuesErr(t *testing.T) { - router := mux.NewRouter() - addValuesRoutes(log.NewNopLogger(), router, sdnSearcher) - - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/ui/values/other", nil) - router.ServeHTTP(w, req) - w.Flush() - - if w.Code != http.StatusBadRequest { - t.Errorf("bogus HTTP status: %d: %s", w.Code, w.Body.String()) - } -} diff --git a/cmd/server/webhook.go b/cmd/server/webhook.go deleted file mode 100644 index 107b22e3..00000000 --- a/cmd/server/webhook.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "fmt" - "net/http" - "net/url" - "os" - "strconv" - "time" - - "github.com/moov-io/base/strx" - "go4.org/syncutil" -) - -var ( - // webhookGate is a goroutine-safe throttler designed to only allow N - // goroutines to run at any given time. - webhookGate *syncutil.Gate - - webhookHTTPClient = &http.Client{ - Timeout: 10 * time.Second, - Transport: &http.Transport{ - MaxIdleConns: 100, - MaxIdleConnsPerHost: 100, - MaxConnsPerHost: 100, - IdleConnTimeout: 1 * time.Minute, - }, - // never follow a redirect as it could lead to a DoS or us being redirected - // to an unexpected location. - CheckRedirect: func(_ *http.Request, _ []*http.Request) error { - return http.ErrUseLastResponse - }, - } -) - -func init() { - maxWorkers, err := strconv.ParseInt(strx.Or(os.Getenv("WEBHOOK_MAX_WORKERS"), "10"), 10, 32) - if err == nil { - webhookGate = syncutil.NewGate(int(maxWorkers)) - } -} - -// callWebhook will take `body` as JSON and make a POST request to the provided webhook url. -// Returned is the HTTP status code. -func callWebhook(body *bytes.Buffer, webhook string, authToken string) (int, error) { - webhook, err := validateWebhook(webhook) - if err != nil { - return 0, err - } - - // Setup HTTP request - req, err := http.NewRequest("POST", webhook, body) - if err != nil { - return 0, fmt.Errorf("unknown error webhook: %v", err) - } - if authToken != "" { - req.Header.Set("Authorization", authToken) - } - - // Guard HTTP calls in-flight - if webhookGate != nil { - webhookGate.Start() - defer webhookGate.Done() - } - - resp, err := webhookHTTPClient.Do(req) - if resp == nil || err != nil { - if resp == nil { - return 0, fmt.Errorf("unable to call webhook: %v", err) - } - return resp.StatusCode, fmt.Errorf("HTTP problem with webhook %v", err) - } - if resp.Body != nil { - resp.Body.Close() - } - if resp.StatusCode > 299 || resp.StatusCode < 200 { - return resp.StatusCode, fmt.Errorf("callWebhook: bogus status code: %d", resp.StatusCode) - } - return resp.StatusCode, nil -} - -// validateWebhook performs some basic checks against the incoming webhook and -// returns a normalized value. -// -// - Must be an HTTPS url -// - Must be a valid URL -func validateWebhook(raw string) (string, error) { - u, err := url.Parse(raw) - if err != nil { - return "", fmt.Errorf("%s is not a valid URL: %v", raw, err) - } - if u.Scheme != "https" { - return "", fmt.Errorf("%s is not an HTTPS url", u.String()) - } - return u.String(), nil -} diff --git a/cmd/server/webhook_test.go b/cmd/server/webhook_test.go deleted file mode 100644 index 1078be74..00000000 --- a/cmd/server/webhook_test.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2022 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "crypto/tls" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/moov-io/base/log" -) - -var ( - downloadWebhook = func(w http.ResponseWriter, r *http.Request) { - var stats DownloadStats - if err := json.NewDecoder(r.Body).Decode(&stats); err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(err.Error())) - } else { - if stats.SDNs != 101 { - w.WriteHeader(http.StatusBadRequest) - return - } - w.WriteHeader(http.StatusOK) - } - } - - redirect = func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Location", "https://example.com") - w.WriteHeader(http.StatusMovedPermanently) - w.Write([]byte("didn't redirect")) - } -) - -// TestWebhook_retry ensures the webhookHTTPClient never follows a redirect. -// This is done to prevent infinite (or costly) redirect cycles which can degrade performance. -func TestWebhook_retry(t *testing.T) { - if testing.Short() { - return - } - - server := httptest.NewServer(http.HandlerFunc(redirect)) - defer server.Close() - - // normal client, ensure redirect is followed - resp, err := server.Client().Get(server.URL) - if err != nil { - t.Fatal(err) - } - - // Ensure we landed on example.com - bs, _ := io.ReadAll(resp.Body) - resp.Body.Close() - if !bytes.Contains(bs, []byte("iana.org")) { - t.Errorf("resp.Body=%s", string(bs)) - } - - // Now ensure our webhookHTTPClient doesn't follow the redirect - resp, err = webhookHTTPClient.Get(server.URL) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - - bs, _ = io.ReadAll(resp.Body) - if !bytes.Contains(bs, []byte("didn't redirect")) { - t.Errorf("resp.Body=%s", string(bs)) - } -} - -func TestWebhook_validate(t *testing.T) { - out, err := validateWebhook("") - if err == nil { - t.Error("expected error") - } - if out != "" { - t.Errorf("got out=%q", out) - } - - // happy path - out, err = validateWebhook("https://ofac.example.com/callback") - if err != nil { - t.Error(err) - } - if out != "https://ofac.example.com/callback" { - t.Errorf("got out=%q", out) - } - - // HTTP endpoint - out, err = validateWebhook("http://bad.example.com/callback") - if err == nil { - t.Error("expected error, but got none") - } - if out != "" { - t.Errorf("out=%q", out) - } -} - -func TestWebhook_call(t *testing.T) { - if testing.Short() { - return - } - - server := httptest.NewTLSServer(http.HandlerFunc(downloadWebhook)) - defer server.Close() - - // override to add test TLS certificate - if tr, ok := webhookHTTPClient.Transport.(*http.Transport); ok { - if ctr, ok := server.Client().Transport.(*http.Transport); ok { - tr.TLSClientConfig = new(tls.Config) - tr.TLSClientConfig.RootCAs = ctr.TLSClientConfig.RootCAs - } else { - t.Errorf("unknown server.Client().Transport type: %T", server.Client().Transport) - } - } else { - t.Fatalf("%T %#v", webhookHTTPClient.Transport, webhookHTTPClient.Transport) - } - - stats := &DownloadStats{ - SDNs: 101, - RefreshedAt: time.Now().In(time.UTC), - } - - t.Setenv("DOWNLOAD_WEBHOOK_URL", server.URL) - t.Setenv("DOWNLOAD_WEBHOOK_AUTH_TOKEN", "authToken") - - logger := log.NewTestLogger() - err := callDownloadWebook(logger, stats) - if err != nil { - t.Fatal(err) - } -} - -func TestWebhook__CallErr(t *testing.T) { - var body bytes.Buffer - body.WriteString(`{"foo": "bar"}`) - - status, err := callWebhook(&body, "https://localhost/12345", "12345") - if err == nil { - t.Fatal(err) - } - if status != 0 { - t.Errorf("bogus HTTP status: %d", status) - } -} diff --git a/configs/config.default.yml b/configs/config.default.yml new file mode 100644 index 00000000..40fada1b --- /dev/null +++ b/configs/config.default.yml @@ -0,0 +1,9 @@ +Watchman: + Servers: + BindAddress: ":8084" + AdminAddress: "9094" + + Download: + RefreshInterval: "12h" + InitialDataDirectory: "" + DisabledLists: [] # us_ofac, eu_csl, etc... diff --git a/go.mod b/go.mod index 848cbb65..d6294f1f 100644 --- a/go.mod +++ b/go.mod @@ -3,20 +3,17 @@ module github.com/moov-io/watchman go 1.20 require ( - fyne.io/fyne v1.4.3 fyne.io/fyne/v2 v2.5.3 github.com/abadojack/whatlanggo v1.0.1 github.com/antchfx/htmlquery v1.3.3 github.com/antihax/optional v1.0.0 github.com/bbalet/stopwords v1.0.0 - github.com/go-kit/kit v0.13.0 github.com/gorilla/mux v1.8.1 github.com/jaswdr/faker v1.19.1 github.com/knieriem/odf v0.1.0 github.com/moov-io/base v0.48.2 github.com/openvenues/gopostal v0.0.0-20240426055609-4fe3a773f519 github.com/pariz/gountries v0.1.6 - github.com/prometheus/client_golang v1.17.0 github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.4.0 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 @@ -41,34 +38,52 @@ require ( github.com/fyne-io/image v0.0.0-20220602074514-4956b0afb3d2 // indirect github.com/go-gl/gl v0.0.0-20211210172815-726fda9656d6 // indirect github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a // indirect + github.com/go-kit/kit v0.13.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-text/render v0.2.0 // indirect github.com/go-text/typesetting v0.2.0 // indirect + github.com/gobuffalo/here v0.6.7 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/jeandeaual/go-locale v0.0.0-20240223122105-ce5225dcaa49 // indirect github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/markbates/pkger v0.17.1 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/nicksnyder/go-i18n/v2 v2.4.0 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.17.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/rickar/cal/v2 v2.1.13 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/rymdport/portal v0.3.0 // indirect + github.com/sagikazarmark/locafero v0.3.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.10.0 // indirect + github.com/spf13/cast v1.5.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.17.0 // indirect github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c // indirect github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/yuin/goldmark v1.7.1 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/image v0.18.0 // indirect golang.org/x/mobile v0.0.0-20231127183840-76ac6878050a // indirect golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 7e35fa66..f0b4d324 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,7 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= @@ -15,6 +16,7 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= @@ -36,9 +38,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -fyne.io/fyne v1.4.3 h1:356CnXCiYrrfaLGsB7qLK3c6ktzyh8WR05v/2RBu51I= -fyne.io/fyne v1.4.3/go.mod h1:8kiPBNSDmuplxs9WnKCkaWYqbcXFy0DeAzwa6PBO9Z8= fyne.io/fyne/v2 v2.5.3 h1:k6LjZx6EzRZhClsuzy6vucLZBstdH2USDGHSGWq8ly8= fyne.io/fyne/v2 v2.5.3/go.mod h1:0GOXKqyvNwk3DLmsFu9v0oYM0ZcD1ysGnlHCerKoAmo= fyne.io/systray v1.11.0 h1:D9HISlxSkx+jHSniMBR6fCFOUjk1x/OOOJLa9lJYAKg= @@ -47,11 +48,9 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Kodeworks/golang-image-ico v0.0.0-20141118225523-73f0f4cfade9/go.mod h1:7uhhqiBaR4CpN0k9rMjOtjpcfGd6DG2m04zQxKnWQ0I= github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4= github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc= -github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/antchfx/htmlquery v1.3.3 h1:x6tVzrRhVNfECDaVxnZi1mEGrQg3mjE/rxbH2Pe6dNE= github.com/antchfx/htmlquery v1.3.3/go.mod h1:WeU3N7/rL6mb6dCwtE30dURBnBieKDC/fR8t6X+cKjU= github.com/antchfx/xpath v1.3.2 h1:LNjzlsSjinu3bQpw9hWMY9ocB80oLOWuQqFvO6xt51U= @@ -95,6 +94,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fredbi/uri v1.1.0 h1:OqLpTXtyRg9ABReqvDGdJPqZUxs8cyBDOMXBbskCaB8= github.com/fredbi/uri v1.1.0/go.mod h1:aYTUoAXBOq7BLfVJ8GnKmfcuURosB1xyHDIfWeC/iW4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -106,15 +106,12 @@ github.com/fyne-io/glfw-js v0.0.0-20241126112943-313d8a0fe1d0 h1:/1YRWFv9bAWkoo3 github.com/fyne-io/glfw-js v0.0.0-20241126112943-313d8a0fe1d0/go.mod h1:gsGA2dotD4v0SR6PmPCYvS9JuOeMwAtmfvDE7mbYXMY= github.com/fyne-io/image v0.0.0-20220602074514-4956b0afb3d2 h1:hnLq+55b7Zh7/2IRzWCpiTcAvjv/P8ERF+N7+xXbZhk= github.com/fyne-io/image v0.0.0-20220602074514-4956b0afb3d2/go.mod h1:eO7W361vmlPOrykIg+Rsh1SZ3tQBaOsfzZhsIOb/Lm0= -github.com/fyne-io/mobile v0.1.2/go.mod h1:/kOrWrZB6sasLbEy2JIvr4arEzQTXBTZGb3Y96yWbHY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/gl v0.0.0-20190320180904-bf2b1f2f34d7/go.mod h1:482civXOzJJCPzJ4ZOX/pwvXBWSnzD4OKMdH4ClKGbk= github.com/go-gl/gl v0.0.0-20211210172815-726fda9656d6 h1:zDw5v7qm4yH7N8C8uWd+8Ii9rROdgWxQuGoJ9WDXxfk= github.com/go-gl/gl v0.0.0-20211210172815-726fda9656d6/go.mod h1:9YTyiznxEY1fVinfM7RvRcjRHbw2xLBJ3AAGIT0I4Nw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200625191551-73d3c3675aa3/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a h1:vxnBhFDDT+xzxf1jTJKMKZw3H0swfWk9RpWbBbDK5+0= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.13.0 h1:OoneCcHKHQ03LfBpoQCUfCluwd2Vt3ohz+kvbJneZAU= @@ -128,13 +125,13 @@ github.com/go-text/render v0.2.0/go.mod h1:CkiqfukRGKJA5vZZISkjSYrcdtgKQWRa2HIzv github.com/go-text/typesetting v0.2.0 h1:fbzsgbmk04KiWtE+c3ZD4W2nmCRzBqrqQOvYlwAOdho= github.com/go-text/typesetting v0.2.0/go.mod h1:2+owI/sxa73XA581LAzVuEBZ3WEEV2pXeDswCH/3i1I= github.com/go-text/typesetting-utils v0.0.0-20240317173224-1986cbe96c66 h1:GUrm65PQPlhFSKjLPGOZNPNxLCybjzjYBzjfoBGaDUY= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM= +github.com/gobuffalo/here v0.6.7 h1:hpfhh+kt2y9JLDfhYUxxCRxQol540jsVfKUZzjlbp8o= +github.com/gobuffalo/here v0.6.7/go.mod h1:vuCfanjqckTuRlqAitJz6QC4ABNnS27wLb816UhsPcc= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/goki/freetype v0.0.0-20181231101311-fa8a33aabaff h1:W71vTCKoxtdXgnm1ECDFkfQnpdqAO00zzGXLA5yaEX8= -github.com/goki/freetype v0.0.0-20181231101311-fa8a33aabaff/go.mod h1:wfqRWLHRBsRgkp5dmbG56SA0DmVtwrF5N3oPdI8t+Aw= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -196,6 +193,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y= @@ -204,6 +202,7 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20211219123610-ec9572f70e60/go.mod h1:cz9oNYuRUWGdHmLF2IodMLkAhcPtXeULvcBNagUrxTI= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= @@ -227,6 +226,7 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -235,12 +235,10 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jackmordaunt/icns v0.0.0-20181231085925-4f16af745526/go.mod h1:UQkeMHVoNcyXYq9otUupF7/h/2tmHlhrS2zw7ZVvUqc= github.com/jaswdr/faker v1.19.1 h1:xBoz8/O6r0QAR8eEvKJZMdofxiRH+F0M/7MU9eNKhsM= github.com/jaswdr/faker v1.19.1/go.mod h1:x7ZlyB1AZqwqKZgyQlnqEG8FDptmHlncA5u2zY/yi6w= github.com/jeandeaual/go-locale v0.0.0-20240223122105-ce5225dcaa49 h1:Po+wkNdMmN+Zj1tDsJQy7mJlPlwGNQd9JZoPjObagf8= github.com/jeandeaual/go-locale v0.0.0-20240223122105-ce5225dcaa49/go.mod h1:YiutDnxPRLk5DLUFj6Rw4pRBBURZY07GFr54NdV9mQg= -github.com/josephspurrier/goversioninfo v0.0.0-20200309025242-14b0ab84c6ca/go.mod h1:eJTEwMjXb7kZ633hO3Ln9mBUCOjX2+FlTljvpl9SYdE= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -255,10 +253,13 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lucor/goinfo v0.0.0-20200401173949-526b5363a13a/go.mod h1:ORP3/rB5IsulLEBwQZCJyyV6niqmI7P4EWSmkug+1Ng= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/markbates/pkger v0.17.1 h1:/MKEtWqtc0mZvu9OinB9UzVN9iYCwLWuyUv4Bw+PCno= +github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= @@ -272,6 +273,8 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -279,21 +282,21 @@ github.com/moov-io/base v0.48.2 h1:BPSNgmwokOVaVzAMJg71L48LCrDYelMfVXJEiZb2zOY= github.com/moov-io/base v0.48.2/go.mod h1:u1/WC3quR6otC9NrM1TtXSwNti1A/m7MR49RIXY1ee4= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= -github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/nicksnyder/go-i18n/v2 v2.4.0 h1:3IcvPOAvnCKwNm0TB0dLDTuawWEj+ax/RERNC+diLMM= github.com/nicksnyder/go-i18n/v2 v2.4.0/go.mod h1:nxYSZE9M0bf3Y70gPQjN9ha7XNHX7gMc814+6wVyEI4= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/openvenues/gopostal v0.0.0-20240426055609-4fe3a773f519 h1:xZ0ZhxCnrs2zaBBvGIHQqzoeXjzctJP61r+aX3QjXhQ= github.com/openvenues/gopostal v0.0.0-20240426055609-4fe3a773f519/go.mod h1:Ycrd7XnwQdumHzpB/6WEa85B4WNdbLC6Wz4FAQNkaV0= github.com/pariz/gountries v0.1.6 h1:Cu8sBSvD6HvAtzinKJ7Yw8q4wAF2dD7oXjA5yDJQt1I= github.com/pariz/gountries v0.1.6/go.mod h1:Et5QWMc75++5nUKSYKNtz/uc+2LHl4LKhNd6zwdTu+0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -308,7 +311,6 @@ github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGy github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rickar/cal/v2 v2.1.13 h1:FENBPXxDPyL1OWGf9ZdpWGcEiGoSjt0UZED8VOxvK0c= -github.com/rickar/cal/v2 v2.1.13/go.mod h1:/fdlMcx7GjPlIBibMzOM9gMvDBsrK+mOtRXdTzUqV/A= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -319,6 +321,10 @@ github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/rymdport/portal v0.3.0 h1:QRHcwKwx3kY5JTQcsVhmhC3TGqGQb9LFghVNUy8AdB8= github.com/rymdport/portal v0.3.0/go.mod h1:kFF4jslnJ8pD5uCi17brj/ODlfIidOxlgUDTO5ncnC4= +github.com/sagikazarmark/locafero v0.3.0 h1:zT7VEGWC2DTflmccN/5T1etyKvxSxpHsjb9cJvm4SvQ= +github.com/sagikazarmark/locafero v0.3.0/go.mod h1:w+v7UsPNFwzF1cHuOajOOzoq4U7v/ig1mpRjqV+Bu1U= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= @@ -326,21 +332,28 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= +github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/srwiley/oksvg v0.0.0-20200311192757-870daf9aa564/go.mod h1:afMbS0qvv1m5tfENCwnOdZGOF8RGR/FsZ7bvBxQGZG4= +github.com/spf13/viper v1.17.0 h1:I5txKw7MJasPL/BrfkbA0Jyo/oELqVmux4pR/UxOMfI= +github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0+yVI= github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c h1:km8GpoQut05eY3GiYWEedbTT0qnSxrCjsVbb7yKY1KE= github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c/go.mod h1:cNQ3dwVJtS5Hmnjxy6AgTPd0Inb3pW05ftPSX7NZO7Q= -github.com/srwiley/rasterx v0.0.0-20200120212402-85cb7272f5e9/go.mod h1:mvWM0+15UqyrFKqdRjY6LuAVJR0HOVhJlEgZ5JWtSWU= github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef h1:Ch6Q+AZUxDBCVqdkI8FSpFyZDtCVBc2VmejdNrm5rRQ= github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef/go.mod h1:nXTWP6+gD5+LUJ8krVhhoeHjvHTutPxMYl5SvkcnJNE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -348,9 +361,12 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/urfave/cli/v2 v2.4.0 h1:m2pxjjDFgDxSPtO8WSdbndj17Wu2y8vOT86wE/tjr+I= github.com/urfave/cli/v2 v2.4.0/go.mod h1:NX9W0zmTvedE5oDoOMs2RTC8RvdK98NTYZE5LbaEYPg= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= @@ -376,6 +392,8 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= @@ -386,8 +404,10 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -399,9 +419,10 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -463,11 +484,13 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= @@ -527,7 +550,6 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -536,12 +558,14 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -583,7 +607,6 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190808195139-e713427fea3f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -605,7 +628,6 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200328031815-3db5fc6bac03/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -621,6 +643,7 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -696,7 +719,9 @@ google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -738,12 +763,14 @@ google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGm google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/internal/download/download.go b/internal/download/download.go new file mode 100644 index 00000000..b2ca88e4 --- /dev/null +++ b/internal/download/download.go @@ -0,0 +1,200 @@ +package download + +import ( + "context" + "fmt" + "time" + + "github.com/moov-io/watchman/pkg/ofac" + "github.com/moov-io/watchman/pkg/search" + + "github.com/moov-io/base/log" + "golang.org/x/sync/errgroup" +) + +type Downloader interface { + RefreshAll(ctx context.Context) (Stats, error) +} + +func NewDownloader(logger log.Logger, conf Config) (Downloader, error) { + return &downloader{ + logger: logger, + conf: conf, + }, nil +} + +type downloader struct { + logger log.Logger + conf Config +} + +func (dl *downloader) RefreshAll(ctx context.Context) (Stats, error) { + stats := Stats{ + Lists: make(map[string]int), + StartedAt: time.Now().In(time.UTC), + } + + logger := dl.logger.Info().With(log.Fields{ + "initial_data_directory": log.String(dl.conf.InitialDataDirectory), + }) + logger.Info().Log("starting list refresh") + + g, ctx := errgroup.WithContext(ctx) + preparedLists := make(chan preparedList) + + g.Go(func() error { + err := loadOFACRecords(ctx, logger, dl.conf, preparedLists) + if err != nil { + return fmt.Errorf("loading OFAC records: %w", err) + } + return nil + }) + + err := g.Wait() + if err != nil { + return stats, fmt.Errorf("problem loading lists: %v", err) + } + + // accumulate the lists + for list := range preparedLists { + stats.Lists[string(list.ListName)] = len(list.Entities) + stats.Entities = append(stats.Entities, list.Entities...) + } + + stats.EndedAt = time.Now().In(time.UTC) + + return stats, nil +} + +type preparedList struct { + ListName search.SourceList + Entities []search.Entity[search.Value] +} + +func loadOFACRecords(ctx context.Context, logger log.Logger, conf Config, responseCh chan preparedList) error { + files, err := ofac.Download(ctx, logger, conf.InitialDataDirectory) + if err != nil { + return fmt.Errorf("download: %v", err) + } + if len(files) == 0 { + return fmt.Errorf("unexpected %d OFAC files found", len(files)) + } + + res, err := ofac.Read(files) + if err != nil { + return err + } + + entities := ofac.GroupIntoEntities(res.SDNs, res.Addresses, res.SDNComments, res.AlternateIdentities) + + responseCh <- preparedList{ + ListName: search.SourceUSOFAC, + Entities: entities, + } + + return nil +} + +// "github.com/moov-io/watchman/pkg/csl_eu" +// "github.com/moov-io/watchman/pkg/csl_uk" +// "github.com/moov-io/watchman/pkg/csl_us" + +// func cslUSRecords(logger log.Logger, initialDir string) (csl_us.CSL, error) { +// file, err := csl_us.Download(logger, initialDir) +// if err != nil { +// logger.Warn().Logf("skipping CSL US download: %v", err) +// return csl_us.CSL{}, nil +// } +// cslRecords, err := csl_us.ReadFile(file["csl.csv"]) +// if err != nil { +// return csl_us.CSL{}, fmt.Errorf("reading CSL US: %w", err) +// } +// return cslRecords, nil +// } + +// func euCSLRecords(logger log.Logger, initialDir string) ([]csl_eu.CSLRecord, error) { +// file, err := csl_eu.DownloadEU(logger, initialDir) +// if err != nil { +// logger.Warn().Logf("skipping EU CSL download: %v", err) +// // no error to return because we skip the download +// return nil, nil +// } + +// cslRecords, _, err := csl_eu.ParseEU(file["eu_csl.csv"]) +// if err != nil { +// return nil, err +// } +// return cslRecords, err + +// } + +// func ukCSLRecords(logger log.Logger, initialDir string) ([]csl_uk.CSLRecord, error) { +// file, err := csl_uk.DownloadCSL(logger, initialDir) +// if err != nil { +// logger.Warn().Logf("skipping UK CSL download: %v", err) +// // no error to return because we skip the download +// return nil, nil +// } +// cslRecords, _, err := csl_uk.ReadCSLFile(file["ConList.csv"]) +// if err != nil { +// return nil, err +// } +// return cslRecords, err +// } + +// func ukSanctionsListRecords(logger log.Logger, initialDir string) ([]csl_uk.SanctionsListRecord, error) { +// file, err := csl_uk.DownloadSanctionsList(logger, initialDir) +// if file == nil || err != nil { +// logger.Warn().Logf("skipping UK Sanctions List download: %v", err) +// // no error to return because we skip the download +// return nil, nil +// } + +// records, _, err := csl_uk.ReadSanctionsListFile(file["UK_Sanctions_List.ods"]) +// if err != nil { +// return nil, err +// } +// return records, err +// } + +// var euCSLs []Result[csl_eu.CSLRecord] +// withEUScreeningList := cmp.Or(os.Getenv("WITH_EU_SCREENING_LIST"), "true") +// if strx.Yes(withEUScreeningList) { +// euConsolidatedList, err := euCSLRecords(s.logger, initialDir) +// if err != nil { +// stats.Errors = append(stats.Errors, fmt.Errorf("EUCSL: %v", err)) +// } +// euCSLs = precomputeCSLEntities[csl_eu.CSLRecord](euConsolidatedList, s.pipe) +// } + +// var ukCSLs []Result[csl_uk.CSLRecord] +// withUKCSLSanctionsList := cmp.Or(os.Getenv("WITH_UK_CSL_SANCTIONS_LIST"), "true") +// if strx.Yes(withUKCSLSanctionsList) { +// ukConsolidatedList, err := ukCSLRecords(s.logger, initialDir) +// if err != nil { +// stats.Errors = append(stats.Errors, fmt.Errorf("UKCSL: %v", err)) +// } +// ukCSLs = precomputeCSLEntities[csl_uk.CSLRecord](ukConsolidatedList, s.pipe) +// } + +// var ukSLs []Result[csl_uk.SanctionsListRecord] +// withUKSanctionsList := os.Getenv("WITH_UK_SANCTIONS_LIST") +// if strings.ToLower(withUKSanctionsList) == "true" { +// ukSanctionsList, err := ukSanctionsListRecords(s.logger, initialDir) +// if err != nil { +// stats.Errors = append(stats.Errors, fmt.Errorf("UKSanctionsList: %v", err)) +// } +// ukSLs = precomputeCSLEntities[csl_uk.SanctionsListRecord](ukSanctionsList, s.pipe) + +// stats.UKSanctionsList = len(ukSLs) +// } + +// // csl records from US downloaded here +// var usConsolidatedLists csl_us.CSL +// withUSConsolidatedLists := cmp.Or(os.Getenv("WITH_US_CSL_SANCTIONS_LIST"), "true") +// if strx.Yes(withUSConsolidatedLists) { +// usConsolidatedLists, err = cslUSRecords(s.logger, initialDir) +// if err != nil { +// stats.Errors = append(stats.Errors, fmt.Errorf("US CSL: %v", err)) +// } +// } diff --git a/internal/download/models.go b/internal/download/models.go new file mode 100644 index 00000000..eb5cbe29 --- /dev/null +++ b/internal/download/models.go @@ -0,0 +1,23 @@ +package download + +import ( + "time" + + "github.com/moov-io/watchman/pkg/search" +) + +type Stats struct { + Entities []search.Entity[search.Value] `json:"-"` + + Lists map[string]int `json:"lists"` + + StartedAt time.Time `json:"startedAt"` + EndedAt time.Time `json:"endedAt"` +} + +type Config struct { + RefreshInterval time.Duration + InitialDataDirectory string + + DisabledLists []string // us_ofac, eu_csl, etc... +} diff --git a/internal/prepare/pipeline.go b/internal/prepare/pipeline.go deleted file mode 100644 index f85c25e7..00000000 --- a/internal/prepare/pipeline.go +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package prepare - -import ( - "errors" - - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/pkg/csl_eu" - "github.com/moov-io/watchman/pkg/csl_uk" - "github.com/moov-io/watchman/pkg/csl_us" - "github.com/moov-io/watchman/pkg/dpl" - "github.com/moov-io/watchman/pkg/ofac" -) - -// Name represents an individual or entity name to be processed for search. -type Name struct { - // Original is the initial value and MUST not be changed by any pipeline step. - Original string - - // Processed is the mutable value that each pipeline step can optionally - // replace and is read as the input to each step. - Processed string - - // optional metadata of where a name came from - alt *ofac.AlternateIdentity - sdn *ofac.SDN - ssi *csl_us.SSI - uvl *csl_us.UVL - isn *csl_us.ISN - fse *csl_us.FSE - plc *csl_us.PLC - cap *csl_us.CAP - dtc *csl_us.DTC - cmic *csl_us.CMIC - ns_mbs *csl_us.NS_MBS - - eu_csl *csl_eu.CSLRecord - - uk_csl *csl_uk.CSLRecord - - uk_sanctionsList *csl_uk.SanctionsListRecord - - dp *dpl.DPL - el *csl_us.EL - addrs []*ofac.Address - - altNames []string -} - -func SdnName(sdn *ofac.SDN, addrs []*ofac.Address) *Name { - return &Name{ - Original: sdn.SDNName, - Processed: sdn.SDNName, - sdn: sdn, - addrs: addrs, - } -} - -func AltName(alt *ofac.AlternateIdentity) *Name { - return &Name{ - Original: alt.AlternateName, - Processed: alt.AlternateName, - alt: alt, - } -} - -func DPName(dp *dpl.DPL) *Name { - return &Name{ - Original: dp.Name, - Processed: dp.Name, - dp: dp, - } -} - -func CSLName(item interface{}) *Name { - switch v := item.(type) { - case *csl_us.EL: - return &Name{ - Original: v.Name, - Processed: v.Name, - el: v, - } - case *csl_us.MEU: - return &Name{ - Original: v.Name, - Processed: v.Name, - } - case *csl_us.SSI: - return &Name{ - Original: v.Name, - Processed: v.Name, - ssi: v, - altNames: v.AlternateNames, - } - case *csl_us.UVL: - return &Name{ - Original: v.Name, - Processed: v.Name, - uvl: v, - } - case *csl_us.ISN: - return &Name{ - Original: v.Name, - Processed: v.Name, - isn: v, - altNames: v.AlternateNames, - } - case *csl_us.FSE: - return &Name{ - Original: v.Name, - Processed: v.Name, - fse: v, - } - case *csl_us.PLC: - return &Name{ - Original: v.Name, - Processed: v.Name, - plc: v, - altNames: v.AlternateNames, - } - case *csl_us.CAP: - return &Name{ - Original: v.Name, - Processed: v.Name, - cap: v, - altNames: v.AlternateNames, - } - case *csl_us.DTC: - return &Name{ - Original: v.Name, - Processed: v.Name, - dtc: v, - altNames: v.AlternateNames, - } - case *csl_us.CMIC: - return &Name{ - Original: v.Name, - Processed: v.Name, - cmic: v, - altNames: v.AlternateNames, - } - case *csl_us.NS_MBS: - return &Name{ - Original: v.Name, - Processed: v.Name, - ns_mbs: v, - altNames: v.AlternateNames, - } - case *csl_eu.CSLRecord: - if len(v.NameAliasWholeNames) >= 1 { - var alts []string - alts = append(alts, v.NameAliasWholeNames...) - return &Name{ - Original: v.NameAliasWholeNames[0], - Processed: v.NameAliasWholeNames[0], - eu_csl: v, - altNames: alts, - } - } - case *csl_uk.CSLRecord: - if len(v.Names) >= 1 { - var alts []string - alts = append(alts, v.Names...) - return &Name{ - Original: v.Names[0], - Processed: v.Names[0], - uk_csl: v, - altNames: alts, - } - } - case *csl_uk.SanctionsListRecord: - if len(v.Names) >= 1 { - var alts []string - alts = append(alts, v.Names...) - return &Name{ - Original: v.Names[0], - Processed: v.Names[0], - uk_sanctionsList: v, - altNames: alts, - } - } - - return &Name{} - } - return &Name{} -} - -type step interface { - apply(*Name) error -} - -type debugStep struct { - step - - logger log.Logger -} - -func (ds *debugStep) apply(in *Name) error { - if err := ds.step.apply(in); err != nil { - return ds.logger.Info().With(log.Fields{ - "original": log.String(in.Original), - }).LogErrorf("%T encountered error: %v", ds.step, err).Err() - } - ds.logger.Info().With(log.Fields{ - "original": log.String(in.Original), - "result": log.String(in.Processed), - }).Logf("%T", ds.step) - return nil -} - -func NewPipeliner(logger log.Logger, debug bool) *Pipeliner { - steps := []step{ - &reorderSDNStep{}, - &companyNameCleanupStep{}, - &stopwordsStep{}, - &normalizeStep{}, - } - if debug { - for i := range steps { - steps[i] = &debugStep{logger: logger, step: steps[i]} - } - } - return &Pipeliner{ - logger: logger, - steps: steps, - } -} - -type Pipeliner struct { - logger log.Logger - steps []step -} - -func (p *Pipeliner) Do(name *Name) error { - if p == nil || p.steps == nil || p.logger == nil || name == nil { - return errors.New("nil Pipeliner or Name") - } - for i := range p.steps { - if name == nil { - return p.logger.Error().LogErrorf("%T: nil Name", p.steps[i]).Err() - } - if err := p.steps[i].apply(name); err != nil { - return p.logger.Error().LogErrorf("pipeline: %v", err).Err() - } - } - return nil -} diff --git a/internal/prepare/pipeline_company_name_cleanup.go b/internal/prepare/pipeline_company_name_cleanup.go index 716ff812..0361ae35 100644 --- a/internal/prepare/pipeline_company_name_cleanup.go +++ b/internal/prepare/pipeline_company_name_cleanup.go @@ -8,20 +8,6 @@ import ( "strings" ) -type companyNameCleanupStep struct { -} - -func (s *companyNameCleanupStep) apply(in *Name) error { - switch { - case in.sdn != nil && in.sdn.SDNType == "": - in.Processed = removeCompanyTitles(in.Processed) - - case in.ssi != nil && in.ssi.Type == "": - in.Processed = removeCompanyTitles(in.Processed) - } - return nil -} - // original list: inc, incorporated, llc, llp, co, ltd, limited, sa de cv, corporation, corp, ltda, // open joint stock company, pty ltd, public limited company, ag, cjsc, plc, as, aps, // oy, sa, gmbh, se, pvt ltd, sp zoo, ooo, sl, pjsc, jsc, bv, pt, tbk @@ -43,6 +29,6 @@ var ( ) ) -func removeCompanyTitles(in string) string { +func RemoveCompanyTitles(in string) string { return companySuffixReplacer.Replace(in) } diff --git a/internal/prepare/pipeline_company_name_cleanup_test.go b/internal/prepare/pipeline_company_name_cleanup_test.go index 374a5bfd..e2fb4381 100644 --- a/internal/prepare/pipeline_company_name_cleanup_test.go +++ b/internal/prepare/pipeline_company_name_cleanup_test.go @@ -7,25 +7,12 @@ package prepare import ( "testing" - "github.com/moov-io/watchman/pkg/ofac" + "github.com/stretchr/testify/require" ) func TestPipeline__companyNameCleanupStep(t *testing.T) { - nn := &Name{ - Processed: "SAI ADVISORS INC.", - sdn: &ofac.SDN{ - SDNType: "", - }, - } - - step := &companyNameCleanupStep{} - if err := step.apply(nn); err != nil { - t.Fatal(err) - } - - if nn.Processed != "SAI ADVISORS" { - t.Errorf("nn.Processed=%s", nn.Processed) - } + out := RemoveCompanyTitles("SAI ADVISORS INC.") + require.Equal(t, "SAI ADVISORS", out) } func TestRemoveCompanyTitles(t *testing.T) { @@ -61,8 +48,7 @@ func TestRemoveCompanyTitles(t *testing.T) { {"PETRO ROYAL FZE", "PETRO ROYAL FZE"}, // SDN 16136 } for i := range cases { - if ans := removeCompanyTitles(cases[i].input); cases[i].expected != ans { - t.Errorf("#%d input=%q expected=%q got=%q", i, cases[i].input, cases[i].expected, ans) - } + got := RemoveCompanyTitles(cases[i].input) + require.Equal(t, cases[i].expected, got) } } diff --git a/internal/prepare/pipeline_normalize.go b/internal/prepare/pipeline_normalize.go index 9a2dbf7a..72a21717 100644 --- a/internal/prepare/pipeline_normalize.go +++ b/internal/prepare/pipeline_normalize.go @@ -18,13 +18,6 @@ var ( punctuationReplacer = strings.NewReplacer(".", "", ",", "", "-", " ", " ", " ") ) -type normalizeStep struct{} - -func (s *normalizeStep) apply(in *Name) error { - in.Processed = LowerAndRemovePunctuation(in.Processed) - return nil -} - // LowerAndRemovePunctuation will lowercase each substring and remove punctuation // // This function is called on every record from the flat files and all diff --git a/internal/prepare/pipeline_normalize_test.go b/internal/prepare/pipeline_normalize_test.go index 677cbc7b..e3dbb054 100644 --- a/internal/prepare/pipeline_normalize_test.go +++ b/internal/prepare/pipeline_normalize_test.go @@ -6,19 +6,13 @@ package prepare import ( "testing" + + "github.com/stretchr/testify/require" ) func TestPipeline__normalizeStep(t *testing.T) { - nn := &Name{Processed: "Nicolás Maduro"} - - step := &normalizeStep{} - if err := step.apply(nn); err != nil { - t.Fatal(err) - } - - if nn.Processed != "nicolas maduro" { - t.Errorf("nn.Processed=%v", nn.Processed) - } + got := LowerAndRemovePunctuation("Nicolás Maduro") + require.Equal(t, "nicolas maduro", got) } // TestLowerAndRemovePunctuation ensures we are trimming and UTF-8 normalizing strings @@ -35,10 +29,8 @@ func TestLowerAndRemovePunctuation(t *testing.T) { {"issue 483 #1", "11420 CORP.", "11420 corp"}, {"issue 483 #2", "11,420.2-1 CORP.", "114202 1 corp"}, } - for i, tc := range tests { + for _, tc := range tests { guess := LowerAndRemovePunctuation(tc.input) - if guess != tc.expected { - t.Errorf("case: %d name: %s LowerAndRemovePunctuation(%q)=%q expected %q", i, tc.name, tc.input, guess, tc.expected) - } + require.Equal(t, tc.expected, guess) } } diff --git a/internal/prepare/pipeline_reorder.go b/internal/prepare/pipeline_reorder.go index 88737710..6d5d106e 100644 --- a/internal/prepare/pipeline_reorder.go +++ b/internal/prepare/pipeline_reorder.go @@ -10,32 +10,18 @@ import ( "strings" ) -type reorderSDNStep struct { -} - -func (s *reorderSDNStep) apply(in *Name) error { - switch { - case in.sdn != nil: - in.Processed = reorderSDNName(in.Processed, in.sdn.SDNType) - - case in.ssi != nil: - in.Processed = reorderSDNName(in.Processed, in.ssi.Type) - } - return nil -} - var ( surnamePrecedes = regexp.MustCompile(`(,?[\s?a-zA-Z\.]{1,})$`) ) -// reorderSDNName will take a given SDN name and if it matches a specific pattern where +// ReorderSDNName will take a given SDN name and if it matches a specific pattern where // the first name is placed after the last name (surname) to return a string where the first name // preceedes the last. // // Example: // SDN EntityID: 19147 has 'FELIX B. MADURO S.A.' // SDN EntityID: 22790 has 'MADURO MOROS, Nicolas' -func reorderSDNName(name string, tpe string) string { +func ReorderSDNName(name string, tpe string) string { if !strings.EqualFold(tpe, "individual") { return name // only reorder individual names } diff --git a/internal/prepare/pipeline_reorder_test.go b/internal/prepare/pipeline_reorder_test.go index 59567449..00898715 100644 --- a/internal/prepare/pipeline_reorder_test.go +++ b/internal/prepare/pipeline_reorder_test.go @@ -7,25 +7,12 @@ package prepare import ( "testing" - "github.com/moov-io/watchman/pkg/ofac" + "github.com/stretchr/testify/require" ) func TestPipeline__reorderSDNStep(t *testing.T) { - nn := &Name{ - Processed: "Last, First Middle", - sdn: &ofac.SDN{ - SDNType: "individual", - }, - } - - step := &reorderSDNStep{} - if err := step.apply(nn); err != nil { - t.Fatal(err) - } - - if nn.Processed != "First Middle Last" { - t.Errorf("nn.Processed=%v", nn.Processed) - } + got := ReorderSDNName("Last, First Middle", "individual") + require.Equal(t, "First Middle Last", got) } func TestReorderSDNName(t *testing.T) { @@ -44,10 +31,8 @@ func TestReorderSDNName(t *testing.T) { {"RIZO MORENO, Jorge Luis", "Jorge Luis RIZO MORENO"}, } for i := range cases { - guess := reorderSDNName(cases[i].input, "individual") - if guess != cases[i].expected { - t.Errorf("reorderSDNName(%q)=%q expected %q", cases[i].input, guess, cases[i].expected) - } + got := ReorderSDNName(cases[i].input, "individual") + require.Equal(t, cases[i].expected, got) } // Entities @@ -59,9 +44,7 @@ func TestReorderSDNName(t *testing.T) { {"11,420.2-1 CORP.", "11,420.2-1 CORP."}, } for i := range cases { - guess := reorderSDNName(cases[i].input, "") // blank refers to a company - if guess != cases[i].expected { - t.Errorf("reorderSDNName(%q)=%q expected %q", cases[i].input, guess, cases[i].expected) - } + got := ReorderSDNName(cases[i].input, "") // blank refers to a company + require.Equal(t, cases[i].expected, got) } } diff --git a/internal/prepare/pipeline_stopwords.go b/internal/prepare/pipeline_stopwords.go index 29febd2c..e9087e1e 100644 --- a/internal/prepare/pipeline_stopwords.go +++ b/internal/prepare/pipeline_stopwords.go @@ -31,28 +31,25 @@ var ( }(os.Getenv("KEEP_STOPWORDS")) ) -type stopwordsStep struct{} - -func (s *stopwordsStep) apply(in *Name) error { - if in == nil { - return nil - } - - switch { - case in.sdn != nil && !strings.EqualFold(in.sdn.SDNType, "individual"): - in.Processed = removeStopwords(in.Processed, detectLanguage(in.Processed, in.addrs)) - case in.ssi != nil && !strings.EqualFold(in.ssi.Type, "individual"): - in.Processed = removeStopwords(in.Processed, detectLanguage(in.Processed, nil)) - case in.alt != nil: - in.Processed = removeStopwords(in.Processed, detectLanguage(in.Processed, nil)) - } - return nil -} +// switch { +// case in.sdn != nil && !strings.EqualFold(in.sdn.SDNType, "individual"): +// in.Processed = removeStopwords(in.Processed, detectLanguage(in.Processed, in.addrs)) +// case in.ssi != nil && !strings.EqualFold(in.ssi.Type, "individual"): +// in.Processed = removeStopwords(in.Processed, detectLanguage(in.Processed, nil)) +// case in.alt != nil: +// in.Processed = removeStopwords(in.Processed, detectLanguage(in.Processed, nil)) +// } var ( numberRegex = regexp.MustCompile(`([\d\.\,\-]{1,}[\d]{1,})`) ) +func RemoveStopwords(in string, addrs []ofac.Address) string { + lang := detectLanguage(in, addrs) + + return removeStopwords(in, lang) +} + func removeStopwords(in string, lang whatlanggo.Lang) string { if keepStopwords { return in @@ -76,7 +73,7 @@ func removeStopwords(in string, lang whatlanggo.Lang) string { // detectLanguage will return a guess as to the appropriate language a given SDN's name // is written in. The addresses must be linked to the SDN whose name is detected. -func detectLanguage(in string, addrs []*ofac.Address) whatlanggo.Lang { +func detectLanguage(in string, addrs []ofac.Address) whatlanggo.Lang { info := whatlanggo.Detect(in) if info.IsReliable() { // Return the detected language if whatlanggo is confident enough diff --git a/internal/prepare/pipeline_stopwords_test.go b/internal/prepare/pipeline_stopwords_test.go index ab9ec6de..ef190174 100644 --- a/internal/prepare/pipeline_stopwords_test.go +++ b/internal/prepare/pipeline_stopwords_test.go @@ -7,8 +7,8 @@ package prepare import ( "testing" - "github.com/moov-io/watchman/pkg/csl_us" "github.com/moov-io/watchman/pkg/ofac" + "github.com/stretchr/testify/require" "github.com/abadojack/whatlanggo" ) @@ -20,8 +20,8 @@ func TestStopwordsEnv(t *testing.T) { } func TestStopwords__detect(t *testing.T) { - addrs := func(country string) []*ofac.Address { - return []*ofac.Address{ + addrs := func(country string) []ofac.Address { + return []ofac.Address{ { Country: country, }, @@ -44,10 +44,9 @@ func TestStopwords__detect(t *testing.T) { {"INTERCONTINENTAL BAUMASCHINEN UND NUTZFAHRZEUGE HANDELS GMBH", "Germany", whatlanggo.Deu}, } - for i := range cases { - if lang := detectLanguage(cases[i].in, addrs(cases[i].country)); lang != cases[i].expected { - t.Errorf("#%d in=%q country=%s lang=%v", i, cases[i].in, cases[i].country, lang) - } + for _, tc := range cases { + got := detectLanguage(tc.in, addrs(tc.country)) + require.Equal(t, tc.expected, got) } } @@ -71,76 +70,34 @@ func TestStopwords__clean(t *testing.T) { for i := range cases { result := removeStopwords(cases[i].in, cases[i].lang) - if result != cases[i].expected { - t.Errorf("\n#%d in=%q lang=%v\n got=%q\n exp=%q", i, cases[i].in, cases[i].lang, result, cases[i].expected) - } + require.Equal(t, cases[i].expected, result) } } func TestStopwords__apply(t *testing.T) { cases := []struct { - testName string - in *Name + in string expected string }{ { - testName: "type missing", - in: &Name{Processed: "Trees and Trucks"}, - expected: "Trees and Trucks", - }, - { - testName: "alt name", - in: &Name{Processed: "Trees and Trucks", alt: &ofac.AlternateIdentity{}}, - expected: "trees trucks", - }, - { - testName: "sdn individual", - in: &Name{Processed: "Trees and Trucks", sdn: &ofac.SDN{SDNType: "individual"}}, - expected: "Trees and Trucks", - }, - { - testName: "sdn business", - in: &Name{Processed: "Trees and Trucks", sdn: &ofac.SDN{SDNType: "business"}}, + in: "Trees and Trucks", expected: "trees trucks", }, { - testName: "ssi individual", - in: &Name{Processed: "Trees and Trucks", ssi: &csl_us.SSI{Type: "individual"}}, - expected: "Trees and Trucks", - }, - { - testName: "ssi business", - in: &Name{Processed: "Trees and Trucks", ssi: &csl_us.SSI{Type: "business"}}, - expected: "trees trucks", - }, - { - testName: "Issue 483 #1", - in: &Name{Processed: "11420 CORP.", sdn: &ofac.SDN{SDNType: "business"}}, + in: "11420 CORP.", // Issue 483 #1 expected: "11420 corp", }, { - testName: "Issue 483 #2", - in: &Name{Processed: "11,420.2-1 CORP.", sdn: &ofac.SDN{SDNType: "business"}}, + in: "11,420.2-1 CORP.", // Issue 483 #2 expected: "11,420.2-1 corp", }, { - testName: "Issue 483 #3", - in: &Name{Processed: "11AA420 CORP.", sdn: &ofac.SDN{SDNType: "business"}}, + in: "11AA420 CORP.", // Issue 483 #3 expected: "11aa420 corp", }, } - for _, test := range cases { - t.Run(test.testName, func(t *testing.T) { - stopwords := stopwordsStep{} - err := stopwords.apply(test.in) - if err != nil { - t.Errorf("\n#%v in=%v err=%v", test.testName, test.in, err) - } - - if test.in.Processed != test.expected { - t.Errorf("\n#%v expected=%v got=%v", test.testName, test.expected, test.in.Processed) - } - }) + got := RemoveStopwords(test.in, nil) + require.Equal(t, test.expected, got) } } diff --git a/internal/prepare/pipeline_test.go b/internal/prepare/pipeline_test.go index dd0e7491..645b3604 100644 --- a/internal/prepare/pipeline_test.go +++ b/internal/prepare/pipeline_test.go @@ -4,85 +4,85 @@ package prepare -import ( - "testing" +// import ( +// "testing" - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/pkg/ofac" -) +// "github.com/moov-io/base/log" +// "github.com/moov-io/watchman/pkg/ofac" +// ) -var ( - noopPipeliner = &Pipeliner{ - logger: log.NewNopLogger(), - steps: []step{}, - } +// var ( +// noopPipeliner = &Pipeliner{ +// logger: log.NewNopLogger(), +// steps: []step{}, +// } - noLogPipeliner = NewPipeliner(log.NewNopLogger(), false) -) +// noLogPipeliner = NewPipeliner(log.NewNopLogger(), false) +// ) -func TestPipelineNoop(t *testing.T) { - if err := noopPipeliner.Do(&Name{}); err != nil { - t.Fatal(err) - } -} +// func TestPipelineNoop(t *testing.T) { +// if err := noopPipeliner.Do(&Name{}); err != nil { +// t.Fatal(err) +// } +// } -func TestFullPipeline(t *testing.T) { - individual := func(in string) *Name { - return &Name{ - Processed: in, - sdn: &ofac.SDN{ - SDNType: "individual", - }, - } - } - company := func(in string) *Name { - return &Name{ - Processed: in, - sdn: &ofac.SDN{ - SDNType: "", // blank refers to a company - }, - } - } +// func TestFullPipeline(t *testing.T) { +// individual := func(in string) *Name { +// return &Name{ +// Processed: in, +// sdn: &ofac.SDN{ +// SDNType: "individual", +// }, +// } +// } +// company := func(in string) *Name { +// return &Name{ +// Processed: in, +// sdn: &ofac.SDN{ +// SDNType: "", // blank refers to a company +// }, +// } +// } - cases := []struct { - in *Name - expected string - }{ - // input edge cases - {individual(""), ""}, - {individual(" "), ""}, - {individual(" "), ""}, - {company(""), ""}, - {company(" "), ""}, - {company(" "), ""}, +// cases := []struct { +// in *Name +// expected string +// }{ +// // input edge cases +// {individual(""), ""}, +// {individual(" "), ""}, +// {individual(" "), ""}, +// {company(""), ""}, +// {company(" "), ""}, +// {company(" "), ""}, - // Re-order individual names - {individual("MADURO MOROS, Nicolas"), "nicolas maduro moros"}, +// // Re-order individual names +// {individual("MADURO MOROS, Nicolas"), "nicolas maduro moros"}, - // Remove Company Suffixes - {company("YAKIMA OIL TRADING, LLP"), "yakima oil trading"}, // SDN 20259 - {company("MKS INTERNATIONAL CO. LTD."), "mks international"}, // SDN 21553 - {company("SHANGHAI NORTH TRANSWAY INTERNATIONAL TRADING CO."), "shanghai north transway international trading"}, // SDN 22246 +// // Remove Company Suffixes +// {company("YAKIMA OIL TRADING, LLP"), "yakima oil trading"}, // SDN 20259 +// {company("MKS INTERNATIONAL CO. LTD."), "mks international"}, // SDN 21553 +// {company("SHANGHAI NORTH TRANSWAY INTERNATIONAL TRADING CO."), "shanghai north transway international trading"}, // SDN 22246 - // Keep numbers - {company("11420 CORP."), "11420 corp"}, - {company("11AA420 CORP."), "11aa420 corp"}, - {company("11,420.2-1 CORP."), "114202 1 corp"}, +// // Keep numbers +// {company("11420 CORP."), "11420 corp"}, +// {company("11AA420 CORP."), "11aa420 corp"}, +// {company("11,420.2-1 CORP."), "114202 1 corp"}, - // Remove stopwords - {company("INVERSIONES LA QUINTA Y CIA. LTDA."), "inversiones la quinta y cia"}, +// // Remove stopwords +// {company("INVERSIONES LA QUINTA Y CIA. LTDA."), "inversiones la quinta y cia"}, - // Normalize ("-" -> " ") - {company("ANGLO-CARIBBEAN CO., LTD."), "anglo caribbean"}, - } +// // Normalize ("-" -> " ") +// {company("ANGLO-CARIBBEAN CO., LTD."), "anglo caribbean"}, +// } - for i := range cases { - if err := noLogPipeliner.Do(cases[i].in); err != nil { - t.Error(err) - } else { - if cases[i].in.Processed != cases[i].expected { - t.Errorf("%d# output=%q expected=%q", i, cases[i].in.Processed, cases[i].expected) - } - } - } -} +// for i := range cases { +// if err := noLogPipeliner.Do(cases[i].in); err != nil { +// t.Error(err) +// } else { +// if cases[i].in.Processed != cases[i].expected { +// t.Errorf("%d# output=%q expected=%q", i, cases[i].in.Processed, cases[i].expected) +// } +// } +// } +// } diff --git a/internal/search/api_search.go b/internal/search/api_search.go index 40f0de3e..c51ce43a 100644 --- a/internal/search/api_search.go +++ b/internal/search/api_search.go @@ -66,6 +66,8 @@ func (c *controller) search(w http.ResponseWriter, r *http.Request) { return } + fmt.Printf("req: %#v\n", req) + q := r.URL.Query() opts := SearchOpts{ Limit: extractSearchLimit(r), @@ -73,6 +75,7 @@ func (c *controller) search(w http.ResponseWriter, r *http.Request) { RequestID: q.Get("requestID"), DebugSourceIDs: strings.Split(q.Get("debugSourceIDs"), ","), } + fmt.Printf("opts: %#v\n", opts) entities, err := c.service.Search(r.Context(), req, opts) if err != nil { @@ -81,11 +84,13 @@ func (c *controller) search(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusBadRequest) return } + fmt.Printf("found %d entities\n", len(entities)) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(searchResponse{ Entities: entities, }) + // TODO(adam): why isn't this returning results??? } var ( diff --git a/internal/search/service.go b/internal/search/service.go index 10963bca..991ea552 100644 --- a/internal/search/service.go +++ b/internal/search/service.go @@ -5,6 +5,7 @@ import ( "fmt" "slices" "sync" + "time" "github.com/moov-io/watchman/internal/largest" "github.com/moov-io/watchman/pkg/search" @@ -14,18 +15,17 @@ import ( ) type Service interface { + UpdateEntities(entities []search.Entity[search.Value]) + Search(ctx context.Context, query search.Entity[search.Value], opts SearchOpts) ([]search.SearchedEntity[search.Value], error) } -func NewService(logger log.Logger, entities []search.Entity[search.Value]) Service { - fmt.Printf("v2search NewService(%d entity types)\n", len(entities)) //nolint:forbidigo - +func NewService(logger log.Logger) Service { gate := syncutil.NewGate(100) // TODO(adam): return &service{ - logger: logger, - entities: entities, - Gate: gate, + logger: logger, + Gate: gate, } } @@ -37,6 +37,13 @@ type service struct { *syncutil.Gate // limits concurrent processing } +func (s *service) UpdateEntities(entities []search.Entity[search.Value]) { + s.Lock() + defer s.Unlock() + + s.entities = entities +} + func (s *service) Search(ctx context.Context, query search.Entity[search.Value], opts SearchOpts) ([]search.SearchedEntity[search.Value], error) { // Grab a read-lock over our data s.RLock() @@ -69,6 +76,9 @@ func (s *service) performSearch(ctx context.Context, query search.Entity[search. var wg sync.WaitGroup wg.Add(len(indices)) + fmt.Printf("indices: %#v ", indices) + + start := time.Now() for idx := range indices { start := idx var end int @@ -79,6 +89,7 @@ func (s *service) performSearch(ctx context.Context, query search.Entity[search. } else { end = indices[start+1] } + fmt.Printf("start=%d end=%v\n", start, end) go func() { defer wg.Done() @@ -88,6 +99,9 @@ func (s *service) performSearch(ctx context.Context, query search.Entity[search. } wg.Wait() + fmt.Printf("concurrent search took: %v\n", time.Since(start)) + start = time.Now() + results := items.Items() var out []search.SearchedEntity[search.Value] @@ -106,6 +120,8 @@ func (s *service) performSearch(ctx context.Context, query search.Entity[search. }) } + fmt.Printf("result mapping took: %v\n", time.Since(start)) + return out, nil } diff --git a/internal/search/service_test.go b/internal/search/service_test.go index 7a1f2f34..0bdeb034 100644 --- a/internal/search/service_test.go +++ b/internal/search/service_test.go @@ -24,18 +24,15 @@ func TestService_Search(t *testing.T) { ofacRecords, err := ofac.Read(files) require.NoError(t, err) - sdns := depointer(ofacRecords.SDNs) - addrs := depointer(ofacRecords.Addresses) - comments := depointer(ofacRecords.SDNComments) - alts := depointer(ofacRecords.AlternateIdentities) - entities := ofac.ToEntities(sdns, addrs, comments, alts) + entities := ofac.GroupIntoEntities(ofacRecords.SDNs, ofacRecords.Addresses, ofacRecords.SDNComments, ofacRecords.AlternateIdentities) ctx := context.Background() logger := log.NewTestLogger() opts := SearchOpts{Limit: 10, MinMatch: 0.01} - svc := NewService(logger, entities) + svc := NewService(logger) + svc.UpdateEntities(entities) t.Run("basic", func(t *testing.T) { results, err := svc.Search(ctx, search.Entity[search.Value]{ @@ -91,13 +88,3 @@ func testInputs(tb testing.TB, paths ...string) map[string]io.ReadCloser { } return input } - -func depointer[T any](input []*T) []T { - out := make([]T, len(input)) - for i := range input { - if input[i] != nil { - out[i] = *input[i] - } - } - return out -} diff --git a/internal/ui/search.go b/internal/ui/search.go index 80b209e7..4bca4e00 100644 --- a/internal/ui/search.go +++ b/internal/ui/search.go @@ -158,7 +158,7 @@ var ( func newSelect(modelName string) *widget.Select { values, err := ast.ExtractVariablesOfType(modelsPath, modelName) if err != nil { - panic(fmt.Sprintf("reading %s values: %w", modelName, err)) //nolint:forbidigo + panic(fmt.Sprintf("reading %s values: %v", modelName, err)) //nolint:forbidigo } selectWidget := widget.NewSelect(values, func(_ string) {}) diff --git a/package.go b/package.go new file mode 100644 index 00000000..fda0c823 --- /dev/null +++ b/package.go @@ -0,0 +1,8 @@ +package watchman + +import ( + "embed" +) + +//go:embed configs/config.default.yml +var ConfigDefaults embed.FS diff --git a/pkg/csl_eu/download_eu.go b/pkg/csl_eu/download_eu.go index 758ccef8..6828dfe0 100644 --- a/pkg/csl_eu/download_eu.go +++ b/pkg/csl_eu/download_eu.go @@ -5,6 +5,7 @@ package csl_eu import ( + "context" "fmt" "io" "os" @@ -23,12 +24,12 @@ var ( euDownloadURL = strx.Or(os.Getenv("EU_CSL_DOWNLOAD_URL"), publicEUDownloadURL) ) -func DownloadEU(logger log.Logger, initialDir string) (map[string]io.ReadCloser, error) { +func DownloadEU(ctx context.Context, logger log.Logger, initialDir string) (map[string]io.ReadCloser, error) { dl := download.New(logger, download.HTTPClient) euCSLNameAndSource := make(map[string]string) euCSLNameAndSource["eu_csl.csv"] = euDownloadURL - return dl.GetFiles(initialDir, euCSLNameAndSource) + return dl.GetFiles(ctx, initialDir, euCSLNameAndSource) } diff --git a/pkg/csl_eu/download_eu_test.go b/pkg/csl_eu/download_eu_test.go index 02949a42..c523c872 100644 --- a/pkg/csl_eu/download_eu_test.go +++ b/pkg/csl_eu/download_eu_test.go @@ -5,6 +5,7 @@ package csl_eu import ( + "context" "fmt" "io" "os" @@ -20,7 +21,7 @@ func TestEUDownload(t *testing.T) { return } - file, err := DownloadEU(log.NewNopLogger(), "") + file, err := DownloadEU(context.Background(), log.NewNopLogger(), "") if err != nil { t.Fatal(err) } @@ -53,7 +54,7 @@ func TestEUDownload_initialDir(t *testing.T) { // create each file mk(t, "eu_csl.csv", "file=eu_csl.csv") - file, err := DownloadEU(log.NewNopLogger(), dir) + file, err := DownloadEU(context.Background(), log.NewNopLogger(), dir) if err != nil { t.Fatal(err) } diff --git a/pkg/csl_eu/reader_eu.go b/pkg/csl_eu/reader_eu.go index a17a8db8..5c865810 100644 --- a/pkg/csl_eu/reader_eu.go +++ b/pkg/csl_eu/reader_eu.go @@ -12,7 +12,7 @@ import ( "strconv" ) -func ParseEU(r io.ReadCloser) ([]*CSLRecord, CSL, error) { +func ParseEU(r io.ReadCloser) ([]CSLRecord, CSL, error) { if r == nil { return nil, nil, errors.New("EU CSL file is empty or missing") } @@ -65,9 +65,9 @@ func ParseEU(r io.ReadCloser) ([]*CSLRecord, CSL, error) { } } - totalReport := make([]*CSLRecord, 0, len(report)) + totalReport := make([]CSLRecord, 0, len(report)) for _, row := range report { - totalReport = append(totalReport, row) + totalReport = append(totalReport, *row) } return totalReport, report, nil } diff --git a/pkg/csl_uk/download_uk.go b/pkg/csl_uk/download_uk.go index d7d907be..b991a7a6 100644 --- a/pkg/csl_uk/download_uk.go +++ b/pkg/csl_uk/download_uk.go @@ -5,6 +5,7 @@ package csl_uk import ( + "context" "fmt" "io" "os" @@ -23,21 +24,21 @@ var ( ukCSLDownloadURL = strx.Or(os.Getenv("UK_CSL_DOWNLOAD_URL"), publicCSLDownloadURL) ) -func DownloadCSL(logger log.Logger, initialDir string) (map[string]io.ReadCloser, error) { +func DownloadCSL(ctx context.Context, logger log.Logger, initialDir string) (map[string]io.ReadCloser, error) { dl := download.New(logger, download.HTTPClient) ukCSLNameAndSource := make(map[string]string) ukCSLNameAndSource["ConList.csv"] = ukCSLDownloadURL - return dl.GetFiles(initialDir, ukCSLNameAndSource) + return dl.GetFiles(ctx, initialDir, ukCSLNameAndSource) } -func DownloadSanctionsList(logger log.Logger, initialDir string) (map[string]io.ReadCloser, error) { +func DownloadSanctionsList(ctx context.Context, logger log.Logger, initialDir string) (map[string]io.ReadCloser, error) { dl := download.New(logger, download.HTTPClient) ukSanctionsNameAndSource := make(map[string]string) - latestURL, err := fetchLatestUKSanctionsListURL(logger, initialDir) + latestURL, err := fetchLatestUKSanctionsListURL(ctx, logger, initialDir) if err != nil { return nil, err } @@ -45,14 +46,14 @@ func DownloadSanctionsList(logger log.Logger, initialDir string) (map[string]io. ukSanctionsNameAndSource["UK_Sanctions_List.ods"] = latestURL - return dl.GetFiles(initialDir, ukSanctionsNameAndSource) + return dl.GetFiles(ctx, initialDir, ukSanctionsNameAndSource) } var ( defaultUKSanctionsListHTML = strx.Or(os.Getenv("UK_CSL_HTML_INDEX_URL"), "https://www.gov.uk/government/publications/the-uk-sanctions-list") ) -func fetchLatestUKSanctionsListURL(logger log.Logger, initialDir string) (string, error) { +func fetchLatestUKSanctionsListURL(ctx context.Context, logger log.Logger, initialDir string) (string, error) { fromEnv := strings.TrimSpace(os.Getenv("UK_SANCTIONS_LIST_URL")) if fromEnv != "" { return fromEnv, nil @@ -64,7 +65,7 @@ func fetchLatestUKSanctionsListURL(logger log.Logger, initialDir string) (string dl := download.New(logger, download.HTTPClient) - pages, err := dl.GetFiles(initialDir, ukSanctionsNameAndSource) + pages, err := dl.GetFiles(ctx, initialDir, ukSanctionsNameAndSource) if err != nil { return "", fmt.Errorf("getting UK Sanctions html index: %w", err) } diff --git a/pkg/csl_uk/download_uk_test.go b/pkg/csl_uk/download_uk_test.go index 85b3543b..c56f3436 100644 --- a/pkg/csl_uk/download_uk_test.go +++ b/pkg/csl_uk/download_uk_test.go @@ -5,6 +5,7 @@ package csl_uk import ( + "context" "fmt" "io" "os" @@ -21,7 +22,7 @@ func TestCSLDownload(t *testing.T) { return } - file, err := DownloadCSL(log.NewNopLogger(), "") + file, err := DownloadCSL(context.Background(), log.NewNopLogger(), "") if err != nil { t.Fatal(err) } @@ -54,7 +55,7 @@ func TestCSLDownload_initialDir(t *testing.T) { // create each file mk(t, "ConList.csv", "file=ConList.csv") - file, err := DownloadCSL(log.NewNopLogger(), dir) + file, err := DownloadCSL(context.Background(), log.NewNopLogger(), dir) if err != nil { t.Fatal(err) } @@ -83,7 +84,7 @@ func TestUKSanctionsListIndex(t *testing.T) { } logger := log.NewTestLogger() - foundURL, err := fetchLatestUKSanctionsListURL(logger, "") + foundURL, err := fetchLatestUKSanctionsListURL(context.Background(), logger, "") require.NoError(t, err) require.Contains(t, foundURL, "UK_Sanctions_List.ods") @@ -95,7 +96,7 @@ func TestUKSanctionsListDownload(t *testing.T) { } logger := log.NewTestLogger() - file, err := DownloadSanctionsList(logger, "") + file, err := DownloadSanctionsList(context.Background(), logger, "") if err != nil { t.Fatal(err) } @@ -128,7 +129,7 @@ func TestUKSanctionsListDownload_initialDir(t *testing.T) { // create each file mk(t, "UK_Sanctions_List.ods", "file=UK_Sanctions_List.ods") - file, err := DownloadSanctionsList(log.NewNopLogger(), dir) + file, err := DownloadSanctionsList(context.Background(), log.NewNopLogger(), dir) if err != nil { t.Fatal(err) } diff --git a/pkg/csl_uk/reader_uk.go b/pkg/csl_uk/reader_uk.go index bb7957c8..18eec3fe 100644 --- a/pkg/csl_uk/reader_uk.go +++ b/pkg/csl_uk/reader_uk.go @@ -15,7 +15,7 @@ import ( "github.com/knieriem/odf/ods" ) -func ReadCSLFile(fd io.ReadCloser) ([]*CSLRecord, CSL, error) { +func ReadCSLFile(fd io.ReadCloser) ([]CSLRecord, CSL, error) { if fd == nil { return nil, nil, errors.New("uk CSL file is empty or missing") } @@ -29,7 +29,7 @@ func ReadCSLFile(fd io.ReadCloser) ([]*CSLRecord, CSL, error) { return rows, rowsMap, nil } -func ParseCSL(r io.Reader) ([]*CSLRecord, CSL, error) { +func ParseCSL(r io.Reader) ([]CSLRecord, CSL, error) { reader := csv.NewReader(r) reader.FieldsPerRecord = 36 @@ -79,9 +79,9 @@ func ParseCSL(r io.Reader) ([]*CSLRecord, CSL, error) { } } - totalReport := make([]*CSLRecord, 0, len(report)) + totalReport := make([]CSLRecord, 0, len(report)) for _, row := range report { - totalReport = append(totalReport, row) + totalReport = append(totalReport, *row) } return totalReport, report, nil } @@ -199,7 +199,7 @@ func unmarshalCSLRecord(csvRecord []string, ukCSLRecord *CSLRecord) { } } -func ReadSanctionsListFile(f io.ReadCloser) ([]*SanctionsListRecord, SanctionsListMap, error) { +func ReadSanctionsListFile(f io.ReadCloser) ([]SanctionsListRecord, SanctionsListMap, error) { if f == nil { return nil, nil, errors.New("uk sanctions list file is empty or missing") } @@ -228,9 +228,9 @@ func ReadSanctionsListFile(f io.ReadCloser) ([]*SanctionsListRecord, SanctionsLi return rows, rowsMap, nil } -func parseSanctionsList(doc *ods.Doc) ([]*SanctionsListRecord, SanctionsListMap, error) { +func parseSanctionsList(doc *ods.Doc) ([]SanctionsListRecord, SanctionsListMap, error) { // read from the ods document - var totalReport []*SanctionsListRecord + var totalReport []SanctionsListRecord report := SanctionsListMap{} // unmarshal each row into a uk sanctions list record @@ -260,7 +260,7 @@ func parseSanctionsList(doc *ods.Doc) ([]*SanctionsListRecord, SanctionsListMap, } for _, row := range report { - totalReport = append(totalReport, row) + totalReport = append(totalReport, *row) } return totalReport, report, nil } diff --git a/pkg/csl_us/csl.go b/pkg/csl_us/csl.go index 5c863612..7925b280 100644 --- a/pkg/csl_us/csl.go +++ b/pkg/csl_us/csl.go @@ -6,17 +6,17 @@ package csl_us // CSL contains each record from the Consolidate Screening List, broken down by the record's original source type CSL struct { - ELs []*EL // Entity List – Bureau of Industry and Security - MEUs []*MEU // Military End User List - SSIs []*SSI // Sectoral Sanctions Identifications List (SSI) - Treasury Department - UVLs []*UVL // Unverified List – Bureau of Industry and Security - FSEs []*FSE // Foreign Sanctions Evaders (FSE) - Treasury Department - ISNs []*ISN // Nonproliferation Sanctions (ISN) - State Department - PLCs []*PLC // Palestinian Legislative Council List (PLC) - Treasury Department - CAPs []*CAP // CAPTA (formerly Foreign Financial Institutions Subject to Part 561 - Treasury Department) - DTCs []*DTC // ITAR Debarred (DTC) - State Department - CMICs []*CMIC // Non-SDN Chinese Military-Industrial Complex Companies List (CMIC) - Treasury Department - NS_MBSs []*NS_MBS // Non-SDN Menu-Based Sanctions List (NS-MBS List) - Treasury Department + ELs []EL // Entity List – Bureau of Industry and Security + MEUs []MEU // Military End User List + SSIs []SSI // Sectoral Sanctions Identifications List (SSI) - Treasury Department + UVLs []UVL // Unverified List – Bureau of Industry and Security + FSEs []FSE // Foreign Sanctions Evaders (FSE) - Treasury Department + ISNs []ISN // Nonproliferation Sanctions (ISN) - State Department + PLCs []PLC // Palestinian Legislative Council List (PLC) - Treasury Department + CAPs []CAP // CAPTA (formerly Foreign Financial Institutions Subject to Part 561 - Treasury Department) + DTCs []DTC // ITAR Debarred (DTC) - State Department + CMICs []CMIC // Non-SDN Chinese Military-Industrial Complex Companies List (CMIC) - Treasury Department + NS_MBSs []NS_MBS // Non-SDN Menu-Based Sanctions List (NS-MBS List) - Treasury Department } // This is the order of the columns in the CSL diff --git a/pkg/csl_us/csl_test.go b/pkg/csl_us/csl_test.go index 5cb6a28b..87fe3c7c 100644 --- a/pkg/csl_us/csl_test.go +++ b/pkg/csl_us/csl_test.go @@ -5,6 +5,7 @@ package csl_us import ( + "context" "os" "testing" @@ -14,7 +15,6 @@ import ( ) func TestCSL(t *testing.T) { - if testing.Short() { t.Skip("ignorning network test") } @@ -23,7 +23,7 @@ func TestCSL(t *testing.T) { dir, err := os.MkdirTemp("", "csl") require.NoError(t, err) - file, err := Download(logger, dir) + file, err := Download(context.Background(), logger, dir) require.NoError(t, err) cslRecords, err := ReadFile(file["csl.csv"]) diff --git a/pkg/csl_us/download.go b/pkg/csl_us/download.go index 48e54671..1196c323 100644 --- a/pkg/csl_us/download.go +++ b/pkg/csl_us/download.go @@ -5,6 +5,7 @@ package csl_us import ( + "context" "fmt" "io" "net/url" @@ -20,7 +21,7 @@ var ( usDownloadURL = strx.Or(os.Getenv("CSL_DOWNLOAD_TEMPLATE"), os.Getenv("US_CSL_DOWNLOAD_URL"), publicUSDownloadURL) ) -func Download(logger log.Logger, initialDir string) (map[string]io.ReadCloser, error) { +func Download(ctx context.Context, logger log.Logger, initialDir string) (map[string]io.ReadCloser, error) { dl := download.New(logger, download.HTTPClient) cslURL, err := buildDownloadURL(usDownloadURL) @@ -31,7 +32,7 @@ func Download(logger log.Logger, initialDir string) (map[string]io.ReadCloser, e cslNameAndSource := make(map[string]string) cslNameAndSource["csl.csv"] = cslURL - return dl.GetFiles(initialDir, cslNameAndSource) + return dl.GetFiles(ctx, initialDir, cslNameAndSource) } func buildDownloadURL(urlStr string) (string, error) { diff --git a/pkg/csl_us/download_test.go b/pkg/csl_us/download_test.go index 1c991003..22ece57f 100644 --- a/pkg/csl_us/download_test.go +++ b/pkg/csl_us/download_test.go @@ -5,6 +5,7 @@ package csl_us import ( + "context" "io" "os" "path/filepath" @@ -19,7 +20,7 @@ func TestDownload(t *testing.T) { return } - file, err := Download(log.NewNopLogger(), "") + file, err := Download(context.Background(), log.NewNopLogger(), "") if err != nil { t.Fatal(err) } @@ -53,7 +54,7 @@ func TestDownload_initialDir(t *testing.T) { mk(t, "csl.csv", "file=csl.csv") mk(t, "csl.csv", "file=csl.csv") - file, err := Download(log.NewNopLogger(), dir) + file, err := Download(context.Background(), log.NewNopLogger(), dir) if err != nil { t.Fatal(err) } diff --git a/pkg/csl_us/reader.go b/pkg/csl_us/reader.go index c16e13f3..b1523c8b 100644 --- a/pkg/csl_us/reader.go +++ b/pkg/csl_us/reader.go @@ -11,15 +11,15 @@ import ( "strings" ) -func ReadFile(fd io.ReadCloser) (*CSL, error) { +func ReadFile(fd io.ReadCloser) (CSL, error) { if fd == nil { - return nil, errors.New("CSL file is empty or missing") + return CSL{}, errors.New("CSL file is empty or missing") } defer fd.Close() return Parse(fd) } -func Parse(r io.Reader) (*CSL, error) { +func Parse(r io.Reader) (CSL, error) { reader := csv.NewReader(r) var report CSL @@ -36,7 +36,7 @@ func Parse(r io.Reader) (*CSL, error) { errors.Is(err, csv.ErrQuote) { continue } - return nil, err + return report, err } if len(record) <= 1 { @@ -96,15 +96,15 @@ func Parse(r io.Reader) (*CSL, error) { } } } - return &report, nil + return report, nil } -func unmarshalEL(row []string, offset int) *EL { +func unmarshalEL(row []string, offset int) EL { id := "" if offset == 1 { id = row[0] // set the ID from the newer CSV format } - return &EL{ + return EL{ ID: id, Name: row[NameIdx+offset], Addresses: expandField(row[AddressesIdx+offset]), @@ -118,8 +118,8 @@ func unmarshalEL(row []string, offset int) *EL { } } -func unmarshalMEU(record []string, offset int) *MEU { - return &MEU{ +func unmarshalMEU(record []string, offset int) MEU { + return MEU{ EntityID: record[0], Name: record[NameIdx+offset], Addresses: record[AddressesIdx+offset], @@ -129,8 +129,8 @@ func unmarshalMEU(record []string, offset int) *MEU { } } -func unmarshalSSI(record []string, offset int) *SSI { - return &SSI{ +func unmarshalSSI(record []string, offset int) SSI { + return SSI{ EntityID: record[EntityNumberIdx+offset], Type: record[TypeIdx+offset], Programs: expandProgramsList(record[ProgramsIdx+offset]), @@ -144,8 +144,8 @@ func unmarshalSSI(record []string, offset int) *SSI { } } -func unmarshalUVL(record []string, offset int) *UVL { - return &UVL{ +func unmarshalUVL(record []string, offset int) UVL { + return UVL{ EntityID: record[0], Name: record[NameIdx+offset], Addresses: expandField(record[AddressesIdx+offset]), @@ -154,8 +154,8 @@ func unmarshalUVL(record []string, offset int) *UVL { } } -func unmarshalISN(record []string, offset int) *ISN { - return &ISN{ +func unmarshalISN(record []string, offset int) ISN { + return ISN{ EntityID: record[0], Programs: expandProgramsList(record[ProgramsIdx+offset]), Name: record[NameIdx+offset], @@ -168,8 +168,8 @@ func unmarshalISN(record []string, offset int) *ISN { } } -func unmarshalFSE(record []string, offset int) *FSE { - return &FSE{ +func unmarshalFSE(record []string, offset int) FSE { + return FSE{ EntityID: record[0], EntityNumber: record[EntityNumberIdx+offset], Type: record[TypeIdx+offset], @@ -184,8 +184,8 @@ func unmarshalFSE(record []string, offset int) *FSE { } } -func unmarshalPLC(record []string, offset int) *PLC { - return &PLC{ +func unmarshalPLC(record []string, offset int) PLC { + return PLC{ EntityID: record[0], EntityNumber: record[EntityNumberIdx+offset], Type: record[TypeIdx+offset], @@ -201,8 +201,8 @@ func unmarshalPLC(record []string, offset int) *PLC { } } -func unmarshalCAP(record []string, offset int) *CAP { - return &CAP{ +func unmarshalCAP(record []string, offset int) CAP { + return CAP{ EntityID: record[0], EntityNumber: record[EntityNumberIdx+offset], Type: record[TypeIdx+offset], @@ -217,8 +217,8 @@ func unmarshalCAP(record []string, offset int) *CAP { } } -func unmarshalNS_MBS(record []string, offset int) *NS_MBS { - return &NS_MBS{ +func unmarshalNS_MBS(record []string, offset int) NS_MBS { + return NS_MBS{ EntityID: record[0], EntityNumber: record[EntityNumberIdx+offset], Type: record[TypeIdx+offset], @@ -232,8 +232,8 @@ func unmarshalNS_MBS(record []string, offset int) *NS_MBS { } } -func unmarshalCMIC(record []string, offset int) *CMIC { - return &CMIC{ +func unmarshalCMIC(record []string, offset int) CMIC { + return CMIC{ EntityID: record[0], EntityNumber: record[EntityNumberIdx+offset], Type: record[TypeIdx+offset], @@ -248,8 +248,8 @@ func unmarshalCMIC(record []string, offset int) *CMIC { } } -func unmarshalDTC(record []string, offset int) *DTC { - return &DTC{ +func unmarshalDTC(record []string, offset int) DTC { + return DTC{ EntityID: record[0], Name: record[NameIdx+offset], FederalRegisterNotice: record[FRNoticeIdx+offset], diff --git a/pkg/csl_us/reader_test.go b/pkg/csl_us/reader_test.go index f753a620..ac6535ad 100644 --- a/pkg/csl_us/reader_test.go +++ b/pkg/csl_us/reader_test.go @@ -18,22 +18,13 @@ import ( func TestRead(t *testing.T) { fd, err := os.Open(filepath.Join("..", "..", "test", "testdata", "csl.csv")) - if err != nil { - t.Error(err) - } + require.NoError(t, err) + csl, err := ReadFile(fd) - if err != nil { - t.Fatal(err) - } - if csl == nil { - t.Fatal("failed to parse csl.csv") - } - if len(csl.SSIs) != 26 { // test CSL csv file has 26 SSI entries - t.Errorf("len(SSIs)=%d", len(csl.SSIs)) - } - if len(csl.ELs) != 22 { - t.Errorf("len(ELs)=%d", len(csl.ELs)) - } + require.NoError(t, err) + + require.Len(t, csl.SSIs, 26) // test CSL csv file has 26 SSI entries + require.Len(t, csl.ELs, 22) } func TestRead__Large(t *testing.T) { @@ -70,30 +61,20 @@ func TestRead_missingRow(t *testing.T) { } func TestRead_invalidRow(t *testing.T) { - fd, err := os.Open(filepath.Join("..", "..", "test", "testdata", "invalidFiles", "csl.csv")) - if err != nil { - t.Error(err) - } + require.NoError(t, err) + csl, err := ReadFile(fd) - if err != nil { - t.Fatal(err) - } - if csl == nil { - t.Fatal("failed to parse csl.csv") - } - if len(csl.SSIs) != 1 { - t.Errorf("len(SSIs)=%d", len(csl.SSIs)) - } - if len(csl.ELs) != 1 { - t.Errorf("len(ELs)=%d", len(csl.ELs)) - } + require.NoError(t, err) + + require.Len(t, csl.SSIs, 1) + require.Len(t, csl.ELs, 1) } func Test_unmarshalEL(t *testing.T) { record := []string{"Entity List (EL) - Bureau of Industry and Security", "", "", "", "GBNTT", "", "No. 34 Mansour Street, Tehran, IR", "73 FR 54506", "2008-09-22", "", "", "For all items subject to the EAR (See §744.11 of the EAR)", "Presumption of denial", "", "", "", "", "", "", "", "http://bit.ly/1L47xrV", "", "", "", "", "", "http://bit.ly/1L47xrV", ""} - expectedEL := &EL{ + expectedEL := EL{ Name: "GBNTT", AlternateNames: nil, Addresses: []string{"No. 34 Mansour Street, Tehran, IR"}, @@ -122,7 +103,7 @@ d54346ef81802673c1b1daeb2ca8bd5d13755abd,Military End User (MEU) List - Bureau o require.NoError(t, err) require.Len(t, report.MEUs, 3) - require.Equal(t, &MEU{ + require.Equal(t, MEU{ EntityID: "26744194bd9b5cbec49db6ee29a4b53c697c7420", Name: "AECC Aviation Power Co. Ltd.", Addresses: "Xiujia Bay, Weiyong Dt, Xian, 710021, CN", @@ -131,7 +112,7 @@ d54346ef81802673c1b1daeb2ca8bd5d13755abd,Military End User (MEU) List - Bureau o EndDate: "", }, report.MEUs[0]) - require.Equal(t, &MEU{ + require.Equal(t, MEU{ EntityID: "d54346ef81802673c1b1daeb2ca8bd5d13755abd", Name: "AECC China Gas Turbine Establishment", Addresses: "No. 1 Hangkong Road, Mianyang, Sichuan, CN", @@ -149,7 +130,7 @@ func Test_unmarshalSSI(t *testing.T) { "OAO AK TRANSNEFT; AKTSIONERNAYA KOMPANIYA PO TRANSPORTUNEFTI TRANSNEFT OAO; OIL TRANSPORTING JOINT-STOCK COMPANY TRANSNEFT; TRANSNEFT, JSC; TRANSNEFT OJSC; TRANSNEFT", "", "", "", "", "http://bit.ly/1MLgou0", "1027700049486, Registration ID; 00044463, Government Gazette Number; 7706061801, Tax ID No.; transneft@ak.transneft.ru, Email Address; www.transneft.ru, Website; Subject to Directive 2, Executive Order 13662 Directive Determination -", } - expectedSSI := &SSI{ + expectedSSI := SSI{ EntityID: "17254", Type: "Entity", Programs: []string{"UKRAINE-EO13662", "SYRIA"}, @@ -175,7 +156,7 @@ func Test_unmarshalUVL(t *testing.T) { "Atlas Sanatgaran", "", "Komitas 26/114, Yerevan, Armenia, AM", "", "", "", "", "", "", "", "", "", "", "", "", "", "http://bit.ly/1iwwTSJ", "", "", "", "", "", "http://bit.ly/1Qi4R7Z", "", } - expectedUVL := &UVL{ + expectedUVL := UVL{ EntityID: "f15fa805ff4ac5e09026f5e78011a1bb6b26dec2", Name: "Atlas Sanatgaran", Addresses: []string{"Komitas 26/114, Yerevan, Armenia, AM"}, @@ -196,7 +177,7 @@ func Test_unmarshalISN(t *testing.T) { "E.O. 13382; Export-Import Bank Act; Nuclear Proliferation Prevention Act", "Abdul Qadeer Khan", "", "", "Vol. 74, No. 11, 01/16/09", "2009-01-09", "", "", "", "", "", "", "", "", "", "", "Associated with the A.Q. Khan Network", "http://bit.ly/1NuVFxV", "ZAMAN; Haydar", "", "", "", "", "http://bit.ly/1NuVFxV", "", } - expectedISN := &ISN{ + expectedISN := ISN{ EntityID: "2d2db09c686e4829d0ef1b0b04145eec3d42cd88", Programs: []string{"E.O. 13382", "Export-Import Bank Act", "Nuclear Proliferation Prevention Act"}, Name: "Abdul Qadeer Khan", @@ -221,7 +202,7 @@ func Test_unmarshalFSE(t *testing.T) { "BEKTAS, Halis", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "https://bit.ly/1QWTIfE", "", "CH", "1966-02-13", "", "", "http://bit.ly/1N1docf", "CH, X0906223, Passport", } - expectedFSE := &FSE{ + expectedFSE := FSE{ EntityID: "17526", EntityNumber: "17526", Type: "Individual", @@ -249,7 +230,7 @@ func Test_unmarshalPLC(t *testing.T) { "", "", "http://bit.ly/2tjOLpx", "", } - expectedPLC := &PLC{ + expectedPLC := PLC{ EntityID: "9702", EntityNumber: "9702", Type: "Individual", @@ -280,7 +261,7 @@ func Test_unmarshalCAP(t *testing.T) { "", "", "", "", "http://bit.ly/2PqohAD", "RU, 1027700159497, Registration Number; RU, 29292940, Government Gazette Number; MOSWRUMM, SWIFT/BIC; www.bm.ru, Website; Subject to Directive 1, Executive Order 13662 Directive Determination -; 044525219, BIK (RU); Financial Institution, Target Type", } - expectedCAP := &CAP{ + expectedCAP := CAP{ EntityID: "20002", EntityNumber: "20002", Type: "Entity", @@ -317,7 +298,7 @@ func Test_unmarshalDTC(t *testing.T) { "http://bit.ly/307FuRQ", "Yasmin Tariq; Fatimah Mohammad", "", "", "", "", "http://bit.ly/307FuRQ", } - expectedDTC := &DTC{ + expectedDTC := DTC{ EntityID: "d44d88d0265d93927b9ff1c13bbbb7c7db64142c", Name: "Yasmin Ahmed", FederalRegisterNotice: "69 FR 17468", @@ -342,7 +323,7 @@ func Test_unmarshalCMIC(t *testing.T) { "Proven Honour Capital Ltd, Issuer Name; Proven Honour Capital Limited, Issuer Name; XS1233275194, ISIN; HK0000216777, ISIN; Private Company, Target Type; XS1401816761, ISIN; HK0000111952, ISIN; 03 Jun 2021, Listing Date (CMIC); 02 Aug 2021, Effective Date (CMIC); 03 Jun 2022, Purchase/Sales For Divestment Date (CMIC)", } - expectedCMIC := &CMIC{ + expectedCMIC := CMIC{ EntityID: "32091", EntityNumber: "32091", Type: "Entity", @@ -374,7 +355,7 @@ func Test_unmarshalNS_MBS(t *testing.T) { "RU, 1027700167110, Registration Number; RU, 09807684, Government Gazette Number; RU, 7744001497, Tax ID No.; www.gazprombank.ru, Website; GAZPRUMM, SWIFT/BIC; Subject to Directive 1, Executive Order 13662 Directive Determination -; Subject to Directive 3 - All transactions in, provision of financing for, and other dealings in new debt of longer than 14 days maturity or new equity where such new debt or new equity is issued on or after the 'Effective Date (EO 14024 Directive)' associated with this name are prohibited., Executive Order 14024 Directive Information; 31 Jul 1990, Organization Established Date; 24 Feb 2022, Listing Date (EO 14024 Directive 3):; 26 Mar 2022, Effective Date (EO 14024 Directive 3):; For more information on directives, please visit the following link: https://home.treasury.gov/policy-issues/financial-sanctions/sanctions-programs-and-country-information/russian-harmful-foreign-activities-sanctions#directives, Executive Order 14024 Directive Information -", } - expectedNS_MBS := &NS_MBS{ + expectedNS_MBS := NS_MBS{ EntityID: "17016", EntityNumber: "17016", Type: "Entity", diff --git a/pkg/download/client.go b/pkg/download/client.go index e810c41a..a9f7662e 100644 --- a/pkg/download/client.go +++ b/pkg/download/client.go @@ -5,6 +5,7 @@ package download import ( + "context" "errors" "fmt" "io" @@ -48,7 +49,7 @@ type Downloader struct { // initialDir is an optional filepath to look for files in before attempting to download. // // Callers are expected to call the io.Closer interface method when they are done with the file -func (dl *Downloader) GetFiles(dir string, namesAndSources map[string]string) (map[string]io.ReadCloser, error) { +func (dl *Downloader) GetFiles(ctx context.Context, dir string, namesAndSources map[string]string) (map[string]io.ReadCloser, error) { if dl == nil { return nil, errors.New("nil Downloader") } @@ -96,7 +97,7 @@ findfiles: logger := dl.createLogger(filename, downloadURL) startTime := time.Now().In(time.UTC) - content, err := dl.retryDownload(downloadURL) + content, err := dl.retryDownload(ctx, downloadURL) dur := time.Now().In(time.UTC).Sub(startTime) if err != nil { @@ -127,10 +128,10 @@ func (dl *Downloader) createLogger(filename, downloadURL string) log.Logger { }) } -func (dl *Downloader) retryDownload(downloadURL string) (io.ReadCloser, error) { +func (dl *Downloader) retryDownload(ctx context.Context, downloadURL string) (io.ReadCloser, error) { // Allow a couple retries for various sources (some are flakey) for i := 0; i < 3; i++ { - req, err := http.NewRequest(http.MethodGet, downloadURL, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) if err != nil { return nil, dl.Logger.Error().LogErrorf("error building HTTP request: %v", err).Err() } diff --git a/pkg/dpl/download.go b/pkg/dpl/download.go deleted file mode 100644 index 989e7b66..00000000 --- a/pkg/dpl/download.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2020 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package dpl - -import ( - "fmt" - "io" - "os" - - "github.com/moov-io/base/log" - "github.com/moov-io/watchman/pkg/download" -) - -var ( - dplDownloadTemplate = func() string { - if w := os.Getenv("DPL_DOWNLOAD_TEMPLATE"); w != "" { - return w - } - return "https://www.bis.doc.gov/dpl/%s" // Denied Persons List (tab separated) - }() -) - -// Download returns an array of absolute filepaths for files downloaded -func Download(logger log.Logger, initialDir string) (map[string]io.ReadCloser, error) { - dl := download.New(logger, download.HTTPClient) - - addrs := make(map[string]string) - addrs["dpl.txt"] = fmt.Sprintf(dplDownloadTemplate, "dpl.txt") - - return dl.GetFiles(initialDir, addrs) -} diff --git a/pkg/dpl/download_test.go b/pkg/dpl/download_test.go deleted file mode 100644 index b20b9a31..00000000 --- a/pkg/dpl/download_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2020 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package dpl - -import ( - "io" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/moov-io/base/log" -) - -func TestDownloader(t *testing.T) { - if testing.Short() { - return - } - - file, err := Download(log.NewNopLogger(), "") - if err != nil { - t.Fatal(err) - } - if len(file) == 0 { - t.Fatal("no DPL file") - } - for filename := range file { - if !strings.EqualFold("dpl.txt", filepath.Base(filename)) { - t.Errorf("unknown file %s", file) - } - } -} - -func TestDownloader__initialDir(t *testing.T) { - dir, err := os.MkdirTemp("", "iniital-dir") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - mk := func(t *testing.T, name string, body string) { - path := filepath.Join(dir, name) - if err := os.WriteFile(path, []byte(body), 0600); err != nil { - t.Fatalf("writing %s: %v", path, err) - } - } - - // create each file - mk(t, "sdn.csv", "file=sdn.csv") - mk(t, "dpl.txt", "file=dpl.txt") - - file, err := Download(log.NewNopLogger(), dir) - if err != nil { - t.Fatal(err) - } - if len(file) == 0 { - t.Fatal("no DPL file") - } - for fn, fd := range file { - if strings.EqualFold("dpl.txt", filepath.Base(fn)) { - bs, err := io.ReadAll(fd) - if err != nil { - t.Fatal(err) - } - if v := string(bs); v != "file=dpl.txt" { - t.Errorf("dpl.txt: %v", v) - } - } else { - t.Fatalf("unknown file: %v", file) - } - } -} diff --git a/pkg/dpl/dpl.go b/pkg/dpl/dpl.go deleted file mode 100644 index e8b2827a..00000000 --- a/pkg/dpl/dpl.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2020 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package dpl - -// DPL is the BIS Denied Persons List -type DPL struct { - // Name is the name of the Denied Person - Name string `json:"name"` - // StreetAddress is the Denied Person's street address - StreetAddress string `json:"streetAddress"` - // City is the Denied Person's city - City string `json:"city"` - // State is the Denied Person's state - State string `json:"state"` - // Country is the Denied Person's country - Country string `json:"country"` - // PostalCode is the Denied Person's postal code - PostalCode string `json:"postalCode"` - // EffectiveDate is the date the denial came into effect - EffectiveDate string `json:"effectiveDate"` - // ExpirationDate is the date the denial expires. If blank, the denial has no expiration - ExpirationDate string `json:"expirationDate"` - // StandardOrder denotes whether or not the Person was added to the list by a "standard" order - StandardOrder string `json:"standardOrder"` - // LastUpdate is the date of the most recent change to the denial - LastUpdate string `json:"lastUpdate"` - // Action is the most recent action taken regarding the denial - Action string `json:"action"` - // FRCitation is the reference to the order's citation in the Federal Register - FRCitation string `json:"frCitation"` -} diff --git a/pkg/dpl/reader.go b/pkg/dpl/reader.go deleted file mode 100644 index 2db09f0c..00000000 --- a/pkg/dpl/reader.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2020 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package dpl - -import ( - "encoding/csv" - "errors" - "io" -) - -// Read parses DPL records from a TXT file and populates the associated arrays. -// -// For more details on the raw DPL files see https://moov-io.github.io/watchman/file-structure.html -func Read(f io.ReadCloser) ([]*DPL, error) { - if f == nil { - return nil, errors.New("DPL file is empty or missing") - } - defer f.Close() - - // create a new csv.Reader and set the delim char to txtDelim char - reader := csv.NewReader(f) - reader.Comma = '\t' - - // Loop through all lines we can - var out []*DPL - for { - line, err := reader.Read() - if err != nil { - // reached the last line - if errors.Is(err, io.EOF) { - break - } - // malformed row - if errors.Is(err, csv.ErrFieldCount) || - errors.Is(err, csv.ErrBareQuote) || - errors.Is(err, csv.ErrQuote) { - continue - } - return nil, err - } - - if len(line) < 12 || (len(line) >= 2 && line[1] == "Street_Address") { - continue // skip malformed headers - } - - deniedPerson := &DPL{ - Name: line[0], - StreetAddress: line[1], - City: line[2], - State: line[3], - Country: line[4], - PostalCode: line[5], - EffectiveDate: line[6], - ExpirationDate: line[7], - StandardOrder: line[8], - LastUpdate: line[9], - Action: line[10], - FRCitation: line[11], - } - out = append(out, deniedPerson) - } - return out, nil -} diff --git a/pkg/dpl/reader_test.go b/pkg/dpl/reader_test.go deleted file mode 100644 index da9fcc01..00000000 --- a/pkg/dpl/reader_test.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2020 The Moov Authors -// Use of this source code is governed by an Apache License -// license that can be found in the LICENSE file. - -package dpl - -import ( - "os" - "path/filepath" - "testing" -) - -func TestDPL__read(t *testing.T) { - fd, err := os.Open(filepath.Join("..", "..", "test", "testdata", "dpl.txt")) - if err != nil { - t.Error(err) - } - dpls, err := Read(fd) - if err != nil { - t.Fatal(err) - } - if len(dpls) != 546 { - t.Errorf("found %d DPL records", len(dpls)) - } - - // this file is formatted incorrectly for DPL, so we expect all rows to be skipped - fd, err = os.Open(filepath.Join("..", "..", "test", "testdata", "sdn.csv")) - if err != nil { - t.Error(err) - } - got, err := Read(fd) - if err != nil { - t.Fatal(err) - } - if len(got) != 0 { - t.Errorf("found %d DPL records, wanted 0", len(got)) - } -} diff --git a/pkg/ofac/download.go b/pkg/ofac/download.go index 4fa2eb73..a38be03c 100644 --- a/pkg/ofac/download.go +++ b/pkg/ofac/download.go @@ -5,6 +5,7 @@ package ofac import ( + "context" "fmt" "io" "os" @@ -29,7 +30,7 @@ var ( }() ) -func Download(logger log.Logger, initialDir string) (map[string]io.ReadCloser, error) { +func Download(ctx context.Context, logger log.Logger, initialDir string) (map[string]io.ReadCloser, error) { dl := download.New(logger, download.HTTPClient) addrs := make(map[string]string) @@ -37,5 +38,5 @@ func Download(logger log.Logger, initialDir string) (map[string]io.ReadCloser, e addrs[ofacFilenames[i]] = fmt.Sprintf(ofacURLTemplate, ofacFilenames[i]) } - return dl.GetFiles(initialDir, addrs) + return dl.GetFiles(ctx, initialDir, addrs) } diff --git a/pkg/ofac/download_test.go b/pkg/ofac/download_test.go index a3626a21..891440da 100644 --- a/pkg/ofac/download_test.go +++ b/pkg/ofac/download_test.go @@ -5,6 +5,7 @@ package ofac import ( + "context" "os" "path/filepath" "strings" @@ -19,7 +20,7 @@ func TestDownloader(t *testing.T) { return } - files, err := Download(log.NewNopLogger(), "") + files, err := Download(context.Background(), log.NewNopLogger(), "") require.NoError(t, err) require.Len(t, files, 4) @@ -52,7 +53,7 @@ func TestDownloader__initialDir(t *testing.T) { mk(t, "sdn.csv", "file=sdn.csv") mk(t, "dpl.txt", "file=dpl.txt") - files, err := Download(log.NewNopLogger(), dir) + files, err := Download(context.Background(), log.NewNopLogger(), dir) if err != nil { t.Fatal(err) } diff --git a/pkg/ofac/mapper_person_test.go b/pkg/ofac/mapper_person_test.go index fb1f854f..0946cca5 100644 --- a/pkg/ofac/mapper_person_test.go +++ b/pkg/ofac/mapper_person_test.go @@ -17,7 +17,7 @@ func TestMapper__Person(t *testing.T) { var sdn *SDN for i := range res.SDNs { if res.SDNs[i].EntityID == "15102" { - sdn = res.SDNs[i] + sdn = &res.SDNs[i] } } require.NotNil(t, sdn) diff --git a/pkg/ofac/mapper_vehicles_test.go b/pkg/ofac/mapper_vehicles_test.go index 1816ca1c..05113f3a 100644 --- a/pkg/ofac/mapper_vehicles_test.go +++ b/pkg/ofac/mapper_vehicles_test.go @@ -17,7 +17,7 @@ func TestMapper__Vessel(t *testing.T) { var sdn *SDN for i := range res.SDNs { if res.SDNs[i].EntityID == "15036" { - sdn = res.SDNs[i] + sdn = &res.SDNs[i] } } require.NotNil(t, sdn) @@ -50,7 +50,7 @@ func TestMapper__Aircraft(t *testing.T) { var sdn *SDN for i := range res.SDNs { if res.SDNs[i].EntityID == "18158" { - sdn = res.SDNs[i] + sdn = &res.SDNs[i] } } require.NotNil(t, sdn) diff --git a/pkg/ofac/reader.go b/pkg/ofac/reader.go index ac919d24..e31558e2 100644 --- a/pkg/ofac/reader.go +++ b/pkg/ofac/reader.go @@ -58,16 +58,16 @@ func Read(files map[string]io.ReadCloser) (*Results, error) { type Results struct { // Addresses returns an array of OFAC Specially Designated National Addresses - Addresses []*Address `json:"address"` + Addresses []Address `json:"address"` // AlternateIdentities returns an array of OFAC Specially Designated National Alternate Identity - AlternateIdentities []*AlternateIdentity `json:"alternateIdentity"` + AlternateIdentities []AlternateIdentity `json:"alternateIdentity"` // SDNs returns an array of OFAC Specially Designated Nationals - SDNs []*SDN `json:"sdn"` + SDNs []SDN `json:"sdn"` // SDNComments returns an array of OFAC Specially Designated National Comments - SDNComments []*SDNComments `json:"sdnComments"` + SDNComments []SDNComments `json:"sdnComments"` } func (r *Results) append(rr *Results, err error) error { @@ -83,7 +83,7 @@ func (r *Results) append(rr *Results, err error) error { func csvAddressFile(f io.ReadCloser) (*Results, error) { defer f.Close() - var out []*Address + var out []Address // Read File into a Variable reader := csv.NewReader(f) @@ -107,7 +107,7 @@ func csvAddressFile(f io.ReadCloser) (*Results, error) { } record = replaceNull(record) - out = append(out, &Address{ + out = append(out, Address{ EntityID: record[0], AddressID: record[1], Address: record[2], @@ -121,7 +121,7 @@ func csvAddressFile(f io.ReadCloser) (*Results, error) { func csvAlternateIdentityFile(f io.ReadCloser) (*Results, error) { defer f.Close() - var out []*AlternateIdentity + var out []AlternateIdentity // Read File into a Variable reader := csv.NewReader(f) @@ -144,7 +144,7 @@ func csvAlternateIdentityFile(f io.ReadCloser) (*Results, error) { continue } record = replaceNull(record) - out = append(out, &AlternateIdentity{ + out = append(out, AlternateIdentity{ EntityID: record[0], AlternateID: record[1], AlternateType: record[2], @@ -157,7 +157,7 @@ func csvAlternateIdentityFile(f io.ReadCloser) (*Results, error) { func csvSDNFile(f io.ReadCloser) (*Results, error) { defer f.Close() - var out []*SDN + var out []SDN // Read File into a Variable reader := csv.NewReader(f) @@ -180,7 +180,7 @@ func csvSDNFile(f io.ReadCloser) (*Results, error) { continue } record = replaceNull(record) - out = append(out, &SDN{ + out = append(out, SDN{ EntityID: record[0], SDNName: record[1], SDNType: record[2], @@ -205,7 +205,7 @@ func csvSDNCommentsFile(f io.ReadCloser) (*Results, error) { r.LazyQuotes = true // Loop through lines & turn into object - var out []*SDNComments + var out []SDNComments for { line, err := r.Read() if err != nil { @@ -225,7 +225,7 @@ func csvSDNCommentsFile(f io.ReadCloser) (*Results, error) { continue } line = replaceNull(line) - out = append(out, &SDNComments{ + out = append(out, SDNComments{ EntityID: line[0], RemarksExtended: line[1], DigitalCurrencyAddresses: readDigitalCurrencyAddresses(line[1]), @@ -380,7 +380,7 @@ func readDigitalCurrencyAddresses(remarks string) []DigitalCurrencyAddress { return out } -func mergeSpilloverRecords(sdns []*SDN, comments []*SDNComments) []*SDN { +func mergeSpilloverRecords(sdns []SDN, comments []SDNComments) []SDN { for i := range sdns { for j := range comments { if sdns[i].EntityID == comments[j].EntityID { From 3aeb64470b8dd6d3a9e0c9a2d0985f5c8a5e5ef6 Mon Sep 17 00:00:00 2001 From: Adam Shannon Date: Thu, 9 Jan 2025 15:53:50 -0600 Subject: [PATCH 4/9] chore: remove old api, admin, and client directories --- .gitleaksignore | 1 - admin/.gitignore | 24 - admin/.openapi-generator-ignore | 23 - admin/.openapi-generator/VERSION | 1 - admin/README.md | 61 - admin/api/openapi.yaml | 205 -- admin/api_admin.go | 274 -- admin/client.go | 545 ---- admin/configuration.go | 128 - admin/docs/AdminApi.md | 103 - admin/docs/DataRefresh.md | 17 - admin/docs/DebugSdn.md | 12 - admin/docs/Error.md | 11 - admin/docs/OfacSdn.md | 17 - admin/docs/SdnDebugMetadata.md | 12 - admin/docs/SdnType.md | 10 - admin/git_push.sh | 58 - admin/model_data_refresh.go | 31 - admin/model_debug_sdn.go | 16 - admin/model_error.go | 16 - admin/model_ofac_sdn.go | 23 - admin/model_sdn_debug_metadata.go | 18 - admin/model_sdn_type.go | 21 - admin/response.go | 46 - api/admin.yaml | 129 - api/client.yaml | 1257 --------- client/.gitignore | 24 - client/.openapi-generator-ignore | 23 - client/.openapi-generator/VERSION | 1 - client/README.md | 81 - client/api/openapi.yaml | 2292 ----------------- client/api_watchman.go | 752 ------ client/client.go | 545 ---- client/configuration.go | 128 - client/docs/BisEntities.md | 21 - client/docs/CaptaList.md | 23 - client/docs/Download.md | 17 - client/docs/Dpl.md | 24 - client/docs/Error.md | 11 - client/docs/EuConsolidatedSanctionsList.md | 29 - client/docs/ForeignSanctionsEvader.md | 23 - client/docs/ItarDebarred.md | 18 - client/docs/MilitaryEndUser.md | 18 - client/docs/NonProliferationSanction.md | 21 - .../NonSdnChineseMilitaryIndustrialComplex.md | 23 - client/docs/NonSdnMenuBasedSanctionsList.md | 22 - client/docs/OfacAlt.md | 17 - client/docs/OfacEntityAddress.md | 16 - client/docs/OfacSdn.md | 18 - client/docs/PalestinianLegislativeCouncil.md | 24 - client/docs/SdnType.md | 10 - client/docs/Search.md | 29 - client/docs/Ssi.md | 22 - client/docs/SsiType.md | 10 - client/docs/UkConsolidatedSanctionsList.md | 16 - client/docs/UkSanctionsList.md | 18 - client/docs/Unverified.md | 17 - client/docs/WatchmanApi.md | 320 --- client/git_push.sh | 58 - client/model_bis_entities.go | 36 - client/model_capta_list.go | 29 - client/model_download.go | 25 - client/model_dpl.go | 42 - client/model_error.go | 16 - .../model_eu_consolidated_sanctions_list.go | 35 - client/model_foreign_sanctions_evader.go | 29 - client/model_itar_debarred.go | 24 - client/model_military_end_user.go | 24 - client/model_non_proliferation_sanction.go | 27 - ...sdn_chinese_military_industrial_complex.go | 29 - ...model_non_sdn_menu_based_sanctions_list.go | 28 - client/model_ofac_alt.go | 23 - client/model_ofac_entity_address.go | 21 - client/model_ofac_sdn.go | 26 - .../model_palestinian_legislative_council.go | 30 - client/model_sdn_type.go | 21 - client/model_search.go | 37 - client/model_ssi.go | 37 - client/model_ssi_type.go | 19 - .../model_uk_consolidated_sanctions_list.go | 22 - client/model_uk_sanctions_list.go | 24 - client/model_unverified.go | 23 - client/response.go | 46 - 83 files changed, 8403 deletions(-) delete mode 100644 .gitleaksignore delete mode 100644 admin/.gitignore delete mode 100644 admin/.openapi-generator-ignore delete mode 100644 admin/.openapi-generator/VERSION delete mode 100644 admin/README.md delete mode 100644 admin/api/openapi.yaml delete mode 100644 admin/api_admin.go delete mode 100644 admin/client.go delete mode 100644 admin/configuration.go delete mode 100644 admin/docs/AdminApi.md delete mode 100644 admin/docs/DataRefresh.md delete mode 100644 admin/docs/DebugSdn.md delete mode 100644 admin/docs/Error.md delete mode 100644 admin/docs/OfacSdn.md delete mode 100644 admin/docs/SdnDebugMetadata.md delete mode 100644 admin/docs/SdnType.md delete mode 100644 admin/git_push.sh delete mode 100644 admin/model_data_refresh.go delete mode 100644 admin/model_debug_sdn.go delete mode 100644 admin/model_error.go delete mode 100644 admin/model_ofac_sdn.go delete mode 100644 admin/model_sdn_debug_metadata.go delete mode 100644 admin/model_sdn_type.go delete mode 100644 admin/response.go delete mode 100644 api/admin.yaml delete mode 100644 api/client.yaml delete mode 100644 client/.gitignore delete mode 100644 client/.openapi-generator-ignore delete mode 100644 client/.openapi-generator/VERSION delete mode 100644 client/README.md delete mode 100644 client/api/openapi.yaml delete mode 100644 client/api_watchman.go delete mode 100644 client/client.go delete mode 100644 client/configuration.go delete mode 100644 client/docs/BisEntities.md delete mode 100644 client/docs/CaptaList.md delete mode 100644 client/docs/Download.md delete mode 100644 client/docs/Dpl.md delete mode 100644 client/docs/Error.md delete mode 100644 client/docs/EuConsolidatedSanctionsList.md delete mode 100644 client/docs/ForeignSanctionsEvader.md delete mode 100644 client/docs/ItarDebarred.md delete mode 100644 client/docs/MilitaryEndUser.md delete mode 100644 client/docs/NonProliferationSanction.md delete mode 100644 client/docs/NonSdnChineseMilitaryIndustrialComplex.md delete mode 100644 client/docs/NonSdnMenuBasedSanctionsList.md delete mode 100644 client/docs/OfacAlt.md delete mode 100644 client/docs/OfacEntityAddress.md delete mode 100644 client/docs/OfacSdn.md delete mode 100644 client/docs/PalestinianLegislativeCouncil.md delete mode 100644 client/docs/SdnType.md delete mode 100644 client/docs/Search.md delete mode 100644 client/docs/Ssi.md delete mode 100644 client/docs/SsiType.md delete mode 100644 client/docs/UkConsolidatedSanctionsList.md delete mode 100644 client/docs/UkSanctionsList.md delete mode 100644 client/docs/Unverified.md delete mode 100644 client/docs/WatchmanApi.md delete mode 100644 client/git_push.sh delete mode 100644 client/model_bis_entities.go delete mode 100644 client/model_capta_list.go delete mode 100644 client/model_download.go delete mode 100644 client/model_dpl.go delete mode 100644 client/model_error.go delete mode 100644 client/model_eu_consolidated_sanctions_list.go delete mode 100644 client/model_foreign_sanctions_evader.go delete mode 100644 client/model_itar_debarred.go delete mode 100644 client/model_military_end_user.go delete mode 100644 client/model_non_proliferation_sanction.go delete mode 100644 client/model_non_sdn_chinese_military_industrial_complex.go delete mode 100644 client/model_non_sdn_menu_based_sanctions_list.go delete mode 100644 client/model_ofac_alt.go delete mode 100644 client/model_ofac_entity_address.go delete mode 100644 client/model_ofac_sdn.go delete mode 100644 client/model_palestinian_legislative_council.go delete mode 100644 client/model_sdn_type.go delete mode 100644 client/model_search.go delete mode 100644 client/model_ssi.go delete mode 100644 client/model_ssi_type.go delete mode 100644 client/model_uk_consolidated_sanctions_list.go delete mode 100644 client/model_uk_sanctions_list.go delete mode 100644 client/model_unverified.go delete mode 100644 client/response.go diff --git a/.gitleaksignore b/.gitleaksignore deleted file mode 100644 index 36792c6b..00000000 --- a/.gitleaksignore +++ /dev/null @@ -1 +0,0 @@ -client/api/openapi.yaml:generic-api-key:2923 \ No newline at end of file diff --git a/admin/.gitignore b/admin/.gitignore deleted file mode 100644 index daf913b1..00000000 --- a/admin/.gitignore +++ /dev/null @@ -1,24 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe -*.test -*.prof diff --git a/admin/.openapi-generator-ignore b/admin/.openapi-generator-ignore deleted file mode 100644 index 7484ee59..00000000 --- a/admin/.openapi-generator-ignore +++ /dev/null @@ -1,23 +0,0 @@ -# OpenAPI Generator Ignore -# Generated by openapi-generator https://github.com/openapitools/openapi-generator - -# Use this file to prevent files from being overwritten by the generator. -# The patterns follow closely to .gitignore or .dockerignore. - -# As an example, the C# client generator defines ApiClient.cs. -# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line: -#ApiClient.cs - -# You can match any string of characters against a directory, file or extension with a single asterisk (*): -#foo/*/qux -# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux - -# You can recursively match patterns against a directory, file or extension with a double asterisk (**): -#foo/**/qux -# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux - -# You can also negate patterns with an exclamation (!). -# For example, you can ignore all files in a docs folder with the file extension .md: -#docs/*.md -# Then explicitly reverse the ignore rule for a single file: -#!docs/README.md diff --git a/admin/.openapi-generator/VERSION b/admin/.openapi-generator/VERSION deleted file mode 100644 index ecedc98d..00000000 --- a/admin/.openapi-generator/VERSION +++ /dev/null @@ -1 +0,0 @@ -4.3.1 \ No newline at end of file diff --git a/admin/README.md b/admin/README.md deleted file mode 100644 index 9ae5fed1..00000000 --- a/admin/README.md +++ /dev/null @@ -1,61 +0,0 @@ -# Go API client for admin - -Watchman is an HTTP API and Go library to download, parse and offer search functions over numerous trade sanction lists from the United States, European Union governments, agencies, and non profits for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - - -## Overview -This API client was generated by the [OpenAPI Generator](https://openapi-generator.tech) project. By using the [OpenAPI-spec](https://www.openapis.org/) from a remote server, you can easily generate an API client. - -- API version: v1 -- Package version: 1.0.0 -- Build package: org.openapitools.codegen.languages.GoClientCodegen -For more information, please visit [https://github.com/moov-io/watchman](https://github.com/moov-io/watchman) - -## Installation - -Install the following dependencies: - -```shell -go get github.com/stretchr/testify/assert -go get golang.org/x/oauth2 -go get golang.org/x/net/context -go get github.com/antihax/optional -``` - -Put the package under your project folder and add the following in import: - -```golang -import "./admin" -``` - -## Documentation for API Endpoints - -All URIs are relative to *http://localhost:9094* - -Class | Method | HTTP request | Description ------------- | ------------- | ------------- | ------------- -*AdminApi* | [**DebugSDN**](docs/AdminApi.md#debugsdn) | **Get** /debug/sdn/{sdnId} | Debug SDN -*AdminApi* | [**GetVersion**](docs/AdminApi.md#getversion) | **Get** /version | Get Version -*AdminApi* | [**RefreshData**](docs/AdminApi.md#refreshdata) | **Post** /data/refresh | Download and reindex all data sources - - -## Documentation For Models - - - [DataRefresh](docs/DataRefresh.md) - - [DebugSdn](docs/DebugSdn.md) - - [Error](docs/Error.md) - - [OfacSdn](docs/OfacSdn.md) - - [SdnDebugMetadata](docs/SdnDebugMetadata.md) - - [SdnType](docs/SdnType.md) - - -## Documentation For Authorization - - Endpoints do not require authorization. - - - -## Author - - - diff --git a/admin/api/openapi.yaml b/admin/api/openapi.yaml deleted file mode 100644 index efc5811b..00000000 --- a/admin/api/openapi.yaml +++ /dev/null @@ -1,205 +0,0 @@ -openapi: 3.0.2 -info: - contact: - url: https://github.com/moov-io/watchman - description: | - Watchman is an HTTP API and Go library to download, parse and offer search functions over numerous trade sanction lists from the United States, European Union governments, agencies, and non profits for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - license: - name: Apache 2.0 - url: http://www.apache.org/licenses/LICENSE-2.0.html - title: Watchman Admin API - version: v1 -servers: -- description: Local development - url: http://localhost:9094 -tags: -- description: Watchman endpoints which are only meant to be exposed for admin dashboards - and operations. - name: Admin -paths: - /version: - get: - description: Show the current version of Watchman - operationId: getVersion - responses: - "200": - content: - text/plain: - schema: - example: v0.13.1 - type: string - description: The current version running - summary: Get Version - tags: - - Admin - /data/refresh: - post: - operationId: refreshData - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/DataRefresh' - description: Data successfully refreshed - "400": - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - description: See error message - summary: Download and reindex all data sources - tags: - - Admin - /debug/sdn/{sdnId}: - get: - description: Get an SDN and search index debug information - operationId: debugSDN - parameters: - - description: SDN ID - explode: false - in: path - name: sdnId - required: true - schema: - example: 564dd7d1 - type: string - style: simple - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/DebugSDN' - description: SDN with debug information - "400": - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - description: See error message - summary: Debug SDN - tags: - - Admin -components: - schemas: - SDNDebugMetadata: - example: - indexedName: mohammad moghisseh - parsedRemarksId: "3603251708570001" - properties: - indexedName: - description: Exact text stored in our index used for string ranking - example: mohammad moghisseh - type: string - parsedRemarksId: - description: ID parsed from remarks field - example: "3603251708570001" - type: string - DebugSDN: - example: - debug: - indexedName: mohammad moghisseh - parsedRemarksId: "3603251708570001" - SDN: - match: 0.91 - entityID: "1231" - programs: - - CUBA - sdnName: BANCO NACIONAL DE CUBA - title: Title of an individual - remarks: remarks - properties: - SDN: - $ref: '#/components/schemas/OfacSDN' - debug: - $ref: '#/components/schemas/SDNDebugMetadata' - DataRefresh: - example: - SDNs: 13131 - altNames: 322 - addresses: 4155 - deniedPersons: 5889 - bisEntities: 6831 - sectoralSanctions: 667 - timestamp: 2000-01-23T04:56:07.000+00:00 - properties: - SDNs: - description: Count of OFAC SDNs after index - example: 13131 - type: integer - altNames: - description: Count of OFAC alternate names after index - example: 322 - type: integer - addresses: - description: Count of OFAC Addresses after index - example: 4155 - type: integer - sectoralSanctions: - description: Count of SSI entities after index - example: 667 - type: integer - deniedPersons: - description: Count of BSL denied persons after index - example: 5889 - type: integer - bisEntities: - description: Count of BIS entities after index - example: 6831 - type: integer - timestamp: - format: date-time - type: string - Error: - properties: - error: - description: An error message describing the problem intended for humans. - example: Example error, see description - type: string - required: - - error - OfacSDN: - description: Specially designated national from OFAC list - example: - match: 0.91 - entityID: "1231" - programs: - - CUBA - sdnName: BANCO NACIONAL DE CUBA - title: Title of an individual - remarks: remarks - properties: - entityID: - example: "1231" - type: string - sdnName: - example: BANCO NACIONAL DE CUBA - type: string - sdnType: - $ref: '#/components/schemas/SdnType' - programs: - description: Programs is the sanction programs this SDN was added from - example: - - CUBA - items: - type: string - type: array - title: - example: Title of an individual - type: string - remarks: - type: string - match: - description: Remarks on SDN and often additional information about the SDN - example: 0.91 - type: number - SdnType: - description: Used for classifying SDNs — typically represents an individual - or company - enum: - - individual - - entity - - vessel - - aircraft - type: string diff --git a/admin/api_admin.go b/admin/api_admin.go deleted file mode 100644 index 5bde3376..00000000 --- a/admin/api_admin.go +++ /dev/null @@ -1,274 +0,0 @@ -/* - * Watchman Admin API - * - * Watchman is an HTTP API and Go library to download, parse and offer search functions over numerous trade sanction lists from the United States, European Union governments, agencies, and non profits for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package admin - -import ( - _context "context" - _ioutil "io/ioutil" - _nethttp "net/http" - _neturl "net/url" - "strings" -) - -// Linger please -var ( - _ _context.Context -) - -// AdminApiService AdminApi service -type AdminApiService service - -/* -DebugSDN Debug SDN -Get an SDN and search index debug information - - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). - - @param sdnId SDN ID - -@return DebugSdn -*/ -func (a *AdminApiService) DebugSDN(ctx _context.Context, sdnId string) (DebugSdn, *_nethttp.Response, error) { - var ( - localVarHTTPMethod = _nethttp.MethodGet - localVarPostBody interface{} - localVarFormFileName string - localVarFileName string - localVarFileBytes []byte - localVarReturnValue DebugSdn - ) - - // create path and map variables - localVarPath := a.client.cfg.BasePath + "/debug/sdn/{sdnId}" - localVarPath = strings.Replace(localVarPath, "{"+"sdnId"+"}", _neturl.QueryEscape(parameterToString(sdnId, "")), -1) - - localVarHeaderParams := make(map[string]string) - localVarQueryParams := _neturl.Values{} - localVarFormParams := _neturl.Values{} - - // to determine the Content-Type header - localVarHTTPContentTypes := []string{} - - // set Content-Type header - localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) - if localVarHTTPContentType != "" { - localVarHeaderParams["Content-Type"] = localVarHTTPContentType - } - - // to determine the Accept header - localVarHTTPHeaderAccepts := []string{"application/json"} - - // set Accept header - localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) - if localVarHTTPHeaderAccept != "" { - localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept - } - r, err := a.client.prepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes) - if err != nil { - return localVarReturnValue, nil, err - } - - localVarHTTPResponse, err := a.client.callAPI(r) - if err != nil || localVarHTTPResponse == nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) - localVarHTTPResponse.Body.Close() - if err != nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - if localVarHTTPResponse.StatusCode >= 300 { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: localVarHTTPResponse.Status, - } - if localVarHTTPResponse.StatusCode == 400 { - var v Error - err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) - if err != nil { - newErr.error = err.Error() - return localVarReturnValue, localVarHTTPResponse, newErr - } - newErr.model = v - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) - if err != nil { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: err.Error(), - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - return localVarReturnValue, localVarHTTPResponse, nil -} - -/* -GetVersion Get Version -Show the current version of Watchman - - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). - -@return string -*/ -func (a *AdminApiService) GetVersion(ctx _context.Context) (string, *_nethttp.Response, error) { - var ( - localVarHTTPMethod = _nethttp.MethodGet - localVarPostBody interface{} - localVarFormFileName string - localVarFileName string - localVarFileBytes []byte - localVarReturnValue string - ) - - // create path and map variables - localVarPath := a.client.cfg.BasePath + "/version" - localVarHeaderParams := make(map[string]string) - localVarQueryParams := _neturl.Values{} - localVarFormParams := _neturl.Values{} - - // to determine the Content-Type header - localVarHTTPContentTypes := []string{} - - // set Content-Type header - localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) - if localVarHTTPContentType != "" { - localVarHeaderParams["Content-Type"] = localVarHTTPContentType - } - - // to determine the Accept header - localVarHTTPHeaderAccepts := []string{"text/plain"} - - // set Accept header - localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) - if localVarHTTPHeaderAccept != "" { - localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept - } - r, err := a.client.prepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes) - if err != nil { - return localVarReturnValue, nil, err - } - - localVarHTTPResponse, err := a.client.callAPI(r) - if err != nil || localVarHTTPResponse == nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) - localVarHTTPResponse.Body.Close() - if err != nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - if localVarHTTPResponse.StatusCode >= 300 { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: localVarHTTPResponse.Status, - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) - if err != nil { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: err.Error(), - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - return localVarReturnValue, localVarHTTPResponse, nil -} - -/* -RefreshData Download and reindex all data sources - - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). - -@return DataRefresh -*/ -func (a *AdminApiService) RefreshData(ctx _context.Context) (DataRefresh, *_nethttp.Response, error) { - var ( - localVarHTTPMethod = _nethttp.MethodPost - localVarPostBody interface{} - localVarFormFileName string - localVarFileName string - localVarFileBytes []byte - localVarReturnValue DataRefresh - ) - - // create path and map variables - localVarPath := a.client.cfg.BasePath + "/data/refresh" - localVarHeaderParams := make(map[string]string) - localVarQueryParams := _neturl.Values{} - localVarFormParams := _neturl.Values{} - - // to determine the Content-Type header - localVarHTTPContentTypes := []string{} - - // set Content-Type header - localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) - if localVarHTTPContentType != "" { - localVarHeaderParams["Content-Type"] = localVarHTTPContentType - } - - // to determine the Accept header - localVarHTTPHeaderAccepts := []string{"application/json"} - - // set Accept header - localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) - if localVarHTTPHeaderAccept != "" { - localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept - } - r, err := a.client.prepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes) - if err != nil { - return localVarReturnValue, nil, err - } - - localVarHTTPResponse, err := a.client.callAPI(r) - if err != nil || localVarHTTPResponse == nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) - localVarHTTPResponse.Body.Close() - if err != nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - if localVarHTTPResponse.StatusCode >= 300 { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: localVarHTTPResponse.Status, - } - if localVarHTTPResponse.StatusCode == 400 { - var v Error - err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) - if err != nil { - newErr.error = err.Error() - return localVarReturnValue, localVarHTTPResponse, newErr - } - newErr.model = v - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) - if err != nil { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: err.Error(), - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - return localVarReturnValue, localVarHTTPResponse, nil -} diff --git a/admin/client.go b/admin/client.go deleted file mode 100644 index f6aacfe4..00000000 --- a/admin/client.go +++ /dev/null @@ -1,545 +0,0 @@ -/* - * Watchman Admin API - * - * Watchman is an HTTP API and Go library to download, parse and offer search functions over numerous trade sanction lists from the United States, European Union governments, agencies, and non profits for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package admin - -import ( - "bytes" - "context" - "encoding/json" - "encoding/xml" - "errors" - "fmt" - "io" - "io/ioutil" - "log" - "mime/multipart" - "net/http" - "net/http/httputil" - "net/url" - "os" - "path/filepath" - "reflect" - "regexp" - "strconv" - "strings" - "time" - "unicode/utf8" - - "golang.org/x/oauth2" -) - -var ( - jsonCheck = regexp.MustCompile(`(?i:(?:application|text)/(?:vnd\.[^;]+\+)?json)`) - xmlCheck = regexp.MustCompile(`(?i:(?:application|text)/xml)`) -) - -// APIClient manages communication with the Watchman Admin API API vv1 -// In most cases there should be only one, shared, APIClient. -type APIClient struct { - cfg *Configuration - common service // Reuse a single struct instead of allocating one for each service on the heap. - - // API Services - - AdminApi *AdminApiService -} - -type service struct { - client *APIClient -} - -// NewAPIClient creates a new API client. Requires a userAgent string describing your application. -// optionally a custom http.Client to allow for advanced features such as caching. -func NewAPIClient(cfg *Configuration) *APIClient { - if cfg.HTTPClient == nil { - cfg.HTTPClient = http.DefaultClient - } - - c := &APIClient{} - c.cfg = cfg - c.common.client = c - - // API Services - c.AdminApi = (*AdminApiService)(&c.common) - - return c -} - -func atoi(in string) (int, error) { - return strconv.Atoi(in) -} - -// selectHeaderContentType select a content type from the available list. -func selectHeaderContentType(contentTypes []string) string { - if len(contentTypes) == 0 { - return "" - } - if contains(contentTypes, "application/json") { - return "application/json" - } - return contentTypes[0] // use the first content type specified in 'consumes' -} - -// selectHeaderAccept join all accept types and return -func selectHeaderAccept(accepts []string) string { - if len(accepts) == 0 { - return "" - } - - if contains(accepts, "application/json") { - return "application/json" - } - - return strings.Join(accepts, ",") -} - -// contains is a case insenstive match, finding needle in a haystack -func contains(haystack []string, needle string) bool { - for _, a := range haystack { - if strings.ToLower(a) == strings.ToLower(needle) { - return true - } - } - return false -} - -// Verify optional parameters are of the correct type. -func typeCheckParameter(obj interface{}, expected string, name string) error { - // Make sure there is an object. - if obj == nil { - return nil - } - - // Check the type is as expected. - if reflect.TypeOf(obj).String() != expected { - return fmt.Errorf("Expected %s to be of type %s but received %s.", name, expected, reflect.TypeOf(obj).String()) - } - return nil -} - -// parameterToString convert interface{} parameters to string, using a delimiter if format is provided. -func parameterToString(obj interface{}, collectionFormat string) string { - var delimiter string - - switch collectionFormat { - case "pipes": - delimiter = "|" - case "ssv": - delimiter = " " - case "tsv": - delimiter = "\t" - case "csv": - delimiter = "," - } - - if reflect.TypeOf(obj).Kind() == reflect.Slice { - return strings.Trim(strings.Replace(fmt.Sprint(obj), " ", delimiter, -1), "[]") - } else if t, ok := obj.(time.Time); ok { - return t.Format(time.RFC3339) - } - - return fmt.Sprintf("%v", obj) -} - -// helper for converting interface{} parameters to json strings -func parameterToJson(obj interface{}) (string, error) { - jsonBuf, err := json.Marshal(obj) - if err != nil { - return "", err - } - return string(jsonBuf), err -} - -// callAPI do the request. -func (c *APIClient) callAPI(request *http.Request) (*http.Response, error) { - if c.cfg.Debug { - dump, err := httputil.DumpRequestOut(request, true) - if err != nil { - return nil, err - } - log.Printf("\n%s\n", string(dump)) - } - - resp, err := c.cfg.HTTPClient.Do(request) - if err != nil { - return resp, err - } - - if c.cfg.Debug { - dump, err := httputil.DumpResponse(resp, true) - if err != nil { - return resp, err - } - log.Printf("\n%s\n", string(dump)) - } - - return resp, err -} - -// ChangeBasePath changes base path to allow switching to mocks -func (c *APIClient) ChangeBasePath(path string) { - c.cfg.BasePath = path -} - -// Allow modification of underlying config for alternate implementations and testing -// Caution: modifying the configuration while live can cause data races and potentially unwanted behavior -func (c *APIClient) GetConfig() *Configuration { - return c.cfg -} - -// prepareRequest build the request -func (c *APIClient) prepareRequest( - ctx context.Context, - path string, method string, - postBody interface{}, - headerParams map[string]string, - queryParams url.Values, - formParams url.Values, - formFileName string, - fileName string, - fileBytes []byte) (localVarRequest *http.Request, err error) { - - var body *bytes.Buffer - - // Detect postBody type and post. - if postBody != nil { - contentType := headerParams["Content-Type"] - if contentType == "" { - contentType = detectContentType(postBody) - headerParams["Content-Type"] = contentType - } - - body, err = setBody(postBody, contentType) - if err != nil { - return nil, err - } - } - - // add form parameters and file if available. - if strings.HasPrefix(headerParams["Content-Type"], "multipart/form-data") && len(formParams) > 0 || (len(fileBytes) > 0 && fileName != "") { - if body != nil { - return nil, errors.New("Cannot specify postBody and multipart form at the same time.") - } - body = &bytes.Buffer{} - w := multipart.NewWriter(body) - - for k, v := range formParams { - for _, iv := range v { - if strings.HasPrefix(k, "@") { // file - err = addFile(w, k[1:], iv) - if err != nil { - return nil, err - } - } else { // form value - w.WriteField(k, iv) - } - } - } - if len(fileBytes) > 0 && fileName != "" { - w.Boundary() - //_, fileNm := filepath.Split(fileName) - part, err := w.CreateFormFile(formFileName, filepath.Base(fileName)) - if err != nil { - return nil, err - } - _, err = part.Write(fileBytes) - if err != nil { - return nil, err - } - } - - // Set the Boundary in the Content-Type - headerParams["Content-Type"] = w.FormDataContentType() - - // Set Content-Length - headerParams["Content-Length"] = fmt.Sprintf("%d", body.Len()) - w.Close() - } - - if strings.HasPrefix(headerParams["Content-Type"], "application/x-www-form-urlencoded") && len(formParams) > 0 { - if body != nil { - return nil, errors.New("Cannot specify postBody and x-www-form-urlencoded form at the same time.") - } - body = &bytes.Buffer{} - body.WriteString(formParams.Encode()) - // Set Content-Length - headerParams["Content-Length"] = fmt.Sprintf("%d", body.Len()) - } - - // Setup path and query parameters - url, err := url.Parse(path) - if err != nil { - return nil, err - } - - // Override request host, if applicable - if c.cfg.Host != "" { - url.Host = c.cfg.Host - } - - // Override request scheme, if applicable - if c.cfg.Scheme != "" { - url.Scheme = c.cfg.Scheme - } - - // Adding Query Param - query := url.Query() - for k, v := range queryParams { - for _, iv := range v { - query.Add(k, iv) - } - } - - // Encode the parameters. - url.RawQuery = query.Encode() - - // Generate a new request - if body != nil { - localVarRequest, err = http.NewRequest(method, url.String(), body) - } else { - localVarRequest, err = http.NewRequest(method, url.String(), nil) - } - if err != nil { - return nil, err - } - - // add header parameters, if any - if len(headerParams) > 0 { - headers := http.Header{} - for h, v := range headerParams { - headers.Set(h, v) - } - localVarRequest.Header = headers - } - - // Add the user agent to the request. - localVarRequest.Header.Add("User-Agent", c.cfg.UserAgent) - - if ctx != nil { - // add context to the request - localVarRequest = localVarRequest.WithContext(ctx) - - // Walk through any authentication. - - // OAuth2 authentication - if tok, ok := ctx.Value(ContextOAuth2).(oauth2.TokenSource); ok { - // We were able to grab an oauth2 token from the context - var latestToken *oauth2.Token - if latestToken, err = tok.Token(); err != nil { - return nil, err - } - - latestToken.SetAuthHeader(localVarRequest) - } - - // Basic HTTP Authentication - if auth, ok := ctx.Value(ContextBasicAuth).(BasicAuth); ok { - localVarRequest.SetBasicAuth(auth.UserName, auth.Password) - } - - // AccessToken Authentication - if auth, ok := ctx.Value(ContextAccessToken).(string); ok { - localVarRequest.Header.Add("Authorization", "Bearer "+auth) - } - - } - - for header, value := range c.cfg.DefaultHeader { - localVarRequest.Header.Add(header, value) - } - - return localVarRequest, nil -} - -func (c *APIClient) decode(v interface{}, b []byte, contentType string) (err error) { - if len(b) == 0 { - return nil - } - if s, ok := v.(*string); ok { - *s = string(b) - return nil - } - if f, ok := v.(**os.File); ok { - *f, err = ioutil.TempFile("", "HttpClientFile") - if err != nil { - return - } - _, err = (*f).Write(b) - _, err = (*f).Seek(0, io.SeekStart) - return - } - if xmlCheck.MatchString(contentType) { - if err = xml.Unmarshal(b, v); err != nil { - return err - } - return nil - } - if jsonCheck.MatchString(contentType) { - if err = json.Unmarshal(b, v); err != nil { - return err - } - return nil - } - return errors.New("undefined response type") -} - -// Add a file to the multipart request -func addFile(w *multipart.Writer, fieldName, path string) error { - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - - part, err := w.CreateFormFile(fieldName, filepath.Base(path)) - if err != nil { - return err - } - _, err = io.Copy(part, file) - - return err -} - -// Prevent trying to import "fmt" -func reportError(format string, a ...interface{}) error { - return fmt.Errorf(format, a...) -} - -// Set request body from an interface{} -func setBody(body interface{}, contentType string) (bodyBuf *bytes.Buffer, err error) { - if bodyBuf == nil { - bodyBuf = &bytes.Buffer{} - } - - if reader, ok := body.(io.Reader); ok { - _, err = bodyBuf.ReadFrom(reader) - } else if b, ok := body.([]byte); ok { - _, err = bodyBuf.Write(b) - } else if s, ok := body.(string); ok { - _, err = bodyBuf.WriteString(s) - } else if s, ok := body.(*string); ok { - _, err = bodyBuf.WriteString(*s) - } else if jsonCheck.MatchString(contentType) { - err = json.NewEncoder(bodyBuf).Encode(body) - } else if xmlCheck.MatchString(contentType) { - var bs []byte - bs, err = xml.Marshal(body) - if err == nil { - bodyBuf.Write(bs) - } - } - - if err != nil { - return nil, err - } - - if bodyBuf.Len() == 0 { - err = fmt.Errorf("Invalid body type %s\n", contentType) - return nil, err - } - return bodyBuf, nil -} - -// detectContentType method is used to figure out `Request.Body` content type for request header -func detectContentType(body interface{}) string { - contentType := "text/plain; charset=utf-8" - kind := reflect.TypeOf(body).Kind() - - switch kind { - case reflect.Struct, reflect.Map, reflect.Ptr: - contentType = "application/json; charset=utf-8" - case reflect.String: - contentType = "text/plain; charset=utf-8" - default: - if b, ok := body.([]byte); ok { - contentType = http.DetectContentType(b) - } else if kind == reflect.Slice { - contentType = "application/json; charset=utf-8" - } - } - - return contentType -} - -// Ripped from https://github.com/gregjones/httpcache/blob/master/httpcache.go -type cacheControl map[string]string - -func parseCacheControl(headers http.Header) cacheControl { - cc := cacheControl{} - ccHeader := headers.Get("Cache-Control") - for _, part := range strings.Split(ccHeader, ",") { - part = strings.Trim(part, " ") - if part == "" { - continue - } - if strings.ContainsRune(part, '=') { - keyval := strings.Split(part, "=") - cc[strings.Trim(keyval[0], " ")] = strings.Trim(keyval[1], ",") - } else { - cc[part] = "" - } - } - return cc -} - -// CacheExpires helper function to determine remaining time before repeating a request. -func CacheExpires(r *http.Response) time.Time { - // Figure out when the cache expires. - var expires time.Time - now, err := time.Parse(time.RFC1123, r.Header.Get("date")) - if err != nil { - return time.Now() - } - respCacheControl := parseCacheControl(r.Header) - - if maxAge, ok := respCacheControl["max-age"]; ok { - lifetime, err := time.ParseDuration(maxAge + "s") - if err != nil { - expires = now - } else { - expires = now.Add(lifetime) - } - } else { - expiresHeader := r.Header.Get("Expires") - if expiresHeader != "" { - expires, err = time.Parse(time.RFC1123, expiresHeader) - if err != nil { - expires = now - } - } - } - return expires -} - -func strlen(s string) int { - return utf8.RuneCountInString(s) -} - -// GenericOpenAPIError Provides access to the body, error and model on returned errors. -type GenericOpenAPIError struct { - body []byte - error string - model interface{} -} - -// Error returns non-empty string if there was an error. -func (e GenericOpenAPIError) Error() string { - return e.error -} - -// Body returns the raw bytes of the response -func (e GenericOpenAPIError) Body() []byte { - return e.body -} - -// Model returns the unpacked model of the error -func (e GenericOpenAPIError) Model() interface{} { - return e.model -} diff --git a/admin/configuration.go b/admin/configuration.go deleted file mode 100644 index 59fb17e2..00000000 --- a/admin/configuration.go +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Watchman Admin API - * - * Watchman is an HTTP API and Go library to download, parse and offer search functions over numerous trade sanction lists from the United States, European Union governments, agencies, and non profits for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package admin - -import ( - "fmt" - "net/http" - "strings" -) - -// contextKeys are used to identify the type of value in the context. -// Since these are string, it is possible to get a short description of the -// context key for logging and debugging using key.String(). - -type contextKey string - -func (c contextKey) String() string { - return "auth " + string(c) -} - -var ( - // ContextOAuth2 takes an oauth2.TokenSource as authentication for the request. - ContextOAuth2 = contextKey("token") - - // ContextBasicAuth takes BasicAuth as authentication for the request. - ContextBasicAuth = contextKey("basic") - - // ContextAccessToken takes a string oauth2 access token as authentication for the request. - ContextAccessToken = contextKey("accesstoken") - - // ContextAPIKey takes an APIKey as authentication for the request - ContextAPIKey = contextKey("apikey") -) - -// BasicAuth provides basic http authentication to a request passed via context using ContextBasicAuth -type BasicAuth struct { - UserName string `json:"userName,omitempty"` - Password string `json:"password,omitempty"` -} - -// APIKey provides API key based authentication to a request passed via context using ContextAPIKey -type APIKey struct { - Key string - Prefix string -} - -// ServerVariable stores the information about a server variable -type ServerVariable struct { - Description string - DefaultValue string - EnumValues []string -} - -// ServerConfiguration stores the information about a server -type ServerConfiguration struct { - Url string - Description string - Variables map[string]ServerVariable -} - -// Configuration stores the configuration of the API client -type Configuration struct { - BasePath string `json:"basePath,omitempty"` - Host string `json:"host,omitempty"` - Scheme string `json:"scheme,omitempty"` - DefaultHeader map[string]string `json:"defaultHeader,omitempty"` - UserAgent string `json:"userAgent,omitempty"` - Debug bool `json:"debug,omitempty"` - Servers []ServerConfiguration - HTTPClient *http.Client -} - -// NewConfiguration returns a new Configuration object -func NewConfiguration() *Configuration { - cfg := &Configuration{ - BasePath: "http://localhost:9094", - DefaultHeader: make(map[string]string), - UserAgent: "OpenAPI-Generator/1.0.0/go", - Debug: false, - Servers: []ServerConfiguration{ - { - Url: "http://localhost:9094", - Description: "Local development", - }, - }, - } - return cfg -} - -// AddDefaultHeader adds a new HTTP header to the default header in the request -func (c *Configuration) AddDefaultHeader(key string, value string) { - c.DefaultHeader[key] = value -} - -// ServerUrl returns URL based on server settings -func (c *Configuration) ServerUrl(index int, variables map[string]string) (string, error) { - if index < 0 || len(c.Servers) <= index { - return "", fmt.Errorf("Index %v out of range %v", index, len(c.Servers)-1) - } - server := c.Servers[index] - url := server.Url - - // go through variables and replace placeholders - for name, variable := range server.Variables { - if value, ok := variables[name]; ok { - found := bool(len(variable.EnumValues) == 0) - for _, enumValue := range variable.EnumValues { - if value == enumValue { - found = true - } - } - if !found { - return "", fmt.Errorf("The variable %s in the server URL has invalid value %v. Must be %v", name, value, variable.EnumValues) - } - url = strings.Replace(url, "{"+name+"}", value, -1) - } else { - url = strings.Replace(url, "{"+name+"}", variable.DefaultValue, -1) - } - } - return url, nil -} diff --git a/admin/docs/AdminApi.md b/admin/docs/AdminApi.md deleted file mode 100644 index 994030da..00000000 --- a/admin/docs/AdminApi.md +++ /dev/null @@ -1,103 +0,0 @@ -# \AdminApi - -All URIs are relative to *http://localhost:9094* - -Method | HTTP request | Description -------------- | ------------- | ------------- -[**DebugSDN**](AdminApi.md#DebugSDN) | **Get** /debug/sdn/{sdnId} | Debug SDN -[**GetVersion**](AdminApi.md#GetVersion) | **Get** /version | Get Version -[**RefreshData**](AdminApi.md#RefreshData) | **Post** /data/refresh | Download and reindex all data sources - - - -## DebugSDN - -> DebugSdn DebugSDN(ctx, sdnId) - -Debug SDN - -Get an SDN and search index debug information - -### Required Parameters - - -Name | Type | Description | Notes -------------- | ------------- | ------------- | ------------- -**ctx** | **context.Context** | context for authentication, logging, cancellation, deadlines, tracing, etc. -**sdnId** | **string**| SDN ID | - -### Return type - -[**DebugSdn**](DebugSDN.md) - -### Authorization - -No authorization required - -### HTTP request headers - -- **Content-Type**: Not defined -- **Accept**: application/json - -[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) -[[Back to Model list]](../README.md#documentation-for-models) -[[Back to README]](../README.md) - - -## GetVersion - -> string GetVersion(ctx, ) - -Get Version - -Show the current version of Watchman - -### Required Parameters - -This endpoint does not need any parameter. - -### Return type - -**string** - -### Authorization - -No authorization required - -### HTTP request headers - -- **Content-Type**: Not defined -- **Accept**: text/plain - -[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) -[[Back to Model list]](../README.md#documentation-for-models) -[[Back to README]](../README.md) - - -## RefreshData - -> DataRefresh RefreshData(ctx, ) - -Download and reindex all data sources - -### Required Parameters - -This endpoint does not need any parameter. - -### Return type - -[**DataRefresh**](DataRefresh.md) - -### Authorization - -No authorization required - -### HTTP request headers - -- **Content-Type**: Not defined -- **Accept**: application/json - -[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) -[[Back to Model list]](../README.md#documentation-for-models) -[[Back to README]](../README.md) - diff --git a/admin/docs/DataRefresh.md b/admin/docs/DataRefresh.md deleted file mode 100644 index 1de7d2a0..00000000 --- a/admin/docs/DataRefresh.md +++ /dev/null @@ -1,17 +0,0 @@ -# DataRefresh - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**SDNs** | **int32** | Count of OFAC SDNs after index | [optional] -**AltNames** | **int32** | Count of OFAC alternate names after index | [optional] -**Addresses** | **int32** | Count of OFAC Addresses after index | [optional] -**SectoralSanctions** | **int32** | Count of SSI entities after index | [optional] -**DeniedPersons** | **int32** | Count of BSL denied persons after index | [optional] -**BisEntities** | **int32** | Count of BIS entities after index | [optional] -**Timestamp** | [**time.Time**](time.Time.md) | | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/admin/docs/DebugSdn.md b/admin/docs/DebugSdn.md deleted file mode 100644 index 0c7d5c11..00000000 --- a/admin/docs/DebugSdn.md +++ /dev/null @@ -1,12 +0,0 @@ -# DebugSdn - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**SDN** | [**OfacSdn**](OfacSDN.md) | | [optional] -**Debug** | [**SdnDebugMetadata**](SDNDebugMetadata.md) | | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/admin/docs/Error.md b/admin/docs/Error.md deleted file mode 100644 index fc134a35..00000000 --- a/admin/docs/Error.md +++ /dev/null @@ -1,11 +0,0 @@ -# Error - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**Error** | **string** | An error message describing the problem intended for humans. | - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/admin/docs/OfacSdn.md b/admin/docs/OfacSdn.md deleted file mode 100644 index 3f3afdb5..00000000 --- a/admin/docs/OfacSdn.md +++ /dev/null @@ -1,17 +0,0 @@ -# OfacSdn - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**EntityID** | **string** | | [optional] -**SdnName** | **string** | | [optional] -**SdnType** | [**SdnType**](SdnType.md) | | [optional] -**Programs** | **[]string** | Programs is the sanction programs this SDN was added from | [optional] -**Title** | **string** | | [optional] -**Remarks** | **string** | | [optional] -**Match** | **float32** | Remarks on SDN and often additional information about the SDN | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/admin/docs/SdnDebugMetadata.md b/admin/docs/SdnDebugMetadata.md deleted file mode 100644 index e0bbad6f..00000000 --- a/admin/docs/SdnDebugMetadata.md +++ /dev/null @@ -1,12 +0,0 @@ -# SdnDebugMetadata - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**IndexedName** | **string** | Exact text stored in our index used for string ranking | [optional] -**ParsedRemarksId** | **string** | ID parsed from remarks field | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/admin/docs/SdnType.md b/admin/docs/SdnType.md deleted file mode 100644 index 10e28810..00000000 --- a/admin/docs/SdnType.md +++ /dev/null @@ -1,10 +0,0 @@ -# SdnType - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/admin/git_push.sh b/admin/git_push.sh deleted file mode 100644 index bc93d187..00000000 --- a/admin/git_push.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -# ref: https://help.github.com/articles/adding-an-existing-project-to-github-using-the-command-line/ -# -# Usage example: /bin/sh ./git_push.sh wing328 openapi-pestore-perl "minor update" "gitlab.com" - -git_user_id=$1 -git_repo_id=$2 -release_note=$3 -git_host=$4 - -if [ "$git_host" = "" ]; then - git_host="github.com" - echo "[INFO] No command line input provided. Set \$git_host to $git_host" -fi - -if [ "$git_user_id" = "" ]; then - git_user_id="moov-io" - echo "[INFO] No command line input provided. Set \$git_user_id to $git_user_id" -fi - -if [ "$git_repo_id" = "" ]; then - git_repo_id="watchman" - echo "[INFO] No command line input provided. Set \$git_repo_id to $git_repo_id" -fi - -if [ "$release_note" = "" ]; then - release_note="Minor update" - echo "[INFO] No command line input provided. Set \$release_note to $release_note" -fi - -# Initialize the local directory as a Git repository -git init - -# Adds the files in the local repository and stages them for commit. -git add . - -# Commits the tracked changes and prepares them to be pushed to a remote repository. -git commit -m "$release_note" - -# Sets the new remote -git_remote=`git remote` -if [ "$git_remote" = "" ]; then # git remote not defined - - if [ "$GIT_TOKEN" = "" ]; then - echo "[INFO] \$GIT_TOKEN (environment variable) is not set. Using the git credential in your environment." - git remote add origin https://${git_host}/${git_user_id}/${git_repo_id}.git - else - git remote add origin https://${git_user_id}:${GIT_TOKEN}@${git_host}/${git_user_id}/${git_repo_id}.git - fi - -fi - -git pull origin master - -# Pushes (Forces) the changes in the local repository up to the remote repository -echo "Git pushing to https://${git_host}/${git_user_id}/${git_repo_id}.git" -git push origin master 2>&1 | grep -v 'To https' - diff --git a/admin/model_data_refresh.go b/admin/model_data_refresh.go deleted file mode 100644 index a75ca915..00000000 --- a/admin/model_data_refresh.go +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Watchman Admin API - * - * Watchman is an HTTP API and Go library to download, parse and offer search functions over numerous trade sanction lists from the United States, European Union governments, agencies, and non profits for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package admin - -import ( - "time" -) - -// DataRefresh struct for DataRefresh -type DataRefresh struct { - // Count of OFAC SDNs after index - SDNs int32 `json:"SDNs,omitempty"` - // Count of OFAC alternate names after index - AltNames int32 `json:"altNames,omitempty"` - // Count of OFAC Addresses after index - Addresses int32 `json:"addresses,omitempty"` - // Count of SSI entities after index - SectoralSanctions int32 `json:"sectoralSanctions,omitempty"` - // Count of BSL denied persons after index - DeniedPersons int32 `json:"deniedPersons,omitempty"` - // Count of BIS entities after index - BisEntities int32 `json:"bisEntities,omitempty"` - Timestamp time.Time `json:"timestamp,omitempty"` -} diff --git a/admin/model_debug_sdn.go b/admin/model_debug_sdn.go deleted file mode 100644 index e3871796..00000000 --- a/admin/model_debug_sdn.go +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Watchman Admin API - * - * Watchman is an HTTP API and Go library to download, parse and offer search functions over numerous trade sanction lists from the United States, European Union governments, agencies, and non profits for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package admin - -// DebugSdn struct for DebugSdn -type DebugSdn struct { - SDN OfacSdn `json:"SDN,omitempty"` - Debug SdnDebugMetadata `json:"debug,omitempty"` -} diff --git a/admin/model_error.go b/admin/model_error.go deleted file mode 100644 index e6405553..00000000 --- a/admin/model_error.go +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Watchman Admin API - * - * Watchman is an HTTP API and Go library to download, parse and offer search functions over numerous trade sanction lists from the United States, European Union governments, agencies, and non profits for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package admin - -// Error struct for Error -type Error struct { - // An error message describing the problem intended for humans. - Error string `json:"error"` -} diff --git a/admin/model_ofac_sdn.go b/admin/model_ofac_sdn.go deleted file mode 100644 index dcdc3ed5..00000000 --- a/admin/model_ofac_sdn.go +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Watchman Admin API - * - * Watchman is an HTTP API and Go library to download, parse and offer search functions over numerous trade sanction lists from the United States, European Union governments, agencies, and non profits for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package admin - -// OfacSdn Specially designated national from OFAC list -type OfacSdn struct { - EntityID string `json:"entityID,omitempty"` - SdnName string `json:"sdnName,omitempty"` - SdnType SdnType `json:"sdnType,omitempty"` - // Programs is the sanction programs this SDN was added from - Programs []string `json:"programs,omitempty"` - Title string `json:"title,omitempty"` - Remarks string `json:"remarks,omitempty"` - // Remarks on SDN and often additional information about the SDN - Match float32 `json:"match,omitempty"` -} diff --git a/admin/model_sdn_debug_metadata.go b/admin/model_sdn_debug_metadata.go deleted file mode 100644 index 91757275..00000000 --- a/admin/model_sdn_debug_metadata.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Watchman Admin API - * - * Watchman is an HTTP API and Go library to download, parse and offer search functions over numerous trade sanction lists from the United States, European Union governments, agencies, and non profits for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package admin - -// SdnDebugMetadata struct for SdnDebugMetadata -type SdnDebugMetadata struct { - // Exact text stored in our index used for string ranking - IndexedName string `json:"indexedName,omitempty"` - // ID parsed from remarks field - ParsedRemarksId string `json:"parsedRemarksId,omitempty"` -} diff --git a/admin/model_sdn_type.go b/admin/model_sdn_type.go deleted file mode 100644 index 120d6c70..00000000 --- a/admin/model_sdn_type.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Watchman Admin API - * - * Watchman is an HTTP API and Go library to download, parse and offer search functions over numerous trade sanction lists from the United States, European Union governments, agencies, and non profits for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package admin - -// SdnType Used for classifying SDNs — typically represents an individual or company -type SdnType string - -// List of SdnType -const ( - INDIVIDUAL SdnType = "individual" - ENTITY SdnType = "entity" - VESSEL SdnType = "vessel" - AIRCRAFT SdnType = "aircraft" -) diff --git a/admin/response.go b/admin/response.go deleted file mode 100644 index 4cc75b3a..00000000 --- a/admin/response.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Watchman Admin API - * - * Watchman is an HTTP API and Go library to download, parse and offer search functions over numerous trade sanction lists from the United States, European Union governments, agencies, and non profits for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package admin - -import ( - "net/http" -) - -// APIResponse stores the API response returned by the server. -type APIResponse struct { - *http.Response `json:"-"` - Message string `json:"message,omitempty"` - // Operation is the name of the OpenAPI operation. - Operation string `json:"operation,omitempty"` - // RequestURL is the request URL. This value is always available, even if the - // embedded *http.Response is nil. - RequestURL string `json:"url,omitempty"` - // Method is the HTTP method used for the request. This value is always - // available, even if the embedded *http.Response is nil. - Method string `json:"method,omitempty"` - // Payload holds the contents of the response body (which may be nil or empty). - // This is provided here as the raw response.Body() reader will have already - // been drained. - Payload []byte `json:"-"` -} - -// NewAPIResponse returns a new APIResonse object. -func NewAPIResponse(r *http.Response) *APIResponse { - - response := &APIResponse{Response: r} - return response -} - -// NewAPIResponseWithError returns a new APIResponse object with the provided error message. -func NewAPIResponseWithError(errorMessage string) *APIResponse { - - response := &APIResponse{Message: errorMessage} - return response -} diff --git a/api/admin.yaml b/api/admin.yaml deleted file mode 100644 index e6efc336..00000000 --- a/api/admin.yaml +++ /dev/null @@ -1,129 +0,0 @@ -openapi: 3.0.2 -info: - description: | - Watchman is an HTTP API and Go library to download, parse and offer search functions over numerous trade sanction lists from the United States, European Union governments, agencies, and non profits for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - version: v1 - title: Watchman Admin API - contact: - url: https://github.com/moov-io/watchman - license: - name: Apache 2.0 - url: http://www.apache.org/licenses/LICENSE-2.0.html - -servers: - - url: http://localhost:9094 - description: Local development - -tags: - - name: Admin - description: Watchman endpoints which are only meant to be exposed for admin dashboards and operations. - -paths: - /version: - get: - tags: ["Admin"] - summary: Get Version - description: Show the current version of Watchman - operationId: getVersion - responses: - '200': - description: The current version running - content: - text/plain: - schema: - type: string - example: v0.13.1 - /data/refresh: - post: - tags: ["Admin"] - summary: Download and reindex all data sources - operationId: refreshData - responses: - '200': - description: Data successfully refreshed - content: - application/json: - schema: - $ref: "#/components/schemas/DataRefresh" - '400': - description: See error message - content: - application/json: - schema: - $ref: 'https://raw.githubusercontent.com/moov-io/base/master/api/common.yaml#/components/schemas/Error' - /debug/sdn/{sdnId}: - get: - tags: ["Admin"] - summary: Debug SDN - description: Get an SDN and search index debug information - operationId: debugSDN - parameters: - - name: sdnId - in: path - description: SDN ID - required: true - schema: - type: string - example: 564dd7d1 - responses: - '200': - description: SDN with debug information - content: - application/json: - schema: - $ref: "#/components/schemas/DebugSDN" - '400': - description: See error message - content: - application/json: - schema: - $ref: 'https://raw.githubusercontent.com/moov-io/base/master/api/common.yaml#/components/schemas/Error' - -components: - schemas: - SDNDebugMetadata: - properties: - indexedName: - type: string - description: Exact text stored in our index used for string ranking - example: mohammad moghisseh - parsedRemarksId: - type: string - description: ID parsed from remarks field - example: '3603251708570001' - DebugSDN: - properties: - SDN: - $ref: './client.yaml#/components/schemas/OfacSDN' - debug: - $ref: '#/components/schemas/SDNDebugMetadata' - DataRefresh: - properties: - SDNs: - type: integer - description: Count of OFAC SDNs after index - example: 13131 - altNames: - type: integer - description: Count of OFAC alternate names after index - example: 322 - addresses: - type: integer - description: Count of OFAC Addresses after index - example: 4155 - sectoralSanctions: - type: integer - description: Count of SSI entities after index - example: 667 - deniedPersons: - type: integer - description: Count of BSL denied persons after index - example: 5889 - bisEntities: - type: integer - description: Count of BIS entities after index - example: 6831 - timestamp: - type: string - format: date-time - example: 2006-01-02T15:04:05Z07:00 diff --git a/api/client.yaml b/api/client.yaml deleted file mode 100644 index 9fa4151f..00000000 --- a/api/client.yaml +++ /dev/null @@ -1,1257 +0,0 @@ -openapi: 3.0.2 -info: - description: Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - version: v1 - title: Watchman API - contact: - url: https://github.com/moov-io/watchman - license: - name: Apache 2.0 - url: http://www.apache.org/licenses/LICENSE-2.0.html - -servers: - - url: http://localhost:8084 - description: Local development - -tags: - - name: Watchman - description: Endpoints for searching individuals and corporations that the U.S. government enforces economic sanctions against and adding webhook notifications for search criteria. - -paths: - /ping: - get: - tags: [Watchman] - summary: Ping Watchman service - description: Check if the Watchman service is running. - operationId: ping - responses: - '200': - description: Service is running properly - - # OFAC endpoints - /ofac/sdn/{sdnID}/alts: - get: - tags: [Watchman] - summary: Get SDN alt names - operationId: getSDNAltNames - parameters: - - name: X-Request-ID - in: header - description: Optional Request ID allows application developer to trace requests through the system's logs - schema: - type: string - example: 94c825ee - - in: path - name: sdnID - description: SDN ID - required: true - schema: - type: string - example: 564dd7d1 - responses: - '200': - description: SDN alternate names - content: - application/json: - schema: - $ref: '#/components/schemas/OfacSDNAltNames' - '400': - description: Error occurred, see response body. - content: - application/json: - schema: - $ref: 'https://raw.githubusercontent.com/moov-io/base/master/api/common.yaml#/components/schemas/Error' - - /ofac/sdn/{sdnID}/addresses: - get: - tags: [Watchman] - summary: Get SDN addresses - operationId: getSDNAddresses - parameters: - - name: X-Request-ID - in: header - description: Optional Request ID allows application developer to trace requests through the system's logs - schema: - type: string - example: 94c825ee - - in: path - name: sdnID - description: SDN ID - required: true - schema: - type: string - example: 564dd7d1 - responses: - '200': - description: SDN addresses - content: - application/json: - schema: - $ref: '#/components/schemas/OfacEntityAddresses' - '400': - description: Error occurred, see response body. - content: - application/json: - schema: - $ref: 'https://raw.githubusercontent.com/moov-io/base/master/api/common.yaml#/components/schemas/Error' - - # Search Endpoint - /search: - get: - tags: [Watchman] - summary: Search - operationId: search - parameters: - - name: X-Request-ID - in: header - description: Optional Request ID allows application developer to trace requests through the system's logs - schema: - type: string - example: 94c825ee - - name: q - in: query - schema: - type: string - example: John Doe - description: Search across Name, Alt Names, and SDN Address fields for all available sanctions lists. Entries may be returned in all response sub-objects. - - name: name - in: query - schema: - type: string - example: Jane Smith - description: Name which could correspond to an entry on the SDN, Denied Persons, Sectoral Sanctions Identifications, or BIS Entity List sanctions lists. Alt names are also searched. - - name: address - in: query - schema: - type: string - example: 123 83rd Ave - description: Physical address which could correspond to a human on the SDN list. Only Address results will be returned. - - name: city - in: query - schema: - type: string - example: USA - description: City name as desginated by SDN guidelines. Only Address results will be returned. - - name: state - in: query - schema: - type: string - example: USA - description: State name as desginated by SDN guidelines. Only Address results will be returned. - - name: providence - in: query - schema: - type: string - example: USA - description: Providence name as desginated by SDN guidelines. Only Address results will be returned. - - name: zip - in: query - schema: - type: string - example: USA - description: Zip code as desginated by SDN guidelines. Only Address results will be returned. - - name: country - in: query - schema: - type: string - example: USA - description: Country name as desginated by SDN guidelines. Only Address results will be returned. - - name: altName - in: query - schema: - type: string - example: Jane Smith - description: Alternate name which could correspond to a human on the SDN list. Only Alt name results will be returned. - - name: id - in: query - schema: - type: string - example: '10517860' - description: ID value often found in remarks property of an SDN. Takes the form of 'No. NNNNN' as an alphanumeric value. - - name: minMatch - in: query - schema: - type: number - format: float - example: 0.95 - description: Match percentage that search query must obtain for results to be returned. - - name: limit - in: query - schema: - type: integer - example: 25 - description: Maximum results returned by a search. Results are sorted by their match percentage in decending order. - - name: sdnType - in: query - schema: - $ref: '#/components/schemas/SdnType' - description: Optional filter to only return SDNs whose type case-insensitively matches. - - name: program - in: query - schema: - type: string - example: SDGT - description: Optional filter to only return SDNs whose program case-insensitively matches. - responses: - '200': - description: SDNs returned from a search - content: - application/json: - schema: - $ref: '#/components/schemas/Search' - '400': - description: Error occurred, see response body. - content: - application/json: - schema: - $ref: 'https://raw.githubusercontent.com/moov-io/base/master/api/common.yaml#/components/schemas/Error' - - # US, UK, and EU Consolidated Screening List endpoints - # TODO(adam): add UK and EU CSL endpoints - /search/us-csl: - get: - tags: [Watchman] - summary: Search US CSL - description: Search the US Consolidated Screening List - operationId: searchUSCSL - parameters: - - name: X-Request-ID - in: header - description: Optional Request ID allows application developer to trace requests through the system's logs - schema: - type: string - example: 94c825ee - - name: name - in: query - schema: - type: string - example: Jane Smith - description: Name which could correspond to an entry on the CSL - - name: limit - in: query - schema: - type: integer - example: 25 - description: Maximum number of downloads to return sorted by their timestamp in decending order. - responses: - '200': - description: SDNs returned from a search - content: - application/json: - schema: - $ref: '#/components/schemas/Search' - '400': - description: Error occurred, see response body. - content: - application/json: - schema: - $ref: 'https://raw.githubusercontent.com/moov-io/base/master/api/common.yaml#/components/schemas/Error' - - # Downloads endpoint - /downloads: - get: - tags: [Watchman] - summary: Get latest downloads - description: Return list of recent downloads of list data. - operationId: getLatestDownloads - parameters: - - name: X-Request-ID - in: header - description: Optional Request ID allows application developer to trace requests through the system's logs - schema: - type: string - example: 94c825ee - - name: limit - in: query - schema: - type: integer - example: 25 - description: Maximum number of downloads to return sorted by their timestamp in decending order. - responses: - '200': - description: Recent timestamps and counts of parsed objects - content: - application/json: - schema: - $ref: '#/components/schemas/Downloads' - '400': - description: Error occurred, see response body. - content: - application/json: - schema: - $ref: 'https://raw.githubusercontent.com/moov-io/base/master/api/common.yaml#/components/schemas/Error' - /ui/values/{key}: - get: - tags: [Watchman] - summary: Get UI values - description: Return an ordered distinct list of keys for an SDN property. - operationId: getUIValues - parameters: - - in: path - name: key - description: SDN property to lookup. Values are sdnType, ofacProgram - required: true - schema: - $ref: '#/components/schemas/SdnType' - - name: limit - in: query - schema: - type: integer - example: 25 - description: Maximum number of UI keys returned - responses: - '200': - description: Ordered and distinct list of values for an SDN property - content: - application/json: - schema: - $ref: '#/components/schemas/UIKeys' - '400': - description: Error occurred, see response body. - content: - application/json: - schema: - $ref: 'https://raw.githubusercontent.com/moov-io/base/master/api/common.yaml#/components/schemas/Error' - -components: - schemas: - OfacSDN: - description: Specially designated national from OFAC list - properties: - entityID: - type: string - example: "1231" - sdnName: - type: string - example: BANCO NACIONAL DE CUBA - sdnType: - $ref: '#/components/schemas/SdnType' - programs: - type: array - items: - type: string - description: Programs is the sanction programs this SDN was added from - example: [CUBA] - title: - type: string - example: Title of an individual - remarks: - type: string - description: Remarks on SDN and often additional information about the SDN - example: Additional info - match: - type: number - description: Match percentage of search query - example: 0.91 - matchedName: - type: string - description: "The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm." - example: "jane doe" - OfacEntityAddresses: - type: array - items: - $ref: '#/components/schemas/OfacEntityAddress' - OfacEntityAddress: - description: Physical address from OFAC list - properties: - entityID: - type: string - example: "2112" - addressID: - type: string - example: "201" - address: - type: string - example: 123 73th St - cityStateProvincePostalCode: - type: string - example: Tokyo 103 - country: - type: string - example: Japan - match: - type: number - description: Match percentage of search query - example: 0.91 - OfacSDNAltNames: - type: array - items: - $ref: '#/components/schemas/OfacAlt' - OfacAlt: - description: Alternate name from OFAC list - properties: - entityID: - type: string - example: "306" - alternateID: - type: string - example: "220" - alternateType: - type: string - example: aka - alternateName: - type: string - example: NATIONAL BANK OF CUBA - alternateRemarks: - type: string - example: Extra information - match: - type: number - description: Match percentage of search query - example: 0.91 - matchedName: - type: string - description: "The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm." - example: "jane doe" - SdnType: - description: 'Used for classifying SDNs — typically represents an individual or company' - type: string - enum: - - individual - - entity - - vessel - - aircraft - example: individual - SsiType: - description: 'Used for classifying SSIs' - type: string - enum: - - individual - - entity - DPL: - description: BIS Denied Persons List item - properties: - name: - type: string - description: Denied Person's name - example: ISMAEL RETA - streetAddress: - type: string - description: "Denied Person's street address" - example: 'REGISTER NUMBER: 78795-379, FEDERAL CORRECTIONAL INSTITUTION, P.O. BOX 4200' - city: - type: string - description: "Denied Person's city" - example: THREE RIVERS - state: - type: string - description: "Denied Person's state" - example: TX - country: - type: string - description: "Denied Person's country" - example: "United States" - postalCode: - type: string - description: "Denied Person's postal code" - example: "78071" - effectiveDate: - type: string - description: Date when denial came into effect - example: '06/15/2016' - expirationDate: - type: string - description: Date when denial expires, if blank denial never expires - example: '06/15/2025' - standardOrder: - type: string - description: Denotes whether or not the Denied Person was added by a standard order - example: 'Y' - lastUpdate: - type: string - description: Date when the Denied Person's record was most recently updated - example: '2016-06-22' - action: - type: string - description: Most recent action taken regarding the denial - example: FR NOTICE ADDED - frCitation: - type: string - description: Reference to the order's citation in the Federal Register - example: '81.F.R. 40658 6/22/2016' - match: - type: number - description: Match percentage of search query - example: 0.92 - matchedName: - type: string - description: "The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm." - example: "jane doe" - SSI: - description: Treasury Department Sectoral Sanctions Identifications List (SSI) - properties: - entityID: - type: string - description: The ID assigned to an entity by the Treasury Department - example: "1231" - type: - $ref: '#/components/schemas/SsiType' - programs: - type: array - items: - type: string - description: Sanction programs for which the entity is flagged - example: ["UKRAINE-EO13662", "SYRIA"] - name: - type: string - description: The name of the entity - example: PJSC VERKHNECHONSKNEFTEGAZ - addresses: - type: array - items: - type: string - description: Addresses associated with the entity - example: ["D. Retyum, Luzhski Raion, Leningradskaya Obl., 188230, RU", "Retiun Village, Lujskiy District, Leningrad Region, RU"] - remarks: - type: array - items: - type: string - description: Additional details regarding the entity - example: ["For more information on directives, please visit the following link: http://www.treasury.gov/resource-center/sanctions/Programs/Pages/ukraine.aspx#directives.", "(Linked To: OPEN JOINT-STOCK COMPANY ROSNEFT OIL COMPANY)"] - alternateNames: - type: array - items: - type: string - description: Known aliases associated with the entity - example: ["VERKHNECHONSKNEFTEGAZ", "OJSC VERKHNECHONSKNEFTEGAZ"] - ids: - type: array - items: - type: string - description: IDs on file for the entity - example: ["Subject to Directive 4, Executive Order 13662 Directive Determination", "vcng@rosneft.ru, Email Address", "Subject to Directive 2, Executive Order 13662 Directive Determination"] - sourceListURL: - type: string - description: The link to the official SSI list - example: http://bit.ly/1MLgou0 - sourceInfoURL: - type: string - description: The link for information regarding the source - example: http://bit.ly/1MLgou0 - match: - type: number - description: Match percentage of search query - example: 0.91 - matchedName: - type: string - description: "The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm." - example: "jane doe" - BISEntities: - description: Bureau of Industry and Security Entity List - properties: - name: - type: string - description: The name of the entity - example: Luhansk People¬ís Republic - addresses: - type: array - items: - type: string - description: Addresses associated with the entity - example: ["D. Retyum, Luzhski Raion, Leningradskaya Obl., 188230, RU", "Retiun Village, Lujskiy District, Leningrad Region, RU"] - alternateNames: - type: array - items: - type: string - description: Known aliases associated with the entity - example: ["VERKHNECHONSKNEFTEGAZ", "OJSC VERKHNECHONSKNEFTEGAZ"] - startDate: - type: string - description: Date when the restriction came into effect - example: 6/21/16 - licenseRequirement: - type: string - description: Specifies the license requirement imposed on the named entity - example: "For all items subject to the EAR. (See ¬ß744.11 of the EAR)." - licensePolicy: - type: string - description: Identifies the policy BIS uses to review the licenseRequirements - example: "Presumption of denial." - frNotice: - type: string - description: Identifies the corresponding Notice in the Federal Register - example: 81 FR 61595 - sourceListURL: - type: string - description: The link to the official SSI list - example: http://bit.ly/1MLgou0 - sourceInfoURL: - type: string - description: The link for information regarding the source - example: http://bit.ly/1MLgou0 - match: - type: number - description: Match percentage of search query - example: 0.91 - matchedName: - type: string - description: "The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm." - example: "jane doe" - MilitaryEndUser: - properties: - entityID: - type: string - example: '26744194bd9b5cbec49db6ee29a4b53c697c7420' - name: - type: string - example: 'AECC Aviation Power Co. Ltd.' - addresses: - type: string - example: 'Xiujia Bay, Weiyong Dt, Xian, 710021, CN' - FRNotice: - type: string - example: '85 FR 83799' - startDate: - type: string - example: '2020-12-23' - endDate: - type: string - example: '' - match: - type: number - description: Match percentage of search query - example: 0.92 - matchedName: - type: string - description: "The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm." - example: "jane doe" - Unverified: - properties: - entityID: - type: string - example: 'f15fa805ff4ac5e09026f5e78011a1bb6b26dec2' - name: - type: string - example: 'Atlas Sanatgaran' - addresses: - type: array - items: - type: string - example: - - 'Komitas 26/114, Yerevan, Armenia, AM' - sourceListURL: - type: string - example: 'http://bit.ly/1iwwTSJ' - sourceInfoURL: - type: string - example: 'http://bit.ly/1Qi4R7Z' - match: - type: number - description: Match percentage of search query - example: 0.92 - matchedName: - type: string - description: "The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm." - example: "jane doe" - NonProliferationSanction: - properties: - entityID: - type: string - example: '2d2db09c686e4829d0ef1b0b04145eec3d42cd88' - programs: - type: array - items: - type: string - example: - - "E.O. 13382" - - "Export-Import Bank Act" - - "Nuclear Proliferation Prevention Act" - name: - type: string - example: 'Abdul Qadeer Khan' - federalRegisterNotice: - type: string - example: 'Vol. 74, No. 11, 01/16/09' - startDate: - type: string - example: '2009-01-09' - remarks: - type: array - items: - type: string - example: 'Associated with the A.Q. Khan Network' - sourceListURL: - type: string - example: 'http://bit.ly/1NuVFxV' - alternateNames: - type: array - items: - type: string - example: - - "ZAMAN" - - "Haydar" - sourceInfoURL: - type: string - example: 'http://bit.ly/1NuVFxV' - match: - type: number - description: Match percentage of search query - example: 0.92 - matchedName: - type: string - description: "The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm." - example: "jane doe" - ForeignSanctionsEvader: - properties: - entityID: - type: string - example: '17526' - entityNumber: - type: string - example: '17526' - type: - type: string - example: 'Individual' - programs: - type: array - items: - type: string - example: - - "SYRIA" - - "FSE-SY" - name: - type: string - example: 'BEKTAS, Halis' - addresses: - type: array - items: - type: string - example: [] - sourceListURL: - type: string - example: 'https://bit.ly/1QWTIfE' - citizenships: - type: string - example: 'CH' - datesOfBirth: - type: string - example: '1966-02-13' - sourceInfoURL: - type: string - example: 'http://bit.ly/1N1docf' - IDs: - type: array - items: - type: string - example: - - "CH, X0906223, Passport" - match: - type: number - description: Match percentage of search query - example: 0.92 - matchedName: - type: string - description: "The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm." - example: "jane doe" - PalestinianLegislativeCouncil: - properties: - entityID: - type: string - example: '9702' - entityNumber: - type: string - example: '9702' - type: - type: string - example: 'Individual' - programs: - type: array - items: - type: string - example: - - "NS-PLC" - - "Office of Misinformation" - name: - type: string - example: 'SALAMEH, Salem' - addresses: - type: array - items: - type: string - example: - - "123 Dunbar Street, Testerville, TX, Palestine" - remarks: - type: string - example: 'HAMAS - Der al-Balah' - sourceListURL: - type: string - example: https://bit.ly/1QWTIfE - alternateNames: - type: array - items: - type: string - example: - - "SALAMEH, Salem Ahmad Abdel Hadi" - datesOfBirth: - type: string - example: '1951' - placesOfBirth: - type: string - example: '' - sourceInfoURL: - type: string - example: 'http://bit.ly/2tjOLpx' - match: - type: number - description: Match percentage of search query - example: 0.92 - matchedName: - type: string - description: "The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm." - example: "jane doe" - CAPTAList: - properties: - entityID: - type: string - example: "20002" - entityNumber: - type: string - example: "20002" - type: - type: string - example: "Entity" - programs: - type: array - items: - type: string - example: - - "UKRAINE-EO13662" - - "RUSSIA-EO14024" - name: - type: string - example: "BM BANK PUBLIC JOINT STOCK COMPANY" - addresses: - type: array - items: - type: string - example: "Bld 3 8/15, Rozhdestvenka St., Moscow, 107996, RU" - remarks: - type: array - items: - type: string - example: - - "All offices worldwide" - sourceListURL: - type: string - example: "" - alternateNames: - type: array - items: - type: string - example: - - "BM BANK JSC" - - "BM BANK AO" - - "AKTSIONERNOE OBSHCHESTVO BM BANK" - sourceInfoURL: - type: string - example: "http://bit.ly/2PqohAD" - IDs: - type: array - items: - type: string - example: - - "RU, 1027700159497, Registration Number" - - "RU, 29292940, Government Gazette Number" - - "MOSWRUMM, SWIFT/BIC" - match: - type: number - description: Match percentage of search query - example: 0.92 - matchedName: - type: string - description: "The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm." - example: "jane doe" - ITARDebarred: - properties: - entityID: - type: string - example: "d44d88d0265d93927b9ff1c13bbbb7c7db64142c" - name: - type: string - example: "Yasmin Ahmed" - federalRegisterNotice: - type: string - example: "69 FR 17468" - sourceListURL: - type: string - example: "http://bit.ly/307FuRQ" - alternateNames: - type: array - items: - type: string - example: - - "Yasmin Tariq" - - "Fatimah Mohammad" - sourceInfoURL: - type: string - example: "http://bit.ly/307FuRQ" - match: - type: number - description: Match percentage of search query - example: 0.92 - matchedName: - type: string - description: "The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm." - example: "jane doe" - NonSDNChineseMilitaryIndustrialComplex: - properties: - entityID: - type: string - example: "32091" - entityNumber: - type: string - example: "32091" - type: - type: string - example: "Entity" - programs: - type: array - items: - type: string - example: - - "CMIC-EO13959" - name: - type: string - example: "PROVEN HONOUR CAPITAL LIMITED" - addresses: - type: array - items: - type: string - example: - - "C/O Vistra Corporate Services Centre, Wickhams Cay II, Road Town, VG1110, VG" - remarks: - type: array - items: - type: string - example: - - "(Linked To: HUAWEI INVESTMENT & HOLDING CO., LTD.)" - sourceListURL: - type: string - example: "https://bit.ly/1QWTIfE" - alternateNames: - type: array - items: - type: string - example: - - "PROVEN HONOUR CAPITAL LTD" - - "PROVEN HONOUR" - sourceInfoURL: - type: string - example: "https://bit.ly/3zsMQ4n" - IDs: - type: array - items: - type: string - example: - - "Proven Honour Capital Ltd, Issuer Name" - - "XS1233275194, ISIN" - match: - type: number - description: Match percentage of search query - example: 0.92 - matchedName: - type: string - description: "The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm." - example: "jane doe" - NonSDNMenuBasedSanctionsList: - properties: - EntityID: - type: string - example: "17016" - EntityNumber: - type: string - example: "17016" - Type: - type: string - example: "Entity" - Programs: - type: array - items: - type: string - example: - - "UKRAINE-EO13662" - - "MBS" - Name: - type: string - example: "GAZPROMBANK JOINT STOCK COMPANY" - Addresses: - type: array - items: - type: string - example: - - "16 Nametkina Street, Bldg. 1, Moscow, 117420, RU" - Remarks: - type: array - items: - type: string - example: - - "For more information on directives, please visit the following link: http://www.treasury.gov/resource-center/sanctions/Programs/Pages/ukraine.aspx#directives." - AlternateNames: - type: array - items: - type: string - example: - - "GAZPROMBANK OPEN JOINT STOCK COMPANY" - - "BANK GPB JSC" - - "GAZPROMBANK AO" - SourceInfoURL: - type: string - example: "https://bit.ly/2MbsybU" - IDs: - type: array - items: - type: string - example: - - "RU, 1027700167110, Registration Number" - - "RU, 09807684, Government Gazette Number" - - "RU, 7744001497, Tax ID No." - match: - type: number - description: Match percentage of search query - example: 0.92 - matchedName: - type: string - description: "The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm." - example: "jane doe" - EUConsolidatedSanctionsList: - properties: - fileGenerationDate: - type: string - example: "28/10/2022" - entityLogicalId: - type: integer - example: 13 - entityRemark: - type: string - entitySubjectType: - type: string - entityPublicationURL: - type: string - entityReferenceNumber: - type: string - nameAliasWholeNames: - type: array - items: - type: string - nameAliasTitles: - type: array - items: - type: string - addressCities: - type: array - items: - type: string - addressStreets: - type: array - items: - type: string - addressPoBoxes: - type: array - items: - type: string - addressZipCodes: - type: array - items: - type: string - addressCountryDescriptions: - type: array - items: - type: string - birthDates: - type: array - items: - type: string - birthCities: - type: array - items: - type: string - birthCountries: - type: array - items: - type: string - validFromTo: - type: object - match: - description: Match percentage of search query - example: 0.92 - type: number - matchedName: - type: string - description: "The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm." - example: "jane doe" - UKConsolidatedSanctionsList: - properties: - names: - type: array - items: - type: string - addresses: - type: array - items: - type: string - countries: - type: array - items: - type: string - groupType: - type: string - match: - description: Match percentage of search query - example: 0.92 - type: number - matchedName: - type: string - description: "The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm." - example: "jane doe" - UKSanctionsList: - properties: - names: - type: array - items: - type: string - nonLatinNames: - type: array - items: - type: string - entityType: - type: string - addresses: - type: array - items: - type: string - addressCountries: - type: array - items: - type: string - stateLocalities: - type: array - items: - type: string - match: - description: Match percentage of search query - example: 0.92 - type: number - matchedName: - type: string - description: "The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm." - example: "jane doe" - Search: - description: Search results containing SDNs, alternate names, and/or addreses - properties: - # OFAC - SDNs: - type: array - items: - $ref: '#/components/schemas/OfacSDN' - altNames: - type: array - items: - $ref: '#/components/schemas/OfacAlt' - addresses: - type: array - items: - $ref: '#/components/schemas/OfacEntityAddress' - # BIS - deniedPersons: - type: array - items: - $ref: '#/components/schemas/DPL' - bisEntities: - type: array - items: - $ref: '#/components/schemas/BISEntities' - # US Consolidated Screening List - militaryEndUsers: - type: array - items: - $ref: '#/components/schemas/MilitaryEndUser' - sectoralSanctions: - type: array - items: - $ref: '#/components/schemas/SSI' - unverifiedCSL: - type: array - items: - $ref: '#/components/schemas/Unverified' - nonproliferationSanctions: - type: array - items: - $ref: '#/components/schemas/NonProliferationSanction' - foreignSanctionsEvaders: - type: array - items: - $ref: '#/components/schemas/ForeignSanctionsEvader' - palestinianLegislativeCouncil: - type: array - items: - $ref: '#/components/schemas/PalestinianLegislativeCouncil' - captaList: - type: array - items: - $ref: '#/components/schemas/CAPTAList' - itarDebarred: - type: array - items: - $ref: '#/components/schemas/ITARDebarred' - nonSDNChineseMilitaryIndustrialComplex: - type: array - items: - $ref: '#/components/schemas/NonSDNChineseMilitaryIndustrialComplex' - nonSDNMenuBasedSanctionsList: - type: array - items: - $ref: '#/components/schemas/NonSDNMenuBasedSanctionsList' - euConsolidatedSanctionsList: - items: - $ref: '#/components/schemas/EUConsolidatedSanctionsList' - type: array - ukConsolidatedSanctionsList: - items: - $ref: '#/components/schemas/UKConsolidatedSanctionsList' - type: array - ukSanctionsList: - items: - $ref: '#/components/schemas/UKSanctionsList' - type: array - # Metadata - refreshedAt: - type: string - format: date-time - example: "2006-01-02T15:04:05" - Downloads: - type: array - items: - $ref: '#/components/schemas/Download' - Download: - description: Metadata and stats about downloaded OFAC data - properties: - # OFAC - SDNs: - type: integer - example: 7414 - altNames: - type: integer - example: 9729 - addresses: - type: integer - example: 11747 - sectoralSanctions: - type: integer - example: 329 - # BIS - deniedPersons: - type: integer - example: 842 - bisEntities: - type: integer - example: 1391 - # Metadata - timestamp: - type: string - format: date-time - example: "2006-01-02T15:04:05" - UIKeys: - type: array - items: - $ref: '#/components/schemas/SdnType' - uniqueItems: true diff --git a/client/.gitignore b/client/.gitignore deleted file mode 100644 index daf913b1..00000000 --- a/client/.gitignore +++ /dev/null @@ -1,24 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe -*.test -*.prof diff --git a/client/.openapi-generator-ignore b/client/.openapi-generator-ignore deleted file mode 100644 index 7484ee59..00000000 --- a/client/.openapi-generator-ignore +++ /dev/null @@ -1,23 +0,0 @@ -# OpenAPI Generator Ignore -# Generated by openapi-generator https://github.com/openapitools/openapi-generator - -# Use this file to prevent files from being overwritten by the generator. -# The patterns follow closely to .gitignore or .dockerignore. - -# As an example, the C# client generator defines ApiClient.cs. -# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line: -#ApiClient.cs - -# You can match any string of characters against a directory, file or extension with a single asterisk (*): -#foo/*/qux -# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux - -# You can recursively match patterns against a directory, file or extension with a double asterisk (**): -#foo/**/qux -# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux - -# You can also negate patterns with an exclamation (!). -# For example, you can ignore all files in a docs folder with the file extension .md: -#docs/*.md -# Then explicitly reverse the ignore rule for a single file: -#!docs/README.md diff --git a/client/.openapi-generator/VERSION b/client/.openapi-generator/VERSION deleted file mode 100644 index ecedc98d..00000000 --- a/client/.openapi-generator/VERSION +++ /dev/null @@ -1 +0,0 @@ -4.3.1 \ No newline at end of file diff --git a/client/README.md b/client/README.md deleted file mode 100644 index 8e5257ba..00000000 --- a/client/README.md +++ /dev/null @@ -1,81 +0,0 @@ -# Go API client for client - -Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - -## Overview -This API client was generated by the [OpenAPI Generator](https://openapi-generator.tech) project. By using the [OpenAPI-spec](https://www.openapis.org/) from a remote server, you can easily generate an API client. - -- API version: v1 -- Package version: 1.0.0 -- Build package: org.openapitools.codegen.languages.GoClientCodegen -For more information, please visit [https://github.com/moov-io/watchman](https://github.com/moov-io/watchman) - -## Installation - -Install the following dependencies: - -```shell -go get github.com/stretchr/testify/assert -go get golang.org/x/oauth2 -go get golang.org/x/net/context -go get github.com/antihax/optional -``` - -Put the package under your project folder and add the following in import: - -```golang -import "./client" -``` - -## Documentation for API Endpoints - -All URIs are relative to *http://localhost:8084* - -Class | Method | HTTP request | Description ------------- | ------------- | ------------- | ------------- -*WatchmanApi* | [**GetLatestDownloads**](docs/WatchmanApi.md#getlatestdownloads) | **Get** /downloads | Get latest downloads -*WatchmanApi* | [**GetSDNAddresses**](docs/WatchmanApi.md#getsdnaddresses) | **Get** /ofac/sdn/{sdnID}/addresses | Get SDN addresses -*WatchmanApi* | [**GetSDNAltNames**](docs/WatchmanApi.md#getsdnaltnames) | **Get** /ofac/sdn/{sdnID}/alts | Get SDN alt names -*WatchmanApi* | [**GetUIValues**](docs/WatchmanApi.md#getuivalues) | **Get** /ui/values/{key} | Get UI values -*WatchmanApi* | [**Ping**](docs/WatchmanApi.md#ping) | **Get** /ping | Ping Watchman service -*WatchmanApi* | [**Search**](docs/WatchmanApi.md#search) | **Get** /search | Search -*WatchmanApi* | [**SearchUSCSL**](docs/WatchmanApi.md#searchuscsl) | **Get** /search/us-csl | Search US CSL - - -## Documentation For Models - - - [BisEntities](docs/BisEntities.md) - - [CaptaList](docs/CaptaList.md) - - [Download](docs/Download.md) - - [Dpl](docs/Dpl.md) - - [Error](docs/Error.md) - - [EuConsolidatedSanctionsList](docs/EuConsolidatedSanctionsList.md) - - [ForeignSanctionsEvader](docs/ForeignSanctionsEvader.md) - - [ItarDebarred](docs/ItarDebarred.md) - - [MilitaryEndUser](docs/MilitaryEndUser.md) - - [NonProliferationSanction](docs/NonProliferationSanction.md) - - [NonSdnChineseMilitaryIndustrialComplex](docs/NonSdnChineseMilitaryIndustrialComplex.md) - - [NonSdnMenuBasedSanctionsList](docs/NonSdnMenuBasedSanctionsList.md) - - [OfacAlt](docs/OfacAlt.md) - - [OfacEntityAddress](docs/OfacEntityAddress.md) - - [OfacSdn](docs/OfacSdn.md) - - [PalestinianLegislativeCouncil](docs/PalestinianLegislativeCouncil.md) - - [SdnType](docs/SdnType.md) - - [Search](docs/Search.md) - - [Ssi](docs/Ssi.md) - - [SsiType](docs/SsiType.md) - - [UkConsolidatedSanctionsList](docs/UkConsolidatedSanctionsList.md) - - [UkSanctionsList](docs/UkSanctionsList.md) - - [Unverified](docs/Unverified.md) - - -## Documentation For Authorization - - Endpoints do not require authorization. - - - -## Author - - - diff --git a/client/api/openapi.yaml b/client/api/openapi.yaml deleted file mode 100644 index 3fd22825..00000000 --- a/client/api/openapi.yaml +++ /dev/null @@ -1,2292 +0,0 @@ -openapi: 3.0.2 -info: - contact: - url: https://github.com/moov-io/watchman - description: Moov Watchman offers download, parse, and search functions over numerous - U.S. trade sanction lists for complying with regional laws. Also included is a - web UI and async webhook notification service to initiate processes on remote - systems. - license: - name: Apache 2.0 - url: http://www.apache.org/licenses/LICENSE-2.0.html - title: Watchman API - version: v1 -servers: -- description: Local development - url: http://localhost:8084 -tags: -- description: Endpoints for searching individuals and corporations that the U.S. - government enforces economic sanctions against and adding webhook notifications - for search criteria. - name: Watchman -paths: - /ping: - get: - description: Check if the Watchman service is running. - operationId: ping - responses: - "200": - description: Service is running properly - summary: Ping Watchman service - tags: - - Watchman - /ofac/sdn/{sdnID}/alts: - get: - operationId: getSDNAltNames - parameters: - - description: Optional Request ID allows application developer to trace requests - through the system's logs - explode: false - in: header - name: X-Request-ID - required: false - schema: - example: 94c825ee - type: string - style: simple - - description: SDN ID - explode: false - in: path - name: sdnID - required: true - schema: - example: 564dd7d1 - type: string - style: simple - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/OfacSDNAltNames' - description: SDN alternate names - "400": - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - description: Error occurred, see response body. - summary: Get SDN alt names - tags: - - Watchman - /ofac/sdn/{sdnID}/addresses: - get: - operationId: getSDNAddresses - parameters: - - description: Optional Request ID allows application developer to trace requests - through the system's logs - explode: false - in: header - name: X-Request-ID - required: false - schema: - example: 94c825ee - type: string - style: simple - - description: SDN ID - explode: false - in: path - name: sdnID - required: true - schema: - example: 564dd7d1 - type: string - style: simple - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/OfacEntityAddresses' - description: SDN addresses - "400": - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - description: Error occurred, see response body. - summary: Get SDN addresses - tags: - - Watchman - /search: - get: - operationId: search - parameters: - - description: Optional Request ID allows application developer to trace requests - through the system's logs - explode: false - in: header - name: X-Request-ID - required: false - schema: - example: 94c825ee - type: string - style: simple - - description: Search across Name, Alt Names, and SDN Address fields for all - available sanctions lists. Entries may be returned in all response sub-objects. - explode: true - in: query - name: q - required: false - schema: - example: John Doe - type: string - style: form - - description: Name which could correspond to an entry on the SDN, Denied Persons, - Sectoral Sanctions Identifications, or BIS Entity List sanctions lists. - Alt names are also searched. - explode: true - in: query - name: name - required: false - schema: - example: Jane Smith - type: string - style: form - - description: Physical address which could correspond to a human on the SDN - list. Only Address results will be returned. - explode: true - in: query - name: address - required: false - schema: - example: 123 83rd Ave - type: string - style: form - - description: City name as desginated by SDN guidelines. Only Address results - will be returned. - explode: true - in: query - name: city - required: false - schema: - example: USA - type: string - style: form - - description: State name as desginated by SDN guidelines. Only Address results - will be returned. - explode: true - in: query - name: state - required: false - schema: - example: USA - type: string - style: form - - description: Providence name as desginated by SDN guidelines. Only Address - results will be returned. - explode: true - in: query - name: providence - required: false - schema: - example: USA - type: string - style: form - - description: Zip code as desginated by SDN guidelines. Only Address results - will be returned. - explode: true - in: query - name: zip - required: false - schema: - example: USA - type: string - style: form - - description: Country name as desginated by SDN guidelines. Only Address results - will be returned. - explode: true - in: query - name: country - required: false - schema: - example: USA - type: string - style: form - - description: Alternate name which could correspond to a human on the SDN list. - Only Alt name results will be returned. - explode: true - in: query - name: altName - required: false - schema: - example: Jane Smith - type: string - style: form - - description: ID value often found in remarks property of an SDN. Takes the - form of 'No. NNNNN' as an alphanumeric value. - explode: true - in: query - name: id - required: false - schema: - example: "10517860" - type: string - style: form - - description: Match percentage that search query must obtain for results to - be returned. - explode: true - in: query - name: minMatch - required: false - schema: - example: 0.95 - format: float - type: number - style: form - - description: Maximum results returned by a search. Results are sorted by their - match percentage in decending order. - explode: true - in: query - name: limit - required: false - schema: - example: 25 - type: integer - style: form - - description: Optional filter to only return SDNs whose type case-insensitively - matches. - explode: true - in: query - name: sdnType - required: false - schema: - $ref: '#/components/schemas/SdnType' - style: form - - description: Optional filter to only return SDNs whose program case-insensitively - matches. - explode: true - in: query - name: program - required: false - schema: - example: SDGT - type: string - style: form - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/Search' - description: SDNs returned from a search - "400": - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - description: Error occurred, see response body. - summary: Search - tags: - - Watchman - /search/us-csl: - get: - description: Search the US Consolidated Screening List - operationId: searchUSCSL - parameters: - - description: Optional Request ID allows application developer to trace requests - through the system's logs - explode: false - in: header - name: X-Request-ID - required: false - schema: - example: 94c825ee - type: string - style: simple - - description: Name which could correspond to an entry on the CSL - explode: true - in: query - name: name - required: false - schema: - example: Jane Smith - type: string - style: form - - description: Maximum number of downloads to return sorted by their timestamp - in decending order. - explode: true - in: query - name: limit - required: false - schema: - example: 25 - type: integer - style: form - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/Search' - description: SDNs returned from a search - "400": - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - description: Error occurred, see response body. - summary: Search US CSL - tags: - - Watchman - /downloads: - get: - description: Return list of recent downloads of list data. - operationId: getLatestDownloads - parameters: - - description: Optional Request ID allows application developer to trace requests - through the system's logs - explode: false - in: header - name: X-Request-ID - required: false - schema: - example: 94c825ee - type: string - style: simple - - description: Maximum number of downloads to return sorted by their timestamp - in decending order. - explode: true - in: query - name: limit - required: false - schema: - example: 25 - type: integer - style: form - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/Downloads' - description: Recent timestamps and counts of parsed objects - "400": - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - description: Error occurred, see response body. - summary: Get latest downloads - tags: - - Watchman - /ui/values/{key}: - get: - description: Return an ordered distinct list of keys for an SDN property. - operationId: getUIValues - parameters: - - description: SDN property to lookup. Values are sdnType, ofacProgram - explode: false - in: path - name: key - required: true - schema: - $ref: '#/components/schemas/SdnType' - style: simple - - description: Maximum number of UI keys returned - explode: true - in: query - name: limit - required: false - schema: - example: 25 - type: integer - style: form - responses: - "200": - content: - application/json: - schema: - $ref: '#/components/schemas/UIKeys' - description: Ordered and distinct list of values for an SDN property - "400": - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - description: Error occurred, see response body. - summary: Get UI values - tags: - - Watchman -components: - schemas: - OfacSDN: - description: Specially designated national from OFAC list - example: - sdnType: individual - match: 0.91 - entityID: "1231" - programs: - - CUBA - sdnName: BANCO NACIONAL DE CUBA - title: Title of an individual - matchedName: jane doe - remarks: Additional info - properties: - entityID: - example: "1231" - type: string - sdnName: - example: BANCO NACIONAL DE CUBA - type: string - sdnType: - $ref: '#/components/schemas/SdnType' - programs: - description: Programs is the sanction programs this SDN was added from - example: - - CUBA - items: - type: string - type: array - title: - example: Title of an individual - type: string - remarks: - description: Remarks on SDN and often additional information about the SDN - example: Additional info - type: string - match: - description: Match percentage of search query - example: 0.91 - type: number - matchedName: - description: The highest scoring term from the search query. This term is - the precomputed indexed value used by the search algorithm. - example: jane doe - type: string - OfacEntityAddresses: - items: - $ref: '#/components/schemas/OfacEntityAddress' - type: array - OfacEntityAddress: - description: Physical address from OFAC list - example: - country: Japan - address: 123 73th St - cityStateProvincePostalCode: Tokyo 103 - match: 0.91 - entityID: "2112" - addressID: "201" - properties: - entityID: - example: "2112" - type: string - addressID: - example: "201" - type: string - address: - example: 123 73th St - type: string - cityStateProvincePostalCode: - example: Tokyo 103 - type: string - country: - example: Japan - type: string - match: - description: Match percentage of search query - example: 0.91 - type: number - OfacSDNAltNames: - items: - $ref: '#/components/schemas/OfacAlt' - type: array - OfacAlt: - description: Alternate name from OFAC list - example: - alternateID: "220" - alternateRemarks: Extra information - alternateType: aka - match: 0.91 - entityID: "306" - alternateName: NATIONAL BANK OF CUBA - matchedName: jane doe - properties: - entityID: - example: "306" - type: string - alternateID: - example: "220" - type: string - alternateType: - example: aka - type: string - alternateName: - example: NATIONAL BANK OF CUBA - type: string - alternateRemarks: - example: Extra information - type: string - match: - description: Match percentage of search query - example: 0.91 - type: number - matchedName: - description: The highest scoring term from the search query. This term is - the precomputed indexed value used by the search algorithm. - example: jane doe - type: string - SdnType: - description: Used for classifying SDNs — typically represents an individual - or company - enum: - - individual - - entity - - vessel - - aircraft - example: individual - type: string - SsiType: - description: Used for classifying SSIs - enum: - - individual - - entity - type: string - DPL: - description: BIS Denied Persons List item - example: - country: United States - city: THREE RIVERS - postalCode: "78071" - match: 0.92 - standardOrder: "Y" - frCitation: 81.F.R. 40658 6/22/2016 - streetAddress: 'REGISTER NUMBER: 78795-379, FEDERAL CORRECTIONAL INSTITUTION, - P.O. BOX 4200' - lastUpdate: 2016-06-22 - name: ISMAEL RETA - action: FR NOTICE ADDED - state: TX - matchedName: jane doe - effectiveDate: 06/15/2016 - expirationDate: 06/15/2025 - properties: - name: - description: Denied Person's name - example: ISMAEL RETA - type: string - streetAddress: - description: Denied Person's street address - example: 'REGISTER NUMBER: 78795-379, FEDERAL CORRECTIONAL INSTITUTION, - P.O. BOX 4200' - type: string - city: - description: Denied Person's city - example: THREE RIVERS - type: string - state: - description: Denied Person's state - example: TX - type: string - country: - description: Denied Person's country - example: United States - type: string - postalCode: - description: Denied Person's postal code - example: "78071" - type: string - effectiveDate: - description: Date when denial came into effect - example: 06/15/2016 - type: string - expirationDate: - description: Date when denial expires, if blank denial never expires - example: 06/15/2025 - type: string - standardOrder: - description: Denotes whether or not the Denied Person was added by a standard - order - example: "Y" - type: string - lastUpdate: - description: Date when the Denied Person's record was most recently updated - example: 2016-06-22 - type: string - action: - description: Most recent action taken regarding the denial - example: FR NOTICE ADDED - type: string - frCitation: - description: Reference to the order's citation in the Federal Register - example: 81.F.R. 40658 6/22/2016 - type: string - match: - description: Match percentage of search query - example: 0.92 - type: number - matchedName: - description: The highest scoring term from the search query. This term is - the precomputed indexed value used by the search algorithm. - example: jane doe - type: string - SSI: - description: Treasury Department Sectoral Sanctions Identifications List (SSI) - example: - sourceInfoURL: http://bit.ly/1MLgou0 - addresses: - - D. Retyum, Luzhski Raion, Leningradskaya Obl., 188230, RU - - Retiun Village, Lujskiy District, Leningrad Region, RU - alternateNames: - - VERKHNECHONSKNEFTEGAZ - - OJSC VERKHNECHONSKNEFTEGAZ - name: PJSC VERKHNECHONSKNEFTEGAZ - match: 0.91 - ids: - - Subject to Directive 4, Executive Order 13662 Directive Determination - - vcng@rosneft.ru, Email Address - - Subject to Directive 2, Executive Order 13662 Directive Determination - entityID: "1231" - programs: - - UKRAINE-EO13662 - - SYRIA - sourceListURL: http://bit.ly/1MLgou0 - matchedName: jane doe - remarks: - - 'For more information on directives, please visit the following link: http://www.treasury.gov/resource-center/sanctions/Programs/Pages/ukraine.aspx#directives.' - - '(Linked To: OPEN JOINT-STOCK COMPANY ROSNEFT OIL COMPANY)' - properties: - entityID: - description: The ID assigned to an entity by the Treasury Department - example: "1231" - type: string - type: - $ref: '#/components/schemas/SsiType' - programs: - description: Sanction programs for which the entity is flagged - example: - - UKRAINE-EO13662 - - SYRIA - items: - type: string - type: array - name: - description: The name of the entity - example: PJSC VERKHNECHONSKNEFTEGAZ - type: string - addresses: - description: Addresses associated with the entity - example: - - D. Retyum, Luzhski Raion, Leningradskaya Obl., 188230, RU - - Retiun Village, Lujskiy District, Leningrad Region, RU - items: - type: string - type: array - remarks: - description: Additional details regarding the entity - example: - - 'For more information on directives, please visit the following link: - http://www.treasury.gov/resource-center/sanctions/Programs/Pages/ukraine.aspx#directives.' - - '(Linked To: OPEN JOINT-STOCK COMPANY ROSNEFT OIL COMPANY)' - items: - type: string - type: array - alternateNames: - description: Known aliases associated with the entity - example: - - VERKHNECHONSKNEFTEGAZ - - OJSC VERKHNECHONSKNEFTEGAZ - items: - type: string - type: array - ids: - description: IDs on file for the entity - example: - - Subject to Directive 4, Executive Order 13662 Directive Determination - - vcng@rosneft.ru, Email Address - - Subject to Directive 2, Executive Order 13662 Directive Determination - items: - type: string - type: array - sourceListURL: - description: The link to the official SSI list - example: http://bit.ly/1MLgou0 - type: string - sourceInfoURL: - description: The link for information regarding the source - example: http://bit.ly/1MLgou0 - type: string - match: - description: Match percentage of search query - example: 0.91 - type: number - matchedName: - description: The highest scoring term from the search query. This term is - the precomputed indexed value used by the search algorithm. - example: jane doe - type: string - BISEntities: - description: Bureau of Industry and Security Entity List - example: - licenseRequirement: For all items subject to the EAR. (See ¬ß744.11 of the - EAR). - sourceInfoURL: http://bit.ly/1MLgou0 - addresses: - - D. Retyum, Luzhski Raion, Leningradskaya Obl., 188230, RU - - Retiun Village, Lujskiy District, Leningrad Region, RU - alternateNames: - - VERKHNECHONSKNEFTEGAZ - - OJSC VERKHNECHONSKNEFTEGAZ - licensePolicy: Presumption of denial. - name: Luhansk People¬ís Republic - match: 0.91 - sourceListURL: http://bit.ly/1MLgou0 - matchedName: jane doe - startDate: 6/21/16 - frNotice: 81 FR 61595 - properties: - name: - description: The name of the entity - example: Luhansk People¬ís Republic - type: string - addresses: - description: Addresses associated with the entity - example: - - D. Retyum, Luzhski Raion, Leningradskaya Obl., 188230, RU - - Retiun Village, Lujskiy District, Leningrad Region, RU - items: - type: string - type: array - alternateNames: - description: Known aliases associated with the entity - example: - - VERKHNECHONSKNEFTEGAZ - - OJSC VERKHNECHONSKNEFTEGAZ - items: - type: string - type: array - startDate: - description: Date when the restriction came into effect - example: 6/21/16 - type: string - licenseRequirement: - description: Specifies the license requirement imposed on the named entity - example: For all items subject to the EAR. (See ¬ß744.11 of the EAR). - type: string - licensePolicy: - description: Identifies the policy BIS uses to review the licenseRequirements - example: Presumption of denial. - type: string - frNotice: - description: Identifies the corresponding Notice in the Federal Register - example: 81 FR 61595 - type: string - sourceListURL: - description: The link to the official SSI list - example: http://bit.ly/1MLgou0 - type: string - sourceInfoURL: - description: The link for information regarding the source - example: http://bit.ly/1MLgou0 - type: string - match: - description: Match percentage of search query - example: 0.91 - type: number - matchedName: - description: The highest scoring term from the search query. This term is - the precomputed indexed value used by the search algorithm. - example: jane doe - type: string - MilitaryEndUser: - example: - addresses: Xiujia Bay, Weiyong Dt, Xian, 710021, CN - endDate: endDate - name: AECC Aviation Power Co. Ltd. - match: 0.92 - FRNotice: 85 FR 83799 - entityID: 26744194bd9b5cbec49db6ee29a4b53c697c7420 - matchedName: jane doe - startDate: 2020-12-23 - properties: - entityID: - example: 26744194bd9b5cbec49db6ee29a4b53c697c7420 - type: string - name: - example: AECC Aviation Power Co. Ltd. - type: string - addresses: - example: Xiujia Bay, Weiyong Dt, Xian, 710021, CN - type: string - FRNotice: - example: 85 FR 83799 - type: string - startDate: - example: 2020-12-23 - type: string - endDate: - type: string - match: - description: Match percentage of search query - example: 0.92 - type: number - matchedName: - description: The highest scoring term from the search query. This term is - the precomputed indexed value used by the search algorithm. - example: jane doe - type: string - Unverified: - example: - sourceInfoURL: http://bit.ly/1Qi4R7Z - addresses: - - Komitas 26/114, Yerevan, Armenia, AM - name: Atlas Sanatgaran - match: 0.92 - entityID: f15fa805ff4ac5e09026f5e78011a1bb6b26dec2 - sourceListURL: http://bit.ly/1iwwTSJ - matchedName: jane doe - properties: - entityID: - example: f15fa805ff4ac5e09026f5e78011a1bb6b26dec2 - type: string - name: - example: Atlas Sanatgaran - type: string - addresses: - example: - - Komitas 26/114, Yerevan, Armenia, AM - items: - type: string - type: array - sourceListURL: - example: http://bit.ly/1iwwTSJ - type: string - sourceInfoURL: - example: http://bit.ly/1Qi4R7Z - type: string - match: - description: Match percentage of search query - example: 0.92 - type: number - matchedName: - description: The highest scoring term from the search query. This term is - the precomputed indexed value used by the search algorithm. - example: jane doe - type: string - NonProliferationSanction: - example: - sourceInfoURL: http://bit.ly/1NuVFxV - alternateNames: - - ZAMAN - - Haydar - federalRegisterNotice: Vol. 74, No. 11, 01/16/09 - name: Abdul Qadeer Khan - match: 0.92 - entityID: 2d2db09c686e4829d0ef1b0b04145eec3d42cd88 - programs: - - E.O. 13382 - - Export-Import Bank Act - - Nuclear Proliferation Prevention Act - sourceListURL: http://bit.ly/1NuVFxV - matchedName: jane doe - startDate: 2009-01-09 - remarks: Associated with the A.Q. Khan Network - properties: - entityID: - example: 2d2db09c686e4829d0ef1b0b04145eec3d42cd88 - type: string - programs: - example: - - E.O. 13382 - - Export-Import Bank Act - - Nuclear Proliferation Prevention Act - items: - type: string - type: array - name: - example: Abdul Qadeer Khan - type: string - federalRegisterNotice: - example: Vol. 74, No. 11, 01/16/09 - type: string - startDate: - example: 2009-01-09 - type: string - remarks: - example: Associated with the A.Q. Khan Network - items: - type: string - type: array - sourceListURL: - example: http://bit.ly/1NuVFxV - type: string - alternateNames: - example: - - ZAMAN - - Haydar - items: - type: string - type: array - sourceInfoURL: - example: http://bit.ly/1NuVFxV - type: string - match: - description: Match percentage of search query - example: 0.92 - type: number - matchedName: - description: The highest scoring term from the search query. This term is - the precomputed indexed value used by the search algorithm. - example: jane doe - type: string - ForeignSanctionsEvader: - example: - addresses: [] - entityNumber: "17526" - match: 0.92 - entityID: "17526" - datesOfBirth: 1966-02-13 - type: Individual - sourceListURL: https://bit.ly/1QWTIfE - sourceInfoURL: http://bit.ly/1N1docf - citizenships: CH - name: BEKTAS, Halis - IDs: - - CH, X0906223, Passport - programs: - - SYRIA - - FSE-SY - matchedName: jane doe - properties: - entityID: - example: "17526" - type: string - entityNumber: - example: "17526" - type: string - type: - example: Individual - type: string - programs: - example: - - SYRIA - - FSE-SY - items: - type: string - type: array - name: - example: BEKTAS, Halis - type: string - addresses: - example: [] - items: - type: string - type: array - sourceListURL: - example: https://bit.ly/1QWTIfE - type: string - citizenships: - example: CH - type: string - datesOfBirth: - example: 1966-02-13 - type: string - sourceInfoURL: - example: http://bit.ly/1N1docf - type: string - IDs: - example: - - CH, X0906223, Passport - items: - type: string - type: array - match: - description: Match percentage of search query - example: 0.92 - type: number - matchedName: - description: The highest scoring term from the search query. This term is - the precomputed indexed value used by the search algorithm. - example: jane doe - type: string - PalestinianLegislativeCouncil: - example: - addresses: - - 123 Dunbar Street, Testerville, TX, Palestine - entityNumber: "9702" - match: 0.92 - entityID: "9702" - datesOfBirth: "1951" - type: Individual - sourceListURL: https://bit.ly/1QWTIfE - sourceInfoURL: http://bit.ly/2tjOLpx - alternateNames: - - SALAMEH, Salem Ahmad Abdel Hadi - placesOfBirth: placesOfBirth - name: SALAMEH, Salem - programs: - - NS-PLC - - Office of Misinformation - matchedName: jane doe - remarks: HAMAS - Der al-Balah - properties: - entityID: - example: "9702" - type: string - entityNumber: - example: "9702" - type: string - type: - example: Individual - type: string - programs: - example: - - NS-PLC - - Office of Misinformation - items: - type: string - type: array - name: - example: SALAMEH, Salem - type: string - addresses: - example: - - 123 Dunbar Street, Testerville, TX, Palestine - items: - type: string - type: array - remarks: - example: HAMAS - Der al-Balah - type: string - sourceListURL: - example: https://bit.ly/1QWTIfE - type: string - alternateNames: - example: - - SALAMEH, Salem Ahmad Abdel Hadi - items: - type: string - type: array - datesOfBirth: - example: "1951" - type: string - placesOfBirth: - type: string - sourceInfoURL: - example: http://bit.ly/2tjOLpx - type: string - match: - description: Match percentage of search query - example: 0.92 - type: number - matchedName: - description: The highest scoring term from the search query. This term is - the precomputed indexed value used by the search algorithm. - example: jane doe - type: string - CAPTAList: - example: - addresses: Bld 3 8/15, Rozhdestvenka St., Moscow, 107996, RU - entityNumber: "20002" - match: 0.92 - entityID: "20002" - type: Entity - sourceListURL: sourceListURL - sourceInfoURL: http://bit.ly/2PqohAD - alternateNames: - - BM BANK JSC - - BM BANK AO - - AKTSIONERNOE OBSHCHESTVO BM BANK - name: BM BANK PUBLIC JOINT STOCK COMPANY - IDs: - - RU, 1027700159497, Registration Number - - RU, 29292940, Government Gazette Number - - MOSWRUMM, SWIFT/BIC - programs: - - UKRAINE-EO13662 - - RUSSIA-EO14024 - matchedName: jane doe - remarks: - - All offices worldwide - properties: - entityID: - example: "20002" - type: string - entityNumber: - example: "20002" - type: string - type: - example: Entity - type: string - programs: - example: - - UKRAINE-EO13662 - - RUSSIA-EO14024 - items: - type: string - type: array - name: - example: BM BANK PUBLIC JOINT STOCK COMPANY - type: string - addresses: - example: Bld 3 8/15, Rozhdestvenka St., Moscow, 107996, RU - items: - type: string - type: array - remarks: - example: - - All offices worldwide - items: - type: string - type: array - sourceListURL: - type: string - alternateNames: - example: - - BM BANK JSC - - BM BANK AO - - AKTSIONERNOE OBSHCHESTVO BM BANK - items: - type: string - type: array - sourceInfoURL: - example: http://bit.ly/2PqohAD - type: string - IDs: - example: - - RU, 1027700159497, Registration Number - - RU, 29292940, Government Gazette Number - - MOSWRUMM, SWIFT/BIC - items: - type: string - type: array - match: - description: Match percentage of search query - example: 0.92 - type: number - matchedName: - description: The highest scoring term from the search query. This term is - the precomputed indexed value used by the search algorithm. - example: jane doe - type: string - ITARDebarred: - example: - sourceInfoURL: http://bit.ly/307FuRQ - alternateNames: - - Yasmin Tariq - - Fatimah Mohammad - federalRegisterNotice: 69 FR 17468 - name: Yasmin Ahmed - match: 0.92 - entityID: d44d88d0265d93927b9ff1c13bbbb7c7db64142c - sourceListURL: http://bit.ly/307FuRQ - matchedName: jane doe - properties: - entityID: - example: d44d88d0265d93927b9ff1c13bbbb7c7db64142c - type: string - name: - example: Yasmin Ahmed - type: string - federalRegisterNotice: - example: 69 FR 17468 - type: string - sourceListURL: - example: http://bit.ly/307FuRQ - type: string - alternateNames: - example: - - Yasmin Tariq - - Fatimah Mohammad - items: - type: string - type: array - sourceInfoURL: - example: http://bit.ly/307FuRQ - type: string - match: - description: Match percentage of search query - example: 0.92 - type: number - matchedName: - description: The highest scoring term from the search query. This term is - the precomputed indexed value used by the search algorithm. - example: jane doe - type: string - NonSDNChineseMilitaryIndustrialComplex: - example: - addresses: - - C/O Vistra Corporate Services Centre, Wickhams Cay II, Road Town, VG1110, - VG - entityNumber: "32091" - match: 0.92 - entityID: "32091" - type: Entity - sourceListURL: https://bit.ly/1QWTIfE - sourceInfoURL: https://bit.ly/3zsMQ4n - alternateNames: - - PROVEN HONOUR CAPITAL LTD - - PROVEN HONOUR - name: PROVEN HONOUR CAPITAL LIMITED - IDs: - - Proven Honour Capital Ltd, Issuer Name - - XS1233275194, ISIN - programs: - - CMIC-EO13959 - matchedName: jane doe - remarks: - - '(Linked To: HUAWEI INVESTMENT & HOLDING CO., LTD.)' - properties: - entityID: - example: "32091" - type: string - entityNumber: - example: "32091" - type: string - type: - example: Entity - type: string - programs: - example: - - CMIC-EO13959 - items: - type: string - type: array - name: - example: PROVEN HONOUR CAPITAL LIMITED - type: string - addresses: - example: - - C/O Vistra Corporate Services Centre, Wickhams Cay II, Road Town, VG1110, - VG - items: - type: string - type: array - remarks: - example: - - '(Linked To: HUAWEI INVESTMENT & HOLDING CO., LTD.)' - items: - type: string - type: array - sourceListURL: - example: https://bit.ly/1QWTIfE - type: string - alternateNames: - example: - - PROVEN HONOUR CAPITAL LTD - - PROVEN HONOUR - items: - type: string - type: array - sourceInfoURL: - example: https://bit.ly/3zsMQ4n - type: string - IDs: - example: - - Proven Honour Capital Ltd, Issuer Name - - XS1233275194, ISIN - items: - type: string - type: array - match: - description: Match percentage of search query - example: 0.92 - type: number - matchedName: - description: The highest scoring term from the search query. This term is - the precomputed indexed value used by the search algorithm. - example: jane doe - type: string - NonSDNMenuBasedSanctionsList: - example: - EntityID: "17016" - Programs: - - UKRAINE-EO13662 - - MBS - Addresses: - - 16 Nametkina Street, Bldg. 1, Moscow, 117420, RU - SourceInfoURL: https://bit.ly/2MbsybU - Type: Entity - Remarks: - - 'For more information on directives, please visit the following link: http://www.treasury.gov/resource-center/sanctions/Programs/Pages/ukraine.aspx#directives.' - match: 0.92 - IDs: - - RU, 1027700167110, Registration Number - - RU, 09807684, Government Gazette Number - - RU, 7744001497, Tax ID No. - EntityNumber: "17016" - AlternateNames: - - GAZPROMBANK OPEN JOINT STOCK COMPANY - - BANK GPB JSC - - GAZPROMBANK AO - matchedName: jane doe - Name: GAZPROMBANK JOINT STOCK COMPANY - properties: - EntityID: - example: "17016" - type: string - EntityNumber: - example: "17016" - type: string - Type: - example: Entity - type: string - Programs: - example: - - UKRAINE-EO13662 - - MBS - items: - type: string - type: array - Name: - example: GAZPROMBANK JOINT STOCK COMPANY - type: string - Addresses: - example: - - 16 Nametkina Street, Bldg. 1, Moscow, 117420, RU - items: - type: string - type: array - Remarks: - example: - - 'For more information on directives, please visit the following link: - http://www.treasury.gov/resource-center/sanctions/Programs/Pages/ukraine.aspx#directives.' - items: - type: string - type: array - AlternateNames: - example: - - GAZPROMBANK OPEN JOINT STOCK COMPANY - - BANK GPB JSC - - GAZPROMBANK AO - items: - type: string - type: array - SourceInfoURL: - example: https://bit.ly/2MbsybU - type: string - IDs: - example: - - RU, 1027700167110, Registration Number - - RU, 09807684, Government Gazette Number - - RU, 7744001497, Tax ID No. - items: - type: string - type: array - match: - description: Match percentage of search query - example: 0.92 - type: number - matchedName: - description: The highest scoring term from the search query. This term is - the precomputed indexed value used by the search algorithm. - example: jane doe - type: string - EUConsolidatedSanctionsList: - example: - birthCities: - - birthCities - - birthCities - entitySubjectType: entitySubjectType - nameAliasTitles: - - nameAliasTitles - - nameAliasTitles - addressPoBoxes: - - addressPoBoxes - - addressPoBoxes - addressCities: - - addressCities - - addressCities - match: 0.92 - birthDates: - - birthDates - - birthDates - entityPublicationURL: entityPublicationURL - entityLogicalId: 13 - nameAliasWholeNames: - - nameAliasWholeNames - - nameAliasWholeNames - validFromTo: '{}' - addressZipCodes: - - addressZipCodes - - addressZipCodes - addressCountryDescriptions: - - addressCountryDescriptions - - addressCountryDescriptions - entityReferenceNumber: entityReferenceNumber - addressStreets: - - addressStreets - - addressStreets - birthCountries: - - birthCountries - - birthCountries - matchedName: jane doe - fileGenerationDate: 28/10/2022 - entityRemark: entityRemark - properties: - fileGenerationDate: - example: 28/10/2022 - type: string - entityLogicalId: - example: 13 - type: integer - entityRemark: - type: string - entitySubjectType: - type: string - entityPublicationURL: - type: string - entityReferenceNumber: - type: string - nameAliasWholeNames: - items: - type: string - type: array - nameAliasTitles: - items: - type: string - type: array - addressCities: - items: - type: string - type: array - addressStreets: - items: - type: string - type: array - addressPoBoxes: - items: - type: string - type: array - addressZipCodes: - items: - type: string - type: array - addressCountryDescriptions: - items: - type: string - type: array - birthDates: - items: - type: string - type: array - birthCities: - items: - type: string - type: array - birthCountries: - items: - type: string - type: array - validFromTo: - type: object - match: - description: Match percentage of search query - example: 0.92 - type: number - matchedName: - description: The highest scoring term from the search query. This term is - the precomputed indexed value used by the search algorithm. - example: jane doe - type: string - UKConsolidatedSanctionsList: - example: - addresses: - - addresses - - addresses - groupType: groupType - names: - - names - - names - match: 0.92 - countries: - - countries - - countries - matchedName: jane doe - properties: - names: - items: - type: string - type: array - addresses: - items: - type: string - type: array - countries: - items: - type: string - type: array - groupType: - type: string - match: - description: Match percentage of search query - example: 0.92 - type: number - matchedName: - description: The highest scoring term from the search query. This term is - the precomputed indexed value used by the search algorithm. - example: jane doe - type: string - UKSanctionsList: - example: - stateLocalities: - - stateLocalities - - stateLocalities - addresses: - - addresses - - addresses - names: - - names - - names - entityType: entityType - addressCountries: - - addressCountries - - addressCountries - match: 0.92 - nonLatinNames: - - nonLatinNames - - nonLatinNames - matchedName: jane doe - properties: - names: - items: - type: string - type: array - nonLatinNames: - items: - type: string - type: array - entityType: - type: string - addresses: - items: - type: string - type: array - addressCountries: - items: - type: string - type: array - stateLocalities: - items: - type: string - type: array - match: - description: Match percentage of search query - example: 0.92 - type: number - matchedName: - description: The highest scoring term from the search query. This term is - the precomputed indexed value used by the search algorithm. - example: jane doe - type: string - Search: - description: Search results containing SDNs, alternate names, and/or addreses - example: - nonSDNChineseMilitaryIndustrialComplex: - - addresses: - - C/O Vistra Corporate Services Centre, Wickhams Cay II, Road Town, VG1110, - VG - entityNumber: "32091" - match: 0.92 - entityID: "32091" - type: Entity - sourceListURL: https://bit.ly/1QWTIfE - sourceInfoURL: https://bit.ly/3zsMQ4n - alternateNames: - - PROVEN HONOUR CAPITAL LTD - - PROVEN HONOUR - name: PROVEN HONOUR CAPITAL LIMITED - IDs: - - Proven Honour Capital Ltd, Issuer Name - - XS1233275194, ISIN - programs: - - CMIC-EO13959 - matchedName: jane doe - remarks: - - '(Linked To: HUAWEI INVESTMENT & HOLDING CO., LTD.)' - - addresses: - - C/O Vistra Corporate Services Centre, Wickhams Cay II, Road Town, VG1110, - VG - entityNumber: "32091" - match: 0.92 - entityID: "32091" - type: Entity - sourceListURL: https://bit.ly/1QWTIfE - sourceInfoURL: https://bit.ly/3zsMQ4n - alternateNames: - - PROVEN HONOUR CAPITAL LTD - - PROVEN HONOUR - name: PROVEN HONOUR CAPITAL LIMITED - IDs: - - Proven Honour Capital Ltd, Issuer Name - - XS1233275194, ISIN - programs: - - CMIC-EO13959 - matchedName: jane doe - remarks: - - '(Linked To: HUAWEI INVESTMENT & HOLDING CO., LTD.)' - altNames: - - alternateID: "220" - alternateRemarks: Extra information - alternateType: aka - match: 0.91 - entityID: "306" - alternateName: NATIONAL BANK OF CUBA - matchedName: jane doe - - alternateID: "220" - alternateRemarks: Extra information - alternateType: aka - match: 0.91 - entityID: "306" - alternateName: NATIONAL BANK OF CUBA - matchedName: jane doe - addresses: - - country: Japan - address: 123 73th St - cityStateProvincePostalCode: Tokyo 103 - match: 0.91 - entityID: "2112" - addressID: "201" - - country: Japan - address: 123 73th St - cityStateProvincePostalCode: Tokyo 103 - match: 0.91 - entityID: "2112" - addressID: "201" - deniedPersons: - - country: United States - city: THREE RIVERS - postalCode: "78071" - match: 0.92 - standardOrder: "Y" - frCitation: 81.F.R. 40658 6/22/2016 - streetAddress: 'REGISTER NUMBER: 78795-379, FEDERAL CORRECTIONAL INSTITUTION, - P.O. BOX 4200' - lastUpdate: 2016-06-22 - name: ISMAEL RETA - action: FR NOTICE ADDED - state: TX - matchedName: jane doe - effectiveDate: 06/15/2016 - expirationDate: 06/15/2025 - - country: United States - city: THREE RIVERS - postalCode: "78071" - match: 0.92 - standardOrder: "Y" - frCitation: 81.F.R. 40658 6/22/2016 - streetAddress: 'REGISTER NUMBER: 78795-379, FEDERAL CORRECTIONAL INSTITUTION, - P.O. BOX 4200' - lastUpdate: 2016-06-22 - name: ISMAEL RETA - action: FR NOTICE ADDED - state: TX - matchedName: jane doe - effectiveDate: 06/15/2016 - expirationDate: 06/15/2025 - ukConsolidatedSanctionsList: - - addresses: - - addresses - - addresses - groupType: groupType - names: - - names - - names - match: 0.92 - countries: - - countries - - countries - matchedName: jane doe - - addresses: - - addresses - - addresses - groupType: groupType - names: - - names - - names - match: 0.92 - countries: - - countries - - countries - matchedName: jane doe - palestinianLegislativeCouncil: - - addresses: - - 123 Dunbar Street, Testerville, TX, Palestine - entityNumber: "9702" - match: 0.92 - entityID: "9702" - datesOfBirth: "1951" - type: Individual - sourceListURL: https://bit.ly/1QWTIfE - sourceInfoURL: http://bit.ly/2tjOLpx - alternateNames: - - SALAMEH, Salem Ahmad Abdel Hadi - placesOfBirth: placesOfBirth - name: SALAMEH, Salem - programs: - - NS-PLC - - Office of Misinformation - matchedName: jane doe - remarks: HAMAS - Der al-Balah - - addresses: - - 123 Dunbar Street, Testerville, TX, Palestine - entityNumber: "9702" - match: 0.92 - entityID: "9702" - datesOfBirth: "1951" - type: Individual - sourceListURL: https://bit.ly/1QWTIfE - sourceInfoURL: http://bit.ly/2tjOLpx - alternateNames: - - SALAMEH, Salem Ahmad Abdel Hadi - placesOfBirth: placesOfBirth - name: SALAMEH, Salem - programs: - - NS-PLC - - Office of Misinformation - matchedName: jane doe - remarks: HAMAS - Der al-Balah - itarDebarred: - - sourceInfoURL: http://bit.ly/307FuRQ - alternateNames: - - Yasmin Tariq - - Fatimah Mohammad - federalRegisterNotice: 69 FR 17468 - name: Yasmin Ahmed - match: 0.92 - entityID: d44d88d0265d93927b9ff1c13bbbb7c7db64142c - sourceListURL: http://bit.ly/307FuRQ - matchedName: jane doe - - sourceInfoURL: http://bit.ly/307FuRQ - alternateNames: - - Yasmin Tariq - - Fatimah Mohammad - federalRegisterNotice: 69 FR 17468 - name: Yasmin Ahmed - match: 0.92 - entityID: d44d88d0265d93927b9ff1c13bbbb7c7db64142c - sourceListURL: http://bit.ly/307FuRQ - matchedName: jane doe - unverifiedCSL: - - sourceInfoURL: http://bit.ly/1Qi4R7Z - addresses: - - Komitas 26/114, Yerevan, Armenia, AM - name: Atlas Sanatgaran - match: 0.92 - entityID: f15fa805ff4ac5e09026f5e78011a1bb6b26dec2 - sourceListURL: http://bit.ly/1iwwTSJ - matchedName: jane doe - - sourceInfoURL: http://bit.ly/1Qi4R7Z - addresses: - - Komitas 26/114, Yerevan, Armenia, AM - name: Atlas Sanatgaran - match: 0.92 - entityID: f15fa805ff4ac5e09026f5e78011a1bb6b26dec2 - sourceListURL: http://bit.ly/1iwwTSJ - matchedName: jane doe - captaList: - - addresses: Bld 3 8/15, Rozhdestvenka St., Moscow, 107996, RU - entityNumber: "20002" - match: 0.92 - entityID: "20002" - type: Entity - sourceListURL: sourceListURL - sourceInfoURL: http://bit.ly/2PqohAD - alternateNames: - - BM BANK JSC - - BM BANK AO - - AKTSIONERNOE OBSHCHESTVO BM BANK - name: BM BANK PUBLIC JOINT STOCK COMPANY - IDs: - - RU, 1027700159497, Registration Number - - RU, 29292940, Government Gazette Number - - MOSWRUMM, SWIFT/BIC - programs: - - UKRAINE-EO13662 - - RUSSIA-EO14024 - matchedName: jane doe - remarks: - - All offices worldwide - - addresses: Bld 3 8/15, Rozhdestvenka St., Moscow, 107996, RU - entityNumber: "20002" - match: 0.92 - entityID: "20002" - type: Entity - sourceListURL: sourceListURL - sourceInfoURL: http://bit.ly/2PqohAD - alternateNames: - - BM BANK JSC - - BM BANK AO - - AKTSIONERNOE OBSHCHESTVO BM BANK - name: BM BANK PUBLIC JOINT STOCK COMPANY - IDs: - - RU, 1027700159497, Registration Number - - RU, 29292940, Government Gazette Number - - MOSWRUMM, SWIFT/BIC - programs: - - UKRAINE-EO13662 - - RUSSIA-EO14024 - matchedName: jane doe - remarks: - - All offices worldwide - SDNs: - - sdnType: individual - match: 0.91 - entityID: "1231" - programs: - - CUBA - sdnName: BANCO NACIONAL DE CUBA - title: Title of an individual - matchedName: jane doe - remarks: Additional info - - sdnType: individual - match: 0.91 - entityID: "1231" - programs: - - CUBA - sdnName: BANCO NACIONAL DE CUBA - title: Title of an individual - matchedName: jane doe - remarks: Additional info - ukSanctionsList: - - stateLocalities: - - stateLocalities - - stateLocalities - addresses: - - addresses - - addresses - names: - - names - - names - entityType: entityType - addressCountries: - - addressCountries - - addressCountries - match: 0.92 - nonLatinNames: - - nonLatinNames - - nonLatinNames - matchedName: jane doe - - stateLocalities: - - stateLocalities - - stateLocalities - addresses: - - addresses - - addresses - names: - - names - - names - entityType: entityType - addressCountries: - - addressCountries - - addressCountries - match: 0.92 - nonLatinNames: - - nonLatinNames - - nonLatinNames - matchedName: jane doe - bisEntities: - - licenseRequirement: For all items subject to the EAR. (See ¬ß744.11 of the - EAR). - sourceInfoURL: http://bit.ly/1MLgou0 - addresses: - - D. Retyum, Luzhski Raion, Leningradskaya Obl., 188230, RU - - Retiun Village, Lujskiy District, Leningrad Region, RU - alternateNames: - - VERKHNECHONSKNEFTEGAZ - - OJSC VERKHNECHONSKNEFTEGAZ - licensePolicy: Presumption of denial. - name: Luhansk People¬ís Republic - match: 0.91 - sourceListURL: http://bit.ly/1MLgou0 - matchedName: jane doe - startDate: 6/21/16 - frNotice: 81 FR 61595 - - licenseRequirement: For all items subject to the EAR. (See ¬ß744.11 of the - EAR). - sourceInfoURL: http://bit.ly/1MLgou0 - addresses: - - D. Retyum, Luzhski Raion, Leningradskaya Obl., 188230, RU - - Retiun Village, Lujskiy District, Leningrad Region, RU - alternateNames: - - VERKHNECHONSKNEFTEGAZ - - OJSC VERKHNECHONSKNEFTEGAZ - licensePolicy: Presumption of denial. - name: Luhansk People¬ís Republic - match: 0.91 - sourceListURL: http://bit.ly/1MLgou0 - matchedName: jane doe - startDate: 6/21/16 - frNotice: 81 FR 61595 - euConsolidatedSanctionsList: - - birthCities: - - birthCities - - birthCities - entitySubjectType: entitySubjectType - nameAliasTitles: - - nameAliasTitles - - nameAliasTitles - addressPoBoxes: - - addressPoBoxes - - addressPoBoxes - addressCities: - - addressCities - - addressCities - match: 0.92 - birthDates: - - birthDates - - birthDates - entityPublicationURL: entityPublicationURL - entityLogicalId: 13 - nameAliasWholeNames: - - nameAliasWholeNames - - nameAliasWholeNames - validFromTo: '{}' - addressZipCodes: - - addressZipCodes - - addressZipCodes - addressCountryDescriptions: - - addressCountryDescriptions - - addressCountryDescriptions - entityReferenceNumber: entityReferenceNumber - addressStreets: - - addressStreets - - addressStreets - birthCountries: - - birthCountries - - birthCountries - matchedName: jane doe - fileGenerationDate: 28/10/2022 - entityRemark: entityRemark - - birthCities: - - birthCities - - birthCities - entitySubjectType: entitySubjectType - nameAliasTitles: - - nameAliasTitles - - nameAliasTitles - addressPoBoxes: - - addressPoBoxes - - addressPoBoxes - addressCities: - - addressCities - - addressCities - match: 0.92 - birthDates: - - birthDates - - birthDates - entityPublicationURL: entityPublicationURL - entityLogicalId: 13 - nameAliasWholeNames: - - nameAliasWholeNames - - nameAliasWholeNames - validFromTo: '{}' - addressZipCodes: - - addressZipCodes - - addressZipCodes - addressCountryDescriptions: - - addressCountryDescriptions - - addressCountryDescriptions - entityReferenceNumber: entityReferenceNumber - addressStreets: - - addressStreets - - addressStreets - birthCountries: - - birthCountries - - birthCountries - matchedName: jane doe - fileGenerationDate: 28/10/2022 - entityRemark: entityRemark - militaryEndUsers: - - addresses: Xiujia Bay, Weiyong Dt, Xian, 710021, CN - endDate: endDate - name: AECC Aviation Power Co. Ltd. - match: 0.92 - FRNotice: 85 FR 83799 - entityID: 26744194bd9b5cbec49db6ee29a4b53c697c7420 - matchedName: jane doe - startDate: 2020-12-23 - - addresses: Xiujia Bay, Weiyong Dt, Xian, 710021, CN - endDate: endDate - name: AECC Aviation Power Co. Ltd. - match: 0.92 - FRNotice: 85 FR 83799 - entityID: 26744194bd9b5cbec49db6ee29a4b53c697c7420 - matchedName: jane doe - startDate: 2020-12-23 - foreignSanctionsEvaders: - - addresses: [] - entityNumber: "17526" - match: 0.92 - entityID: "17526" - datesOfBirth: 1966-02-13 - type: Individual - sourceListURL: https://bit.ly/1QWTIfE - sourceInfoURL: http://bit.ly/1N1docf - citizenships: CH - name: BEKTAS, Halis - IDs: - - CH, X0906223, Passport - programs: - - SYRIA - - FSE-SY - matchedName: jane doe - - addresses: [] - entityNumber: "17526" - match: 0.92 - entityID: "17526" - datesOfBirth: 1966-02-13 - type: Individual - sourceListURL: https://bit.ly/1QWTIfE - sourceInfoURL: http://bit.ly/1N1docf - citizenships: CH - name: BEKTAS, Halis - IDs: - - CH, X0906223, Passport - programs: - - SYRIA - - FSE-SY - matchedName: jane doe - nonSDNMenuBasedSanctionsList: - - EntityID: "17016" - Programs: - - UKRAINE-EO13662 - - MBS - Addresses: - - 16 Nametkina Street, Bldg. 1, Moscow, 117420, RU - SourceInfoURL: https://bit.ly/2MbsybU - Type: Entity - Remarks: - - 'For more information on directives, please visit the following link: - http://www.treasury.gov/resource-center/sanctions/Programs/Pages/ukraine.aspx#directives.' - match: 0.92 - IDs: - - RU, 1027700167110, Registration Number - - RU, 09807684, Government Gazette Number - - RU, 7744001497, Tax ID No. - EntityNumber: "17016" - AlternateNames: - - GAZPROMBANK OPEN JOINT STOCK COMPANY - - BANK GPB JSC - - GAZPROMBANK AO - matchedName: jane doe - Name: GAZPROMBANK JOINT STOCK COMPANY - - EntityID: "17016" - Programs: - - UKRAINE-EO13662 - - MBS - Addresses: - - 16 Nametkina Street, Bldg. 1, Moscow, 117420, RU - SourceInfoURL: https://bit.ly/2MbsybU - Type: Entity - Remarks: - - 'For more information on directives, please visit the following link: - http://www.treasury.gov/resource-center/sanctions/Programs/Pages/ukraine.aspx#directives.' - match: 0.92 - IDs: - - RU, 1027700167110, Registration Number - - RU, 09807684, Government Gazette Number - - RU, 7744001497, Tax ID No. - EntityNumber: "17016" - AlternateNames: - - GAZPROMBANK OPEN JOINT STOCK COMPANY - - BANK GPB JSC - - GAZPROMBANK AO - matchedName: jane doe - Name: GAZPROMBANK JOINT STOCK COMPANY - refreshedAt: 2000-01-23T04:56:07.000+00:00 - sectoralSanctions: - - sourceInfoURL: http://bit.ly/1MLgou0 - addresses: - - D. Retyum, Luzhski Raion, Leningradskaya Obl., 188230, RU - - Retiun Village, Lujskiy District, Leningrad Region, RU - alternateNames: - - VERKHNECHONSKNEFTEGAZ - - OJSC VERKHNECHONSKNEFTEGAZ - name: PJSC VERKHNECHONSKNEFTEGAZ - match: 0.91 - ids: - - Subject to Directive 4, Executive Order 13662 Directive Determination - - vcng@rosneft.ru, Email Address - - Subject to Directive 2, Executive Order 13662 Directive Determination - entityID: "1231" - programs: - - UKRAINE-EO13662 - - SYRIA - sourceListURL: http://bit.ly/1MLgou0 - matchedName: jane doe - remarks: - - 'For more information on directives, please visit the following link: - http://www.treasury.gov/resource-center/sanctions/Programs/Pages/ukraine.aspx#directives.' - - '(Linked To: OPEN JOINT-STOCK COMPANY ROSNEFT OIL COMPANY)' - - sourceInfoURL: http://bit.ly/1MLgou0 - addresses: - - D. Retyum, Luzhski Raion, Leningradskaya Obl., 188230, RU - - Retiun Village, Lujskiy District, Leningrad Region, RU - alternateNames: - - VERKHNECHONSKNEFTEGAZ - - OJSC VERKHNECHONSKNEFTEGAZ - name: PJSC VERKHNECHONSKNEFTEGAZ - match: 0.91 - ids: - - Subject to Directive 4, Executive Order 13662 Directive Determination - - vcng@rosneft.ru, Email Address - - Subject to Directive 2, Executive Order 13662 Directive Determination - entityID: "1231" - programs: - - UKRAINE-EO13662 - - SYRIA - sourceListURL: http://bit.ly/1MLgou0 - matchedName: jane doe - remarks: - - 'For more information on directives, please visit the following link: - http://www.treasury.gov/resource-center/sanctions/Programs/Pages/ukraine.aspx#directives.' - - '(Linked To: OPEN JOINT-STOCK COMPANY ROSNEFT OIL COMPANY)' - nonproliferationSanctions: - - sourceInfoURL: http://bit.ly/1NuVFxV - alternateNames: - - ZAMAN - - Haydar - federalRegisterNotice: Vol. 74, No. 11, 01/16/09 - name: Abdul Qadeer Khan - match: 0.92 - entityID: 2d2db09c686e4829d0ef1b0b04145eec3d42cd88 - programs: - - E.O. 13382 - - Export-Import Bank Act - - Nuclear Proliferation Prevention Act - sourceListURL: http://bit.ly/1NuVFxV - matchedName: jane doe - startDate: 2009-01-09 - remarks: Associated with the A.Q. Khan Network - - sourceInfoURL: http://bit.ly/1NuVFxV - alternateNames: - - ZAMAN - - Haydar - federalRegisterNotice: Vol. 74, No. 11, 01/16/09 - name: Abdul Qadeer Khan - match: 0.92 - entityID: 2d2db09c686e4829d0ef1b0b04145eec3d42cd88 - programs: - - E.O. 13382 - - Export-Import Bank Act - - Nuclear Proliferation Prevention Act - sourceListURL: http://bit.ly/1NuVFxV - matchedName: jane doe - startDate: 2009-01-09 - remarks: Associated with the A.Q. Khan Network - properties: - SDNs: - items: - $ref: '#/components/schemas/OfacSDN' - type: array - altNames: - items: - $ref: '#/components/schemas/OfacAlt' - type: array - addresses: - items: - $ref: '#/components/schemas/OfacEntityAddress' - type: array - deniedPersons: - items: - $ref: '#/components/schemas/DPL' - type: array - bisEntities: - items: - $ref: '#/components/schemas/BISEntities' - type: array - militaryEndUsers: - items: - $ref: '#/components/schemas/MilitaryEndUser' - type: array - sectoralSanctions: - items: - $ref: '#/components/schemas/SSI' - type: array - unverifiedCSL: - items: - $ref: '#/components/schemas/Unverified' - type: array - nonproliferationSanctions: - items: - $ref: '#/components/schemas/NonProliferationSanction' - type: array - foreignSanctionsEvaders: - items: - $ref: '#/components/schemas/ForeignSanctionsEvader' - type: array - palestinianLegislativeCouncil: - items: - $ref: '#/components/schemas/PalestinianLegislativeCouncil' - type: array - captaList: - items: - $ref: '#/components/schemas/CAPTAList' - type: array - itarDebarred: - items: - $ref: '#/components/schemas/ITARDebarred' - type: array - nonSDNChineseMilitaryIndustrialComplex: - items: - $ref: '#/components/schemas/NonSDNChineseMilitaryIndustrialComplex' - type: array - nonSDNMenuBasedSanctionsList: - items: - $ref: '#/components/schemas/NonSDNMenuBasedSanctionsList' - type: array - euConsolidatedSanctionsList: - items: - $ref: '#/components/schemas/EUConsolidatedSanctionsList' - type: array - ukConsolidatedSanctionsList: - items: - $ref: '#/components/schemas/UKConsolidatedSanctionsList' - type: array - ukSanctionsList: - items: - $ref: '#/components/schemas/UKSanctionsList' - type: array - refreshedAt: - format: date-time - type: string - Downloads: - items: - $ref: '#/components/schemas/Download' - type: array - Download: - description: Metadata and stats about downloaded OFAC data - example: - SDNs: 7414 - altNames: 9729 - addresses: 11747 - deniedPersons: 842 - bisEntities: 1391 - sectoralSanctions: 329 - timestamp: 2000-01-23T04:56:07.000+00:00 - properties: - SDNs: - example: 7414 - type: integer - altNames: - example: 9729 - type: integer - addresses: - example: 11747 - type: integer - sectoralSanctions: - example: 329 - type: integer - deniedPersons: - example: 842 - type: integer - bisEntities: - example: 1391 - type: integer - timestamp: - format: date-time - type: string - UIKeys: - items: - $ref: '#/components/schemas/SdnType' - type: array - uniqueItems: true - Error: - properties: - error: - description: An error message describing the problem intended for humans. - example: Example error, see description - type: string - required: - - error diff --git a/client/api_watchman.go b/client/api_watchman.go deleted file mode 100644 index a3f225f0..00000000 --- a/client/api_watchman.go +++ /dev/null @@ -1,752 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -import ( - _context "context" - "github.com/antihax/optional" - _ioutil "io/ioutil" - _nethttp "net/http" - _neturl "net/url" - "strings" -) - -// Linger please -var ( - _ _context.Context -) - -// WatchmanApiService WatchmanApi service -type WatchmanApiService service - -// GetLatestDownloadsOpts Optional parameters for the method 'GetLatestDownloads' -type GetLatestDownloadsOpts struct { - XRequestID optional.String - Limit optional.Int32 -} - -/* -GetLatestDownloads Get latest downloads -Return list of recent downloads of list data. - - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). - - @param optional nil or *GetLatestDownloadsOpts - Optional Parameters: - - @param "XRequestID" (optional.String) - Optional Request ID allows application developer to trace requests through the system's logs - - @param "Limit" (optional.Int32) - Maximum number of downloads to return sorted by their timestamp in decending order. - -@return []Download -*/ -func (a *WatchmanApiService) GetLatestDownloads(ctx _context.Context, localVarOptionals *GetLatestDownloadsOpts) ([]Download, *_nethttp.Response, error) { - var ( - localVarHTTPMethod = _nethttp.MethodGet - localVarPostBody interface{} - localVarFormFileName string - localVarFileName string - localVarFileBytes []byte - localVarReturnValue []Download - ) - - // create path and map variables - localVarPath := a.client.cfg.BasePath + "/downloads" - localVarHeaderParams := make(map[string]string) - localVarQueryParams := _neturl.Values{} - localVarFormParams := _neturl.Values{} - - if localVarOptionals != nil && localVarOptionals.Limit.IsSet() { - localVarQueryParams.Add("limit", parameterToString(localVarOptionals.Limit.Value(), "")) - } - // to determine the Content-Type header - localVarHTTPContentTypes := []string{} - - // set Content-Type header - localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) - if localVarHTTPContentType != "" { - localVarHeaderParams["Content-Type"] = localVarHTTPContentType - } - - // to determine the Accept header - localVarHTTPHeaderAccepts := []string{"application/json"} - - // set Accept header - localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) - if localVarHTTPHeaderAccept != "" { - localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept - } - if localVarOptionals != nil && localVarOptionals.XRequestID.IsSet() { - localVarHeaderParams["X-Request-ID"] = parameterToString(localVarOptionals.XRequestID.Value(), "") - } - r, err := a.client.prepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes) - if err != nil { - return localVarReturnValue, nil, err - } - - localVarHTTPResponse, err := a.client.callAPI(r) - if err != nil || localVarHTTPResponse == nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) - localVarHTTPResponse.Body.Close() - if err != nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - if localVarHTTPResponse.StatusCode >= 300 { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: localVarHTTPResponse.Status, - } - if localVarHTTPResponse.StatusCode == 400 { - var v Error - err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) - if err != nil { - newErr.error = err.Error() - return localVarReturnValue, localVarHTTPResponse, newErr - } - newErr.model = v - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) - if err != nil { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: err.Error(), - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - return localVarReturnValue, localVarHTTPResponse, nil -} - -// GetSDNAddressesOpts Optional parameters for the method 'GetSDNAddresses' -type GetSDNAddressesOpts struct { - XRequestID optional.String -} - -/* -GetSDNAddresses Get SDN addresses - - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). - - @param sdnID SDN ID - - @param optional nil or *GetSDNAddressesOpts - Optional Parameters: - - @param "XRequestID" (optional.String) - Optional Request ID allows application developer to trace requests through the system's logs - -@return []OfacEntityAddress -*/ -func (a *WatchmanApiService) GetSDNAddresses(ctx _context.Context, sdnID string, localVarOptionals *GetSDNAddressesOpts) ([]OfacEntityAddress, *_nethttp.Response, error) { - var ( - localVarHTTPMethod = _nethttp.MethodGet - localVarPostBody interface{} - localVarFormFileName string - localVarFileName string - localVarFileBytes []byte - localVarReturnValue []OfacEntityAddress - ) - - // create path and map variables - localVarPath := a.client.cfg.BasePath + "/ofac/sdn/{sdnID}/addresses" - localVarPath = strings.Replace(localVarPath, "{"+"sdnID"+"}", _neturl.QueryEscape(parameterToString(sdnID, "")), -1) - - localVarHeaderParams := make(map[string]string) - localVarQueryParams := _neturl.Values{} - localVarFormParams := _neturl.Values{} - - // to determine the Content-Type header - localVarHTTPContentTypes := []string{} - - // set Content-Type header - localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) - if localVarHTTPContentType != "" { - localVarHeaderParams["Content-Type"] = localVarHTTPContentType - } - - // to determine the Accept header - localVarHTTPHeaderAccepts := []string{"application/json"} - - // set Accept header - localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) - if localVarHTTPHeaderAccept != "" { - localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept - } - if localVarOptionals != nil && localVarOptionals.XRequestID.IsSet() { - localVarHeaderParams["X-Request-ID"] = parameterToString(localVarOptionals.XRequestID.Value(), "") - } - r, err := a.client.prepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes) - if err != nil { - return localVarReturnValue, nil, err - } - - localVarHTTPResponse, err := a.client.callAPI(r) - if err != nil || localVarHTTPResponse == nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) - localVarHTTPResponse.Body.Close() - if err != nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - if localVarHTTPResponse.StatusCode >= 300 { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: localVarHTTPResponse.Status, - } - if localVarHTTPResponse.StatusCode == 400 { - var v Error - err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) - if err != nil { - newErr.error = err.Error() - return localVarReturnValue, localVarHTTPResponse, newErr - } - newErr.model = v - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) - if err != nil { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: err.Error(), - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - return localVarReturnValue, localVarHTTPResponse, nil -} - -// GetSDNAltNamesOpts Optional parameters for the method 'GetSDNAltNames' -type GetSDNAltNamesOpts struct { - XRequestID optional.String -} - -/* -GetSDNAltNames Get SDN alt names - - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). - - @param sdnID SDN ID - - @param optional nil or *GetSDNAltNamesOpts - Optional Parameters: - - @param "XRequestID" (optional.String) - Optional Request ID allows application developer to trace requests through the system's logs - -@return []OfacAlt -*/ -func (a *WatchmanApiService) GetSDNAltNames(ctx _context.Context, sdnID string, localVarOptionals *GetSDNAltNamesOpts) ([]OfacAlt, *_nethttp.Response, error) { - var ( - localVarHTTPMethod = _nethttp.MethodGet - localVarPostBody interface{} - localVarFormFileName string - localVarFileName string - localVarFileBytes []byte - localVarReturnValue []OfacAlt - ) - - // create path and map variables - localVarPath := a.client.cfg.BasePath + "/ofac/sdn/{sdnID}/alts" - localVarPath = strings.Replace(localVarPath, "{"+"sdnID"+"}", _neturl.QueryEscape(parameterToString(sdnID, "")), -1) - - localVarHeaderParams := make(map[string]string) - localVarQueryParams := _neturl.Values{} - localVarFormParams := _neturl.Values{} - - // to determine the Content-Type header - localVarHTTPContentTypes := []string{} - - // set Content-Type header - localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) - if localVarHTTPContentType != "" { - localVarHeaderParams["Content-Type"] = localVarHTTPContentType - } - - // to determine the Accept header - localVarHTTPHeaderAccepts := []string{"application/json"} - - // set Accept header - localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) - if localVarHTTPHeaderAccept != "" { - localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept - } - if localVarOptionals != nil && localVarOptionals.XRequestID.IsSet() { - localVarHeaderParams["X-Request-ID"] = parameterToString(localVarOptionals.XRequestID.Value(), "") - } - r, err := a.client.prepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes) - if err != nil { - return localVarReturnValue, nil, err - } - - localVarHTTPResponse, err := a.client.callAPI(r) - if err != nil || localVarHTTPResponse == nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) - localVarHTTPResponse.Body.Close() - if err != nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - if localVarHTTPResponse.StatusCode >= 300 { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: localVarHTTPResponse.Status, - } - if localVarHTTPResponse.StatusCode == 400 { - var v Error - err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) - if err != nil { - newErr.error = err.Error() - return localVarReturnValue, localVarHTTPResponse, newErr - } - newErr.model = v - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) - if err != nil { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: err.Error(), - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - return localVarReturnValue, localVarHTTPResponse, nil -} - -// GetUIValuesOpts Optional parameters for the method 'GetUIValues' -type GetUIValuesOpts struct { - Limit optional.Int32 -} - -/* -GetUIValues Get UI values -Return an ordered distinct list of keys for an SDN property. - - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). - - @param key SDN property to lookup. Values are sdnType, ofacProgram - - @param optional nil or *GetUIValuesOpts - Optional Parameters: - - @param "Limit" (optional.Int32) - Maximum number of UI keys returned - -@return []SdnType -*/ -func (a *WatchmanApiService) GetUIValues(ctx _context.Context, key SdnType, localVarOptionals *GetUIValuesOpts) ([]SdnType, *_nethttp.Response, error) { - var ( - localVarHTTPMethod = _nethttp.MethodGet - localVarPostBody interface{} - localVarFormFileName string - localVarFileName string - localVarFileBytes []byte - localVarReturnValue []SdnType - ) - - // create path and map variables - localVarPath := a.client.cfg.BasePath + "/ui/values/{key}" - localVarPath = strings.Replace(localVarPath, "{"+"key"+"}", _neturl.QueryEscape(parameterToString(key, "")), -1) - - localVarHeaderParams := make(map[string]string) - localVarQueryParams := _neturl.Values{} - localVarFormParams := _neturl.Values{} - - if localVarOptionals != nil && localVarOptionals.Limit.IsSet() { - localVarQueryParams.Add("limit", parameterToString(localVarOptionals.Limit.Value(), "")) - } - // to determine the Content-Type header - localVarHTTPContentTypes := []string{} - - // set Content-Type header - localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) - if localVarHTTPContentType != "" { - localVarHeaderParams["Content-Type"] = localVarHTTPContentType - } - - // to determine the Accept header - localVarHTTPHeaderAccepts := []string{"application/json"} - - // set Accept header - localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) - if localVarHTTPHeaderAccept != "" { - localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept - } - r, err := a.client.prepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes) - if err != nil { - return localVarReturnValue, nil, err - } - - localVarHTTPResponse, err := a.client.callAPI(r) - if err != nil || localVarHTTPResponse == nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) - localVarHTTPResponse.Body.Close() - if err != nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - if localVarHTTPResponse.StatusCode >= 300 { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: localVarHTTPResponse.Status, - } - if localVarHTTPResponse.StatusCode == 400 { - var v Error - err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) - if err != nil { - newErr.error = err.Error() - return localVarReturnValue, localVarHTTPResponse, newErr - } - newErr.model = v - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) - if err != nil { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: err.Error(), - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - return localVarReturnValue, localVarHTTPResponse, nil -} - -/* -Ping Ping Watchman service -Check if the Watchman service is running. - - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). -*/ -func (a *WatchmanApiService) Ping(ctx _context.Context) (*_nethttp.Response, error) { - var ( - localVarHTTPMethod = _nethttp.MethodGet - localVarPostBody interface{} - localVarFormFileName string - localVarFileName string - localVarFileBytes []byte - ) - - // create path and map variables - localVarPath := a.client.cfg.BasePath + "/ping" - localVarHeaderParams := make(map[string]string) - localVarQueryParams := _neturl.Values{} - localVarFormParams := _neturl.Values{} - - // to determine the Content-Type header - localVarHTTPContentTypes := []string{} - - // set Content-Type header - localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) - if localVarHTTPContentType != "" { - localVarHeaderParams["Content-Type"] = localVarHTTPContentType - } - - // to determine the Accept header - localVarHTTPHeaderAccepts := []string{} - - // set Accept header - localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) - if localVarHTTPHeaderAccept != "" { - localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept - } - r, err := a.client.prepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes) - if err != nil { - return nil, err - } - - localVarHTTPResponse, err := a.client.callAPI(r) - if err != nil || localVarHTTPResponse == nil { - return localVarHTTPResponse, err - } - - localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) - localVarHTTPResponse.Body.Close() - if err != nil { - return localVarHTTPResponse, err - } - - if localVarHTTPResponse.StatusCode >= 300 { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: localVarHTTPResponse.Status, - } - return localVarHTTPResponse, newErr - } - - return localVarHTTPResponse, nil -} - -// SearchOpts Optional parameters for the method 'Search' -type SearchOpts struct { - XRequestID optional.String - Q optional.String - Name optional.String - Address optional.String - City optional.String - State optional.String - Providence optional.String - Zip optional.String - Country optional.String - AltName optional.String - Id optional.String - MinMatch optional.Float32 - Limit optional.Int32 - SdnType optional.Interface - Program optional.String -} - -/* -Search Search - - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). - - @param optional nil or *SearchOpts - Optional Parameters: - - @param "XRequestID" (optional.String) - Optional Request ID allows application developer to trace requests through the system's logs - - @param "Q" (optional.String) - Search across Name, Alt Names, and SDN Address fields for all available sanctions lists. Entries may be returned in all response sub-objects. - - @param "Name" (optional.String) - Name which could correspond to an entry on the SDN, Denied Persons, Sectoral Sanctions Identifications, or BIS Entity List sanctions lists. Alt names are also searched. - - @param "Address" (optional.String) - Physical address which could correspond to a human on the SDN list. Only Address results will be returned. - - @param "City" (optional.String) - City name as desginated by SDN guidelines. Only Address results will be returned. - - @param "State" (optional.String) - State name as desginated by SDN guidelines. Only Address results will be returned. - - @param "Providence" (optional.String) - Providence name as desginated by SDN guidelines. Only Address results will be returned. - - @param "Zip" (optional.String) - Zip code as desginated by SDN guidelines. Only Address results will be returned. - - @param "Country" (optional.String) - Country name as desginated by SDN guidelines. Only Address results will be returned. - - @param "AltName" (optional.String) - Alternate name which could correspond to a human on the SDN list. Only Alt name results will be returned. - - @param "Id" (optional.String) - ID value often found in remarks property of an SDN. Takes the form of 'No. NNNNN' as an alphanumeric value. - - @param "MinMatch" (optional.Float32) - Match percentage that search query must obtain for results to be returned. - - @param "Limit" (optional.Int32) - Maximum results returned by a search. Results are sorted by their match percentage in decending order. - - @param "SdnType" (optional.Interface of SdnType) - Optional filter to only return SDNs whose type case-insensitively matches. - - @param "Program" (optional.String) - Optional filter to only return SDNs whose program case-insensitively matches. - -@return Search -*/ -func (a *WatchmanApiService) Search(ctx _context.Context, localVarOptionals *SearchOpts) (Search, *_nethttp.Response, error) { - var ( - localVarHTTPMethod = _nethttp.MethodGet - localVarPostBody interface{} - localVarFormFileName string - localVarFileName string - localVarFileBytes []byte - localVarReturnValue Search - ) - - // create path and map variables - localVarPath := a.client.cfg.BasePath + "/search" - localVarHeaderParams := make(map[string]string) - localVarQueryParams := _neturl.Values{} - localVarFormParams := _neturl.Values{} - - if localVarOptionals != nil && localVarOptionals.Q.IsSet() { - localVarQueryParams.Add("q", parameterToString(localVarOptionals.Q.Value(), "")) - } - if localVarOptionals != nil && localVarOptionals.Name.IsSet() { - localVarQueryParams.Add("name", parameterToString(localVarOptionals.Name.Value(), "")) - } - if localVarOptionals != nil && localVarOptionals.Address.IsSet() { - localVarQueryParams.Add("address", parameterToString(localVarOptionals.Address.Value(), "")) - } - if localVarOptionals != nil && localVarOptionals.City.IsSet() { - localVarQueryParams.Add("city", parameterToString(localVarOptionals.City.Value(), "")) - } - if localVarOptionals != nil && localVarOptionals.State.IsSet() { - localVarQueryParams.Add("state", parameterToString(localVarOptionals.State.Value(), "")) - } - if localVarOptionals != nil && localVarOptionals.Providence.IsSet() { - localVarQueryParams.Add("providence", parameterToString(localVarOptionals.Providence.Value(), "")) - } - if localVarOptionals != nil && localVarOptionals.Zip.IsSet() { - localVarQueryParams.Add("zip", parameterToString(localVarOptionals.Zip.Value(), "")) - } - if localVarOptionals != nil && localVarOptionals.Country.IsSet() { - localVarQueryParams.Add("country", parameterToString(localVarOptionals.Country.Value(), "")) - } - if localVarOptionals != nil && localVarOptionals.AltName.IsSet() { - localVarQueryParams.Add("altName", parameterToString(localVarOptionals.AltName.Value(), "")) - } - if localVarOptionals != nil && localVarOptionals.Id.IsSet() { - localVarQueryParams.Add("id", parameterToString(localVarOptionals.Id.Value(), "")) - } - if localVarOptionals != nil && localVarOptionals.MinMatch.IsSet() { - localVarQueryParams.Add("minMatch", parameterToString(localVarOptionals.MinMatch.Value(), "")) - } - if localVarOptionals != nil && localVarOptionals.Limit.IsSet() { - localVarQueryParams.Add("limit", parameterToString(localVarOptionals.Limit.Value(), "")) - } - if localVarOptionals != nil && localVarOptionals.SdnType.IsSet() { - localVarQueryParams.Add("sdnType", parameterToString(localVarOptionals.SdnType.Value(), "")) - } - if localVarOptionals != nil && localVarOptionals.Program.IsSet() { - localVarQueryParams.Add("program", parameterToString(localVarOptionals.Program.Value(), "")) - } - // to determine the Content-Type header - localVarHTTPContentTypes := []string{} - - // set Content-Type header - localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) - if localVarHTTPContentType != "" { - localVarHeaderParams["Content-Type"] = localVarHTTPContentType - } - - // to determine the Accept header - localVarHTTPHeaderAccepts := []string{"application/json"} - - // set Accept header - localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) - if localVarHTTPHeaderAccept != "" { - localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept - } - if localVarOptionals != nil && localVarOptionals.XRequestID.IsSet() { - localVarHeaderParams["X-Request-ID"] = parameterToString(localVarOptionals.XRequestID.Value(), "") - } - r, err := a.client.prepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes) - if err != nil { - return localVarReturnValue, nil, err - } - - localVarHTTPResponse, err := a.client.callAPI(r) - if err != nil || localVarHTTPResponse == nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) - localVarHTTPResponse.Body.Close() - if err != nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - if localVarHTTPResponse.StatusCode >= 300 { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: localVarHTTPResponse.Status, - } - if localVarHTTPResponse.StatusCode == 400 { - var v Error - err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) - if err != nil { - newErr.error = err.Error() - return localVarReturnValue, localVarHTTPResponse, newErr - } - newErr.model = v - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) - if err != nil { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: err.Error(), - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - return localVarReturnValue, localVarHTTPResponse, nil -} - -// SearchUSCSLOpts Optional parameters for the method 'SearchUSCSL' -type SearchUSCSLOpts struct { - XRequestID optional.String - Name optional.String - Limit optional.Int32 -} - -/* -SearchUSCSL Search US CSL -Search the US Consolidated Screening List - - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). - - @param optional nil or *SearchUSCSLOpts - Optional Parameters: - - @param "XRequestID" (optional.String) - Optional Request ID allows application developer to trace requests through the system's logs - - @param "Name" (optional.String) - Name which could correspond to an entry on the CSL - - @param "Limit" (optional.Int32) - Maximum number of downloads to return sorted by their timestamp in decending order. - -@return Search -*/ -func (a *WatchmanApiService) SearchUSCSL(ctx _context.Context, localVarOptionals *SearchUSCSLOpts) (Search, *_nethttp.Response, error) { - var ( - localVarHTTPMethod = _nethttp.MethodGet - localVarPostBody interface{} - localVarFormFileName string - localVarFileName string - localVarFileBytes []byte - localVarReturnValue Search - ) - - // create path and map variables - localVarPath := a.client.cfg.BasePath + "/search/us-csl" - localVarHeaderParams := make(map[string]string) - localVarQueryParams := _neturl.Values{} - localVarFormParams := _neturl.Values{} - - if localVarOptionals != nil && localVarOptionals.Name.IsSet() { - localVarQueryParams.Add("name", parameterToString(localVarOptionals.Name.Value(), "")) - } - if localVarOptionals != nil && localVarOptionals.Limit.IsSet() { - localVarQueryParams.Add("limit", parameterToString(localVarOptionals.Limit.Value(), "")) - } - // to determine the Content-Type header - localVarHTTPContentTypes := []string{} - - // set Content-Type header - localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) - if localVarHTTPContentType != "" { - localVarHeaderParams["Content-Type"] = localVarHTTPContentType - } - - // to determine the Accept header - localVarHTTPHeaderAccepts := []string{"application/json"} - - // set Accept header - localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) - if localVarHTTPHeaderAccept != "" { - localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept - } - if localVarOptionals != nil && localVarOptionals.XRequestID.IsSet() { - localVarHeaderParams["X-Request-ID"] = parameterToString(localVarOptionals.XRequestID.Value(), "") - } - r, err := a.client.prepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes) - if err != nil { - return localVarReturnValue, nil, err - } - - localVarHTTPResponse, err := a.client.callAPI(r) - if err != nil || localVarHTTPResponse == nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) - localVarHTTPResponse.Body.Close() - if err != nil { - return localVarReturnValue, localVarHTTPResponse, err - } - - if localVarHTTPResponse.StatusCode >= 300 { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: localVarHTTPResponse.Status, - } - if localVarHTTPResponse.StatusCode == 400 { - var v Error - err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) - if err != nil { - newErr.error = err.Error() - return localVarReturnValue, localVarHTTPResponse, newErr - } - newErr.model = v - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) - if err != nil { - newErr := GenericOpenAPIError{ - body: localVarBody, - error: err.Error(), - } - return localVarReturnValue, localVarHTTPResponse, newErr - } - - return localVarReturnValue, localVarHTTPResponse, nil -} diff --git a/client/client.go b/client/client.go deleted file mode 100644 index 9082221f..00000000 --- a/client/client.go +++ /dev/null @@ -1,545 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -import ( - "bytes" - "context" - "encoding/json" - "encoding/xml" - "errors" - "fmt" - "io" - "io/ioutil" - "log" - "mime/multipart" - "net/http" - "net/http/httputil" - "net/url" - "os" - "path/filepath" - "reflect" - "regexp" - "strconv" - "strings" - "time" - "unicode/utf8" - - "golang.org/x/oauth2" -) - -var ( - jsonCheck = regexp.MustCompile(`(?i:(?:application|text)/(?:vnd\.[^;]+\+)?json)`) - xmlCheck = regexp.MustCompile(`(?i:(?:application|text)/xml)`) -) - -// APIClient manages communication with the Watchman API API vv1 -// In most cases there should be only one, shared, APIClient. -type APIClient struct { - cfg *Configuration - common service // Reuse a single struct instead of allocating one for each service on the heap. - - // API Services - - WatchmanApi *WatchmanApiService -} - -type service struct { - client *APIClient -} - -// NewAPIClient creates a new API client. Requires a userAgent string describing your application. -// optionally a custom http.Client to allow for advanced features such as caching. -func NewAPIClient(cfg *Configuration) *APIClient { - if cfg.HTTPClient == nil { - cfg.HTTPClient = http.DefaultClient - } - - c := &APIClient{} - c.cfg = cfg - c.common.client = c - - // API Services - c.WatchmanApi = (*WatchmanApiService)(&c.common) - - return c -} - -func atoi(in string) (int, error) { - return strconv.Atoi(in) -} - -// selectHeaderContentType select a content type from the available list. -func selectHeaderContentType(contentTypes []string) string { - if len(contentTypes) == 0 { - return "" - } - if contains(contentTypes, "application/json") { - return "application/json" - } - return contentTypes[0] // use the first content type specified in 'consumes' -} - -// selectHeaderAccept join all accept types and return -func selectHeaderAccept(accepts []string) string { - if len(accepts) == 0 { - return "" - } - - if contains(accepts, "application/json") { - return "application/json" - } - - return strings.Join(accepts, ",") -} - -// contains is a case insenstive match, finding needle in a haystack -func contains(haystack []string, needle string) bool { - for _, a := range haystack { - if strings.ToLower(a) == strings.ToLower(needle) { - return true - } - } - return false -} - -// Verify optional parameters are of the correct type. -func typeCheckParameter(obj interface{}, expected string, name string) error { - // Make sure there is an object. - if obj == nil { - return nil - } - - // Check the type is as expected. - if reflect.TypeOf(obj).String() != expected { - return fmt.Errorf("Expected %s to be of type %s but received %s.", name, expected, reflect.TypeOf(obj).String()) - } - return nil -} - -// parameterToString convert interface{} parameters to string, using a delimiter if format is provided. -func parameterToString(obj interface{}, collectionFormat string) string { - var delimiter string - - switch collectionFormat { - case "pipes": - delimiter = "|" - case "ssv": - delimiter = " " - case "tsv": - delimiter = "\t" - case "csv": - delimiter = "," - } - - if reflect.TypeOf(obj).Kind() == reflect.Slice { - return strings.Trim(strings.Replace(fmt.Sprint(obj), " ", delimiter, -1), "[]") - } else if t, ok := obj.(time.Time); ok { - return t.Format(time.RFC3339) - } - - return fmt.Sprintf("%v", obj) -} - -// helper for converting interface{} parameters to json strings -func parameterToJson(obj interface{}) (string, error) { - jsonBuf, err := json.Marshal(obj) - if err != nil { - return "", err - } - return string(jsonBuf), err -} - -// callAPI do the request. -func (c *APIClient) callAPI(request *http.Request) (*http.Response, error) { - if c.cfg.Debug { - dump, err := httputil.DumpRequestOut(request, true) - if err != nil { - return nil, err - } - log.Printf("\n%s\n", string(dump)) - } - - resp, err := c.cfg.HTTPClient.Do(request) - if err != nil { - return resp, err - } - - if c.cfg.Debug { - dump, err := httputil.DumpResponse(resp, true) - if err != nil { - return resp, err - } - log.Printf("\n%s\n", string(dump)) - } - - return resp, err -} - -// ChangeBasePath changes base path to allow switching to mocks -func (c *APIClient) ChangeBasePath(path string) { - c.cfg.BasePath = path -} - -// Allow modification of underlying config for alternate implementations and testing -// Caution: modifying the configuration while live can cause data races and potentially unwanted behavior -func (c *APIClient) GetConfig() *Configuration { - return c.cfg -} - -// prepareRequest build the request -func (c *APIClient) prepareRequest( - ctx context.Context, - path string, method string, - postBody interface{}, - headerParams map[string]string, - queryParams url.Values, - formParams url.Values, - formFileName string, - fileName string, - fileBytes []byte) (localVarRequest *http.Request, err error) { - - var body *bytes.Buffer - - // Detect postBody type and post. - if postBody != nil { - contentType := headerParams["Content-Type"] - if contentType == "" { - contentType = detectContentType(postBody) - headerParams["Content-Type"] = contentType - } - - body, err = setBody(postBody, contentType) - if err != nil { - return nil, err - } - } - - // add form parameters and file if available. - if strings.HasPrefix(headerParams["Content-Type"], "multipart/form-data") && len(formParams) > 0 || (len(fileBytes) > 0 && fileName != "") { - if body != nil { - return nil, errors.New("Cannot specify postBody and multipart form at the same time.") - } - body = &bytes.Buffer{} - w := multipart.NewWriter(body) - - for k, v := range formParams { - for _, iv := range v { - if strings.HasPrefix(k, "@") { // file - err = addFile(w, k[1:], iv) - if err != nil { - return nil, err - } - } else { // form value - w.WriteField(k, iv) - } - } - } - if len(fileBytes) > 0 && fileName != "" { - w.Boundary() - //_, fileNm := filepath.Split(fileName) - part, err := w.CreateFormFile(formFileName, filepath.Base(fileName)) - if err != nil { - return nil, err - } - _, err = part.Write(fileBytes) - if err != nil { - return nil, err - } - } - - // Set the Boundary in the Content-Type - headerParams["Content-Type"] = w.FormDataContentType() - - // Set Content-Length - headerParams["Content-Length"] = fmt.Sprintf("%d", body.Len()) - w.Close() - } - - if strings.HasPrefix(headerParams["Content-Type"], "application/x-www-form-urlencoded") && len(formParams) > 0 { - if body != nil { - return nil, errors.New("Cannot specify postBody and x-www-form-urlencoded form at the same time.") - } - body = &bytes.Buffer{} - body.WriteString(formParams.Encode()) - // Set Content-Length - headerParams["Content-Length"] = fmt.Sprintf("%d", body.Len()) - } - - // Setup path and query parameters - url, err := url.Parse(path) - if err != nil { - return nil, err - } - - // Override request host, if applicable - if c.cfg.Host != "" { - url.Host = c.cfg.Host - } - - // Override request scheme, if applicable - if c.cfg.Scheme != "" { - url.Scheme = c.cfg.Scheme - } - - // Adding Query Param - query := url.Query() - for k, v := range queryParams { - for _, iv := range v { - query.Add(k, iv) - } - } - - // Encode the parameters. - url.RawQuery = query.Encode() - - // Generate a new request - if body != nil { - localVarRequest, err = http.NewRequest(method, url.String(), body) - } else { - localVarRequest, err = http.NewRequest(method, url.String(), nil) - } - if err != nil { - return nil, err - } - - // add header parameters, if any - if len(headerParams) > 0 { - headers := http.Header{} - for h, v := range headerParams { - headers.Set(h, v) - } - localVarRequest.Header = headers - } - - // Add the user agent to the request. - localVarRequest.Header.Add("User-Agent", c.cfg.UserAgent) - - if ctx != nil { - // add context to the request - localVarRequest = localVarRequest.WithContext(ctx) - - // Walk through any authentication. - - // OAuth2 authentication - if tok, ok := ctx.Value(ContextOAuth2).(oauth2.TokenSource); ok { - // We were able to grab an oauth2 token from the context - var latestToken *oauth2.Token - if latestToken, err = tok.Token(); err != nil { - return nil, err - } - - latestToken.SetAuthHeader(localVarRequest) - } - - // Basic HTTP Authentication - if auth, ok := ctx.Value(ContextBasicAuth).(BasicAuth); ok { - localVarRequest.SetBasicAuth(auth.UserName, auth.Password) - } - - // AccessToken Authentication - if auth, ok := ctx.Value(ContextAccessToken).(string); ok { - localVarRequest.Header.Add("Authorization", "Bearer "+auth) - } - - } - - for header, value := range c.cfg.DefaultHeader { - localVarRequest.Header.Add(header, value) - } - - return localVarRequest, nil -} - -func (c *APIClient) decode(v interface{}, b []byte, contentType string) (err error) { - if len(b) == 0 { - return nil - } - if s, ok := v.(*string); ok { - *s = string(b) - return nil - } - if f, ok := v.(**os.File); ok { - *f, err = ioutil.TempFile("", "HttpClientFile") - if err != nil { - return - } - _, err = (*f).Write(b) - _, err = (*f).Seek(0, io.SeekStart) - return - } - if xmlCheck.MatchString(contentType) { - if err = xml.Unmarshal(b, v); err != nil { - return err - } - return nil - } - if jsonCheck.MatchString(contentType) { - if err = json.Unmarshal(b, v); err != nil { - return err - } - return nil - } - return errors.New("undefined response type") -} - -// Add a file to the multipart request -func addFile(w *multipart.Writer, fieldName, path string) error { - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - - part, err := w.CreateFormFile(fieldName, filepath.Base(path)) - if err != nil { - return err - } - _, err = io.Copy(part, file) - - return err -} - -// Prevent trying to import "fmt" -func reportError(format string, a ...interface{}) error { - return fmt.Errorf(format, a...) -} - -// Set request body from an interface{} -func setBody(body interface{}, contentType string) (bodyBuf *bytes.Buffer, err error) { - if bodyBuf == nil { - bodyBuf = &bytes.Buffer{} - } - - if reader, ok := body.(io.Reader); ok { - _, err = bodyBuf.ReadFrom(reader) - } else if b, ok := body.([]byte); ok { - _, err = bodyBuf.Write(b) - } else if s, ok := body.(string); ok { - _, err = bodyBuf.WriteString(s) - } else if s, ok := body.(*string); ok { - _, err = bodyBuf.WriteString(*s) - } else if jsonCheck.MatchString(contentType) { - err = json.NewEncoder(bodyBuf).Encode(body) - } else if xmlCheck.MatchString(contentType) { - var bs []byte - bs, err = xml.Marshal(body) - if err == nil { - bodyBuf.Write(bs) - } - } - - if err != nil { - return nil, err - } - - if bodyBuf.Len() == 0 { - err = fmt.Errorf("Invalid body type %s\n", contentType) - return nil, err - } - return bodyBuf, nil -} - -// detectContentType method is used to figure out `Request.Body` content type for request header -func detectContentType(body interface{}) string { - contentType := "text/plain; charset=utf-8" - kind := reflect.TypeOf(body).Kind() - - switch kind { - case reflect.Struct, reflect.Map, reflect.Ptr: - contentType = "application/json; charset=utf-8" - case reflect.String: - contentType = "text/plain; charset=utf-8" - default: - if b, ok := body.([]byte); ok { - contentType = http.DetectContentType(b) - } else if kind == reflect.Slice { - contentType = "application/json; charset=utf-8" - } - } - - return contentType -} - -// Ripped from https://github.com/gregjones/httpcache/blob/master/httpcache.go -type cacheControl map[string]string - -func parseCacheControl(headers http.Header) cacheControl { - cc := cacheControl{} - ccHeader := headers.Get("Cache-Control") - for _, part := range strings.Split(ccHeader, ",") { - part = strings.Trim(part, " ") - if part == "" { - continue - } - if strings.ContainsRune(part, '=') { - keyval := strings.Split(part, "=") - cc[strings.Trim(keyval[0], " ")] = strings.Trim(keyval[1], ",") - } else { - cc[part] = "" - } - } - return cc -} - -// CacheExpires helper function to determine remaining time before repeating a request. -func CacheExpires(r *http.Response) time.Time { - // Figure out when the cache expires. - var expires time.Time - now, err := time.Parse(time.RFC1123, r.Header.Get("date")) - if err != nil { - return time.Now() - } - respCacheControl := parseCacheControl(r.Header) - - if maxAge, ok := respCacheControl["max-age"]; ok { - lifetime, err := time.ParseDuration(maxAge + "s") - if err != nil { - expires = now - } else { - expires = now.Add(lifetime) - } - } else { - expiresHeader := r.Header.Get("Expires") - if expiresHeader != "" { - expires, err = time.Parse(time.RFC1123, expiresHeader) - if err != nil { - expires = now - } - } - } - return expires -} - -func strlen(s string) int { - return utf8.RuneCountInString(s) -} - -// GenericOpenAPIError Provides access to the body, error and model on returned errors. -type GenericOpenAPIError struct { - body []byte - error string - model interface{} -} - -// Error returns non-empty string if there was an error. -func (e GenericOpenAPIError) Error() string { - return e.error -} - -// Body returns the raw bytes of the response -func (e GenericOpenAPIError) Body() []byte { - return e.body -} - -// Model returns the unpacked model of the error -func (e GenericOpenAPIError) Model() interface{} { - return e.model -} diff --git a/client/configuration.go b/client/configuration.go deleted file mode 100644 index faa25fb5..00000000 --- a/client/configuration.go +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -import ( - "fmt" - "net/http" - "strings" -) - -// contextKeys are used to identify the type of value in the context. -// Since these are string, it is possible to get a short description of the -// context key for logging and debugging using key.String(). - -type contextKey string - -func (c contextKey) String() string { - return "auth " + string(c) -} - -var ( - // ContextOAuth2 takes an oauth2.TokenSource as authentication for the request. - ContextOAuth2 = contextKey("token") - - // ContextBasicAuth takes BasicAuth as authentication for the request. - ContextBasicAuth = contextKey("basic") - - // ContextAccessToken takes a string oauth2 access token as authentication for the request. - ContextAccessToken = contextKey("accesstoken") - - // ContextAPIKey takes an APIKey as authentication for the request - ContextAPIKey = contextKey("apikey") -) - -// BasicAuth provides basic http authentication to a request passed via context using ContextBasicAuth -type BasicAuth struct { - UserName string `json:"userName,omitempty"` - Password string `json:"password,omitempty"` -} - -// APIKey provides API key based authentication to a request passed via context using ContextAPIKey -type APIKey struct { - Key string - Prefix string -} - -// ServerVariable stores the information about a server variable -type ServerVariable struct { - Description string - DefaultValue string - EnumValues []string -} - -// ServerConfiguration stores the information about a server -type ServerConfiguration struct { - Url string - Description string - Variables map[string]ServerVariable -} - -// Configuration stores the configuration of the API client -type Configuration struct { - BasePath string `json:"basePath,omitempty"` - Host string `json:"host,omitempty"` - Scheme string `json:"scheme,omitempty"` - DefaultHeader map[string]string `json:"defaultHeader,omitempty"` - UserAgent string `json:"userAgent,omitempty"` - Debug bool `json:"debug,omitempty"` - Servers []ServerConfiguration - HTTPClient *http.Client -} - -// NewConfiguration returns a new Configuration object -func NewConfiguration() *Configuration { - cfg := &Configuration{ - BasePath: "http://localhost:8084", - DefaultHeader: make(map[string]string), - UserAgent: "OpenAPI-Generator/1.0.0/go", - Debug: false, - Servers: []ServerConfiguration{ - { - Url: "http://localhost:8084", - Description: "Local development", - }, - }, - } - return cfg -} - -// AddDefaultHeader adds a new HTTP header to the default header in the request -func (c *Configuration) AddDefaultHeader(key string, value string) { - c.DefaultHeader[key] = value -} - -// ServerUrl returns URL based on server settings -func (c *Configuration) ServerUrl(index int, variables map[string]string) (string, error) { - if index < 0 || len(c.Servers) <= index { - return "", fmt.Errorf("Index %v out of range %v", index, len(c.Servers)-1) - } - server := c.Servers[index] - url := server.Url - - // go through variables and replace placeholders - for name, variable := range server.Variables { - if value, ok := variables[name]; ok { - found := bool(len(variable.EnumValues) == 0) - for _, enumValue := range variable.EnumValues { - if value == enumValue { - found = true - } - } - if !found { - return "", fmt.Errorf("The variable %s in the server URL has invalid value %v. Must be %v", name, value, variable.EnumValues) - } - url = strings.Replace(url, "{"+name+"}", value, -1) - } else { - url = strings.Replace(url, "{"+name+"}", variable.DefaultValue, -1) - } - } - return url, nil -} diff --git a/client/docs/BisEntities.md b/client/docs/BisEntities.md deleted file mode 100644 index 0e6200e7..00000000 --- a/client/docs/BisEntities.md +++ /dev/null @@ -1,21 +0,0 @@ -# BisEntities - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**Name** | **string** | The name of the entity | [optional] -**Addresses** | **[]string** | Addresses associated with the entity | [optional] -**AlternateNames** | **[]string** | Known aliases associated with the entity | [optional] -**StartDate** | **string** | Date when the restriction came into effect | [optional] -**LicenseRequirement** | **string** | Specifies the license requirement imposed on the named entity | [optional] -**LicensePolicy** | **string** | Identifies the policy BIS uses to review the licenseRequirements | [optional] -**FrNotice** | **string** | Identifies the corresponding Notice in the Federal Register | [optional] -**SourceListURL** | **string** | The link to the official SSI list | [optional] -**SourceInfoURL** | **string** | The link for information regarding the source | [optional] -**Match** | **float32** | Match percentage of search query | [optional] -**MatchedName** | **string** | The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/CaptaList.md b/client/docs/CaptaList.md deleted file mode 100644 index 4f863a03..00000000 --- a/client/docs/CaptaList.md +++ /dev/null @@ -1,23 +0,0 @@ -# CaptaList - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**EntityID** | **string** | | [optional] -**EntityNumber** | **string** | | [optional] -**Type** | **string** | | [optional] -**Programs** | **[]string** | | [optional] -**Name** | **string** | | [optional] -**Addresses** | **[]string** | | [optional] -**Remarks** | **[]string** | | [optional] -**SourceListURL** | **string** | | [optional] -**AlternateNames** | **[]string** | | [optional] -**SourceInfoURL** | **string** | | [optional] -**IDs** | **[]string** | | [optional] -**Match** | **float32** | Match percentage of search query | [optional] -**MatchedName** | **string** | The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/Download.md b/client/docs/Download.md deleted file mode 100644 index c6334b11..00000000 --- a/client/docs/Download.md +++ /dev/null @@ -1,17 +0,0 @@ -# Download - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**SDNs** | **int32** | | [optional] -**AltNames** | **int32** | | [optional] -**Addresses** | **int32** | | [optional] -**SectoralSanctions** | **int32** | | [optional] -**DeniedPersons** | **int32** | | [optional] -**BisEntities** | **int32** | | [optional] -**Timestamp** | [**time.Time**](time.Time.md) | | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/Dpl.md b/client/docs/Dpl.md deleted file mode 100644 index 2a21e534..00000000 --- a/client/docs/Dpl.md +++ /dev/null @@ -1,24 +0,0 @@ -# Dpl - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**Name** | **string** | Denied Person's name | [optional] -**StreetAddress** | **string** | Denied Person's street address | [optional] -**City** | **string** | Denied Person's city | [optional] -**State** | **string** | Denied Person's state | [optional] -**Country** | **string** | Denied Person's country | [optional] -**PostalCode** | **string** | Denied Person's postal code | [optional] -**EffectiveDate** | **string** | Date when denial came into effect | [optional] -**ExpirationDate** | **string** | Date when denial expires, if blank denial never expires | [optional] -**StandardOrder** | **string** | Denotes whether or not the Denied Person was added by a standard order | [optional] -**LastUpdate** | **string** | Date when the Denied Person's record was most recently updated | [optional] -**Action** | **string** | Most recent action taken regarding the denial | [optional] -**FrCitation** | **string** | Reference to the order's citation in the Federal Register | [optional] -**Match** | **float32** | Match percentage of search query | [optional] -**MatchedName** | **string** | The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/Error.md b/client/docs/Error.md deleted file mode 100644 index fc134a35..00000000 --- a/client/docs/Error.md +++ /dev/null @@ -1,11 +0,0 @@ -# Error - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**Error** | **string** | An error message describing the problem intended for humans. | - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/EuConsolidatedSanctionsList.md b/client/docs/EuConsolidatedSanctionsList.md deleted file mode 100644 index 0583718b..00000000 --- a/client/docs/EuConsolidatedSanctionsList.md +++ /dev/null @@ -1,29 +0,0 @@ -# EuConsolidatedSanctionsList - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**FileGenerationDate** | **string** | | [optional] -**EntityLogicalId** | **int32** | | [optional] -**EntityRemark** | **string** | | [optional] -**EntitySubjectType** | **string** | | [optional] -**EntityPublicationURL** | **string** | | [optional] -**EntityReferenceNumber** | **string** | | [optional] -**NameAliasWholeNames** | **[]string** | | [optional] -**NameAliasTitles** | **[]string** | | [optional] -**AddressCities** | **[]string** | | [optional] -**AddressStreets** | **[]string** | | [optional] -**AddressPoBoxes** | **[]string** | | [optional] -**AddressZipCodes** | **[]string** | | [optional] -**AddressCountryDescriptions** | **[]string** | | [optional] -**BirthDates** | **[]string** | | [optional] -**BirthCities** | **[]string** | | [optional] -**BirthCountries** | **[]string** | | [optional] -**ValidFromTo** | [**map[string]interface{}**](.md) | | [optional] -**Match** | **float32** | Match percentage of search query | [optional] -**MatchedName** | **string** | The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/ForeignSanctionsEvader.md b/client/docs/ForeignSanctionsEvader.md deleted file mode 100644 index 24d73319..00000000 --- a/client/docs/ForeignSanctionsEvader.md +++ /dev/null @@ -1,23 +0,0 @@ -# ForeignSanctionsEvader - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**EntityID** | **string** | | [optional] -**EntityNumber** | **string** | | [optional] -**Type** | **string** | | [optional] -**Programs** | **[]string** | | [optional] -**Name** | **string** | | [optional] -**Addresses** | **[]string** | | [optional] -**SourceListURL** | **string** | | [optional] -**Citizenships** | **string** | | [optional] -**DatesOfBirth** | **string** | | [optional] -**SourceInfoURL** | **string** | | [optional] -**IDs** | **[]string** | | [optional] -**Match** | **float32** | Match percentage of search query | [optional] -**MatchedName** | **string** | The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/ItarDebarred.md b/client/docs/ItarDebarred.md deleted file mode 100644 index 66d56c85..00000000 --- a/client/docs/ItarDebarred.md +++ /dev/null @@ -1,18 +0,0 @@ -# ItarDebarred - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**EntityID** | **string** | | [optional] -**Name** | **string** | | [optional] -**FederalRegisterNotice** | **string** | | [optional] -**SourceListURL** | **string** | | [optional] -**AlternateNames** | **[]string** | | [optional] -**SourceInfoURL** | **string** | | [optional] -**Match** | **float32** | Match percentage of search query | [optional] -**MatchedName** | **string** | The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/MilitaryEndUser.md b/client/docs/MilitaryEndUser.md deleted file mode 100644 index 47c34574..00000000 --- a/client/docs/MilitaryEndUser.md +++ /dev/null @@ -1,18 +0,0 @@ -# MilitaryEndUser - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**EntityID** | **string** | | [optional] -**Name** | **string** | | [optional] -**Addresses** | **string** | | [optional] -**FRNotice** | **string** | | [optional] -**StartDate** | **string** | | [optional] -**EndDate** | **string** | | [optional] -**Match** | **float32** | Match percentage of search query | [optional] -**MatchedName** | **string** | The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/NonProliferationSanction.md b/client/docs/NonProliferationSanction.md deleted file mode 100644 index 537b30c8..00000000 --- a/client/docs/NonProliferationSanction.md +++ /dev/null @@ -1,21 +0,0 @@ -# NonProliferationSanction - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**EntityID** | **string** | | [optional] -**Programs** | **[]string** | | [optional] -**Name** | **string** | | [optional] -**FederalRegisterNotice** | **string** | | [optional] -**StartDate** | **string** | | [optional] -**Remarks** | **[]string** | | [optional] -**SourceListURL** | **string** | | [optional] -**AlternateNames** | **[]string** | | [optional] -**SourceInfoURL** | **string** | | [optional] -**Match** | **float32** | Match percentage of search query | [optional] -**MatchedName** | **string** | The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/NonSdnChineseMilitaryIndustrialComplex.md b/client/docs/NonSdnChineseMilitaryIndustrialComplex.md deleted file mode 100644 index 36faaa08..00000000 --- a/client/docs/NonSdnChineseMilitaryIndustrialComplex.md +++ /dev/null @@ -1,23 +0,0 @@ -# NonSdnChineseMilitaryIndustrialComplex - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**EntityID** | **string** | | [optional] -**EntityNumber** | **string** | | [optional] -**Type** | **string** | | [optional] -**Programs** | **[]string** | | [optional] -**Name** | **string** | | [optional] -**Addresses** | **[]string** | | [optional] -**Remarks** | **[]string** | | [optional] -**SourceListURL** | **string** | | [optional] -**AlternateNames** | **[]string** | | [optional] -**SourceInfoURL** | **string** | | [optional] -**IDs** | **[]string** | | [optional] -**Match** | **float32** | Match percentage of search query | [optional] -**MatchedName** | **string** | The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/NonSdnMenuBasedSanctionsList.md b/client/docs/NonSdnMenuBasedSanctionsList.md deleted file mode 100644 index 12f50369..00000000 --- a/client/docs/NonSdnMenuBasedSanctionsList.md +++ /dev/null @@ -1,22 +0,0 @@ -# NonSdnMenuBasedSanctionsList - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**EntityID** | **string** | | [optional] -**EntityNumber** | **string** | | [optional] -**Type** | **string** | | [optional] -**Programs** | **[]string** | | [optional] -**Name** | **string** | | [optional] -**Addresses** | **[]string** | | [optional] -**Remarks** | **[]string** | | [optional] -**AlternateNames** | **[]string** | | [optional] -**SourceInfoURL** | **string** | | [optional] -**IDs** | **[]string** | | [optional] -**Match** | **float32** | Match percentage of search query | [optional] -**MatchedName** | **string** | The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/OfacAlt.md b/client/docs/OfacAlt.md deleted file mode 100644 index 4f56279a..00000000 --- a/client/docs/OfacAlt.md +++ /dev/null @@ -1,17 +0,0 @@ -# OfacAlt - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**EntityID** | **string** | | [optional] -**AlternateID** | **string** | | [optional] -**AlternateType** | **string** | | [optional] -**AlternateName** | **string** | | [optional] -**AlternateRemarks** | **string** | | [optional] -**Match** | **float32** | Match percentage of search query | [optional] -**MatchedName** | **string** | The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/OfacEntityAddress.md b/client/docs/OfacEntityAddress.md deleted file mode 100644 index a2aa9682..00000000 --- a/client/docs/OfacEntityAddress.md +++ /dev/null @@ -1,16 +0,0 @@ -# OfacEntityAddress - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**EntityID** | **string** | | [optional] -**AddressID** | **string** | | [optional] -**Address** | **string** | | [optional] -**CityStateProvincePostalCode** | **string** | | [optional] -**Country** | **string** | | [optional] -**Match** | **float32** | Match percentage of search query | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/OfacSdn.md b/client/docs/OfacSdn.md deleted file mode 100644 index 2ec499cf..00000000 --- a/client/docs/OfacSdn.md +++ /dev/null @@ -1,18 +0,0 @@ -# OfacSdn - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**EntityID** | **string** | | [optional] -**SdnName** | **string** | | [optional] -**SdnType** | [**SdnType**](SdnType.md) | | [optional] -**Programs** | **[]string** | Programs is the sanction programs this SDN was added from | [optional] -**Title** | **string** | | [optional] -**Remarks** | **string** | Remarks on SDN and often additional information about the SDN | [optional] -**Match** | **float32** | Match percentage of search query | [optional] -**MatchedName** | **string** | The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/PalestinianLegislativeCouncil.md b/client/docs/PalestinianLegislativeCouncil.md deleted file mode 100644 index 63ffae26..00000000 --- a/client/docs/PalestinianLegislativeCouncil.md +++ /dev/null @@ -1,24 +0,0 @@ -# PalestinianLegislativeCouncil - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**EntityID** | **string** | | [optional] -**EntityNumber** | **string** | | [optional] -**Type** | **string** | | [optional] -**Programs** | **[]string** | | [optional] -**Name** | **string** | | [optional] -**Addresses** | **[]string** | | [optional] -**Remarks** | **string** | | [optional] -**SourceListURL** | **string** | | [optional] -**AlternateNames** | **[]string** | | [optional] -**DatesOfBirth** | **string** | | [optional] -**PlacesOfBirth** | **string** | | [optional] -**SourceInfoURL** | **string** | | [optional] -**Match** | **float32** | Match percentage of search query | [optional] -**MatchedName** | **string** | The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/SdnType.md b/client/docs/SdnType.md deleted file mode 100644 index 10e28810..00000000 --- a/client/docs/SdnType.md +++ /dev/null @@ -1,10 +0,0 @@ -# SdnType - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/Search.md b/client/docs/Search.md deleted file mode 100644 index 6e10c59c..00000000 --- a/client/docs/Search.md +++ /dev/null @@ -1,29 +0,0 @@ -# Search - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**SDNs** | [**[]OfacSdn**](OfacSDN.md) | | [optional] -**AltNames** | [**[]OfacAlt**](OfacAlt.md) | | [optional] -**Addresses** | [**[]OfacEntityAddress**](OfacEntityAddress.md) | | [optional] -**DeniedPersons** | [**[]Dpl**](DPL.md) | | [optional] -**BisEntities** | [**[]BisEntities**](BISEntities.md) | | [optional] -**MilitaryEndUsers** | [**[]MilitaryEndUser**](MilitaryEndUser.md) | | [optional] -**SectoralSanctions** | [**[]Ssi**](SSI.md) | | [optional] -**UnverifiedCSL** | [**[]Unverified**](Unverified.md) | | [optional] -**NonproliferationSanctions** | [**[]NonProliferationSanction**](NonProliferationSanction.md) | | [optional] -**ForeignSanctionsEvaders** | [**[]ForeignSanctionsEvader**](ForeignSanctionsEvader.md) | | [optional] -**PalestinianLegislativeCouncil** | [**[]PalestinianLegislativeCouncil**](PalestinianLegislativeCouncil.md) | | [optional] -**CaptaList** | [**[]CaptaList**](CAPTAList.md) | | [optional] -**ItarDebarred** | [**[]ItarDebarred**](ITARDebarred.md) | | [optional] -**NonSDNChineseMilitaryIndustrialComplex** | [**[]NonSdnChineseMilitaryIndustrialComplex**](NonSDNChineseMilitaryIndustrialComplex.md) | | [optional] -**NonSDNMenuBasedSanctionsList** | [**[]NonSdnMenuBasedSanctionsList**](NonSDNMenuBasedSanctionsList.md) | | [optional] -**EuConsolidatedSanctionsList** | [**[]EuConsolidatedSanctionsList**](EUConsolidatedSanctionsList.md) | | [optional] -**UkConsolidatedSanctionsList** | [**[]UkConsolidatedSanctionsList**](UKConsolidatedSanctionsList.md) | | [optional] -**UkSanctionsList** | [**[]UkSanctionsList**](UKSanctionsList.md) | | [optional] -**RefreshedAt** | [**time.Time**](time.Time.md) | | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/Ssi.md b/client/docs/Ssi.md deleted file mode 100644 index 991adfc9..00000000 --- a/client/docs/Ssi.md +++ /dev/null @@ -1,22 +0,0 @@ -# Ssi - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**EntityID** | **string** | The ID assigned to an entity by the Treasury Department | [optional] -**Type** | [**SsiType**](SsiType.md) | | [optional] -**Programs** | **[]string** | Sanction programs for which the entity is flagged | [optional] -**Name** | **string** | The name of the entity | [optional] -**Addresses** | **[]string** | Addresses associated with the entity | [optional] -**Remarks** | **[]string** | Additional details regarding the entity | [optional] -**AlternateNames** | **[]string** | Known aliases associated with the entity | [optional] -**Ids** | **[]string** | IDs on file for the entity | [optional] -**SourceListURL** | **string** | The link to the official SSI list | [optional] -**SourceInfoURL** | **string** | The link for information regarding the source | [optional] -**Match** | **float32** | Match percentage of search query | [optional] -**MatchedName** | **string** | The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/SsiType.md b/client/docs/SsiType.md deleted file mode 100644 index 073d3906..00000000 --- a/client/docs/SsiType.md +++ /dev/null @@ -1,10 +0,0 @@ -# SsiType - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/UkConsolidatedSanctionsList.md b/client/docs/UkConsolidatedSanctionsList.md deleted file mode 100644 index d13316e0..00000000 --- a/client/docs/UkConsolidatedSanctionsList.md +++ /dev/null @@ -1,16 +0,0 @@ -# UkConsolidatedSanctionsList - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**Names** | **[]string** | | [optional] -**Addresses** | **[]string** | | [optional] -**Countries** | **[]string** | | [optional] -**GroupType** | **string** | | [optional] -**Match** | **float32** | Match percentage of search query | [optional] -**MatchedName** | **string** | The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/UkSanctionsList.md b/client/docs/UkSanctionsList.md deleted file mode 100644 index 36bfadef..00000000 --- a/client/docs/UkSanctionsList.md +++ /dev/null @@ -1,18 +0,0 @@ -# UkSanctionsList - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**Names** | **[]string** | | [optional] -**NonLatinNames** | **[]string** | | [optional] -**EntityType** | **string** | | [optional] -**Addresses** | **[]string** | | [optional] -**AddressCountries** | **[]string** | | [optional] -**StateLocalities** | **[]string** | | [optional] -**Match** | **float32** | Match percentage of search query | [optional] -**MatchedName** | **string** | The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/Unverified.md b/client/docs/Unverified.md deleted file mode 100644 index 8394192a..00000000 --- a/client/docs/Unverified.md +++ /dev/null @@ -1,17 +0,0 @@ -# Unverified - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**EntityID** | **string** | | [optional] -**Name** | **string** | | [optional] -**Addresses** | **[]string** | | [optional] -**SourceListURL** | **string** | | [optional] -**SourceInfoURL** | **string** | | [optional] -**Match** | **float32** | Match percentage of search query | [optional] -**MatchedName** | **string** | The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/client/docs/WatchmanApi.md b/client/docs/WatchmanApi.md deleted file mode 100644 index a746759a..00000000 --- a/client/docs/WatchmanApi.md +++ /dev/null @@ -1,320 +0,0 @@ -# \WatchmanApi - -All URIs are relative to *http://localhost:8084* - -Method | HTTP request | Description -------------- | ------------- | ------------- -[**GetLatestDownloads**](WatchmanApi.md#GetLatestDownloads) | **Get** /downloads | Get latest downloads -[**GetSDNAddresses**](WatchmanApi.md#GetSDNAddresses) | **Get** /ofac/sdn/{sdnID}/addresses | Get SDN addresses -[**GetSDNAltNames**](WatchmanApi.md#GetSDNAltNames) | **Get** /ofac/sdn/{sdnID}/alts | Get SDN alt names -[**GetUIValues**](WatchmanApi.md#GetUIValues) | **Get** /ui/values/{key} | Get UI values -[**Ping**](WatchmanApi.md#Ping) | **Get** /ping | Ping Watchman service -[**Search**](WatchmanApi.md#Search) | **Get** /search | Search -[**SearchUSCSL**](WatchmanApi.md#SearchUSCSL) | **Get** /search/us-csl | Search US CSL - - - -## GetLatestDownloads - -> []Download GetLatestDownloads(ctx, optional) - -Get latest downloads - -Return list of recent downloads of list data. - -### Required Parameters - - -Name | Type | Description | Notes -------------- | ------------- | ------------- | ------------- -**ctx** | **context.Context** | context for authentication, logging, cancellation, deadlines, tracing, etc. - **optional** | ***GetLatestDownloadsOpts** | optional parameters | nil if no parameters - -### Optional Parameters - -Optional parameters are passed through a pointer to a GetLatestDownloadsOpts struct - - -Name | Type | Description | Notes -------------- | ------------- | ------------- | ------------- - **xRequestID** | **optional.String**| Optional Request ID allows application developer to trace requests through the system's logs | - **limit** | **optional.Int32**| Maximum number of downloads to return sorted by their timestamp in decending order. | - -### Return type - -[**[]Download**](Download.md) - -### Authorization - -No authorization required - -### HTTP request headers - -- **Content-Type**: Not defined -- **Accept**: application/json - -[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) -[[Back to Model list]](../README.md#documentation-for-models) -[[Back to README]](../README.md) - - -## GetSDNAddresses - -> []OfacEntityAddress GetSDNAddresses(ctx, sdnID, optional) - -Get SDN addresses - -### Required Parameters - - -Name | Type | Description | Notes -------------- | ------------- | ------------- | ------------- -**ctx** | **context.Context** | context for authentication, logging, cancellation, deadlines, tracing, etc. -**sdnID** | **string**| SDN ID | - **optional** | ***GetSDNAddressesOpts** | optional parameters | nil if no parameters - -### Optional Parameters - -Optional parameters are passed through a pointer to a GetSDNAddressesOpts struct - - -Name | Type | Description | Notes -------------- | ------------- | ------------- | ------------- - - **xRequestID** | **optional.String**| Optional Request ID allows application developer to trace requests through the system's logs | - -### Return type - -[**[]OfacEntityAddress**](OfacEntityAddress.md) - -### Authorization - -No authorization required - -### HTTP request headers - -- **Content-Type**: Not defined -- **Accept**: application/json - -[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) -[[Back to Model list]](../README.md#documentation-for-models) -[[Back to README]](../README.md) - - -## GetSDNAltNames - -> []OfacAlt GetSDNAltNames(ctx, sdnID, optional) - -Get SDN alt names - -### Required Parameters - - -Name | Type | Description | Notes -------------- | ------------- | ------------- | ------------- -**ctx** | **context.Context** | context for authentication, logging, cancellation, deadlines, tracing, etc. -**sdnID** | **string**| SDN ID | - **optional** | ***GetSDNAltNamesOpts** | optional parameters | nil if no parameters - -### Optional Parameters - -Optional parameters are passed through a pointer to a GetSDNAltNamesOpts struct - - -Name | Type | Description | Notes -------------- | ------------- | ------------- | ------------- - - **xRequestID** | **optional.String**| Optional Request ID allows application developer to trace requests through the system's logs | - -### Return type - -[**[]OfacAlt**](OfacAlt.md) - -### Authorization - -No authorization required - -### HTTP request headers - -- **Content-Type**: Not defined -- **Accept**: application/json - -[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) -[[Back to Model list]](../README.md#documentation-for-models) -[[Back to README]](../README.md) - - -## GetUIValues - -> []SdnType GetUIValues(ctx, key, optional) - -Get UI values - -Return an ordered distinct list of keys for an SDN property. - -### Required Parameters - - -Name | Type | Description | Notes -------------- | ------------- | ------------- | ------------- -**ctx** | **context.Context** | context for authentication, logging, cancellation, deadlines, tracing, etc. -**key** | [**SdnType**](.md)| SDN property to lookup. Values are sdnType, ofacProgram | - **optional** | ***GetUIValuesOpts** | optional parameters | nil if no parameters - -### Optional Parameters - -Optional parameters are passed through a pointer to a GetUIValuesOpts struct - - -Name | Type | Description | Notes -------------- | ------------- | ------------- | ------------- - - **limit** | **optional.Int32**| Maximum number of UI keys returned | - -### Return type - -[**[]SdnType**](SdnType.md) - -### Authorization - -No authorization required - -### HTTP request headers - -- **Content-Type**: Not defined -- **Accept**: application/json - -[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) -[[Back to Model list]](../README.md#documentation-for-models) -[[Back to README]](../README.md) - - -## Ping - -> Ping(ctx, ) - -Ping Watchman service - -Check if the Watchman service is running. - -### Required Parameters - -This endpoint does not need any parameter. - -### Return type - - (empty response body) - -### Authorization - -No authorization required - -### HTTP request headers - -- **Content-Type**: Not defined -- **Accept**: Not defined - -[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) -[[Back to Model list]](../README.md#documentation-for-models) -[[Back to README]](../README.md) - - -## Search - -> Search Search(ctx, optional) - -Search - -### Required Parameters - - -Name | Type | Description | Notes -------------- | ------------- | ------------- | ------------- -**ctx** | **context.Context** | context for authentication, logging, cancellation, deadlines, tracing, etc. - **optional** | ***SearchOpts** | optional parameters | nil if no parameters - -### Optional Parameters - -Optional parameters are passed through a pointer to a SearchOpts struct - - -Name | Type | Description | Notes -------------- | ------------- | ------------- | ------------- - **xRequestID** | **optional.String**| Optional Request ID allows application developer to trace requests through the system's logs | - **q** | **optional.String**| Search across Name, Alt Names, and SDN Address fields for all available sanctions lists. Entries may be returned in all response sub-objects. | - **name** | **optional.String**| Name which could correspond to an entry on the SDN, Denied Persons, Sectoral Sanctions Identifications, or BIS Entity List sanctions lists. Alt names are also searched. | - **address** | **optional.String**| Physical address which could correspond to a human on the SDN list. Only Address results will be returned. | - **city** | **optional.String**| City name as desginated by SDN guidelines. Only Address results will be returned. | - **state** | **optional.String**| State name as desginated by SDN guidelines. Only Address results will be returned. | - **providence** | **optional.String**| Providence name as desginated by SDN guidelines. Only Address results will be returned. | - **zip** | **optional.String**| Zip code as desginated by SDN guidelines. Only Address results will be returned. | - **country** | **optional.String**| Country name as desginated by SDN guidelines. Only Address results will be returned. | - **altName** | **optional.String**| Alternate name which could correspond to a human on the SDN list. Only Alt name results will be returned. | - **id** | **optional.String**| ID value often found in remarks property of an SDN. Takes the form of 'No. NNNNN' as an alphanumeric value. | - **minMatch** | **optional.Float32**| Match percentage that search query must obtain for results to be returned. | - **limit** | **optional.Int32**| Maximum results returned by a search. Results are sorted by their match percentage in decending order. | - **sdnType** | [**optional.Interface of SdnType**](.md)| Optional filter to only return SDNs whose type case-insensitively matches. | - **program** | **optional.String**| Optional filter to only return SDNs whose program case-insensitively matches. | - -### Return type - -[**Search**](Search.md) - -### Authorization - -No authorization required - -### HTTP request headers - -- **Content-Type**: Not defined -- **Accept**: application/json - -[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) -[[Back to Model list]](../README.md#documentation-for-models) -[[Back to README]](../README.md) - - -## SearchUSCSL - -> Search SearchUSCSL(ctx, optional) - -Search US CSL - -Search the US Consolidated Screening List - -### Required Parameters - - -Name | Type | Description | Notes -------------- | ------------- | ------------- | ------------- -**ctx** | **context.Context** | context for authentication, logging, cancellation, deadlines, tracing, etc. - **optional** | ***SearchUSCSLOpts** | optional parameters | nil if no parameters - -### Optional Parameters - -Optional parameters are passed through a pointer to a SearchUSCSLOpts struct - - -Name | Type | Description | Notes -------------- | ------------- | ------------- | ------------- - **xRequestID** | **optional.String**| Optional Request ID allows application developer to trace requests through the system's logs | - **name** | **optional.String**| Name which could correspond to an entry on the CSL | - **limit** | **optional.Int32**| Maximum number of downloads to return sorted by their timestamp in decending order. | - -### Return type - -[**Search**](Search.md) - -### Authorization - -No authorization required - -### HTTP request headers - -- **Content-Type**: Not defined -- **Accept**: application/json - -[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) -[[Back to Model list]](../README.md#documentation-for-models) -[[Back to README]](../README.md) - diff --git a/client/git_push.sh b/client/git_push.sh deleted file mode 100644 index bc93d187..00000000 --- a/client/git_push.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -# ref: https://help.github.com/articles/adding-an-existing-project-to-github-using-the-command-line/ -# -# Usage example: /bin/sh ./git_push.sh wing328 openapi-pestore-perl "minor update" "gitlab.com" - -git_user_id=$1 -git_repo_id=$2 -release_note=$3 -git_host=$4 - -if [ "$git_host" = "" ]; then - git_host="github.com" - echo "[INFO] No command line input provided. Set \$git_host to $git_host" -fi - -if [ "$git_user_id" = "" ]; then - git_user_id="moov-io" - echo "[INFO] No command line input provided. Set \$git_user_id to $git_user_id" -fi - -if [ "$git_repo_id" = "" ]; then - git_repo_id="watchman" - echo "[INFO] No command line input provided. Set \$git_repo_id to $git_repo_id" -fi - -if [ "$release_note" = "" ]; then - release_note="Minor update" - echo "[INFO] No command line input provided. Set \$release_note to $release_note" -fi - -# Initialize the local directory as a Git repository -git init - -# Adds the files in the local repository and stages them for commit. -git add . - -# Commits the tracked changes and prepares them to be pushed to a remote repository. -git commit -m "$release_note" - -# Sets the new remote -git_remote=`git remote` -if [ "$git_remote" = "" ]; then # git remote not defined - - if [ "$GIT_TOKEN" = "" ]; then - echo "[INFO] \$GIT_TOKEN (environment variable) is not set. Using the git credential in your environment." - git remote add origin https://${git_host}/${git_user_id}/${git_repo_id}.git - else - git remote add origin https://${git_user_id}:${GIT_TOKEN}@${git_host}/${git_user_id}/${git_repo_id}.git - fi - -fi - -git pull origin master - -# Pushes (Forces) the changes in the local repository up to the remote repository -echo "Git pushing to https://${git_host}/${git_user_id}/${git_repo_id}.git" -git push origin master 2>&1 | grep -v 'To https' - diff --git a/client/model_bis_entities.go b/client/model_bis_entities.go deleted file mode 100644 index fa861283..00000000 --- a/client/model_bis_entities.go +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// BisEntities Bureau of Industry and Security Entity List -type BisEntities struct { - // The name of the entity - Name string `json:"name,omitempty"` - // Addresses associated with the entity - Addresses []string `json:"addresses,omitempty"` - // Known aliases associated with the entity - AlternateNames []string `json:"alternateNames,omitempty"` - // Date when the restriction came into effect - StartDate string `json:"startDate,omitempty"` - // Specifies the license requirement imposed on the named entity - LicenseRequirement string `json:"licenseRequirement,omitempty"` - // Identifies the policy BIS uses to review the licenseRequirements - LicensePolicy string `json:"licensePolicy,omitempty"` - // Identifies the corresponding Notice in the Federal Register - FrNotice string `json:"frNotice,omitempty"` - // The link to the official SSI list - SourceListURL string `json:"sourceListURL,omitempty"` - // The link for information regarding the source - SourceInfoURL string `json:"sourceInfoURL,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` - // The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. - MatchedName string `json:"matchedName,omitempty"` -} diff --git a/client/model_capta_list.go b/client/model_capta_list.go deleted file mode 100644 index 8c80b222..00000000 --- a/client/model_capta_list.go +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// CaptaList struct for CaptaList -type CaptaList struct { - EntityID string `json:"entityID,omitempty"` - EntityNumber string `json:"entityNumber,omitempty"` - Type string `json:"type,omitempty"` - Programs []string `json:"programs,omitempty"` - Name string `json:"name,omitempty"` - Addresses []string `json:"addresses,omitempty"` - Remarks []string `json:"remarks,omitempty"` - SourceListURL string `json:"sourceListURL,omitempty"` - AlternateNames []string `json:"alternateNames,omitempty"` - SourceInfoURL string `json:"sourceInfoURL,omitempty"` - IDs []string `json:"IDs,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` - // The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. - MatchedName string `json:"matchedName,omitempty"` -} diff --git a/client/model_download.go b/client/model_download.go deleted file mode 100644 index 97828ec9..00000000 --- a/client/model_download.go +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -import ( - "time" -) - -// Download Metadata and stats about downloaded OFAC data -type Download struct { - SDNs int32 `json:"SDNs,omitempty"` - AltNames int32 `json:"altNames,omitempty"` - Addresses int32 `json:"addresses,omitempty"` - SectoralSanctions int32 `json:"sectoralSanctions,omitempty"` - DeniedPersons int32 `json:"deniedPersons,omitempty"` - BisEntities int32 `json:"bisEntities,omitempty"` - Timestamp time.Time `json:"timestamp,omitempty"` -} diff --git a/client/model_dpl.go b/client/model_dpl.go deleted file mode 100644 index 4fec6645..00000000 --- a/client/model_dpl.go +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// Dpl BIS Denied Persons List item -type Dpl struct { - // Denied Person's name - Name string `json:"name,omitempty"` - // Denied Person's street address - StreetAddress string `json:"streetAddress,omitempty"` - // Denied Person's city - City string `json:"city,omitempty"` - // Denied Person's state - State string `json:"state,omitempty"` - // Denied Person's country - Country string `json:"country,omitempty"` - // Denied Person's postal code - PostalCode string `json:"postalCode,omitempty"` - // Date when denial came into effect - EffectiveDate string `json:"effectiveDate,omitempty"` - // Date when denial expires, if blank denial never expires - ExpirationDate string `json:"expirationDate,omitempty"` - // Denotes whether or not the Denied Person was added by a standard order - StandardOrder string `json:"standardOrder,omitempty"` - // Date when the Denied Person's record was most recently updated - LastUpdate string `json:"lastUpdate,omitempty"` - // Most recent action taken regarding the denial - Action string `json:"action,omitempty"` - // Reference to the order's citation in the Federal Register - FrCitation string `json:"frCitation,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` - // The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. - MatchedName string `json:"matchedName,omitempty"` -} diff --git a/client/model_error.go b/client/model_error.go deleted file mode 100644 index d2f8e6c6..00000000 --- a/client/model_error.go +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// Error struct for Error -type Error struct { - // An error message describing the problem intended for humans. - Error string `json:"error"` -} diff --git a/client/model_eu_consolidated_sanctions_list.go b/client/model_eu_consolidated_sanctions_list.go deleted file mode 100644 index 1a600b2b..00000000 --- a/client/model_eu_consolidated_sanctions_list.go +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// EuConsolidatedSanctionsList struct for EuConsolidatedSanctionsList -type EuConsolidatedSanctionsList struct { - FileGenerationDate string `json:"fileGenerationDate,omitempty"` - EntityLogicalId int32 `json:"entityLogicalId,omitempty"` - EntityRemark string `json:"entityRemark,omitempty"` - EntitySubjectType string `json:"entitySubjectType,omitempty"` - EntityPublicationURL string `json:"entityPublicationURL,omitempty"` - EntityReferenceNumber string `json:"entityReferenceNumber,omitempty"` - NameAliasWholeNames []string `json:"nameAliasWholeNames,omitempty"` - NameAliasTitles []string `json:"nameAliasTitles,omitempty"` - AddressCities []string `json:"addressCities,omitempty"` - AddressStreets []string `json:"addressStreets,omitempty"` - AddressPoBoxes []string `json:"addressPoBoxes,omitempty"` - AddressZipCodes []string `json:"addressZipCodes,omitempty"` - AddressCountryDescriptions []string `json:"addressCountryDescriptions,omitempty"` - BirthDates []string `json:"birthDates,omitempty"` - BirthCities []string `json:"birthCities,omitempty"` - BirthCountries []string `json:"birthCountries,omitempty"` - ValidFromTo map[string]interface{} `json:"validFromTo,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` - // The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. - MatchedName string `json:"matchedName,omitempty"` -} diff --git a/client/model_foreign_sanctions_evader.go b/client/model_foreign_sanctions_evader.go deleted file mode 100644 index d231bfb8..00000000 --- a/client/model_foreign_sanctions_evader.go +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// ForeignSanctionsEvader struct for ForeignSanctionsEvader -type ForeignSanctionsEvader struct { - EntityID string `json:"entityID,omitempty"` - EntityNumber string `json:"entityNumber,omitempty"` - Type string `json:"type,omitempty"` - Programs []string `json:"programs,omitempty"` - Name string `json:"name,omitempty"` - Addresses []string `json:"addresses,omitempty"` - SourceListURL string `json:"sourceListURL,omitempty"` - Citizenships string `json:"citizenships,omitempty"` - DatesOfBirth string `json:"datesOfBirth,omitempty"` - SourceInfoURL string `json:"sourceInfoURL,omitempty"` - IDs []string `json:"IDs,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` - // The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. - MatchedName string `json:"matchedName,omitempty"` -} diff --git a/client/model_itar_debarred.go b/client/model_itar_debarred.go deleted file mode 100644 index fdc1a6c8..00000000 --- a/client/model_itar_debarred.go +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// ItarDebarred struct for ItarDebarred -type ItarDebarred struct { - EntityID string `json:"entityID,omitempty"` - Name string `json:"name,omitempty"` - FederalRegisterNotice string `json:"federalRegisterNotice,omitempty"` - SourceListURL string `json:"sourceListURL,omitempty"` - AlternateNames []string `json:"alternateNames,omitempty"` - SourceInfoURL string `json:"sourceInfoURL,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` - // The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. - MatchedName string `json:"matchedName,omitempty"` -} diff --git a/client/model_military_end_user.go b/client/model_military_end_user.go deleted file mode 100644 index e2c61a1d..00000000 --- a/client/model_military_end_user.go +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// MilitaryEndUser struct for MilitaryEndUser -type MilitaryEndUser struct { - EntityID string `json:"entityID,omitempty"` - Name string `json:"name,omitempty"` - Addresses string `json:"addresses,omitempty"` - FRNotice string `json:"FRNotice,omitempty"` - StartDate string `json:"startDate,omitempty"` - EndDate string `json:"endDate,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` - // The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. - MatchedName string `json:"matchedName,omitempty"` -} diff --git a/client/model_non_proliferation_sanction.go b/client/model_non_proliferation_sanction.go deleted file mode 100644 index fb8ac607..00000000 --- a/client/model_non_proliferation_sanction.go +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// NonProliferationSanction struct for NonProliferationSanction -type NonProliferationSanction struct { - EntityID string `json:"entityID,omitempty"` - Programs []string `json:"programs,omitempty"` - Name string `json:"name,omitempty"` - FederalRegisterNotice string `json:"federalRegisterNotice,omitempty"` - StartDate string `json:"startDate,omitempty"` - Remarks []string `json:"remarks,omitempty"` - SourceListURL string `json:"sourceListURL,omitempty"` - AlternateNames []string `json:"alternateNames,omitempty"` - SourceInfoURL string `json:"sourceInfoURL,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` - // The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. - MatchedName string `json:"matchedName,omitempty"` -} diff --git a/client/model_non_sdn_chinese_military_industrial_complex.go b/client/model_non_sdn_chinese_military_industrial_complex.go deleted file mode 100644 index 12eba732..00000000 --- a/client/model_non_sdn_chinese_military_industrial_complex.go +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// NonSdnChineseMilitaryIndustrialComplex struct for NonSdnChineseMilitaryIndustrialComplex -type NonSdnChineseMilitaryIndustrialComplex struct { - EntityID string `json:"entityID,omitempty"` - EntityNumber string `json:"entityNumber,omitempty"` - Type string `json:"type,omitempty"` - Programs []string `json:"programs,omitempty"` - Name string `json:"name,omitempty"` - Addresses []string `json:"addresses,omitempty"` - Remarks []string `json:"remarks,omitempty"` - SourceListURL string `json:"sourceListURL,omitempty"` - AlternateNames []string `json:"alternateNames,omitempty"` - SourceInfoURL string `json:"sourceInfoURL,omitempty"` - IDs []string `json:"IDs,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` - // The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. - MatchedName string `json:"matchedName,omitempty"` -} diff --git a/client/model_non_sdn_menu_based_sanctions_list.go b/client/model_non_sdn_menu_based_sanctions_list.go deleted file mode 100644 index 9cf0f7f3..00000000 --- a/client/model_non_sdn_menu_based_sanctions_list.go +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// NonSdnMenuBasedSanctionsList struct for NonSdnMenuBasedSanctionsList -type NonSdnMenuBasedSanctionsList struct { - EntityID string `json:"EntityID,omitempty"` - EntityNumber string `json:"EntityNumber,omitempty"` - Type string `json:"Type,omitempty"` - Programs []string `json:"Programs,omitempty"` - Name string `json:"Name,omitempty"` - Addresses []string `json:"Addresses,omitempty"` - Remarks []string `json:"Remarks,omitempty"` - AlternateNames []string `json:"AlternateNames,omitempty"` - SourceInfoURL string `json:"SourceInfoURL,omitempty"` - IDs []string `json:"IDs,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` - // The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. - MatchedName string `json:"matchedName,omitempty"` -} diff --git a/client/model_ofac_alt.go b/client/model_ofac_alt.go deleted file mode 100644 index 3bfe7b53..00000000 --- a/client/model_ofac_alt.go +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// OfacAlt Alternate name from OFAC list -type OfacAlt struct { - EntityID string `json:"entityID,omitempty"` - AlternateID string `json:"alternateID,omitempty"` - AlternateType string `json:"alternateType,omitempty"` - AlternateName string `json:"alternateName,omitempty"` - AlternateRemarks string `json:"alternateRemarks,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` - // The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. - MatchedName string `json:"matchedName,omitempty"` -} diff --git a/client/model_ofac_entity_address.go b/client/model_ofac_entity_address.go deleted file mode 100644 index b7a2e319..00000000 --- a/client/model_ofac_entity_address.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// OfacEntityAddress Physical address from OFAC list -type OfacEntityAddress struct { - EntityID string `json:"entityID,omitempty"` - AddressID string `json:"addressID,omitempty"` - Address string `json:"address,omitempty"` - CityStateProvincePostalCode string `json:"cityStateProvincePostalCode,omitempty"` - Country string `json:"country,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` -} diff --git a/client/model_ofac_sdn.go b/client/model_ofac_sdn.go deleted file mode 100644 index 11a1e34b..00000000 --- a/client/model_ofac_sdn.go +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// OfacSdn Specially designated national from OFAC list -type OfacSdn struct { - EntityID string `json:"entityID,omitempty"` - SdnName string `json:"sdnName,omitempty"` - SdnType SdnType `json:"sdnType,omitempty"` - // Programs is the sanction programs this SDN was added from - Programs []string `json:"programs,omitempty"` - Title string `json:"title,omitempty"` - // Remarks on SDN and often additional information about the SDN - Remarks string `json:"remarks,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` - // The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. - MatchedName string `json:"matchedName,omitempty"` -} diff --git a/client/model_palestinian_legislative_council.go b/client/model_palestinian_legislative_council.go deleted file mode 100644 index 669a414c..00000000 --- a/client/model_palestinian_legislative_council.go +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// PalestinianLegislativeCouncil struct for PalestinianLegislativeCouncil -type PalestinianLegislativeCouncil struct { - EntityID string `json:"entityID,omitempty"` - EntityNumber string `json:"entityNumber,omitempty"` - Type string `json:"type,omitempty"` - Programs []string `json:"programs,omitempty"` - Name string `json:"name,omitempty"` - Addresses []string `json:"addresses,omitempty"` - Remarks string `json:"remarks,omitempty"` - SourceListURL string `json:"sourceListURL,omitempty"` - AlternateNames []string `json:"alternateNames,omitempty"` - DatesOfBirth string `json:"datesOfBirth,omitempty"` - PlacesOfBirth string `json:"placesOfBirth,omitempty"` - SourceInfoURL string `json:"sourceInfoURL,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` - // The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. - MatchedName string `json:"matchedName,omitempty"` -} diff --git a/client/model_sdn_type.go b/client/model_sdn_type.go deleted file mode 100644 index 3e48d872..00000000 --- a/client/model_sdn_type.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// SdnType Used for classifying SDNs — typically represents an individual or company -type SdnType string - -// List of SdnType -const ( - SDNTYPE_INDIVIDUAL SdnType = "individual" - SDNTYPE_ENTITY SdnType = "entity" - SDNTYPE_VESSEL SdnType = "vessel" - SDNTYPE_AIRCRAFT SdnType = "aircraft" -) diff --git a/client/model_search.go b/client/model_search.go deleted file mode 100644 index a7d37339..00000000 --- a/client/model_search.go +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -import ( - "time" -) - -// Search Search results containing SDNs, alternate names, and/or addreses -type Search struct { - SDNs []OfacSdn `json:"SDNs,omitempty"` - AltNames []OfacAlt `json:"altNames,omitempty"` - Addresses []OfacEntityAddress `json:"addresses,omitempty"` - DeniedPersons []Dpl `json:"deniedPersons,omitempty"` - BisEntities []BisEntities `json:"bisEntities,omitempty"` - MilitaryEndUsers []MilitaryEndUser `json:"militaryEndUsers,omitempty"` - SectoralSanctions []Ssi `json:"sectoralSanctions,omitempty"` - UnverifiedCSL []Unverified `json:"unverifiedCSL,omitempty"` - NonproliferationSanctions []NonProliferationSanction `json:"nonproliferationSanctions,omitempty"` - ForeignSanctionsEvaders []ForeignSanctionsEvader `json:"foreignSanctionsEvaders,omitempty"` - PalestinianLegislativeCouncil []PalestinianLegislativeCouncil `json:"palestinianLegislativeCouncil,omitempty"` - CaptaList []CaptaList `json:"captaList,omitempty"` - ItarDebarred []ItarDebarred `json:"itarDebarred,omitempty"` - NonSDNChineseMilitaryIndustrialComplex []NonSdnChineseMilitaryIndustrialComplex `json:"nonSDNChineseMilitaryIndustrialComplex,omitempty"` - NonSDNMenuBasedSanctionsList []NonSdnMenuBasedSanctionsList `json:"nonSDNMenuBasedSanctionsList,omitempty"` - EuConsolidatedSanctionsList []EuConsolidatedSanctionsList `json:"euConsolidatedSanctionsList,omitempty"` - UkConsolidatedSanctionsList []UkConsolidatedSanctionsList `json:"ukConsolidatedSanctionsList,omitempty"` - UkSanctionsList []UkSanctionsList `json:"ukSanctionsList,omitempty"` - RefreshedAt time.Time `json:"refreshedAt,omitempty"` -} diff --git a/client/model_ssi.go b/client/model_ssi.go deleted file mode 100644 index bc093856..00000000 --- a/client/model_ssi.go +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// Ssi Treasury Department Sectoral Sanctions Identifications List (SSI) -type Ssi struct { - // The ID assigned to an entity by the Treasury Department - EntityID string `json:"entityID,omitempty"` - Type SsiType `json:"type,omitempty"` - // Sanction programs for which the entity is flagged - Programs []string `json:"programs,omitempty"` - // The name of the entity - Name string `json:"name,omitempty"` - // Addresses associated with the entity - Addresses []string `json:"addresses,omitempty"` - // Additional details regarding the entity - Remarks []string `json:"remarks,omitempty"` - // Known aliases associated with the entity - AlternateNames []string `json:"alternateNames,omitempty"` - // IDs on file for the entity - Ids []string `json:"ids,omitempty"` - // The link to the official SSI list - SourceListURL string `json:"sourceListURL,omitempty"` - // The link for information regarding the source - SourceInfoURL string `json:"sourceInfoURL,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` - // The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. - MatchedName string `json:"matchedName,omitempty"` -} diff --git a/client/model_ssi_type.go b/client/model_ssi_type.go deleted file mode 100644 index 70be3523..00000000 --- a/client/model_ssi_type.go +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// SsiType Used for classifying SSIs -type SsiType string - -// List of SsiType -const ( - SSITYPE_INDIVIDUAL SsiType = "individual" - SSITYPE_ENTITY SsiType = "entity" -) diff --git a/client/model_uk_consolidated_sanctions_list.go b/client/model_uk_consolidated_sanctions_list.go deleted file mode 100644 index c7d96a4d..00000000 --- a/client/model_uk_consolidated_sanctions_list.go +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// UkConsolidatedSanctionsList struct for UkConsolidatedSanctionsList -type UkConsolidatedSanctionsList struct { - Names []string `json:"names,omitempty"` - Addresses []string `json:"addresses,omitempty"` - Countries []string `json:"countries,omitempty"` - GroupType string `json:"groupType,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` - // The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. - MatchedName string `json:"matchedName,omitempty"` -} diff --git a/client/model_uk_sanctions_list.go b/client/model_uk_sanctions_list.go deleted file mode 100644 index 22abc4ed..00000000 --- a/client/model_uk_sanctions_list.go +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// UkSanctionsList struct for UkSanctionsList -type UkSanctionsList struct { - Names []string `json:"names,omitempty"` - NonLatinNames []string `json:"nonLatinNames,omitempty"` - EntityType string `json:"entityType,omitempty"` - Addresses []string `json:"addresses,omitempty"` - AddressCountries []string `json:"addressCountries,omitempty"` - StateLocalities []string `json:"stateLocalities,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` - // The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. - MatchedName string `json:"matchedName,omitempty"` -} diff --git a/client/model_unverified.go b/client/model_unverified.go deleted file mode 100644 index 194f4f23..00000000 --- a/client/model_unverified.go +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -// Unverified struct for Unverified -type Unverified struct { - EntityID string `json:"entityID,omitempty"` - Name string `json:"name,omitempty"` - Addresses []string `json:"addresses,omitempty"` - SourceListURL string `json:"sourceListURL,omitempty"` - SourceInfoURL string `json:"sourceInfoURL,omitempty"` - // Match percentage of search query - Match float32 `json:"match,omitempty"` - // The highest scoring term from the search query. This term is the precomputed indexed value used by the search algorithm. - MatchedName string `json:"matchedName,omitempty"` -} diff --git a/client/response.go b/client/response.go deleted file mode 100644 index e6aef386..00000000 --- a/client/response.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Watchman API - * - * Moov Watchman offers download, parse, and search functions over numerous U.S. trade sanction lists for complying with regional laws. Also included is a web UI and async webhook notification service to initiate processes on remote systems. - * - * API version: v1 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package client - -import ( - "net/http" -) - -// APIResponse stores the API response returned by the server. -type APIResponse struct { - *http.Response `json:"-"` - Message string `json:"message,omitempty"` - // Operation is the name of the OpenAPI operation. - Operation string `json:"operation,omitempty"` - // RequestURL is the request URL. This value is always available, even if the - // embedded *http.Response is nil. - RequestURL string `json:"url,omitempty"` - // Method is the HTTP method used for the request. This value is always - // available, even if the embedded *http.Response is nil. - Method string `json:"method,omitempty"` - // Payload holds the contents of the response body (which may be nil or empty). - // This is provided here as the raw response.Body() reader will have already - // been drained. - Payload []byte `json:"-"` -} - -// NewAPIResponse returns a new APIResonse object. -func NewAPIResponse(r *http.Response) *APIResponse { - - response := &APIResponse{Response: r} - return response -} - -// NewAPIResponseWithError returns a new APIResponse object with the provided error message. -func NewAPIResponseWithError(errorMessage string) *APIResponse { - - response := &APIResponse{Message: errorMessage} - return response -} From fb2e973630ce78433d16f056c1b34278f5ebf299 Mon Sep 17 00:00:00 2001 From: Adam Shannon Date: Thu, 9 Jan 2025 15:54:34 -0600 Subject: [PATCH 5/9] build: remove openapi-generator configs --- .openapi-generator/admin-generator-config.yml | 9 --------- .openapi-generator/client-generator-config.yml | 10 ---------- 2 files changed, 19 deletions(-) delete mode 100644 .openapi-generator/admin-generator-config.yml delete mode 100644 .openapi-generator/client-generator-config.yml diff --git a/.openapi-generator/admin-generator-config.yml b/.openapi-generator/admin-generator-config.yml deleted file mode 100644 index 08df3cb0..00000000 --- a/.openapi-generator/admin-generator-config.yml +++ /dev/null @@ -1,9 +0,0 @@ -inputSpec: /local/api/admin.yaml -outputDir: /local/admin -generatorName: go -artifactId: "watchman-admin" -gitUserId: moov-io -gitRepoId: "watchman" -additionalProperties: - isGoSubmodule: true - packageName: admin diff --git a/.openapi-generator/client-generator-config.yml b/.openapi-generator/client-generator-config.yml deleted file mode 100644 index fd7a3bf0..00000000 --- a/.openapi-generator/client-generator-config.yml +++ /dev/null @@ -1,10 +0,0 @@ -inputSpec: /local/api/client.yaml -outputDir: /local/client -generatorName: go -artifactId: "watchman-client" -gitUserId: moov-io -gitRepoId: "watchman" -additionalProperties: - isGoSubmodule: true - packageName: client - enumClassPrefix: true From 8499d5d249362a852743d3cfadcfea7ce5186cc0 Mon Sep 17 00:00:00 2001 From: Adam Shannon Date: Thu, 9 Jan 2025 16:01:56 -0600 Subject: [PATCH 6/9] cmd/server: add quick tests --- cmd/server/config_test.go | 24 ++++++++++++++++++++++++ cmd/server/download_test.go | 23 +++++++++++++++++++++-- internal/download/models.go | 2 +- 3 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 cmd/server/config_test.go diff --git a/cmd/server/config_test.go b/cmd/server/config_test.go new file mode 100644 index 00000000..fac3e270 --- /dev/null +++ b/cmd/server/config_test.go @@ -0,0 +1,24 @@ +// Copyright The Moov Authors +// Use of this source code is governed by an Apache License +// license that can be found in the LICENSE file. + +package main + +import ( + "testing" + "time" + + "github.com/moov-io/base/log" + "github.com/stretchr/testify/require" +) + +func TestLoadConfig(t *testing.T) { + logger := log.NewTestLogger() + + conf, err := LoadConfig(logger) + require.NoError(t, err) + require.NotNil(t, conf) + + require.Equal(t, ":8084", conf.Servers.BindAddress) + require.Equal(t, 12*time.Hour, conf.Download.RefreshInterval) +} diff --git a/cmd/server/download_test.go b/cmd/server/download_test.go index 831bf483..9df6abda 100644 --- a/cmd/server/download_test.go +++ b/cmd/server/download_test.go @@ -17,8 +17,27 @@ package main // "github.com/moov-io/base/log" // "github.com/moov-io/watchman/pkg/ofac" -// "github.com/stretchr/testify/require" -// ) +import ( + "testing" + "time" + + "github.com/moov-io/watchman/internal/download" + + "github.com/stretchr/testify/require" +) + +func TestGetRefreshInterval(t *testing.T) { + conf := download.Config{ + RefreshInterval: 2 * time.Minute, + } + got := getRefreshInterval(conf) + require.Equal(t, 2*time.Minute, got) + + t.Setenv("DATA_REFRESH_INTERVAL", "1h") + + got = getRefreshInterval(conf) + require.Equal(t, 1*time.Hour, got) +} // func TestDownloadStats(t *testing.T) { // when := time.Date(2022, time.May, 21, 9, 4, 0, 0, time.UTC) diff --git a/internal/download/models.go b/internal/download/models.go index eb5cbe29..a1a0f8ce 100644 --- a/internal/download/models.go +++ b/internal/download/models.go @@ -19,5 +19,5 @@ type Config struct { RefreshInterval time.Duration InitialDataDirectory string - DisabledLists []string // us_ofac, eu_csl, etc... + DisabledLists []string // us_ofac, eu_csl, etc... // TODO(adam): check when we pull in other lists } From 7605ab1aaacab1eeccba0d8837cf4ce9bfb457aa Mon Sep 17 00:00:00 2001 From: Adam Shannon Date: Fri, 10 Jan 2025 15:30:38 -0600 Subject: [PATCH 7/9] search: revamp similarity scoring, further test OFAC mapping and scoring --- internal/download/download.go | 23 +- internal/indices/indices.go | 77 ++ internal/indices/indices_test.go | 38 + internal/prepare/pipeline_reorder.go | 58 +- internal/prepare/pipeline_reorder_test.go | 1 + internal/prepare/pipeline_stopwords.go | 12 +- internal/prepare/pipeline_stopwords_test.go | 13 +- internal/search/service.go | 21 +- internal/search/service_test.go | 11 +- pkg/ofac/mapper.go | 122 +- pkg/ofac/mapper_business_test.go | 188 ++- pkg/ofac/mapper_person_private_test.go | 36 + pkg/ofac/mapper_person_test.go | 150 ++- pkg/ofac/reader.go | 2 +- pkg/search/model_searched_entity_test.go | 7 +- pkg/search/models.go | 11 +- pkg/search/models_test.go | 7 +- pkg/search/similarity.go | 1184 +++++-------------- pkg/search/similarity_address.go | 116 ++ pkg/search/similarity_close.go | 217 ++++ pkg/search/similarity_exact.go | 771 ++++++++++++ pkg/search/similarity_fuzzy.go | 666 +++++++++++ pkg/search/similarity_meta.go | 91 ++ pkg/search/similarity_ofac_test.go | 533 ++++----- 24 files changed, 2998 insertions(+), 1357 deletions(-) create mode 100644 internal/indices/indices.go create mode 100644 internal/indices/indices_test.go create mode 100644 pkg/ofac/mapper_person_private_test.go create mode 100644 pkg/search/similarity_address.go create mode 100644 pkg/search/similarity_close.go create mode 100644 pkg/search/similarity_exact.go create mode 100644 pkg/search/similarity_fuzzy.go create mode 100644 pkg/search/similarity_meta.go diff --git a/internal/download/download.go b/internal/download/download.go index b2ca88e4..46a9df24 100644 --- a/internal/download/download.go +++ b/internal/download/download.go @@ -37,10 +37,13 @@ func (dl *downloader) RefreshAll(ctx context.Context) (Stats, error) { logger := dl.logger.Info().With(log.Fields{ "initial_data_directory": log.String(dl.conf.InitialDataDirectory), }) + start := time.Now() logger.Info().Log("starting list refresh") g, ctx := errgroup.WithContext(ctx) - preparedLists := make(chan preparedList) + preparedLists := make(chan preparedList, 1) + + fmt.Println("starting all lists") g.Go(func() error { err := loadOFACRecords(ctx, logger, dl.conf, preparedLists) @@ -51,15 +54,21 @@ func (dl *downloader) RefreshAll(ctx context.Context) (Stats, error) { }) err := g.Wait() + close(preparedLists) + fmt.Printf("finished all lists: %v\n", time.Since(start)) + start = time.Now() + if err != nil { return stats, fmt.Errorf("problem loading lists: %v", err) } // accumulate the lists + fmt.Println("starting to accumulate preparedLists") for list := range preparedLists { stats.Lists[string(list.ListName)] = len(list.Entities) stats.Entities = append(stats.Entities, list.Entities...) } + fmt.Printf("finished accumulating preparedLists: %v\n", time.Since(start)) stats.EndedAt = time.Now().In(time.UTC) @@ -72,7 +81,11 @@ type preparedList struct { } func loadOFACRecords(ctx context.Context, logger log.Logger, conf Config, responseCh chan preparedList) error { + start := time.Now() + fmt.Println("starting OFAC download") files, err := ofac.Download(ctx, logger, conf.InitialDataDirectory) + fmt.Printf("finished OFAC download: %v\n", time.Since(start)) + start = time.Now() if err != nil { return fmt.Errorf("download: %v", err) } @@ -80,17 +93,25 @@ func loadOFACRecords(ctx context.Context, logger log.Logger, conf Config, respon return fmt.Errorf("unexpected %d OFAC files found", len(files)) } + fmt.Println("starting OFAC parse") res, err := ofac.Read(files) + fmt.Printf("finished OFAC parse: %v\n", time.Since(start)) + start = time.Now() if err != nil { return err } + fmt.Println("starting OFAC grouping") entities := ofac.GroupIntoEntities(res.SDNs, res.Addresses, res.SDNComments, res.AlternateIdentities) + fmt.Printf("finished OFAC grouping: %v\n", time.Since(start)) + start = time.Now() + fmt.Println("sending OFAC preparedList") responseCh <- preparedList{ ListName: search.SourceUSOFAC, Entities: entities, } + fmt.Printf("finished sending OFAC preparedList: %v\n", time.Since(start)) return nil } diff --git a/internal/indices/indices.go b/internal/indices/indices.go new file mode 100644 index 00000000..c0dc212f --- /dev/null +++ b/internal/indices/indices.go @@ -0,0 +1,77 @@ +package indices + +import ( + "fmt" + "sync" +) + +// New creates slice indices for parallel processing +func New(total, groups int) []int { + if groups <= 0 { + return []int{0, total} + } + if groups == 1 || groups >= total { + return []int{0, total} + } + + chunkSize := total / groups + remaining := total % groups + + // Pre-allocate slice with exact capacity + xs := make([]int, 0, groups+1) + xs = append(xs, 0) + + pos := 0 + for i := 0; i < groups-1; i++ { + pos += chunkSize + if remaining > 0 { + pos++ + remaining-- + } + xs = append(xs, pos) + } + return append(xs, total) +} + +// ProcessSlice processes input slice concurrently using the provided function +func ProcessSlice[T any, F any](in []T, groups int, f func(T) F) []F { + if len(in) == 0 { + return []F{} + } + + indices := New(len(in), groups) + numGroups := len(indices) - 1 // Number of actual chunks + + // Pre-allocate output slice + out := make([]F, len(in)) + + // Use WaitGroup for synchronization + var wg sync.WaitGroup + wg.Add(numGroups) + + // Process each chunk concurrently + for i := 0; i < numGroups; i++ { + start := indices[i] + end := indices[i+1] + + go func(start, end int) { + defer wg.Done() + + // Process chunk and write directly to pre-allocated output slice + for idx, v := range in[start:end] { + out[start+idx] = f(v) + } + + if debug { + fmt.Printf("Processed chunk [%d:%d]\n", start, end) + } + }(start, end) + } + + // Wait for all goroutines to complete + wg.Wait() + return out +} + +// Enable/disable debug output +const debug = false diff --git a/internal/indices/indices_test.go b/internal/indices/indices_test.go new file mode 100644 index 00000000..b075ff64 --- /dev/null +++ b/internal/indices/indices_test.go @@ -0,0 +1,38 @@ +package indices + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewIndicies(t *testing.T) { + indices := New(122, 5) + require.Len(t, indices, 6) + + expected := []int{0, 25, 50, 74, 98, 122} + require.Equal(t, expected, indices) +} + +func TestProcessSlice(t *testing.T) { + input := []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} + + originalInput := make([]int, len(input)) + copy(originalInput, input) + require.Equal(t, input, originalInput) + + fn := func(in int) string { + return fmt.Sprintf("%d", in*5) + } + expected := []string{"5", "10", "15", "20", "25", "30", "35", "40", "45", "50"} + + require.Equal(t, expected, ProcessSlice(input, 3, fn)) + require.Equal(t, originalInput, input) // input is unchanged + + require.Equal(t, expected, ProcessSlice(input, 7, fn)) + require.Equal(t, originalInput, input) // input is unchanged + + require.Equal(t, expected, ProcessSlice(input, 10, fn)) + require.Equal(t, originalInput, input) // input is unchanged +} diff --git a/internal/prepare/pipeline_reorder.go b/internal/prepare/pipeline_reorder.go index 6d5d106e..57fe63de 100644 --- a/internal/prepare/pipeline_reorder.go +++ b/internal/prepare/pipeline_reorder.go @@ -10,24 +10,52 @@ import ( "strings" ) +// This pattern looks for: +// +// a comma, optional whitespace, then +// 1+ Unicode letters and diacritics (\p{L}\p{M}), plus allowed punctuation/apostrophes/hyphens/spaces +// until the end of the string. +// +// Examples that should match (and reorder): +// +// "AL-ZAYDI, Shibl Muhsin 'Ubayd" --> "Shibl Muhsin 'Ubayd AL-ZAYDI" +// "MADURO MOROS, Nicolas" --> "Nicolas MADURO MOROS" var ( - surnamePrecedes = regexp.MustCompile(`(,?[\s?a-zA-Z\.]{1,})$`) + surnamePrecedes = regexp.MustCompile(`,(?:\s+)?([\p{L}\p{M}'’\-\.\s]+)$`) ) -// ReorderSDNName will take a given SDN name and if it matches a specific pattern where -// the first name is placed after the last name (surname) to return a string where the first name -// preceedes the last. -// -// Example: -// SDN EntityID: 19147 has 'FELIX B. MADURO S.A.' -// SDN EntityID: 22790 has 'MADURO MOROS, Nicolas' -func ReorderSDNName(name string, tpe string) string { - if !strings.EqualFold(tpe, "individual") { - return name // only reorder individual names +func ReorderSDNNames(names []string, sdnType string) []string { + out := make([]string, len(names)) + for idx := range names { + out[idx] = ReorderSDNName(names[idx], sdnType) } - v := surnamePrecedes.FindString(name) - if v == "" { - return name // no match on 'Doe, John' + return out +} + +// ReorderSDNName will take a given SDN name and, if it matches "Surname, FirstName(s)", +// reorder it to "FirstName(s) Surname" (only for type == "individual"). +func ReorderSDNName(name, sdnType string) string { + // Only reorder for individuals + if !strings.EqualFold(sdnType, "individual") { + return name } - return strings.TrimSpace(fmt.Sprintf("%s %s", strings.TrimPrefix(v, ","), strings.TrimSuffix(name, v))) + + // Try matching the pattern + match := surnamePrecedes.FindStringSubmatch(name) + if len(match) < 2 { + // No match => no reordering + return name + } + + // match[1] is the part after the comma (the "first/given names" portion). + givenNames := strings.TrimSpace(match[1]) + + // match[0] is the entire substring matching the pattern, including the comma; + // remove it from the original to isolate the "surname" part. + surname := strings.TrimSuffix(name, match[0]) + surname = strings.TrimSpace(surname) + + // Rebuild as "GivenName(s) Surname" + out := fmt.Sprintf("%s %s", givenNames, surname) + return strings.TrimSpace(out) } diff --git a/internal/prepare/pipeline_reorder_test.go b/internal/prepare/pipeline_reorder_test.go index 00898715..b68c8995 100644 --- a/internal/prepare/pipeline_reorder_test.go +++ b/internal/prepare/pipeline_reorder_test.go @@ -26,6 +26,7 @@ func TestReorderSDNName(t *testing.T) { {"MADURO MOROS, Nicolas", "Nicolas MADURO MOROS"}, {"IBRAHIM, Sadr", "Sadr IBRAHIM"}, {"AL ZAWAHIRI, Dr. Ayman", "Dr. Ayman AL ZAWAHIRI"}, + {"AL-ZAYDI, Shibl Muhsin 'Ubayd", "Shibl Muhsin 'Ubayd AL-ZAYDI"}, // Issue 115 {"Bush, George W", "George W Bush"}, {"RIZO MORENO, Jorge Luis", "Jorge Luis RIZO MORENO"}, diff --git a/internal/prepare/pipeline_stopwords.go b/internal/prepare/pipeline_stopwords.go index e9087e1e..ef824743 100644 --- a/internal/prepare/pipeline_stopwords.go +++ b/internal/prepare/pipeline_stopwords.go @@ -10,8 +10,6 @@ import ( "strconv" "strings" - "github.com/moov-io/watchman/pkg/ofac" - "github.com/abadojack/whatlanggo" "github.com/bbalet/stopwords" "github.com/pariz/gountries" @@ -44,8 +42,8 @@ var ( numberRegex = regexp.MustCompile(`([\d\.\,\-]{1,}[\d]{1,})`) ) -func RemoveStopwords(in string, addrs []ofac.Address) string { - lang := detectLanguage(in, addrs) +func RemoveStopwords(in string, countryName string) string { + lang := detectLanguage(in, countryName) return removeStopwords(in, lang) } @@ -73,14 +71,14 @@ func removeStopwords(in string, lang whatlanggo.Lang) string { // detectLanguage will return a guess as to the appropriate language a given SDN's name // is written in. The addresses must be linked to the SDN whose name is detected. -func detectLanguage(in string, addrs []ofac.Address) whatlanggo.Lang { +func detectLanguage(in string, countryName string) whatlanggo.Lang { info := whatlanggo.Detect(in) if info.IsReliable() { // Return the detected language if whatlanggo is confident enough return info.Lang } - if len(addrs) == 0 { + if countryName == "" { // If no addresses are associated to this text blob then fallback to English return whatlanggo.Eng } @@ -89,7 +87,7 @@ func detectLanguage(in string, addrs []ofac.Address) whatlanggo.Lang { // // TODO(adam): Should we do this only if there's one address? If there are multiple should we // fallback to English or a mixed set? - country, err := gountries.New().FindCountryByName(addrs[0].Country) + country, err := gountries.New().FindCountryByName(countryName) if len(country.Languages) == 0 || err != nil { return whatlanggo.Eng } diff --git a/internal/prepare/pipeline_stopwords_test.go b/internal/prepare/pipeline_stopwords_test.go index ef190174..4f704019 100644 --- a/internal/prepare/pipeline_stopwords_test.go +++ b/internal/prepare/pipeline_stopwords_test.go @@ -7,7 +7,6 @@ package prepare import ( "testing" - "github.com/moov-io/watchman/pkg/ofac" "github.com/stretchr/testify/require" "github.com/abadojack/whatlanggo" @@ -20,14 +19,6 @@ func TestStopwordsEnv(t *testing.T) { } func TestStopwords__detect(t *testing.T) { - addrs := func(country string) []ofac.Address { - return []ofac.Address{ - { - Country: country, - }, - } - } - cases := []struct { in string country string @@ -45,7 +36,7 @@ func TestStopwords__detect(t *testing.T) { } for _, tc := range cases { - got := detectLanguage(tc.in, addrs(tc.country)) + got := detectLanguage(tc.in, tc.country) require.Equal(t, tc.expected, got) } } @@ -97,7 +88,7 @@ func TestStopwords__apply(t *testing.T) { }, } for _, test := range cases { - got := RemoveStopwords(test.in, nil) + got := RemoveStopwords(test.in, "") require.Equal(t, test.expected, got) } } diff --git a/internal/search/service.go b/internal/search/service.go index 991ea552..a9f19250 100644 --- a/internal/search/service.go +++ b/internal/search/service.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "github.com/moov-io/watchman/internal/indices" "github.com/moov-io/watchman/internal/largest" "github.com/moov-io/watchman/pkg/search" @@ -71,7 +72,7 @@ type SearchOpts struct { func (s *service) performSearch(ctx context.Context, query search.Entity[search.Value], opts SearchOpts) ([]search.SearchedEntity[search.Value], error) { items := largest.NewItems(opts.Limit, opts.MinMatch) - indices := makeIndices(len(s.entities), opts.Limit/3) // limit goroutines + indices := indices.New(len(s.entities), opts.Limit/3) // limit goroutines var wg sync.WaitGroup wg.Add(len(indices)) @@ -143,21 +144,3 @@ func performSubSearch(items *largest.Items, query search.Entity[search.Value], e }) } } - -func makeIndices(total, groups int) []int { - if groups == 1 || groups >= total { - return []int{total} - } - xs := []int{0} - i := 0 - for { - if i > total { - break - } - i += total / groups - if i < total { - xs = append(xs, i) - } - } - return append(xs, total) -} diff --git a/internal/search/service_test.go b/internal/search/service_test.go index 0bdeb034..4ecc3b2c 100644 --- a/internal/search/service_test.go +++ b/internal/search/service_test.go @@ -37,6 +37,7 @@ func TestService_Search(t *testing.T) { t.Run("basic", func(t *testing.T) { results, err := svc.Search(ctx, search.Entity[search.Value]{ Name: "SHIPPING LIMITED", + Type: search.EntityBusiness, }, opts) require.NoError(t, err) require.Greater(t, len(results), 1) @@ -63,15 +64,9 @@ func TestService_Search(t *testing.T) { res := results[0] require.InDelta(t, 1.00, res.Match, 0.001) - }) -} -func TestService_makeIndicies(t *testing.T) { - indices := makeIndices(122, 5) - require.Len(t, indices, 7) - - expected := []int{0, 24, 48, 72, 96, 120, 122} - require.Equal(t, expected, indices) + // 36216 + }) } func testInputs(tb testing.TB, paths ...string) map[string]io.ReadCloser { diff --git a/pkg/ofac/mapper.go b/pkg/ofac/mapper.go index 2250fd07..2cc9605c 100644 --- a/pkg/ofac/mapper.go +++ b/pkg/ofac/mapper.go @@ -8,10 +8,13 @@ import ( "cmp" "fmt" "regexp" + "runtime" "strconv" "strings" "time" + "github.com/moov-io/watchman/internal/indices" + "github.com/moov-io/watchman/internal/prepare" "github.com/moov-io/watchman/pkg/address" "github.com/moov-io/watchman/pkg/search" ) @@ -24,11 +27,15 @@ var ( // Matches both "POB Baghdad, Iraq" and "Alt. POB: Keren Eritrea" pobRegex = regexp.MustCompile(`(?i)(?:Alt\.)?\s*POB:?\s+([^;]+)`) // Contact information patterns - emailRegex = regexp.MustCompile(`(?i)(?:Email|EMAIL)[:\s]+([^;]+)`) - phoneRegex = regexp.MustCompile(`(?i)(?:Telephone|Phone|PHONE)[:\s]+([^;]+)`) - faxRegex = regexp.MustCompile(`(?i)Fax[:\s]+([^;]+)`) + emailRegex = regexp.MustCompile(`(?i)(?:Email|EMAIL)[:\s](Address)?[:\s]+([^;]+)`) + phoneRegex = regexp.MustCompile(`(?i)(?:Telephone|Phone|PHONE)[:\s](No.?)?[:\s]+([^;]+)`) + faxRegex = regexp.MustCompile(`(?i)Fax[:\s](No.?)?[:\s]+([^;]+)`) // Website patterns websiteRegex = regexp.MustCompile(`(?i)(?:Website|http)[:\s]+([^;\s]+)`) + + // Identifier Regex + identifierRegex = regexp.MustCompile(`(?i)%s[\s.:]+([^\s](?:[^;()]*[^\s;()])?)(?:\s*\(([^)]+)\))?`) + // Country extraction pattern countryParenRegex = regexp.MustCompile(`\(([\w\s]+)\)`) ) @@ -41,6 +48,15 @@ var ( } ) +// Company Number 05527424 (United Kingdom) +// Company Number IMO 1991835. +// Commercial Registry Number 0411518776478 (Iran) +// Enterprise Number 0430.033.662 (Belgium). +// Tax ID No. 230810605961 (Russia). +// Trade License No. 04110179 (United Kingdom). +// UK Company Number 01019769 (United Kingdom) +// US FEIN 000920912 (United States). + func makeIdentifiers(remarks []string, needles []string) []search.Identifier { seen := make(map[string]bool) var out []search.Identifier @@ -60,12 +76,19 @@ func makeIdentifiers(remarks []string, needles []string) []search.Identifier { func makeIdentifier(remarks []string, suffix string) *search.Identifier { found := findMatchingRemarks(remarks, suffix) + if len(found) == 0 { + for _, rmk := range remarks { + if matches := identifierRegex.FindStringSubmatch(rmk); len(matches) > 1 { + found = append(found, remark{fullName: suffix, value: matches[1]}) + break + } + } + } if len(found) == 0 { return nil } // Often the country is in parenthesis at the end, so let's look for that - // Example: Business Number 51566843 (Hong Kong) country := "" value := found[0].value @@ -123,11 +146,7 @@ func extractCountry(remark string) string { } func GroupIntoEntities(sdns []SDN, addresses []Address, comments []SDNComments, altIds []AlternateIdentity) []search.Entity[search.Value] { - out := make([]search.Entity[search.Value], len(sdns)) - - for idx, sdn := range sdns { - // find addresses, comments, and altIDs which match - + fn := func(sdn SDN) search.Entity[search.Value] { var addrs []Address for _, addr := range addresses { if sdn.EntityID == addr.EntityID { @@ -149,17 +168,20 @@ func GroupIntoEntities(sdns []SDN, addresses []Address, comments []SDNComments, } } - out[idx] = ToEntity(sdn, addrs, cmts, alts) + return ToEntity(sdn, addrs, cmts, alts) } - return out + groups := runtime.NumCPU() // arbitrary group size // TODO(adam): + + return indices.ProcessSlice(sdns, groups, fn) } func ToEntity(sdn SDN, addresses []Address, comments []SDNComments, altIds []AlternateIdentity) search.Entity[search.Value] { out := search.Entity[search.Value]{ - Name: sdn.SDNName, + Name: prepare.ReorderSDNName(sdn.SDNName, sdn.SDNType), Source: search.SourceUSOFAC, SourceData: sdn, + SourceID: sdn.EntityID, } remarks := splitRemarks(sdn.Remarks) @@ -172,6 +194,7 @@ func ToEntity(sdn SDN, addresses []Address, comments []SDNComments, altIds []Alt out.CryptoAddresses = parseCryptoAddresses(remarks) // Extract common fields regardless of entity type + out.Contact = parseContactInfo(remarks) out.Addresses = parseAddresses(addresses) // Get all alternate names from both remarks and AlternateIdentity entries @@ -179,12 +202,13 @@ func ToEntity(sdn SDN, addresses []Address, comments []SDNComments, altIds []Alt altNames = append(altNames, parseAltNames(remarks)...) altNames = append(altNames, parseAltIdentities(altIds)...) altNames = deduplicateStrings(altNames) + altNames = prepare.ReorderSDNNames(altNames, sdn.SDNType) switch strings.ToLower(strings.TrimSpace(sdn.SDNType)) { case "-0-", "": out.Type = search.EntityBusiness out.Business = &search.Business{ - Name: sdn.SDNName, + Name: prepare.RemoveCompanyTitles(sdn.SDNName), AltNames: altNames, } out.Business.Created = findDateStamp(findMatchingRemarks(remarks, "Organization Established Date")) @@ -198,18 +222,33 @@ func ToEntity(sdn SDN, addresses []Address, comments []SDNComments, altIds []Alt "Certificate of Incorporation Number", "Chamber of Commerce Number", "Chinese Commercial Code", - "Registered Charity No.", "Commercial Registry Number", "Company Number", + "Company ID", // new: e.g., "Company ID: No. 59 531..." + "D-U-N-S Number", // new: e.g., "D-U-N-S Number 33-843-5672" + "Dubai Chamber of Commerce Membership No", // new "Enterprise Number", + "Fiscal Code", // new: business tax identifiers + "Folio Mercantil No", // new: Mexican business registration "Legal Entity Number", + "Matricula Mercantil No", // new: Colombian business registration + "Public Registration Number", // new "Registration Number", + "RIF", // new: Venezuelan tax ID + "RUC", // new: Panama business registration + "Romanian C.R", // new: Romanian Commercial Registry + "Tax ID No.", // new: Important business identifier + "Trade License No", // new + "UK Company Number", // new: Specific UK format + "US FEIN", // new: US Federal Employer ID Number + "United Social Credit Code Certificate", // new: Chinese business ID + "V.A.T. Number", // new: VAT registration numbers }) case "individual": out.Type = search.EntityPerson out.Person = &search.Person{ - Name: sdn.SDNName, + Name: out.Name, AltNames: altNames, Gender: search.Gender(strings.ToLower(firstValue(findMatchingRemarks(remarks, "Gender")))), } @@ -271,11 +310,13 @@ func ToEntity(sdn SDN, addresses []Address, comments []SDNComments, altIds []Alt } func parseAddresses(inputs []Address) []search.Address { - out := make([]search.Address, len(inputs)) + var out []search.Address for i := range inputs { - addr := fmt.Sprintf("%s %s %s", inputs[i].Address, inputs[i].CityStateProvincePostalCode, inputs[i].Country) - - out[i] = address.ParseAddress(addr) + input := fmt.Sprintf("%s %s %s", inputs[i].Address, inputs[i].CityStateProvincePostalCode, inputs[i].Country) + addr := address.ParseAddress(input) + if addr.Line1 != "" { + out = append(out, addr) + } } return out } @@ -611,6 +652,49 @@ func deduplicateTitles(titles []string) []string { return result } +func parseContactInfo(remarks []string) search.ContactInfo { + var out search.ContactInfo + + // Look for emails, phone numbers, fax numbers, websites, etc.. + for _, remark := range remarks { + remark = strings.TrimSuffix(remark, ".") + + // Emails + matches := emailRegex.FindAllStringSubmatch(remark, -1) + for _, m := range matches { + if len(m) > 1 { + out.EmailAddresses = append(out.EmailAddresses, m[len(m)-1]) + } + } + + // Phone Numbers + matches = phoneRegex.FindAllStringSubmatch(remark, -1) + for _, m := range matches { + if len(m) > 1 { + out.PhoneNumbers = append(out.PhoneNumbers, m[len(m)-1]) + } + } + + // Fax Numbers + matches = faxRegex.FindAllStringSubmatch(remark, -1) + for _, m := range matches { + if len(m) > 1 { + out.FaxNumbers = append(out.FaxNumbers, m[len(m)-1]) + } + } + + // Websites + matches = websiteRegex.FindAllStringSubmatch(remark, -1) + for _, m := range matches { + if len(m) > 1 { + out.Websites = append(out.Websites, m[len(m)-1]) + } + } + } + + return out +} + func parseCryptoAddresses(remarks []string) []search.CryptoAddress { var addresses []search.CryptoAddress diff --git a/pkg/ofac/mapper_business_test.go b/pkg/ofac/mapper_business_test.go index 6c980adb..cc2a335a 100644 --- a/pkg/ofac/mapper_business_test.go +++ b/pkg/ofac/mapper_business_test.go @@ -1,23 +1,177 @@ -package ofac +package ofac_test import ( + "context" + "path/filepath" "sort" "testing" + "time" + "github.com/moov-io/watchman/internal/download" + "github.com/moov-io/watchman/pkg/ofac" "github.com/moov-io/watchman/pkg/search" + "github.com/moov-io/base/log" "github.com/stretchr/testify/require" ) +func TestMapperBusiness__FromSource(t *testing.T) { + t.Run("33151 - crypto addresses", func(t *testing.T) { + found := findOFACEntity(t, "33151") + require.Equal(t, "SUEX OTC, S.R.O.", found.Name) + require.Equal(t, search.EntityBusiness, found.Type) + require.Equal(t, search.SourceUSOFAC, found.Source) + require.Equal(t, "33151", found.SourceID) + + require.Nil(t, found.Person) + require.NotNil(t, found.Business) + require.Nil(t, found.Organization) + require.Nil(t, found.Aircraft) + require.Nil(t, found.Vessel) + + business := found.Business + require.Equal(t, "SUEX OTC, S.R.O.", business.Name) + require.ElementsMatch(t, []string{"SUCCESSFUL EXCHANGE"}, business.AltNames) + + createdAt := time.Date(2018, time.September, 25, 0, 0, 0, 0, time.UTC) + require.Equal(t, createdAt.Format(time.RFC3339), business.Created.Format(time.RFC3339)) + require.Nil(t, business.Dissolved) + + expectedIdentifiers := []search.Identifier{ + {Name: "Company Number", Country: "Czech Republic", Identifier: "07486049"}, + {Name: "Legal Entity Number", Country: "Czech Republic", Identifier: "5299007NTWCC3U23WM81"}, + } + require.ElementsMatch(t, expectedIdentifiers, business.Identifier) + + expectedContact := search.ContactInfo{ + Websites: []string{"suex.io"}, + } + require.Equal(t, expectedContact, found.Contact) + require.Empty(t, found.Addresses) + + expectedCryptoAddresses := []search.CryptoAddress{ + {Currency: "XBT", Address: "12HQDsicffSBaYdJ6BhnE22sfjTESmmzKx"}, + {Currency: "XBT", Address: "1L4ncif9hh9TnUveqWq77HfWWt6CJWtrnb"}, + {Currency: "XBT", Address: "13mnk8SvDGqsQTHbiGiHBXqtaQCUKfcsnP"}, + {Currency: "XBT", Address: "1Edue8XZCWNoDBNZgnQkCCivDyr9GEo4x6"}, + {Currency: "XBT", Address: "1ECeZBxCVJ8Wm2JSN3Cyc6rge2gnvD3W5K"}, + {Currency: "XBT", Address: "1J9oGoAiHeRfeMZeUnJ9W7RpV55CdKtgYE"}, + {Currency: "XBT", Address: "1295rkVyNfFpqZpXvKGhDqwhP1jZcNNDMV"}, + {Currency: "XBT", Address: "1LiNmTUPSJEd92ZgVJjAV3RT9BzUjvUCkx"}, + {Currency: "XBT", Address: "1LrxsRd7zNuxPJcL5rttnoeJFy1y4AffYY"}, + {Currency: "XBT", Address: "1KUUJPkyDhamZXgpsyXqNGc3x1QPXtdhgz"}, + {Currency: "XBT", Address: "1CF46Rfbp97absrs7zb7dFfZS6qBXUm9EP"}, + {Currency: "XBT", Address: "1Df883c96LVauVsx9FEgnsourD8DELwCUQ"}, + {Currency: "XBT", Address: "bc1qdt3gml5z5n50y5hm04u2yjdphefkm0fl2zdj68"}, + {Currency: "XBT", Address: "1B64QRxfaa35MVkf7sDjuGUYAP5izQt7Qi"}, + {Currency: "ETH", Address: "0x2f389ce8bd8ff92de3402ffce4691d17fc4f6535"}, + {Currency: "ETH", Address: "0x19aa5fe80d33a56d56c78e82ea5e50e5d80b4dff"}, + {Currency: "ETH", Address: "0xe7aa314c77f4233c18c6cc84384a9247c0cf367b"}, + {Currency: "ETH", Address: "0x308ed4b7b49797e1a98d3818bff6fe5385410370"}, + {Currency: "USDT", Address: "0x2f389ce8bd8ff92de3402ffce4691d17fc4f6535"}, + {Currency: "USDT", Address: "0x19aa5fe80d33a56d56c78e82ea5e50e5d80b4dff"}, + {Currency: "USDT", Address: "1KUUJPkyDhamZXgpsyXqNGc3x1QPXtdhgz"}, + {Currency: "USDT", Address: "1CF46Rfbp97absrs7zb7dFfZS6qBXUm9EP"}, + {Currency: "USDT", Address: "1LrxsRd7zNuxPJcL5rttnoeJFy1y4AffYY"}, + {Currency: "USDT", Address: "1Df883c96LVauVsx9FEgnsourD8DELwCUQ"}, + {Currency: "USDT", Address: "16iWn2J1McqjToYLHSsAyS6En3QA8YQ91H"}, + } + require.ElementsMatch(t, expectedCryptoAddresses, found.CryptoAddresses) + + require.Empty(t, found.Affiliations) + require.Nil(t, found.SanctionsInfo) + require.Empty(t, found.HistoricalInfo) + require.Empty(t, found.Titles) + + sdn, ok := found.SourceData.(ofac.SDN) + require.True(t, ok) + require.Equal(t, "33151", sdn.EntityID) + require.Equal(t, "SUEX OTC, S.R.O.", sdn.SDNName) + }) + + t.Run("12685", func(t *testing.T) { + found := findOFACEntity(t, "12685") + require.Equal(t, "GADDAFI INTERNATIONAL CHARITY AND DEVELOPMENT FOUNDATION", found.Name) + + expectedContact := search.ContactInfo{ + EmailAddresses: []string{"info@gicdf.org"}, + PhoneNumbers: []string{"(218) (0)214778301", "(022) 7363030"}, + FaxNumbers: []string{"(218) (0)214778766", "(022) 7363196"}, + Websites: []string{"www.gicdf.org"}, + } + require.Equal(t, expectedContact, found.Contact) + }) + + t.Run("50544", func(t *testing.T) { + found := findOFACEntity(t, "50544") + + require.Equal(t, "AUTONOMOUS NON-PROFIT ORGANIZATION DIALOG REGIONS", found.Name) + require.Equal(t, search.EntityBusiness, found.Type) + require.Equal(t, search.SourceUSOFAC, found.Source) + require.Equal(t, "50544", found.SourceID) + + require.Nil(t, found.Person) + require.NotNil(t, found.Business) + require.Nil(t, found.Organization) + require.Nil(t, found.Aircraft) + require.Nil(t, found.Vessel) + + business := found.Business + require.Equal(t, "AUTONOMOUS NON-PROFIT ORGANIZATION DIALOG REGIONS", business.Name) + + expectedAltNames := []string{ + "DIALOGUE REGIONS", + "DIALOG REGIONY", + "DIALOGUE", + "AUTONOMOUS NON-PROFIT ORGANIZATION FOR THE DEVELOPMENT OF DIGITAL PROJECTS IN THE FIELD OF PUBLIC RELATIONS AND COMMUNICATIONS DIALOG REGIONS", + "ANO DIALOG REGIONS", + "AVTONOMNAYA NEKOMMERCHESKAYA ORGANIZATSIYA PO RAZVITIYU TSIFROVYKH PROEKTOV V SFERE OBSHCHESTEVENNYKH SVYAZEI I KOMMUNIKATSII DIALOG REGIONY", + } + require.ElementsMatch(t, expectedAltNames, business.AltNames) + + createdAt := time.Date(2020, time.July, 21, 0, 0, 0, 0, time.UTC) + require.Equal(t, createdAt.Format(time.RFC3339), business.Created.Format(time.RFC3339)) + require.Nil(t, business.Dissolved) + + expectedIdentifiers := []search.Identifier{ + {Name: "Business Registration Number", Country: "Russia", Identifier: "1207700248030"}, + {Name: "Tax ID No.", Country: "Russia", Identifier: "9709063550"}, + } + require.ElementsMatch(t, expectedIdentifiers, business.Identifier) + + expectedContact := search.ContactInfo{ + Websites: []string{"www.dialog.info", "www.dialog-regions.ru"}, + } + require.Equal(t, expectedContact, found.Contact) + require.Empty(t, found.Addresses) + require.Empty(t, found.CryptoAddresses) + + expectedAffiliations := []search.Affiliation{ + {EntityName: "AUTONOMOUS NON-PROFIT ORGANIZATION DIALOG", Type: "Linked To", Details: ""}, + } + require.ElementsMatch(t, expectedAffiliations, found.Affiliations) + + require.Nil(t, found.SanctionsInfo) + require.Empty(t, found.HistoricalInfo) + require.Empty(t, found.Titles) + + sdn, ok := found.SourceData.(ofac.SDN) + require.True(t, ok) + require.Equal(t, "50544", sdn.EntityID) + require.Equal(t, "AUTONOMOUS NON-PROFIT ORGANIZATION DIALOG REGIONS", sdn.SDNName) + + }) +} + func TestMapper__CompleteBusiness(t *testing.T) { - sdn := &SDN{ + sdn := &ofac.SDN{ EntityID: "12345", SDNName: "ACME CORPORATION", SDNType: "-0-", Remarks: "Business Registration Number 51566843 (Hong Kong); Commercial Registry Number CH-020.1.066.499-9 (Switzerland); Company Number 05527424 (United Kingdom)", } - e := ToEntity(*sdn, nil, nil, nil) + e := ofac.ToEntity(*sdn, nil, nil, nil) require.Equal(t, "ACME CORPORATION", e.Name) require.Equal(t, search.EntityBusiness, e.Type) require.Equal(t, search.SourceUSOFAC, e.Source) @@ -53,14 +207,14 @@ func TestMapper__CompleteBusiness(t *testing.T) { } func TestMapper__CompleteBusinessWithRemarks(t *testing.T) { - sdn := &SDN{ + sdn := &ofac.SDN{ EntityID: "12345", SDNName: "ACME CORPORATION", SDNType: "-0-", Remarks: "Business Registration Number 51566843 (Hong Kong); Subsidiary Of: PARENT CORP; Former Name: OLD ACME LTD; Additional Sanctions Information - Subject to Secondary Sanctions", } - e := ToEntity(*sdn, nil, nil, nil) + e := ofac.ToEntity(*sdn, nil, nil, nil) // Test affiliations require.Len(t, e.Affiliations, 1) @@ -76,3 +230,27 @@ func TestMapper__CompleteBusinessWithRemarks(t *testing.T) { require.Equal(t, "Former Name", e.HistoricalInfo[0].Type) require.Equal(t, "OLD ACME LTD", e.HistoricalInfo[0].Value) } + +func findOFACEntity(tb testing.TB, entityID string) search.Entity[search.Value] { + tb.Helper() + + logger := log.NewTestLogger() + conf := download.Config{ + InitialDataDirectory: filepath.Join("..", "ofac", "testdata"), + } + dl, err := download.NewDownloader(logger, conf) + require.NoError(tb, err) + + stats, err := dl.RefreshAll(context.Background()) + require.NoError(tb, err) + + for _, entity := range stats.Entities { + if entityID == entity.SourceID { + return entity + } + } + + tb.Fatalf("OFAC entity %s not found", entityID) + + return search.Entity[search.Value]{} +} diff --git a/pkg/ofac/mapper_person_private_test.go b/pkg/ofac/mapper_person_private_test.go new file mode 100644 index 00000000..92653053 --- /dev/null +++ b/pkg/ofac/mapper_person_private_test.go @@ -0,0 +1,36 @@ +package ofac + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseAltNames(t *testing.T) { + tests := []struct { + remarks []string + expected []string + }{ + { + remarks: []string{"a.k.a. 'SMITH, John'"}, + expected: []string{"SMITH, John"}, + }, + { + remarks: []string{"a.k.a. 'SMITH, John'; a.k.a. 'DOE, Jane'"}, + expected: []string{"SMITH, John", "DOE, Jane"}, + }, + { + remarks: []string{"Some other remark", "a.k.a. 'SMITH, John'"}, + expected: []string{"SMITH, John"}, + }, + { + remarks: []string{}, + expected: nil, + }, + } + + for _, tt := range tests { + result := parseAltNames(tt.remarks) + require.Equal(t, tt.expected, result) + } +} diff --git a/pkg/ofac/mapper_person_test.go b/pkg/ofac/mapper_person_test.go index 0946cca5..df7290af 100644 --- a/pkg/ofac/mapper_person_test.go +++ b/pkg/ofac/mapper_person_test.go @@ -1,34 +1,103 @@ -package ofac +package ofac_test import ( - "path/filepath" "testing" "time" + "github.com/moov-io/watchman/pkg/ofac" "github.com/moov-io/watchman/pkg/search" "github.com/stretchr/testify/require" ) -func TestMapper__Person(t *testing.T) { - res, err := Read(testInputs(t, filepath.Join("..", "..", "test", "testdata", "sdn.csv"))) - require.NoError(t, err) +func TestMapperPerson__FromSource(t *testing.T) { + t.Run("48603", func(t *testing.T) { + found := findOFACEntity(t, "48603") + + require.Equal(t, "Dmitry Yuryevich KHOROSHEV", found.Name) + require.Equal(t, search.EntityPerson, found.Type) + require.Equal(t, search.SourceUSOFAC, found.Source) + require.Equal(t, "48603", found.SourceID) + + require.NotNil(t, found.Person) + require.Nil(t, found.Business) + require.Nil(t, found.Organization) + require.Nil(t, found.Aircraft) + require.Nil(t, found.Vessel) - var sdn *SDN - for i := range res.SDNs { - if res.SDNs[i].EntityID == "15102" { - sdn = &res.SDNs[i] + person := found.Person + require.Equal(t, "Dmitry Yuryevich KHOROSHEV", person.Name) + + expectedAltNames := []string{ + "LOCKBITSUPP", "Dmitriy Yurevich KHOROSHEV", + "Dmitry YURIEVICH", "Dmitrii Yuryevich KHOROSHEV", } - } - require.NotNil(t, sdn) + require.ElementsMatch(t, expectedAltNames, person.AltNames) + + require.Equal(t, search.GenderMale, person.Gender) + + expectedBirthDate := time.Date(1993, time.April, 17, 0, 0, 0, 0, time.UTC) + require.Equal(t, expectedBirthDate.Format(time.RFC3339), person.BirthDate.Format(time.RFC3339)) + require.Nil(t, person.DeathDate) + + require.Empty(t, person.Titles) + + expectedGovernmentIDs := []search.GovernmentID{ + { + Type: search.GovernmentIDPassport, + Country: "Russia", + Identifier: "2018278055", + }, + { + Type: search.GovernmentIDPassport, + Country: "Russia", + Identifier: "2006801524", + }, + { + Type: search.GovernmentIDTax, + Country: "Russia", + Identifier: "366110340670", + }, + } + require.ElementsMatch(t, expectedGovernmentIDs, person.GovernmentIDs) + + // "DOB 17 Apr 1993; POB Russian Federation; nationality Russia; citizen Russia; Email Address khoroshev1@icloud.com; + // alt. Email Address sitedev5@yandex.ru; Gender Male; Digital Currency Address - XBT bc1qvhnfknw852ephxyc5hm4q520zmvf9maphetc9z; + // Secondary sanctions risk: Ukraine-/Russia-Related Sanctions Regulations, 31 CFR 589.201; Passport 2018278055 (Russia); + // alt. Passport 2006801524 (Russia); Tax ID No. 366110340670 (Russia); a.k.a. 'LOCKBITSUPP'." + + expectedContact := search.ContactInfo{ + EmailAddresses: []string{"khoroshev1@icloud.com", "sitedev5@yandex.ru"}, + } + require.Equal(t, expectedContact, found.Contact) + require.Empty(t, found.Addresses) + + expectedCryptoAddresses := []search.CryptoAddress{ + {Currency: "XBT", Address: "bc1qvhnfknw852ephxyc5hm4q520zmvf9maphetc9z"}, + } + require.ElementsMatch(t, expectedCryptoAddresses, found.CryptoAddresses) + + require.Empty(t, found.Affiliations) + require.Nil(t, found.SanctionsInfo) + require.Empty(t, found.HistoricalInfo) + require.Empty(t, found.Titles) + + sdn, ok := found.SourceData.(ofac.SDN) + require.True(t, ok) + require.Equal(t, "48603", sdn.EntityID) + require.Equal(t, "KHOROSHEV, Dmitry Yuryevich", sdn.SDNName) + }) +} - e := ToEntity(*sdn, nil, nil, nil) - require.Equal(t, "MORENO, Daniel", e.Name) +func TestMapper__Person(t *testing.T) { + e := findOFACEntity(t, "15102") + + require.Equal(t, "Daniel MORENO", e.Name) require.Equal(t, search.EntityPerson, e.Type) require.Equal(t, search.SourceUSOFAC, e.Source) require.NotNil(t, e.Person) - require.Equal(t, "MORENO, Daniel", e.Person.Name) + require.Equal(t, "Daniel MORENO", e.Person.Name) require.Equal(t, "", string(e.Person.Gender)) require.Equal(t, "1972-10-12T00:00:00Z", e.Person.BirthDate.Format(time.RFC3339)) require.Nil(t, e.Person.DeathDate) @@ -44,34 +113,34 @@ func TestMapper__Person(t *testing.T) { require.Nil(t, e.Aircraft) require.Nil(t, e.Vessel) - sourceData, ok := e.SourceData.(SDN) + sourceData, ok := e.SourceData.(ofac.SDN) require.True(t, ok) require.Equal(t, "15102", sourceData.EntityID) } func TestMapper__CompletePerson(t *testing.T) { - sdn := &SDN{ + sdn := ofac.SDN{ EntityID: "26057", SDNName: "AL-ZAYDI, Shibl Muhsin 'Ubayd", SDNType: "individual", Remarks: "DOB 28 Oct 1968; POB Baghdad, Iraq; Additional Sanctions Information - Subject to Secondary Sanctions Pursuant to the Hizballah Financial Sanctions Regulations; alt. Additional Sanctions Information - Subject to Secondary Sanctions; Gender Male; a.k.a. 'SHIBL, Hajji'; nationality Iran; Passport A123456 (Iran) expires 2024; Driver's License No. 04900377 (Moldova) issued 02 Jul 2004; Email Address test@example.com; Phone: +1-123-456-7890; Fax: +1-123-456-7899", } - e := ToEntity(*sdn, nil, nil, nil) - require.Equal(t, "AL-ZAYDI, Shibl Muhsin 'Ubayd", e.Name) + e := ofac.ToEntity(sdn, nil, nil, nil) + require.Equal(t, "Shibl Muhsin 'Ubayd AL-ZAYDI", e.Name) require.Equal(t, search.EntityPerson, e.Type) require.Equal(t, search.SourceUSOFAC, e.Source) // Person specific fields require.NotNil(t, e.Person) - require.Equal(t, "AL-ZAYDI, Shibl Muhsin 'Ubayd", e.Person.Name) + require.Equal(t, "Shibl Muhsin 'Ubayd AL-ZAYDI", e.Person.Name) require.Equal(t, search.GenderMale, e.Person.Gender) require.Equal(t, "1968-10-28T00:00:00Z", e.Person.BirthDate.Format(time.RFC3339)) require.Nil(t, e.Person.DeathDate) // Test alt names require.Len(t, e.Person.AltNames, 1) - require.Equal(t, "SHIBL, Hajji", e.Person.AltNames[0]) + require.Equal(t, "Hajji SHIBL", e.Person.AltNames[0]) // Test government IDs require.Len(t, e.Person.GovernmentIDs, 2) @@ -98,44 +167,15 @@ func TestMapper__CompletePerson(t *testing.T) { require.Nil(t, e.Vessel) } -func TestParseAltNames(t *testing.T) { - tests := []struct { - remarks []string - expected []string - }{ - { - remarks: []string{"a.k.a. 'SMITH, John'"}, - expected: []string{"SMITH, John"}, - }, - { - remarks: []string{"a.k.a. 'SMITH, John'; a.k.a. 'DOE, Jane'"}, - expected: []string{"SMITH, John", "DOE, Jane"}, - }, - { - remarks: []string{"Some other remark", "a.k.a. 'SMITH, John'"}, - expected: []string{"SMITH, John"}, - }, - { - remarks: []string{}, - expected: nil, - }, - } - - for _, tt := range tests { - result := parseAltNames(tt.remarks) - require.Equal(t, tt.expected, result) - } -} - func TestMapper__CompletePersonWithRemarks(t *testing.T) { - sdn := &SDN{ + sdn := ofac.SDN{ EntityID: "26057", - SDNName: "AL-ZAYDI, Shibl Muhsin 'Ubayd", + SDNName: "Shibl Muhsin Ubayd al-Zaydi", SDNType: "individual", Remarks: "DOB 28 Oct 1968; POB Baghdad, Iraq; Gender Male; Title: Commander; Former Name: AL-ZAYDI, Muhammad; Linked To: ISLAMIC REVOLUTIONARY GUARD CORPS (IRGC)-QODS FORCE; Additional Sanctions Information - Subject to Secondary Sanctions", } - e := ToEntity(*sdn, nil, nil, nil) + e := ofac.ToEntity(sdn, nil, nil, nil) // Test affiliations require.Len(t, e.Affiliations, 1) @@ -157,7 +197,7 @@ func TestMapper__CompletePersonWithRemarks(t *testing.T) { } func TestMapper__PersonWithTitle(t *testing.T) { - sdn := &SDN{ + sdn := ofac.SDN{ EntityID: "12345", SDNName: "SMITH, John", SDNType: "individual", @@ -165,8 +205,8 @@ func TestMapper__PersonWithTitle(t *testing.T) { Remarks: "Title: Regional Director", } - e := ToEntity(*sdn, nil, nil, nil) - require.Equal(t, "SMITH, John", e.Name) + e := ofac.ToEntity(sdn, nil, nil, nil) + require.Equal(t, "John SMITH", e.Name) require.Equal(t, search.EntityPerson, e.Type) // Should have both titles - from SDN field and remarks diff --git a/pkg/ofac/reader.go b/pkg/ofac/reader.go index e31558e2..90868417 100644 --- a/pkg/ofac/reader.go +++ b/pkg/ofac/reader.go @@ -258,7 +258,7 @@ func splitPrograms(in string) []string { } func splitRemarks(input string) []string { - return strings.Split(input, ";") + return strings.Split(strings.TrimSuffix(input, "."), ";") } type remark struct { diff --git a/pkg/search/model_searched_entity_test.go b/pkg/search/model_searched_entity_test.go index e386bc42..4ec5fa1c 100644 --- a/pkg/search/model_searched_entity_test.go +++ b/pkg/search/model_searched_entity_test.go @@ -33,7 +33,12 @@ func TestSearchedEntityJSON(t *testing.T) { "organization": null, "aircraft": null, "vessel": null, - "contact": null, + "contact": { + "emailAddresses": null, + "phoneNumbers": null, + "faxNumbers": null, + "websites": null + }, "addresses": null, "cryptoAddresses": null, "affiliations": null, diff --git a/pkg/search/models.go b/pkg/search/models.go index 8bafa76b..f9cbfe9a 100644 --- a/pkg/search/models.go +++ b/pkg/search/models.go @@ -23,8 +23,8 @@ type Entity[T Value] struct { Aircraft *Aircraft `json:"aircraft"` Vessel *Vessel `json:"vessel"` - Contact []ContactInfo `json:"contact"` - Addresses []Address `json:"addresses"` + Contact ContactInfo `json:"contact"` + Addresses []Address `json:"addresses"` CryptoAddresses []CryptoAddress `json:"cryptoAddresses"` @@ -77,9 +77,10 @@ var ( ) type ContactInfo struct { - EmailAddresses []string - PhoneNumbers []string - FaxNumbers []string + EmailAddresses []string `json:"emailAddresses"` + PhoneNumbers []string `json:"phoneNumbers"` + FaxNumbers []string `json:"faxNumbers"` + Websites []string `json:"websites"` } // TODO(adam): diff --git a/pkg/search/models_test.go b/pkg/search/models_test.go index 09a517cc..d9150a1f 100644 --- a/pkg/search/models_test.go +++ b/pkg/search/models_test.go @@ -29,7 +29,12 @@ func TestEntityJSON(t *testing.T) { "organization": null, "aircraft": null, "vessel": null, - "contact": null, + "contact": { + "emailAddresses": null, + "phoneNumbers": null, + "faxNumbers": null, + "websites": null + }, "addresses": null, "cryptoAddresses": null, "affiliations": null, diff --git a/pkg/search/similarity.go b/pkg/search/similarity.go index 41cb7edf..5ce964db 100644 --- a/pkg/search/similarity.go +++ b/pkg/search/similarity.go @@ -5,1022 +5,446 @@ import ( "io" "math" "strings" - "time" +) - "github.com/moov-io/watchman/internal/stringscore" +const ( + // Score thresholds + exactMatchThreshold = 0.99 + highConfidenceThreshold = 0.95 + + // Weights for different categories + criticalIdWeight = 50.0 + nameWeight = 35.0 + supportingInfoWeight = 15.0 ) // Similarity calculates a match score between a query and an index entity. func Similarity[Q any, I any](query Entity[Q], index Entity[I]) float64 { - return DebugSimilarity[Q, I](nil, query, index) + return DebugSimilarity(nil, query, index) } +// DebugSimilarity does the same as Similarity, but logs debug info to w. func DebugSimilarity[Q any, I any](w io.Writer, query Entity[Q], index Entity[I]) float64 { - pieces := make([]scorePiece, 0) + if w == nil { + w = io.Discard // TODO(adam): remove + } + fmt.Fprintf(w, "\n=== Starting Similarity Comparison ===\n") - // Primary identifiers (IMO number, Call Sign, etc.) - highest weight - exactMatchWeight := 50.0 - pieces = append(pieces, compareExactIdentifiers(w, query, index, exactMatchWeight)) + // Log the business data we're comparing + if query.Business != nil && index.Business != nil { + fmt.Fprintf(w, "\nBusiness Details:\n") + fmt.Fprintf(w, "Query Name: %q\n", query.Business.Name) + fmt.Fprintf(w, "Index Name: %q\n", index.Business.Name) + fmt.Fprintf(w, "Query Identifiers: %+v\n", query.Business.Identifier) + fmt.Fprintf(w, "Index Identifiers: %+v\n", index.Business.Identifier) + } - // Name match is critical - nameWeight := 30.0 - pieces = append(pieces, compareName(w, query, index, nameWeight)) + var pieces []scorePiece - // Entity-specific comparisons (type, flag, etc) - entityWeight := 15.0 - pieces = append(pieces, compareEntitySpecific(w, query, index, entityWeight)) + // Critical identifiers (highest weight) + exactIdentifiers := compareExactIdentifiers(w, query, index, criticalIdWeight) + if exactIdentifiers.matched && exactIdentifiers.fieldsCompared > 0 { + return exactIdentifiers.score + } + exactCryptoAddresses := compareExactCryptoAddresses(w, query, index, criticalIdWeight) + if exactCryptoAddresses.matched && exactCryptoAddresses.fieldsCompared > 0 { + return exactCryptoAddresses.score + } + exactGovernmentIDs := compareExactGovernmentIDs(w, query, index, criticalIdWeight) + if exactGovernmentIDs.matched && exactGovernmentIDs.fieldsCompared > 0 { + return exactGovernmentIDs.score + } + pieces = append(pieces, exactIdentifiers, exactCryptoAddresses, exactGovernmentIDs) + fmt.Println("Critical pieces") + fmt.Printf("exact identifiers: %#v\n", pieces[0]) + fmt.Printf("crypto addresses: %#v\n", pieces[1]) + fmt.Printf("gov IDs: %#v\n", pieces[2]) - // Supporting information (addresses, sanctions, etc) - supportingWeight := 5.0 - pieces = append(pieces, compareSupportingInfo(w, query, index, supportingWeight)) + // Name comparison (second highest weight) + pieces = append(pieces, + compareName(w, query, index, nameWeight), + compareEntityTitlesFuzzy(w, query, index, nameWeight), + ) + fmt.Println("name comparison") + fmt.Printf("name: %#v\n", pieces[3]) + fmt.Printf("titles: %#v\n", pieces[4]) + + // Supporting information (lower weight) + pieces = append(pieces, + compareEntityDates(w, query, index, supportingInfoWeight), + compareAddresses(w, query, index, supportingInfoWeight), + compareSupportingInfo(w, query, index, supportingInfoWeight), + ) + fmt.Println("supporting info") + fmt.Printf("dates: %#v\n", pieces[5]) + fmt.Printf("addresses: %#v\n", pieces[6]) + fmt.Printf("supporting into: %#v\n", pieces[7]) - // Compute final score with coverage logic - return calculateFinalScore(w, pieces, index) + finalScore := calculateFinalScore(w, pieces, query, index) + fmt.Printf("final score: %.2f\n", finalScore) + return finalScore } // scorePiece is a partial scoring result from one comparison function type scorePiece struct { - score float64 // 0-1 score for this piece - weight float64 // Weight for this piece - matched bool // Whether there was a "match" - required bool // Whether this piece is "required" for a high overall score - exact bool // Whether this was an exact match - fieldsCompared int // Number of fields actually compared in this piece - pieceType string // e.g. "name", "entity", "identifiers", etc. + score float64 // 0-1 for this piece + weight float64 // weight for final + matched bool // whether there's a "match" + required bool // if this piece is "required" for a high overall score + exact bool // whether it's an exact match + fieldsCompared int // how many fields were actually compared + pieceType string // e.g. "identifiers", "name", etc. } -func compareExactIdentifiers[Q any, I any](w io.Writer, query Entity[Q], index Entity[I], weight float64) scorePiece { - matches := 0 - totalWeight := 0.0 - score := 0.0 - fieldsCompared := 0 - hasMatch := false - - // For Vessel example: - if query.Vessel != nil && index.Vessel != nil { - // IMO Number - if query.Vessel.IMONumber != "" { - fieldsCompared++ - if index.Vessel.IMONumber != "" && strings.EqualFold(query.Vessel.IMONumber, index.Vessel.IMONumber) { - matches++ - hasMatch = true - score += 10.0 - totalWeight += 10.0 - } - } - // Call Sign - if query.Vessel.CallSign != "" { - fieldsCompared++ - if index.Vessel.CallSign != "" && strings.EqualFold(query.Vessel.CallSign, index.Vessel.CallSign) { - matches++ - hasMatch = true - score += 8.0 - totalWeight += 8.0 - } - } - // MMSI - if query.Vessel.MMSI != "" { - fieldsCompared++ - if index.Vessel.MMSI != "" && strings.EqualFold(query.Vessel.MMSI, index.Vessel.MMSI) { - matches++ - hasMatch = true - score += 6.0 - totalWeight += 6.0 - } - } - } - - // Similar logic for Person, Aircraft, etc. as needed... - - finalScore := 0.0 - if totalWeight > 0 { - finalScore = score / totalWeight - } - // Penalty if we compared but found no matches - if fieldsCompared > 0 && matches == 0 { - finalScore = 0.1 - } - - return scorePiece{ - score: finalScore, - weight: weight, - matched: hasMatch, - required: fieldsCompared > 0, - exact: finalScore > 0.95, - fieldsCompared: fieldsCompared, - pieceType: "identifiers", +func boolToScore(b bool) float64 { + if b { + return 1.0 } + return 0.0 } -func compareName[Q any, I any](w io.Writer, query Entity[Q], index Entity[I], weight float64) scorePiece { - qName := strings.TrimSpace(strings.ToLower(query.Name)) - iName := strings.TrimSpace(strings.ToLower(index.Name)) - - // If the query name is empty, skip - if qName == "" { - return scorePiece{score: 0, weight: 0, fieldsCompared: 0, pieceType: "name"} - } - - // Exact match - if qName == iName { - return scorePiece{ - score: 1.0, - weight: weight, - matched: true, - required: true, - exact: true, - fieldsCompared: 1, - pieceType: "name", - } - } - - // Fuzzy match - qTerms := strings.Fields(qName) - bestScore := stringscore.BestPairsJaroWinkler(qTerms, iName) - - // Check alternate names if both are Person entities - if query.Person != nil && index.Person != nil { - for _, altName := range index.Person.AltNames { - altScore := stringscore.BestPairsJaroWinkler(qTerms, strings.ToLower(altName)) - if altScore > bestScore { - bestScore = altScore - } - } - } - // Check historical info for "Former Name" - for _, hist := range index.HistoricalInfo { - if strings.EqualFold(hist.Type, "Former Name") { - histScore := stringscore.BestPairsJaroWinkler(qTerms, strings.ToLower(hist.Value)) - if histScore > bestScore { - bestScore = histScore - } - } +func calculateAverage(scores []float64) float64 { + if len(scores) == 0 { + return 0 } - - return scorePiece{ - score: bestScore, - weight: weight, - matched: bestScore > 0.8, - required: true, - exact: bestScore > 0.99, - fieldsCompared: 1, - pieceType: "name", + var sum float64 + for _, score := range scores { + sum += score } + return sum / float64(len(scores)) } -func compareEntitySpecific[Q any, I any](w io.Writer, query Entity[Q], index Entity[I], weight float64) scorePiece { - // If types don't match, it's an immediate 0 - if query.Type != index.Type { - return scorePiece{ - score: 0, - weight: weight, - pieceType: "entity", - fieldsCompared: 1, - } - } - - var typeScore float64 - var matched bool - var fieldsCompared int - - switch query.Type { - case EntityVessel: - typeScore, matched, fieldsCompared = compareVesselFields(w, query.Vessel, index.Vessel) - case EntityPerson: - typeScore, matched, fieldsCompared = comparePersonFields(w, query.Person, index.Person) - case EntityAircraft: - typeScore, matched, fieldsCompared = compareAircraftFields(w, query.Aircraft, index.Aircraft) - case EntityBusiness: - typeScore, matched, fieldsCompared = compareBusinessFields(w, query.Business, index.Business) - case EntityOrganization: - typeScore, matched, fieldsCompared = compareOrganizationFields(w, query.Organization, index.Organization) - } - debug(w, "compareEntitySpecific\n ") - debug(w, "%v typeScore=%.4f matched=%v fieldsCompared=%v\n", query.Type, typeScore, matched, fieldsCompared) - - return scorePiece{ - score: typeScore, - weight: weight, - matched: matched, - required: fieldsCompared > 0, - exact: typeScore > 0.95, - fieldsCompared: fieldsCompared + 1, // +1 for the type comparison - pieceType: "entity", +// debug prints if w is non-nil +func debug(w io.Writer, pattern string, args ...any) { + if w != nil { + fmt.Fprintf(w, pattern, args...) } } const ( - day = 24 * time.Hour + // Score thresholds + typeMismatchScore = 0.667 + criticalCovThreshold = 0.7 + minCoverageThreshold = 0.28 // how many fields did the query compare the index against? + perfectMatchBoost = 1.15 + criticalFieldMultiplier = 1.2 + + // Minimum field requirements by entity type + minPersonFields = 3 // e.g., name, DOB, gender + minBusinessFields = 3 // e.g., name, identifier, creation date + minOrgFields = 3 // e.g., name, identifier, creation date + minVesselFields = 3 // e.g., IMO, name, flag + minAircraftFields = 3 // e.g., serial number, model, flag ) -// ------------------------------- -// Person-Specific Fields -// ------------------------------- -func comparePersonFields(w io.Writer, query *Person, index *Person) (float64, bool, int) { - if query == nil || index == nil { - return 0, false, 0 +// entityFields tracks required and available fields for an entity +type entityFields struct { + required int + available int + hasName bool + hasID bool + hasCritical bool +} + +func calculateFinalScore[Q any, I any](w io.Writer, pieces []scorePiece, query Entity[Q], index Entity[I]) float64 { + if len(pieces) == 0 || query.Type != index.Type { + return 0 } - scores := make([]float64, 0) - fieldsCompared := 0 + // Get field counts and critical field information + fields := countFieldsByImportance(pieces) + coverage := calculateCoverage(pieces, index) + fmt.Printf("calculateFinalScore: fields=%#v coverage=%#v ", fields, coverage) - // Birthdate - if query.BirthDate != nil && index.BirthDate != nil { - fieldsCompared++ + // Calculate base score with weighted importance + baseScore := calculateBaseScore(pieces, fields) + fmt.Printf(" baseScore=%v ", baseScore) - qb := query.BirthDate.Truncate(day) - ib := index.BirthDate.Truncate(day) + // Apply coverage penalties + finalScore := applyPenaltiesAndBonuses(baseScore, coverage, fields, query.Type == index.Type) + fmt.Printf(" finalScore=%.2f\n", finalScore) - if qb.Equal(ib) { - scores = append(scores, 1.0) - } else { - scores = append(scores, 0.0) - } - } + return finalScore +} + +func countFieldsByImportance(pieces []scorePiece) entityFields { + var fields entityFields - // Gender - if query.Gender != "" { - fieldsCompared++ - if strings.EqualFold(string(query.Gender), string(index.Gender)) { - scores = append(scores, 1.0) - } else { - scores = append(scores, 0.0) + for _, piece := range pieces { + if piece.weight <= 0 || piece.fieldsCompared == 0 { + continue } - } - // Titles - if len(query.Titles) > 0 && len(index.Titles) > 0 { - fieldsCompared++ - matches := 0 - for _, qTitle := range query.Titles { - for _, iTitle := range index.Titles { - if strings.EqualFold(qTitle, iTitle) { - matches++ - break - } + if piece.required { + fields.required += piece.fieldsCompared + } + if piece.matched { + if piece.pieceType == "name" { + fields.hasName = true + } + if piece.exact && (piece.pieceType == "identifiers" || piece.pieceType == "gov-ids-exact") { + fields.hasID = true + } + if piece.exact { + fields.hasCritical = true } } - scores = append(scores, float64(matches)/float64(len(query.Titles))) } - if len(scores) == 0 { - return 0, false, fieldsCompared - } - - sum := 0.0 - for _, s := range scores { - sum += s - } - avg := sum / float64(len(scores)) - - return avg, avg > 0.9, fieldsCompared + return fields } -// ------------------------------- -// Vessel-Specific Fields -// ------------------------------- -func compareVesselFields(w io.Writer, query *Vessel, index *Vessel) (float64, bool, int) { - if query == nil || index == nil { - return 0, false, 0 - } - - type fieldScore struct { - score float64 - weight float64 - } - var ( - scores []fieldScore - fieldsCompared int - ) +func calculateBaseScore(pieces []scorePiece, fields entityFields) float64 { + var totalScore, totalWeight float64 - // Compare only fields present in query - if query.CallSign != "" { - fieldsCompared++ - if strings.EqualFold(query.CallSign, index.CallSign) { - scores = append(scores, fieldScore{1.0, 4.0}) - } else { - scores = append(scores, fieldScore{0.0, 4.0}) + for _, piece := range pieces { + if piece.weight <= 0 || piece.fieldsCompared == 0 { + continue } - } - if query.IMONumber != "" && index.IMONumber != "" { - fieldsCompared++ - if strings.EqualFold(query.IMONumber, index.IMONumber) { - scores = append(scores, fieldScore{1.0, 4.0}) - } else { - scores = append(scores, fieldScore{0.0, 4.0}) + // Apply importance multiplier for critical fields + multiplier := 1.0 + if piece.required { + multiplier = criticalFieldMultiplier } - } - - if query.Owner != "" { - fieldsCompared++ - ownerTerms := strings.Fields(strings.ToLower(query.Owner)) - ownerScore := stringscore.BestPairsJaroWinkler(ownerTerms, strings.ToLower(index.Owner)) - scores = append(scores, fieldScore{ownerScore, 2.0}) - } - if query.Flag != "" { - fieldsCompared++ - if strings.EqualFold(query.Flag, index.Flag) { - scores = append(scores, fieldScore{1.0, 1.5}) - } else { - scores = append(scores, fieldScore{0.0, 1.5}) - } + totalScore += piece.score * piece.weight * multiplier + totalWeight += piece.weight * multiplier } - if query.Type != "" { - fieldsCompared++ - if strings.EqualFold(string(query.Type), string(index.Type)) { - scores = append(scores, fieldScore{1.0, 1.0}) - } else { - scores = append(scores, fieldScore{0.0, 1.0}) - } + if totalWeight == 0 { + return 0 } - if query.Tonnage > 0 && index.Tonnage > 0 { - fieldsCompared++ - diff := math.Abs(float64(query.Tonnage - index.Tonnage)) - s := vesselTonnageScore(diff) - scores = append(scores, fieldScore{s, 1.0}) - } + return totalScore / totalWeight +} - if query.GrossRegisteredTonnage > 0 && index.GrossRegisteredTonnage > 0 { - fieldsCompared++ - diff := math.Abs(float64(query.GrossRegisteredTonnage - index.GrossRegisteredTonnage)) - s := vesselTonnageScore(diff) - scores = append(scores, fieldScore{s, 1.0}) +func calculateCoverage[I any](pieces []scorePiece, index Entity[I]) coverage { + indexFields := countAvailableFields(index) + if indexFields == 0 { + return coverage{ratio: 1.0, criticalRatio: 1.0} } - if len(scores) == 0 { - return 0, false, fieldsCompared - } + var fieldsCompared, criticalFieldsCompared int + var criticalTotal int - var totalScore, totalWeight float64 - for _, fs := range scores { - totalScore += fs.score * fs.weight - totalWeight += fs.weight + for _, p := range pieces { + fieldsCompared += p.fieldsCompared + if p.required { + criticalFieldsCompared += p.fieldsCompared + criticalTotal += p.fieldsCompared + } } - avgScore := totalScore / totalWeight - return avgScore, avgScore > 0.9, fieldsCompared -} + fmt.Printf("calculateCoverage: fieldsCompared=%v indexFields=%v criticalFieldsCompared=%v criticalTotal=%v\n", + fieldsCompared, indexFields, criticalFieldsCompared, criticalTotal) -// Helper for vessel tonnage diffs -func vesselTonnageScore(diff float64) float64 { - switch { - case diff == 0: - return 1.0 - case diff < 100: - return 0.8 - case diff < 500: - return 0.5 - default: - return 0.0 + return coverage{ + ratio: float64(fieldsCompared) / float64(indexFields), + criticalRatio: float64(criticalFieldsCompared) / float64(criticalTotal), } } -// ------------------------------- -// Aircraft-Specific Fields -// ------------------------------- -func compareAircraftFields(w io.Writer, query *Aircraft, index *Aircraft) (float64, bool, int) { - if query == nil || index == nil { - return 0, false, 0 - } +type coverage struct { + ratio float64 + criticalRatio float64 +} - var scores []float64 - fieldsCompared := 0 +func applyPenaltiesAndBonuses(baseScore float64, cov coverage, fields entityFields, sameType bool) float64 { + score := baseScore - debug(w, "compareAircraftFields\n ") + fmt.Printf("cov: %#v\n", cov) - if query.ICAOCode != "" { - fieldsCompared++ - if strings.EqualFold(query.ICAOCode, index.ICAOCode) { - scores = append(scores, 1.0) - } else { - scores = append(scores, 0.0) - } - debug(w, " .ICAOCode") + // Apply coverage penalties + if cov.ratio < minCoverageThreshold { + score *= 0.98 // Significant penalty for low overall coverage } - - if query.Model != "" { - fieldsCompared++ - if strings.EqualFold(query.Model, index.Model) { - scores = append(scores, 1.0) - } else { - // fuzzy - qTerms := strings.Fields(strings.ToLower(query.Model)) - modelScore := stringscore.BestPairsJaroWinkler(qTerms, strings.ToLower(index.Model)) - scores = append(scores, modelScore) - } - debug(w, " .Model") + if cov.criticalRatio < criticalCovThreshold { + score *= 0.95 // Penalty for missing critical fields } - if query.Flag != "" { - fieldsCompared++ - if strings.EqualFold(query.Flag, index.Flag) { - scores = append(scores, 1.0) - } else { - scores = append(scores, 0.0) - } - debug(w, " .Flag") + // Apply perfect match bonus + if fields.hasName && fields.hasID && fields.hasCritical && cov.ratio > 0.95 && score > highConfidenceThreshold { + score = math.Min(1.0, score*perfectMatchBoost) } - if len(scores) == 0 { - return 0, false, fieldsCompared + // Handle type mismatches + if !sameType { + score = math.Min(score, typeMismatchScore) } - debug(w, " (Scores: %v)", scores) - - sum := 0.0 - for _, s := range scores { - sum += s - } + return score +} - debug(w, " [totalScore=%v ", sum) +func countAvailableFields[I any](index Entity[I]) int { + var count int - avg := sum / float64(len(scores)) + // Count type-specific fields + switch index.Type { + case EntityPerson: + count = countPersonFields(index.Person) + case EntityBusiness: + count = countBusinessFields(index.Business) + case EntityOrganization: + count = countOrganizationFields(index.Organization) + case EntityVessel: + count = countVesselFields(index.Vessel) + case EntityAircraft: + count = countAircraftFields(index.Aircraft) + } - debug(w, "avgScore=%.4f fieldsCompared=%v]\n", avg, fieldsCompared) + // Count common fields + count += countCommonFields(index) - return avg, avg > 0.5, fieldsCompared + return count } -// ------------------------------- -// Business-Specific Fields -// ------------------------------- -// compareBusinessFields compares fields for the Business entity -func compareBusinessFields(w io.Writer, query *Business, index *Business) (float64, bool, int) { - if query == nil || index == nil { - return 0, false, 0 - } +func countCommonFields[I any](index Entity[I]) int { + count := 0 - // We'll collect sub-scores with weights - type fieldScore struct { - score float64 - weight float64 + if strings.TrimSpace(index.Name) != "" { + count++ } - var scores []fieldScore - fieldsCompared := 0 - - debug(w, "compareBusinessFields\n ") - - // 1) Primary Name check (fuzzy or exact) - if query.Name != "" { - fieldsCompared++ - if strings.EqualFold(query.Name, index.Name) { - // exact match - scores = append(scores, fieldScore{score: 1.0, weight: 4.0}) - } else { - // fuzzy match - qTerms := strings.Fields(strings.ToLower(query.Name)) - iName := strings.ToLower(index.Name) - nameScore := stringscore.BestPairsJaroWinkler(qTerms, iName) - scores = append(scores, fieldScore{score: nameScore, weight: 4.0}) - } - debug(w, " .Name") + if index.Source != "" { + count++ } - - // 2) AltNames check - // If the query has alt names, let's see if any overlap. Or, if the index has alt names, - // we can see if the query.Name matches them. - // Typically you'd do something like your `Person.AltNames` logic. For simplicity: - if len(query.AltNames) > 0 && len(index.AltNames) > 0 { - fieldsCompared++ - bestAltScore := 0.0 - for _, qAlt := range query.AltNames { - for _, iAlt := range index.AltNames { - altScore := stringscore.BestPairsJaroWinkler( - strings.Fields(strings.ToLower(qAlt)), - strings.ToLower(iAlt), - ) - if altScore > bestAltScore { - bestAltScore = altScore - } - } - } - // Weight alt names a bit lower than primary name - scores = append(scores, fieldScore{score: bestAltScore, weight: 2.0}) - - debug(w, " .AltName") + if len(index.Contact.EmailAddresses) > 0 { + count++ } - - // 3) Created date - if query.Created != nil && index.Created != nil { - fieldsCompared++ - if query.Created.Equal(*index.Created) { - scores = append(scores, fieldScore{score: 1.0, weight: 1.0}) - } else { - // partial credit if close - diffDays := math.Abs(query.Created.Sub(*index.Created).Hours() / 24) - switch { - case diffDays <= 1: - scores = append(scores, fieldScore{score: 0.9, weight: 1.0}) - case diffDays <= 7: - scores = append(scores, fieldScore{score: 0.7, weight: 1.0}) - default: - scores = append(scores, fieldScore{score: 0.0, weight: 1.0}) - } - } - - debug(w, " .Created") + if len(index.Contact.PhoneNumbers) > 0 { + count++ } - - // 4) Dissolved date - if query.Dissolved != nil && index.Dissolved != nil { - fieldsCompared++ - if query.Dissolved.Equal(*index.Dissolved) { - scores = append(scores, fieldScore{score: 1.0, weight: 1.0}) - } else { - // partial logic if you want to consider near-dates as partial matches - diffDays := math.Abs(query.Dissolved.Sub(*index.Dissolved).Hours() / 24) - switch { - case diffDays <= 1: - scores = append(scores, fieldScore{score: 0.9, weight: 1.0}) - case diffDays <= 7: - scores = append(scores, fieldScore{score: 0.7, weight: 1.0}) - default: - scores = append(scores, fieldScore{score: 0.0, weight: 1.0}) - } - } - - debug(w, " .Dissolved") + if len(index.Contact.FaxNumbers) > 0 { + count++ } - - // 5) Identifiers - // If you have multiple IDs in each, you might do a best match approach. - // For each query.Identifier, find best match in index.Identifier. - // Possibly weigh "Tax ID" or other critical IDs more heavily. - if len(query.Identifier) > 0 && len(index.Identifier) > 0 { - fieldsCompared++ - bestIDScore := 0.0 - for _, qID := range query.Identifier { - for _, iID := range index.Identifier { - // Example logic: exact match of "Identifier" + Country -> 1.0 - // partial or mismatch -> 0. - // Could also do fuzzy or partial logic for qID.Identifier vs. iID.Identifier - if strings.EqualFold(qID.Identifier, iID.Identifier) && - strings.EqualFold(qID.Country, iID.Country) && - strings.EqualFold(qID.Name, iID.Name) { - // perfect - bestIDScore = 1.0 - break - } else if strings.EqualFold(qID.Identifier, iID.Identifier) { - // partial - if bestIDScore < 0.8 { - bestIDScore = 0.8 - } - } else { - // could do fuzzy on qID.Identifier vs. iID.Identifier if you want - } - } - if bestIDScore == 1.0 { - break - } - } - // Weight ID matches strongly - scores = append(scores, fieldScore{score: bestIDScore, weight: 5.0}) - - debug(w, " .Identifier") + if len(index.CryptoAddresses) > 0 { + count++ } - - if len(scores) == 0 { - return 0, false, fieldsCompared + if len(index.Affiliations) > 0 { + count++ } - - // Weighted average - var totalScore, totalWeight float64 - for _, fs := range scores { - totalScore += fs.score * fs.weight - totalWeight += fs.weight + if len(index.Titles) > 0 { + count++ + } + if len(index.Addresses) > 0 { + count++ } - avgScore := totalScore / totalWeight - - debug(w, " (Scores: %v)", scores) - debug(w, " [totalScore=%v totalWeight=%v avgScore=%.4f fieldsCompared=%v]\n", totalScore, totalWeight, avgScore, fieldsCompared) - // We'll say it's "matched" if > 0.5 on average / - return avgScore, avgScore > 0.9, fieldsCompared + return count } -// ------------------------------- -// Organization-Specific Fields -// ------------------------------- -func compareOrganizationFields(w io.Writer, query *Organization, index *Organization) (float64, bool, int) { - if query == nil || index == nil { - return 0, false, 0 +func countPersonFields(p *Person) int { + if p == nil { + return 0 } - fieldsCompared := 0 - scores := make([]float64, 0) - - debug(w, "compareOrganizationFields\n ") - - // Created date - if query.Created != nil && index.Created != nil { - fieldsCompared++ - if query.Created.Equal(*index.Created) { - scores = append(scores, 1.0) - } else { - diff := math.Abs(query.Created.Sub(*index.Created).Hours() / 24) - switch { - case diff <= 1: - scores = append(scores, 0.9) - case diff <= 7: - scores = append(scores, 0.7) - default: - scores = append(scores, 0.0) - } - } - debug(w, " .Created") + count := 0 + if p.BirthDate != nil { + count++ } - - if len(scores) == 0 { - return 0, false, fieldsCompared + if p.Gender != "" { + count++ } - - debug(w, " (Scores: %v)", scores) - - sum := 0.0 - for _, s := range scores { - sum += s + if len(p.Titles) > 0 { + count++ + } + if len(p.GovernmentIDs) > 0 { + count++ } - debug(w, " [totalScore=%v ", sum) - - avg := sum / float64(len(scores)) - - debug(w, "avgScore=%.4f fieldsCompared=%v]\n", avg, fieldsCompared) - - return avg, avg > 0.9, fieldsCompared + return count } -// ------------------------------- -// Supporting Info (addresses, etc.) -// ------------------------------- -func compareSupportingInfo[Q any, I any](w io.Writer, query Entity[Q], index Entity[I], weight float64) scorePiece { - var pieces []float64 - fieldsCompared := 0 - - debug(w, "compareSupportingInfo\n ") - - // Compare addresses - if len(query.Addresses) > 0 && len(index.Addresses) > 0 { - bestAddress := 0.0 - fieldsCompared++ - for _, qAddr := range query.Addresses { - for _, iAddr := range index.Addresses { - addrScore := compareAddress(w, qAddr, iAddr) - if addrScore > bestAddress { - bestAddress = addrScore - } - } - } - pieces = append(pieces, bestAddress) - debug(w, " .Addresses") +func countBusinessFields(b *Business) int { + if b == nil { + return 0 } - // Compare sanctions programs - if query.SanctionsInfo != nil && index.SanctionsInfo != nil { - fieldsCompared++ - programScore := compareSanctionsPrograms(w, query.SanctionsInfo, index.SanctionsInfo) - pieces = append(pieces, programScore) - debug(w, " .SanctionsInfo") + count := 0 + if strings.TrimSpace(b.Name) != "" { + count++ } - - // Compare crypto addresses (exact matches only) - if len(query.CryptoAddresses) > 0 && len(index.CryptoAddresses) > 0 { - fieldsCompared++ - matches := 0 - for _, qCA := range query.CryptoAddresses { - for _, iCA := range index.CryptoAddresses { - if strings.EqualFold(qCA.Currency, iCA.Currency) && - strings.EqualFold(qCA.Address, iCA.Address) { - matches++ - } - } - } - score := float64(matches) / float64(len(query.CryptoAddresses)) - pieces = append(pieces, score) - debug(w, " .CryptoAddresses") + if len(b.AltNames) > 0 { + count++ } - - if len(pieces) == 0 { - return scorePiece{score: 0, weight: 0, fieldsCompared: 0, pieceType: "supporting"} + if b.Created != nil { + count++ } - - debug(w, " (Scores: %v)", pieces) - - // Average of these pieces - sum := 0.0 - for _, s := range pieces { - sum += s + if len(b.Identifier) > 0 { + count++ } - debug(w, " [totalScore=%v ", sum) - - avgScore := sum / float64(len(pieces)) - debug(w, "avgScore=%.4f fieldsCompared=%v]\n", avgScore, fieldsCompared) - - return scorePiece{ - score: avgScore, - weight: weight, - matched: avgScore > 0.5, - required: false, - exact: avgScore > 0.99, - fieldsCompared: fieldsCompared, - pieceType: "supporting", - } + return count } -// ------------------------------- -// Address comparison -// ------------------------------- -func compareAddress(w io.Writer, query Address, index Address) float64 { - var ( - pieces []float64 - weights []float64 - ) - - debug(w, "compareAddress\n ") - - // Line1 - if query.Line1 != "" { - qTerms := strings.Fields(query.Line1) - score := stringscore.BestPairsJaroWinkler(qTerms, index.Line1) - pieces = append(pieces, score) - weights = append(weights, 3.0) - debug(w, ".Line1") - } - // Line2 - if query.Line2 != "" { - qTerms := strings.Fields(query.Line2) - score := stringscore.BestPairsJaroWinkler(qTerms, index.Line2) - pieces = append(pieces, score) - weights = append(weights, 1.0) - debug(w, ".Line2") - } - // City - if query.City != "" { - qTerms := strings.Fields(query.City) - score := stringscore.BestPairsJaroWinkler(qTerms, index.City) - pieces = append(pieces, score) - weights = append(weights, 2.0) - debug(w, ".City") - } - // State (exact) - if query.State != "" { - if strings.EqualFold(query.State, index.State) { - pieces = append(pieces, 1.0) - } else { - pieces = append(pieces, 0.0) - } - weights = append(weights, 1.0) - debug(w, ".State") +func countOrganizationFields(o *Organization) int { + if o == nil { + return 0 } - // Postal code (exact) - if query.PostalCode != "" { - if strings.EqualFold(query.PostalCode, index.PostalCode) { - pieces = append(pieces, 1.0) - } else { - pieces = append(pieces, 0.0) - } - weights = append(weights, 1.5) - debug(w, ".PosalCode") + + count := 0 + if strings.TrimSpace(o.Name) != "" { + count++ } - // Country (exact) - if query.Country != "" { - if strings.EqualFold(query.Country, index.Country) { - pieces = append(pieces, 1.0) - } else { - pieces = append(pieces, 0.0) - } - weights = append(weights, 2.0) - debug(w, ".Country") + if len(o.AltNames) > 0 { + count++ } - - if len(pieces) == 0 { - return 0 + if o.Created != nil { + count++ } - - debug(w, " (Scores: %v)", pieces) - - var totalScore, totalWeight float64 - for i := range pieces { - totalScore += pieces[i] * weights[i] - totalWeight += weights[i] + if len(o.Identifier) > 0 { + count++ } - debug(w, " [totalScore=%v totalWeight=%v]\n", totalScore, totalWeight) - return totalScore / totalWeight + return count } -func compareSanctionsPrograms(w io.Writer, query *SanctionsInfo, index *SanctionsInfo) float64 { - if query == nil || index == nil { - return 0 - } - if len(query.Programs) == 0 { +func countVesselFields(v *Vessel) int { + if v == nil { return 0 } - matches := 0 - for _, qProgram := range query.Programs { - for _, iProgram := range index.Programs { - if strings.EqualFold(qProgram, iProgram) { - matches++ - break - } - } + count := 0 + if v.IMONumber != "" { + count++ } - - score := float64(matches) / float64(len(query.Programs)) - - // Adjust for mismatch on "secondary" - if query.Secondary != index.Secondary { - score *= 0.8 + if v.CallSign != "" { + count++ } - return score -} - -// ------------------------------- -// Coverage Logic -// ------------------------------- - -// countIndexUniqueFields only counts fields relevant to the entity's type. -// This prevents penalizing a Person for Vessel fields, etc. -func countIndexUniqueFields[I any](index Entity[I]) int { - count := 0 - - switch index.Type { - case EntityVessel: - if index.Vessel != nil { - if index.Vessel.IMONumber != "" { - count++ - } - if index.Vessel.CallSign != "" { - count++ - } - if index.Vessel.MMSI != "" { - count++ - } - if index.Vessel.Owner != "" { - count++ - } - // If you want to treat name as part of "unique fields," do that outside or here - } - case EntityPerson: - if index.Person != nil { - if index.Person.BirthDate != nil { - count++ - } - if index.Person.Gender != "" { - count++ - } - if len(index.Person.Titles) > 0 { - count++ - } - } - case EntityAircraft: - if index.Aircraft != nil { - if index.Aircraft.ICAOCode != "" { - count++ - } - if index.Aircraft.Model != "" { - count++ - } - if index.Aircraft.Flag != "" { - count++ - } - } - case EntityBusiness: - if index.Business != nil { - // If there's a Name - if strings.TrimSpace(index.Business.Name) != "" { - count++ - } - // If there's at least one alt name - if len(index.Business.AltNames) > 0 { - count++ - } - // If there's a created date - if index.Business.Created != nil { - count++ - } - // If there's a dissolved date - if index.Business.Dissolved != nil { - count++ - } - // If there's at least one Identifier - if len(index.Business.Identifier) > 0 { - count++ - } - } - case EntityOrganization: - if index.Organization != nil { - if index.Organization.Created != nil { - count++ - } - } + if v.MMSI != "" { + count++ } - - // Regardless of type, if there's a non-blank Name, count it - if strings.TrimSpace(index.Name) != "" { + if v.Flag != "" { + count++ + } + if v.Model != "" { + count++ + } + if v.Owner != "" { count++ } - - // You could also count addresses, sanctions, etc. if relevant: - // if len(index.Addresses) > 0 { count++ } - // if index.SanctionsInfo != nil { count++ } - // etc. return count } -// calculateFinalScore applies coverage logic and final adjustments. -func calculateFinalScore[I any](w io.Writer, pieces []scorePiece, index Entity[I]) float64 { - if len(pieces) == 0 { +func countAircraftFields(a *Aircraft) int { + if a == nil { return 0 } - var ( - totalScore float64 - totalWeight float64 - hasExactMatch bool - hasNameMatch bool - ) - - debug(w, "\ncalculateFinalScore\n") - - // Sum up the piece scores - for _, piece := range pieces { - debug(w, "%#v\n", piece) - - // Skip zero-weight pieces entirely - if piece.weight <= 0 { - continue - } - - // If "entity" piece has score=0 but fieldsCompared=1, that indicates a type mismatch => overall 0 - // if piece.pieceType == "entity" && piece.fieldsCompared == 1 && piece.score == 0 { - // debug(w, "entity - mismatch") - // return 0 - // } - - // Only accumulate if we actually compared some fields - if piece.fieldsCompared > 0 { - totalScore += piece.score * piece.weight - totalWeight += piece.weight - - if piece.exact { - hasExactMatch = true - } - // If the piece is "required" and "matched," track if it's the name - if piece.required && piece.matched && piece.pieceType == "name" { - hasNameMatch = true - } - } - } - - if totalWeight == 0 { - return 0 + count := 0 + if a.ICAOCode != "" { + count++ } - - baseScore := totalScore / totalWeight - debug(w, "baseScore=%.4f ", baseScore) - - // Coverage check: only count fields relevant to the index type - coveragePenalty := 1.0 - indexUniqueCount := countIndexUniqueFields(index) - fieldsCompared := 0 - for _, p := range pieces { - fieldsCompared += p.fieldsCompared + if a.Model != "" { + count++ } - debug(w, "fieldsCompared=%d ", fieldsCompared) - - if indexUniqueCount > 0 { - coverage := float64(fieldsCompared) / float64(indexUniqueCount) - - // If coverage is very low (< 0.5) but the base score is high, reduce a bit - if coverage < 0.5 && baseScore > 0.6 { - coveragePenalty = 0.9 - } + if a.Flag != "" { + count++ } - - finalScore := baseScore * coveragePenalty - debug(w, "coveragePenalty=%.2f ", coveragePenalty) - - // Perfect match boost: only if coverage wasn't penalized - if hasExactMatch && hasNameMatch && finalScore > 0.9 && coveragePenalty == 1.0 { - debug(w, "PERFECT MATCH BOOST ") - - finalScore = math.Min(1.0, finalScore*1.15) + if a.SerialNumber != "" { + count++ } - debug(w, "finalScore=%.2f", finalScore) - return finalScore -} - -func debug(w io.Writer, pattern string, args ...any) { - if w != nil { - fmt.Fprintf(w, pattern, args...) - } + return count } diff --git a/pkg/search/similarity_address.go b/pkg/search/similarity_address.go new file mode 100644 index 00000000..6313209e --- /dev/null +++ b/pkg/search/similarity_address.go @@ -0,0 +1,116 @@ +package search + +import ( + "io" + "strings" + + "github.com/moov-io/watchman/internal/stringscore" +) + +const ( + // Field weights for addresses + line1Weight = 3.0 // Primary address line - most important + line2Weight = 1.0 // Secondary address info - less important + cityWeight = 2.0 // City - moderately important + stateWeight = 1.0 // State - helps confirm location + postalWeight = 1.5 // Postal code - good verification + countryWeight = 2.0 // Country - important for international +) + +func compareAddresses[Q any, I any](w io.Writer, query Entity[Q], index Entity[I], weight float64) scorePiece { + fieldsCompared := 0 + var scores []float64 + + // Compare addresses + if len(query.Addresses) > 0 && len(index.Addresses) > 0 { + fieldsCompared++ + if score := findBestAddressMatch(query.Addresses, index.Addresses); score > 0 { + scores = append(scores, score) + } + } + + if len(scores) == 0 { + return scorePiece{score: 0, weight: weight, fieldsCompared: 0, pieceType: "supporting"} + } + + avgScore := calculateAverage(scores) + return scorePiece{ + score: avgScore, + weight: weight, + matched: avgScore > 0.5, + required: false, + exact: avgScore > 0.99, + fieldsCompared: fieldsCompared, + pieceType: "supporting", + } +} + +func findBestAddressMatch(queryAddrs, indexAddrs []Address) float64 { + bestScore := 0.0 + for _, qa := range queryAddrs { + for _, ia := range indexAddrs { + if score := compareAddress(qa, ia); score > bestScore { + bestScore = score + if score > highConfidenceThreshold { + return score // Early exit on high confidence match + } + } + } + } + return bestScore +} + +func compareAddress(query, index Address) float64 { + var totalScore, totalWeight float64 + + // Compare line1 (highest weight) + if query.Line1 != "" && index.Line1 != "" { + similarity := stringscore.JaroWinkler(query.Line1, index.Line1) + totalScore += similarity * line1Weight + totalWeight += line1Weight + } + + // Compare line2 + if query.Line2 != "" && index.Line2 != "" { + similarity := stringscore.JaroWinkler(query.Line2, index.Line2) + totalScore += similarity * line2Weight + totalWeight += line2Weight + } + + // Compare city + if query.City != "" && index.City != "" { + similarity := stringscore.JaroWinkler(query.City, index.City) + totalScore += similarity * cityWeight + totalWeight += cityWeight + } + + // Compare state (exact match) + if query.State != "" && index.State != "" { + if strings.EqualFold(query.State, index.State) { + totalScore += stateWeight + } + totalWeight += stateWeight + } + + // Compare postal code (exact match) + if query.PostalCode != "" && index.PostalCode != "" { + if strings.EqualFold(query.PostalCode, index.PostalCode) { + totalScore += postalWeight + } + totalWeight += postalWeight + } + + // Compare country (exact match) + if query.Country != "" && index.Country != "" { + if strings.EqualFold(query.Country, index.Country) { + totalScore += countryWeight + } + totalWeight += countryWeight + } + + if totalWeight == 0 { + return 0 + } + + return totalScore / totalWeight +} diff --git a/pkg/search/similarity_close.go b/pkg/search/similarity_close.go new file mode 100644 index 00000000..6429db6d --- /dev/null +++ b/pkg/search/similarity_close.go @@ -0,0 +1,217 @@ +package search + +import ( + "io" + "math" + "time" +) + +const ( + // Date thresholds in days + exactMatch = 0 + veryClose = 2 // Within 2 days + close = 7 // Within a week + moderate = 30 // Within a month + distant = 365 // Within a year +) + +// compareEntityDates performs date comparisons based on entity type +func compareEntityDates[Q any, I any](w io.Writer, query Entity[Q], index Entity[I], weight float64) scorePiece { + if query.Type != index.Type { + return scorePiece{score: 0, weight: weight, pieceType: "dates", fieldsCompared: 0} + } + + var dateScore float64 + var fieldsCompared int + var matched bool + + switch query.Type { + case EntityPerson: + dateScore, matched, fieldsCompared = comparePersonDates(query.Person, index.Person) + case EntityBusiness: + dateScore, matched, fieldsCompared = compareBusinessDates(query.Business, index.Business) + case EntityOrganization: + dateScore, matched, fieldsCompared = compareOrgDates(query.Organization, index.Organization) + case EntityVessel, EntityAircraft: + dateScore, matched, fieldsCompared = compareAssetDates(query.Type, query, index) + } + + return scorePiece{ + score: dateScore, + weight: weight, + matched: matched, + required: fieldsCompared > 0, + exact: dateScore > 0.99, + fieldsCompared: fieldsCompared, + pieceType: "dates", + } +} + +// comparePersonDates handles birth and death dates +func comparePersonDates(query *Person, index *Person) (float64, bool, int) { + if query == nil || index == nil { + return 0, false, 0 + } + + fieldsCompared := 0 + var scores []float64 + + // Birth date comparison + if query.BirthDate != nil && index.BirthDate != nil { + fieldsCompared++ + scores = append(scores, compareDates(query.BirthDate, index.BirthDate)) + } + + // Death date comparison + if query.DeathDate != nil && index.DeathDate != nil { + fieldsCompared++ + scores = append(scores, compareDates(query.DeathDate, index.DeathDate)) + } + + if len(scores) == 0 { + return 0, false, fieldsCompared + } + + // Calculate average score + avgScore := calculateAverage(scores) + + // Check date consistency if both dates are present + if fieldsCompared == 2 && !areDatesLogical(query, index) { + avgScore *= 0.5 // Penalty for illogical dates + } + + return avgScore, avgScore > 0.7, fieldsCompared +} + +// compareBusinessDates handles business dates +func compareBusinessDates(query *Business, index *Business) (float64, bool, int) { + if query == nil || index == nil { + return 0, false, 0 + } + + fieldsCompared := 0 + var scores []float64 + + // Created date comparison + if query.Created != nil && index.Created != nil { + fieldsCompared++ + scores = append(scores, compareDates(query.Created, index.Created)) + } + + // Dissolved date comparison (if present) + if query.Dissolved != nil && index.Dissolved != nil { + fieldsCompared++ + scores = append(scores, compareDates(query.Dissolved, index.Dissolved)) + } + + if len(scores) == 0 { + return 0, false, fieldsCompared + } + + avgScore := calculateAverage(scores) + return avgScore, avgScore > 0.7, fieldsCompared +} + +// compareOrgDates handles organization dates +func compareOrgDates(query *Organization, index *Organization) (float64, bool, int) { + if query == nil || index == nil { + return 0, false, 0 + } + + fieldsCompared := 0 + var scores []float64 + + // Created date comparison + if query.Created != nil && index.Created != nil { + fieldsCompared++ + scores = append(scores, compareDates(query.Created, index.Created)) + } + + // Dissolved date comparison + if query.Dissolved != nil && index.Dissolved != nil { + fieldsCompared++ + scores = append(scores, compareDates(query.Dissolved, index.Dissolved)) + } + + if len(scores) == 0 { + return 0, false, fieldsCompared + } + + avgScore := calculateAverage(scores) + return avgScore, avgScore > 0.7, fieldsCompared +} + +// compareAssetDates handles vessel and aircraft dates +func compareAssetDates[Q any, I any](entityType EntityType, query Entity[Q], index Entity[I]) (float64, bool, int) { + fieldsCompared := 0 + var builtDate1, builtDate2 *time.Time + + switch entityType { + case EntityVessel: + if query.Vessel != nil && index.Vessel != nil { + builtDate1 = query.Vessel.Built + builtDate2 = index.Vessel.Built + } + case EntityAircraft: + if query.Aircraft != nil && index.Aircraft != nil { + builtDate1 = query.Aircraft.Built + builtDate2 = index.Aircraft.Built + } + } + + if builtDate1 == nil || builtDate2 == nil { + return 0, false, fieldsCompared + } + + fieldsCompared = 1 + score := compareDates(builtDate1, builtDate2) + return score, score > 0.7, fieldsCompared +} + +// compareDates calculates similarity score between two dates +func compareDates(date1, date2 *time.Time) float64 { + // Normalize to same time of day + d1 := date1.Truncate(24 * time.Hour) + d2 := date2.Truncate(24 * time.Hour) + + // Calculate difference in days + diffDays := math.Abs(d1.Sub(d2).Hours() / 24) + + switch { + case diffDays <= float64(exactMatch): + return 1.0 + case diffDays <= float64(veryClose): + return 0.95 - (0.05 * (diffDays / float64(veryClose))) + case diffDays <= float64(close): + return 0.9 - (0.1 * (diffDays / float64(close))) + case diffDays <= float64(moderate): + return 0.8 - (0.2 * (diffDays / float64(moderate))) + case diffDays <= float64(distant): + return 0.6 - (0.3 * (diffDays / float64(distant))) + default: + return 0.0 + } +} + +// areDatesLogical checks if dates make temporal sense +func areDatesLogical(person *Person, index *Person) bool { + if person.BirthDate != nil && person.DeathDate != nil && + index.BirthDate != nil && index.DeathDate != nil { + + // Check that birth precedes death in both records + personValid := person.BirthDate.Before(*person.DeathDate) + indexValid := index.BirthDate.Before(*index.DeathDate) + + if !personValid || !indexValid { + return false + } + + // Check relative lifespans (within 20%) + personSpan := person.DeathDate.Sub(*person.BirthDate) + indexSpan := index.DeathDate.Sub(*index.BirthDate) + + ratio := math.Max(personSpan.Hours(), indexSpan.Hours()) / math.Max(1, math.Min(personSpan.Hours(), indexSpan.Hours())) + return ratio <= 1.2 + } + return true +} diff --git a/pkg/search/similarity_exact.go b/pkg/search/similarity_exact.go new file mode 100644 index 00000000..00d664c3 --- /dev/null +++ b/pkg/search/similarity_exact.go @@ -0,0 +1,771 @@ +package search + +import ( + "io" + "strings" + "unicode" +) + +// compareExactIdentifiers covers exact matches for identifiers across all entity types +func compareExactIdentifiers[Q any, I any](w io.Writer, query Entity[Q], index Entity[I], weight float64) scorePiece { + // If types don't match, return early + if query.Type != index.Type { + return scorePiece{ + score: 0, + weight: weight, + matched: false, + required: false, + exact: false, + fieldsCompared: 1, + pieceType: "identifiers", + } + } + + // Call appropriate helper based on entity type + switch query.Type { + case EntityPerson: + return comparePersonExactIDs(w, query.Person, index.Person, weight) + case EntityBusiness: + return compareBusinessExactIDs(w, query.Business, index.Business, weight) + case EntityOrganization: + return compareOrgExactIDs(w, query.Organization, index.Organization, weight) + case EntityVessel: + return compareVesselExactIDs(w, query.Vessel, index.Vessel, weight) + case EntityAircraft: + return compareAircraftExactIDs(w, query.Aircraft, index.Aircraft, weight) + default: + return scorePiece{score: 0, weight: 0, fieldsCompared: 0, pieceType: "identifiers"} + } +} + +// comparePersonExactIDs checks exact matches for Person-specific identifiers +func comparePersonExactIDs(w io.Writer, query *Person, index *Person, weight float64) scorePiece { + if query == nil || index == nil { + return scorePiece{score: 0, weight: 0, fieldsCompared: 0, pieceType: "identifiers"} + } + + fieldsCompared := 0 + totalWeight := 0.0 + score := 0.0 + hasMatch := false + + // Government IDs (extremely high weight for exact matches) + if len(query.GovernmentIDs) > 0 && len(index.GovernmentIDs) > 0 { + fieldsCompared++ + totalWeight += 15.0 + for _, qID := range query.GovernmentIDs { + for _, iID := range index.GovernmentIDs { + if strings.EqualFold(string(qID.Type), string(iID.Type)) && + strings.EqualFold(qID.Country, iID.Country) && + strings.EqualFold(qID.Identifier, iID.Identifier) { + score += 15.0 + hasMatch = true + goto GovIDDone // Break both loops on first match + } + } + } + } +GovIDDone: + + finalScore := 0.0 + if totalWeight > 0 { + finalScore = score / totalWeight + } + + return scorePiece{ + score: finalScore, + weight: weight, + matched: hasMatch, + required: fieldsCompared > 0, + exact: finalScore > 0.99, + fieldsCompared: fieldsCompared, + pieceType: "identifiers", + } +} + +// compareBusinessExactIDs checks exact matches for Business-specific identifiers +func compareBusinessExactIDs(w io.Writer, query *Business, index *Business, weight float64) scorePiece { + if query == nil || index == nil { + return scorePiece{score: 0, weight: 0, fieldsCompared: 0, pieceType: "identifiers"} + } + + fieldsCompared := 0 + totalWeight := 0.0 + score := 0.0 + hasMatch := false + + // Business Registration/Tax IDs + if len(query.Identifier) > 0 && len(index.Identifier) > 0 { + fieldsCompared++ + totalWeight += 15.0 + for _, qID := range query.Identifier { + for _, iID := range index.Identifier { + // Exact match on all identifier fields + if strings.EqualFold(qID.Name, iID.Name) && + strings.EqualFold(qID.Country, iID.Country) && + strings.EqualFold(qID.Identifier, iID.Identifier) { + score += 15.0 + hasMatch = true + goto IdentifierDone + } + } + } + } +IdentifierDone: + + finalScore := 0.0 + if totalWeight > 0 { + finalScore = score / totalWeight + } + + return scorePiece{ + score: finalScore, + weight: weight, + matched: hasMatch, + required: fieldsCompared > 0, + exact: finalScore > 0.99, + fieldsCompared: fieldsCompared, + pieceType: "identifiers", + } +} + +// compareOrgExactIDs checks exact matches for Organization-specific identifiers +func compareOrgExactIDs(w io.Writer, query *Organization, index *Organization, weight float64) scorePiece { + if query == nil || index == nil { + return scorePiece{score: 0, weight: 0, fieldsCompared: 0, pieceType: "identifiers"} + } + + fieldsCompared := 0 + totalWeight := 0.0 + score := 0.0 + hasMatch := false + + // Organization Registration/Tax IDs + if len(query.Identifier) > 0 && len(index.Identifier) > 0 { + fieldsCompared++ + totalWeight += 15.0 + for _, qID := range query.Identifier { + for _, iID := range index.Identifier { + // Exact match on all identifier fields + if strings.EqualFold(qID.Name, iID.Name) && + strings.EqualFold(qID.Country, iID.Country) && + strings.EqualFold(qID.Identifier, iID.Identifier) { + score += 15.0 + hasMatch = true + goto IdentifierDone + } + } + } + } +IdentifierDone: + + finalScore := 0.0 + if totalWeight > 0 { + finalScore = score / totalWeight + } + + return scorePiece{ + score: finalScore, + weight: weight, + matched: hasMatch, + required: fieldsCompared > 0, + exact: finalScore > 0.99, + fieldsCompared: fieldsCompared, + pieceType: "identifiers", + } +} + +// compareVesselExactIDs checks exact matches for Vessel-specific identifiers +func compareVesselExactIDs(w io.Writer, query *Vessel, index *Vessel, weight float64) scorePiece { + if query == nil || index == nil { + return scorePiece{score: 0, weight: 0, fieldsCompared: 0, pieceType: "identifiers"} + } + + fieldsCompared := 0 + totalWeight := 0.0 + score := 0.0 + hasMatch := false + + // IMO Number (highest weight) + if query.IMONumber != "" { + fieldsCompared++ + totalWeight += 15.0 + if strings.EqualFold(query.IMONumber, index.IMONumber) { + score += 15.0 + hasMatch = true + } + } + + // Call Sign + if query.CallSign != "" { + fieldsCompared++ + totalWeight += 12.0 + if strings.EqualFold(query.CallSign, index.CallSign) { + score += 12.0 + hasMatch = true + } + } + + // MMSI (Maritime Mobile Service Identity) + if query.MMSI != "" { + fieldsCompared++ + totalWeight += 12.0 + if strings.EqualFold(query.MMSI, index.MMSI) { + score += 12.0 + hasMatch = true + } + } + + finalScore := 0.0 + if totalWeight > 0 { + finalScore = score / totalWeight + } + + return scorePiece{ + score: finalScore, + weight: weight, + matched: hasMatch, + required: fieldsCompared > 0, + exact: finalScore > 0.99, + fieldsCompared: fieldsCompared, + pieceType: "identifiers", + } +} + +// compareAircraftExactIDs checks exact matches for Aircraft-specific identifiers +func compareAircraftExactIDs(w io.Writer, query *Aircraft, index *Aircraft, weight float64) scorePiece { + if query == nil || index == nil { + return scorePiece{score: 0, weight: 0, fieldsCompared: 0, pieceType: "identifiers"} + } + + fieldsCompared := 0 + totalWeight := 0.0 + score := 0.0 + hasMatch := false + + // Serial Number (highest weight) + if query.SerialNumber != "" { + fieldsCompared++ + totalWeight += 15.0 + if strings.EqualFold(query.SerialNumber, index.SerialNumber) { + score += 15.0 + hasMatch = true + } + } + + // ICAO Code + if query.ICAOCode != "" { + fieldsCompared++ + totalWeight += 12.0 + if strings.EqualFold(query.ICAOCode, index.ICAOCode) { + score += 12.0 + hasMatch = true + } + } + + finalScore := 0.0 + if totalWeight > 0 { + finalScore = score / totalWeight + } + + return scorePiece{ + score: finalScore, + weight: weight, + matched: hasMatch, + required: fieldsCompared > 0, + exact: finalScore > 0.99, + fieldsCompared: fieldsCompared, + pieceType: "identifiers", + } +} + +func compareExactCryptoAddresses[Q any, I any](w io.Writer, query Entity[Q], index Entity[I], weight float64) scorePiece { + fieldsCompared := 0 + hasMatch := false + score := 0.0 + + qCAs := query.CryptoAddresses + iCAs := index.CryptoAddresses + + // Early return if either list is empty + if len(qCAs) == 0 || len(iCAs) == 0 { + return scorePiece{ + score: 0, + weight: weight, + matched: false, + required: false, + exact: false, + fieldsCompared: 0, + pieceType: "crypto-exact", + } + } + + fieldsCompared++ + + // First try exact matches (both currency and address) + for _, qCA := range qCAs { + qAddr := strings.ToLower(strings.TrimSpace(qCA.Address)) + qCurr := strings.ToLower(strings.TrimSpace(qCA.Currency)) + + if qAddr == "" { + continue // Skip empty addresses + } + + for _, iCA := range iCAs { + iAddr := strings.ToLower(strings.TrimSpace(iCA.Address)) + iCurr := strings.ToLower(strings.TrimSpace(iCA.Currency)) + + // Case 1: Both have currency specified - need both to match + if qCurr != "" && iCurr != "" { + if qCurr == iCurr && qAddr == iAddr { + score = 1.0 + hasMatch = true + goto Done + } + } else { + // Case 2: At least one currency empty - match on address only + if qAddr == iAddr { + score = 1.0 + hasMatch = true + goto Done + } + } + } + } + +Done: + return scorePiece{ + score: score, + weight: weight, + matched: hasMatch, + required: false, + exact: score > 0.99, + fieldsCompared: fieldsCompared, + pieceType: "crypto-exact", + } +} + +// compareExactGovernmentIDs compares government IDs across entity types +func compareExactGovernmentIDs[Q any, I any](w io.Writer, query Entity[Q], index Entity[I], weight float64) scorePiece { + if query.Type != index.Type { + return scorePiece{ + score: 0, + weight: weight, + matched: false, + required: false, + exact: false, + fieldsCompared: 0, + pieceType: "gov-ids-exact", + } + } + + switch query.Type { + case EntityPerson: + return comparePersonGovernmentIDs(query.Person, index.Person, weight) + case EntityBusiness: + return compareBusinessGovernmentIDs(query.Business, index.Business, weight) + case EntityOrganization: + return compareOrgGovernmentIDs(query.Organization, index.Organization, weight) + default: + return scorePiece{ + score: 0, + weight: weight, + matched: false, + required: false, + exact: false, + fieldsCompared: 0, + pieceType: "gov-ids-exact", + } + } +} + +// idMatch represents the result of comparing two identifiers +type idMatch struct { + score float64 + found bool + exact bool + hasCountry bool +} + +// compareIdentifiers handles the core logic of comparing two identifier values +func compareIdentifiers(queryID, indexID string, queryCountry, indexCountry string) idMatch { + // Early return if identifiers don't match + if !strings.EqualFold(strings.TrimSpace(queryID), strings.TrimSpace(indexID)) { + return idMatch{score: 0, found: false, exact: false} + } + + // If we get here, identifiers match exactly + queryCountry = strings.TrimSpace(queryCountry) + indexCountry = strings.TrimSpace(indexCountry) + + // If neither has country, it's an exact match but flag no country + if queryCountry == "" && indexCountry == "" { + return idMatch{score: 1.0, found: true, exact: true, hasCountry: false} + } + + // If only one has country, slight penalty + if (queryCountry == "" && indexCountry != "") || (queryCountry != "" && indexCountry == "") { + return idMatch{score: 0.9, found: true, exact: false, hasCountry: true} + } + + // Both have country - check if they match + if strings.EqualFold(queryCountry, indexCountry) { + return idMatch{score: 1.0, found: true, exact: true, hasCountry: true} + } + + // Countries don't match - significant penalty but still count as a match + return idMatch{score: 0.7, found: true, exact: false, hasCountry: true} +} + +func comparePersonGovernmentIDs(query *Person, index *Person, weight float64) scorePiece { + if query == nil || index == nil { + return scorePiece{score: 0, weight: weight, fieldsCompared: 0, pieceType: "gov-ids-exact"} + } + + qIDs := query.GovernmentIDs + iIDs := index.GovernmentIDs + + if len(qIDs) == 0 || len(iIDs) == 0 { + return scorePiece{score: 0, weight: weight, fieldsCompared: 0, pieceType: "gov-ids-exact"} + } + + fieldsCompared := 1 + bestMatch := idMatch{score: 0} + + for _, qID := range qIDs { + for _, iID := range iIDs { + match := compareIdentifiers(qID.Identifier, iID.Identifier, qID.Country, iID.Country) + if match.found && match.score > bestMatch.score { + bestMatch = match + } + if bestMatch.exact { + goto Done + } + } + } + +Done: + return scorePiece{ + score: bestMatch.score, + weight: weight, + matched: bestMatch.found, + required: false, + exact: bestMatch.exact, + fieldsCompared: fieldsCompared, + pieceType: "gov-ids-exact", + } +} + +func compareBusinessGovernmentIDs(query *Business, index *Business, weight float64) scorePiece { + if query == nil || index == nil { + return scorePiece{score: 0, weight: weight, fieldsCompared: 0, pieceType: "gov-ids-exact"} + } + + qIDs := query.Identifier + iIDs := index.Identifier + + if len(qIDs) == 0 || len(iIDs) == 0 { + return scorePiece{score: 0, weight: weight, fieldsCompared: 0, pieceType: "gov-ids-exact"} + } + + fieldsCompared := 1 + bestMatch := idMatch{score: 0} + + for _, qID := range qIDs { + for _, iID := range iIDs { + // For business, we'll check the identifier and country, ignoring name for now + match := compareIdentifiers(qID.Identifier, iID.Identifier, qID.Country, iID.Country) + if match.found && match.score > bestMatch.score { + bestMatch = match + } + if bestMatch.exact { + goto Done + } + } + } + +Done: + return scorePiece{ + score: bestMatch.score, + weight: weight, + matched: bestMatch.found, + required: false, + exact: bestMatch.exact, + fieldsCompared: fieldsCompared, + pieceType: "gov-ids-exact", + } +} + +func compareOrgGovernmentIDs(query *Organization, index *Organization, weight float64) scorePiece { + if query == nil || index == nil { + return scorePiece{score: 0, weight: weight, fieldsCompared: 0, pieceType: "gov-ids-exact"} + } + + qIDs := query.Identifier + iIDs := index.Identifier + + if len(qIDs) == 0 || len(iIDs) == 0 { + return scorePiece{score: 0, weight: weight, fieldsCompared: 0, pieceType: "gov-ids-exact"} + } + + fieldsCompared := 1 + bestMatch := idMatch{score: 0} + + for _, qID := range qIDs { + for _, iID := range iIDs { + // For orgs, we'll check the identifier and country, ignoring name for now + match := compareIdentifiers(qID.Identifier, iID.Identifier, qID.Country, iID.Country) + if match.found && match.score > bestMatch.score { + bestMatch = match + } + if bestMatch.exact { + goto Done + } + } + } + +Done: + return scorePiece{ + score: bestMatch.score, + weight: weight, + matched: bestMatch.found, + required: false, + exact: bestMatch.exact, + fieldsCompared: fieldsCompared, + pieceType: "gov-ids-exact", + } +} + +func compareExactSourceID[Q any, I any](w io.Writer, query Entity[Q], index Entity[I], weight float64) scorePiece { + // Early return if query has no source ID + if strings.TrimSpace(query.SourceID) == "" { + return scorePiece{ + score: 0, + weight: weight, + matched: false, + required: false, + exact: false, + fieldsCompared: 0, + pieceType: "source-id-exact", + } + } + + // Always count as field compared if query has a source ID + fieldsCompared := 1 + + // Handle case where index has no source ID + if strings.TrimSpace(index.SourceID) == "" { + return scorePiece{ + score: 0, + weight: weight, + matched: false, + required: false, + exact: false, + fieldsCompared: fieldsCompared, + pieceType: "source-id-exact", + } + } + + // Compare normalized source IDs + hasMatch := strings.EqualFold( + strings.TrimSpace(query.SourceID), + strings.TrimSpace(index.SourceID), + ) + + return scorePiece{ + score: boolToScore(hasMatch), + weight: weight, + matched: hasMatch, + required: false, + exact: hasMatch, // If matched, it's exact by definition + fieldsCompared: fieldsCompared, + pieceType: "source-id-exact", + } +} + +func compareExactSourceList[Q any, I any](w io.Writer, query Entity[Q], index Entity[I], weight float64) scorePiece { + // Early return if query has no source + if query.Source == "" { + return scorePiece{ + score: 0, + weight: weight, + matched: false, + required: false, + exact: false, + fieldsCompared: 0, + pieceType: "source-list", + } + } + + // Always count as field compared if query has a source + fieldsCompared := 1 + + // Handle case where index has no source + if index.Source == "" { + return scorePiece{ + score: 0, + weight: weight, + matched: false, + required: false, + exact: false, + fieldsCompared: fieldsCompared, + pieceType: "source-list", + } + } + + // Compare normalized sources + hasMatch := strings.EqualFold( + string(query.Source), + string(index.Source), + ) + + return scorePiece{ + score: boolToScore(hasMatch), + weight: weight, + matched: hasMatch, + required: false, + exact: hasMatch, // If matched, it's exact by definition + fieldsCompared: fieldsCompared, + pieceType: "source-list", + } +} + +// contactFieldMatch handles matching logic for a single contact field type (email, phone, fax) +type contactFieldMatch struct { + matches int + totalQuery int + score float64 +} + +func compareExactContactInfo[Q any, I any](w io.Writer, query Entity[Q], index Entity[I], weight float64) scorePiece { + fieldsCompared := 0 + var matches []contactFieldMatch + + // Compare emails (exact match) + if len(query.Contact.EmailAddresses) > 0 && len(index.Contact.EmailAddresses) > 0 { + fieldsCompared++ + matches = append(matches, compareContactField( + query.Contact.EmailAddresses, + index.Contact.EmailAddresses, + normalizeEmail, + )) + } + + // Compare phone numbers (normalized) + if len(query.Contact.PhoneNumbers) > 0 && len(index.Contact.PhoneNumbers) > 0 { + fieldsCompared++ + matches = append(matches, compareContactField( + query.Contact.PhoneNumbers, + index.Contact.PhoneNumbers, + normalizePhoneNumber, + )) + } + + // Compare fax numbers (normalized same as phones) + if len(query.Contact.FaxNumbers) > 0 && len(index.Contact.FaxNumbers) > 0 { + fieldsCompared++ + matches = append(matches, compareContactField( + query.Contact.FaxNumbers, + index.Contact.FaxNumbers, + normalizePhoneNumber, + )) + } + + if fieldsCompared == 0 { + return scorePiece{ + score: 0, + weight: weight, + matched: false, + required: false, + exact: false, + fieldsCompared: 0, + pieceType: "contact-exact", + } + } + + // Calculate final scores + totalScore := 0.0 + totalMatches := 0 + totalQueryItems := 0 + + for _, m := range matches { + totalScore += m.score + totalMatches += m.matches + totalQueryItems += m.totalQuery + } + + finalScore := totalScore / float64(len(matches)) + + return scorePiece{ + score: finalScore, + weight: weight, + matched: totalMatches > 0, + required: false, + exact: finalScore > 0.99, + fieldsCompared: fieldsCompared, + pieceType: "contact-exact", + } +} + +// compareContactField handles the comparison logic for a single type of contact field +func compareContactField(queryValues, indexValues []string, normalize func(string) string) contactFieldMatch { + matches := 0 + + // Create map of normalized index values for faster lookup + indexMap := make(map[string]bool, len(indexValues)) + for _, iv := range indexValues { + indexMap[normalize(iv)] = true + } + + // Check each query value against the map + for _, qv := range queryValues { + if indexMap[normalize(qv)] { + matches++ + } + } + + score := float64(matches) / float64(len(queryValues)) + + return contactFieldMatch{ + matches: matches, + totalQuery: len(queryValues), + score: score, + } +} + +// normalizeEmail normalizes email addresses for comparison +// TODO(adam): +func normalizeEmail(email string) string { + return strings.ToLower(strings.TrimSpace(email)) +} + +// normalizePhoneNumber strips all non-numeric characters and normalizes phone numbers +// TODO(adam): +func normalizePhoneNumber(phone string) string { + var normalized strings.Builder + + // Strip everything except digits and plus sign (for international prefix) + for _, r := range phone { + if unicode.IsDigit(r) || r == '+' { + normalized.WriteRune(r) + } + } + + // Handle international format + result := normalized.String() + if strings.HasPrefix(result, "+") { + return result // Keep international format as is + } + + // If it's a 10-digit number without country code, keep as is + if len(result) == 10 { + return result + } + + // If it has more digits but no plus, assume it includes country code + if len(result) > 10 { + return "+" + result + } + + return result +} diff --git a/pkg/search/similarity_fuzzy.go b/pkg/search/similarity_fuzzy.go new file mode 100644 index 00000000..c72014b3 --- /dev/null +++ b/pkg/search/similarity_fuzzy.go @@ -0,0 +1,666 @@ +package search + +import ( + "io" + "math" + "regexp" + "strings" + "unicode" + + "github.com/moov-io/watchman/internal/stringscore" +) + +const ( + // Minimum length for a name term to be considered significant + minTermLength = 3 + + // Minimum number of significant terms that must match for a high confidence match + minMatchingTerms = 2 + + // Score thresholds for term matching + termMatchThreshold = 0.90 // Individual term match threshold + nameMatchThreshold = 0.85 // Overall name match threshold +) + +// nameMatch tracks detailed matching information +type nameMatch struct { + score float64 + matchingTerms int + totalTerms int + isExact bool + isHistorical bool +} + +func compareName[Q any, I any](w io.Writer, query Entity[Q], index Entity[I], weight float64) scorePiece { + qName := normalizeName(query.Name) + iName := normalizeName(index.Name) + + // Early return for empty query + if qName == "" { + return scorePiece{score: 0, weight: 0, fieldsCompared: 0, pieceType: "name"} + } + + // Exact match fast path + if qName == iName { + return scorePiece{ + score: 1.0, + weight: weight, + matched: true, + required: true, + exact: true, + fieldsCompared: 1, + pieceType: "name", + } + } + + // Get query terms and filter out insignificant ones + qTerms := filterSignificantTerms(strings.Fields(qName)) + if len(qTerms) == 0 { + return scorePiece{score: 0, weight: 0, fieldsCompared: 0, pieceType: "name"} + } + + // Check primary name + bestMatch := compareNameTerms(qTerms, iName) + + // Check alternate names for persons + if query.Person != nil && index.Person != nil { + for _, altName := range index.Person.AltNames { + altMatch := compareNameTerms(qTerms, normalizeName(altName)) + if altMatch.score > bestMatch.score { + bestMatch = altMatch + } + } + } + + // Check historical names with penalty + for _, hist := range index.HistoricalInfo { + if strings.EqualFold(hist.Type, "Former Name") { + histMatch := compareNameTerms(qTerms, normalizeName(hist.Value)) + histMatch.score *= 0.95 // Apply penalty for historical names + histMatch.isHistorical = true + if histMatch.score > bestMatch.score { + bestMatch = histMatch + } + } + } + + // Apply additional criteria for match quality + finalScore := adjustScoreBasedOnQuality(bestMatch, len(qTerms)) + + return scorePiece{ + score: finalScore, + weight: weight, + matched: isHighConfidenceMatch(bestMatch, finalScore), + required: true, + exact: finalScore > exactMatchThreshold, + fieldsCompared: 1, + pieceType: "name", + } +} + +// normalizeName performs thorough name normalization +func normalizeName(name string) string { + // Convert to lowercase and trim spaces + name = strings.ToLower(strings.TrimSpace(name)) + + // Remove all punctuation and normalize whitespace + var normalized strings.Builder + lastWasSpace := true // Start with true to trim leading spaces + + for _, r := range name { + if unicode.IsPunct(r) || unicode.IsSymbol(r) { + if !lastWasSpace { + normalized.WriteRune(' ') + lastWasSpace = true + } + continue + } + + if unicode.IsSpace(r) { + if !lastWasSpace { + normalized.WriteRune(' ') + lastWasSpace = true + } + continue + } + + if unicode.IsLetter(r) || unicode.IsNumber(r) { + normalized.WriteRune(r) + lastWasSpace = false + } + } + + return strings.TrimSpace(normalized.String()) +} + +// filterSignificantTerms removes common noise and insignificant terms +func filterSignificantTerms(terms []string) []string { + filtered := make([]string, 0, len(terms)) + for _, term := range terms { + // Skip terms that are too short + if len(term) < minTermLength { + continue + } + + // Skip common noise terms (expand this list as needed) + switch term { + case "the", "and", "or", "of", "in", "at", "by": + continue + } + + filtered = append(filtered, term) + } + return filtered +} + +// compareNameTerms performs detailed term-by-term comparison +func compareNameTerms(queryTerms []string, indexName string) nameMatch { + indexTerms := filterSignificantTerms(strings.Fields(indexName)) + if len(indexTerms) == 0 { + return nameMatch{score: 0} + } + + // Track individual term matches + termScores := make([]float64, len(queryTerms)) + matchingTerms := 0 + + // For each query term, find its best match in index terms + for i, qTerm := range queryTerms { + bestTermScore := 0.0 + for _, iTerm := range indexTerms { + score := stringscore.JaroWinkler(qTerm, iTerm) + if score > bestTermScore { + bestTermScore = score + if score > termMatchThreshold { + matchingTerms++ + } + } + } + termScores[i] = bestTermScore + } + + // Calculate overall score + var totalScore float64 + for _, score := range termScores { + totalScore += score + } + avgScore := totalScore / float64(len(termScores)) + + return nameMatch{ + score: avgScore, + matchingTerms: matchingTerms, + totalTerms: len(queryTerms), + isExact: avgScore > exactMatchThreshold, + } +} + +// adjustScoreBasedOnQuality applies additional quality criteria +func adjustScoreBasedOnQuality(match nameMatch, queryTermCount int) float64 { + // Require minimum number of matching terms for high scores + if match.matchingTerms < minMatchingTerms && queryTermCount >= minMatchingTerms { + return match.score * 0.8 // Significant penalty for too few matching terms + } + + // Historical names already have a penalty applied + if match.isHistorical { + return match.score + } + + return match.score +} + +// isHighConfidenceMatch determines if the match quality is sufficient +func isHighConfidenceMatch(match nameMatch, finalScore float64) bool { + // Must meet both term matching and score criteria + return match.matchingTerms >= minMatchingTerms && finalScore > nameMatchThreshold +} + +const ( + titleMatchThreshold = 0.85 // Threshold for considering titles matched + minTitleTermLength = 2 // Minimum length for title terms + abbreviationThreshold = 0.92 // Threshold for matching abbreviated titles +) + +var ( + // Common title abbreviations and their full forms + titleAbbreviations = map[string]string{ + "ceo": "chief executive officer", + "cfo": "chief financial officer", + "coo": "chief operating officer", + "pres": "president", + "vp": "vice president", + "dir": "director", + "exec": "executive", + "mgr": "manager", + "sr": "senior", + "jr": "junior", + "asst": "assistant", + "assoc": "associate", + "tech": "technical", + "admin": "administrator", + "eng": "engineer", + "dev": "developer", + } + + // Patterns to clean up titles + punctRegexp = regexp.MustCompile(`[^\w\s-]`) + spaceRegexp = regexp.MustCompile(`\s+`) +) + +func compareEntityTitlesFuzzy[Q any, I any](w io.Writer, query Entity[Q], index Entity[I], weight float64) scorePiece { + if len(query.Titles) == 0 { + return scorePiece{score: 0, weight: 0, fieldsCompared: 0, pieceType: "titles"} + } + + // Prepare normalized index titles once + normalizedIndexTitles := make([]string, 0, len(index.Titles)) + for _, title := range index.Titles { + if normalized := normalizeTitle(title); normalized != "" { + normalizedIndexTitles = append(normalizedIndexTitles, normalized) + } + } + + if len(normalizedIndexTitles) == 0 { + return scorePiece{score: 0, weight: 0, fieldsCompared: 0, pieceType: "titles"} + } + + fieldsCompared := 0 + matches := 0 + total := 0 + + for _, qTitle := range query.Titles { + normalizedQuery := normalizeTitle(qTitle) + if normalizedQuery == "" { + continue + } + + fieldsCompared++ + total++ + + // Try exact match first + if score := findBestTitleMatch(normalizedQuery, normalizedIndexTitles); score > 0 { + if score > titleMatchThreshold { + matches++ + } + continue + } + + // Try matching with expanded abbreviations + expandedQuery := expandAbbreviations(normalizedQuery) + if score := findBestTitleMatch(expandedQuery, normalizedIndexTitles); score > titleMatchThreshold { + matches++ + continue + } + + // Try matching each index title with expanded abbreviations + bestScore := 0.0 + for _, iTitle := range normalizedIndexTitles { + expandedIndex := expandAbbreviations(iTitle) + score := calculateTitleSimilarity(normalizedQuery, expandedIndex) + if score > bestScore { + bestScore = score + } + } + + if bestScore > titleMatchThreshold { + matches++ + } + } + + var finalScore float64 + if total > 0 { + finalScore = float64(matches) / float64(total) + } + + return scorePiece{ + score: finalScore, + weight: weight, + matched: finalScore > 0.5, + required: false, + exact: finalScore > exactMatchThreshold, + fieldsCompared: fieldsCompared, + pieceType: "titles", + } +} + +// normalizeTitle cleans and normalizes a title string +func normalizeTitle(title string) string { + // Convert to lowercase and trim + title = strings.TrimSpace(strings.ToLower(title)) + + // Remove punctuation except hyphens + title = punctRegexp.ReplaceAllString(title, " ") + + // Normalize spaces + title = spaceRegexp.ReplaceAllString(title, " ") + + // Final trim + return strings.TrimSpace(title) +} + +// expandAbbreviations expands known abbreviations in a title +func expandAbbreviations(title string) string { + words := strings.Fields(title) + expanded := make([]string, 0, len(words)) + + for _, word := range words { + if full, exists := titleAbbreviations[word]; exists { + expanded = append(expanded, full) + } else { + expanded = append(expanded, word) + } + } + + return strings.Join(expanded, " ") +} + +// calculateTitleSimilarity computes similarity between two titles +func calculateTitleSimilarity(title1, title2 string) float64 { + // Handle exact matches + if title1 == title2 { + return 1.0 + } + + // Split into terms + terms1 := strings.Fields(title1) + terms2 := strings.Fields(title2) + + // Filter out very short terms + terms1 = filterTerms(terms1) + terms2 = filterTerms(terms2) + + if len(terms1) == 0 || len(terms2) == 0 { + return 0.0 + } + + // Use JaroWinkler for term comparison + score := stringscore.BestPairsJaroWinkler(terms1, title2) + + // Adjust score based on length difference + lengthDiff := math.Abs(float64(len(terms1) - len(terms2))) + if lengthDiff > 0 { + score *= (1.0 - (lengthDiff * 0.1)) + } + + return score +} + +// findBestTitleMatch finds the best matching index title for a query title +func findBestTitleMatch(queryTitle string, indexTitles []string) float64 { + bestScore := 0.0 + + for _, indexTitle := range indexTitles { + score := calculateTitleSimilarity(queryTitle, indexTitle) + if score > bestScore { + bestScore = score + if score > abbreviationThreshold { + break // Good enough match found + } + } + } + + return bestScore +} + +// filterTerms removes terms that are too short +func filterTerms(terms []string) []string { + filtered := make([]string, 0, len(terms)) + for _, term := range terms { + if len(term) >= minTitleTermLength { + filtered = append(filtered, term) + } + } + return filtered +} + +const ( + // Thresholds for name matching + affiliationNameThreshold = 0.85 + + // Type match bonuses/penalties + exactTypeBonus = 0.15 // Bonus for exact type match + relatedTypeBonues = 0.08 // Bonus for related type match + typeMatchPenalty = 0.15 // Penalty for type mismatch +) + +// affiliationTypeGroup groups similar affiliation types +var affiliationTypeGroups = map[string][]string{ + "ownership": { + "owned by", "subsidiary of", "parent of", "holding company", + "owner", "owned", "subsidiary", "parent", + }, + "control": { + "controlled by", "controls", "managed by", "manages", + "operated by", "operates", + }, + "association": { + "linked to", "associated with", "affiliated with", "related to", + "connection to", "connected with", + }, + "leadership": { + "led by", "leader of", "directed by", "directs", + "headed by", "heads", + }, +} + +// affiliationMatch tracks match details +type affiliationMatch struct { + nameScore float64 + typeScore float64 + finalScore float64 + exactMatch bool +} + +func compareAffiliationsFuzzy[Q any, I any](w io.Writer, query Entity[Q], index Entity[I], weight float64) scorePiece { + // Early return if no affiliations to compare + if len(query.Affiliations) == 0 { + return scorePiece{ + score: 0, + weight: weight, + matched: false, + required: false, + exact: false, + fieldsCompared: 0, + pieceType: "affiliations", + } + } + + // Validate index affiliations + if len(index.Affiliations) == 0 { + return scorePiece{ + score: 0, + weight: weight, + matched: false, + required: false, + exact: false, + fieldsCompared: 1, // We had query affiliations but no index matches + pieceType: "affiliations", + } + } + + // Process each query affiliation + var matches []affiliationMatch + for _, qAff := range query.Affiliations { + // Skip empty affiliations + if match := findBestAffiliationMatch(qAff, index.Affiliations); match.nameScore > 0 { + matches = append(matches, match) + } + } + + if len(matches) == 0 { + return scorePiece{ + score: 0, + weight: weight, + matched: false, + required: false, + exact: false, + fieldsCompared: 1, + pieceType: "affiliations", + } + } + + // Calculate final score + finalScore := calculateFinalAffiliateScore(matches) + + return scorePiece{ + score: finalScore, + weight: weight, + matched: finalScore > affiliationNameThreshold, + required: false, + exact: finalScore > exactMatchThreshold, + fieldsCompared: 1, + pieceType: "affiliations", + } +} + +// findBestAffiliationMatch finds the best matching index affiliation +func findBestAffiliationMatch(queryAff Affiliation, indexAffs []Affiliation) affiliationMatch { + qName := normalizeAffiliationName(queryAff.EntityName) + if qName == "" { + return affiliationMatch{} + } + + var bestMatch affiliationMatch + + for _, iAff := range indexAffs { + iName := normalizeAffiliationName(iAff.EntityName) + if iName == "" { + continue + } + + // Calculate name match score + nameScore := calculateNameScore(qName, iName) + if nameScore <= bestMatch.nameScore { + continue + } + + // Calculate type match score + typeScore := calculateTypeScore(queryAff.Type, iAff.Type) + + // Calculate combined score with type influence + finalScore := calculateCombinedScore(nameScore, typeScore) + + if finalScore > bestMatch.finalScore { + bestMatch = affiliationMatch{ + nameScore: nameScore, + typeScore: typeScore, + finalScore: finalScore, + exactMatch: nameScore > exactMatchThreshold && typeScore > 0.9, + } + } + } + + return bestMatch +} + +// normalizeAffiliationName normalizes an entity name for comparison +func normalizeAffiliationName(name string) string { + // Basic normalization + name = strings.TrimSpace(strings.ToLower(name)) + + // Remove common business suffixes + suffixes := []string{" inc", " ltd", " llc", " corp", " co", " company"} + for _, suffix := range suffixes { + name = strings.TrimSuffix(name, suffix) + } + + return strings.TrimSpace(name) +} + +// calculateNameScore calculates similarity between entity names +func calculateNameScore(queryName, indexName string) float64 { + // Exact match check + if queryName == indexName { + return 1.0 + } + + // Calculate similarity using terms + queryTerms := strings.Fields(queryName) + if len(queryTerms) == 0 { + return 0.0 + } + + return stringscore.BestPairsJaroWinkler(queryTerms, indexName) +} + +// calculateTypeScore determines how well affiliation types match +func calculateTypeScore(queryType, indexType string) float64 { + queryType = strings.ToLower(strings.TrimSpace(queryType)) + indexType = strings.ToLower(strings.TrimSpace(indexType)) + + // Exact type match + if queryType == indexType { + return 1.0 + } + + // Check if types are in the same group + queryGroup := getTypeGroup(queryType) + indexGroup := getTypeGroup(indexType) + + if queryGroup != "" && queryGroup == indexGroup { + return 0.8 + } + + return 0.0 +} + +// getTypeGroup finds which group a type belongs to +func getTypeGroup(affType string) string { + for group, types := range affiliationTypeGroups { + for _, t := range types { + if strings.Contains(affType, t) { + return group + } + } + } + return "" +} + +// calculateCombinedScore combines name and type scores +func calculateCombinedScore(nameScore, typeScore float64) float64 { + // Base score is the name match score + score := nameScore + + // Apply type match bonus/penalty + if typeScore > 0.9 { + score += exactTypeBonus + } else if typeScore > 0.7 { + score += relatedTypeBonues + } else { + score -= typeMatchPenalty + } + + // Ensure score stays in valid range + if score > 1.0 { + score = 1.0 + } + if score < 0.0 { + score = 0.0 + } + + return score +} + +// calculateFinalAffiliateScore determines overall affiliation match score +func calculateFinalAffiliateScore(matches []affiliationMatch) float64 { + if len(matches) == 0 { + return 0.0 + } + + // Calculate weighted average giving more weight to better matches + var weightedSum float64 + var totalWeight float64 + + for _, match := range matches { + // Weight is the square of the score to emphasize better matches + weight := match.finalScore * match.finalScore + weightedSum += match.finalScore * weight + totalWeight += weight + } + + if totalWeight == 0 { + return 0.0 + } + + return weightedSum / totalWeight +} diff --git a/pkg/search/similarity_meta.go b/pkg/search/similarity_meta.go new file mode 100644 index 00000000..6992ddf4 --- /dev/null +++ b/pkg/search/similarity_meta.go @@ -0,0 +1,91 @@ +package search + +import ( + "io" + "strings" + + "github.com/moov-io/watchman/internal/stringscore" +) + +func compareSupportingInfo[Q any, I any](w io.Writer, query Entity[Q], index Entity[I], weight float64) scorePiece { + fieldsCompared := 0 + var scores []float64 + + // Compare sanctions + if query.SanctionsInfo != nil && index.SanctionsInfo != nil { + fieldsCompared++ + if score := compareSanctionsPrograms(w, query.SanctionsInfo, index.SanctionsInfo); score > 0 { + scores = append(scores, score) + } + } + + // Compare historical info + if len(query.HistoricalInfo) > 0 && len(index.HistoricalInfo) > 0 { + fieldsCompared++ + if score := compareHistoricalValues(query.HistoricalInfo, index.HistoricalInfo); score > 0 { + scores = append(scores, score) + } + } + + if len(scores) == 0 { + return scorePiece{score: 0, weight: weight, fieldsCompared: 0, pieceType: "supporting"} + } + + avgScore := calculateAverage(scores) + return scorePiece{ + score: avgScore, + weight: weight, + matched: avgScore > 0.5, + required: false, + exact: avgScore > 0.99, + fieldsCompared: fieldsCompared, + pieceType: "supporting", + } +} + +func compareSanctionsPrograms(w io.Writer, query *SanctionsInfo, index *SanctionsInfo) float64 { + if query == nil || index == nil { + return 0 + } + + // Compare programs + programScore := 0.0 + if len(query.Programs) > 0 && len(index.Programs) > 0 { + matches := 0 + for _, qp := range query.Programs { + for _, ip := range index.Programs { + if strings.EqualFold(qp, ip) { + matches++ + break + } + } + } + programScore = float64(matches) / float64(len(query.Programs)) + } + + // Adjust score based on secondary sanctions match + if query.Secondary != index.Secondary { + programScore *= 0.8 + } + + return programScore +} + +func compareHistoricalValues(queryHist, indexHist []HistoricalInfo) float64 { + bestScore := 0.0 + for _, qh := range queryHist { + for _, ih := range indexHist { + // Type must match exactly + if !strings.EqualFold(qh.Type, ih.Type) { + continue + } + + // Compare values + similarity := stringscore.JaroWinkler(qh.Value, ih.Value) + if similarity > bestScore { + bestScore = similarity + } + } + } + return bestScore +} diff --git a/pkg/search/similarity_ofac_test.go b/pkg/search/similarity_ofac_test.go index c93c54ae..da7a1c1d 100644 --- a/pkg/search/similarity_ofac_test.go +++ b/pkg/search/similarity_ofac_test.go @@ -2,250 +2,33 @@ package search_test import ( "bytes" + "context" "fmt" "io" + "path/filepath" "testing" "time" + "github.com/moov-io/watchman/internal/download" "github.com/moov-io/watchman/pkg/ofac" "github.com/moov-io/watchman/pkg/search" + "github.com/moov-io/base/log" "github.com/stretchr/testify/require" ) -func TestSimilarity_OFAC_SDN_Vessel(t *testing.T) { - baseSDN := ofac.SDN{ - EntityID: "123", - SDNName: "BLUE TANKER", - SDNType: "vessel", - Programs: []string{"SDGT", "IRGC"}, - CallSign: "BTANK123", - VesselType: "Cargo", - Tonnage: "15000", - GrossRegisteredTonnage: "18000", - VesselFlag: "PA", - VesselOwner: "GLOBAL SHIPPING CORP", - Remarks: "Known aliases: Sea Transporter, Ocean Carrier", - } - - // Create the base entity to match against - indexEntity := ofac.ToEntity(baseSDN, - []ofac.Address{ - { - AddressID: "addr1", - Address: "123 Harbor Drive", - CityStateProvincePostalCode: "Panama City 12345", - Country: "PA", - }, - }, - []ofac.SDNComments{}, - []ofac.AlternateIdentity{ - { - AlternateID: "alt1", - AlternateType: "Vessel Registration", - AlternateName: "REG123456", - }, - }, - ) - - testCases := []struct { - name string - query search.Entity[any] - expected float64 - }{ - { - name: "Exact match - All fields", - query: search.Entity[any]{ - Name: "BLUE TANKER", - Type: search.EntityVessel, - Vessel: &search.Vessel{ - Name: "BLUE TANKER", - Type: search.VesselTypeCargo, - Flag: "PA", - Tonnage: 15000, - CallSign: "BTANK123", - GrossRegisteredTonnage: 18000, - Owner: "GLOBAL SHIPPING CORP", - IMONumber: "IMO123456", - }, - Addresses: []search.Address{ - { - Line1: "123 Harbor Drive", - City: "Panama City", - Country: "PA", - PostalCode: "12345", - }, - }, - }, - expected: 1.0, - }, - { - name: "High confidence - Key fields match", - query: search.Entity[any]{ - Name: "BLUE TANKER", - Type: search.EntityVessel, - Vessel: &search.Vessel{ - Name: "BLUE TANKER", - Type: search.VesselTypeCargo, - IMONumber: "IMO123456", - CallSign: "BTANK123", - }, - }, - expected: 1.0, - }, - { - name: "Similar name with matching identifiers", - query: search.Entity[any]{ - Name: "Blue Tanker II", // Similar but not exact - Type: search.EntityVessel, - Vessel: &search.Vessel{ - Name: "Blue Tanker II", - CallSign: "BTANK123", // Exact match - IMONumber: "IMO123456", - Type: search.VesselTypeCargo, - Flag: "PA", - }, - }, - expected: 1.0, - }, - { - name: "Matching identifiers with different name", - query: search.Entity[any]{ - Name: "Sea Transporter", // Known alias - Type: search.EntityVessel, - Vessel: &search.Vessel{ - Name: "Sea Transporter", - CallSign: "BTANK123", - Type: search.VesselTypeCargo, - }, - }, - expected: 0.814, - }, - { - name: "Similar vessel details but key mismatch", - query: search.Entity[any]{ - Name: "BLUE TANKER", - Type: search.EntityVessel, - Vessel: &search.Vessel{ - Name: "BLUE TANKER", - Type: search.VesselTypeCargo, - Flag: "PA", - CallSign: "BTANK124", // One digit off - Tonnage: 14800, // Close but not exact - }, - }, - expected: 0.431, - }, - { - name: "Partial info with some matches", - query: search.Entity[any]{ - Name: "BLUE TANKER", - Type: search.EntityVessel, - Vessel: &search.Vessel{ - Name: "BLUE TANKER", - Flag: "PA", - Owner: "GLOBAL SHIPPING CORP", - }, - }, - expected: 1.0, - }, - { - name: "Different vessel with similar name", - query: search.Entity[any]{ - Name: "BLUE TANKER STAR", - Type: search.EntityVessel, - Vessel: &search.Vessel{ - Name: "BLUE TANKER STAR", - Type: search.VesselTypeCargo, - Flag: "SG", // Different flag - CallSign: "BSTAR789", - Owner: "STAR SHIPPING LTD", - }, - }, - expected: 0.36, - }, - { - name: "Complete mismatch", - query: search.Entity[any]{ - Name: "GOLDEN FREIGHTER", - Type: search.EntityVessel, - Vessel: &search.Vessel{ - Name: "GOLDEN FREIGHTER", - Type: search.VesselTypeCargo, - Flag: "LR", - CallSign: "GOLD999", - }, - }, - expected: 0.154, - }, - { - name: "Wrong entity type", - query: search.Entity[any]{ - Name: "BLUE TANKER", - Type: search.EntityBusiness, - Business: &search.Business{ - Name: "BLUE TANKER", - }, - }, - expected: 0.667, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - score := search.DebugSimilarity(debug(t), tc.query, indexEntity) - require.InDelta(t, tc.expected, score, 0.02) - - // Additional assertions for specific score thresholds - if tc.expected >= 0.95 { - require.GreaterOrEqual(t, score, 0.95, "High confidence matches should score >= 0.95") - } - if tc.expected <= 0.40 { - require.LessOrEqual(t, score, 0.40, "Clear mismatches should score <= 0.40") - } - }) - } -} - func TestSimilarity_OFAC_SDN_Person(t *testing.T) { - // Create a base SDN that represents a Person - baseSDN := ofac.SDN{ - EntityID: "999", - SDNName: "JOHN SMITH", - SDNType: "individual", // or "person", depending on your data - Title: "MR", - Remarks: "Some remarks about JOHN SMITH; Gender Male; DOB 05 Jan 1959", - } + indexEntity := findOFACEntity(t, "48603") - // For demonstration, we can embed a "DOB" note in Remarks or store it in an address, - // but your actual logic might parse or store it differently. We'll keep it simple. - sdnAddress := ofac.Address{ - EntityID: "999", - AddressID: "addr-person-1", - Address: "1234 MAIN ST", - CityStateProvincePostalCode: "LOS ANGELES CA 90001", - Country: "UNITED STATES", - AddressRemarks: "", // Example way to store - } - - // Alternate identity or name - altIdentity := ofac.AlternateIdentity{ - EntityID: "999", - AlternateID: "alt-person-1", - AlternateType: "fka", - AlternateName: "JONATHAN SMITH", - AlternateRemarks: "Formerly known as Jonathan", - } - - // Convert this SDN into your internal Entity - indexEntity := ofac.ToEntity(baseSDN, []ofac.Address{sdnAddress}, []ofac.SDNComments{}, []ofac.AlternateIdentity{altIdentity}) - - // For the Person-specific comparison logic to work (compare birth date, gender, titles, etc.), - // we’ll fill out the Person struct in the query. The watchman code’s Person comparison - // checks for exact birth date matches, gender, titles, etc. + // 48603,"KHOROSHEV, Dmitry Yuryevich","individual","CYBER2",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- , + // "DOB 17 Apr 1993; POB Russian Federation; nationality Russia; citizen Russia; Email Address khoroshev1@icloud.com; + // alt. Email Address sitedev5@yandex.ru; Gender Male; Digital Currency Address - XBT bc1qvhnfknw852ephxyc5hm4q520zmvf9maphetc9z; + // Secondary sanctions risk: Ukraine-/Russia-Related Sanctions Regulations, 31 CFR 589.201; Passport 2018278055 (Russia); + // alt. Passport 2006801524 (Russia); Tax ID No. 366110340670 (Russia); a.k.a. 'LOCKBITSUPP'." // Let's define a sample time to represent the person's birthday - dob := time.Date(1959, time.January, 5, 10, 32, 0, 0, time.UTC) + birthDate := time.Date(1993, time.April, 17, 0, 0, 0, 0, time.UTC) + laterBirthDate := birthDate.Add(30 * 24 * time.Hour) testCases := []struct { name string @@ -255,58 +38,60 @@ func TestSimilarity_OFAC_SDN_Person(t *testing.T) { { name: "Exact match - Person with all fields", query: search.Entity[any]{ - Name: "JOHN SMITH", + Name: "Dmitry Yuryevich KHOROSHEV", Type: search.EntityPerson, Person: &search.Person{ - BirthDate: &dob, - Gender: "male", - Titles: []string{"MR"}, + Name: "Dmitry Yuryevich KHOROSHEV", + BirthDate: &birthDate, + Gender: search.GenderMale, + }, + Contact: search.ContactInfo{ + EmailAddresses: []string{"khoroshev1@icloud.com"}, }, }, - expected: 1.0, + expected: 1.00, }, { name: "Partial match - Missing birthdate", query: search.Entity[any]{ - Name: "JOHN SMITH", + Name: "Dmitry Yuryevich KHOROSHEV", Type: search.EntityPerson, Person: &search.Person{ - // No BirthDate provided - Gender: "male", - Titles: []string{"MR"}, + Name: "Dmitry Yuryevich KHOROSHEV", + }, + Contact: search.ContactInfo{ + EmailAddresses: []string{"khoroshev1@icloud.com"}, }, }, - expected: 1.0, + expected: 0.98, + }, + { + name: "Name match only", + query: search.Entity[any]{ + Name: "Dmitry Yuryevich KHOROSHEV", + Type: search.EntityPerson, + }, + expected: 0.98, // TODO(adam): should be lower? }, { name: "Fuzzy name match - Alternate identity", query: search.Entity[any]{ - Name: "Jonathan Smith", + Name: "Dmitri Yuryevich", Type: search.EntityPerson, - Person: &search.Person{ - Titles: []string{"MR"}, - }, }, - // Expect a high score, but maybe slightly less than perfect - // if "Jonathan" vs. "John" is considered a close fuzzy match or if alt name is recognized. - expected: 0.667, + expected: 0.95, }, { name: "Close name but different person details", query: search.Entity[any]{ - Name: "JOHN SMYTH", // Slightly different spelling + Name: "Dmitri Yuryvich", Type: search.EntityPerson, Person: &search.Person{ - // Different birth date - BirthDate: func() *time.Time { - d := time.Date(1975, 1, 1, 0, 0, 0, 0, time.UTC) - return &d - }(), - Gender: "male", - Titles: []string{"MR"}, + BirthDate: &laterBirthDate, + Gender: search.GenderMale, }, }, - expected: 0.7938, + expected: 0.862, }, { name: "Mismatch - Wrong name and no matching details", @@ -317,7 +102,7 @@ func TestSimilarity_OFAC_SDN_Person(t *testing.T) { Gender: "F", }, }, - expected: 0.4559, + expected: 0.3110, }, { name: "Wrong entity type", @@ -325,7 +110,7 @@ func TestSimilarity_OFAC_SDN_Person(t *testing.T) { Name: "JOHN SMITH", Type: search.EntityVessel, // intentionally vessel to mismatch }, - expected: 0.667, + expected: 0.0, }, } @@ -338,53 +123,14 @@ func TestSimilarity_OFAC_SDN_Person(t *testing.T) { } func TestSimilarity_OFAC_SDN_Business(t *testing.T) { - // Create an OFAC SDN that represents a Business - baseSDN := ofac.SDN{ - EntityID: "B-101", - SDNName: "ACME Corporation", - SDNType: "", - Remarks: "Organization Established Date 15 Feb 2010; Registration Number 12345;", - // (Optionally add more fields, e.g. Programs, etc.) - } - - // We'll define an Address or AlternateIdentity if needed - sdnAddr := ofac.Address{ - EntityID: "B-101", - AddressID: "addr-bus-1", - Address: "1000 Industrial Way", - Country: "US", - // ... - } - - // For demonstration, a single AlternateIdentity for the business (like a DBA name) - altID := ofac.AlternateIdentity{ - EntityID: "B-101", - AlternateID: "alt-bus-1", - AlternateType: "DBA", - AlternateName: "ACME Co", - } + indexEntity := findOFACEntity(t, "50544") - // Convert this SDN into your internal Entity (which should set Type=Business, - // and fill out Business{Name, AltNames, etc.}) - // The details here depend on your ofac.ToEntity(...) implementation. - indexEntity := ofac.ToEntity(baseSDN, []ofac.Address{sdnAddr}, nil, []ofac.AlternateIdentity{altID}) + // 50544,"AUTONOMOUS NON-PROFIT ORGANIZATION DIALOG REGIONS",-0- ,"RUSSIA-EO14024",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- , + // "Website www.dialog.info; alt. Website www.dialog-regions.ru; Secondary sanctions risk: See Section 11 of Executive Order 14024.; + // Organization Established Date 21 Jul 2020; Tax ID No. 9709063550 (Russia); Business Registration Number 1207700248030 (Russia); + // a.k.a. 'DIALOGUE REGIONS'; a.k.a. 'DIALOG REGIONY'; a.k.a. 'DIALOGUE'; Linked To: AUTONOMOUS NON-PROFIT ORGANIZATION DIALOG." - // For this example, let's assume your ofac.ToEntity() sets: - // indexEntity.Type = search.EntityBusiness - // indexEntity.Business = &search.Business{ - // Name: "ACME Corporation", - // AltNames: []string{"ACME Co"}, - // Created: &time.Date(2010, time.February, 15, ...), - // Dissolved: nil, - // Identifier: []search.Identifier{ - // { Name: "Registration", Identifier: "12345", Country: "US" }, - // }, - // } - // - // Adjust your actual mapping logic accordingly. - - // We'll define a Created date to match the "Organization Established Date" data - businessCreatedAt := time.Date(2010, time.February, 15, 0, 0, 0, 0, time.UTC) + businessCreatedAt := time.Date(2020, time.July, 21, 12, 0, 0, 0, time.UTC) testCases := []struct { name string @@ -394,56 +140,50 @@ func TestSimilarity_OFAC_SDN_Business(t *testing.T) { { name: "Exact match - All fields", query: search.Entity[any]{ - Name: "ACME Corporation", + Name: "AUTONOMOUS NON-PROFIT ORGANIZATION DIALOG REGIONS", Type: search.EntityBusiness, Business: &search.Business{ - Name: "ACME Corporation", - AltNames: []string{"ACME Co"}, - Created: &businessCreatedAt, + Name: "AUTONOMOUS NON-PROFIT ORGANIZATION DIALOG REGIONS", + Created: &businessCreatedAt, Identifier: []search.Identifier{ { - Name: "Registration", - Country: "US", - Identifier: "12345", + Name: "Business Registration Number", + Country: "Russia", + Identifier: "1207700248030", }, }, }, }, - // If everything lines up exactly, we expect near 1.0 expected: 1.0, }, { name: "Partial match - Fuzzy name, same identifier", query: search.Entity[any]{ - Name: "ACME Corp", + Name: "AUTO NON-PROFIT ORGANIZATION", Type: search.EntityBusiness, Business: &search.Business{ - Name: "ACME Corp", // Fuzzy match to "ACME Corporation" + Name: "AUTO NON-PROFIT ORGANIZATION", Identifier: []search.Identifier{ { - Name: "Registration Number", - Identifier: "12345", // exact ID match + Name: "Tax ID No.", + Country: "Russia", + Identifier: "9709063550", }, }, }, }, - // If your fuzzy logic sees "ACME Inc." ~ "ACME Corporation" ~0.8 or so, - // plus an exact ID match, you might get something ~0.9 or 0.95 - expected: 0.8438, + expected: 0.9647, }, { name: "Alt name only + missing ID", query: search.Entity[any]{ - Name: "ACME Co", // matches alt name + Name: "DIALOGUE REGIONS", Type: search.EntityBusiness, Business: &search.Business{ - Name: "ACME Co", - // No identifier + Name: "DIALOGUE REGIONS", }, }, - // Expect a moderate score (fuzzy match or alt name success), - // but not 1.0 since the ID is missing, or other fields not matched. - expected: 0.7, + expected: 0.9555, }, { name: "Different name, different ID", @@ -457,8 +197,7 @@ func TestSimilarity_OFAC_SDN_Business(t *testing.T) { }, }, }, - // Expect a low score for mismatch - expected: 0.3347, + expected: 0.0954, }, { name: "Wrong entity type", @@ -466,8 +205,7 @@ func TestSimilarity_OFAC_SDN_Business(t *testing.T) { Name: "ACME Corporation", Type: search.EntityVessel, }, - // Mismatched types typically yield a near-zero or a partial coverage score - expected: 0.6, + expected: 0.0, }, } @@ -479,6 +217,118 @@ func TestSimilarity_OFAC_SDN_Business(t *testing.T) { } } +func TestSimilarity_OFAC_SDN_Vessel(t *testing.T) { + t.Run("47371", func(t *testing.T) { + indexEntity := findOFACEntity(t, "47371") + + // 47371,"NS LEADER","vessel","RUSSIA-EO14024",-0- ,"A8LU7","Crude Oil Tanker",-0- ,-0- ,"Gabon",-0- , + // "Secondary sanctions risk: See Section 11 of Executive Order 14024.; Identification Number IMO 9339301; + // MMSI 636013272; Linked To: NS LEADER SHIPPING INCORPORATED." + + testCases := []struct { + name string + query search.Entity[any] + expected float64 + }{ + { + name: "Exact match - All fields", + query: search.Entity[any]{ + Name: "NS LEADER", + Type: search.EntityVessel, + Vessel: &search.Vessel{ + CallSign: "A8LU7", + MMSI: "636013272", + IMONumber: "9339301", + }, + }, + expected: 1.0, + }, + { + name: "High confidence - Key fields match", + query: search.Entity[any]{ + Name: "NS LEADER", + Type: search.EntityVessel, + Vessel: &search.Vessel{ + CallSign: "A8LU7", + }, + }, + expected: 1.0, + }, + { + name: "Similar name with matching identifiers", + query: search.Entity[any]{ + Name: "NS LEADER II", // Similar but not exact + Type: search.EntityVessel, + Vessel: &search.Vessel{ + MMSI: "636013272", + }, + }, + expected: 1.0, + }, + { + name: "Matching identifiers with different name", + query: search.Entity[any]{ + Name: "Sea Transporter", + Type: search.EntityVessel, + Vessel: &search.Vessel{ + IMONumber: "9339301", + }, + }, + expected: 1.0, + }, + { + name: "Similar vessel details but key mismatch", + query: search.Entity[any]{ + Name: "NS LEADER", + Type: search.EntityVessel, + Vessel: &search.Vessel{ + Flag: "Iran", // wrong + CallSign: "BTANK124", // other callsign + }, + }, + expected: 0.431, + }, + { + name: "Complete mismatch", + query: search.Entity[any]{ + Name: "GOLDEN FREIGHTER", + Type: search.EntityVessel, + Vessel: &search.Vessel{ + Name: "GOLDEN FREIGHTER", + Type: search.VesselTypeCargo, + Flag: "LR", + CallSign: "GOLD999", + }, + }, + expected: 0.2104, + }, + { + name: "Wrong entity type", + query: search.Entity[any]{ + Name: "BLUE TANKER", + Type: search.EntityBusiness, + }, + expected: 0.0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + score := search.DebugSimilarity(debug(t), tc.query, indexEntity) + require.InDelta(t, tc.expected, score, 0.02) + + // Additional assertions for specific score thresholds + if tc.expected >= 0.95 { + require.GreaterOrEqual(t, score, 0.95, "High confidence matches should score >= 0.95") + } + if tc.expected <= 0.40 { + require.LessOrEqual(t, score, 0.40, "Clear mismatches should score <= 0.40") + } + }) + } + }) +} + func TestSimilarity_Edge_Cases(t *testing.T) { baseSDN := ofac.SDN{ EntityID: "123", @@ -501,8 +351,9 @@ func TestSimilarity_Edge_Cases(t *testing.T) { name: "Name only", query: search.Entity[any]{ Name: "TEST ENTITY", + Type: search.EntityVessel, }, - expected: 0.667, + expected: 1.0, }, { name: "Mismatched types", @@ -510,7 +361,7 @@ func TestSimilarity_Edge_Cases(t *testing.T) { Name: "TEST ENTITY", Type: search.EntityBusiness, }, - expected: 0.667, + expected: 0.0, }, } @@ -522,6 +373,30 @@ func TestSimilarity_Edge_Cases(t *testing.T) { } } +func findOFACEntity(tb testing.TB, entityID string) search.Entity[search.Value] { + tb.Helper() + + logger := log.NewTestLogger() + conf := download.Config{ + InitialDataDirectory: filepath.Join("..", "ofac", "testdata"), + } + dl, err := download.NewDownloader(logger, conf) + require.NoError(tb, err) + + stats, err := dl.RefreshAll(context.Background()) + require.NoError(tb, err) + + for _, entity := range stats.Entities { + if entityID == entity.SourceID { + return entity + } + } + + tb.Fatalf("OFAC entity %s not found", entityID) + + return search.Entity[search.Value]{} +} + func debug(t *testing.T) io.Writer { t.Helper() From 15faa22de6ef1604f3343bbbb53b721f0772bbfd Mon Sep 17 00:00:00 2001 From: Adam Shannon Date: Fri, 10 Jan 2025 15:31:21 -0600 Subject: [PATCH 8/9] meta: gofmt --- internal/ui/search.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/ui/search.go b/internal/ui/search.go index 4bca4e00..40b182c8 100644 --- a/internal/ui/search.go +++ b/internal/ui/search.go @@ -102,8 +102,8 @@ func showResults(env Environment, results *fyne.Container, entities []search.Sea results.Add(header) var data = [][]string{ - []string{"top left", "top right"}, - []string{"bottom left", "bottom right"}, + {"top left", "top right"}, + {"bottom left", "bottom right"}, } for _, row := range data { From 9dd017036a1149560de8fe2be55b3db0af3650cf Mon Sep 17 00:00:00 2001 From: Adam Shannon Date: Fri, 10 Jan 2025 16:08:53 -0600 Subject: [PATCH 9/9] search: add fuzzy similarity tests --- pkg/search/similarity_fuzzy.go | 2 + pkg/search/similarity_fuzzy_test.go | 761 ++++++++++++++++++++++++++++ 2 files changed, 763 insertions(+) create mode 100644 pkg/search/similarity_fuzzy_test.go diff --git a/pkg/search/similarity_fuzzy.go b/pkg/search/similarity_fuzzy.go index c72014b3..692620dd 100644 --- a/pkg/search/similarity_fuzzy.go +++ b/pkg/search/similarity_fuzzy.go @@ -143,6 +143,8 @@ func filterSignificantTerms(terms []string) []string { } // Skip common noise terms (expand this list as needed) + term = strings.TrimSpace(strings.ToLower(term)) + switch term { case "the", "and", "or", "of", "in", "at", "by": continue diff --git a/pkg/search/similarity_fuzzy_test.go b/pkg/search/similarity_fuzzy_test.go new file mode 100644 index 00000000..14bf4e41 --- /dev/null +++ b/pkg/search/similarity_fuzzy_test.go @@ -0,0 +1,761 @@ +package search + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCompareName(t *testing.T) { + var buf bytes.Buffer + + tests := []struct { + name string + query Entity[any] + index Entity[any] + expectedScore float64 + shouldMatch bool + exact bool + }{ + { + name: "exact match", + query: Entity[any]{ + Name: "AEROCARIBBEAN AIRLINES", + }, + index: Entity[any]{ + Name: "AEROCARIBBEAN AIRLINES", + }, + expectedScore: 1.0, + shouldMatch: true, + exact: true, + }, + { + name: "case insensitive match", + query: Entity[any]{ + Name: "aerocaribbean airlines", + }, + index: Entity[any]{ + Name: "AEROCARIBBEAN AIRLINES", + }, + expectedScore: 1.0, + shouldMatch: true, + exact: true, + }, + { + name: "punctuation differences", + query: Entity[any]{ + Name: "ANGLO CARIBBEAN CO LTD", + }, + index: Entity[any]{ + Name: "ANGLO-CARIBBEAN CO., LTD.", + }, + expectedScore: 1.0, + shouldMatch: true, + exact: true, + }, + { + name: "slight misspelling", + query: Entity[any]{ + Name: "AEROCARRIBEAN AIRLINES", + }, + index: Entity[any]{ + Name: "AEROCARIBBEAN AIRLINES", + }, + expectedScore: 0.95, + shouldMatch: true, + exact: false, + }, + { + name: "word reordering", + query: Entity[any]{ + Name: "AIRLINES AEROCARIBBEAN", + }, + index: Entity[any]{ + Name: "AEROCARIBBEAN AIRLINES", + }, + expectedScore: 0.90, + shouldMatch: true, + exact: true, + }, + { + name: "extra words in query", + query: Entity[any]{ + Name: "THE AEROCARIBBEAN AIRLINES COMPANY", + }, + index: Entity[any]{ + Name: "AEROCARIBBEAN AIRLINES", + }, + expectedScore: 0.85, + shouldMatch: false, + exact: false, + }, + { + name: "historical name match", + query: Entity[any]{ + Name: "OLD AEROCARIBBEAN", + }, + index: Entity[any]{ + Name: "AEROCARIBBEAN AIRLINES", + HistoricalInfo: []HistoricalInfo{ + { + Type: "Former Name", + Value: "OLD AEROCARIBBEAN", + }, + }, + }, + expectedScore: 0.90, + shouldMatch: true, + exact: false, + }, + { + name: "alternative name match for person", + query: Entity[any]{ + Name: "JOHN MICHAEL SMITH", + Type: EntityPerson, + Person: &Person{ + Name: "JOHN MICHAEL SMITH", + }, + }, + index: Entity[any]{ + Name: "JOHN SMITH", + Type: EntityPerson, + Person: &Person{ + Name: "JOHN SMITH", + AltNames: []string{"JOHN MICHAEL SMITH", "J.M. SMITH"}, + }, + }, + expectedScore: 0.95, + shouldMatch: true, + exact: true, + }, + { + name: "minimum term matches", + query: Entity[any]{ + Name: "CARIBBEAN TRADING LIMITED", + }, + index: Entity[any]{ + Name: "PACIFIC TRADING LIMITED", + }, + expectedScore: 0.8628, + shouldMatch: true, + exact: false, + }, + { + name: "completely different names", + query: Entity[any]{ + Name: "AEROCARIBBEAN AIRLINES", + }, + index: Entity[any]{ + Name: "BANCO NACIONAL DE CUBA", + }, + expectedScore: 0.4479, + shouldMatch: false, + exact: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := compareName(&buf, tt.query, tt.index, 1.0) + + assert.InDelta(t, tt.expectedScore, result.score, 0.1, + "expected score %v but got %v", tt.expectedScore, result.score) + assert.Equal(t, tt.shouldMatch, result.matched, + "expected matched=%v but got matched=%v", tt.shouldMatch, result.matched) + assert.Equal(t, tt.exact, result.exact, + "expected exact=%v but got exact=%v", tt.exact, result.exact) + }) + } +} + +func TestCompareEntityTitlesFuzzy(t *testing.T) { + var buf bytes.Buffer + + tests := []struct { + name string + query Entity[any] + index Entity[any] + expectedScore float64 + shouldMatch bool + exact bool + }{ + { + name: "exact title match", + query: Entity[any]{ + Titles: []string{"Chief Executive Officer"}, + }, + index: Entity[any]{ + Titles: []string{"Chief Executive Officer"}, + }, + expectedScore: 1.0, + shouldMatch: true, + exact: true, + }, + { + name: "abbreviated title match", + query: Entity[any]{ + Titles: []string{"CEO"}, + }, + index: Entity[any]{ + Titles: []string{"Chief Executive Officer"}, + }, + expectedScore: 0.0, // TODO(adam): needs fixed + shouldMatch: false, + exact: false, + }, + { + name: "multiple titles with partial matches", + query: Entity[any]{ + Titles: []string{"CEO", "Director of Operations"}, + }, + index: Entity[any]{ + Titles: []string{"Chief Executive Officer", "Operations Director"}, + }, + expectedScore: 0.50, + shouldMatch: false, + exact: false, + }, + { + name: "similar but not exact titles", + query: Entity[any]{ + Titles: []string{"Senior Technical Manager"}, + }, + index: Entity[any]{ + Titles: []string{"Technical Manager"}, + }, + expectedScore: 0.0, // TODO(adam): needs fixed + shouldMatch: false, + exact: false, + }, + { + name: "no matching titles", + query: Entity[any]{ + Titles: []string{"Chief Financial Officer"}, + }, + index: Entity[any]{ + Titles: []string{"Sales Director", "Regional Manager"}, + }, + expectedScore: 0.0, + shouldMatch: false, + exact: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := compareEntityTitlesFuzzy(&buf, tt.query, tt.index, 1.0) + + assert.InDelta(t, tt.expectedScore, result.score, 0.1, + "expected score %v but got %v", tt.expectedScore, result.score) + assert.Equal(t, tt.shouldMatch, result.matched, + "expected matched=%v but got matched=%v", tt.shouldMatch, result.matched) + assert.Equal(t, tt.exact, result.exact, + "expected exact=%v but got exact=%v", tt.exact, result.exact) + }) + } +} + +func TestCompareAffiliationsFuzzy(t *testing.T) { + var buf bytes.Buffer + + tests := []struct { + name string + query Entity[any] + index Entity[any] + expectedScore float64 + shouldMatch bool + exact bool + }{ + { + name: "exact affiliation match", + query: Entity[any]{ + Affiliations: []Affiliation{ + { + EntityName: "BANCO NACIONAL DE CUBA", + Type: "subsidiary of", + }, + }, + }, + index: Entity[any]{ + Affiliations: []Affiliation{ + { + EntityName: "BANCO NACIONAL DE CUBA", + Type: "subsidiary of", + }, + }, + }, + expectedScore: 1.0, + shouldMatch: true, + exact: true, + }, + { + name: "similar affiliation with related type", + query: Entity[any]{ + Affiliations: []Affiliation{ + { + EntityName: "Banco Nacional Cuba", + Type: "owned by", + }, + }, + }, + index: Entity[any]{ + Affiliations: []Affiliation{ + { + EntityName: "BANCO NACIONAL DE CUBA", + Type: "subsidiary of", + }, + }, + }, + expectedScore: 0.90, + shouldMatch: true, + exact: true, + }, + { + name: "multiple affiliations with partial matches", + query: Entity[any]{ + Affiliations: []Affiliation{ + { + EntityName: "CARIBBEAN TRADING CO", + Type: "linked to", + }, + { + EntityName: "BANCO CUBA", + Type: "subsidiary of", + }, + }, + }, + index: Entity[any]{ + Affiliations: []Affiliation{ + { + EntityName: "CARIBBEAN TRADING COMPANY", + Type: "associated with", + }, + { + EntityName: "BANCO NACIONAL DE CUBA", + Type: "parent company", + }, + }, + }, + expectedScore: 1.0, + shouldMatch: true, + exact: true, + }, + { + name: "no matching affiliations", + query: Entity[any]{ + Affiliations: []Affiliation{ + { + EntityName: "ACME CORPORATION", + Type: "owned by", + }, + }, + }, + index: Entity[any]{ + Affiliations: []Affiliation{ + { + EntityName: "BANCO NACIONAL DE CUBA", + Type: "subsidiary of", + }, + }, + }, + expectedScore: 0.3956, + shouldMatch: false, + exact: false, + }, + { + name: "empty affiliations", + query: Entity[any]{ + Affiliations: []Affiliation{}, + }, + index: Entity[any]{ + Affiliations: []Affiliation{ + { + EntityName: "BANCO NACIONAL DE CUBA", + Type: "subsidiary of", + }, + }, + }, + expectedScore: 0.0, + shouldMatch: false, + exact: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := compareAffiliationsFuzzy(&buf, tt.query, tt.index, 1.0) + + assert.InDelta(t, tt.expectedScore, result.score, 0.1, + "expected score %v but got %v", tt.expectedScore, result.score) + assert.Equal(t, tt.shouldMatch, result.matched, + "expected matched=%v but got matched=%v", tt.shouldMatch, result.matched) + assert.Equal(t, tt.exact, result.exact, + "expected exact=%v but got exact=%v", tt.exact, result.exact) + }) + } +} + +// Helper function tests + +func TestNormalizeName(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "standard name", + input: "AEROCARIBBEAN AIRLINES", + expected: "aerocaribbean airlines", + }, + { + name: "name with punctuation", + input: "ANGLO-CARIBBEAN CO., LTD.", + expected: "anglo caribbean co ltd", + }, + { + name: "extra whitespace", + input: " BANCO NACIONAL DE CUBA ", + expected: "banco nacional de cuba", + }, + { + name: "mixed case with special chars", + input: "Banco.Nacional_de@Cuba", + expected: "banco nacional de cuba", + }, + { + name: "empty string", + input: "", + expected: "", + }, + { + name: "only special chars", + input: ".,!@#$%^&*()", + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := normalizeName(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestNormalizeTitle(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "standard title", + input: "Chief Executive Officer", + expected: "chief executive officer", + }, + { + name: "title with punctuation", + input: "Sr. Vice-President, Operations", + expected: "sr vice-president operations", + }, + { + name: "abbreviated title", + input: "CEO & CFO", + expected: "ceo cfo", + }, + { + name: "extra whitespace", + input: " Senior Technical Manager ", + expected: "senior technical manager", + }, + { + name: "empty string", + input: "", + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := normalizeTitle(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestCalculateNameScore(t *testing.T) { + tests := []struct { + name string + queryName string + indexName string + expectedScore float64 + }{ + { + name: "exact match", + queryName: "banco nacional de cuba", + indexName: "banco nacional de cuba", + expectedScore: 1.0, + }, + { + name: "close match", + queryName: "banco nacional cuba", + indexName: "banco nacional de cuba", + expectedScore: 0.95, + }, + { + name: "partial match", + queryName: "banco cuba", + indexName: "banco nacional de cuba", + expectedScore: 0.9210, + }, + { + name: "word reordering", + queryName: "nacional banco cuba", + indexName: "banco nacional de cuba", + expectedScore: 0.9842, + }, + { + name: "completely different", + queryName: "aerocaribbean airlines", + indexName: "banco nacional de cuba", + expectedScore: 0.0, + }, + { + name: "empty query", + queryName: "", + indexName: "banco nacional de cuba", + expectedScore: 0.0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + score := calculateNameScore(tt.queryName, tt.indexName) + assert.InDelta(t, tt.expectedScore, score, 0.1) + }) + } +} + +func TestCalculateTypeScore(t *testing.T) { + tests := []struct { + name string + queryType string + indexType string + expectedScore float64 + }{ + { + name: "exact match", + queryType: "owned by", + indexType: "owned by", + expectedScore: 1.0, + }, + { + name: "same group - ownership", + queryType: "owned by", + indexType: "subsidiary of", + expectedScore: 0.8, + }, + { + name: "same group - control", + queryType: "controlled by", + indexType: "operates", + expectedScore: 0.8, + }, + { + name: "same group - association", + queryType: "linked to", + indexType: "associated with", + expectedScore: 0.8, + }, + { + name: "same group - leadership", + queryType: "led by", + indexType: "headed by", + expectedScore: 0.8, + }, + { + name: "different groups", + queryType: "owned by", + indexType: "linked to", + expectedScore: 0.0, + }, + { + name: "case insensitive", + queryType: "OWNED BY", + indexType: "owned by", + expectedScore: 1.0, + }, + { + name: "with extra spaces", + queryType: " owned by ", + indexType: "owned by", + expectedScore: 0.8, + }, + { + name: "unknown types", + queryType: "unknown relation", + indexType: "other relation", + expectedScore: 0.0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + score := calculateTypeScore(tt.queryType, tt.indexType) + assert.InDelta(t, tt.expectedScore, score, 0.1) + }) + } +} + +func TestFilterSignificantTerms(t *testing.T) { + tests := []struct { + name string + input []string + expected []string + }{ + { + name: "all significant terms", + input: []string{"banco", "nacional", "cuba"}, + expected: []string{"banco", "nacional", "cuba"}, + }, + { + name: "with noise terms", + input: []string{"the", "banco", "of", "nacional", "and", "cuba"}, + expected: []string{"banco", "nacional", "cuba"}, + }, + { + name: "with short terms", + input: []string{"al", "banco", "de", "nacional"}, + expected: []string{"banco", "nacional"}, + }, + { + name: "only noise terms", + input: []string{"the", "of", "and", "in", "at"}, + expected: []string{}, + }, + { + name: "empty input", + input: []string{}, + expected: []string{}, + }, + { + name: "mixed case terms", + input: []string{"THE", "Banco", "OF", "Nacional"}, + expected: []string{"banco", "nacional"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := filterSignificantTerms(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestCalculateCombinedScore(t *testing.T) { + tests := []struct { + name string + nameScore float64 + typeScore float64 + expectedScore float64 + }{ + { + name: "perfect match", + nameScore: 1.0, + typeScore: 1.0, + expectedScore: 1.0, + }, + { + name: "high name score with exact type", + nameScore: 0.9, + typeScore: 1.0, + expectedScore: 1.0, // With exactTypeBonus but capped at 1.0 + }, + { + name: "high name score with related type", + nameScore: 0.9, + typeScore: 0.8, + expectedScore: 0.98, // With relatedTypeBonues + }, + { + name: "high name score with mismatched type", + nameScore: 0.9, + typeScore: 0.0, + expectedScore: 0.75, // With typeMatchPenalty + }, + { + name: "low scores", + nameScore: 0.3, + typeScore: 0.0, + expectedScore: 0.15, // With typeMatchPenalty + }, + { + name: "zero name score", + nameScore: 0.0, + typeScore: 1.0, + expectedScore: 0.15, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + score := calculateCombinedScore(tt.nameScore, tt.typeScore) + assert.InDelta(t, tt.expectedScore, score, 0.1) + }) + } +} + +func TestCalculateFinalAffiliateScore(t *testing.T) { + tests := []struct { + name string + matches []affiliationMatch + expectedScore float64 + }{ + { + name: "single perfect match", + matches: []affiliationMatch{ + {nameScore: 1.0, typeScore: 1.0, finalScore: 1.0, exactMatch: true}, + }, + expectedScore: 1.0, + }, + { + name: "multiple high quality matches", + matches: []affiliationMatch{ + {nameScore: 0.95, typeScore: 1.0, finalScore: 0.95, exactMatch: false}, + {nameScore: 0.90, typeScore: 0.8, finalScore: 0.90, exactMatch: false}, + }, + expectedScore: 0.93, + }, + { + name: "mixed quality matches", + matches: []affiliationMatch{ + {nameScore: 0.95, typeScore: 1.0, finalScore: 0.95, exactMatch: false}, + {nameScore: 0.50, typeScore: 0.0, finalScore: 0.35, exactMatch: false}, + }, + expectedScore: 0.85, + }, + { + name: "no matches", + matches: []affiliationMatch{}, + expectedScore: 0.0, + }, + { + name: "all low quality matches", + matches: []affiliationMatch{ + {nameScore: 0.3, typeScore: 0.0, finalScore: 0.15, exactMatch: false}, + {nameScore: 0.4, typeScore: 0.0, finalScore: 0.25, exactMatch: false}, + }, + expectedScore: 0.21, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + score := calculateFinalAffiliateScore(tt.matches) + assert.InDelta(t, tt.expectedScore, score, 0.1) + }) + } +}