Skip to content

Commit

Permalink
test: refresh testing for v2 code
Browse files Browse the repository at this point in the history
  • Loading branch information
@adamdecaf
adamdecaf committed Jan 13, 2025
1 parent 0cffbf6 commit 14c3d73
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 125 deletions.
7 changes: 6 additions & 1 deletion cmd/server/download.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
package main

import (
"cmp"
"context"
"os"
"strings"
Expand Down Expand Up @@ -45,6 +46,10 @@ func setupPeriodicRefreshing(ctx context.Context, logger log.Logger, conf downlo
return <-errs
}

const (
defaultRefreshInterval = 12 * time.Hour
)

func getRefreshInterval(conf download.Config) time.Duration {
override := strings.TrimSpace(os.Getenv("DATA_REFRESH_INTERVAL"))
if override != "" {
Expand All @@ -53,7 +58,7 @@ func getRefreshInterval(conf download.Config) time.Duration {
return dur
}
}
return conf.RefreshInterval
return cmp.Or(conf.RefreshInterval, defaultRefreshInterval)
}

func refreshAllSources(ctx context.Context, logger log.Logger, downloader download.Downloader, searchService search.Service) error {
Expand Down
144 changes: 20 additions & 124 deletions cmd/server/download_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,16 @@

package main

// import (
// "bytes"
// "encoding/json"
// "errors"
// "os"
// "path/filepath"
// "slices"
// "testing"
// "time"

// "github.com/moov-io/base/log"
// "github.com/moov-io/watchman/pkg/ofac"

import (
"context"
"path/filepath"
"testing"
"time"

"github.com/moov-io/watchman/internal/download"
"github.com/moov-io/watchman/internal/search"

"github.com/moov-io/base/log"
"github.com/stretchr/testify/require"
)

Expand All @@ -39,119 +30,24 @@ func TestGetRefreshInterval(t *testing.T) {
require.Equal(t, 1*time.Hour, got)
}

// func TestDownloadStats(t *testing.T) {
// when := time.Date(2022, time.May, 21, 9, 4, 0, 0, time.UTC)
// bs, err := json.Marshal(&DownloadStats{
// SDNs: 1,
// Errors: []error{
// errors.New("bad thing"),
// },
// RefreshedAt: when,
// })
// require.NoError(t, err)

// var wrapper struct {
// SDNs int
// Errors []string
// Timestamp time.Time
// }
// err = json.NewDecoder(bytes.NewReader(bs)).Decode(&wrapper)
// require.NoError(t, err)

// require.Equal(t, 1, wrapper.SDNs)
// require.Len(t, wrapper.Errors, 1)
// require.Equal(t, when, wrapper.Timestamp)
// }

// func TestSearcher__refreshInterval(t *testing.T) {
// if v := getDataRefreshInterval(log.NewNopLogger(), ""); v.String() != "12h0m0s" {
// t.Errorf("Got %v", v)
// }
// if v := getDataRefreshInterval(log.NewNopLogger(), "60s"); v.String() != "1m0s" {
// t.Errorf("Got %v", v)
// }
// if v := getDataRefreshInterval(log.NewNopLogger(), "off"); v != 0*time.Second {
// t.Errorf("got %v", v)
// }

// // cover another branch
// s := newSearcher(log.NewNopLogger(), noLogPipeliner, 1)
// s.periodicDataRefresh(0*time.Second, nil)
// }

// func TestSearcher__refreshData(t *testing.T) {
// s := createTestSearcher(t) // TODO(adam): initial setup
// stats := testSearcherStats

// if len(s.Addresses) == 0 || stats.Addresses == 0 {
// t.Errorf("empty Addresses=%d stats.Addresses=%d", len(s.Addresses), stats.Addresses)
// }
// if len(s.Alts) == 0 || stats.Alts == 0 {
// t.Errorf("empty Alts=%d or stats.Alts=%d", len(s.Alts), stats.Alts)
// }
// if len(s.SDNs) == 0 || stats.SDNs == 0 {
// t.Errorf("empty SDNs=%d or stats.SDNs=%d", len(s.SDNs), stats.SDNs)
// }
// if len(s.DPs) == 0 || stats.DeniedPersons == 0 {
// t.Errorf("empty DPs=%d or stats.DeniedPersons=%d", len(s.DPs), stats.DeniedPersons)
// }
// if len(s.SSIs) == 0 || stats.SectoralSanctions == 0 {
// t.Errorf("empty SSIs=%d or stats.SectoralSanctions=%d", len(s.SSIs), stats.SectoralSanctions)
// }
// if len(s.BISEntities) == 0 || stats.BISEntities == 0 {
// t.Errorf("empty searcher.BISEntities=%d or stats.BISEntities=%d", len(s.BISEntities), stats.BISEntities)
// }
// }
func TestDownloader_setupPeriodicRefreshing(t *testing.T) {
ctx, cancelFunc := context.WithCancel(context.Background())
logger := log.NewTestLogger()

// func TestDownload__lastRefresh(t *testing.T) {
// start := time.Now()
// time.Sleep(5 * time.Millisecond) // force start to be before our calls

// if when := lastRefresh(""); when.Before(start) {
// t.Errorf("expected time.Now()")
// }

// // make a temp dir (initially with nothing in it)
// dir, err := os.MkdirTemp("", "lastRefresh")
// if err != nil {
// t.Fatal(err)
// }

// if when := lastRefresh(dir); !when.IsZero() {
// t.Errorf("expected zero time: %v", t)
// }

// // add a file and get it's mtime
// path := filepath.Join(dir, "out.txt")
// if err := os.WriteFile(path, []byte("hello, world"), 0600); err != nil {
// t.Fatal(err)
// }
// if info, err := os.Stat(path); err != nil {
// t.Fatal(err)
// } else {
// if when := lastRefresh(dir); !when.Equal(info.ModTime()) {
// t.Errorf("t=%v", when)
// }
// }
// }
conf := download.Config{
InitialDataDirectory: filepath.Join("..", "..", "pkg", "ofac", "testdata"),
}

// func TestDownload__OFAC_Spillover(t *testing.T) {
// logger := log.NewTestLogger()
// initialDir := filepath.Join("..", "..", "test", "testdata", "static")
dl, err := download.NewDownloader(logger, conf)
require.NoError(t, err)

// res, err := ofacRecords(logger, initialDir)
// require.NoError(t, err)
searchService := search.NewService(logger)

// var sdn *ofac.SDN
// idx := slices.IndexFunc(res.SDNs, func(s *ofac.SDN) bool {
// return s.EntityID == "12300"
// })
// if idx >= 0 {
// sdn = res.SDNs[idx]
// }
// require.NotNil(t, sdn)
go func() {
time.Sleep(500 * time.Millisecond)
cancelFunc()
}()

// //nolint:misspell
// expected := `DOB 13 May 1965; alt. DOB 13 Apr 1968; alt. DOB 07 Jul 1964; POB Medellin, Colombia; alt. POB Marinilla, Antioquia, Colombia; alt. POB Ciudad Victoria, Tamaulipas, Mexico; Cedula No. 7548733 (Colombia); alt. Cedula No. 70163752 (Colombia); alt. Cedula No. 172489729-1 (Ecuador); Passport AL720622 (Colombia); R.F.C. CIVJ650513LJA (Mexico); alt. R.F.C. OUSV-640707 (Mexico); C.U.R.P. CIVJ650513HNEFLR06 (Mexico); alt. C.U.R.P. OUVS640707HTSSLR07 (Mexico); Matricula Mercantil No 181301-1 Cali (Colombia); alt. Matricula Mercantil No 405885 Bogota (Colombia); Linked To: BIO FORESTAL S.A.S.; Linked To: CUBI CAFE CLICK CUBE MEXICO, S.A. DE C.V.; Linked To: DOLPHIN DIVE SCHOOL S.A.; Linked To: GANADERIA LA SORGUITA S.A.S.; Linked To: GESTORES DEL ECUADOR GESTORUM S.A.; Linked To: INVERPUNTO DEL VALLE S.A.; Linked To: INVERSIONES CIFUENTES Y CIA. S. EN C.; Linked To: LE CLAUDE, S.A. DE C.V.; Linked To: OPERADORA NUEVA GRANADA, S.A. DE C.V.; Linked To: PARQUES TEMATICOS S.A.S.; Linked To: PROMO RAIZ S.A.S.; Linked To: RED MUNDIAL INMOBILIARIA, S.A. DE C.V.; Linked To: FUNDACION PARA EL BIENESTAR Y EL PORVENIR; Linked To: C.I. METALURGIA EXTRACTIVA DE COLOMBIA S.A.S.; Linked To: GRUPO MUNDO MARINO, S.A.; Linked To: C.I. DISERCOM S.A.S.; Linked To: C.I. OKCOFFEE COLOMBIA S.A.S.; Linked To: C.I. OKCOFFEE INTERNATIONAL S.A.S.; Linked To: FUNDACION OKCOFFEE COLOMBIA; Linked To: CUBICAFE S.A.S.; Linked To: HOTELES Y BIENES S.A.; Linked To: FUNDACION SALVA LA SELVA; Linked To: LINEA AEREA PUEBLOS AMAZONICOS S.A.S.; Linked To: DESARROLLO MINERO RESPONSABLE C.I. S.A.S.; Linked To: R D I S.A.`
// require.Equal(t, expected, sdn.Remarks)
// }
err = setupPeriodicRefreshing(ctx, logger, conf, dl, searchService)
require.NoError(t, err)
}
26 changes: 26 additions & 0 deletions pkg/ofac/reader_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@
package ofac

import (
"context"
"io"
"os"
"path/filepath"
"reflect"
"testing"

"github.com/moov-io/base/log"
"github.com/stretchr/testify/require"
)

Expand Down Expand Up @@ -187,3 +189,27 @@ func TestSDNComments_CryptoCurrencies(t *testing.T) {
}
require.ElementsMatch(t, expected, addresses)
}

func TestDownload__OFAC_Spillover(t *testing.T) {
ctx := context.Background()
logger := log.NewTestLogger()
initialDir := filepath.Join("..", "..", "test", "testdata", "static")

files, err := Download(ctx, logger, initialDir)
require.NoError(t, err)

res, err := Read(files)
require.NoError(t, err)

var found *SDN
for _, sdn := range res.SDNs {
if sdn.EntityID == "12300" {
found = &sdn
break
}
}
require.NotNil(t, found)

expected := `DOB 13 May 1965; alt. DOB 13 Apr 1968; alt. DOB 07 Jul 1964; POB Medellin, Colombia; alt. POB Marinilla, Antioquia, Colombia; alt. POB Ciudad Victoria, Tamaulipas, Mexico; Cedula No. 7548733 (Colombia); alt. Cedula No. 70163752 (Colombia); alt. Cedula No. 172489729-1 (Ecuador); Passport AL720622 (Colombia); R.F.C. CIVJ650513LJA (Mexico); alt. R.F.C. OUSV-640707 (Mexico); C.U.R.P. CIVJ650513HNEFLR06 (Mexico); alt. C.U.R.P. OUVS640707HTSSLR07 (Mexico); Matricula Mercantil No 181301-1 Cali (Colombia); alt. Matricula Mercantil No 405885 Bogota (Colombia); Linked To: BIO FORESTAL S.A.S.; Linked To: CUBI CAFE CLICK CUBE MEXICO, S.A. DE C.V.; Linked To: DOLPHIN DIVE SCHOOL S.A.; Linked To: GANADERIA LA SORGUITA S.A.S.; Linked To: GESTORES DEL ECUADOR GESTORUM S.A.; Linked To: INVERPUNTO DEL VALLE S.A.; Linked To: INVERSIONES CIFUENTES Y CIA. S. EN C.; Linked To: LE CLAUDE, S.A. DE C.V.; Linked To: OPERADORA NUEVA GRANADA, S.A. DE C.V.; Linked To: PARQUES TEMATICOS S.A.S.; Linked To: PROMO RAIZ S.A.S.; Linked To: RED MUNDIAL INMOBILIARIA, S.A. DE C.V.; Linked To: FUNDACION PARA EL BIENESTAR Y EL PORVENIR; Linked To: C.I. METALURGIA EXTRACTIVA DE COLOMBIA S.A.S.; Linked To: GRUPO MUNDO MARINO, S.A.; Linked To: C.I. DISERCOM S.A.S.; Linked To: C.I. OKCOFFEE COLOMBIA S.A.S.; Linked To: C.I. OKCOFFEE INTERNATIONAL S.A.S.; Linked To: FUNDACION OKCOFFEE COLOMBIA; Linked To: CUBICAFE S.A.S.; Linked To: HOTELES Y BIENES S.A.; Linked To: FUNDACION SALVA LA SELVA; Linked To: LINEA AEREA PUEBLOS AMAZONICOS S.A.S.; Linked To: DESARROLLO MINERO RESPONSABLE C.I. S.A.S.; Linked To: R D I S.A.`
require.Equal(t, expected, found.Remarks)
}

0 comments on commit 14c3d73

Please sign in to comment.