Merge pull request #1778 from RafaelTaranto/refactor/sanctions-module

LAM-470 refactor: sanctions module
lamassu · Dec 20, 2024 · 7cf2788 · 7cf2788
2 parents 0698482 + de7583c
commit 7cf2788
Show file tree

Hide file tree

Showing 9 changed files with 141 additions and 195 deletions.
diff --git a/.sample.env b/.sample.env
@@ -31,15 +31,6 @@ OPERATOR_DATA_DIR=
 
 COIN_ATM_RADAR_URL=
 
-## OFAC Sources variables
-
-# These variables map to each other, similar to a zip HOF. Entries are separated by commas
-# Example:
-# OFAC_SOURCES_NAMES=name1,name2
-# OFAC_SOURCES_URLS=url1,url2
-OFAC_SOURCES_NAMES=
-OFAC_SOURCES_URLS=
-
 ## Misc
 
 HOSTNAME=

diff --git a/bin/lamassu-ofac-update-sources b/bin/lamassu-ofac-update-sources
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -27,8 +27,6 @@ services:
       - FRONT_CAMERA_DIR=/lamassu-data/frontcamera
       - OPERATOR_DATA_DIR=/lamassu-data/operatordata
       - COIN_ATM_RADAR_URL=https://coinatmradar.info/api/lamassu/
-      - OFAC_SOURCES_NAMES=sdn_advanced,cons_advanced
-      - OFAC_SOURCES_URLS=https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml,https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml
       - HOSTNAME=localhost
       - LOG_LEVEL=info
 
@@ -58,8 +56,6 @@ services:
       - FRONT_CAMERA_DIR=/lamassu-data/frontcamera
       - OPERATOR_DATA_DIR=/lamassu-data/operatordata
       - COIN_ATM_RADAR_URL=https://coinatmradar.info/api/lamassu/
-      - OFAC_SOURCES_NAMES=sdn_advanced,cons_advanced
-      - OFAC_SOURCES_URLS=https://www.treasury.gov/ofac/downloads/sanctions/1.0/sdn_advanced.xml,https://www.treasury.gov/ofac/downloads/sanctions/1.0/cons_advanced.xml
       - HOSTNAME=172.29.0.3
       - LOG_LEVEL=info
     depends_on:

diff --git a/lib/compliance.js b/lib/compliance.js
@@ -72,8 +72,8 @@ function validateOfac (deviceId, sanctionsActive, customer) {
 
 function validationPatch (deviceId, sanctionsActive, customer) {
   return validateOfac(deviceId, sanctionsActive, customer)
-    .then(sactions =>
-      _.isNil(customer.sanctions) || customer.sanctions !== sactions ?
+    .then(sanctions =>
+      _.isNil(customer.sanctions) || customer.sanctions !== sanctions ?
         { sanctions } :
         {}
     )

diff --git a/lib/ofac/update.js b/lib/ofac/update.js
@@ -1,58 +1,50 @@
 const parser = require('./parsing')
-const https = require('https')
-const URL = require('url')
+const axios = require('axios')
 const { createWriteStream } = require('fs')
-const fs = require('fs/promises')
-const { readFile, writeFile, rename, unlink } = fs
+const { rename, writeFile, readFile, mkdir, copyFile, unlink } = require('fs/promises')
 const path = require('path')
 const _ = require('lodash/fp')
-const logger = require('../logger')
 
 const DOWNLOAD_DIR = path.resolve('/tmp')
-
 const OFAC_DATA_DIR = process.env.OFAC_DATA_DIR
-const OFAC_SOURCES_NAMES = process.env.OFAC_SOURCES_NAMES.split(',')
-const OFAC_SOURCES_URLS = process.env.OFAC_SOURCES_URLS.split(',')
-
-const ofacSources = _.map(
-  ([name, url]) => ({ name, url }),
-  _.zip(OFAC_SOURCES_NAMES, OFAC_SOURCES_URLS)
-)
-
-const mkdir = path =>
-  fs.mkdir(path)
+const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources')
+const LAST_UPDATED_FILE = path.resolve(OFAC_DATA_DIR, 'last_updated.dat')
+
+const OFAC_SOURCES = [{
+  name: 'sdn_advanced',
+  url: 'https://sanctionslistservice.ofac.treas.gov/api/download/sdn_advanced.xml'
+}, {
+  name: 'cons_advanced',
+  url: 'https://sanctionslistservice.ofac.treas.gov/api/download/cons_advanced.xml'
+}]
+
+const _mkdir = path =>
+  mkdir(path)
     .catch(err => err.code === 'EEXIST' ? Promise.resolve() : Promise.reject(err))
 
-const promiseGetEtag = ({ url }) =>
-  new Promise((resolve, reject) => {
-    const parsed = URL.parse(url)
-    const requestOptions = {
-      hostname: parsed.hostname,
-      path: parsed.path,
-      method: 'HEAD'
-    }
-
-    const request = https.request(requestOptions, _.flow(
-      _.get(['headers', 'etag']),
-      resolve
-    ))
-
-    request.on('error', reject)
-
-    request.end()
-  })
-
 const download = (dstDir, { name, url }) => {
   const dstFile = path.join(dstDir, name + '.xml')
-  const file = createWriteStream(dstFile)
-
-  return new Promise((resolve, reject) => {
-    const request = https.get(url, response => {
-      response.pipe(file)
-      file.on('finish', () => file.close(() => resolve(dstFile)))
+  const writer = createWriteStream(dstFile)
+
+  return axios({
+    method: 'get',
+    url: url,
+    responseType: 'stream',
+  }).then(response => {
+    return new Promise((resolve, reject) => {
+      response.data.pipe(writer)
+      let error = null
+      writer.on('error', err => {
+        error = err
+        writer.close()
+        reject(err)
+      })
+      writer.on('close', () => {
+        if (!error) {
+          resolve(dstFile)
+        }
+      })
     })
-
-    request.on('error', reject)
   })
 }
 
@@ -81,78 +73,63 @@ const parseToJson = srcFile => {
   })
 }
 
-const moveToSourcesDir = (srcFile, ofacSourcesDir) => {
+const moveToSourcesDir = async (srcFile, ofacSourcesDir) => {
   const name = path.basename(srcFile)
   const dstFile = path.join(ofacSourcesDir, name)
-  return rename(srcFile, dstFile)
+  try {
+    await rename(srcFile, dstFile)
+  } catch (err) {
+    if (err.code === 'EXDEV') {
+      // If rename fails due to cross-device link, fallback to copy + delete
+      await copyFile(srcFile, dstFile)
+      await unlink(srcFile)
+    } else {
+      throw err
+    }
+  }
+  return dstFile
 }
 
 function update () {
   if (!OFAC_DATA_DIR) {
     throw new Error('ofacDataDir must be defined in the environment')
   }
 
-  if (!ofacSources) {
-    logger.error('ofacSources must be defined in the environment')
-  }
-
-  const OFAC_SOURCES_DIR = path.join(OFAC_DATA_DIR, 'sources')
-  const OFAC_ETAGS_FILE = path.join(OFAC_DATA_DIR, 'etags.json')
-
-  return mkdir(OFAC_DATA_DIR)
-    .then(() => mkdir(OFAC_SOURCES_DIR))
-    .then(() => writeFile(OFAC_ETAGS_FILE, '{}', {encoding: 'utf-8', flag: 'wx'}))
+  return _mkdir(OFAC_DATA_DIR)
+    .then(() => _mkdir(OFAC_SOURCES_DIR))
     .catch(err => {
       if (err.code === 'EEXIST') return
       throw err
     })
-    .then(() => {
-      const promiseOldEtags = readFile(OFAC_ETAGS_FILE, {encoding: 'utf-8'})
-      .then(json => JSON.parse(json))
-      .catch(_ => {
-        logger.error('Can\'t parse etags.json, getting new data...')
-        return {}
-      })
+    .then(() => readFile(LAST_UPDATED_FILE))
+    .then(data => {
+      const lastUpdate = new Date(data.toString())
+      const now = new Date()
+      const hoursSinceUpdate = (now - lastUpdate) / (1000 * 60 * 60)
+
+      return hoursSinceUpdate < 24
+    })
+    .catch(err => {
+      // If file doesn't exist, continue with update
+      if (err.code === 'ENOENT') return false
+      throw err
+    })
+    .then(skipUpdate => {
+      if (skipUpdate) return Promise.resolve()
+
+      const downloads = _.flow(
+        _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson))
+      )(OFAC_SOURCES)
+
+      return Promise.all(downloads)
+        .then(parsed => {
+          const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed)
+          const timestamp = new Date().toISOString()
 
-      const promiseNewEtags = Promise.resolve(ofacSources || [])
-        .then(sources => Promise.all(_.map(promiseGetEtag, sources))
-          .then(etags => _.map(
-            ([source, etag]) => _.set('etag', etag,  source),
-            _.zip(sources, etags)
-          ))
-        )
-
-      return Promise.all([promiseOldEtags, promiseNewEtags])
-        .then(([oldEtags, newEtags]) => {
-          const hasNotChanged = ({name, etag}) => oldEtags[name] === etag
-
-          const downloads = _.flow(
-            _.reject(hasNotChanged),
-            _.map(file => download(DOWNLOAD_DIR, file).then(parseToJson))
-          )(newEtags)
-
-          const oldFileNames = _.keys(oldEtags)
-          const newFileNames = _.map(_.get('name'), newEtags)
-          const missingFileNames = _.difference(oldFileNames, newFileNames)
-          const resolve = name => path.join(OFAC_SOURCES_DIR, name + '.json')
-          const missing = _.map(resolve, missingFileNames)
-
-          const etagsJson = _.flow(
-            _.map(source => [source.name, source.etag]),
-            _.fromPairs,
-            obj => JSON.stringify(obj, null, 4)
-          )(newEtags)
-
-          return Promise.all(downloads)
-            .then(parsed => {
-              const moves = _.map(src => moveToSourcesDir(src, OFAC_SOURCES_DIR), parsed)
-              const deletions = _.map(unlink, missing)
-              const updateEtags = writeFile(OFAC_ETAGS_FILE, etagsJson)
-
-              return Promise.all([updateEtags, ...moves, ...deletions])
-            })
+          return Promise.all([...moves])
+            .then(() => writeFile(LAST_UPDATED_FILE, timestamp))
         })
     })
 }
 
-module.exports = {update}
+module.exports = { update }