feat: Retrieve rloi_ids with offset and limit in dts-process.js

DEFRA · May 9, 2024 · 4356d72 · 4356d72
1 parent 183fe4b
commit 4356d72
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 33 deletions.
diff --git a/lib/functions/dts-process.js b/lib/functions/dts-process.js
@@ -6,6 +6,7 @@ const Joi = require('joi')
 const axios = require('axios')
 const { HTTP_NOT_FOUND } = require('../constants')
 const deleteStation = require('../helpers/imtd-api').deleteStation
+const getRloiIds = require('../helpers/imtd-api').getRloiIds
 
 async function insertStation (stationDataArray) {
   try {
@@ -78,29 +79,15 @@ async function validateStationData (stationDataArray) {
   }
 }
 
-async function getRloiIds ({ limit, offset } = {}) {
-  try {
-    logger.info(`Retrieving up to ${limit} rloi_ids with an offset of ${offset}`)
-    const result = await pg('rivers_mview')
-      .distinct('rloi_id')
-      .whereNotNull('rloi_id')
-      .orderBy('rloi_id', 'asc')
-      .limit(limit)
-      .offset(offset)
-    logger.info(`Retrieved ${result.length} rloi_ids`)
-    return result
-  } catch (error) {
-    throw Error(`Could not get list of id's from database (Error: ${error.message})`)
-  }
-}
-
 async function handler ({ offset = 0 } = {}) {
   const BATCH_SIZE = parseInt(process.env.IMTD_BATCH_SIZE || '500')
 
+  logger.info(`Retrieving up to ${BATCH_SIZE} rloi_ids with an offset of ${offset}`)
   const stations = await getRloiIds({
     offset,
     limit: BATCH_SIZE
   })
+  logger.info(`Retrieved ${stations.length} rloi_ids`)
 
   for (const station of stations) {
     await getData(station.rloi_id)

diff --git a/lib/functions/imtd-process.js b/lib/functions/imtd-process.js
@@ -4,6 +4,7 @@ const logger = require('../helpers/logging')
 const pg = require('../helpers/db')
 const invokeLambda = require('../helpers/invoke-lambda')
 const deleteThresholds = require('../helpers/imtd-api').deleteStation
+const getRloiIds = require('../helpers/imtd-api').getRloiIds
 const tableName = 'station_imtd_threshold'
 const { HTTP_NOT_FOUND } = require('../constants')
 
@@ -67,29 +68,15 @@ async function getData (stationId) {
   }
 }
 
-async function getRloiIds ({ limit, offset } = {}) {
-  try {
-    logger.info(`Retrieving up to ${limit} rloi_ids with an offset of ${offset}`)
-    const result = await pg('rivers_mview')
-      .distinct('rloi_id')
-      .whereNotNull('rloi_id')
-      .orderBy('rloi_id', 'asc')
-      .limit(limit)
-      .offset(offset)
-    logger.info(`Retrieved ${result.length} rloi_ids`)
-    return result
-  } catch (error) {
-    throw Error(`Could not get list of id's from database (Error: ${error.message})`)
-  }
-}
-
 async function handler ({ offset = 0 } = {}) {
   const BATCH_SIZE = parseInt(process.env.IMTD_BATCH_SIZE || '500')
 
+  logger.info(`Retrieving up to ${BATCH_SIZE} rloi_ids with an offset of ${offset}`)
   const stations = await getRloiIds({
     offset,
     limit: BATCH_SIZE
   })
+  logger.info(`Retrieved ${stations.length} rloi_ids`)
 
   for (const station of stations) {
     await getData(station.rloi_id)

diff --git a/lib/helpers/imtd-api.js b/lib/helpers/imtd-api.js
@@ -4,4 +4,18 @@ async function deleteStation (stationId, tableName) {
   await pg(tableName).where({ station_id: stationId }).delete()
 }
 
-module.exports = { deleteStation }
+async function getRloiIds ({ limit, offset } = {}) {
+  try {
+    const result = await pg('rivers_mview')
+      .distinct('rloi_id')
+      .whereNotNull('rloi_id')
+      .orderBy('rloi_id', 'asc')
+      .limit(limit)
+      .offset(offset)
+    return result
+  } catch (error) {
+    throw Error(`Could not get list of id's from database (Error: ${error.message})`)
+  }
+}
+
+module.exports = { deleteStation, getRloiIds }