From d2497cf4eaab63ec1965f6a4f5c96aeb68db0e7c Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Thu, 21 Sep 2023 15:39:25 +0200 Subject: [PATCH 001/114] feat(wip): first version of unpaywall history --- .../lib/controllers/changefiles.js | 10 + .../snapshots/changefiles-day-example.json | 18 + src/fakeUnpaywall/snapshots/fake1.jsonl.gz | Bin 19896 -> 19896 bytes src/fakeUnpaywall/snapshots/fake3.jsonl.gz | Bin 877494 -> 877494 bytes src/fakeUnpaywall/snapshots/history1.jsonl.gz | Bin 0 -> 1952 bytes src/fakeUnpaywall/snapshots/history2.jsonl.gz | Bin 0 -> 1934 bytes src/update/lib/controllers/job.js | 51 +++ src/update/lib/download.js | 2 +- src/update/lib/insert.js | 337 ++++++++++++++++-- src/update/lib/job.js | 60 +++- src/update/lib/middlewares/job.js | 23 ++ src/update/lib/routers/job.js | 11 + src/update/lib/services/elastic.js | 41 +++ src/update/mapping/unpaywall_enriched.json | 250 +++++++++++++ src/update/mapping/unpaywall_history.json | 141 ++++++++ 15 files changed, 916 insertions(+), 28 deletions(-) create mode 100644 src/fakeUnpaywall/snapshots/history1.jsonl.gz create mode 100644 src/fakeUnpaywall/snapshots/history2.jsonl.gz create mode 100644 src/update/mapping/unpaywall_enriched.json create mode 100644 src/update/mapping/unpaywall_history.json diff --git a/src/fakeUnpaywall/lib/controllers/changefiles.js b/src/fakeUnpaywall/lib/controllers/changefiles.js index 7ad3cb24..63f5dc6f 100644 --- a/src/fakeUnpaywall/lib/controllers/changefiles.js +++ b/src/fakeUnpaywall/lib/controllers/changefiles.js @@ -78,6 +78,16 @@ async function updateChangefilesExample(interval) { changefilesDay.list[4].last_modified = new Date(now - oneYear - (1 * oneDay)) .toISOString().slice(0, 19); + changefilesDay.list[5].date = new Date(now - (2 * oneYear) - (1 * oneDay)) + .toISOString().slice(0, 10); + changefilesDay.list[5].last_modified = new Date(now - (2 * oneYear) - (1 * oneDay)) + .toISOString().slice(0, 19); + + changefilesDay.list[6].date = new Date(now - (2 * oneYear) - (2 * oneDay)) + .toISOString().slice(0, 10); + changefilesDay.list[6].last_modified = new Date(now - (2 * oneYear) - (2 * oneDay)) + .toISOString().slice(0, 19); + try { await fs.writeFile(changefilesDayPath, JSON.stringify(changefilesDay, null, 2), 'utf8'); } catch (err) { diff --git a/src/fakeUnpaywall/snapshots/changefiles-day-example.json b/src/fakeUnpaywall/snapshots/changefiles-day-example.json index d40c80d4..028a59c5 100644 --- a/src/fakeUnpaywall/snapshots/changefiles-day-example.json +++ b/src/fakeUnpaywall/snapshots/changefiles-day-example.json @@ -44,6 +44,24 @@ "lines": 50, "size": 19896, "url": "http://fakeUnpaywall:3000/daily-feed/changefiles/fake1.jsonl.gz?api_key=default" + }, + { + "date": "", + "filename": "history1.jsonl.gz", + "filetype": "jsonl", + "last_modified": "", + "lines": 50, + "size": 19918, + "url": "http://fakeUnpaywall:3000/daily-feed/changefiles/history1.jsonl.gz?api_key=default" + }, + { + "date": "", + "filename": "history2.jsonl.gz", + "filetype": "jsonl", + "last_modified": "", + "lines": 50, + "size": 19896, + "url": "http://fakeUnpaywall:3000/daily-feed/changefiles/history2.gz?api_key=default" } ] } \ No newline at end of file diff --git a/src/fakeUnpaywall/snapshots/fake1.jsonl.gz b/src/fakeUnpaywall/snapshots/fake1.jsonl.gz index ffd5bbb02211fef2689f0fbfe6f1c3da584cd01a..684673cf52cac0a83b1cb9ac2232e8749efff889 100644 GIT binary patch delta 26 icmdlnn{mf%Mt=Ej4vsL7webwhX^GjXh8y`;`T_uT{s^4_ delta 26 icmdlnn{mf%Mt=Ej4vsL7webwhX^GjX#vA!p`T_uU2ne45 diff --git a/src/fakeUnpaywall/snapshots/fake3.jsonl.gz b/src/fakeUnpaywall/snapshots/fake3.jsonl.gz index af21e056109e679f7fcec8688d0eed6814f123d2..91007baa68280cae286b0d36dcf9a427088936ef 100644 GIT binary patch delta 65 zcmdmX+;rPoT&`VX^GjX#*O@~{EV&qOs)LPt^6#l{H(3~Y_0t4t^6FV V{G6@)T&?`vt^7P&`FWQo0|0H%5f%Ud delta 65 zcmdmX+;rPBK{EV&qOs)LPt^6#l{H(3~Y_0t4t^6FV V{G6@)T&?`vt^7P&`FWQo0|0^~5qtmu diff --git a/src/fakeUnpaywall/snapshots/history1.jsonl.gz b/src/fakeUnpaywall/snapshots/history1.jsonl.gz new file mode 100644 index 0000000000000000000000000000000000000000..b8159e18e958216d0149237d708c6d66eed134ad GIT binary patch literal 1952 zcmV;R2VeLfiwFoRLJVa9188Y;bZ>HbF)nIzZ*FV=?O1DX+c*~ezQ2MHFhC1LvZ#kG z30N%B%%JT|lVHFIdzNq?|UiR@gwmo3A*{vATbsddup z!4pTSQE!(E+Bzo<%fZTXJ=ew#vig`6nXQ?AW>i$#dRtDM-0y^*bl4 zR{w4fvU=0OUgrOrKRU74gy(Z7NrL81 zKrbf1%3PA`moQ{3b(1#Jnyt*BjH$?L$wjkr67WP?NR`%kvH7SZUD$);N9;&%DOO9( z8>yx#7m!k#w&~7e7BEa$`X44_tw|AeMZ0MU55%wSv~1I6L5)dtd?6?B65cSRQm{XrN}5ul>)O;!>qLLm_u(cU}cJ= zgJ&M+z1*iBp>AaN1B&ec$AO)>A$$aw_%vXHASRC@$1H~30fhDy4iYa+{J_D1AGy(S zG5nOI85C~hg_H)E1w~A+M z1-%vUS#`LhE07*SmU1)&@@=z*dx9CcfdER95rF$HH*aok7Ux(D@717RNO?3*VVdMw zaJ3I~y~1AN1qqIwXwdhABk!9hO>hMXa7Ke1)Fv4egMw`5>f_Pa!`mGTFwt8>OE0Ce zJ(J3|ENmlP?YUI8VGVmBmF;?LOJUEYvb_|<-KeEhUVQ`etw?42Rw~<_RJL!W@{2>5 z^U@hNRk)p|DM~z_|KQfcUzIl#HDL=RM;eqi`WpJOu7Z1ARQXj>Rov_@%xGE zJAUY69IOFOC}nl?W3pkPgW_g#J-%tLtE$eYT2Qq$9sUS3_OR_oY+-A{`WOYQ&~@Eum1Ym_tkc( z8}#bh=?|RHi9PSOQ3`iEUN%bS%h>aqa@i!EUoDF+=_UB99s@b@X~`N8Zn7km$uz^M zXvR$QuKjdIH0%kzVnm?dL=GlQRr|Y|cXoRBzVf0@z7LWRN=-iolZ}361~tw%w$af% zt(XFZ393s{&Su5XC1BQ|SKO3JAZ0araXP$w^>S-hb4bbwfm1Pby4Kemn!X&}=-%D! zYQAH-X8VzhM8~%MJ;ST9tcXU#HT!c3`!VKiQSh(kT2RiT+lZZCF2_^dxn{b% zeK=QLwK1w^HXWW6?lP!Pp(^QxJpj`wfX*8#XLo+st5|Vl2!Z=oxS!xKaXrV2apd~P zB!wDk;VvB%iZKM+drV4rutSc?B6nisI2C)CME1hS#ZO;c1Xr$Wv?lhPFvQ;Ak)`)@ zEG`zFJE{Q;o6wGC73uUJD9Vzk8EFQY-}B?prA0r;C?!!p4x<6~eVpOApd`z3*7yBj zcVr!n7mmBy02{oz-!-6gn^n9yuqs#_3(e1UG+tJts?j$kBK9SiejVibr23|}^?{$v zlY7W;-x4)}TdrH^kahNPcB3dh!6AI}!XUa&s^1Q~PpTVrV7o@9 m?3L<2uYl)$1?-dcKk<9!_}y0m%tL=i4e&ofhxVZ}ApihcZp@JY literal 0 HcmV?d00001 diff --git a/src/fakeUnpaywall/snapshots/history2.jsonl.gz b/src/fakeUnpaywall/snapshots/history2.jsonl.gz new file mode 100644 index 0000000000000000000000000000000000000000..6b53693e388bbe1bf8faba0cfbc2214fc74950f2 GIT binary patch literal 1934 zcmV;92XXixiwFoqK@4R8188Y;bZ>HbGA?R!Z*FV=?O17V+%^#X-d{loD3AcwK^?o2 zfTD2RB1dD#&ALs4q7W#F+NCX#3Ms9#3HskV)b4r@doSBa{UJbN*c6Ayyf?!)^VP@& zGhzcf7RF)ER=yKp$8>N2|DDL}8Ary15^2P)<2Xmgka9_3{-tOoC#6Yb!?KdXET$4c zjJT04Ju({BlzQ4=!l@iG1(Ogh3Ys-Y6zG~sic-pR#)pXT94S(=F_9Bgze0i|)yjfo z6xCAH6gHr$n^nsz6S5hJTyVXMY6#;cRSJHiv_wQ^BfX!3Bf?daR+ezC zQPb2awrv=(gdE!2uHV)oHX-Z4RuV7HoPpw`Vh6^TDWBhxxF#kySiXB}wAbHLjCIHbD zt2c}+Tm9AUW%a6MpW78Xw%>N|82~KvWz#_;%5pE`-@&~~@Zqxc$1VH*(^0l262Sf3 z7;!;L1(rYf6j)E3_51{%t?Zv znG?{nF|aZhD z(XZWV==}7=SVdlQsct2;WIMEYXh~$TA;OzlcbhNAyn8+~4~<(!>20-N>guIO{ywOtgG>DFxSWGe`E6Vidtq z&}$Aqs}6T`0n&quQVb_RzD?HfPEaj35I{*%0&w5v=JoaU{2ue^b`AQ4ln3(^mWiJP zS9?I$3+%?OALGah`#sM;@V$A|1XmCrrko2UL!MZwls}9lt+$tqsn|RHLEe zjO4TeQU87(S=#sn6d_54oce7gOa~LVNwp#Dc$P8%qOv#}Swk__Z_nZ(f~REh5N*(9 z>j5(TygfQ@sWFVB2WE0OH_K5e#8q4G9~k#lW(T@%f03pap9h8DP{dH{GHWfGw-#w_ zA5bRA%HCw}L`T#qrXe=f^&_!Hmxuy5Qj?lydcbf{IUd*R~P>&1>| zd4Y$qzXCYPGFms^CL0#YFRsUzqwDsvs_J~AcqTWd!*7Ac4z_)ZBWz4qZ=mK=kYB{r z@*8j7q@Q=)p%7`%oin?dQosGW5x{d@$;R%YqFiZ^EC!XX9t{Ye9#F@3jn!{Sx%TlU zTijU+YUUpRGMA{CTs1?6iS>gbgiLzTu})E6GWHD7@gKa=KxuQUNBuL zfs$3|+411~#q*6#%>gOL1a3vq@k(2BpxbiUTJ!F9Q}ZoVC8OxK726MF$UCy_ZW&&T zL`4)Dtk|ClIFB}Oi-LXDuY$B5+(qpCaz2`<&NY+m-NTvasB zr=;npp6x|}lNCKb%`y`9q9E*J&%-H>ij1UbPJ5o`Z;z~l@xoD88eolA_iqI#)ut6| z_ACnK*Fv*<9gLUJ$ZGgS@{oS=C!hOyKCZr~O>N*u^W+XP+_gnb;Faq(+Gn3#9N9;n zuO;xgK6f0?HJvbwj&K0Kxq% Date.now()) { + return res.status(400).json({ message: 'startDate cannot be in the futur' }); + } + + if (startDate && endDate) { + if (new Date(endDate).getTime() < new Date(startDate).getTime()) { + return res.status(400).json({ message: 'endDate cannot be lower than startDate' }); + } + } + + const jobConfig = { + index, + interval, + startDate, + endDate, + offset: 0, + limit: -1, + }; + + if (!startDate && !endDate) { + jobConfig.endDate = format(new Date(), 'yyyy-MM-dd'); + if (interval === 'week') jobConfig.startDate = format(new Date() - (7 * 24 * 60 * 60 * 1000), 'yyyy-MM-dd'); + if (interval === 'day') jobConfig.startDate = format(new Date(), 'yyyy-MM-dd'); + + insertWithOaHistoryJob(jobConfig); + return res.status(202).json(); + } + + if (startDate && !endDate) jobConfig.endDate = format(new Date(), 'yyyy-MM-dd'); + + insertWithOaHistoryJob(jobConfig); + return res.status(202).json(); +} + module.exports = { downloadAndInsertSnapshotJob, insertChangefilesOnPeriodJob, insertChangefileJob, + insertWithOaHistory, }; diff --git a/src/update/lib/download.js b/src/update/lib/download.js index 9dc29541..ff2feb99 100644 --- a/src/update/lib/download.js +++ b/src/update/lib/download.js @@ -47,7 +47,7 @@ async function updatePercentStepDownload(filepath, size, start) { step.took = (new Date() - start) / 1000; step.percent = ((bytes.size / size) * 100).toFixed(2); updateLatestStep(step); - await new Promise((resolve) => setTimeout(resolve, 1000)); + await new Promise((resolve) => { setTimeout(resolve, 1000); }); updatePercentStepDownload(filepath, size, start); } diff --git a/src/update/lib/insert.js b/src/update/lib/insert.js index b76a1f2a..f352c3d7 100644 --- a/src/update/lib/insert.js +++ b/src/update/lib/insert.js @@ -9,7 +9,8 @@ const config = require('config'); const zlib = require('zlib'); const logger = require('./logger'); -const unpaywallMapping = require('../mapping/unpaywall.json'); +const unpaywallEnrichedMapping = require('../mapping/unpaywall.json'); +const unpaywallHistoryMapping = require('../mapping/unpaywall_history.json'); const { addStepInsert, @@ -19,8 +20,11 @@ const { } = require('./models/state'); const { - elasticClient, + refreshIndex, + bulk, + initAlias, createIndex, + getDataByListOfDOI, } = require('./services/elastic'); const indexAlias = config.get('elasticsearch.indexAlias'); @@ -35,11 +39,11 @@ const snapshotsDir = path.resolve(__dirname, '..', 'data', 'snapshots'); * * @returns {Promise} Success or not. */ -async function insertDataInElastic(data) { +async function insertUnpaywallDataInElastic(data) { const step = getLatestStep(); let res; try { - res = await elasticClient.bulk({ body: data }); + res = await bulk(data); } catch (err) { logger.error('[elastic] Cannot bulk', err); await fail(err?.[0]?.reason); @@ -106,29 +110,305 @@ async function insertDataUnpaywall(insertConfig) { step.index = index; updateLatestStep(step); + await initAlias(index, unpaywallEnrichedMapping, indexAlias); + + const filePath = path.resolve(snapshotsDir, filename); + + // get information "bytes" for state + let bytes; try { - await createIndex(index, unpaywallMapping); + bytes = await fs.stat(filePath); } catch (err) { - logger.error(`[elastic] Cannot create index [${index}]`, err); + logger.error(`[job: insert] Cannot stat [${filePath}]`, err); await fail(err); return false; } + // read file with stream + let readStream; try { - const { body: aliasExists } = await elasticClient.indices.existsAlias({ name: indexAlias }); + readStream = fs.createReadStream(filePath); + } catch (err) { + logger.error(`[job: insert] Cannot read [${filePath}]`, err); + await fail(err); + return false; + } - if (aliasExists) { - logger.info(`[elastic] Alias [${indexAlias}] pointing to index [${index}] already exists`); - } else { - logger.info(`[elastic] Creating alias [${indexAlias}] pointing to index [${index}]`); - await elasticClient.indices.putAlias({ index, name: indexAlias }); - } + // get information "loaded" for state + let loaded = 0; + readStream.on('data', (chunk) => { + loaded += chunk.length; + }); + + let decompressedStream; + try { + decompressedStream = readStream.pipe(zlib.createGunzip()); } catch (err) { - logger.error(`[elastic] Cannot create alias [${indexAlias}] pointing to index [${index}]`, err); + logger.error(`[job: insert] Cannot pipe [${readStream?.filename}]`, err); await fail(err); return false; } + const rl = readline.createInterface({ + input: decompressedStream, + crlfDelay: Infinity, + }); + + // array that will contain the packet of 1000 unpaywall data + let bulkOps = []; + + let success; + + logger.info(`[job: insert] Start insert with [${filename}]`); + + // Reads line by line the output of the decompression stream to make packets of 1000 + // to insert them in bulk in an elastic + for await (const line of rl) { + // limit + if (step.linesRead === limit) { + break; + } + + step.linesRead += 1; + + // offset + if (step.linesRead >= offset + 1) { + // fill the array + try { + const doc = JSON.parse(line); + bulkOps.push({ index: { _index: index, _id: doc.doi } }); + bulkOps.push(doc); + } catch (err) { + logger.error(`[job: insert] Cannot parse [${line}] in json format`, err); + await fail(err); + return false; + } + } + // bulk insertion + if (bulkOps.length >= maxBulkSize) { + success = await insertUnpaywallDataInElastic(bulkOps, step); + bulkOps = []; + if (!success) return false; + step.percent = ((loaded / bytes.size) * 100).toFixed(2); + step.took = (new Date() - start) / 1000; + updateLatestStep(step); + } + if (step.linesRead % 100000 === 0) { + logger.info(`[job: insert] ${step.linesRead} Lines reads`); + updateLatestStep(step); + } + } + // last insertion if there is data left + if (bulkOps.length > 0) { + success = await insertUnpaywallDataInElastic(bulkOps, step); + bulkOps = []; + if (!success) return false; + } + + logger.info('[job: insert] insertion completed'); + + try { + await refreshIndex(); + } catch (err) { + logger.warn('[elastic] Cannot refresh the index', err); + } + + // last update of step + step.status = 'success'; + step.took = (new Date() - start) / 1000; + step.percent = 100; + updateLatestStep(step); + return true; +} + +/** + * Gets difference between 2 objects + * + * @param v1 First object to compare + * @param v2 Second object to compare + * @param prefix Prefix of returned keys. Should be empty on the first iteration + * + * @returns Keys that are in v1 but not in v2 + */ +function objectDiff(v1, v2, prefix = '') { + const diffs = Object.keys(v1 ?? {}).reduce( + (previous, key) => { + const prefixedKey = `${prefix}${key}`; + if (typeof v1[key] === 'object') { + return [ + ...previous, + ...objectDiff(v1[key], v2[key], `${prefixedKey}.`), + ]; + } + if (v1[key] !== v2[key]) { + return [ + ...previous, + prefixedKey, + ]; + } + return previous; + }, + [], + ); + return diffs; + // const diffs = Object.keys(v1 ?? {}).reduce( + // (d, key) => { + // const prefixedKey = `${prefix}${key}`; + // if ( + // Array.isArray(v2[key]) + // ? !v2[key].map((v) => typeof v).includes(typeof v1[key]) + // : typeof v1[key] !== typeof v2[key] + // ) { + // d.push(prefixedKey); + // } else if (typeof v1[key] === 'object' && !Array.isArray(v1[key])) { + // d.push( + // ...objectDiff( + // v1[key], + // v2[key], + // `${prefixedKey}.`, + // ), + // ); + // } + // return d; + // }, + // [], + // ); + // return diffs; +} + +/** + * 1. Insert unpaywall data in elastic with history + * 2. Insert history data in elastic + * + * @param {Array} listOfDoi - List of DOI + * @param {Array} newData - List of unpaywall data + * @param {Date} date - Date of file + */ +async function insertData(listOfDoi, newData, date) { + // TODO date of file + date = new Date(); + + const oldData = await getDataByListOfDOI(listOfDoi, 'unpaywall'); + const historyData = await getDataByListOfDOI(listOfDoi, 'unpaywall_history'); + + let oldHistoryDataMap; + let oldUnpaywallDataMap; + const resHistoryData = []; + + if (historyData.length > 0) { + oldHistoryDataMap = new Map( + historyData.map((data) => { + const copyData = { ...data }; + return [data.doi, copyData]; + }), + ); + + oldUnpaywallDataMap = new Map( + oldData.map((data) => { + const copyData = { ...data }; + return [data.doi, copyData]; + }), + ); + // enrich history + newData.forEach((data) => { + const oldDataUnpaywall = oldUnpaywallDataMap.get(data.doi); + // if document already exist + if (oldDataUnpaywall) { + const diffs = objectDiff(data, oldDataUnpaywall); + if (diffs.length > 0) { + const enrichedHistoryData = oldHistoryDataMap.get(data.doi); + // If document hasn't got history + if (!enrichedHistoryData) { + resHistoryData.push({ index: { _index: 'unpaywall_history', _id: data.doi } }); + resHistoryData.push({ + doi: data.doi, + referencedAt: date, + history: [], + }); + } else { + const newEntryHistory = {}; + newEntryHistory.date = new Date(); + + diffs.forEach((diff) => { + newEntryHistory[diff] = oldDataUnpaywall[diff]; + }); + + enrichedHistoryData.history.push(newEntryHistory); + + resHistoryData.push({ index: { _index: 'unpaywall_history', _id: data.doi } }); + resHistoryData.push(enrichedHistoryData); + } + } + } + }); + } else { + // First insertion + newData.forEach((data) => { + const newEntry = { + doi: data.doi, + referencedAt: date, + history: [], + }; + resHistoryData.push({ index: { _index: 'unpaywall_history', _id: newEntry.doi } }); + resHistoryData.push(newEntry); + }); + } + + if (resHistoryData.length > 0) { + try { + await bulk(resHistoryData, 'unpaywall_history'); + } catch (err) { + logger.error(err); + } + } + + const bulkOps = []; + + newData.forEach((data) => { + bulkOps.push({ index: { _index: 'unpaywall', _id: data.doi } }); + bulkOps.push(data); + }); + + try { + await bulk(bulkOps, 'unpaywall'); + } catch (err) { + logger.error(err); + } + + // #endregion 2 + return true; +} + +/** + * Inserts the contents of an unpaywall data update file with oa history. + * + * @param {Object} insertConfig - Config of insertion. + * @param {string} insertConfig.filename - Name of the snapshot file from which the data will + * be retrieved to be inserted into elastic. + * @param {string} insertConfig.index - Name of the index to which the data will be inserted. + * @param {number} insertConfig.offset - Line of the snapshot at which the data insertion starts. + * @param {number} insertConfig.limit - Line in the file where the insertion stops. + * + * @returns {Promise} Success or not. + */ +async function insertHistoryDataUnpaywall(insertConfig) { + const { + filename, index, offset, limit, + } = insertConfig; + + // step insertion in the state + const start = new Date(); + addStepInsert(filename); + const step = getLatestStep(); + step.file = filename; + step.index = index; + updateLatestStep(step); + + await createIndex('unpaywall', unpaywallEnrichedMapping); + await createIndex('unpaywall_history', unpaywallHistoryMapping); + + await initAlias(index, unpaywallEnrichedMapping, indexAlias); + const filePath = path.resolve(snapshotsDir, filename); // get information "bytes" for state @@ -172,7 +452,8 @@ async function insertDataUnpaywall(insertConfig) { }); // array that will contain the packet of 1000 unpaywall data - let bulkOps = []; + let newData = []; + let listOfDoi = []; let success; @@ -193,8 +474,8 @@ async function insertDataUnpaywall(insertConfig) { // fill the array try { const doc = JSON.parse(line); - bulkOps.push({ index: { _index: index, _id: doc.doi } }); - bulkOps.push(doc); + listOfDoi.push(doc.doi); + newData.push(doc); } catch (err) { logger.error(`[job: insert] Cannot parse [${line}] in json format`, err); await fail(err); @@ -202,11 +483,11 @@ async function insertDataUnpaywall(insertConfig) { } } // bulk insertion - if (bulkOps.length >= maxBulkSize) { - const dataToInsert = bulkOps.slice(); - bulkOps = []; - success = await insertDataInElastic(dataToInsert, step); + if (newData.length >= maxBulkSize) { + success = await insertData(listOfDoi, newData); if (!success) return false; + listOfDoi = []; + newData = []; step.percent = ((loaded / bytes.size) * 100).toFixed(2); step.took = (new Date() - start) / 1000; updateLatestStep(step); @@ -217,16 +498,17 @@ async function insertDataUnpaywall(insertConfig) { } } // last insertion if there is data left - if (bulkOps.length > 0) { - success = await insertDataInElastic(bulkOps, step); + if (newData.length > 0) { + success = await insertData(listOfDoi, newData); if (!success) return false; - bulkOps = []; + listOfDoi = []; + newData = []; } logger.info('[job: insert] insertion completed'); try { - await elasticClient.indices.refresh({ index }); + await refreshIndex(); } catch (err) { logger.warn('[elastic] Cannot refresh the index', err); } @@ -239,4 +521,7 @@ async function insertDataUnpaywall(insertConfig) { return true; } -module.exports = insertDataUnpaywall; +module.exports = { + insertDataUnpaywall, + insertHistoryDataUnpaywall, +}; diff --git a/src/update/lib/job.js b/src/update/lib/job.js index 4c61acbf..8fcbd2cb 100644 --- a/src/update/lib/job.js +++ b/src/update/lib/job.js @@ -22,7 +22,10 @@ const { getLatestStep, } = require('./models/state'); -const insertDataUnpaywall = require('./insert'); +const { + insertDataUnpaywall, + insertHistoryDataUnpaywall, +} = require('./insert'); const { getChangefiles, @@ -128,8 +131,63 @@ async function insertChangefile(jobConfig) { } } +/** + * Download and insert on elastic the changefiles from unpaywall between a period. + * + * @param {Object} jobConfig - Config of job. + * @param {string} jobConfig.index - Name of the index to which the data will be inserted. + * @param {string} jobConfig.interval - Interval of changefile, day or week are available. + * @param {string} jobConfig.startDate - Start date for the changefile period. + * @param {string} jobConfig.endDate - End date for the changefile period. + * @param {number} jobConfig.offset - Line of the snapshot at which the data insertion starts. + * @param {number} jobConfig.limit - Line in the file where the insertion stops. + * + * @returns {Promise} + */ +async function insertWithOaHistoryJob(jobConfig) { + setInUpdate(true); + const { + interval, startDate, endDate, + } = jobConfig; + await createState(); + const start = new Date(); + addStepGetChangefiles(); + const step = getLatestStep(); + const changefilesInfo = await getChangefiles(interval, startDate, endDate); + + if (!changefilesInfo) { + step.status = 'error'; + updateLatestStep(step); + await fail(); + return; + } + + step.took = (new Date() - start) / 1000; + step.status = 'success'; + updateLatestStep(step); + + if (changefilesInfo.length === 0) { + sendMailNoChangefile(startDate, endDate).catch((err) => { + logger.errorRequest(err); + }); + await endState(); + return; + } + + let success = true; + for (let i = 0; i < changefilesInfo.length; i += 1) { + success = await downloadChangefile(changefilesInfo[i], interval); + if (!success) return; + jobConfig.filename = changefilesInfo[i].filename; + success = await insertHistoryDataUnpaywall(jobConfig); + if (!success) return; + } + await endState(); +} + module.exports = { downloadAndInsertSnapshot, insertChangefilesOnPeriod, + insertWithOaHistoryJob, insertChangefile, }; diff --git a/src/update/lib/middlewares/job.js b/src/update/lib/middlewares/job.js index 03baad5c..7695a51b 100644 --- a/src/update/lib/middlewares/job.js +++ b/src/update/lib/middlewares/job.js @@ -72,8 +72,31 @@ async function validateInsertFile(req, res, next) { return next(); } +/** + * Joi middleware to check if job config for insert history of open access. + * + * @param {import('express').Request} req - HTTP request. + * @param {import('express').Response} res - HTTP response. + * @param {import('express').NextFunction} next - Do the following. + */ +async function validateHistoryJob(req, res, next) { + const { error, value } = joi.object({ + index: joi.string().trim().default('unpaywall_history'), + interval: joi.string().trim().valid('day', 'week').default('day'), + startDate: joi.date().format('YYYY-MM-DD'), + endDate: joi.date().format('YYYY-MM-DD').min(joi.ref('startDate')), + }).with('endDate', 'startDate').validate(req.body); + + if (error) return res.status(400).json({ message: error.details[0].message }); + + req.data = value; + + return next(); +} + module.exports = { validateSnapshotJob, validateJobChangefilesConfig, validateInsertFile, + validateHistoryJob, }; diff --git a/src/update/lib/routers/job.js b/src/update/lib/routers/job.js index 47f3312d..51d24e2a 100644 --- a/src/update/lib/routers/job.js +++ b/src/update/lib/routers/job.js @@ -3,6 +3,7 @@ const router = require('express').Router(); const { downloadAndInsertSnapshotJob, insertChangefilesOnPeriodJob, + insertWithOaHistory, insertChangefileJob, } = require('../controllers/job'); @@ -10,6 +11,7 @@ const { validateSnapshotJob, validateJobChangefilesConfig, validateInsertFile, + validateHistoryJob, } = require('../middlewares/job'); const checkStatus = require('../middlewares/status'); @@ -44,4 +46,13 @@ router.post('/job/period', checkStatus, checkAuth, validateJobChangefilesConfig, */ router.post('/job/changefile/:filename', checkStatus, checkAuth, validateInsertFile, insertChangefileJob); +/** + * Route that download and insert on elastic the changefiles from unpaywall between a period. + * Auth required. + * No update process should be in progress. + * + * This route need a body that contains a config of job. + */ +router.post('/job/history/insert', checkStatus, checkAuth, validateHistoryJob, insertWithOaHistory); + module.exports = router; diff --git a/src/update/lib/services/elastic.js b/src/update/lib/services/elastic.js index 1d32092f..9b975b99 100644 --- a/src/update/lib/services/elastic.js +++ b/src/update/lib/services/elastic.js @@ -118,9 +118,50 @@ async function initAlias(indexName, mapping, aliasName) { } } +async function getDataByListOfDOI(dois, index) { + // Normalize request + const normalizeDOI = dois.map((doi) => doi.toLowerCase()); + + const filter = [{ terms: { doi: normalizeDOI } }]; + + const query = { + bool: { + filter, + }, + }; + + let res; + try { + res = await elasticClient.search({ + index, + size: normalizeDOI.length || 1000, + body: { + query, + }, + + }); + } catch (err) { + logger.error('[elastic] Cannot request elastic', err); + return null; + } + // eslint-disable-next-line no-underscore-dangle + return res.body.hits.hits.map((hit) => hit._source); +} + +async function refreshIndex(index) { + return elasticClient.indices.refresh({ index }); +} + +async function bulk(data) { + return elasticClient.bulk({ body: data }); +} + module.exports = { elasticClient, pingElastic, createIndex, initAlias, + getDataByListOfDOI, + refreshIndex, + bulk, }; diff --git a/src/update/mapping/unpaywall_enriched.json b/src/update/mapping/unpaywall_enriched.json new file mode 100644 index 00000000..1dfefb79 --- /dev/null +++ b/src/update/mapping/unpaywall_enriched.json @@ -0,0 +1,250 @@ +{ + "settings": { + "number_of_shards": 3 + }, + "mappings": { + "dynamic_templates": [ + { + "strings_as_keywords": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword" + } + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "best_oa_location": { + "properties": { + "evidence": { "type": "keyword" }, + "host_type":{ "type": "keyword" }, + "is_best": { "type": "boolean" }, + "license": { "type": "keyword" }, + "pmh_id": { "type": "keyword" }, + "updated": { "type": "date" }, + "url": { "type": "keyword" }, + "url_for_landing_page": { "type": "keyword" }, + "url_for_pdf": { "type": "keyword" }, + "version": { "type": "keyword" } + } + }, + "data_standard": { + "type": "long" + }, + "doi": { + "type": "keyword" + }, + "doi_url": { + "type": "keyword" + }, + "first_oa_location": { + "properties": { + "evidence": { "type": "keyword" }, + "host_type":{ "type": "keyword" }, + "is_best": { "type": "boolean" }, + "license": { "type": "keyword" }, + "pmh_id": { "type": "keyword" }, + "updated": { "type": "date" }, + "url": { "type": "keyword" }, + "url_for_landing_page": { "type": "keyword" }, + "url_for_pdf": { "type": "keyword" }, + "version": { "type": "keyword" } + } + }, + "genre": { + "type": "keyword" + }, + "history": { + "properties": { + "best_oa_location": { + "properties": { + "evidence": { "type": "keyword" }, + "host_type":{ "type": "keyword" }, + "is_best": { "type": "boolean" }, + "license": { "type": "keyword" }, + "pmh_id": { "type": "keyword" }, + "updated": { "type": "date" }, + "url": { "type": "keyword" }, + "url_for_landing_page": { "type": "keyword" }, + "url_for_pdf": { "type": "keyword" }, + "version": { "type": "keyword" } + } + }, + "data_standard": { + "type": "long" + }, + "doi_url": { + "type": "keyword" + }, + "first_oa_location": { + "properties": { + "evidence": { "type": "keyword" }, + "host_type":{ "type": "keyword" }, + "is_best": { "type": "boolean" }, + "license": { "type": "keyword" }, + "pmh_id": { "type": "keyword" }, + "updated": { "type": "date" }, + "url": { "type": "keyword" }, + "url_for_landing_page": { "type": "keyword" }, + "url_for_pdf": { "type": "keyword" }, + "version": { "type": "keyword" } + } + }, + "genre": { + "type": "keyword" + }, + "has_repository_copy": { + "type": "boolean" + }, + "is_oa": { + "type": "boolean" + }, + "is_paratext": { + "type": "boolean" + }, + "journal_is_in_doaj": { + "type": "boolean" + }, + "journal_is_oa": { + "type": "boolean" + }, + "journal_issn_l": { + "type": "keyword" + }, + "journal_issns": { + "type": "keyword" + }, + "journal_name": { + "type": "keyword" + }, + "oa_locations": { + "properties": { + "evidence": { "type": "keyword" }, + "host_type":{ "type": "keyword" }, + "is_best": { "type": "boolean" }, + "license": { "type": "keyword" }, + "pmh_id": { "type": "keyword" }, + "updated": { "type": "date" }, + "url": { "type": "keyword" }, + "url_for_landing_page": { "type": "keyword" }, + "url_for_pdf": { "type": "keyword" }, + "version": { "type": "keyword" } + } + }, + "oa_status": { + "type": "keyword" + }, + "published_date": { + "type": "date", + "format": "iso8601" + }, + "publisher": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "updated": { + "type": "date", + "format": "iso8601" + }, + "year": { + "type": "keyword" + }, + "z_authors": { + "properties": { + "family": { "type": "keyword" }, + "given": { "type": "keyword" }, + "sequence": { "type": "keyword" }, + "ORCID": { "type": "keyword" }, + "authenticated-orcid": { "type": "boolean" }, + "affiliation": { + "properties": { + "name": { "type": "keyword" } + } + } + } + } + } + }, + "has_repository_copy": { + "type": "boolean" + }, + "is_oa": { + "type": "boolean" + }, + "is_paratext": { + "type": "boolean" + }, + "journal_is_in_doaj": { + "type": "boolean" + }, + "journal_is_oa": { + "type": "boolean" + }, + "journal_issn_l": { + "type": "keyword" + }, + "journal_issns": { + "type": "keyword" + }, + "journal_name": { + "type": "keyword" + }, + "oa_locations": { + "properties": { + "evidence": { "type": "keyword" }, + "host_type":{ "type": "keyword" }, + "is_best": { "type": "boolean" }, + "license": { "type": "keyword" }, + "pmh_id": { "type": "keyword" }, + "updated": { "type": "date" }, + "url": { "type": "keyword" }, + "url_for_landing_page": { "type": "keyword" }, + "url_for_pdf": { "type": "keyword" }, + "version": { "type": "keyword" } + } + }, + "oa_status": { + "type": "keyword" + }, + "published_date": { + "type": "date", + "format": "iso8601" + }, + "publisher": { + "type": "keyword" + }, + "referencedAt": { + "type": "date" + }, + "title": { + "type": "text" + }, + "updated": { + "type": "date", + "format": "iso8601" + }, + "year": { + "type": "keyword" + }, + "z_authors": { + "properties": { + "family": { "type": "keyword" }, + "given": { "type": "keyword" }, + "sequence": { "type": "keyword" }, + "ORCID": { "type": "keyword" }, + "authenticated-orcid": { "type": "boolean" }, + "affiliation": { + "properties": { + "name": { "type": "keyword" } + } + } + } + } + } + } +} diff --git a/src/update/mapping/unpaywall_history.json b/src/update/mapping/unpaywall_history.json new file mode 100644 index 00000000..aaf2743d --- /dev/null +++ b/src/update/mapping/unpaywall_history.json @@ -0,0 +1,141 @@ +{ + "settings": { + "number_of_shards": 3 + }, + "mappings": { + "dynamic_templates": [ + { + "strings_as_keywords": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword" + } + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "doi": { + "type": "keyword" + }, + "history": { + "properties": { + "best_oa_location": { + "properties": { + "evidence": { "type": "keyword" }, + "host_type":{ "type": "keyword" }, + "is_best": { "type": "boolean" }, + "license": { "type": "keyword" }, + "pmh_id": { "type": "keyword" }, + "updated": { "type": "date" }, + "url": { "type": "keyword" }, + "url_for_landing_page": { "type": "keyword" }, + "url_for_pdf": { "type": "keyword" }, + "version": { "type": "keyword" } + } + }, + "data_standard": { + "type": "long" + }, + "doi_url": { + "type": "keyword" + }, + "first_oa_location": { + "properties": { + "evidence": { "type": "keyword" }, + "host_type":{ "type": "keyword" }, + "is_best": { "type": "boolean" }, + "license": { "type": "keyword" }, + "pmh_id": { "type": "keyword" }, + "updated": { "type": "date" }, + "url": { "type": "keyword" }, + "url_for_landing_page": { "type": "keyword" }, + "url_for_pdf": { "type": "keyword" }, + "version": { "type": "keyword" } + } + }, + "genre": { + "type": "keyword" + }, + "has_repository_copy": { + "type": "boolean" + }, + "is_oa": { + "type": "boolean" + }, + "is_paratext": { + "type": "boolean" + }, + "journal_is_in_doaj": { + "type": "boolean" + }, + "journal_is_oa": { + "type": "boolean" + }, + "journal_issn_l": { + "type": "keyword" + }, + "journal_issns": { + "type": "keyword" + }, + "journal_name": { + "type": "keyword" + }, + "oa_locations": { + "properties": { + "evidence": { "type": "keyword" }, + "host_type":{ "type": "keyword" }, + "is_best": { "type": "boolean" }, + "license": { "type": "keyword" }, + "pmh_id": { "type": "keyword" }, + "updated": { "type": "date" }, + "url": { "type": "keyword" }, + "url_for_landing_page": { "type": "keyword" }, + "url_for_pdf": { "type": "keyword" }, + "version": { "type": "keyword" } + } + }, + "oa_status": { + "type": "keyword" + }, + "published_date": { + "type": "date", + "format": "iso8601" + }, + "publisher": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "updated": { + "type": "date", + "format": "iso8601" + }, + "year": { + "type": "keyword" + }, + "z_authors": { + "properties": { + "family": { "type": "keyword" }, + "given": { "type": "keyword" }, + "sequence": { "type": "keyword" }, + "ORCID": { "type": "keyword" }, + "authenticated-orcid": { "type": "boolean" }, + "affiliation": { + "properties": { + "name": { "type": "keyword" } + } + } + } + } + } + }, + "referencedAt": { + "type": "date" + } + } + } +} From 7812f67d5918b738cc06ce4fed5d5a4fdc3df5be Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Thu, 28 Sep 2023 10:15:01 +0200 Subject: [PATCH 002/114] feat: history update --- .gitignore | 1 + .../lib/controllers/changefiles.js | 5 + .../snapshots/changefiles-day-example.json | 9 + src/fakeUnpaywall/snapshots/history1.jsonl.gz | Bin 1952 -> 520 bytes src/fakeUnpaywall/snapshots/history2.jsonl.gz | Bin 1934 -> 526 bytes src/fakeUnpaywall/snapshots/history3.jsonl.gz | Bin 0 -> 526 bytes src/fakeUnpaywall/sources/history1.jsonl | 2 + src/fakeUnpaywall/sources/history2.jsonl | 2 + src/fakeUnpaywall/sources/history3.jsonl | 2 + src/graphql/lib/models/index.js | 3 + src/graphql/lib/models/unpaywall2.js | 30 ++ src/graphql/lib/resolvers/index.js | 2 + src/graphql/lib/resolvers/unpaywall2.js | 158 ++++++++ src/update/lib/download.js | 10 +- src/update/lib/history.js | 327 ++++++++++++++++ src/update/lib/insert.js | 319 +--------------- src/update/lib/job.js | 10 +- src/update/lib/lab/history-v1.js | 351 ++++++++++++++++++ src/update/lib/models/state.js | 17 +- src/update/lib/routers/job.js | 2 +- src/update/mapping/unpaywall_enriched.json | 241 ++++++------ src/update/mapping/unpaywall_history.json | 254 +++++++++---- src/update/test/auth.js | 2 +- src/update/test/history.js | 179 +++++++++ src/update/test/insertionError.js | 2 +- src/update/test/insertionFile.js | 8 +- src/update/test/insertionPeriodDay.js | 6 +- src/update/test/insertionPeriodWeek.js | 6 +- src/update/test/insertionSnapshot.js | 2 +- src/update/test/updateDay.js | 8 +- src/update/test/updateWeek.js | 4 +- src/update/test/utils/reset.js | 5 + 32 files changed, 1413 insertions(+), 554 deletions(-) create mode 100644 src/fakeUnpaywall/snapshots/history3.jsonl.gz create mode 100644 src/fakeUnpaywall/sources/history1.jsonl create mode 100644 src/fakeUnpaywall/sources/history2.jsonl create mode 100644 src/fakeUnpaywall/sources/history3.jsonl create mode 100644 src/graphql/lib/models/unpaywall2.js create mode 100644 src/graphql/lib/resolvers/unpaywall2.js create mode 100644 src/update/lib/history.js create mode 100644 src/update/lib/lab/history-v1.js create mode 100644 src/update/test/history.js diff --git a/.gitignore b/.gitignore index b9eda26d..302a4864 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ ezunpaywall.local.env.sh +ezunpaywall.env.local.sh config/kibana/certificates/ config/elastic/certificates/ certs/instances.yml diff --git a/src/fakeUnpaywall/lib/controllers/changefiles.js b/src/fakeUnpaywall/lib/controllers/changefiles.js index 63f5dc6f..ca0e06d3 100644 --- a/src/fakeUnpaywall/lib/controllers/changefiles.js +++ b/src/fakeUnpaywall/lib/controllers/changefiles.js @@ -88,6 +88,11 @@ async function updateChangefilesExample(interval) { changefilesDay.list[6].last_modified = new Date(now - (2 * oneYear) - (2 * oneDay)) .toISOString().slice(0, 19); + changefilesDay.list[7].date = new Date(now - (2 * oneYear) - (3 * oneDay)) + .toISOString().slice(0, 10); + changefilesDay.list[7].last_modified = new Date(now - (2 * oneYear) - (3 * oneDay)) + .toISOString().slice(0, 19); + try { await fs.writeFile(changefilesDayPath, JSON.stringify(changefilesDay, null, 2), 'utf8'); } catch (err) { diff --git a/src/fakeUnpaywall/snapshots/changefiles-day-example.json b/src/fakeUnpaywall/snapshots/changefiles-day-example.json index 028a59c5..549f774b 100644 --- a/src/fakeUnpaywall/snapshots/changefiles-day-example.json +++ b/src/fakeUnpaywall/snapshots/changefiles-day-example.json @@ -62,6 +62,15 @@ "lines": 50, "size": 19896, "url": "http://fakeUnpaywall:3000/daily-feed/changefiles/history2.gz?api_key=default" + }, + { + "date": "", + "filename": "history3.jsonl.gz", + "filetype": "jsonl", + "last_modified": "", + "lines": 50, + "size": 19896, + "url": "http://fakeUnpaywall:3000/daily-feed/changefiles/history3.gz?api_key=default" } ] } \ No newline at end of file diff --git a/src/fakeUnpaywall/snapshots/history1.jsonl.gz b/src/fakeUnpaywall/snapshots/history1.jsonl.gz index b8159e18e958216d0149237d708c6d66eed134ad..9cef31cc9d1cebafcde3587c1e22342a930f56c9 100644 GIT binary patch literal 520 zcmV+j0{8tNiwFqI6%=Iv188Y;bZ>HbF)nIzZ*FV=?Nz~!(=ZUd^A(X#Af;O;$DTQH zKpa+F5VG9ZNxX7B#&#+~wg1kHlWhabVGAojt3-(sznOSu-WzL9Oo3s$f>r#6bH3&D z!kd+B)H0x@M!001c0rTh^ieVs1xN%^7<9>!Nj95r^G8X$iXm8n`jmX!ZMO*BKU^vHQ=tN-qN@_cfuRl{R9rcQ6De zt#I<7rycXUcu(t`rG-W&18$D}WFq{b7lwFngS9#OUdME1*_uYXm}550W23OqBc{m{cEUtYrFV)B`CXi@EUP_0h;efxLTzK0U-F(&+Va4-!B-<-xeL ztO%J9Ed=3J`v|krDdRdMywgvkD`*eVBnYE0E@BT3X{v=bCi#$q+%itdKuOdW@qX;z z3mUU%jOf|mLtHM6!v1VM)3y+TUxXk5#iAw9VQq+i(-7r!uvazy6Ezk`(KtZ~pQ-F+ z6pP;*@Z)T|e|7rx$g2L@$ht}vep$odXl K{LLR_4FCWQ{rr0X literal 1952 zcmV;R2VeLfiwFoRLJVa9188Y;bZ>HbF)nIzZ*FV=?O1DX+c*~ezQ2MHFhC1LvZ#kG z30N%B%%JT|lVHFIdzNq?|UiR@gwmo3A*{vATbsddup z!4pTSQE!(E+Bzo<%fZTXJ=ew#vig`6nXQ?AW>i$#dRtDM-0y^*bl4 zR{w4fvU=0OUgrOrKRU74gy(Z7NrL81 zKrbf1%3PA`moQ{3b(1#Jnyt*BjH$?L$wjkr67WP?NR`%kvH7SZUD$);N9;&%DOO9( z8>yx#7m!k#w&~7e7BEa$`X44_tw|AeMZ0MU55%wSv~1I6L5)dtd?6?B65cSRQm{XrN}5ul>)O;!>qLLm_u(cU}cJ= zgJ&M+z1*iBp>AaN1B&ec$AO)>A$$aw_%vXHASRC@$1H~30fhDy4iYa+{J_D1AGy(S zG5nOI85C~hg_H)E1w~A+M z1-%vUS#`LhE07*SmU1)&@@=z*dx9CcfdER95rF$HH*aok7Ux(D@717RNO?3*VVdMw zaJ3I~y~1AN1qqIwXwdhABk!9hO>hMXa7Ke1)Fv4egMw`5>f_Pa!`mGTFwt8>OE0Ce zJ(J3|ENmlP?YUI8VGVmBmF;?LOJUEYvb_|<-KeEhUVQ`etw?42Rw~<_RJL!W@{2>5 z^U@hNRk)p|DM~z_|KQfcUzIl#HDL=RM;eqi`WpJOu7Z1ARQXj>Rov_@%xGE zJAUY69IOFOC}nl?W3pkPgW_g#J-%tLtE$eYT2Qq$9sUS3_OR_oY+-A{`WOYQ&~@Eum1Ym_tkc( z8}#bh=?|RHi9PSOQ3`iEUN%bS%h>aqa@i!EUoDF+=_UB99s@b@X~`N8Zn7km$uz^M zXvR$QuKjdIH0%kzVnm?dL=GlQRr|Y|cXoRBzVf0@z7LWRN=-iolZ}361~tw%w$af% zt(XFZ393s{&Su5XC1BQ|SKO3JAZ0araXP$w^>S-hb4bbwfm1Pby4Kemn!X&}=-%D! zYQAH-X8VzhM8~%MJ;ST9tcXU#HT!c3`!VKiQSh(kT2RiT+lZZCF2_^dxn{b% zeK=QLwK1w^HXWW6?lP!Pp(^QxJpj`wfX*8#XLo+st5|Vl2!Z=oxS!xKaXrV2apd~P zB!wDk;VvB%iZKM+drV4rutSc?B6nisI2C)CME1hS#ZO;c1Xr$Wv?lhPFvQ;Ak)`)@ zEG`zFJE{Q;o6wGC73uUJD9Vzk8EFQY-}B?prA0r;C?!!p4x<6~eVpOApd`z3*7yBj zcVr!n7mmBy02{oz-!-6gn^n9yuqs#_3(e1UG+tJts?j$kBK9SiejVibr23|}^?{$v zlY7W;-x4)}TdrH^kahNPcB3dh!6AI}!XUa&s^1Q~PpTVrV7o@9 m?3L<2uYl)$1?-dcKk<9!_}y0m%tL=i4e&ofhxVZ}ApihcZp@JY diff --git a/src/fakeUnpaywall/snapshots/history2.jsonl.gz b/src/fakeUnpaywall/snapshots/history2.jsonl.gz index 6b53693e388bbe1bf8faba0cfbc2214fc74950f2..d15c4fb6b1c145c0495ab5740803174d521a63df 100644 GIT binary patch literal 526 zcmV+p0`dJHiwFoA78GRw188Y;bZ>HbGA?R!Z*FV=?NvQ*+b|H_`zr)a4IsyqM$er( z6di&tMM2OaB{86REI*7YM*e$vlxz!02RjMSB9I_K;=2>??!6=KKm;hVDM&?cDWz+Q zuXK}=43fKKsS{k1P8uiCZ~Q2U@(g$aTCTk;Nl-y&`(6Ghaa$IhMyLlZ$AT^MO46%4J&aQgVO7x!8M%&T!2P z-^p>uye_`sI%lb&;X#3!VqZ*z-Stc%4<>4zqi?0h$ic-FSgS@FpJO)7YCDX#Z-9ld9QdOfZyLFpB{W*VX$((2MQpKu%Jws zSCp7wEjVUH`vTM42xYoNcqcET%Sj7f1#kmX#-|>PVXOr=M)?v4xn+#t4wI}#J_2Xay;0J8vltJv!kdC!-P*% z_T01C?+w^tvORt}{C#Fs7tgFQ#Op`aH3ELHbGA?R!Z*FV=?O17V+%^#X-d{loD3AcwK^?o2 zfTD2RB1dD#&ALs4q7W#F+NCX#3Ms9#3HskV)b4r@doSBa{UJbN*c6Ayyf?!)^VP@& zGhzcf7RF)ER=yKp$8>N2|DDL}8Ary15^2P)<2Xmgka9_3{-tOoC#6Yb!?KdXET$4c zjJT04Ju({BlzQ4=!l@iG1(Ogh3Ys-Y6zG~sic-pR#)pXT94S(=F_9Bgze0i|)yjfo z6xCAH6gHr$n^nsz6S5hJTyVXMY6#;cRSJHiv_wQ^BfX!3Bf?daR+ezC zQPb2awrv=(gdE!2uHV)oHX-Z4RuV7HoPpw`Vh6^TDWBhxxF#kySiXB}wAbHLjCIHbD zt2c}+Tm9AUW%a6MpW78Xw%>N|82~KvWz#_;%5pE`-@&~~@Zqxc$1VH*(^0l262Sf3 z7;!;L1(rYf6j)E3_51{%t?Zv znG?{nF|aZhD z(XZWV==}7=SVdlQsct2;WIMEYXh~$TA;OzlcbhNAyn8+~4~<(!>20-N>guIO{ywOtgG>DFxSWGe`E6Vidtq z&}$Aqs}6T`0n&quQVb_RzD?HfPEaj35I{*%0&w5v=JoaU{2ue^b`AQ4ln3(^mWiJP zS9?I$3+%?OALGah`#sM;@V$A|1XmCrrko2UL!MZwls}9lt+$tqsn|RHLEe zjO4TeQU87(S=#sn6d_54oce7gOa~LVNwp#Dc$P8%qOv#}Swk__Z_nZ(f~REh5N*(9 z>j5(TygfQ@sWFVB2WE0OH_K5e#8q4G9~k#lW(T@%f03pap9h8DP{dH{GHWfGw-#w_ zA5bRA%HCw}L`T#qrXe=f^&_!Hmxuy5Qj?lydcbf{IUd*R~P>&1>| zd4Y$qzXCYPGFms^CL0#YFRsUzqwDsvs_J~AcqTWd!*7Ac4z_)ZBWz4qZ=mK=kYB{r z@*8j7q@Q=)p%7`%oin?dQosGW5x{d@$;R%YqFiZ^EC!XX9t{Ye9#F@3jn!{Sx%TlU zTijU+YUUpRGMA{CTs1?6iS>gbgiLzTu})E6GWHD7@gKa=KxuQUNBuL zfs$3|+411~#q*6#%>gOL1a3vq@k(2BpxbiUTJ!F9Q}ZoVC8OxK726MF$UCy_ZW&&T zL`4)Dtk|ClIFB}Oi-LXDuY$B5+(qpCaz2`<&NY+m-NTvasB zr=;npp6x|}lNCKb%`y`9q9E*J&%-H>ij1UbPJ5o`Z;z~l@xoD88eolA_iqI#)ut6| z_ACnK*Fv*<9gLUJ$ZGgS@{oS=C!hOyKCZr~O>N*u^W+XP+_gnb;Faq(+Gn3#9N9;n zuO;xgK6f0?HJvbwj&K0Kxq%HbGcIa#Z*FV=?NvQ*+b|H_`zr)a4IsyqM$er( z6d8gpMM2OaB{85$h95>%BmX^jlwzw*2RjXrAdn!z;=42N?!BY#K!jCfQ;>>YQ%ctq zf8Zu186@{$sR1rYCykfrH@=ia1!g$_(I;7wsG`p1+x$}Et}JcUoAnx?YwNr9 z^GL}F9;HC%n$~nhE4Xi~dPC_Z```tBma`CfOd%&?wE)betyZCzm@2tBE=uy7@f3UO z(fP-LbZRdR`h2y$0dvcDN)HHpv6yg>_eq*27ttyoB6;|kt90B$a=#)NTCM{WduNk3 zTrK4f}Vsx0okl2z+q{Kw`quSJlKmG??jE+QB;Ou!bd85 z9@y;fjdjChd-!zt_sFWw9$C|8{N*$25(Pi>%&L|{amk^mKK#sD8se23;=g9rqQ { 'foo.bar': 'foo' } + * + * @param {Object} obj - Object need to be flatten. + * + * @returns {Object} Flatten object. + */ +function flatten(obj) { + const flattened = []; + + function flattenProp(data, keys) { + Object.entries(data).forEach(([key, value]) => { + if (typeof value !== 'object') { return; } + + const newKeys = [...keys, key]; + + if (Object.keys(value).length === 0) { + flattened.push(newKeys.join('.')); + } else { + flattenProp(value, newKeys); + } + }); + } + + flattenProp(obj, []); + + return flattened; +} + +async function unpaywall(parent, args, req, info) { + const apikey = req.get('X-API-KEY'); + + if (!apikey) { + throw Error('Not authorized'); + } + + let key; + try { + key = await redisClient.get(apikey); + } catch (err) { + logger.error(`[redis] Cannot get [${apikey}]`, err); + throw Error('Internal server error'); + } + + let apiKeyConfig; + try { + apiKeyConfig = JSON.parse(key); + } catch (err) { + logger.error(`[redis] Cannot parse [${key}]`, err); + throw Error('Internal server error'); + } + + if (!Array.isArray(apiKeyConfig?.access) || !apiKeyConfig?.access?.includes('graphql') || !apiKeyConfig?.allowed) { + throw Error('Not authorized'); + } + + // Demo apikey + if (apikey === 'demo') { + if ((apiKeyConfig.count - args.dois.length) < 0) { + throw Error('Not authorized'); + } + apiKeyConfig.count -= args.dois.length; + try { + await redisClient.set(apikey, `${JSON.stringify(apiKeyConfig)}`); + } catch (err) { + logger.error(`[redis] Cannot update apikey [${apikey}] with config [${JSON.stringify(apiKeyConfig)}]`, err); + throw err; + } + } + + let index = req?.get('index'); + + const { attributes } = req; + + if (!index) { + index = 'unpaywall_history'; + } + + if (!attributes.includes('*')) { + const test = graphqlFields(info); + const requestedField = flatten(test); + + requestedField.forEach((field) => { + if (!attributes.includes(field)) { + throw Error(`You don't have access to ${field}`); + } + }); + } + + const dois = []; + + req.countDOI = args?.dois?.length; + + // Normalize request + args.dois.forEach((doi) => { + dois.push(doi.toLowerCase()); + }); + + const filter = [{ terms: { doi: dois } }]; + + const query = { + bool: { + filter, + }, + }; + + const { date } = args; + if (date) { + const rangeLte = { + range: { + updated: { + lte: date, + }, + }, + }; + + const rangeGte = { + range: { + endValidity: { + gt: date, + }, + }, + }; + + filter.push(rangeLte); + filter.push(rangeGte); + } + + let res; + try { + res = await elasticClient.search({ + index, + body: { + query, + sort: 'updated', + _source: attributes, + }, + + }); + } catch (err) { + logger.error('[elastic] Cannot request elastic', err); + return null; + } + // eslint-disable-next-line no-underscore-dangle + return res.body.hits.hits.map((hit) => hit._source); +} + +module.exports = unpaywall; diff --git a/src/update/lib/download.js b/src/update/lib/download.js index ff2feb99..a764f27b 100644 --- a/src/update/lib/download.js +++ b/src/update/lib/download.js @@ -2,6 +2,7 @@ const path = require('path'); const fs = require('fs-extra'); const { Readable } = require('stream'); const { format } = require('date-fns'); +const { setTimeout } = require('node:timers/promises'); const logger = require('./logger'); const { @@ -34,6 +35,7 @@ async function updatePercentStepDownload(filepath, size, start) { return; } const step = getLatestStep(); + logger.debug(`updatePercentStepDownload : ${step.percent}`); let bytes; try { bytes = await fs.stat(filepath); @@ -47,7 +49,7 @@ async function updatePercentStepDownload(filepath, size, start) { step.took = (new Date() - start) / 1000; step.percent = ((bytes.size / size) * 100).toFixed(2); updateLatestStep(step); - await new Promise((resolve) => { setTimeout(resolve, 1000); }); + await setTimeout(1000); updatePercentStepDownload(filepath, size, start); } @@ -68,8 +70,10 @@ async function download(file, filepath, size) { const writeStream = file.pipe(fs.createWriteStream(filepath)); const start = new Date(); - // update the percentage of the download step in parallel - updatePercentStepDownload(filepath, size, start); + writeStream.on('ready', async () => { + // update the percentage of the download step in parallel + updatePercentStepDownload(filepath, size, start); + }); writeStream.on('finish', async () => { step.status = 'success'; diff --git a/src/update/lib/history.js b/src/update/lib/history.js new file mode 100644 index 00000000..147fd375 --- /dev/null +++ b/src/update/lib/history.js @@ -0,0 +1,327 @@ +/* eslint-disable consistent-return */ +/* eslint-disable no-param-reassign */ +/* eslint-disable no-restricted-syntax */ + +const path = require('path'); +const fs = require('fs-extra'); +const readline = require('readline'); +const config = require('config'); +const zlib = require('zlib'); + +const logger = require('./logger'); +const unpaywallEnrichedMapping = require('../mapping/unpaywall.json'); +const unpaywallHistoryMapping = require('../mapping/unpaywall_history.json'); + +const { + addStepInsert, + getLatestStep, + updateLatestStep, + fail, +} = require('./models/state'); + +const { + refreshIndex, + getDataByListOfDOI, + bulk, + createIndex, +} = require('./services/elastic'); + +const maxBulkSize = config.get('elasticsearch.maxBulkSize'); + +const snapshotsDir = path.resolve(__dirname, '..', 'data', 'snapshots'); + +/** + * Insert data on elastic with elastic bulk request. + * + * @param {Array} data - Array of unpaywall data. + * + * @returns {Promise} Success or not. + */ +async function insertUnpaywallDataInElastic(data, index) { + const step = getLatestStep(); + let res; + try { + res = await bulk(data); + } catch (err) { + logger.error('[elastic] Cannot bulk', err); + await fail(err?.[0]?.reason); + return false; + } + + const errors = []; + const items = Array.isArray(res?.body?.items) ? res?.body?.items : []; + + items.forEach((i) => { + if (i?.index?.result === 'created') { + step.index[index].insertedDocs += 1; + return; + } + if (i?.index?.result === 'updated') { + step.index[index].updatedDocs += 1; + return; + } + + if (i?.index?.error !== undefined) { + step.index[index].failedDocs += 1; + errors.push(i?.index?.error); + } + }); + + if (errors.length > 0) { + logger.error('[elastic] Error in bulk insertion'); + errors.forEach((error) => { + logger.error(`[elastic] ${JSON.stringify(error, null, 2)}`); + }); + step.status = 'error'; + updateLatestStep(step); + await fail(errors); + return false; + } + + updateLatestStep(step); + + return true; +} +/** + * // TODO DOC + * @param {*} listOfDoi + * @param {*} newData + * @param {*} date + * @param {*} step + * @returns + */ +async function insertData(listOfDoi, newData, date) { + // TODO not hardcode + const oldData = await getDataByListOfDOI(listOfDoi, 'unpaywall_enriched'); + + const resHistoryData = []; + const resData = []; + + if (!oldData) { + newData.forEach((data) => { + const copyData = data; + copyData.referencedAt = date; + // classic insertion + // TODO not hardcode + resData.push({ index: { _index: 'unpaywall_enriched', _id: data.doi } }); + resData.push(copyData); + }); + return true; + } + + const oldUnpaywallDataMap = new Map( + oldData.map((data) => { + const copyData = { ...data }; + return [data.doi, copyData]; + }), + ); + + newData.forEach((data) => { + const copyData = data; + copyData.referencedAt = date; + // history insertion + const oldDataUnpaywall = oldUnpaywallDataMap.get(data.doi); + + if (oldDataUnpaywall) { + const newEntry = { + ...oldDataUnpaywall, + date, + }; + newEntry.endValidity = data.updated; + + // TODO not hardcode + resHistoryData.push({ index: { _index: 'unpaywall_history' } }); + resHistoryData.push(newEntry); + + // if data, get the old referencedAt + copyData.referencedAt = oldDataUnpaywall.referencedAt; + } + + // classic insertion + // TODO not hardcode + resData.push({ index: { _index: 'unpaywall_enriched', _id: data.doi } }); + resData.push(copyData); + }); + + // TODO not hardcode + await insertUnpaywallDataInElastic(resData, 'unpaywall_enriched'); + + if (resHistoryData.length > 0) { + // TODO not hardcode + await insertUnpaywallDataInElastic(resHistoryData, 'unpaywall_history'); + } + + return true; +} + +/** + * Inserts the contents of an unpaywall data update file with history. + * + * @param {Object} insertConfig - Config of insertion. + * @param {string} insertConfig.filename - Name of the snapshot file from which the data will + * be retrieved to be inserted into elastic. + * @param {string} insertConfig.date - Date of file. + * @param {string} insertConfig.index - Name of the index to which the data will be inserted. + * @param {number} insertConfig.offset - Line of the snapshot at which the data insertion starts. + * @param {number} insertConfig.limit - Line in the file where the insertion stops. + * + * @returns {Promise} Success or not. + */ +async function insertHistoryDataUnpaywall(insertConfig) { + const { + filename, date, index, offset, limit, + } = insertConfig; + + try { + // TODO not hardcode index + await createIndex('unpaywall_enriched', unpaywallEnrichedMapping); + } catch (err) { + logger.error(`[elastic] Cannot create index [${index}]`, err); + await fail(err); + return false; + } + + try { + // TODO not hardcode index + await createIndex('unpaywall_history', unpaywallHistoryMapping); + } catch (err) { + logger.error(`[elastic] Cannot create index [${index}]`, err); + await fail(err); + return false; + } + + // step insertion in the state + const start = new Date(); + addStepInsert(filename); + const step = getLatestStep(); + step.file = filename; + step.index = { + // TODO not hardcode + unpaywall_enriched: { + insertedDocs: 0, + updatedDocs: 0, + failedDocs: 0, + }, + unpaywall_history: { + insertedDocs: 0, + updatedDocs: 0, + failedDocs: 0, + }, + }; + updateLatestStep(step); + + const filePath = path.resolve(snapshotsDir, filename); + + // get information bytes for state + let bytes; + try { + bytes = await fs.stat(filePath); + } catch (err) { + logger.error(`[job: insert] Cannot stat [${filePath}]`, err); + await fail(err); + return false; + } + + // read file with stream + let readStream; + try { + readStream = fs.createReadStream(filePath); + } catch (err) { + logger.error(`[job: insert] Cannot read [${filePath}]`, err); + await fail(err); + return false; + } + + // get information "loaded" for state + let loaded = 0; + readStream.on('data', (chunk) => { + loaded += chunk.length; + }); + + let decompressedStream; + try { + decompressedStream = readStream.pipe(zlib.createGunzip()); + } catch (err) { + logger.error(`[job: insert] Cannot pipe [${readStream?.filename}]`, err); + await fail(err); + return false; + } + + const rl = readline.createInterface({ + input: decompressedStream, + crlfDelay: Infinity, + }); + + let newData = []; + let listOfDoi = []; + + let success; + + logger.info(`[job: insert] Start insert with [${filename}]`); + + // Reads line by line the output of the decompression stream to make packets of 1000 + // to insert them in bulk in an elastic + for await (const line of rl) { + // limit + if (step.linesRead === limit) { + break; + } + + step.linesRead += 1; + + // offset + if (step.linesRead >= offset + 1) { + // fill the array + try { + const doc = JSON.parse(line); + // [history] + listOfDoi.push(doc.doi); + newData.push(doc); + } catch (err) { + logger.error(`[job: insert] Cannot parse [${line}] in json format`, err); + await fail(err); + return false; + } + } + // bulk insertion + if (newData.length >= maxBulkSize) { + success = await insertData(listOfDoi, newData, date); + listOfDoi = []; + newData = []; + + if (!success) return false; + step.percent = ((loaded / bytes.size) * 100).toFixed(2); + step.took = (new Date() - start) / 1000; + updateLatestStep(step); + } + if (step.linesRead % 100000 === 0) { + logger.info(`[job: insert] ${step.linesRead} Lines reads`); + updateLatestStep(step); + } + } + // last insertion if there is data left + if (newData.length > 0) { + // [history] + success = await insertData(listOfDoi, newData, date); + listOfDoi = []; + newData = []; + if (!success) return false; + } + + logger.info('[job: insert] insertion completed'); + + try { + await refreshIndex(); + } catch (err) { + logger.warn('[elastic] Cannot refresh the index', err); + } + + // last update of step + step.status = 'success'; + step.took = (new Date() - start) / 1000; + step.percent = 100; + updateLatestStep(step); + return true; +} + +module.exports = insertHistoryDataUnpaywall; diff --git a/src/update/lib/insert.js b/src/update/lib/insert.js index f352c3d7..5e9f8969 100644 --- a/src/update/lib/insert.js +++ b/src/update/lib/insert.js @@ -9,8 +9,7 @@ const config = require('config'); const zlib = require('zlib'); const logger = require('./logger'); -const unpaywallEnrichedMapping = require('../mapping/unpaywall.json'); -const unpaywallHistoryMapping = require('../mapping/unpaywall_history.json'); +const unpaywallMapping = require('../mapping/unpaywall.json'); const { addStepInsert, @@ -24,7 +23,6 @@ const { bulk, initAlias, createIndex, - getDataByListOfDOI, } = require('./services/elastic'); const indexAlias = config.get('elasticsearch.indexAlias'); @@ -110,7 +108,15 @@ async function insertDataUnpaywall(insertConfig) { step.index = index; updateLatestStep(step); - await initAlias(index, unpaywallEnrichedMapping, indexAlias); + try { + await createIndex(index, unpaywallMapping); + } catch (err) { + logger.error(`[elastic] Cannot create index [${index}]`, err); + await fail(err); + return false; + } + + await initAlias(index, unpaywallMapping, indexAlias); const filePath = path.resolve(snapshotsDir, filename); @@ -221,307 +227,4 @@ async function insertDataUnpaywall(insertConfig) { return true; } -/** - * Gets difference between 2 objects - * - * @param v1 First object to compare - * @param v2 Second object to compare - * @param prefix Prefix of returned keys. Should be empty on the first iteration - * - * @returns Keys that are in v1 but not in v2 - */ -function objectDiff(v1, v2, prefix = '') { - const diffs = Object.keys(v1 ?? {}).reduce( - (previous, key) => { - const prefixedKey = `${prefix}${key}`; - if (typeof v1[key] === 'object') { - return [ - ...previous, - ...objectDiff(v1[key], v2[key], `${prefixedKey}.`), - ]; - } - if (v1[key] !== v2[key]) { - return [ - ...previous, - prefixedKey, - ]; - } - return previous; - }, - [], - ); - return diffs; - // const diffs = Object.keys(v1 ?? {}).reduce( - // (d, key) => { - // const prefixedKey = `${prefix}${key}`; - // if ( - // Array.isArray(v2[key]) - // ? !v2[key].map((v) => typeof v).includes(typeof v1[key]) - // : typeof v1[key] !== typeof v2[key] - // ) { - // d.push(prefixedKey); - // } else if (typeof v1[key] === 'object' && !Array.isArray(v1[key])) { - // d.push( - // ...objectDiff( - // v1[key], - // v2[key], - // `${prefixedKey}.`, - // ), - // ); - // } - // return d; - // }, - // [], - // ); - // return diffs; -} - -/** - * 1. Insert unpaywall data in elastic with history - * 2. Insert history data in elastic - * - * @param {Array} listOfDoi - List of DOI - * @param {Array} newData - List of unpaywall data - * @param {Date} date - Date of file - */ -async function insertData(listOfDoi, newData, date) { - // TODO date of file - date = new Date(); - - const oldData = await getDataByListOfDOI(listOfDoi, 'unpaywall'); - const historyData = await getDataByListOfDOI(listOfDoi, 'unpaywall_history'); - - let oldHistoryDataMap; - let oldUnpaywallDataMap; - const resHistoryData = []; - - if (historyData.length > 0) { - oldHistoryDataMap = new Map( - historyData.map((data) => { - const copyData = { ...data }; - return [data.doi, copyData]; - }), - ); - - oldUnpaywallDataMap = new Map( - oldData.map((data) => { - const copyData = { ...data }; - return [data.doi, copyData]; - }), - ); - // enrich history - newData.forEach((data) => { - const oldDataUnpaywall = oldUnpaywallDataMap.get(data.doi); - // if document already exist - if (oldDataUnpaywall) { - const diffs = objectDiff(data, oldDataUnpaywall); - if (diffs.length > 0) { - const enrichedHistoryData = oldHistoryDataMap.get(data.doi); - // If document hasn't got history - if (!enrichedHistoryData) { - resHistoryData.push({ index: { _index: 'unpaywall_history', _id: data.doi } }); - resHistoryData.push({ - doi: data.doi, - referencedAt: date, - history: [], - }); - } else { - const newEntryHistory = {}; - newEntryHistory.date = new Date(); - - diffs.forEach((diff) => { - newEntryHistory[diff] = oldDataUnpaywall[diff]; - }); - - enrichedHistoryData.history.push(newEntryHistory); - - resHistoryData.push({ index: { _index: 'unpaywall_history', _id: data.doi } }); - resHistoryData.push(enrichedHistoryData); - } - } - } - }); - } else { - // First insertion - newData.forEach((data) => { - const newEntry = { - doi: data.doi, - referencedAt: date, - history: [], - }; - resHistoryData.push({ index: { _index: 'unpaywall_history', _id: newEntry.doi } }); - resHistoryData.push(newEntry); - }); - } - - if (resHistoryData.length > 0) { - try { - await bulk(resHistoryData, 'unpaywall_history'); - } catch (err) { - logger.error(err); - } - } - - const bulkOps = []; - - newData.forEach((data) => { - bulkOps.push({ index: { _index: 'unpaywall', _id: data.doi } }); - bulkOps.push(data); - }); - - try { - await bulk(bulkOps, 'unpaywall'); - } catch (err) { - logger.error(err); - } - - // #endregion 2 - return true; -} - -/** - * Inserts the contents of an unpaywall data update file with oa history. - * - * @param {Object} insertConfig - Config of insertion. - * @param {string} insertConfig.filename - Name of the snapshot file from which the data will - * be retrieved to be inserted into elastic. - * @param {string} insertConfig.index - Name of the index to which the data will be inserted. - * @param {number} insertConfig.offset - Line of the snapshot at which the data insertion starts. - * @param {number} insertConfig.limit - Line in the file where the insertion stops. - * - * @returns {Promise} Success or not. - */ -async function insertHistoryDataUnpaywall(insertConfig) { - const { - filename, index, offset, limit, - } = insertConfig; - - // step insertion in the state - const start = new Date(); - addStepInsert(filename); - const step = getLatestStep(); - step.file = filename; - step.index = index; - updateLatestStep(step); - - await createIndex('unpaywall', unpaywallEnrichedMapping); - await createIndex('unpaywall_history', unpaywallHistoryMapping); - - await initAlias(index, unpaywallEnrichedMapping, indexAlias); - - const filePath = path.resolve(snapshotsDir, filename); - - // get information "bytes" for state - let bytes; - try { - bytes = await fs.stat(filePath); - } catch (err) { - logger.error(`[job: insert] Cannot stat [${filePath}]`, err); - await fail(err); - return false; - } - - // read file with stream - let readStream; - try { - readStream = fs.createReadStream(filePath); - } catch (err) { - logger.error(`[job: insert] Cannot read [${filePath}]`, err); - await fail(err); - return false; - } - - // get information "loaded" for state - let loaded = 0; - readStream.on('data', (chunk) => { - loaded += chunk.length; - }); - - let decompressedStream; - try { - decompressedStream = readStream.pipe(zlib.createGunzip()); - } catch (err) { - logger.error(`[job: insert] Cannot pipe [${readStream?.filename}]`, err); - await fail(err); - return false; - } - - const rl = readline.createInterface({ - input: decompressedStream, - crlfDelay: Infinity, - }); - - // array that will contain the packet of 1000 unpaywall data - let newData = []; - let listOfDoi = []; - - let success; - - logger.info(`[job: insert] Start insert with [${filename}]`); - - // Reads line by line the output of the decompression stream to make packets of 1000 - // to insert them in bulk in an elastic - for await (const line of rl) { - // limit - if (step.linesRead === limit) { - break; - } - - step.linesRead += 1; - - // offset - if (step.linesRead >= offset + 1) { - // fill the array - try { - const doc = JSON.parse(line); - listOfDoi.push(doc.doi); - newData.push(doc); - } catch (err) { - logger.error(`[job: insert] Cannot parse [${line}] in json format`, err); - await fail(err); - return false; - } - } - // bulk insertion - if (newData.length >= maxBulkSize) { - success = await insertData(listOfDoi, newData); - if (!success) return false; - listOfDoi = []; - newData = []; - step.percent = ((loaded / bytes.size) * 100).toFixed(2); - step.took = (new Date() - start) / 1000; - updateLatestStep(step); - } - if (step.linesRead % 100000 === 0) { - logger.info(`[job: insert] ${step.linesRead} Lines reads`); - updateLatestStep(step); - } - } - // last insertion if there is data left - if (newData.length > 0) { - success = await insertData(listOfDoi, newData); - if (!success) return false; - listOfDoi = []; - newData = []; - } - - logger.info('[job: insert] insertion completed'); - - try { - await refreshIndex(); - } catch (err) { - logger.warn('[elastic] Cannot refresh the index', err); - } - - // last update of step - step.status = 'success'; - step.took = (new Date() - start) / 1000; - step.percent = 100; - updateLatestStep(step); - return true; -} - -module.exports = { - insertDataUnpaywall, - insertHistoryDataUnpaywall, -}; +module.exports = insertDataUnpaywall; diff --git a/src/update/lib/job.js b/src/update/lib/job.js index 8fcbd2cb..8ff1bc8c 100644 --- a/src/update/lib/job.js +++ b/src/update/lib/job.js @@ -22,10 +22,9 @@ const { getLatestStep, } = require('./models/state'); -const { - insertDataUnpaywall, - insertHistoryDataUnpaywall, -} = require('./insert'); +const insertDataUnpaywall = require('./insert'); + +const insertHistoryDataUnpaywall = require('./history'); const { getChangefiles, @@ -132,7 +131,7 @@ async function insertChangefile(jobConfig) { } /** - * Download and insert on elastic the changefiles from unpaywall between a period. + * Download and insert on elastic the changefiles from unpaywall between a period with history. * * @param {Object} jobConfig - Config of job. * @param {string} jobConfig.index - Name of the index to which the data will be inserted. @@ -179,6 +178,7 @@ async function insertWithOaHistoryJob(jobConfig) { success = await downloadChangefile(changefilesInfo[i], interval); if (!success) return; jobConfig.filename = changefilesInfo[i].filename; + jobConfig.date = changefilesInfo[i].date; success = await insertHistoryDataUnpaywall(jobConfig); if (!success) return; } diff --git a/src/update/lib/lab/history-v1.js b/src/update/lib/lab/history-v1.js new file mode 100644 index 00000000..6cd71b48 --- /dev/null +++ b/src/update/lib/lab/history-v1.js @@ -0,0 +1,351 @@ +/* eslint-disable consistent-return */ +/* eslint-disable no-param-reassign */ +/* eslint-disable no-restricted-syntax */ + +const path = require('path'); +const fs = require('fs-extra'); +const readline = require('readline'); +const config = require('config'); +const zlib = require('zlib'); + +const logger = require('../logger'); +const unpaywallEnrichedMapping = require('../../mapping/unpaywall.json'); +const unpaywallHistoryMapping = require('../../mapping/unpaywall_history.json'); + +const { + addStepInsert, + getLatestStep, + updateLatestStep, + fail, +} = require('../models/state'); + +const { + refreshIndex, + bulk, + initAlias, + createIndex, + getDataByListOfDOI, +} = require('../services/elastic'); + +const indexAlias = config.get('elasticsearch.indexAlias'); +const maxBulkSize = config.get('elasticsearch.maxBulkSize'); + +const snapshotsDir = path.resolve(__dirname, '..', 'data', 'snapshots'); + +/** + * Gets difference value between 2 objects + * + * @param v1 First object to compare + * @param v2 Second object to compare + * @param prefix Prefix of returned keys. Should be empty on the first iteration + * + * @returns Keys that are in v1 but not in v2 + */ +function objectValueDiff(v1, v2, prefix = '') { + const diffs = Object.keys(v1 ?? {}).reduce( + (previous, key) => { + const prefixedKey = `${prefix}${key}`; + if (typeof v1[key] === 'object') { + return [ + ...previous, + ...objectValueDiff(v1[key], v2[key], `${prefixedKey}.`), + ]; + } + if (v1[key] !== v2[key]) { + return [ + ...previous, + prefixedKey, + ]; + } + return previous; + }, + [], + ); + return diffs; +} + +/** + * Gets difference keys between 2 objects + * + * @param v1 First object to compare + * @param v2 Second object to compare + * @param prefix Prefix of returned keys. Should be empty on the first iteration + * + * @returns Keys that are in v1 but not in v2 + */ +function objectKeyDiff(v1, v2, prefix = '') { + const diffs = Object.keys(v1 ?? {}).reduce( + (d, key) => { + const prefixedKey = `${prefix}${key}`; + if ( + Array.isArray(v2[key]) + ? !v2[key].map((v) => typeof v).includes(typeof v1[key]) + : typeof v1[key] !== typeof v2[key] + ) { + d.push(prefixedKey); + } else if (typeof v1[key] === 'object' && !Array.isArray(v1[key])) { + d.push( + ...objectKeyDiff( + v1[key], + v2[key], + `${prefixedKey}.`, + ), + ); + } + return d; + }, + [], + ); + return diffs; +} + +/** + * 1. Insert unpaywall data in elastic with history + * 2. Insert history data in elastic + * + * @param {Array} listOfDoi - List of DOI + * @param {Array} newData - List of unpaywall data + * @param {Date} date - Date of file + */ +async function insertData(listOfDoi, newData, date) { + // TODO date of file + date = new Date(); + + const oldData = await getDataByListOfDOI(listOfDoi, 'unpaywall'); + const historyData = await getDataByListOfDOI(listOfDoi, 'unpaywall_history'); + + let oldHistoryDataMap; + let oldUnpaywallDataMap; + const resHistoryData = []; + + if (historyData.length > 0) { + oldHistoryDataMap = new Map( + historyData.map((data) => { + const copyData = { ...data }; + return [data.doi, copyData]; + }), + ); + + oldUnpaywallDataMap = new Map( + oldData.map((data) => { + const copyData = { ...data }; + return [data.doi, copyData]; + }), + ); + // enrich history + newData.forEach((data) => { + const oldDataUnpaywall = oldUnpaywallDataMap.get(data.doi); + // if document already exist + if (oldDataUnpaywall) { + const diffs = objectValueDiff(data, oldDataUnpaywall); + if (diffs.length > 0) { + const enrichedHistoryData = oldHistoryDataMap.get(data.doi); + // If document hasn't got history + if (!enrichedHistoryData) { + resHistoryData.push({ index: { _index: 'unpaywall_history', _id: data.doi } }); + resHistoryData.push({ + doi: data.doi, + referencedAt: date, + history: [], + }); + } else { + const newEntryHistory = {}; + newEntryHistory.date = new Date(); + + diffs.forEach((diff) => { + newEntryHistory[diff] = oldDataUnpaywall[diff]; + }); + + enrichedHistoryData.history.push(newEntryHistory); + + resHistoryData.push({ index: { _index: 'unpaywall_history', _id: data.doi } }); + resHistoryData.push(enrichedHistoryData); + } + } + } + }); + } else { + // First insertion + newData.forEach((data) => { + const newEntry = { + doi: data.doi, + referencedAt: date, + history: [], + }; + resHistoryData.push({ index: { _index: 'unpaywall_history', _id: newEntry.doi } }); + resHistoryData.push(newEntry); + }); + } + + if (resHistoryData.length > 0) { + try { + await bulk(resHistoryData, 'unpaywall_history'); + } catch (err) { + logger.error(err); + } + } + + const bulkOps = []; + + newData.forEach((data) => { + bulkOps.push({ index: { _index: 'unpaywall', _id: data.doi } }); + bulkOps.push(data); + }); + + try { + await bulk(bulkOps, 'unpaywall'); + } catch (err) { + logger.error(err); + } + + // #endregion 2 + return true; +} + +/** + * Inserts the contents of an unpaywall data update file with oa history. + * + * @param {Object} insertConfig - Config of insertion. + * @param {string} insertConfig.filename - Name of the snapshot file from which the data will + * be retrieved to be inserted into elastic. + * @param {string} insertConfig.date - Date of file inserted. + * @param {string} insertConfig.index - Name of the index to which the data will be inserted. + * @param {number} insertConfig.offset - Line of the snapshot at which the data insertion starts. + * @param {number} insertConfig.limit - Line in the file where the insertion stops. + * + * @returns {Promise} Success or not. + */ +async function insertHistoryDataUnpaywall(insertConfig) { + const { + filename, index, offset, limit, + } = insertConfig; + + // step insertion in the state + const start = new Date(); + addStepInsert(filename); + const step = getLatestStep(); + step.file = filename; + step.index = index; + updateLatestStep(step); + + await createIndex('unpaywall', unpaywallEnrichedMapping); + await createIndex('unpaywall_history', unpaywallHistoryMapping); + + await initAlias(index, unpaywallEnrichedMapping, indexAlias); + + const filePath = path.resolve(snapshotsDir, filename); + + // get information "bytes" for state + let bytes; + try { + bytes = await fs.stat(filePath); + } catch (err) { + logger.error(`[job: insert] Cannot stat [${filePath}]`, err); + await fail(err); + return false; + } + + // read file with stream + let readStream; + try { + readStream = fs.createReadStream(filePath); + } catch (err) { + logger.error(`[job: insert] Cannot read [${filePath}]`, err); + await fail(err); + return false; + } + + // get information "loaded" for state + let loaded = 0; + readStream.on('data', (chunk) => { + loaded += chunk.length; + }); + + let decompressedStream; + try { + decompressedStream = readStream.pipe(zlib.createGunzip()); + } catch (err) { + logger.error(`[job: insert] Cannot pipe [${readStream?.filename}]`, err); + await fail(err); + return false; + } + + const rl = readline.createInterface({ + input: decompressedStream, + crlfDelay: Infinity, + }); + + // array that will contain the packet of 1000 unpaywall data + let newData = []; + let listOfDoi = []; + + let success; + + logger.info(`[job: insert] Start insert with [${filename}]`); + + // Reads line by line the output of the decompression stream to make packets of 1000 + // to insert them in bulk in an elastic + for await (const line of rl) { + // limit + if (step.linesRead === limit) { + break; + } + + step.linesRead += 1; + + // offset + if (step.linesRead >= offset + 1) { + // fill the array + try { + const doc = JSON.parse(line); + listOfDoi.push(doc.doi); + newData.push(doc); + } catch (err) { + logger.error(`[job: insert] Cannot parse [${line}] in json format`, err); + await fail(err); + return false; + } + } + // bulk insertion + if (newData.length >= maxBulkSize) { + success = await insertData(listOfDoi, newData); + if (!success) return false; + listOfDoi = []; + newData = []; + step.percent = ((loaded / bytes.size) * 100).toFixed(2); + step.took = (new Date() - start) / 1000; + updateLatestStep(step); + } + if (step.linesRead % 100000 === 0) { + logger.info(`[job: insert] ${step.linesRead} Lines reads`); + updateLatestStep(step); + } + } + // last insertion if there is data left + if (newData.length > 0) { + success = await insertData(listOfDoi, newData); + if (!success) return false; + listOfDoi = []; + newData = []; + } + + logger.info('[job: insert] insertion completed'); + + try { + await refreshIndex(); + } catch (err) { + logger.warn('[elastic] Cannot refresh the index', err); + } + + // last update of step + step.status = 'success'; + step.took = (new Date() - start) / 1000; + step.percent = 100; + updateLatestStep(step); + return true; +} + +module.exports = { + insertHistoryDataUnpaywall, + objectKeyDiff, +}; diff --git a/src/update/lib/models/state.js b/src/update/lib/models/state.js index 233d0d63..5326aa1e 100644 --- a/src/update/lib/models/state.js +++ b/src/update/lib/models/state.js @@ -1,3 +1,4 @@ +/* eslint-disable no-restricted-syntax */ const { createReport, } = require('../report'); @@ -116,13 +117,19 @@ function end() { const insertSteps = state.steps.filter((e) => e.task === 'insert'); let totalInsertedDocs = 0; let totalUpdatedDocs = 0; - insertSteps.forEach((e) => { - totalInsertedDocs += e?.insertedDocs || 0; + insertSteps.forEach((step) => { + totalInsertedDocs += step?.insertedDocs || 0; + totalUpdatedDocs += step?.updatedDocs || 0; + if (typeof step.index === 'object') { + const indices = Object.keys(step.index); + indices.forEach((i) => { + const data = step.index[i]; + totalInsertedDocs += data.insertedDocs || 0; + totalUpdatedDocs += data.updatedDocs || 0; + }); + } }); state.totalInsertedDocs = totalInsertedDocs; - insertSteps.forEach((e) => { - totalUpdatedDocs += e?.updatedDocs || 0; - }); state.totalUpdatedDocs = totalUpdatedDocs; } diff --git a/src/update/lib/routers/job.js b/src/update/lib/routers/job.js index 51d24e2a..4dddc32e 100644 --- a/src/update/lib/routers/job.js +++ b/src/update/lib/routers/job.js @@ -53,6 +53,6 @@ router.post('/job/changefile/:filename', checkStatus, checkAuth, validateInsertF * * This route need a body that contains a config of job. */ -router.post('/job/history/insert', checkStatus, checkAuth, validateHistoryJob, insertWithOaHistory); +router.post('/job/history', checkStatus, checkAuth, validateHistoryJob, insertWithOaHistory); module.exports = router; diff --git a/src/update/mapping/unpaywall_enriched.json b/src/update/mapping/unpaywall_enriched.json index 1dfefb79..c9ee4574 100644 --- a/src/update/mapping/unpaywall_enriched.json +++ b/src/update/mapping/unpaywall_enriched.json @@ -19,16 +19,42 @@ }, "best_oa_location": { "properties": { - "evidence": { "type": "keyword" }, - "host_type":{ "type": "keyword" }, - "is_best": { "type": "boolean" }, - "license": { "type": "keyword" }, - "pmh_id": { "type": "keyword" }, - "updated": { "type": "date" }, - "url": { "type": "keyword" }, - "url_for_landing_page": { "type": "keyword" }, - "url_for_pdf": { "type": "keyword" }, - "version": { "type": "keyword" } + "endpoint_id": { + "type": "keyword" + }, + "evidence": { + "type": "keyword" + }, + "host_type": { + "type": "keyword" + }, + "is_best": { + "type": "boolean" + }, + "license": { + "type": "keyword" + }, + "pmh_id": { + "type": "keyword" + }, + "repository_instution": { + "type": "keyword" + }, + "updated": { + "type": "date" + }, + "url": { + "type": "keyword" + }, + "url_for_landing_page": { + "type": "keyword" + }, + "url_for_pdf": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } } }, "data_standard": { @@ -42,134 +68,47 @@ }, "first_oa_location": { "properties": { - "evidence": { "type": "keyword" }, - "host_type":{ "type": "keyword" }, - "is_best": { "type": "boolean" }, - "license": { "type": "keyword" }, - "pmh_id": { "type": "keyword" }, - "updated": { "type": "date" }, - "url": { "type": "keyword" }, - "url_for_landing_page": { "type": "keyword" }, - "url_for_pdf": { "type": "keyword" }, - "version": { "type": "keyword" } - } - }, - "genre": { - "type": "keyword" - }, - "history": { - "properties": { - "best_oa_location": { - "properties": { - "evidence": { "type": "keyword" }, - "host_type":{ "type": "keyword" }, - "is_best": { "type": "boolean" }, - "license": { "type": "keyword" }, - "pmh_id": { "type": "keyword" }, - "updated": { "type": "date" }, - "url": { "type": "keyword" }, - "url_for_landing_page": { "type": "keyword" }, - "url_for_pdf": { "type": "keyword" }, - "version": { "type": "keyword" } - } - }, - "data_standard": { - "type": "long" - }, - "doi_url": { + "endpoint_id": { "type": "keyword" }, - "first_oa_location": { - "properties": { - "evidence": { "type": "keyword" }, - "host_type":{ "type": "keyword" }, - "is_best": { "type": "boolean" }, - "license": { "type": "keyword" }, - "pmh_id": { "type": "keyword" }, - "updated": { "type": "date" }, - "url": { "type": "keyword" }, - "url_for_landing_page": { "type": "keyword" }, - "url_for_pdf": { "type": "keyword" }, - "version": { "type": "keyword" } - } - }, - "genre": { + "evidence": { "type": "keyword" }, - "has_repository_copy": { - "type": "boolean" - }, - "is_oa": { - "type": "boolean" - }, - "is_paratext": { - "type": "boolean" - }, - "journal_is_in_doaj": { - "type": "boolean" + "host_type": { + "type": "keyword" }, - "journal_is_oa": { + "is_best": { "type": "boolean" }, - "journal_issn_l": { + "license": { "type": "keyword" }, - "journal_issns": { + "pmh_id": { "type": "keyword" }, - "journal_name": { + "repository_instution": { "type": "keyword" }, - "oa_locations": { - "properties": { - "evidence": { "type": "keyword" }, - "host_type":{ "type": "keyword" }, - "is_best": { "type": "boolean" }, - "license": { "type": "keyword" }, - "pmh_id": { "type": "keyword" }, - "updated": { "type": "date" }, - "url": { "type": "keyword" }, - "url_for_landing_page": { "type": "keyword" }, - "url_for_pdf": { "type": "keyword" }, - "version": { "type": "keyword" } - } + "updated": { + "type": "date" }, - "oa_status": { + "url": { "type": "keyword" }, - "published_date": { - "type": "date", - "format": "iso8601" - }, - "publisher": { + "url_for_landing_page": { "type": "keyword" }, - "title": { - "type": "text" - }, - "updated": { - "type": "date", - "format": "iso8601" - }, - "year": { + "url_for_pdf": { "type": "keyword" }, - "z_authors": { - "properties": { - "family": { "type": "keyword" }, - "given": { "type": "keyword" }, - "sequence": { "type": "keyword" }, - "ORCID": { "type": "keyword" }, - "authenticated-orcid": { "type": "boolean" }, - "affiliation": { - "properties": { - "name": { "type": "keyword" } - } - } - } + "version": { + "type": "keyword" } } }, + "genre": { + "type": "keyword" + }, "has_repository_copy": { "type": "boolean" }, @@ -196,16 +135,42 @@ }, "oa_locations": { "properties": { - "evidence": { "type": "keyword" }, - "host_type":{ "type": "keyword" }, - "is_best": { "type": "boolean" }, - "license": { "type": "keyword" }, - "pmh_id": { "type": "keyword" }, - "updated": { "type": "date" }, - "url": { "type": "keyword" }, - "url_for_landing_page": { "type": "keyword" }, - "url_for_pdf": { "type": "keyword" }, - "version": { "type": "keyword" } + "endpoint_id": { + "type": "keyword" + }, + "evidence": { + "type": "keyword" + }, + "host_type": { + "type": "keyword" + }, + "is_best": { + "type": "boolean" + }, + "license": { + "type": "keyword" + }, + "pmh_id": { + "type": "keyword" + }, + "repository_instution": { + "type": "keyword" + }, + "updated": { + "type": "date" + }, + "url": { + "type": "keyword" + }, + "url_for_landing_page": { + "type": "keyword" + }, + "url_for_pdf": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } } }, "oa_status": { @@ -233,18 +198,30 @@ }, "z_authors": { "properties": { - "family": { "type": "keyword" }, - "given": { "type": "keyword" }, - "sequence": { "type": "keyword" }, - "ORCID": { "type": "keyword" }, - "authenticated-orcid": { "type": "boolean" }, + "family": { + "type": "keyword" + }, + "given": { + "type": "keyword" + }, + "sequence": { + "type": "keyword" + }, + "ORCID": { + "type": "keyword" + }, + "authenticated-orcid": { + "type": "boolean" + }, "affiliation": { "properties": { - "name": { "type": "keyword" } + "name": { + "type": "keyword" + } } } } } } } -} +} \ No newline at end of file diff --git a/src/update/mapping/unpaywall_history.json b/src/update/mapping/unpaywall_history.json index aaf2743d..3aa2ad16 100644 --- a/src/update/mapping/unpaywall_history.json +++ b/src/update/mapping/unpaywall_history.json @@ -17,125 +17,217 @@ "@timestamp": { "type": "date" }, + "best_oa_location": { + "properties": { + "endpoint_id": { + "type": "keyword" + }, + "evidence": { + "type": "keyword" + }, + "host_type": { + "type": "keyword" + }, + "is_best": { + "type": "boolean" + }, + "license": { + "type": "keyword" + }, + "pmh_id": { + "type": "keyword" + }, + "repository_instution": { + "type": "keyword" + }, + "updated": { + "type": "date" + }, + "url": { + "type": "keyword" + }, + "url_for_landing_page": { + "type": "keyword" + }, + "url_for_pdf": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + }, + "date": { + "type": "date" + }, + "data_standard": { + "type": "long" + }, "doi": { "type": "keyword" }, - "history": { + "doi_url": { + "type": "keyword" + }, + "endValidity": { + "type": "date" + }, + "first_oa_location": { "properties": { - "best_oa_location": { - "properties": { - "evidence": { "type": "keyword" }, - "host_type":{ "type": "keyword" }, - "is_best": { "type": "boolean" }, - "license": { "type": "keyword" }, - "pmh_id": { "type": "keyword" }, - "updated": { "type": "date" }, - "url": { "type": "keyword" }, - "url_for_landing_page": { "type": "keyword" }, - "url_for_pdf": { "type": "keyword" }, - "version": { "type": "keyword" } - } + "endpoint_id": { + "type": "keyword" }, - "data_standard": { - "type": "long" + "evidence": { + "type": "keyword" }, - "doi_url": { + "host_type": { "type": "keyword" }, - "first_oa_location": { - "properties": { - "evidence": { "type": "keyword" }, - "host_type":{ "type": "keyword" }, - "is_best": { "type": "boolean" }, - "license": { "type": "keyword" }, - "pmh_id": { "type": "keyword" }, - "updated": { "type": "date" }, - "url": { "type": "keyword" }, - "url_for_landing_page": { "type": "keyword" }, - "url_for_pdf": { "type": "keyword" }, - "version": { "type": "keyword" } - } + "is_best": { + "type": "boolean" }, - "genre": { + "license": { "type": "keyword" }, - "has_repository_copy": { - "type": "boolean" + "pmh_id": { + "type": "keyword" }, - "is_oa": { - "type": "boolean" + "repository_instution": { + "type": "keyword" }, - "is_paratext": { - "type": "boolean" + "updated": { + "type": "date" }, - "journal_is_in_doaj": { - "type": "boolean" + "url": { + "type": "keyword" + }, + "url_for_landing_page": { + "type": "keyword" + }, + "url_for_pdf": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + }, + "genre": { + "type": "keyword" + }, + "has_repository_copy": { + "type": "boolean" + }, + "is_oa": { + "type": "boolean" + }, + "is_paratext": { + "type": "boolean" + }, + "journal_is_in_doaj": { + "type": "boolean" + }, + "journal_is_oa": { + "type": "boolean" + }, + "journal_issn_l": { + "type": "keyword" + }, + "journal_issns": { + "type": "keyword" + }, + "journal_name": { + "type": "keyword" + }, + "oa_locations": { + "properties": { + "endpoint_id": { + "type": "keyword" + }, + "evidence": { + "type": "keyword" }, - "journal_is_oa": { + "host_type": { + "type": "keyword" + }, + "is_best": { "type": "boolean" }, - "journal_issn_l": { + "license": { "type": "keyword" }, - "journal_issns": { + "pmh_id": { "type": "keyword" }, - "journal_name": { + "repository_instution": { "type": "keyword" }, - "oa_locations": { - "properties": { - "evidence": { "type": "keyword" }, - "host_type":{ "type": "keyword" }, - "is_best": { "type": "boolean" }, - "license": { "type": "keyword" }, - "pmh_id": { "type": "keyword" }, - "updated": { "type": "date" }, - "url": { "type": "keyword" }, - "url_for_landing_page": { "type": "keyword" }, - "url_for_pdf": { "type": "keyword" }, - "version": { "type": "keyword" } - } + "updated": { + "type": "date" + }, + "url": { + "type": "keyword" }, - "oa_status": { + "url_for_landing_page": { "type": "keyword" }, - "published_date": { - "type": "date", - "format": "iso8601" + "url_for_pdf": { + "type": "keyword" }, - "publisher": { + "version": { + "type": "keyword" + } + } + }, + "oa_status": { + "type": "keyword" + }, + "published_date": { + "type": "date", + "format": "iso8601" + }, + "publisher": { + "type": "keyword" + }, + "referencedAt": { + "type": "date" + }, + "title": { + "type": "text" + }, + "updated": { + "type": "date", + "format": "iso8601" + }, + "year": { + "type": "keyword" + }, + "z_authors": { + "properties": { + "family": { "type": "keyword" }, - "title": { - "type": "text" + "given": { + "type": "keyword" }, - "updated": { - "type": "date", - "format": "iso8601" + "sequence": { + "type": "keyword" }, - "year": { + "ORCID": { "type": "keyword" }, - "z_authors": { + "authenticated-orcid": { + "type": "boolean" + }, + "affiliation": { "properties": { - "family": { "type": "keyword" }, - "given": { "type": "keyword" }, - "sequence": { "type": "keyword" }, - "ORCID": { "type": "keyword" }, - "authenticated-orcid": { "type": "boolean" }, - "affiliation": { - "properties": { - "name": { "type": "keyword" } - } + "name": { + "type": "keyword" } } } } - }, - "referencedAt": { - "type": "date" } } } -} +} \ No newline at end of file diff --git a/src/update/test/auth.js b/src/update/test/auth.js index a721c82f..d4e6e2d3 100644 --- a/src/update/test/auth.js +++ b/src/update/test/auth.js @@ -43,7 +43,7 @@ describe('Test: auth service in update service', () => { let isUpdate = true; while (isUpdate) { - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); isUpdate = await checkIfInUpdate(); } expect(res).have.status(202); diff --git a/src/update/test/history.js b/src/update/test/history.js new file mode 100644 index 00000000..4cf011e6 --- /dev/null +++ b/src/update/test/history.js @@ -0,0 +1,179 @@ +/* eslint-disable no-await-in-loop */ +const { expect } = require('chai'); +const chai = require('chai'); +const chaiHttp = require('chai-http'); + +const { + updateChangeFile, +} = require('./utils/snapshot'); + +const { + countDocuments, +} = require('./utils/elastic'); + +const { + getState, +} = require('./utils/state'); + +const checkIfInUpdate = require('./utils/status'); + +const ping = require('./utils/ping'); + +const reset = require('./utils/reset'); + +const updateURL = process.env.UPDATE_HOST || 'http://localhost:59702'; + +chai.use(chaiHttp); + +describe('Test: daily update route test with history', () => { + const now = Date.now(); + const oneDay = 1 * 24 * 60 * 60 * 1000; + const oneYear = 1 * 24 * 60 * 60 * 1000 * 365; + + // create date in a format (YYYY-mm-dd) to be use by ezunpaywall + // 2 years - one day + const date1 = new Date(now - (2 * oneYear) - (1 * oneDay)).toISOString().slice(0, 10); + // 2 years - three days + const date3 = new Date(now - (2 * oneYear) - (3 * oneDay)).toISOString().slice(0, 10); + + before(async function () { + this.timeout(30000); + await ping(); + await updateChangeFile('day'); + }); + + describe('Day: Do daily update', () => { + before(async () => { + await reset(); + }); + + // test response + it('Should return a status code 202', async () => { + const res = await chai.request(updateURL) + .post('/job/history') + .send({ + startDate: date3, + endDate: date1, + interval: 'day', + }) + .set('x-api-key', 'changeme'); + + expect(res).have.status(202); + }); + + it('Should insert 2 data', async () => { + // wait for the update to finish + let isUpdate = true; + while (isUpdate) { + await new Promise((resolve) => { setTimeout(resolve, 100); }); + isUpdate = await checkIfInUpdate(); + } + const count = await countDocuments('unpaywall_enriched'); + expect(count).to.equal(2); + }); + + it('Should insert 4 data', async () => { + // wait for the update to finish + let isUpdate = true; + while (isUpdate) { + await new Promise((resolve) => { setTimeout(resolve, 100); }); + isUpdate = await checkIfInUpdate(); + } + const count = await countDocuments('unpaywall_history'); + expect(count).to.equal(4); + }); + + it('Should get state with all informations from the download and insertion', async () => { + const state = await getState(); + + expect(state).have.property('done').equal(true); + expect(state).have.property('createdAt').to.not.equal(undefined); + expect(state).have.property('endAt').to.not.equal(undefined); + expect(state).have.property('steps').to.be.an('array'); + expect(state).have.property('error').equal(false); + expect(state).have.property('took').to.not.equal(undefined); + expect(state).have.property('totalInsertedDocs').equal(6); + expect(state).have.property('totalUpdatedDocs').equal(4); + + expect(state.steps[0]).have.property('task').equal('getChangefiles'); + expect(state.steps[0]).have.property('took').to.not.equal(undefined); + expect(state.steps[0]).have.property('status').equal('success'); + + expect(state.steps[1]).have.property('task').equal('download'); + expect(state.steps[1]).have.property('file').equal('history3.jsonl.gz'); + expect(state.steps[1]).have.property('percent').equal(100); + expect(state.steps[1]).have.property('took').to.not.equal(undefined); + expect(state.steps[1]).have.property('status').equal('success'); + + expect(state.steps[2]).have.property('task').equal('insert'); + expect(state.steps[2]).have.property('file').equal('history3.jsonl.gz'); + expect(state.steps[2]).have.property('percent').equal(100); + expect(state.steps[2]).have.property('linesRead').equal(2); + expect(state.steps[2]).have.property('insertedDocs').equal(0); + expect(state.steps[2]).have.property('updatedDocs').equal(0); + expect(state.steps[2]).have.property('failedDocs').equal(0); + expect(state.steps[2]).have.property('took').to.not.equal(undefined); + expect(state.steps[2]).have.property('status').equal('success'); + + const step2 = state.steps[2].index; + expect(step2.unpaywall_enriched).have.property('insertedDocs').equal(2); + expect(step2.unpaywall_enriched).have.property('updatedDocs').equal(0); + expect(step2.unpaywall_enriched).have.property('failedDocs').equal(0); + expect(step2.unpaywall_history).have.property('insertedDocs').equal(0); + expect(step2.unpaywall_history).have.property('updatedDocs').equal(0); + expect(step2.unpaywall_history).have.property('failedDocs').equal(0); + + expect(state.steps[3]).have.property('task').equal('download'); + expect(state.steps[3]).have.property('file').equal('history2.jsonl.gz'); + expect(state.steps[3]).have.property('percent').equal(100); + expect(state.steps[3]).have.property('took').to.not.equal(undefined); + expect(state.steps[3]).have.property('status').equal('success'); + + expect(state.steps[4]).have.property('task').equal('insert'); + expect(state.steps[4]).have.property('file').equal('history2.jsonl.gz'); + expect(state.steps[4]).have.property('percent').equal(100); + expect(state.steps[4]).have.property('linesRead').equal(2); + expect(state.steps[4]).have.property('insertedDocs').equal(0); + expect(state.steps[4]).have.property('updatedDocs').equal(0); + expect(state.steps[4]).have.property('failedDocs').equal(0); + expect(state.steps[4]).have.property('took').to.not.equal(undefined); + expect(state.steps[4]).have.property('status').equal('success'); + + const step4 = state.steps[4].index; + expect(step4.unpaywall_enriched).have.property('insertedDocs').equal(0); + expect(step4.unpaywall_enriched).have.property('updatedDocs').equal(2); + expect(step4.unpaywall_enriched).have.property('failedDocs').equal(0); + expect(step4.unpaywall_history).have.property('insertedDocs').equal(2); + expect(step4.unpaywall_history).have.property('updatedDocs').equal(0); + expect(step4.unpaywall_history).have.property('failedDocs').equal(0); + + expect(state.steps[5]).have.property('task').equal('download'); + expect(state.steps[5]).have.property('file').equal('history1.jsonl.gz'); + expect(state.steps[5]).have.property('percent').equal(100); + expect(state.steps[5]).have.property('took').to.not.equal(undefined); + expect(state.steps[5]).have.property('status').equal('success'); + + expect(state.steps[6]).have.property('task').equal('insert'); + expect(state.steps[6]).have.property('file').equal('history1.jsonl.gz'); + expect(state.steps[6]).have.property('percent').equal(100); + expect(state.steps[6]).have.property('linesRead').equal(2); + expect(state.steps[6]).have.property('insertedDocs').equal(0); + expect(state.steps[6]).have.property('updatedDocs').equal(0); + expect(state.steps[6]).have.property('failedDocs').equal(0); + expect(state.steps[6]).have.property('took').to.not.equal(undefined); + expect(state.steps[6]).have.property('status').equal('success'); + + const step6 = state.steps[6].index; + expect(step6.unpaywall_enriched).have.property('insertedDocs').equal(0); + expect(step6.unpaywall_enriched).have.property('updatedDocs').equal(2); + expect(step6.unpaywall_enriched).have.property('failedDocs').equal(0); + expect(step6.unpaywall_history).have.property('insertedDocs').equal(2); + expect(step6.unpaywall_history).have.property('updatedDocs').equal(0); + expect(step6.unpaywall_history).have.property('failedDocs').equal(0); + }); + }); + + after(async () => { + await reset('unpaywall_history'); + }); +}); diff --git a/src/update/test/insertionError.js b/src/update/test/insertionError.js index ed5e66fb..071213a9 100644 --- a/src/update/test/insertionError.js +++ b/src/update/test/insertionError.js @@ -61,7 +61,7 @@ describe('Test: insert the content of a file already installed on ezunpaywall', let isUpdate = true; do { isUpdate = await checkIfInUpdate(); - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); } while (isUpdate); const count = await countDocuments('unpaywall-test'); diff --git a/src/update/test/insertionFile.js b/src/update/test/insertionFile.js index 420ba8d0..96e89d69 100644 --- a/src/update/test/insertionFile.js +++ b/src/update/test/insertionFile.js @@ -57,7 +57,7 @@ describe('Test: insert the content of a file already installed on ezunpaywall', let isUpdate = true; do { isUpdate = await checkIfInUpdate(); - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); } while (isUpdate); const count = await countDocuments('unpaywall-test'); @@ -137,7 +137,7 @@ describe('Test: insert the content of a file already installed on ezunpaywall', let isUpdate = true; do { isUpdate = await checkIfInUpdate(); - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); } while (isUpdate); const count = await countDocuments('unpaywall-test'); @@ -216,7 +216,7 @@ describe('Test: insert the content of a file already installed on ezunpaywall', let isUpdate = true; do { isUpdate = await checkIfInUpdate(); - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); } while (isUpdate); const count = await countDocuments('unpaywall-test'); @@ -296,7 +296,7 @@ describe('Test: insert the content of a file already installed on ezunpaywall', let isUpdate = true; do { isUpdate = await checkIfInUpdate(); - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); } while (isUpdate); const count = await countDocuments('unpaywall-test'); diff --git a/src/update/test/insertionPeriodDay.js b/src/update/test/insertionPeriodDay.js index 9c40ece6..52330e84 100644 --- a/src/update/test/insertionPeriodDay.js +++ b/src/update/test/insertionPeriodDay.js @@ -74,7 +74,7 @@ describe('Test: download and insert file from unpaywall between a period', () => // wait for the update to finish let isUpdate = true; while (isUpdate) { - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); isUpdate = await checkIfInUpdate(); } const count = await countDocuments('unpaywall-test'); @@ -210,7 +210,7 @@ describe('Test: download and insert file from unpaywall between a period', () => // wait for the update to finish let isUpdate = true; while (isUpdate) { - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); isUpdate = await checkIfInUpdate(); } const count = await countDocuments('unpaywall-test'); @@ -346,7 +346,7 @@ describe('Test: download and insert file from unpaywall between a period', () => // wait for the update to finish let isUpdate = true; while (isUpdate) { - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); isUpdate = await checkIfInUpdate(); } const count = await countDocuments('unpaywall-test'); diff --git a/src/update/test/insertionPeriodWeek.js b/src/update/test/insertionPeriodWeek.js index faa59d28..c0f94a96 100644 --- a/src/update/test/insertionPeriodWeek.js +++ b/src/update/test/insertionPeriodWeek.js @@ -73,7 +73,7 @@ describe('Week: Test: download and insert file from unpaywall between a period', // wait for the update to finish let isUpdate = true; while (isUpdate) { - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); isUpdate = await checkIfInUpdate(); } const count = await countDocuments('unpaywall-test'); @@ -207,7 +207,7 @@ describe('Week: Test: download and insert file from unpaywall between a period', // wait for the update to finish let isUpdate = true; while (isUpdate) { - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); isUpdate = await checkIfInUpdate(); } const count = await countDocuments('unpaywall-test'); @@ -343,7 +343,7 @@ describe('Week: Test: download and insert file from unpaywall between a period', // wait for the update to finish let isUpdate = true; while (isUpdate) { - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); isUpdate = await checkIfInUpdate(); } const count = await countDocuments('unpaywall-test'); diff --git a/src/update/test/insertionSnapshot.js b/src/update/test/insertionSnapshot.js index 5a0a6e32..e50848e8 100644 --- a/src/update/test/insertionSnapshot.js +++ b/src/update/test/insertionSnapshot.js @@ -53,7 +53,7 @@ describe('Test: download and insert snapshot from unpaywall', () => { let isUpdate = true; do { isUpdate = await checkIfInUpdate(); - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); } while (isUpdate); const count = await countDocuments('unpaywall-test'); diff --git a/src/update/test/updateDay.js b/src/update/test/updateDay.js index f82413c4..e6f68399 100644 --- a/src/update/test/updateDay.js +++ b/src/update/test/updateDay.js @@ -58,7 +58,7 @@ describe('Test: daily update route test', () => { it('Should insert 50 data', async () => { let isUpdate = true; while (isUpdate) { - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); isUpdate = await checkIfInUpdate(); } const count = await countDocuments('unpaywall-test'); @@ -160,7 +160,7 @@ describe('Test: daily update route test', () => { it('Should insert 50 data', async () => { let isUpdate = true; while (isUpdate) { - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); isUpdate = await checkIfInUpdate(); } const count = await countDocuments('unpaywall-test'); @@ -252,7 +252,7 @@ describe('Test: daily update route test', () => { it('Should insert 50 data', async () => { let isUpdate = true; while (isUpdate) { - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); isUpdate = await checkIfInUpdate(); } const count = await countDocuments('unpaywall-test'); @@ -343,7 +343,7 @@ describe('Test: daily update route test', () => { it('should insert 50 data', async () => { let isUpdate = true; while (isUpdate) { - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); isUpdate = await checkIfInUpdate(); } const count = await countDocuments('unpaywall-test'); diff --git a/src/update/test/updateWeek.js b/src/update/test/updateWeek.js index 55eda5c6..b3143865 100644 --- a/src/update/test/updateWeek.js +++ b/src/update/test/updateWeek.js @@ -58,7 +58,7 @@ describe('Week: Test: weekly update route test', () => { it('Should insert 50 data', async () => { let isUpdate = true; while (isUpdate) { - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); isUpdate = await checkIfInUpdate(); } const count = await countDocuments('unpaywall-test'); @@ -161,7 +161,7 @@ describe('Week: Test: weekly update route test', () => { it('should insert 50 data', async () => { let isUpdate = true; while (isUpdate) { - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => { setTimeout(resolve, 100); }); isUpdate = await checkIfInUpdate(); } const count = await countDocuments('unpaywall-test'); diff --git a/src/update/test/utils/reset.js b/src/update/test/utils/reset.js index a9ac2da7..e0919134 100644 --- a/src/update/test/utils/reset.js +++ b/src/update/test/utils/reset.js @@ -14,12 +14,17 @@ const resetCronConfig = require('./cron'); * @returns {Promise} */ async function reset() { + await deleteFile('history1.jsonl.gz'); + await deleteFile('history2.jsonl.gz'); + await deleteFile('history3.jsonl.gz'); await deleteFile('fake1.jsonl.gz'); await deleteFile('fake2.jsonl.gz'); await deleteFile('fake3.jsonl.gz'); await deleteFile('fake1-error.jsonl.gz'); await deleteFile('snapshot.jsonl.gz'); await deleteIndex('unpaywall-test'); + await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_history'); await resetCronConfig(); } From 7f7ef05b666c03d3507b188ccc578de1ae0de832 Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Wed, 4 Oct 2023 07:39:22 +0200 Subject: [PATCH 003/114] wip: rollback work date by date --- .../lib/controllers/changefiles.js | 12 +-- src/fakeUnpaywall/snapshots/history1.jsonl.gz | Bin 520 -> 527 bytes src/fakeUnpaywall/snapshots/history2.jsonl.gz | Bin 526 -> 531 bytes src/fakeUnpaywall/snapshots/history3.jsonl.gz | Bin 526 -> 526 bytes src/fakeUnpaywall/sources/history1.jsonl | 4 +- src/fakeUnpaywall/sources/history2.jsonl | 4 +- src/fakeUnpaywall/sources/history3.jsonl | 4 +- src/graphql/lib/resolvers/unpaywall2.js | 12 +-- src/update/lib/controllers/job.js | 22 +++++ src/update/lib/history.js | 77 ++++++++++++++++- src/update/lib/job.js | 2 +- src/update/lib/middlewares/job.js | 21 +++++ src/update/lib/routers/job.js | 11 +++ src/update/lib/services/elastic.js | 81 ++++++++++++++++++ 14 files changed, 230 insertions(+), 20 deletions(-) diff --git a/src/fakeUnpaywall/lib/controllers/changefiles.js b/src/fakeUnpaywall/lib/controllers/changefiles.js index ca0e06d3..43a61b5d 100644 --- a/src/fakeUnpaywall/lib/controllers/changefiles.js +++ b/src/fakeUnpaywall/lib/controllers/changefiles.js @@ -78,19 +78,19 @@ async function updateChangefilesExample(interval) { changefilesDay.list[4].last_modified = new Date(now - oneYear - (1 * oneDay)) .toISOString().slice(0, 19); - changefilesDay.list[5].date = new Date(now - (2 * oneYear) - (1 * oneDay)) + changefilesDay.list[5].date = new Date('2020-01-03') .toISOString().slice(0, 10); - changefilesDay.list[5].last_modified = new Date(now - (2 * oneYear) - (1 * oneDay)) + changefilesDay.list[5].last_modified = new Date('2020-01-03') .toISOString().slice(0, 19); - changefilesDay.list[6].date = new Date(now - (2 * oneYear) - (2 * oneDay)) + changefilesDay.list[6].date = new Date('2020-01-02') .toISOString().slice(0, 10); - changefilesDay.list[6].last_modified = new Date(now - (2 * oneYear) - (2 * oneDay)) + changefilesDay.list[6].last_modified = new Date('2020-01-02') .toISOString().slice(0, 19); - changefilesDay.list[7].date = new Date(now - (2 * oneYear) - (3 * oneDay)) + changefilesDay.list[7].date = new Date('2020-01-01') .toISOString().slice(0, 10); - changefilesDay.list[7].last_modified = new Date(now - (2 * oneYear) - (3 * oneDay)) + changefilesDay.list[7].last_modified = new Date('2020-01-01') .toISOString().slice(0, 19); try { diff --git a/src/fakeUnpaywall/snapshots/history1.jsonl.gz b/src/fakeUnpaywall/snapshots/history1.jsonl.gz index 9cef31cc9d1cebafcde3587c1e22342a930f56c9..366577fa85260886cf0b262ad7fb4794d434e253 100644 GIT binary patch literal 527 zcmV+q0`UDGiwFo~DI8@0188Y;bZ>HbF)nIzZ*FV=?Nv>0oG=i*_g5gFT8YgXa@%uH zIYd1~y;POu#ReRCZPy>Dbd~?!8ME1?O(LaDlvF7Q1fF?#X5JfX_C#1kHU+8Z8Krbh z@dY<2$tbx8PYvdhbkcZCF0W zeK$;Uf=4M3T+^DaXtko}S5>{Cbdxc70nTz3B9AE`va?!%&84kYp_iB^`E*>AZ%7Qt?r?8kUl`7h`VJ{d_LGpA_paINwK^*6kfa^6fcD3*B=1W+YnhCot*E1 z0!SmARVL0W0w!2%J#(VHW7FISWjY|dlaHgzOJ{?M){jh?06m!ER10p5@&O0AWsKk+ zlc-PP{o21LYsjJzq-O;WayfT0{k!)_+e`?05&{Kevz7qB(h#rH5ao2R7d8G7HD*Uq z8HNcTsqA@Rv)>!*#)I$X-TwZ0RlUr-dPu$hZeCT(f%uaHQN8(jwKT*tH^h_X)uP6; RtMPa9>L-oEmJMGG005%L_ZHbF)nIzZ*FV=?Nz~!(=ZUd^A(X#Af;O;$DTQH zKpa+F5VG9ZNxX7B#&#+~wg1kHlWhabVGAojt3-(sznOSu-WzL9Oo3s$f>r#6bH3&D z!kd+B)H0x@M!001c0rTh^ieVs1xN%^7<9>!Nj95r^G8X$iXm8n`jmX!ZMO*BKU^vHQ=tN-qN@_cfuRl{R9rcQ6De zt#I<7rycXUcu(t`rG-W&18$D}WFq{b7lwFngS9#OUdME1*_uYXm}550W23OqBc{m{cEUtYrFV)B`CXi@EUP_0h;efxLTzK0U-F(&+Va4-!B-<-xeL ztO%J9Ed=3J`v|krDdRdMywgvkD`*eVBnYE0E@BT3X{v=bCi#$q+%itdKuOdW@qX;z z3mUU%jOf|mLtHM6!v1VM)3y+TUxXk5#iAw9VQq+i(-7r!uvazy6Ezk`(KtZ~pQ-F+ z6pP;*@Z)T|e|7rx$g2L@$ht}vep$odXl K{LLR_4FCWQ{rr0X diff --git a/src/fakeUnpaywall/snapshots/history2.jsonl.gz b/src/fakeUnpaywall/snapshots/history2.jsonl.gz index d15c4fb6b1c145c0495ab5740803174d521a63df..3951f327013c57b6669590d418ef811a51a30235 100644 GIT binary patch literal 531 zcmV+u0_^=CiwFpz92{i;188Y;bZ>HbGA?R!Z*FV=?N!Nc+b|Hl_bUXR8bFRI9ewVp zhoXm|mm(l&k&+nD9F~hv)yThxLwOOJ1aQ&-DOwpaEY3X7^4<`$Cju1N6r`fpl+rcD zpY)KDjFP+Lsgt=Roit7&Zm5$>c?LY4TCTk;Nl-y&=S_YoF(!*nqp1(U`ptTsOxNJL z_2WFz3Ld0Da7}BvqScDlH&wl%bdxdoNjTul2OgqFWCvO#n~htod@r$3^7V8m$uGuZ z=)ocQ`<`@akOncIp*P9h@}1Hn8a@FQ?B$OrO_Q5wmGgnz-{v+gcd59a;oR&UL}$2W zhVSGQF`tVsc+OR7Xn0Uy4!JKT!tQ#ekOvdB&e^xpW9HyuN~~2Qjn6roX0;usWXRn_ zT{TfxJws{r1f_#?J{5|%Ba`CW!B!GEC~+hOcyVaF_&6wD2;g^D3DbiQER0spW1s-i z2n))Dc}Ix})`DYJw2y3>8=*{>2=C-&bUA6ks{n3f%J?*cF-*N+V3aR$kVnP{?y!jZ zEZ%W|CuYx9pG6}`&nkV9%ej;3-@PZ=W~ literal 526 zcmV+p0`dJHiwFoA78GRw188Y;bZ>HbGA?R!Z*FV=?NvQ*+b|H_`zr)a4IsyqM$er( z6di&tMM2OaB{86REI*7YM*e$vlxz!02RjMSB9I_K;=2>??!6=KKm;hVDM&?cDWz+Q zuXK}=43fKKsS{k1P8uiCZ~Q2U@(g$aTCTk;Nl-y&`(6Ghaa$IhMyLlZ$AT^MO46%4J&aQgVO7x!8M%&T!2P z-^p>uye_`sI%lb&;X#3!VqZ*z-Stc%4<>4zqi?0h$ic-FSgS@FpJO)7YCDX#Z-9ld9QdOfZyLFpB{W*VX$((2MQpKu%Jws zSCp7wEjVUH`vTM42xYoNcqcET%Sj7f1#kmX#-|>PVXOr=M)?v4xn+#t4wI}#J_2Xay;0J8vltJv!kdC!-P*% z_T01C?+w^tvORt}{C#Fs7tgFQ#Op`aH3ELM{-81j*y`%0(gjHl$kc!?=O1Bgrut-Tp$vt>#Fqfp0#!JKv6}VJj zmcuUkBuf%i)Y*5Jf0TG6OJ1X?kI{wQb_=E(>$~mqFv$rXr9f~^Yr3Hoth;}z-ch>C z82kXvauyZ+N#D<~bL50EJ0o|qJ0W?M<* zpu`g?mKSs3#rr|=LRfzP0WiG{k;T!;Xk!#^Nd}!!Y3!l|2t^@%P5MarQmFJN|iIRhP}H y`5pf9fpv|1Z+c)=>y5bPMpSQpV66@D$_?>X18Y^|)zx^}!1@XMzY2R`4FCYt)$?2c delta 508 zcmV$~;yNXZEvr9kJJ)^tTHxNm=}dPC_Z z```tBma`CfOd%&?wE)betyZCzm@2tBE=uy7@f3UO(fP-LbZRdR`h2y$0dvcDN)HHp zv6yg>_eq*27ttyoB6;|kt90B$a=#)NTCM{WduNk3TrsJMsSZw)F<)&?B5d_vS e._source.doi); + const listOfDoiFiltered = new Set(listOfDoi); + + for (const doi of listOfDoiFiltered) { + const doc = data.filter((e) => doi === e._source.doi); + let latestDoc; + let id; + doc.forEach((e) => { + const unpaywallData = e._source; + if (!latestDoc) { + latestDoc = unpaywallData; + id = e._id; + } + const date1 = new Date(startDate) - new Date(unpaywallData.endValidity); + const date2 = new Date(startDate) - new Date(latestDoc.endValidity); + + if (date1 > date2) { + latestDoc = unpaywallData; + id = e._id; + } + }); + const tt = await getDataByListOfDOI([latestDoc.doi], 'unpaywall_enriched'); + console.log('oldData :', new Date(tt[0].updated)); + console.log('Request :', new Date(startDate)); + console.log(new Date(tt[0].updated) >= new Date(startDate)); + console.log('====================='); + if (new Date(tt[0].updated) >= new Date(startDate)) { + bulkDelete.push({ delete: { _index: 'unpaywall_history', _id: id } }); + bulkOps.push({ index: { _index: 'unpaywall_enriched', _id: latestDoc.doi } }); + bulkOps.push(latestDoc); + } + } + + console.log('update :', bulkOps.length / 2); + + // try { + // await bulk(bulkOps); + // } catch (err) { + // logger.error('[elastic] Cannot bulk', err); + // return false; + // } + + // TODO update index "unpaywall" + + console.log('delete :', bulkDelete.length); + + // try { + // await bulk(bulkDelete); + // } catch (err) { + // logger.error('[elastic] Cannot bulk', err); + // return false; + // } +} + +module.exports = { + insertHistoryDataUnpaywall, + rollBack, +}; diff --git a/src/update/lib/job.js b/src/update/lib/job.js index 8ff1bc8c..d5808074 100644 --- a/src/update/lib/job.js +++ b/src/update/lib/job.js @@ -24,7 +24,7 @@ const { const insertDataUnpaywall = require('./insert'); -const insertHistoryDataUnpaywall = require('./history'); +const { insertHistoryDataUnpaywall } = require('./history'); const { getChangefiles, diff --git a/src/update/lib/middlewares/job.js b/src/update/lib/middlewares/job.js index 7695a51b..db7e72ab 100644 --- a/src/update/lib/middlewares/job.js +++ b/src/update/lib/middlewares/job.js @@ -94,9 +94,30 @@ async function validateHistoryJob(req, res, next) { return next(); } +/** + * Joi middleware to check if job config for insert history of open access. + * + * @param {import('express').Request} req - HTTP request. + * @param {import('express').Response} res - HTTP response. + * @param {import('express').NextFunction} next - Do the following. + */ +async function validateHistoryReset(req, res, next) { + const { error, value } = joi.object({ + index: joi.string().trim().default('unpaywall_history'), + startDate: joi.date(), + }).with('endDate', 'startDate').validate(req.body); + + if (error) return res.status(400).json({ message: error.details[0].message }); + + req.data = value; + + return next(); +} + module.exports = { validateSnapshotJob, validateJobChangefilesConfig, validateInsertFile, validateHistoryJob, + validateHistoryReset, }; diff --git a/src/update/lib/routers/job.js b/src/update/lib/routers/job.js index 4dddc32e..1ae478d6 100644 --- a/src/update/lib/routers/job.js +++ b/src/update/lib/routers/job.js @@ -5,6 +5,7 @@ const { insertChangefilesOnPeriodJob, insertWithOaHistory, insertChangefileJob, + historyRollBack, } = require('../controllers/job'); const { @@ -12,6 +13,7 @@ const { validateJobChangefilesConfig, validateInsertFile, validateHistoryJob, + validateHistoryReset, } = require('../middlewares/job'); const checkStatus = require('../middlewares/status'); @@ -55,4 +57,13 @@ router.post('/job/changefile/:filename', checkStatus, checkAuth, validateInsertF */ router.post('/job/history', checkStatus, checkAuth, validateHistoryJob, insertWithOaHistory); +/** + * Route that roll back the current and the history index according to a date. + * Auth required. + * No update process should be in progress. + * + * This route need a body that contains a config of job. + */ +router.post('/job/history/reset', checkStatus, checkAuth, validateHistoryReset, historyRollBack); + module.exports = router; diff --git a/src/update/lib/services/elastic.js b/src/update/lib/services/elastic.js index 9b975b99..2589e232 100644 --- a/src/update/lib/services/elastic.js +++ b/src/update/lib/services/elastic.js @@ -148,11 +148,90 @@ async function getDataByListOfDOI(dois, index) { return res.body.hits.hits.map((hit) => hit._source); } +/** + * Search old data in history with date + * @param {string} date - The date on which you want to obtain all the changes + * @param {string} index - Elastic index name + * @returns {Promise} + */ +async function searchOldInHistoryData(date, index) { + const query = { + bool: { + filter: [ + { + range: { + endValidity: { + lte: date, + }, + }, + }, + ], + }, + }; + + let res; + try { + res = await elasticClient.search({ + index, + body: { + query, + sort: 'endValidity', + }, + }); + } catch (err) { + logger.error('[elastic] Cannot request elastic', err); + return null; + } + // eslint-disable-next-line no-underscore-dangle + return res.body.hits.hits; +} + +/** + * Search to recent data in history with date + * @param {string} date - The date on which you want to obtain all the changes + * @param {string} index - Elastic index name + * @returns {Promise} + */ +async function searchToRecentInHistoryData(date, index) { + const query = { + bool: { + filter: [ + { + range: { + endValidity: { + gt: date, + }, + }, + }, + ], + }, + }; + + let res; + try { + res = await elasticClient.search({ + index, + body: { + query, + }, + }); + } catch (err) { + logger.error('[elastic] Cannot request elastic', err); + return null; + } + // eslint-disable-next-line no-underscore-dangle + return res.body.hits.hits; +} + async function refreshIndex(index) { return elasticClient.indices.refresh({ index }); } async function bulk(data) { + if (data.length === 0) { + logger.warn('[elastic]: No data is send for bulk'); + return []; + } return elasticClient.bulk({ body: data }); } @@ -164,4 +243,6 @@ module.exports = { getDataByListOfDOI, refreshIndex, bulk, + searchOldInHistoryData, + searchToRecentInHistoryData, }; From 9505539445e1ddb7486b8d3f4556899f2df76889 Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Mon, 9 Oct 2023 08:10:34 +0200 Subject: [PATCH 004/114] wip: history update --- docker-compose.debug.yml | 1 + src/update/lib/history.js | 221 ++++++++++++++---- src/update/lib/services/elastic.js | 55 +---- src/update/test/history.js | 2 +- src/update/test/rollback.js | 158 +++++++++++++ src/update/test/sources/unpaywall.jsonl | 2 + .../test/sources/unpaywall_history.jsonl | 4 + src/update/test/utils/elastic.js | 78 +++++++ 8 files changed, 424 insertions(+), 97 deletions(-) create mode 100644 src/update/test/rollback.js create mode 100644 src/update/test/sources/unpaywall.jsonl create mode 100644 src/update/test/sources/unpaywall_history.jsonl diff --git a/docker-compose.debug.yml b/docker-compose.debug.yml index 2f86b873..912bad45 100644 --- a/docker-compose.debug.yml +++ b/docker-compose.debug.yml @@ -41,6 +41,7 @@ services: container_name: ezunpaywall-update-dev environment: - NODE_ENV=development + - LOG_LEVEL=debug # unpaywall - UNPAYWALL_APIKEY - UNPAYWALL_HOST diff --git a/src/update/lib/history.js b/src/update/lib/history.js index a370a388..163941b0 100644 --- a/src/update/lib/history.js +++ b/src/update/lib/history.js @@ -24,7 +24,7 @@ const { getDataByListOfDOI, bulk, createIndex, - searchOldInHistoryData, + searchWithRange, } = require('./services/elastic'); const maxBulkSize = config.get('elasticsearch.maxBulkSize'); @@ -330,72 +330,195 @@ async function insertHistoryDataUnpaywall(insertConfig) { * @param {string} startDate - The date you wish to return to */ async function rollBack(startDate) { - // TODO allow to rollBack more than one day - // TODO go back in time with more than one data + // STEP 2 + const nearestDataBulk = []; + const nearestDataDeleteBulk = []; + const oldData = await searchWithRange(startDate, 'endValidity', 'lt', 'unpaywall_history'); - console.log('==========================================================='); - const bulkOps = []; - const bulkDelete = []; - - console.log('startDate :', startDate); - - const data = await searchOldInHistoryData(startDate, 'unpaywall_history'); - console.log(data.length); - - const listOfDoi = data.map((e) => e._source.doi); + const listOfDoi = oldData.map((e) => e._source.doi); const listOfDoiFiltered = new Set(listOfDoi); for (const doi of listOfDoiFiltered) { - const doc = data.filter((e) => doi === e._source.doi); - let latestDoc; - let id; - doc.forEach((e) => { - const unpaywallData = e._source; - if (!latestDoc) { - latestDoc = unpaywallData; - id = e._id; + console.log(doi); + const docs = oldData.filter((e) => doi === e._source.doi); + let nearestData; + let nearestID; + docs.forEach((doc) => { + const unpaywallData = doc._source; + if (!nearestData) { + nearestID = doc._id; + nearestData = unpaywallData; } const date1 = new Date(startDate) - new Date(unpaywallData.endValidity); - const date2 = new Date(startDate) - new Date(latestDoc.endValidity); + const date2 = new Date(startDate) - new Date(nearestData.endValidity); - if (date1 > date2) { - latestDoc = unpaywallData; - id = e._id; + if (date1 < date2) { + nearestID = doc._id; + nearestData = unpaywallData; } }); - const tt = await getDataByListOfDOI([latestDoc.doi], 'unpaywall_enriched'); - console.log('oldData :', new Date(tt[0].updated)); - console.log('Request :', new Date(startDate)); - console.log(new Date(tt[0].updated) >= new Date(startDate)); - console.log('====================='); - if (new Date(tt[0].updated) >= new Date(startDate)) { - bulkDelete.push({ delete: { _index: 'unpaywall_history', _id: id } }); - bulkOps.push({ index: { _index: 'unpaywall_enriched', _id: latestDoc.doi } }); - bulkOps.push(latestDoc); + + const oldDoc = await getDataByListOfDOI([nearestData.doi], 'unpaywall_enriched'); + + if (new Date(oldDoc[0].updated) >= new Date(startDate)) { + nearestDataDeleteBulk.push({ delete: { _index: 'unpaywall_history', _id: nearestID } }); + nearestDataBulk.push({ index: { _index: 'unpaywall_enriched', _id: nearestData.doi } }); + nearestDataBulk.push(nearestData); } } - console.log('update :', bulkOps.length / 2); + // STEP 2 + // STEP 3 + await bulk(nearestDataBulk, true); + await refreshIndex('unpaywall_enriched'); + console.log('Doc mit à jour dans l\'index courant', nearestDataBulk.length / 2); + await bulk(nearestDataDeleteBulk, true); + await refreshIndex('unpaywall_history'); + console.log('Doc supprimé dans l\'historique', nearestDataDeleteBulk.length); + + // STEP 4 + const toRecentDataBulk = []; + const toRecentData = await searchWithRange(startDate, 'updated', 'gte', 'unpaywall_enriched'); + + const listOfIDToRecentData = toRecentData.map((e) => e._id); - // try { - // await bulk(bulkOps); - // } catch (err) { - // logger.error('[elastic] Cannot bulk', err); - // return false; - // } + listOfIDToRecentData.forEach((id) => { + toRecentDataBulk.push({ delete: { _index: 'unpaywall_enriched', _id: id } }); + }); + + await bulk(toRecentDataBulk, true); + await refreshIndex('unpaywall_enriched'); - // TODO update index "unpaywall" + // STEP 1 + const toRecentDataInHistoryBulk = []; + const toRecentDataInHistory = await searchWithRange(startDate, 'endValidity', 'gte', 'unpaywall_history'); + + toRecentDataInHistory.forEach((e) => { + console.log(e._source.updated); + }); - console.log('delete :', bulkDelete.length); + const listOfIDToRecentDataInHistory = toRecentDataInHistory.map((e) => e._id); - // try { - // await bulk(bulkDelete); - // } catch (err) { - // logger.error('[elastic] Cannot bulk', err); - // return false; - // } + listOfIDToRecentDataInHistory.forEach((id) => { + toRecentDataInHistoryBulk.push({ delete: { _index: 'unpaywall_history', _id: id } }); + }); + + await bulk(toRecentDataInHistoryBulk, true); + console.log('valeur trop recente', toRecentDataInHistoryBulk.length); + await refreshIndex('unpaywall_enriched'); } +// /** +// * Resets the unpaywall index according to a date and deletes the data in the history. +// * @param {string} startDate - The date you wish to return to +// */ +// async function rollBack(startDate) { +// const bulkUpdate = []; +// const bulkDelete = []; +// const bulkHistoryDelete = []; +// const bulkHistoryDelete2 = []; + +// let data = await searchWithRange(startDate, 'endValidity', 'lte', 'unpaywall_history'); +// logger.debug(`unpaywall_history.length: ${data.length}`); +// logger.info(format(startDate, 'yyyy-MM-dd-HH')); + +// const listOfDoi = data.map((e) => e._source.doi); +// const listOfDoiFiltered = new Set(listOfDoi); + +// for (const doi of listOfDoiFiltered) { +// const doc = data.filter((e) => doi === e._source.doi); +// let latestDoc; +// doc.forEach((e) => { +// const unpaywallData = e._source; +// if (!latestDoc) { +// latestDoc = unpaywallData; +// } +// const date1 = new Date(startDate) - new Date(unpaywallData.endValidity); +// const date2 = new Date(startDate) - new Date(latestDoc.endValidity); + +// if (date1 < date2) { +// latestDoc = unpaywallData; +// } +// }); + +// const tt = await getDataByListOfDOI([latestDoc.doi], 'unpaywall_enriched'); + +// logger.debug(`oldData : ${new Date(tt[0].updated)}`); +// logger.debug(`Request : ${new Date(startDate)}`); +// logger.debug(`doi : ${doi}`); +// if (new Date(tt[0].updated) >= new Date(startDate)) { +// bulkUpdate.push({ index: { _index: 'unpaywall_enriched', _id: latestDoc.doi } }); +// bulkUpdate.push(latestDoc); +// } +// logger.debug('$$$$$$$$$$$$$$$'); +// logger.debug(latestDoc.genre); +// } + +// logger.debug(`[unpaywall] update : ${bulkUpdate.length / 2}`); + +// // UPDATE in unpaywall index +// try { +// await bulk(bulkUpdate, true); +// } catch (err) { +// logger.error('[elastic] Cannot bulk', err); +// return false; +// } + +// if (bulkUpdate.length > 0) { +// const dataThatWillBeDeleted = +// await searchWithRange(startDate, 'endValidity', 'gte', 'unpaywall_history'); + +// const listOfID = dataThatWillBeDeleted.map((e) => e._id); + +// listOfID.forEach((id) => { +// bulkHistoryDelete.push({ delete: { _index: 'unpaywall_history', _id: id } }); +// }); +// } + +// logger.debug(`[unpaywall_history] delete: ${bulkHistoryDelete.length}`); + +// // DELETE in history index +// try { +// await bulk(bulkHistoryDelete, true); +// } catch (err) { +// logger.error('[elastic] Cannot bulk', err); +// return false; +// } + +// data = await searchWithRange(startDate, 'updated', 'gt', 'unpaywall_enriched'); + +// data.forEach((e) => { +// bulkDelete.push({ delete: { _index: 'unpaywall_enriched', _id: e._source.doi } }); +// }); + +// logger.debug(`[unpaywall] delete: ${bulkDelete.length}`); + +// // DELETE in unpaywall index +// try { +// await bulk(bulkDelete, true); +// } catch (err) { +// logger.error('[elastic] Cannot bulk', err); +// return false; +// } + +// logger.debug(`[unpaywall_history] delete: ${bulkHistoryDelete.length}`); + +// data = await searchWithRange(startDate, 'updated', 'gt', 'unpaywall_history'); +// data.forEach((e) => { +// bulkHistoryDelete2.push({ delete: { _index: 'unpaywall_history', _id: e._id } }); +// }); + +// logger.debug(`[unpaywall_history] second delete: ${bulkHistoryDelete2.length}`); + +// // DELETE in unpaywall history index +// try { +// await bulk(bulkHistoryDelete2, true); +// } catch (err) { +// logger.error('[elastic] Cannot bulk', err); +// return false; +// } +// } + module.exports = { insertHistoryDataUnpaywall, rollBack, diff --git a/src/update/lib/services/elastic.js b/src/update/lib/services/elastic.js index 2589e232..bd453e41 100644 --- a/src/update/lib/services/elastic.js +++ b/src/update/lib/services/elastic.js @@ -149,57 +149,19 @@ async function getDataByListOfDOI(dois, index) { } /** - * Search old data in history with date + * Search data with range * @param {string} date - The date on which you want to obtain all the changes * @param {string} index - Elastic index name * @returns {Promise} */ -async function searchOldInHistoryData(date, index) { +async function searchWithRange(date, param, rangeParam, index) { const query = { bool: { filter: [ { range: { - endValidity: { - lte: date, - }, - }, - }, - ], - }, - }; - - let res; - try { - res = await elasticClient.search({ - index, - body: { - query, - sort: 'endValidity', - }, - }); - } catch (err) { - logger.error('[elastic] Cannot request elastic', err); - return null; - } - // eslint-disable-next-line no-underscore-dangle - return res.body.hits.hits; -} - -/** - * Search to recent data in history with date - * @param {string} date - The date on which you want to obtain all the changes - * @param {string} index - Elastic index name - * @returns {Promise} - */ -async function searchToRecentInHistoryData(date, index) { - const query = { - bool: { - filter: [ - { - range: { - endValidity: { - gt: date, + [param]: { + [rangeParam]: date, }, }, }, @@ -227,12 +189,12 @@ async function refreshIndex(index) { return elasticClient.indices.refresh({ index }); } -async function bulk(data) { +async function bulk(data, refresh = false) { if (data.length === 0) { - logger.warn('[elastic]: No data is send for bulk'); + // logger.warn('[elastic]: No data is send for bulk'); return []; } - return elasticClient.bulk({ body: data }); + return elasticClient.bulk({ body: data, refresh }); } module.exports = { @@ -243,6 +205,5 @@ module.exports = { getDataByListOfDOI, refreshIndex, bulk, - searchOldInHistoryData, - searchToRecentInHistoryData, + searchWithRange, }; diff --git a/src/update/test/history.js b/src/update/test/history.js index 4cf011e6..10f1a9ae 100644 --- a/src/update/test/history.js +++ b/src/update/test/history.js @@ -174,6 +174,6 @@ describe('Test: daily update route test with history', () => { }); after(async () => { - await reset('unpaywall_history'); + await reset(); }); }); diff --git a/src/update/test/rollback.js b/src/update/test/rollback.js new file mode 100644 index 00000000..70374f0d --- /dev/null +++ b/src/update/test/rollback.js @@ -0,0 +1,158 @@ +const { expect } = require('chai'); +const chai = require('chai'); +const chaiHttp = require('chai-http'); +const { setTimeout } = require('node:timers/promises'); + +const unpaywallEnrichedMapping = require('../mapping/unpaywall.json'); +const unpaywallHistoryMapping = require('../mapping/unpaywall_history.json'); + +const { + countDocuments, + insertHistoryDataUnpaywall, + deleteIndex, + getAllData, + createIndex, +} = require('./utils/elastic'); + +const updateURL = process.env.UPDATE_HOST || 'http://localhost:59702'; + +chai.use(chaiHttp); + +describe('Test: rollback history test', () => { + const date1 = '2020-01-04T13:00:00.000'; + const date2 = '2020-01-03T13:00:00.000'; + const date3 = '2020-01-02T13:00:00.000'; + // const date4 = '2020-01-01T13:00:00.000'; + + describe(`Rollback: history rollback at ${date1}`, () => { + before(async () => { + await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_history'); + await createIndex('unpaywall_enriched', unpaywallEnrichedMapping); + await createIndex('unpaywall_history', unpaywallHistoryMapping); + await insertHistoryDataUnpaywall(); + }); + + it('Should return a status code 202', async () => { + const res = await chai.request(updateURL) + .post('/job/history/reset') + .send({ + startDate: date1, + }) + .set('x-api-key', 'changeme'); + + expect(res).have.status(202); + }); + + it('Should get unpaywall data', async () => { + const count1 = await countDocuments('unpaywall_enriched'); + const count2 = await countDocuments('unpaywall_history'); + + expect(count1).to.equal(2); + expect(count2).to.equal(4); + + const data = await getAllData('unpaywall_enriched'); + + data.forEach((e) => { + expect(e.genre).to.equal('v3'); + }); + + const historyData = await getAllData('unpaywall_history'); + historyData.forEach((e) => { + expect(e.genre).not.equal('v3'); + }); + }); + + after(async () => { + // await deleteIndex('unpaywall_enriched'); + // await deleteIndex('unpaywall_history'); + }); + }); + + describe(`Rollback: history rollback at ${date2}`, () => { + before(async () => { + await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_history'); + await createIndex('unpaywall_enriched', unpaywallEnrichedMapping); + await createIndex('unpaywall_history', unpaywallHistoryMapping); + await insertHistoryDataUnpaywall(); + }); + + it('Should return a status code 202', async () => { + const res = await chai.request(updateURL) + .post('/job/history/reset') + .send({ + startDate: date2, + }) + .set('x-api-key', 'changeme'); + + expect(res).have.status(202); + }); + + it('Should get unpaywall data', async () => { + await setTimeout(1000); + const count1 = await countDocuments('unpaywall_enriched'); + const count2 = await countDocuments('unpaywall_history'); + + expect(count1).to.equal(2); + expect(count2).to.equal(2); + + const data = await getAllData('unpaywall_enriched'); + + data.forEach((e) => { + expect(e.genre).to.equal('v1'); + }); + + const historyData = await getAllData('unpaywall_history'); + historyData.forEach((e) => { + expect(e.genre).not.equal('v2'); + expect(e.genre).not.equal('v3'); + }); + }); + + after(async () => { + await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_history'); + }); + }); + + describe(`Rollback: history rollback at ${date3}`, () => { + before(async () => { + await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_history'); + await createIndex('unpaywall_enriched', unpaywallEnrichedMapping); + await createIndex('unpaywall_history', unpaywallHistoryMapping); + await insertHistoryDataUnpaywall(); + }); + + it('Should return a status code 202', async () => { + const res = await chai.request(updateURL) + .post('/job/history/reset') + .send({ + startDate: date3, + }) + .set('x-api-key', 'changeme'); + + expect(res).have.status(202); + }); + + it('Should get unpaywall data', async () => { + const count1 = await countDocuments('unpaywall_enriched'); + const count2 = await countDocuments('unpaywall_history'); + + expect(count1).to.equal(0); + expect(count2).to.equal(0); + + const data = await getAllData('unpaywall_enriched'); + + data.forEach((e) => { + expect(e.genre).to.equal('v1'); + }); + }); + + after(async () => { + await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_history'); + }); + }); +}); diff --git a/src/update/test/sources/unpaywall.jsonl b/src/update/test/sources/unpaywall.jsonl new file mode 100644 index 00000000..65d07a90 --- /dev/null +++ b/src/update/test/sources/unpaywall.jsonl @@ -0,0 +1,2 @@ +{"doi": "10.000/00.00.00","year": 2000,"genre": "v3","is_oa": true,"title": "Title","doi_url": "https://doi.org/10.000/00.00.00","updated": "2020-01-03T12:00:00.000000","oa_status": "gold","publisher": "Publisher","z_authors": [{"given": "John","family": "Doe","sequence": "first"}],"is_paratext": false,"journal_name": "Journal Name","oa_locations": [{"url": "https://doi.org/10.000/00.00.00","pmh_id": null,"is_best": true,"license": "cc-by","oa_date": "2010-01-01","updated": "2020-01-01T12:00:00.000000","version": "publishedVersion","evidence": "oa journal (via doaj)","host_type": "publisher","endpoint_id": null,"url_for_pdf": null,"url_for_landing_page": "https://doi.org/10.000/00.00.00","repository_institution": null}],"data_standard": 2,"journal_is_oa": true,"journal_issns": "0000-0001","journal_issn_l": "0000-0001","published_date": "2000-01-01","best_oa_location": {"url": "https://doi.org/10.000/00.00.00","pmh_id": null,"is_best": true,"license": "cc-by","oa_date": "2010-01-01","updated": "2020-01-01T12:00:00.000000","version": "publishedVersion","evidence": "oa journal (via doaj)","host_type": "publisher","endpoint_id": null,"url_for_pdf": null,"url_for_landing_page": "https://doi.org/10.000/00.00.00","repository_institution": null},"first_oa_location": {"url": "https://doi.org/10.000/00.00.00","pmh_id": null,"is_best": true,"license": "cc-by","oa_date": "2010-01-01","updated": "2020-01-01T12:00:00.000000","version": "publishedVersion","evidence": "oa journal (via doaj)","host_type": "publisher","endpoint_id": null,"url_for_pdf": null,"url_for_landing_page": "https://doi.org/10.000/00.00.00","repository_institution": null},"journal_is_in_doaj": true,"has_repository_copy": true} +{"doi": "10.000/00.00.01","year": 2000,"genre": "v3","is_oa": true,"title": "Title","doi_url": "https://doi.org/10.000/00.00.01","updated": "2020-01-03T12:00:00.000000","oa_status": "gold","publisher": "Publisher","z_authors": [{"given": "John","family": "Doe","sequence": "first"}],"is_paratext": false,"journal_name": "Journal Name","oa_locations": [{"url": "https://doi.org/10.000/00.00.01","pmh_id": null,"is_best": true,"license": "cc-by","oa_date": "2010-01-01","updated": "2020-01-01T12:00:00.000000","version": "publishedVersion","evidence": "oa journal (via doaj)","host_type": "publisher","endpoint_id": null,"url_for_pdf": null,"url_for_landing_page": "https://doi.org/10.000/00.00.01","repository_institution": null}],"data_standard": 2,"journal_is_oa": true,"journal_issns": "0000-0001","journal_issn_l": "0000-0001","published_date": "2000-01-01","best_oa_location": {"url": "https://doi.org/10.000/00.00.01","pmh_id": null,"is_best": true,"license": "cc-by","oa_date": "2010-01-01","updated": "2020-01-01T12:00:00.000000","version": "publishedVersion","evidence": "oa journal (via doaj)","host_type": "publisher","endpoint_id": null,"url_for_pdf": null,"url_for_landing_page": "https://doi.org/10.000/00.00.01","repository_institution": null},"first_oa_location": {"url": "https://doi.org/10.000/00.00.01","pmh_id": null,"is_best": true,"license": "cc-by","oa_date": "2010-01-01","updated": "2020-01-01T12:00:00.000000","version": "publishedVersion","evidence": "oa journal (via doaj)","host_type": "publisher","endpoint_id": null,"url_for_pdf": null,"url_for_landing_page": "https://doi.org/10.000/00.00.01","repository_institution": null},"journal_is_in_doaj": true,"has_repository_copy": true} \ No newline at end of file diff --git a/src/update/test/sources/unpaywall_history.jsonl b/src/update/test/sources/unpaywall_history.jsonl new file mode 100644 index 00000000..130669f0 --- /dev/null +++ b/src/update/test/sources/unpaywall_history.jsonl @@ -0,0 +1,4 @@ +{"doi":"10.000/00.00.01","year":2000,"genre":"v1","is_oa":false,"title":"Title","doi_url":"https://doi.org/10.000/00.00.01","updated":"2020-01-01T12:00:00.000000","oa_status":"gold","publisher":"Publisher","z_authors":[{"given":"John","family":"Doe","sequence":"first"}],"is_paratext":false,"journal_name":"Journal Name","oa_locations":[{"url":"https://doi.org/10.000/00.00.01","pmh_id":null,"is_best":true,"license":"cc-by","oa_date":"2010-01-01","updated":"2020-01-01T12:00:00.000000","version":"publishedVersion","evidence":"oa journal (via doaj)","host_type":"publisher","endpoint_id":null,"url_for_pdf":null,"url_for_landing_page":"https://doi.org/10.000/00.00.01","repository_institution":null}],"data_standard":2,"journal_is_oa":true,"journal_issns":"0000-0001","journal_issn_l":"0000-0001","published_date":"2000-01-01","best_oa_location":{"url":"https://doi.org/10.000/00.00.01","pmh_id":null,"is_best":true,"license":"cc-by","oa_date":"2010-01-01","updated":"2020-01-01T12:00:00.000000","version":"publishedVersion","evidence":"oa journal (via doaj)","host_type":"publisher","endpoint_id":null,"url_for_pdf":null,"url_for_landing_page":"https://doi.org/10.000/00.00.01","repository_institution":null},"first_oa_location":{"url":"https://doi.org/10.000/00.00.01","pmh_id":null,"is_best":true,"license":"cc-by","oa_date":"2010-01-01","updated":"2020-01-01T12:00:00.000000","version":"publishedVersion","evidence":"oa journal (via doaj)","host_type":"publisher","endpoint_id":null,"url_for_pdf":null,"url_for_landing_page":"https://doi.org/10.000/00.00.01","repository_institution":null},"journal_is_in_doaj":true,"has_repository_copy":true,"referencedAt":"2020-01-01","date":"2020-01-02","endValidity":"2020-01-02T12:00:00.000000"} +{"doi":"10.000/00.00.00","year":2000,"genre":"v1","is_oa":true,"title":"Title","doi_url":"https://doi.org/10.000/00.00.00","updated":"2020-01-01T12:00:00.000000","oa_status":"gold","publisher":"Publisher","z_authors":[{"given":"John","family":"Doe","sequence":"first"}],"is_paratext":false,"journal_name":"Journal Name","oa_locations":[{"url":"https://doi.org/10.000/00.00.00","pmh_id":null,"is_best":true,"license":"cc-by","oa_date":"2010-01-01","updated":"2020-01-01T12:00:00.000000","version":"publishedVersion","evidence":"oa journal (via doaj)","host_type":"publisher","endpoint_id":null,"url_for_pdf":null,"url_for_landing_page":"https://doi.org/10.000/00.00.00","repository_institution":null}],"data_standard":2,"journal_is_oa":true,"journal_issns":"0000-0001","journal_issn_l":"0000-0001","published_date":"2000-01-01","best_oa_location":{"url":"https://doi.org/10.000/00.00.00","pmh_id":null,"is_best":true,"license":"cc-by","oa_date":"2010-01-01","updated":"2020-01-01T12:00:00.000000","version":"publishedVersion","evidence":"oa journal (via doaj)","host_type":"publisher","endpoint_id":null,"url_for_pdf":null,"url_for_landing_page":"https://doi.org/10.000/00.00.00","repository_institution":null},"first_oa_location":{"url":"https://doi.org/10.000/00.00.00","pmh_id":null,"is_best":true,"license":"cc-by","oa_date":"2010-01-01","updated":"2020-01-01T12:00:00.000000","version":"publishedVersion","evidence":"oa journal (via doaj)","host_type":"publisher","endpoint_id":null,"url_for_pdf":null,"url_for_landing_page":"https://doi.org/10.000/00.00.00","repository_institution":null},"journal_is_in_doaj":true,"has_repository_copy":true,"referencedAt":"2020-01-01","date":"2020-01-02","endValidity":"2020-01-02T12:00:00.000000"} +{"doi":"10.000/00.00.00","year":2000,"genre":"v2","is_oa":false,"title":"Title","doi_url":"https://doi.org/10.000/00.00.00","updated":"2020-01-02T12:00:00.000000","oa_status":"gold","publisher":"Publisher","z_authors":[{"given":"John","family":"Doe","sequence":"first"}],"is_paratext":false,"journal_name":"Journal Name","oa_locations":[{"url":"https://doi.org/10.000/00.00.00","pmh_id":null,"is_best":true,"license":"cc-by","oa_date":"2010-01-01","updated":"2020-01-01T12:00:00.000000","version":"publishedVersion","evidence":"oa journal (via doaj)","host_type":"publisher","endpoint_id":null,"url_for_pdf":null,"url_for_landing_page":"https://doi.org/10.000/00.00.00","repository_institution":null}],"data_standard":2,"journal_is_oa":true,"journal_issns":"0000-0001","journal_issn_l":"0000-0001","published_date":"2000-01-01","best_oa_location":{"url":"https://doi.org/10.000/00.00.00","pmh_id":null,"is_best":true,"license":"cc-by","oa_date":"2010-01-01","updated":"2020-01-01T12:00:00.000000","version":"publishedVersion","evidence":"oa journal (via doaj)","host_type":"publisher","endpoint_id":null,"url_for_pdf":null,"url_for_landing_page":"https://doi.org/10.000/00.00.00","repository_institution":null},"first_oa_location":{"url":"https://doi.org/10.000/00.00.00","pmh_id":null,"is_best":true,"license":"cc-by","oa_date":"2010-01-01","updated":"2020-01-01T12:00:00.000000","version":"publishedVersion","evidence":"oa journal (via doaj)","host_type":"publisher","endpoint_id":null,"url_for_pdf":null,"url_for_landing_page":"https://doi.org/10.000/00.00.00","repository_institution":null},"journal_is_in_doaj":true,"has_repository_copy":true,"referencedAt":"2020-01-01","date":"2020-01-03","endValidity":"2020-01-03T12:00:00.000000"} +{"doi":"10.000/00.00.01","year":2000,"genre":"v2","is_oa":true,"title":"Title","doi_url":"https://doi.org/10.000/00.00.01","updated":"2020-01-02T12:00:00.000000","oa_status":"gold","publisher":"Publisher","z_authors":[{"given":"John","family":"Doe","sequence":"first"}],"is_paratext":false,"journal_name":"Journal Name","oa_locations":[{"url":"https://doi.org/10.000/00.00.01","pmh_id":null,"is_best":true,"license":"cc-by","oa_date":"2010-01-01","updated":"2020-01-01T12:00:00.000000","version":"publishedVersion","evidence":"oa journal (via doaj)","host_type":"publisher","endpoint_id":null,"url_for_pdf":null,"url_for_landing_page":"https://doi.org/10.000/00.00.01","repository_institution":null}],"data_standard":2,"journal_is_oa":true,"journal_issns":"0000-0001","journal_issn_l":"0000-0001","published_date":"2000-01-01","best_oa_location":{"url":"https://doi.org/10.000/00.00.01","pmh_id":null,"is_best":true,"license":"cc-by","oa_date":"2010-01-01","updated":"2020-01-01T12:00:00.000000","version":"publishedVersion","evidence":"oa journal (via doaj)","host_type":"publisher","endpoint_id":null,"url_for_pdf":null,"url_for_landing_page":"https://doi.org/10.000/00.00.01","repository_institution":null},"first_oa_location":{"url":"https://doi.org/10.000/00.00.01","pmh_id":null,"is_best":true,"license":"cc-by","oa_date":"2010-01-01","updated":"2020-01-01T12:00:00.000000","version":"publishedVersion","evidence":"oa journal (via doaj)","host_type":"publisher","endpoint_id":null,"url_for_pdf":null,"url_for_landing_page":"https://doi.org/10.000/00.00.01","repository_institution":null},"journal_is_in_doaj":true,"has_repository_copy":true,"referencedAt":"2020-01-01","date":"2020-01-03","endValidity":"2020-01-03T12:00:00.000000"} diff --git a/src/update/test/utils/elastic.js b/src/update/test/utils/elastic.js index c22f9aca..fdc92ab3 100644 --- a/src/update/test/utils/elastic.js +++ b/src/update/test/utils/elastic.js @@ -49,6 +49,62 @@ async function insertDataUnpaywall() { } } +async function insertHistoryDataUnpaywall() { + const filepath1 = path.resolve(__dirname, '..', 'sources', 'unpaywall_history.jsonl'); + let readStream1; + try { + readStream1 = await fs.createReadStream(filepath1); + } catch (err) { + console.error(`fs.createReadStream in insertDataUnpaywall: ${err}`); + } + + const rl1 = readline.createInterface({ + input: readStream1, + crlfDelay: Infinity, + }); + + const data1 = []; + + for await (const line of rl1) { + data1.push(JSON.parse(line)); + } + + const body1 = data1.flatMap((doc) => [{ index: { _index: 'unpaywall_history' } }, doc]); + + try { + await elasticClient.bulk({ refresh: true, body: body1 }); + } catch (err) { + console.error(err); + } + + const filepath2 = path.resolve(__dirname, '..', 'sources', 'unpaywall_history.jsonl'); + let readStream2; + try { + readStream2 = await fs.createReadStream(filepath2); + } catch (err) { + console.error(`fs.createReadStream in insertDataUnpaywall: ${err}`); + } + + const rl2 = readline.createInterface({ + input: readStream2, + crlfDelay: Infinity, + }); + + const data2 = []; + + for await (const line of rl2) { + data2.push(JSON.parse(line)); + } + + const body2 = data2.flatMap((doc) => [{ index: { _index: 'unpaywall_enriched', _id: doc.doi } }, doc]); + + try { + await elasticClient.bulk({ refresh: true, body: body2 }); + } catch (err) { + console.error(err); + } +} + /** * Check if index exit. * @@ -132,11 +188,33 @@ async function countDocuments(name) { return data?.body?.count ? data?.body?.count : 0; } +async function getAllData(index) { + let res; + try { + res = await elasticClient.search({ + index, + body: { + query: { + match_all: { }, + }, + }, + + }); + } catch (err) { + console.error('[elastic] Cannot request elastic', err); + return null; + } + // eslint-disable-next-line no-underscore-dangle + return res.body.hits.hits.map((hit) => hit._source); +} + module.exports = { elasticClient, createIndex, deleteIndex, checkIfIndexExist, insertDataUnpaywall, + insertHistoryDataUnpaywall, countDocuments, + getAllData, }; From cada7022d7732d5d67fa60bcf7d46cfc93d7c058 Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Fri, 13 Oct 2023 17:53:29 +0200 Subject: [PATCH 005/114] feat: rollback work and test done --- examples/example.csv | 1011 --------------------------- examples/example.jsonl | 1025 ---------------------------- src/update/lib/history.js | 233 ++----- src/update/lib/routers/job.js | 42 ++ src/update/lib/services/elastic.js | 1 + src/update/test/history.js | 13 +- src/update/test/rollback.js | 73 +- src/update/test/utils/elastic.js | 2 +- 8 files changed, 168 insertions(+), 2232 deletions(-) delete mode 100644 examples/example.csv delete mode 100644 examples/example.jsonl diff --git a/examples/example.csv b/examples/example.csv deleted file mode 100644 index 2d6b295e..00000000 --- a/examples/example.csv +++ /dev/null @@ -1,1011 +0,0 @@ -doi -10.3390/s19071619 -10.3390/su8090895 -10.3989/ris.2013.i1.505 -10.1007/s004390050363 -10.4000/studifrancesi.30097 -10.17576/geo-2019-1502-09 -10.4055/cios.2014.6.1.62 -10.22203/ecm.v006a02 -10.1253/circj.cj-17-1281 -10.22201/fq.18708404e.2018.4.64701 -10.1590/s0103-90162013000500006 -10.22201/fq.18708404e.2018.4.65745 -10.22201/ia.01851101p.2019.55.01.02 -10.22201/ia.14052059p.2019.51.06 -10.3389/fnins.2021.652552 -10.22201/ib.20078706e.2008.002.563 -10.5771/9783748909620 -10.1103/physrevstab.6.114801 -10.22201/ib.9786070271854e2015 -10.22201/ie.20074484e.1997.2.1.115 -10.1103/physrevlett.104.184502 -10.22201/ie.20074484e.1997.2.1.117 -10.1016/j.aspen.2020.10.009 -10.1002/bjs.1800570627 -10.1016/j.jeconom.2010.03.007 -10.1007/s41970-018-0045-2 -10.1051/0004-6361/201220248 -10.1051/0004-6361/201220256 -10.22201/ie.20074484e.1997.2.1.119 -10.1128/aem.01914-12 -10.21203/rs.3.rs-198787/v1 -10.1128/aem.01931-07 -10.1128/aem.01943-08 -10.1128/aem.01945-06 -10.1128/aem.01954-12 -10.1128/ec.05184-11 -10.1128/genomea.00376-14 -10.1128/genomea.01005-13 -10.1051/0004-6361/201321695 -10.1088/0004-637x/804/2/117 -10.1515/ahr-2016-0012 -10.15381/rivep.v30i1.15673 -10.7202/016182ar -10.22201/ie.20074484e.1997.2.1.75 -10.1128/genomea.01019-15 -10.4000/studifrancesi.32957 -10.1128/genomea.01542-17 -10.1590/s0102-79722001000200009 -10.3390/s16020266 -10.1093/molbev/msaa311 -10.3989/egeol.91473-4419 -10.1371/journal.pone.0248369 -10.1111/1759-7714.13706 -10.4102/aveh.v70i4.117 -10.1016/j.tet.2013.06.052 -10.4000/studifrancesi.29881 -10.4000/studifrancesi.30267 -10.22201/ie.20074484e.1998.3.1.59 -10.5040/9781350032675.ch-013 -10.1007/s10489-021-02291-9 -10.22201/ie.20074484e.1998.3.1.67 -10.1016/j.envpol.2019.112976 -10.1126/science.aau8461 -10.1016/j.respe.2019.08.002 -10.22201/ie.20074484e.1999.4.1.77 -10.1128/aac.01818-09 -10.21236/ada400633 -10.1093/sleep/zsaa030 -10.1021/la903052t -10.4000/studifrancesi.30251 -10.1016/j.foodres.2019.108737 -10.22260/isarc2016/0120 -10.1128/jb.00791-15 -10.22260/isarc2017/0001 -10.46743/2160-3715/2018.2934 -10.1128/jb.00798-10 -10.1128/jb.00801-10 -10.4000/studifrancesi.30438 -10.1128/jb.00838-08 -10.22260/isarc2017/0009 -10.1093/humrep/dep308 -10.22260/isarc2017/0010 -10.46743/2160-3715/2018.3118 -10.46743/2160-3715/2018.3169 -10.22260/isarc2017/0016 -10.3390/app9091843 -10.22260/isarc2017/0020 -10.1128/jb.00847-06 -10.22260/isarc2017/0022 -10.46743/2160-3715/2018.3180 -10.4000/studifrancesi.30436 -10.46743/2160-3715/2018.3209 -10.46743/2160-3715/2018.3242 -10.1371/journal.pcbi.1008854.s008 -10.5220/0006422104370443 -10.46743/2160-3715/2018.3259 -10.1002/cjoc.201200189 -10.46743/2160-3715/2018.3297 -10.1021/ct500566k -10.1016/j.jcis.2019.01.109 -10.46743/2160-3715/2018.3312 -10.22260/isarc2017/0024 -10.46743/2160-3715/2018.3358 -10.5860/choice.46-6298 -10.22260/isarc2017/0032 -10.1021/la9510745 -10.1128/jb.179.14.4513-4522.1997 -10.14712/23363398.2017.22 -10.5194/amt-11-5489-2018 -10.22260/isarc2017/0038 -10.4000/studifrancesi.32963 -10.5194/bg-2018-207 -10.5194/bg-5-407-2008 -10.1128/jb.179.15.4741-4746.1997 -10.22260/isarc2017/0040 -10.22260/isarc2017/0044 -10.22260/isarc2017/0046 -10.5194/bg-5-523-2008 -10.22260/isarc2017/0053 -10.5194/bgd-10-8611-2013 -10.1055/s-0033-1358369 -10.22260/isarc2017/0059 -10.7554/elife.43676 -10.22260/isarc2017/0060 -10.22260/isarc2017/0064 -10.4000/studifrancesi.30058 -10.1128/jcm.02861-13 -10.22260/isarc2017/0070 -10.4000/studifrancesi.30091 -10.1128/jcm.03068-13 -10.22260/isarc2017/0075 -10.22260/isarc2017/0078 -10.4000/studifrancesi.30153 -10.1111/j.1095-8339.1996.tb00757.x -10.1128/jcm.03094-14 -10.1128/jcm.03105-15 -10.1006/jmaa.2000.7104 -10.1128/jcm.03108-14 -10.1111/j.1708-8305.1994.tb00602.x -10.22260/isarc2017/0080 -10.1016/j.msea.2009.03.078 -10.22260/isarc2017/0081 -10.3389/fpubh.2018.00054 -10.22260/isarc2017/0085 -10.1128/jcm.39.2.631-635.2001 -10.3389/fchem.2019.00373 -10.1007/978-3-642-13495-1_52 -10.2166/9781789062106_xxi -10.1016/j.colsurfa.2019.124166 -10.1111/ppl.13413 -10.22260/isarc2017/0089 -10.1128/jcm.39.3.990-994.2001 -10.1128/jcm.42.9.4209-4213.2004 -10.3390/bios6030041 -10.1128/jcm.42.9.4329-4331.2004 -10.1128/jcm.44.4.1224-1228.2006 -10.3390/bios7030033 -10.1136/bmj.j3932 -10.14199/ppp-2015-005 -10.1051/0004-6361/201014837 -10.1128/jcm.44.4.1509-1518.2006 -10.22260/isarc2017/0095 -10.1140/epjh/e2013-40018-4 -10.1128/jmbe.v17i3.1243 -10.2169/internalmedicine.55.4418 -10.1143/jpsj.81.064711 -10.22260/isarc2017/0097 -10.3390/medsci7040056 -10.1055/b-006-160132 -10.1128/jmbe.v19i1.1435 -10.1073/pnas.76.6.2522 -10.22260/isarc2017/0101 -10.1128/jmbe.v20i1.1635 -10.7903/cmr.409 -10.22260/isarc2017/0107 -10.22260/isarc2017/0109 -10.1128/jvi.00644-11 -10.1128/jvi.00652-13 -10.22260/isarc2017/0111 -10.22260/isarc2017/0117 -10.22260/isarc2017/0124 -10.1051/0004-6361/201014922 -10.22260/isarc2017/0130 -10.22260/isarc2017/0132 -10.2139/ssrn.310240 -10.22260/isarc2017/0134 -10.22260/isarc2017/0139 -10.1128/jvi.00661-06 -10.29171/azu_acku_ds485_b21_t467_1876 -10.1128/jvi.01865-18 -10.1128/mcb.00917-10 -10.1128/mcb.00922-10 -10.11646/zootaxa.3613.5.5 -10.30837/2522-9818.2019.7.113 -10.22260/isarc2017/0143 -10.3390/ijms20071672 -10.22260/isarc2017/0149 -10.19062/2247-3173.2017.19.2.31 -10.22260/isarc2018/0003 -10.1371/journal.pone.0248369.s002 -10.5935/0103-507x.20190014 -10.22260/isarc2018/0005 -10.22260/isarc2018/0008 -10.1155/2020/6435043 -10.22260/isarc2018/0011 -10.22260/isarc2018/0018 -10.22260/isarc2018/0020 -10.1177/097380100800200404 -10.22260/isarc2018/0022 -10.22260/isarc2018/0025 -10.1002/9781119081418.ch44 -10.1177/1715163521992037 -10.3390/cryst8110401 -10.1055/s-0034-1373782 -10.22260/isarc2018/0028 -10.3389/fspas.2017.00049 -10.33830/jp.v22i1.1431.2021 -10.22260/isarc2018/0030 -10.22260/isarc2018/0032 -10.1016/j.asr.2019.03.031 -10.1016/j.jsat.2012.08.005 -10.22260/isarc2018/0034 -10.3897/zookeys.609.8852 -10.3897/zookeys.506.9796 -10.22260/isarc2018/0039 -10.22260/isarc2018/0041 -10.22260/isarc2018/0045 -10.22260/isarc2018/0048 -10.22260/isarc2018/0051 -10.22260/isarc2018/0054 -10.2754/avb201483s10s45 -10.22260/isarc2018/0056 -10.1021/ja9100886 -10.5194/acp-17-13017-2017 -10.1289/ehp.01109983 -10.1051/0004-6361/201937362 -10.1007/s00199-013-0796-5 -10.24138/jcomss.v7i3.175 -10.1029/2000gl011599 -10.29407/jetar.v1i1.277 -10.22260/isarc2018/0058 -10.1002/anie.201801122 -10.1111/j.1757-1707.2010.01031.x -10.1063/1.5050633 -10.1111/nph.12381 -10.22260/isarc2018/0060 -10.5186/aasfm.2010.3523 -10.5194/acp-3-1999-2003 -10.5194/acpd-14-21659-2014 -10.1016/j.arabjc.2021.103149 -10.22260/isarc2018/0062 -10.22260/isarc2018/0066 -10.22260/isarc2018/0069 -10.5194/bg-2019-489 -10.22260/isarc2018/0073 -10.3329/baj.v18i2.28899 -10.5194/nhess-1-93-2001 -10.5194/nhess-18-1279-2018 -10.5194/nhess-18-1617-2018 -10.1016/j.idcr.2021.e01115 -10.5194/nhessd-3-3761-2015 -10.5194/osd-6-429-2009 -10.2307/1565632 -10.3892/ol.2015.3845 -10.1088/0004-6256/150/1/13 -10.22260/isarc2018/0077 -10.3390/cryst8110431 -10.22260/isarc2018/0086 -10.1002/psp.2018 -10.1016/j.tws.2014.07.002 -10.1007/978-3-319-67615-9_2 -10.1080/03031853.2001.9524950 -10.5194/acpd-4-1449-2004 -10.22260/isarc2018/0088 -10.1063/1.3517251 -10.22260/isarc2018/0097 -10.22260/isarc2018/0099 -10.1161/01.hyp.24.5.633 -10.1371/journal.pntd.0001204 -10.22260/isarc2018/0103 -10.18632/oncotarget.6737 -10.1049/iet-map.2020.0415 -10.1053/euhj.1998.1351 -10.22260/isarc2018/0106 -10.1371/journal.pone.0214672 -10.1086/377027 -10.22260/isarc2018/0108 -10.22260/isarc2018/0112 -10.1007/10704703_3 -10.22260/isarc2018/0116 -10.6000/2292-2598.2020.08.03.25 -10.3389/fvets.2018.00024 -10.3390/f10040310 -10.1021/acs.langmuir.6b02927 -10.22260/isarc2018/0118 -10.5012/bkcs.2003.24.5.653 -10.22266/ijies2012.1231.01 -10.22266/ijies2012.1231.03 -10.33476/jky.v27i1.647 -10.5114/aoms.2016.62905 -10.1017/s135048270200110x -10.1016/s0006-3495(93)81340-7 -10.22260/isarc2018/0121 -10.22260/isarc2018/0123 -10.1029/2000gl011533 -10.22260/isarc2018/0129 -10.22260/isarc2018/0134 -10.1038/s41586-018-0222-z -10.3346/jkms.1986.1.1.37 -10.1108/s1479-364420150000017015 -10.22260/isarc2018/0136 -10.22260/isarc2018/0138 -10.22260/isarc2018/0141 -10.22260/isarc2018/0143 -10.1088/1126-6708/2008/04/069 -10.1155/2013/438384 -10.22260/isarc2018/0145 -10.5194/acp-3-1981-2003 -10.5194/acp-8-3149-2008 -10.5194/amt-11-5507-2018 -10.22260/isarc2018/0146 -10.22260/isarc2018/0149 -10.5194/amt-11-5549-2018 -10.5194/amt-11-5687-2018 -10.5194/bg-2018-198 -10.22260/isarc2018/0151 -10.22260/isarc2018/0153 -10.1007/s11214-017-0449-2 -10.5194/esdd-4-279-2013 -10.1111/j.1444-0938.1941.tb01195.x -10.5194/fr-21-67-2018 -10.5194/nhess-18-1351-2018 -10.5194/nhess-18-1427-2018 -10.22260/isarc2018/0154 -10.5194/nhess-19-41-2019 -10.1155/2020/5123830 -10.5194/nhessd-3-319-2015 -10.22260/isarc2018/0157 -10.22260/isarc2018/0158 -10.22266/ijies2012.9030.02 -10.4000/ree.3013 -10.22266/ijies2013.0331.01 -10.22266/ijies2013.0630.01 -10.21272/mmi.2019.4-29 -10.1007/978-3-319-55486-0_2 -10.22260/isarc2018/0160 -10.1080/1354750x.2021.1910343 -10.1038/ncomms10776 -10.4000/books.editionscnrs.2984 -10.1371/journal.pcbi.1008847.s001 -10.22260/isarc2018/0163 -10.22260/isarc2018/0166 -10.22260/isarc2018/0169 -10.2307/j.ctvwrm691.28 -10.22260/isarc2019/0001 -10.1128/aac.01120-16 -10.1128/aac.01124-10 -10.1128/aac.01129-17 -10.1128/aac.01136-15 -10.22260/isarc2019/0003 -10.1155/2017/6301026 -10.1128/aac.01479-15 -10.1128/aac.01481-13 -10.1155/s0161171203210516 -10.22260/isarc2019/0008 -10.1080/03066150.2016.1176023 -10.3390/e21040365 -10.5958/2349-2996.2018.00056.3 -10.1109/mcc.2000.846196 -10.3390/ani9030096 -10.1128/aac.01508-09 -10.35848/1347-4065/abe0a0 -10.1128/aac.01597-08 -10.1128/aac.01600-08 -10.1016/j.ijepes.2019.03.065 -10.1289/ehp.0800126 -10.1111/j.1574-6941.2012.01367.x -10.22260/isarc2019/0010 -10.22260/isarc2019/0016 -10.2131/jts.34.611 -10.1128/aem.00120-10 -10.22260/isarc2019/0021 -10.22260/isarc2019/0024 -10.22260/isarc2019/0027 -10.1128/aem.01905-15 -10.1128/aem.01912-08 -10.1128/aem.01944-08 -10.13172/2052-7829-1-3-846 -10.22266/ijies2014.0630.02 -10.1128/aem.01952-09 -10.1128/ec.05106-11 -10.1103/physrevd.99.096018 -10.1128/ec.2.4.678-689.2003 -10.1128/ec.3.1.91-99.2004 -10.1128/genomea.00368-18 -10.1128/genomea.01012-15 -10.1128/genomea.01329-14 -10.1128/genomea.01331-15 -10.1128/genomea.01539-16 -10.22266/ijies2014.1231.02 -10.3928/01477447-20190906-05 -10.1128/iai.00416-15 -10.1128/iai.00423-07 -10.1128/iai.00429-18 -10.1128/iai.00442-07 -10.1128/iai.00473-12 -10.1128/jb.00304-15 -10.1128/jb.00390-09 -10.1016/j.janxdis.2006.10.004 -10.1128/jb.01492-08 -10.1042/bj20101427 -10.18413/2712-746x-2020-45-4-623-632 -10.22260/isarc2019/0031 -10.1155/2019/8054640 -10.22260/isarc2019/0033 -10.22260/isarc2019/0038 -10.4000/clio.112 -10.1051/0004-6361/201116548 -10.2340/00015555-0919 -10.1161/jaha.118.009442 -10.22260/isarc2019/0044 -10.1099/jmm.0.45997-0 -10.22266/ijies2015.0930.01 -10.3889/oamjms.2019.617 -10.22260/isarc2019/0046 -10.1201/b16378-7 -10.4236/wjv.2015.51002 -10.1002/mbo3.941 -10.22260/isarc2019/0048 -10.1093/aje/kwq197 -10.1016/0029-554x(70)90052-2 -10.22260/isarc2019/0050 -10.22260/isarc2019/0054 -10.7202/902095ar -10.1371/journal.pone.0065137 -10.1371/journal.pgen.1007774 -10.22260/isarc2019/0056 -10.22260/isarc2019/0061 -10.1371/journal.pmed.1002257 -10.14507/epaa.v23.1915 -10.1371/journal.pmed.1002260 -10.14789/jmj.62.s78 -10.18011/bioeng2014v8n1p65-72 -10.20431/2347-3134.0506001 -10.1016/j.scitotenv.2018.10.259 -10.22260/isarc2019/0063 -10.20546/ijcmas.2019.807.213 -10.1371/journal.pmed.1002807 -10.22266/ijies2015.1231.01 -10.22260/isarc2019/0067 -10.1016/0197-4580(91)90081-t -10.22260/isarc2019/0069 -10.2136/sh1978.1.0013 -10.1371/journal.pone.0186465 -10.1371/journal.pone.0050219 -10.22260/isarc2019/0075 -10.5794/jjoms.41.705 -10.4319/lo.2007.52.2.0912 -10.1371/journal.pone.0210031 -10.5194/acp-8-2975-2008 -10.5194/acp-9-9349-2009 -10.1007/3-540-56282-6_20 -10.1007/s10068-020-00875-8 -10.5194/acpd-14-22045-2014 -10.4236/jmmce.2012.1110097 -10.5194/amt-11-5673-2018 -10.5194/amt-11-569-2018 -10.1190/geo2015-0387.1 -10.3390/brainsci3010344 -10.22266/ijies2015.1231.03 -10.22266/ijies2016.0930.01 -10.1093/hmg/ddab105 -10.22260/isarc2019/0077 -10.22260/isarc2019/0084 -10.22260/isarc2019/0086 -10.5194/bg-2018-201 -10.1088/2058-6272/aab490 -10.1128/aac.01249-16 -10.1051/matecconf/201819401034 -10.5194/bg-2019-476 -10.5194/bg-5-707-2008 -10.5040/9781628927139 -10.22260/isarc2019/0088 -10.1152/ajpheart.2000.279.5.h2124 -10.1088/1757-899x/777/1/011001 -10.22260/isarc2019/0091 -10.1558/eph.v20i2.105 -10.22260/isarc2019/0095 -10.12927/hcq.2016.24703 -10.1093/icvts/ivx352 -10.1155/2015/764187 -10.1371/journal.pntd.0008298 -10.22260/isarc2019/0106 -10.1051/0004-6361/201015251 -10.20944/preprints202102.0379.v1 -10.1002/cjce.5450730418 -10.1128/jb.00605-07 -10.1039/b918311f -10.5369/jsst.2012.21.1.59 -10.22260/isarc2019/0109 -10.1371/journal.pone.0201390 -10.22260/isarc2019/0119 -10.1128/jb.179.1.135-140.1997 -10.22260/isarc2019/0121 -10.22260/isarc2019/0123 -10.1128/jb.183.24.7094-7101.2001 -10.22266/ijies2016.0930.02 -10.18041/entramado.2016v12n1.23124 -10.1002/jcu.1870210903 -10.22266/ijies2016.0930.03 -10.22266/ijies2016.0930.05 -10.21203/rs.3.rs-238650/v1 -10.1155/2020/6662389 -10.21203/rs.2.23128/v5 -10.22260/isarc2019/0127 -10.1016/j.jalgebra.2016.09.019 -10.1371/journal.pone.0011574 -10.1155/2020/4360184 -10.22260/isarc2019/0129 -10.22260/isarc2019/0132 -10.3390/catal8020058 -10.22260/isarc2019/0135 -10.1073/pnas.80.17.5330 -10.1016/j.ocecoaman.2019.104864 -10.1021/acs.langmuir.9b01833 -10.1121/1.3249303 -10.1128/jb.00553-16 -10.22260/isarc2019/0138 -10.22260/isarc2019/0139 -10.22260/isarc2019/0146 -10.22260/isarc2019/0152 -10.1126/sciadv.aay8507 -10.22260/isarc2019/0155 -10.22260/isarc2019/0158 -10.1126/science.1056496 -10.3745/jips.2005.1.1.009 -10.1523/jneurosci.15-10-06846.1995 -10.1126/science.1057022 -10.1371/journal.pone.0040655 -10.3329/jce.v30i1.34793 -10.22260/isarc2019/0160 -10.3390/cells6040037 -10.1371/journal.pone.0135360 -10.1126/science.1142021 -10.1061/(asce)0899-1561(2009)21:6(286) -10.21032/jhis.2018.43.4.274 -10.1126/science.1216556 -10.22203/ecm.v031a21 -10.1002/jlcr.2580220809 -10.1007/978-3-540-70575-8_5 -10.1155/2013/569161 -10.1007/s00041-002-0001-x -10.21203/rs.3.rs-221618/v1 -10.1126/science.270.5240.1347 -10.1155/2020/1346051 -10.1080/10852352.2014.881172 -10.1126/science.270.5242.1633 -10.1016/j.nima.2019.03.071 -10.3390/ijms21249669 -10.22260/isarc2019/0164 -10.22260/isarc2019/0177 -10.22260/isarc2019/0179 -10.22263/2312-4156.2016.4.18 -10.22266/ijies2008.0331.05 -10.1017/s0074180900158371 -10.22266/ijies2008.0630.01 -10.22266/ijies2008.0630.03 -10.1172/jci38468 -10.1128/aem.71.4.1865-1869.2005 -10.5040/9781350052444-0789 -10.2478/ijame-2019-0004 -10.22266/ijies2008.0630.05 -10.1128/aem.71.4.1941-1945.2005 -10.22266/ijies2008.0930.01 -10.1128/cvi.00275-09 -10.1128/cvi.00457-14 -10.5916/jkosme.2018.42.9.719 -10.1128/ec.00065-08 -10.1128/ec.00267-09 -10.4314/jab.v81i1.3 -10.1128/ec.4.7.1203-1210.2005 -10.22266/ijies2008.1231.01 -10.1128/genomea.00173-13 -10.1038/ncomms4130 -10.1128/genomea.00174-17 -10.1093/oxfordjournals.jhered.a023131 -10.1128/genomea.00175-17 -10.1128/genomea.00311-17 -10.1371/journal.pone.0068127 -10.22266/ijies2009.0331.02 -10.22266/ijies2009.0630.01 -10.1093/plankt/21.9.1725 -10.22266/ijies2009.0630.04 -10.1128/jb.01946-14 -10.1103/physrevb.87.174412 -10.1103/physrevlett.95.117005 -10.1128/jb.01956-05 -10.1111/j.1467-9469.2011.00738.x -10.22266/ijies2010.0331.01 -10.1136/bmj.301.6766.1480 -10.1016/j.ecolecon.2019.106426 -10.22266/ijies2010.0331.03 -10.12681/eadd/25784 -10.22266/ijies2010.0630.02 -10.22266/ijies2010.1231.01 -10.22266/ijies2010.1231.03 -10.22266/ijies2010.1231.05 -10.1016/j.apcata.2021.118139 -10.1016/j.apcata.2021.118143 -10.22266/ijies2011.0331.04 -10.4236/ce.2015.613147 -10.1016/j.apcatb.2021.120203 -10.5040/9781472546395.0009 -10.1038/nbt.2601 -10.22266/ijies2011.0630.01 -10.1073/pnas.1304097110 -10.22266/ijies2011.0630.05 -10.1108/00346659910290222 -10.1016/j.apcatb.2021.120207 -10.22266/ijies2011.0930.03 -10.22266/ijies2012.0331.03 -10.1007/s11060-012-0973-6 -10.22266/ijies2012.0630.02 -10.22266/ijies2012.1231.02 -10.13187/ejced.2019.1.136 -10.1371/journal.pbio.2001307 -10.5580/d9b -10.1128/jb.184.7.2045-2049.2002 -10.22266/ijies2013.0630.02 -10.1128/jcm.00619-11 -10.22266/ijies2014.0331.01 -10.22266/ijies2014.1231.01 -10.22266/ijies2014.9030.01 -10.1063/1.1306910 -10.1073/pnas.2008791117 -10.1042/cs20060043 -10.1128/jcm.01061-10 -10.4236/ojms.2015.51012 -10.25047/jtit.v5i1.73 -10.22266/ijies2015.0630.02 -10.22266/ijies2015.1231.02 -10.22260/isarc2016/0052 -10.22260/isarc2016/0053 -10.22260/isarc2016/0058 -10.1371/journal.pgen.1005041 -10.1371/journal.pmed.1002245 -10.1371/journal.pmed.1002781 -10.1484/j.rea.5.104103 -10.22260/isarc2016/0061 -10.3727/096504018x15267574931782 -10.22260/isarc2016/0071 -10.1590/s0104-06182012000100002 -10.22260/isarc2016/0080 -10.22260/isarc2016/0081 -10.22260/isarc2016/0087 -10.1021/acs.energyfuels.0c03860 -10.15448/1980-6523.2014.2.12666 -10.1590/s1415-43662013000700001 -10.5713/ajas.18.0264 -10.3847/1538-3881/aab1f6 -10.1128/genomea.00188-14 -10.22260/isarc2016/0090 -10.22266/ijies2016.0930.06 -10.1371/journal.pone.0152394 -10.1371/journal.pntd.0004076 -10.1007/978-3-0348-0043-3_4 -10.1161/jaha.112.002832 -10.1002/cphc.201402895 -10.2134/agronj2008.0139s -10.22260/isarc2016/0095 -10.22260/isarc2016/0100 -10.22260/isarc2016/0105 -10.22266/ijies2016.0930.07 -10.1051/0004-6361/201015772 -10.1037/a0014611 -10.4054/demres.2004.s3.1 -10.1117/12.632157 -10.22260/isarc2016/0114 -10.30837/2522-9818.2019.7.059 -10.22260/isarc2016/0116 -10.22260/isarc2016/0119 -10.5130/cjlg.v0i6.1623 -10.1016/j.parco.2018.12.007 -10.22260/isarc2016/0125 -10.3390/sym11040460 -10.1287/mnsc.1080.0886 -10.22260/isarc2016/0126 -10.1088/1361-6420/abd29a -10.22260/isarc2016/0128 -10.22260/isarc2016/0130 -10.22260/isarc2016/0132 -10.22260/isarc2017/0003 -10.22260/isarc2017/0004 -10.5465/amj.2011.59215082 -10.1073/pnas.39.9.924 -10.22260/isarc2017/0006 -10.22260/isarc2017/0015 -10.22260/isarc2017/0021 -10.1122/1.1567751 -10.1126/science.1217416 -10.21608/resoncol.2017.656.1016 -10.22260/isarc2017/0023 -10.3389/fpls.2017.01539 -10.1126/scisignal.aaf2223 -10.1128/aac.02175-18 -10.22260/isarc2017/0025 -10.22260/isarc2017/0030 -10.3389/fpsyg.2019.00588 -10.5799/ahinjs.01.2014.02.0416 -10.22260/isarc2017/0037 -10.1021/jp025626l -10.1073/pnas.1602214113 -10.22260/isarc2017/0039 -10.32725/jab.2006.011 -10.1016/j.trc.2014.09.015 -10.1039/c8cp06992a -10.22260/isarc2017/0041 -10.1128/aac.37.2.178 -10.22260/isarc2017/0047 -10.22586/csp.v52i3.11167 -10.1109/cvpr.2018.00132 -10.1021/acsenergylett.9b02478 -10.22260/isarc2017/0055 -10.3928/15428877-20120920-02 -10.1063/5.0021818 -10.1128/aac.37.4.775 -10.22260/isarc2017/0058 -10.1051/rnd:19940364 -10.11118/actaun201260040165 -10.22260/isarc2017/0069 -10.1016/j.conbuildmat.2019.08.019 -10.1016/j.drugalcdep.2019.01.020 -10.1016/j.nima.2019.03.035 -10.14720/aas.2018.111.2.07 -10.18616/ce.v0i0.2946 -10.1128/cvi.00133-10 -10.1128/ec.00113-07 -10.1128/genomea.00163-14 -10.1093/intqhc/mzv026 -10.1128/genomea.01082-15 -10.22260/isarc2017/0071 -10.1190/geo2012-0334.1 -10.1093/fampra/cml054 -10.3989/aeamer.2002.v59.i2.189 -10.22260/isarc2017/0083 -10.1177/2324709612473274 -10.1016/j.ymssp.2019.02.018 -10.1099/00207713-46-3-792 -10.1177/2042098616646726 -10.6113/jpe.2016.16.2.786 -10.22260/isarc2017/0086 -10.1890/10-0318.1 -10.22260/isarc2017/0088 -10.22260/isarc2017/0093 -10.1042/bj20101360 -10.22260/isarc2017/0096 -10.22260/isarc2017/0098 -10.22260/isarc2017/0103 -10.1140/epjs/s11734-021-00082-4 -10.22260/isarc2017/0110 -10.1007/jhep02(2021)163 -10.22260/isarc2017/0120 -10.1016/j.tcs.2017.01.028 -10.22260/isarc2017/0125 -10.1016/j.enconman.2007.01.023 -10.22260/isarc2017/0137 -10.22260/isarc2017/0147 -10.22260/isarc2017/0151 -10.1161/01.str.30.9.1875 -10.3390/12071410 -10.1177/0192512107088390 -10.22260/isarc2018/0004 -10.22260/isarc2018/0007 -10.22260/isarc2018/0009 -10.1080/15384047.2015.1056416 -10.22260/isarc2018/0012 -10.33588/fem.62.258 -10.22260/isarc2018/0016 -10.22260/isarc2018/0021 -10.1128/iai.69.4.2621-2629.2001 -10.1017/s1748499517000124 -10.1128/iai.69.5.3110-3119.2001 -10.22260/isarc2018/0024 -10.1677/joe-06-0076 -10.1128/iai.69.5.3295-3304.2001 -10.2134/jeq2007.0310 -10.1163/9789004217881_020 -10.22260/isarc2018/0027 -10.22260/isarc2018/0029 -10.22260/isarc2018/0031 -10.5802/smai-jcm.52 -10.22260/isarc2018/0033 -10.1007/978-3-319-57123-2_4 -10.1128/jb.182.19.5606-5610.2000 -10.22260/isarc2018/0035 -10.1128/jb.187.18.6281-6289.2005 -10.1128/jcm.01231-14 -10.22260/isarc2018/0040 -10.1056/nejmoa1005372 -10.1128/jcm.01245-11 -10.1128/jcm.01248-12 -10.22260/isarc2018/0044 -10.1128/jcm.01250-09 -10.22260/isarc2018/0047 -10.1128/jcm.01849-09 -10.1128/jcm.01854-07 -10.22260/isarc2018/0049 -10.1128/jcm.02483-14 -10.22260/isarc2018/0053 -10.1049/el:20070162 -10.1007/s11442-015-1184-9 -10.3928/00220124-20070901-09 -10.1128/jcm.06221-11 -10.22266/ijies2016.0930.08 -10.1159/000381911 -10.1128/jvi.00713-10 -10.22266/ijies2016.0930.10 -10.1128/mcb.19.8.5685 -10.22266/ijies2016.0930.11 -10.22260/isarc2018/0055 -10.1016/j.smrv.2021.101490 -10.22260/isarc2018/0057 -10.1016/0020-711x(80)90151-2 -10.1371/journal.pone.0137890 -10.1137/090772514 -10.1016/j.jallcom.2021.159740 -10.22260/isarc2018/0061 -10.22260/isarc2018/0064 -10.22260/isarc2018/0067 -10.22260/isarc2018/0072 -10.1016/j.wombi.2018.11.012 -10.22260/isarc2018/0074 -10.24310/abm.v30i0.7209 -10.1130/g23860a.1 -10.22260/isarc2018/0084 -10.1134/s0001434619070034 -10.22260/isarc2018/0087 -10.22260/isarc2018/0091 -10.1016/j.tcm.2021.04.001 -10.22260/isarc2018/0098 -10.14348/molcells.2015.0052 -10.22260/isarc2018/0100 -10.22260/isarc2018/0105 -10.1590/s0103-166x2013000200011 -10.3406/cea.1989.2142 -10.22260/isarc2018/0107 -10.18535/ijecs/v5i8.54 -10.18542/amazonica.v2i1.347 -10.1590/s0037-86821980000100008 -10.1038/063396c0 -10.1051/0004-6361/202039071 -10.2147/jpr.s118945 -10.22260/isarc2018/0109 -10.22260/isarc2018/0117 -10.22260/isarc2018/0120 -10.4000/laboratoireitalien.72 -10.17721/2227-1481.7.23-29 -10.22260/isarc2018/0122 -10.1007/s10751-016-1374-6 -10.23865/noasp.74 -10.4103/0975-1483.62213 -10.1007/s41906-021-1006-x -10.22260/isarc2018/0124 -10.1006/jagm.1995.1041 -10.22260/isarc2018/0127 -10.22260/isarc2018/0135 -10.22260/isarc2018/0137 -10.3928/01477447-20190906-02 -10.22260/isarc2018/0140 -10.22260/isarc2018/0144 -10.1541/ieejeiss.129.1949 -10.1126/scirobotics.aaw6326 -10.22260/isarc2018/0147 -10.1128/aac.00829-05 -10.1001/archotol.134.10.1094 -10.1128/aac.00829-19 -10.22260/isarc2018/0152 -10.22260/isarc2018/0156 -10.1128/aem.06444-11 -10.1128/aem.69.9.5463-5471.2003 -10.1128/genomea.01115-16 -10.1128/iai.02016-05 -10.22260/isarc2018/0159 -10.1128/iai.02020-14 -10.1042/bj20101769 -10.1080/01402380902779048 -10.22260/isarc2018/0161 -10.1128/iai.06174-11 -10.1128/iai.06176-11 -10.22260/isarc2018/0167 -10.1093/ageing/27.2.123 -10.1073/pnas.1313548110 -10.22260/isarc2019/0002 -10.13065/jksdh.2012.12.1.157 -10.1080/0141620910140108 -10.22260/isarc2019/0007 -10.22266/ijies2016.0930.15 -10.3390/computation3030479 -10.1128/iai.06184-11 -10.22260/isarc2019/0015 -10.1128/iai.06308-11 -10.1097/sla.0b013e3182891efb -10.22266/ijies2016.0930.16 -10.1093/comjnl/bxs137 -10.22260/isarc2019/0022 -10.22260/isarc2019/0025 -10.22266/ijies2016.1231.02 -10.1111/j.1527-3458.1998.tb00073.x -10.1128/iai.36.2.450-454.1982 -10.1128/iai.71.6.3357-3360.2003 -10.1504/ijemr.2018.10009113 -10.1890/0012-9658(2003)084[0791:eopsap]2.0.co;2 -10.1136/bjsm.2004.015271 -10.1128/iai.71.6.3512-3520.2003 -10.1155/2008/782358 -10.22266/ijies2016.1231.03 -10.1128/iai.72.7.3914-3924.2004 -10.1128/iai.72.7.3932-3940.2004 -10.1128/iai.72.7.4240-4248.2004 -10.22266/ijies2016.1231.04 -10.1128/iai.72.8.4401-4409.2004 -10.1128/iai.72.8.4432-4438.2004 -10.22266/ijies2016.1231.05 -10.1128/jcm.42.9.4386-4389.2004 -10.1111/j.1365-246x.2012.05533.x -10.1111/jvim.15287 -10.1111/jvim.15291 -10.1128/jcm.44.4.1347-1351.2006 -10.1128/jmbe.v20i1.1669 -10.3390/a10030082 -10.11606/2179-0892.ra.2017.137322 -10.22260/isarc2019/0030 -10.1101/819987 -10.22260/isarc2019/0037 -10.21500/16578031.220 -10.4148/2334-4415.1197 -10.22260/isarc2019/0040 -10.1186/s13550-021-00771-0 -10.22260/isarc2019/0045 -10.1128/jvi.01853-16 -10.1128/jvi.01868-17 -10.2514/6.1991-4090 -10.1128/mbio.00098-13 -10.1128/mbio.00103-12 -10.22260/isarc2019/0047 -10.1128/mbio.00105-15 -10.1128/mbio.00114-15 -10.1128/mbio.00115-13 -10.1128/mbio.00118-12 -10.4236/ojs.2016.65070 -10.1128/mbio.00129-15 -10.1128/mbio.00139-12 -10.1128/mbio.00144-13 -10.23865/noasp.80 -10.1128/mcb.00086-19 -10.1016/j.cortex.2020.04.002 -10.1128/mcb.00087-14 -10.1128/mcb.00266-15 -10.1128/mcb.00420-13 -10.1371/journal.pone.0077108 -10.22260/isarc2019/0049 -10.1093/hmg/ddab103 -10.22260/isarc2019/0053 -10.1016/j.dsr.2020.103227 -10.3847/1538-4357/aab719 -10.1051/epjconf/20136205005 -10.1542/peds.101.5.e12 -10.1016/j.envint.2020.106248 -10.1155/2013/260351 -10.1016/j.lpm.2019.04.005 -10.1016/j.jcrc.2018.06.002 -10.1016/j.ijpvp.2019.02.011 -10.3889/oamjms.2018.182 -10.1016/j.conbuildmat.2019.117340 -10.1155/2020/1062325 -10.22260/isarc2019/0055 -10.31618/esu.2413-9335.2020.5.81.1166 -10.22260/isarc2019/0059 -10.1249/mss.0b013e3181677530 -10.22260/isarc2019/0062 -10.22260/isarc2019/0066 -10.21236/ada247549 -10.1364/oe.18.015484 -10.3389/fphar.2018.00348 -10.1590/s0103-84782014000600010 -10.22260/isarc2019/0068 -10.1590/s1414-462x2013000200009 -10.22260/isarc2019/0073 -10.5944/rppc.vol.2.num.2.1997.3837 -10.22260/isarc2019/0076 -10.1155/2021/5529486 -10.22146/gamaijb.9292 -10.22260/isarc2019/0078 -10.1177/2514848620945310 -10.1002/lt.25925 -10.1074/jbc.ra119.011766 -10.1111/j.1440-1797.2009.01116.x -10.22260/isarc2019/0081 -10.22260/isarc2019/0085 -10.3390/ijerph13040364 -10.1093/aje/kwn279 -10.3390/en6020748 -10.3390/en6020884 -10.22260/isarc2019/0087 -10.1042/bj20150325 -10.22260/isarc2019/0090 -10.1128/jb.173.9.2800-2808.1991 -10.1523/jneurosci.0860-20.2020 -10.22260/isarc2019/0093 -10.22260/isarc2019/0098 \ No newline at end of file diff --git a/examples/example.jsonl b/examples/example.jsonl deleted file mode 100644 index 478344ef..00000000 --- a/examples/example.jsonl +++ /dev/null @@ -1,1025 +0,0 @@ -{ "doi": "10.22260/isarc2019/0068" } -{ "doi": "10.1590/s1414-462x2013000200009" } -{ "doi": "10.22260/isarc2019/0073" } -{ "doi": "10.5944/rppc.vol.2.num.2.1997.3837" } -{ "doi": "10.22260/isarc2019/0076" } -{ "doi": "10.1155/2021/5529486" } -{ "doi": "10.22146/gamaijb.9292" } -{ "doi": "10.22260/isarc2019/0078" } -{ "doi": "10.1177/2514848620945310" } -{ "doi": "10.1002/lt.25925" } -{ "doi": "10.1074/jbc.ra119.011766" } -{ "doi": "10.1111/j.1440-1797.2009.01116.x" } -{ "doi": "10.22260/isarc2019/0081" } -{ "doi": "10.22260/isarc2019/0085" } -{ "doi": "10.3390/ijerph13040364" } -{ "doi": "10.1093/aje/kwn279" } -{ "doi": "10.3390/en6020748" } -{ "doi": "10.3390/en6020884" } -{ "doi": "10.22260/isarc2019/0087" } -{ "doi": "10.1042/bj20150325" } -{ "doi": "10.22260/isarc2019/0090" } -{ "doi": "10.1128/jb.173.9.2800-2808.1991" } -{ "doi": "10.1523/jneurosci.0860-20.2020" } -{ "doi": "10.22260/isarc2019/0093" } -{ "doi": "10.22260/isarc2019/0098" } -{ "doi": "10.3389/fnhum.2014.00785" } -{ "doi": "10.1002/uog.23647" } -{ "doi": "10.1128/jb.174.4.1179-1188.1992" } -{ "doi": "10.22260/isarc2019/0113" } -{ "doi": "10.22260/isarc2019/0115" } -{ "doi": "10.22260/isarc2019/0120" } -{ "doi": "10.31235/osf.io/dgq23" } -{ "doi": "10.31235/osf.io/y943w" } -{ "doi": "10.23865/noasp.89" } -{ "doi": "10.1049/iet-rsn.2016.0063" } -{ "doi": "10.22260/isarc2019/0122" } -{ "doi": "10.1038/s41598-017-05475-x" } -{ "doi": "10.1093/infdis/53.3.312" } -{ "doi": "10.1016/j.geomphys.2018.05.014" } -{ "doi": "10.1088/2515-7620/abef5d" } -{ "doi": "10.22260/isarc2019/0126" } -{ "doi": "10.1128/jb.175.2.503-509.1993" } -{ "doi": "10.1037/a0018912" } -{ "doi": "10.22260/isarc2019/0128" } -{ "doi": "10.22260/isarc2019/0131" } -{ "doi": "10.1093/shm/hkr051" } -{ "doi": "10.1088/0004-637x/693/2/l118" } -{ "doi": "10.1364/oe.22.028997" } -{ "doi": "10.1098/rstl.1686.0023" } -{ "doi": "10.22260/isarc2019/0133" } -{ "doi": "10.22260/isarc2019/0136" } -{ "doi": "10.1109/tip.2021.3058783" } -{ "doi": "10.22260/isarc2019/0140" } -{ "doi": "10.22260/isarc2019/0150" } -{ "doi": "10.22260/isarc2019/0151" } -{ "doi": "10.22260/isarc2019/0154" } -{ "doi": "10.5194/isprsarchives-xl-8-745-2014" } -{ "doi": "10.22260/isarc2019/0156" } -{ "doi": "10.22260/isarc2019/0159" } -{ "doi": "10.5539/gjhs.v7n5p208" } -{ "doi": "10.33767/osf.io/9frgd" } -{ "doi": "10.1002/chin.198919124" } -{ "doi": "10.1109/rivf48685.2020.9140776" } -{ "doi": "10.3389/fonc.2020.595527" } -{ "doi": "10.18260/1-2--6466" } -{ "doi": "10.1128/jvi.01356-06" } -{ "doi": "10.3389/fonc.2021.626659" } -{ "doi": "10.22260/isarc2019/0163" } -{ "doi": "10.1364/oe.15.005966" } -{ "doi": "10.1128/jb.177.21.6118-6125.1995" } -{ "doi": "10.1158/0008-5472.can-07-1356" } -{ "doi": "10.22260/isarc2019/0166" } -{ "doi": "10.22260/isarc2019/0169" } -{ "doi": "10.1063/5.0029716" } -{ "doi": "10.22260/isarc2019/0173" } -{ "doi": "10.1016/j.agsy.2019.102668" } -{ "doi": "10.22260/isarc2019/0178" } -{ "doi": "10.22263/2312-4156.2016.1.93" } -{ "doi": "10.1038/509030a" } -{ "doi": "10.7554/elife.19484" } -{ "doi": "10.1016/j.nutres.2007.03.006" } -{ "doi": "10.1007/s11614-020-00408-x" } -{ "doi": "10.4236/ojps.2021.111008" } -{ "doi": "10.1088/0004-637x/746/2/119" } -{ "doi": "10.22201/iij.24484873e.2016.146.10511" } -{ "doi": "10.1016/j.evolhumbehav.2016.05.001" } -{ "doi": "10.22201/iij.24484873e.2016.146.10515" } -{ "doi": "10.22201/iij.24484873e.2016.146.10516" } -{ "doi": "10.1088/0953-8984/24/30/305402" } -{ "doi": "10.22201/iij.24484881e.2017.36.10857" } -{ "doi": "10.22201/iij.24487872e.2016.16.521" } -{ "doi": "10.1093/bioinformatics/btu153" } -{ "doi": "10.1007/978-1-4419-5906-5_824" } -{ "doi": "10.1021/jo100462w" } -{ "doi": "10.22201/iij.24487872e.2018.18.12110" } -{ "doi": "10.5902/198050981650" } -{ "doi": "10.22201/iij.24487899e.2017.24.10815" } -{ "doi": "10.1016/j.spmi.2017.11.007" } -{ "doi": "10.22201/iij.24487899e.2017.24.10818" } -{ "doi": "10.1128/jb.184.24.6760-6767.2002" } -{ "doi": "10.22201/iij.24487910e.2013.3.10011" } -{ "doi": "10.1128/jb.184.24.6906-6917.2002" } -{ "doi": "10.5194/amt-11-2523-2018" } -{ "doi": "10.22201/iij.24487910e.2013.4.10037" } -{ "doi": "10.1128/jb.186.12.3777-3784.2004" } -{ "doi": "10.1128/jb.186.12.4046-4050.2004" } -{ "doi": "10.22201/iij.24487910e.2013.4.10041" } -{ "doi": "10.22201/iij.24487910e.2014.5.10052" } -{ "doi": "10.1136/jnnp.2003.026542" } -{ "doi": "10.22201/iij.24487910e.2014.6.10067" } -{ "doi": "10.1038/s41598-018-23915-0" } -{ "doi": "10.1061/(asce)mt.1943-5533.0002511" } -{ "doi": "10.1016/j.electacta.2018.10.184" } -{ "doi": "10.1016/s0168-9002(02)00430-8" } -{ "doi": "10.1103/physrevd.90.112016" } -{ "doi": "10.22201/iij.24487910e.2014.6.10078" } -{ "doi": "10.1109/ipdpsw.2012.11" } -{ "doi": "10.22201/iij.24487910e.2016.9.10100" } -{ "doi": "10.1002/mmng.200900010" } -{ "doi": "10.1111/nph.12375" } -{ "doi": "10.1109/wowmom.2011.5986202" } -{ "doi": "10.22201/iij.24487937e.2007.1.8030" } -{ "doi": "10.1140/epjc/s10052-019-7399-7" } -{ "doi": "10.1111/j.1151-2916.1988.tb05935.x" } -{ "doi": "10.1111/j.1444-0938.1973.tb00721.x" } -{ "doi": "10.22201/iij.24487937e.2007.1.8033" } -{ "doi": "10.22201/iij.24487937e.2007.1.8037" } -{ "doi": "10.1007/978-3-319-74412-4_7" } -{ "doi": "10.1103/physrevlett.94.147004" } -{ "doi": "10.22201/iij.24487937e.2008.2.8049" } -{ "doi": "10.22263/2312-4156.2016.4.65" } -{ "doi": "10.1109/iccet.2019.8726910" } -{ "doi": "10.11646/zootaxa.3957.5.2" } -{ "doi": "10.22266/ijies2008.0331.02" } -{ "doi": "10.1121/1.3249486" } -{ "doi": "10.22201/iij.24487937e.2008.2.8054" } -{ "doi": "10.1121/1.3257203" } -{ "doi": "10.1017/cbo9780511660108.003" } -{ "doi": "10.1021/ac101453s" } -{ "doi": "10.1073/pnas.1513718112" } -{ "doi": "10.1073/pnas.1617636114" } -{ "doi": "10.1039/c4nr06069e" } -{ "doi": "10.1371/journal.pone.0248967" } -{ "doi": "10.3389/feart.2019.00134" } -{ "doi": "10.1002/cbic.202000122" } -{ "doi": "10.1515/ordo-2019-0004" } -{ "doi": "10.15621/ijphy/2019/v6i4/185417" } -{ "doi": "10.1126/sciadv.aay8299" } -{ "doi": "10.5194/bg-2018-324" } -{ "doi": "10.1126/science.1057726" } -{ "doi": "10.1088/1742-6596/405/1/012023" } -{ "doi": "10.1126/science.1215985" } -{ "doi": "10.1126/science.1216682" } -{ "doi": "10.3390/ani10050803" } -{ "doi": "10.1126/science.1216765" } -{ "doi": "10.1302/1863-2548.15.200169" } -{ "doi": "10.1093/isle/isaa158" } -{ "doi": "10.22201/iij.24487937e.2008.2.8059" } -{ "doi": "10.1126/science.41.1054.404" } -{ "doi": "10.22201/iij.24487937e.2009.3.8071" } -{ "doi": "10.1128/aem.71.4.1899-1908.2005" } -{ "doi": "10.1371/journal.pone.0248967.s001" } -{ "doi": "10.1186/s41687-020-00231-8" } -{ "doi": "10.22201/iij.24487937e.2009.3.8074" } -{ "doi": "10.22266/ijies2008.0331.04" } -{ "doi": "10.22201/iij.24487937e.2009.3.8080" } -{ "doi": "10.1289/ehp.9272" } -{ "doi": "10.22201/iij.24487937e.2010.4.8089" } -{ "doi": "10.1371/journal.pone.0248967.s002" } -{ "doi": "10.22201/iij.24487937e.2011.5.8100" } -{ "doi": "10.22201/iij.24487937e.2011.5.8110" } -{ "doi": "10.1128/aem.71.4.1959-1963.2005" } -{ "doi": "10.22201/iij.24487937e.2011.5.8118" } -{ "doi": "10.1128/cvi.05630-11" } -{ "doi": "10.22201/iij.24487937e.2012.6.8136" } -{ "doi": "10.1128/ec.00242-09" } -{ "doi": "10.1128/ec.00271-09" } -{ "doi": "10.1128/ec.4.6.991-998.2005" } -{ "doi": "10.1128/ec.4.7.1211-1220.2005" } -{ "doi": "10.1128/genomea.00146-13" } -{ "doi": "10.1371/journal.pone.0248967.s003" } -{ "doi": "10.1371/journal.pone.0248967.s004" } -{ "doi": "10.1128/genomea.00147-15" } -{ "doi": "10.1128/genomea.00150-15" } -{ "doi": "10.1183/09059180.00010101" } -{ "doi": "10.1371/journal.pone.0248967.s005" } -{ "doi": "10.19168/jyasar.801080" } -{ "doi": "10.24269/jin.v4n1.2019.pp75-87" } -{ "doi": "10.1371/journal.pone.0248967.s006" } -{ "doi": "10.22266/ijies2008.0331.06" } -{ "doi": "10.22266/ijies2008.0630.02" } -{ "doi": "10.22266/ijies2008.0630.04" } -{ "doi": "10.22201/iij.24487937e.2013.7.8150" } -{ "doi": "10.1371/journal.pone.0248967.s009" } -{ "doi": "10.1371/journal.pone.0248967.s010" } -{ "doi": "10.3390/pharmaceutics11040166" } -{ "doi": "10.1088/1742-6596/1129/1/012015" } -{ "doi": "10.22266/ijies2008.0930.02" } -{ "doi": "10.22266/ijies2008.1231.02" } -{ "doi": "10.1371/journal.pone.0248967.s012" } -{ "doi": "10.3847/1538-4357/aab366" } -{ "doi": "10.22266/ijies2009.0331.01" } -{ "doi": "10.1371/journal.pone.0248967.s013" } -{ "doi": "10.22266/ijies2009.0331.04" } -{ "doi": "10.1103/physrevd.91.024025" } -{ "doi": "10.1371/journal.pone.0248967.s015" } -{ "doi": "10.22201/iij.24487937e.2015.9.8180" } -{ "doi": "10.9790/487x-0351316" } -{ "doi": "10.7202/032878ar" } -{ "doi": "10.1104/pp.104.058719" } -{ "doi": "10.1128/jb.174.23.7791-7797.1992" } -{ "doi": "10.22201/iij.24487937e.2015.9.8182" } -{ "doi": "10.22201/iij.24487937e.2017.11.11079" } -{ "doi": "10.22201/iij.24487937e.2018.12.12443" } -{ "doi": "10.1093/oso/9780198814153.003.0012" } -{ "doi": "10.22201/iij.9683695272e.2016" } -{ "doi": "10.11646/zootaxa.3820.1.1" } -{ "doi": "10.1046/j.1464-410x.1998.00683.x" } -{ "doi": "10.1002/adom.201801816" } -{ "doi": "10.1128/jb.182.2.546-550.2000" } -{ "doi": "10.2478/fcds-2019-0016" } -{ "doi": "10.1128/jb.183.1.328-335.2001" } -{ "doi": "10.22201/iisue.20072872e.2010.1.13" } -{ "doi": "10.22201/iisue.20072872e.2010.2.7" } -{ "doi": "10.22201/iisue.20072872e.2011.5.47" } -{ "doi": "10.22201/iisue.20072872e.2012.7.68" } -{ "doi": "10.22201/iisue.20072872e.2016.20.199" } -{ "doi": "10.1128/jcm.01048-14" } -{ "doi": "10.1016/j.enpol.2016.03.017" } -{ "doi": "10.22201/iisue.20072872e.2016.20.203" } -{ "doi": "10.1051/0004-6361/201118622" } -{ "doi": "10.1016/j.infsof.2020.106318" } -{ "doi": "10.22201/iisue.20072872e.2016.20.207" } -{ "doi": "10.1002/anie.201609690" } -{ "doi": "10.1083/jcb.200711167" } -{ "doi": "10.22201/iisue.20072872e.2017.21.217" } -{ "doi": "10.22586/csp.v52i3.11064" } -{ "doi": "10.1007/978-3-662-45803-7_14" } -{ "doi": "10.1128/jcm.01056-18" } -{ "doi": "10.2139/ssrn.1067241" } -{ "doi": "10.1183/23120541.00578-2020" } -{ "doi": "10.22201/iisue.20072872e.2017.23.3010" } -{ "doi": "10.1190/1.2944165" } -{ "doi": "10.22201/iisue.20072872e.2017.23.3013" } -{ "doi": "10.1130/g22421.1" } -{ "doi": "10.1080/09518398.2015.1077402" } -{ "doi": "10.1128/jcm.01065-08" } -{ "doi": "10.2307/590742" } -{ "doi": "10.1128/jcm.01176-10" } -{ "doi": "10.22201/iisue.20072872e.2018.25.282" } -{ "doi": "10.1002/ejoc.202000503" } -{ "doi": "10.1128/jcm.01196-07" } -{ "doi": "10.1371/journal.pone.0218033" } -{ "doi": "10.1161/jaha.118.009169" } -{ "doi": "10.1103/physreve.89.052136" } -{ "doi": "10.22201/iisue.20072872e.2018.25.285" } -{ "doi": "10.1537/ase1911.55.370" } -{ "doi": "10.3945/ajcn.2010.28674j" } -{ "doi": "10.3390/en9070490" } -{ "doi": "10.22201/iisue.20072872e.2018.26.298" } -{ "doi": "10.22201/iisue.20072872e.2018.26.300" } -{ "doi": "10.26444/ms/133102" } -{ "doi": "10.5114/dr.2014.45121" } -{ "doi": "10.1186/s13148-020-0813-z" } -{ "doi": "10.5552/nms.2020.2" } -{ "doi": "10.1007/s00421-020-04415-4" } -{ "doi": "10.1128/aac.02143-15" } -{ "doi": "10.1177/0967010616671642" } -{ "doi": "10.1021/acs.jpcc.6b03237.s001" } -{ "doi": "10.22201/iisue.20072872e.2019.27.341" } -{ "doi": "10.22201/iisue.20072872e.2019.27.345" } -{ "doi": "10.22201/iisue.20072872e.2019.27.347" } -{ "doi": "10.22201/iisue.20072872e.2019.28.430" } -{ "doi": "10.22201/iisue.20072872e.2019.28.432" } -{ "doi": "10.1016/j.ijrobp.2016.08.015" } -{ "doi": "10.1038/srep30155" } -{ "doi": "10.1590/0037-8682-0259-2017" } -{ "doi": "10.20868/upm.thesis.43487" } -{ "doi": "10.1071/wf13086" } -{ "doi": "10.21037/cdt.2018.04.04" } -{ "doi": "10.21608/jfabsu.2020.92406" } -{ "doi": "10.1890/09-1172.1" } -{ "doi": "10.1057/978-1-137-40808-2_8" } -{ "doi": "10.1002/ccr3.260" } -{ "doi": "10.1038/ncomms13916" } -{ "doi": "10.3892/ijo.2016.3499" } -{ "doi": "10.29327/212070.7.2-20" } -{ "doi": "10.22201/ppela.24487988e.2016.6.58438" } -{ "doi": "10.3389/fpsyt.2019.00125" } -{ "doi": "10.22201/ppela.24487988e.2018.10.67370" } -{ "doi": "10.1038/s41377-020-0288-x" } -{ "doi": "10.22201/ppela.24487988e.2018.9.64764" } -{ "doi": "10.1123/jpah.2012-0238" } -{ "doi": "10.1039/b612671e" } -{ "doi": "10.1016/j.biochi.2019.07.019" } -{ "doi": "10.22202/bc.2017.v3i1.2723" } -{ "doi": "10.1016/j.anplas.2019.01.007" } -{ "doi": "10.1126/scisignal.aae0374" } -{ "doi": "10.22202/bc.2017.v3i2.2765" } -{ "doi": "10.22202/bc.2018.v4i2.3016" } -{ "doi": "10.1128/aac.37.10.2132" } -{ "doi": "10.22202/js.v4i1.1870" } -{ "doi": "10.1590/s1415-43662006000300025" } -{ "doi": "10.21273/hortsci.36.6.1022" } -{ "doi": "10.4000/studifrancesi.30573" } -{ "doi": "10.3390/f9100630" } -{ "doi": "10.22203/ecm.v001a03" } -{ "doi": "10.1039/c5sc04920b" } -{ "doi": "10.31899/rh6.1011" } -{ "doi": "10.4000/studifrancesi.32912" } -{ "doi": "10.1155/2016/8713924" } -{ "doi": "10.1017/jpr.2019.9" } -{ "doi": "10.22203/ecm.v001a07" } -{ "doi": "10.3762/bjnano.6.68" } -{ "doi": "10.4000/studifrancesi.33022" } -{ "doi": "10.4000/studifrancesi.30033" } -{ "doi": "10.4000/books.cths.5008" } -{ "doi": "10.4000/conflits.399" } -{ "doi": "10.1016/j.fusengdes.2019.02.050" } -{ "doi": "10.22203/ecm.v002a02" } -{ "doi": "10.4000/laboreal.14098" } -{ "doi": "10.1128/aac.37.2.218" } -{ "doi": "10.1021/es9914083" } -{ "doi": "10.1086/599707" } -{ "doi": "10.22203/ecm.v002a06" } -{ "doi": "10.1038/ncomms3479" } -{ "doi": "10.4236/ajcm.2019.92002" } -{ "doi": "10.22203/ecm.v003a02" } -{ "doi": "10.1109/iccvw.2015.79" } -{ "doi": "10.5040/9781350052437-0338" } -{ "doi": "10.1177/1368431008099643" } -{ "doi": "10.1128/aac.37.2.371" } -{ "doi": "10.1128/aac.37.5.1025" } -{ "doi": "10.1109/tpwrd.2019.2955007" } -{ "doi": "10.22203/ecm.v003a04" } -{ "doi": "10.4000/studifrancesi.32967" } -{ "doi": "10.22203/ecm.v004a02" } -{ "doi": "10.1093/eurpub/ckq097" } -{ "doi": "10.1029/2000pa000528" } -{ "doi": "10.1016/j.rchic.2016.10.012" } -{ "doi": "10.3233/jpd-150712" } -{ "doi": "10.22266/ijies2009.0630.02" } -{ "doi": "10.1136/bmj.2.1766.994" } -{ "doi": "10.1136/thx.47.5.393" } -{ "doi": "10.22203/ecm.v004a05" } -{ "doi": "10.22203/ecm.v005a05" } -{ "doi": "10.1142/s0218216503002585" } -{ "doi": "10.22203/ecm.v006a04" } -{ "doi": "10.22203/ecm.v006a06" } -{ "doi": "10.22203/ecm.v007a02" } -{ "doi": "10.1103/physrevc.78.045205" } -{ "doi": "10.22203/ecm.v007a04" } -{ "doi": "10.1128/genomea.00346-16" } -{ "doi": "10.1128/iai.00340-08" } -{ "doi": "10.22203/ecm.v007a05" } -{ "doi": "10.22203/ecm.v008a02" } -{ "doi": "10.22203/ecm.v008a04" } -{ "doi": "10.1101/lm.810208" } -{ "doi": "10.3389/fnhum.2016.00585" } -{ "doi": "10.1194/jlr.m300431-jlr200" } -{ "doi": "10.1128/iai.58.5.1141-1145.1990" } -{ "doi": "10.1103/physrevd.78.123513" } -{ "doi": "10.22260/isarc2016/0046" } -{ "doi": "10.1108/jd-10-2017-0139" } -{ "doi": "10.1002/ece3.4459" } -{ "doi": "10.4314/wsa.v39i2.14" } -{ "doi": "10.22260/isarc2016/0049" } -{ "doi": "10.1073/pnas.1001673107" } -{ "doi": "10.1128/jb.175.7.2060-2066.1993" } -{ "doi": "10.22260/isarc2016/0051" } -{ "doi": "10.1128/jcm.01236-14" } -{ "doi": "10.1128/jcm.01237-15" } -{ "doi": "10.22266/ijies2009.0930.01" } -{ "doi": "10.1120/jacmp.v5i1.1944" } -{ "doi": "10.1120/jacmp.v6i2.2083" } -{ "doi": "10.1128/jcm.01248-07" } -{ "doi": "10.2174/1874120701509010256" } -{ "doi": "10.1128/jcm.01855-09" } -{ "doi": "10.22260/isarc2016/0055" } -{ "doi": "10.1128/jcm.02483-05" } -{ "doi": "10.1128/jcm.02489-06" } -{ "doi": "10.22260/isarc2016/0056" } -{ "doi": "10.1021/cm901772q" } -{ "doi": "10.1128/jcm.06133-11" } -{ "doi": "10.22260/isarc2016/0065" } -{ "doi": "10.1128/jcm.37.5.1573-1574.1999" } -{ "doi": "10.1128/jvi.00715-09" } -{ "doi": "10.1057/9780230524071_4" } -{ "doi": "10.1128/jvi.00743-08" } -{ "doi": "10.22260/isarc2016/0068" } -{ "doi": "10.1128/jvi.00746-16" } -{ "doi": "10.4067/s0718-22442013000100006" } -{ "doi": "10.1128/jvi.02116-08" } -{ "doi": "10.1128/mcb.1.9.797" } -{ "doi": "10.22260/isarc2016/0072" } -{ "doi": "10.22260/isarc2016/0078" } -{ "doi": "10.2169/internalmedicine.56.8300" } -{ "doi": "10.1109/ms.2017.3" } -{ "doi": "10.22266/ijies2009.1231.04" } -{ "doi": "10.1111/j.1365-2028.2006.00657.x" } -{ "doi": "10.22260/isarc2016/0085" } -{ "doi": "10.1080/00036846.2016.1158917" } -{ "doi": "10.22260/isarc2016/0088" } -{ "doi": "10.22260/isarc2016/0089" } -{ "doi": "10.4000/studifrancesi.30483" } -{ "doi": "10.22260/isarc2016/0092" } -{ "doi": "10.22260/isarc2016/0099" } -{ "doi": "10.4000/studifrancesi.30162" } -{ "doi": "10.22260/isarc2016/0101" } -{ "doi": "10.1177/1471301214532111" } -{ "doi": "10.22260/isarc2016/0104" } -{ "doi": "10.3390/math9010055" } -{ "doi": "10.4000/studifrancesi.29938" } -{ "doi": "10.1371/journal.pone.0151514" } -{ "doi": "10.4000/studifrancesi.30222" } -{ "doi": "10.1158/1078-0432.ccr-07-1124" } -{ "doi": "10.22266/ijies2010.0630.01" } -{ "doi": "10.22266/ijies2010.0630.03" } -{ "doi": "10.1051/0004-6361:20078318" } -{ "doi": "10.22266/ijies2010.0930.01" } -{ "doi": "10.15452/studiagermanistica.2020.27.0005" } -{ "doi": "10.22266/ijies2010.0930.02" } -{ "doi": "10.3944/aott.2009.412" } -{ "doi": "10.1002/edm2.235" } -{ "doi": "10.1093/cid/cit360" } -{ "doi": "10.22266/ijies2010.0930.06" } -{ "doi": "10.22266/ijies2011.0331.03" } -{ "doi": "10.30631/al-risalah.v13i02.410" } -{ "doi": "10.1093/mnras/stu830" } -{ "doi": "10.17265/2162-5263/2015.12.001" } -{ "doi": "10.22266/ijies2011.0630.02" } -{ "doi": "10.1103/physreva.97.032305" } -{ "doi": "10.7202/1022886ar" } -{ "doi": "10.22266/ijies2011.0630.06" } -{ "doi": "10.1016/j.acra.2021.01.002" } -{ "doi": "10.1016/j.jhealeco.2015.08.001" } -{ "doi": "10.22266/ijies2011.0930.02" } -{ "doi": "10.22266/ijies2011.1231.04" } -{ "doi": "10.1002/anie.201804068" } -{ "doi": "10.2139/ssrn.2722712" } -{ "doi": "10.2134/agronj2008.0182s" } -{ "doi": "10.22260/isarc2016/0107" } -{ "doi": "10.3390/ijgi8080358" } -{ "doi": "10.1128/genomea.01234-15" } -{ "doi": "10.22260/isarc2016/0113" } -{ "doi": "10.1128/iai.01685-06" } -{ "doi": "10.22260/isarc2016/0115" } -{ "doi": "10.4000/studifrancesi.30118" } -{ "doi": "10.4000/studifrancesi.30307" } -{ "doi": "10.1128/iai.69.6.3800-3808.2001" } -{ "doi": "10.1128/iai.69.6.3860-3868.2001" } -{ "doi": "10.1128/iai.69.6.3947-3953.2001" } -{ "doi": "10.1042/bj20060705" } -{ "doi": "10.1128/iai.69.6.4007-4018.2001" } -{ "doi": "10.1128/iai.70.12.6592-6596.2002" } -{ "doi": "10.2215/cjn.01840506" } -{ "doi": "10.1155/2009/385757" } -{ "doi": "10.2215/cjn.01870309" } -{ "doi": "10.1080/09639489.2017.1375649" } -{ "doi": "10.2215/cjn.01880310" } -{ "doi": "10.22271/ortho.2017.v3.i3a.07" } -{ "doi": "10.2215/cjn.01880606" } -{ "doi": "10.1192/bjp.157.6.853" } -{ "doi": "10.1103/physrevd.75.037301" } -{ "doi": "10.2215/cjn.01891105" } -{ "doi": "10.2215/cjn.01900309" } -{ "doi": "10.1128/jb.174.8.2431-2439.1992" } -{ "doi": "10.22271/ortho.2017.v3.i3a.08" } -{ "doi": "10.2215/cjn.01950606" } -{ "doi": "10.21622/resd.2017.03.2.185" } -{ "doi": "10.22178/pos.41-6" } -{ "doi": "10.1136/adc.2002.021154" } -{ "doi": "10.1007/s00414-016-1432-2" } -{ "doi": "10.1007/978-3-531-92886-9_4" } -{ "doi": "10.2215/cjn.01961205" } -{ "doi": "10.2215/cjn.01970309" } -{ "doi": "10.2215/cjn.01970507" } -{ "doi": "10.1103/physrevd.89.102001" } -{ "doi": "10.2215/cjn.01990213" } -{ "doi": "10.2215/cjn.02000507" } -{ "doi": "10.1364/josaa.20.001371" } -{ "doi": "10.3390/12030607" } -{ "doi": "10.1371/journal.pone.0012585" } -{ "doi": "10.1007/978-3-319-17043-5_3" } -{ "doi": "10.1371/journal.pone.0012589" } -{ "doi": "10.2215/cjn.02010606" } -{ "doi": "10.2215/cjn.02020309" } -{ "doi": "10.2215/cjn.02021205" } -{ "doi": "10.1371/journal.pone.0012836" } -{ "doi": "10.1371/journal.pone.0041427" } -{ "doi": "10.2215/cjn.02030606" } -{ "doi": "10.1371/journal.pone.0015106" } -{ "doi": "10.2215/cjn.02040408" } -{ "doi": "10.2215/cjn.02090212" } -{ "doi": "10.1016/j.mednuc.2007.03.011" } -{ "doi": "10.1016/j.tibs.2018.09.003" } -{ "doi": "10.2215/cjn.02120508" } -{ "doi": "10.1016/s0008-6363(99)00337-5" } -{ "doi": "10.2215/cjn.02160311" } -{ "doi": "10.1115/detc2006-99483" } -{ "doi": "10.2215/cjn.02180606" } -{ "doi": "10.1161/01.cir.15.4.481" } -{ "doi": "10.1161/01.cir.47.4.681" } -{ "doi": "10.1094/pdis.2004.88.1.63" } -{ "doi": "10.1182/blood-2013-08-521419" } -{ "doi": "10.2215/cjn.02190216" } -{ "doi": "10.1371/journal.pone.0024817" } -{ "doi": "10.1016/j.automatica.2011.03.001" } -{ "doi": "10.2215/cjn.02190314" } -{ "doi": "10.1051/0004-6361/201424369" } -{ "doi": "10.1371/journal.pone.0041353" } -{ "doi": "10.1073/pnas.1019446108" } -{ "doi": "10.1537/ase1887.26.45" } -{ "doi": "10.3390/12071367" } -{ "doi": "10.1093/toxsci/kfg090" } -{ "doi": "10.2215/cjn.02211205" } -{ "doi": "10.2215/cjn.02230409" } -{ "doi": "10.2215/cjn.02280215" } -{ "doi": "10.1107/s1600536804007081" } -{ "doi": "10.2140/pjm.1975.58.1" } -{ "doi": "10.3406/cea.1992.2098" } -{ "doi": "10.1103/physrevd.90.105020" } -{ "doi": "10.1051/0004-6361/201424382" } -{ "doi": "10.1073/pnas.90.2.750" } -{ "doi": "10.1080/00461520.2017.1333430" } -{ "doi": "10.1103/physrevd.94.083505" } -{ "doi": "10.1051/0004-6361/201424389" } -{ "doi": "10.1051/0004-6361/201630325" } -{ "doi": "10.1051/0004-6361/201731426" } -{ "doi": "10.1093/jigpal/jzi040" } -{ "doi": "10.1051/0004-6361/201833507" } -{ "doi": "10.13075/ijomeh.1896.01261" } -{ "doi": "10.1016/j.jmmm.2014.11.070" } -{ "doi": "10.1038/nphys4100" } -{ "doi": "10.1371/journal.pone.0041384" } -{ "doi": "10.3390/antiox8080243" } -{ "doi": "10.2215/cjn.02291205" } -{ "doi": "10.2215/cjn.02300606" } -{ "doi": "10.1128/aac.44.1.173-177.2000" } -{ "doi": "10.1128/aac.47.9.2951-2957.2003" } -{ "doi": "10.2215/cjn.02311205" } -{ "doi": "10.1128/aac.48.3.843-851.2004" } -{ "doi": "10.1128/aac.50.2.810-812.2006" } -{ "doi": "10.1128/aem.00862-16" } -{ "doi": "10.1128/aem.01394-08" } -{ "doi": "10.2215/cjn.02330315" } -{ "doi": "10.2215/cjn.02340310" } -{ "doi": "10.1182/asheducation-2014.1.66" } -{ "doi": "10.2215/cjn.02340607" } -{ "doi": "10.2215/cjn.02360311" } -{ "doi": "10.2172/824271" } -{ "doi": "10.1016/j.fertnstert.2018.11.022" } -{ "doi": "10.2215/cjn.02360409" } -{ "doi": "10.2172/824272" } -{ "doi": "10.2215/cjn.02370508" } -{ "doi": "10.1128/aem.66.1.10-14.2000" } -{ "doi": "10.1128/aem.66.1.15-22.2000" } -{ "doi": "10.2215/cjn.02400311" } -{ "doi": "10.20546/ijcmas.2019.807.209" } -{ "doi": "10.1128/aem.66.1.154-162.2000" } -{ "doi": "10.1128/aem.66.1.310-319.2000" } -{ "doi": "10.2991/icemct-15.2015.373" } -{ "doi": "10.1128/aem.66.11.4790-4797.2000" } -{ "doi": "10.1128/aem.66.11.4940-4944.2000" } -{ "doi": "10.1128/aem.66.11.4962-4971.2000" } -{ "doi": "10.31533/pubvet.v13n7a364.1-10" } -{ "doi": "10.22271/ortho.2017.v3.i3a.10" } -{ "doi": "10.2215/cjn.02490311" } -{ "doi": "10.2215/cjn.02490508" } -{ "doi": "10.2215/cjn.02500607" } -{ "doi": "10.1128/aem.66.12.5107-5109.2000" } -{ "doi": "10.2215/cjn.02510311" } -{ "doi": "10.2215/cjn.02550607" } -{ "doi": "10.14720/aas.2018.111.2.23" } -{ "doi": "10.3126/jcmsn.v11i4.14322" } -{ "doi": "10.2215/cjn.02570315" } -{ "doi": "10.34105/j.kmel.2010.02.020" } -{ "doi": "10.1128/jb.184.1.76-81.2002" } -{ "doi": "10.1128/jb.184.16.4529-4535.2002" } -{ "doi": "10.1128/jb.184.16.4536-4543.2002" } -{ "doi": "10.3998/ark.5550190.0010.302" } -{ "doi": "10.1128/aem.66.12.5161-5166.2000" } -{ "doi": "10.1128/jb.184.16.4555-4572.2002" } -{ "doi": "10.1371/journal.pone.0077504" } -{ "doi": "10.3390/ani10081341" } -{ "doi": "10.2215/cjn.02580607" } -{ "doi": "10.2215/cjn.02590508" } -{ "doi": "10.2215/cjn.02620707" } -{ "doi": "10.2215/cjn.02630608" } -{ "doi": "10.1371/journal.pone.0077660" } -{ "doi": "10.5915/24-1-15451" } -{ "doi": "10.36026/rpgeo.v4i2.2520" } -{ "doi": "10.2172/892378" } -{ "doi": "10.1128/aem.66.12.5301-5305.2000" } -{ "doi": "10.4067/s0034-98872016000900005" } -{ "doi": "10.5194/acp-17-12659-2017" } -{ "doi": "10.5125/jkaoms.2013.39.4.161" } -{ "doi": "10.1007/978-94-6300-208-0_9" } -{ "doi": "10.1177/026975800901600101" } -{ "doi": "10.1007/978-981-13-6504-1_114" } -{ "doi": "10.1128/jb.184.17.4733-4738.2002" } -{ "doi": "10.1128/aem.66.12.5437-5447.2000" } -{ "doi": "10.1128/aem.66.12.5518-5520.2000" } -{ "doi": "10.1128/aem.71.7.3935-3941.2005" } -{ "doi": "10.1128/ec.4.2.262-273.2005" } -{ "doi": "10.1128/jb.184.17.4757-4766.2002" } -{ "doi": "10.1128/ec.4.5.931-936.2005" } -{ "doi": "10.1128/genomea.00109-17" } -{ "doi": "10.1128/genomea.00379-17" } -{ "doi": "10.1128/genomea.00380-13" } -{ "doi": "10.1083/jcb.102.6.2023" } -{ "doi": "10.1128/genomea.00385-18" } -{ "doi": "10.1128/genomea.00399-13" } -{ "doi": "10.1128/genomea.00404-13" } -{ "doi": "10.1128/genomea.00406-15" } -{ "doi": "10.1128/genomea.00408-13" } -{ "doi": "10.2215/cjn.02640310" } -{ "doi": "10.15700/saje.v31n3a540" } -{ "doi": "10.1128/genomea.00617-17" } -{ "doi": "10.1128/genomea.00629-15" } -{ "doi": "10.2215/cjn.02650707" } -{ "doi": "10.1128/genomea.01177-16" } -{ "doi": "10.1128/iai.74.4.2102-2114.2006" } -{ "doi": "10.1128/iai.74.4.2233-2244.2006" } -{ "doi": "10.2215/cjn.02660706" } -{ "doi": "10.1128/jb.184.19.5502-5507.2002" } -{ "doi": "10.3390/s140610124" } -{ "doi": "10.1128/jb.185.15.4620-4625.2003" } -{ "doi": "10.1128/jb.185.16.4948-4955.2003" } -{ "doi": "10.1128/jb.06465-11" } -{ "doi": "10.1128/jb.06490-11" } -{ "doi": "10.1023/b:plso.0000035563.71887.15" } -{ "doi": "10.21236/ada200288" } -{ "doi": "10.14720/aas.2018.111.3.05" } -{ "doi": "10.1128/jb.187.11.3884-3888.2005" } -{ "doi": "10.1128/jb.188.6.2096-2105.2006" } -{ "doi": "10.1007/978-3-662-53120-4_300593" } -{ "doi": "10.1057/9781137032225_8" } -{ "doi": "10.1073/pnas.40.12.1155" } -{ "doi": "10.2169/internalmedicine.40.920" } -{ "doi": "10.1128/jb.182.18.5046-5051.2000" } -{ "doi": "10.1063/1.4920959" } -{ "doi": "10.2215/cjn.02740806" } -{ "doi": "10.2215/cjn.02760310" } -{ "doi": "10.15381/is.v22i40.15929" } -{ "doi": "10.1128/jvi.02060-08" } -{ "doi": "10.5937/tekstind1901025r" } -{ "doi": "10.1093/geronb/gbs161" } -{ "doi": "10.1177/0278364906068393" } -{ "doi": "10.1038/519266a" } -{ "doi": "10.1128/jvi.65.2.981-984.1991" } -{ "doi": "10.1134/s0001434618070179" } -{ "doi": "10.1155/2018/8474389" } -{ "doi": "10.22533/at.ed.4141916012" } -{ "doi": "10.15740/has/ijpp/9.2/514-519" } -{ "doi": "10.2215/cjn.02760315" } -{ "doi": "10.2215/cjn.02770409" } -{ "doi": "10.1063/1.4946808" } -{ "doi": "10.1007/s11227-021-03665-z" } -{ "doi": "10.4314/jae.v17i2.23" } -{ "doi": "10.3390/cryst8060243" } -{ "doi": "10.1002/chem.201901912" } -{ "doi": "10.1016/j.eurpsy.2017.01.002" } -{ "doi": "10.1016/j.jcss.2021.03.005" } -{ "doi": "10.18805/ijar.10276" } -{ "doi": "10.1371/journal.pgen.1007046" } -{ "doi": "10.2215/cjn.02780806" } -{ "doi": "10.2215/cjn.02790608" } -{ "doi": "10.1016/j.crwh.2018.e00060" } -{ "doi": "10.1128/jvi.01461-13" } -{ "doi": "10.1128/jvi.01461-17" } -{ "doi": "10.5897/ajb2014.14109" } -{ "doi": "10.1002/evl3.49" } -{ "doi": "10.35248/2684-1614.18.3.118" } -{ "doi": "10.4067/s0718-48162010000100009" } -{ "doi": "10.1016/j.jfa.2021.109027" } -{ "doi": "10.2215/cjn.02810707" } -{ "doi": "10.1139/cjfas-2018-0207" } -{ "doi": "10.1038/nmicrobiol.2016.250" } -{ "doi": "10.1002/ppp3.10159" } -{ "doi": "10.1021/acs.jpcc.6b05794" } -{ "doi": "10.1017/cbo9781139481373" } -{ "doi": "10.1002/tie.21672" } -{ "doi": "10.1051/alr:2008038" } -{ "doi": "10.2215/cjn.02820311" } -{ "doi": "10.3390/e19030087" } -{ "doi": "10.1371/journal.pone.0021863" } -{ "doi": "10.1007/bf02441950" } -{ "doi": "10.1371/journal.pone.0023339" } -{ "doi": "10.2215/cjn.02830409" } -{ "doi": "10.1155/2016/4364761" } -{ "doi": "10.1371/journal.pone.0032936" } -{ "doi": "10.1103/physrevlett.119.058003" } -{ "doi": "10.1590/1414-431x20165734" } -{ "doi": "10.3390/toxins11040203" } -{ "doi": "10.2215/cjn.02840314" } -{ "doi": "10.1016/j.jfa.2021.109034" } -{ "doi": "10.4103/0974-9233.97939" } -{ "doi": "10.4209/aaqr.2013.06.0208" } -{ "doi": "10.2215/cjn.02840806" } -{ "doi": "10.7188/bvsz.2013.89.4.4" } -{ "doi": "10.7717/peerj.6064" } -{ "doi": "10.2215/cjn.02860409" } -{ "doi": "10.2215/cjn.02870311" } -{ "doi": "10.1128/jcm.00710-16" } -{ "doi": "10.1128/jcm.00715-15" } -{ "doi": "10.1128/jcm.00717-15" } -{ "doi": "10.1007/s11740-021-01023-9" } -{ "doi": "10.1109/joe.2020.2984293" } -{ "doi": "10.1155/2016/5130346" } -{ "doi": "10.1128/jcm.39.8.2860-2863.2001" } -{ "doi": "10.1038/129672b0" } -{ "doi": "10.2215/cjn.02880312" } -{ "doi": "10.1128/jcm.39.9.3040-3046.2001" } -{ "doi": "10.1128/jcm.39.9.3373-3375.2001" } -{ "doi": "10.1128/jcm.43.1.277-283.2005" } -{ "doi": "10.1128/jcm.43.1.356-362.2005" } -{ "doi": "10.1371/journal.pone.0050129" } -{ "doi": "10.1371/journal.pcbi.1006188" } -{ "doi": "10.1128/jcm.43.12.6139-6143.2005" } -{ "doi": "10.1128/jvi.01251-15" } -{ "doi": "10.1038/s41467-018-06777-y" } -{ "doi": "10.1111/1365-2435.13215" } -{ "doi": "10.1021/acssynbio.6b00189" } -{ "doi": "10.2215/cjn.02890806" } -{ "doi": "10.1128/mcb.25.12.5171-5182.2005" } -{ "doi": "10.1080/037454809494575" } -{ "doi": "10.9756/sijifbm/v3i2/03040100201" } -{ "doi": "10.1155/2018/5287138" } -{ "doi": "10.2215/cjn.02900313" } -{ "doi": "10.2215/cjn.01760215" } -{ "doi": "10.2215/cjn.01760506" } -{ "doi": "10.1130/gsatg223a.1" } -{ "doi": "10.2215/cjn.01770214" } -{ "doi": "10.1130/gsatg278gw.1" } -{ "doi": "10.2172/7039964" } -{ "doi": "10.1130/gsatg291a.1" } -{ "doi": "10.2215/cjn.02920608" } -{ "doi": "10.2215/cjn.01790506" } -{ "doi": "10.18632/oncotarget.2626" } -{ "doi": "10.1017/cbo9781107338364" } -{ "doi": "10.2215/cjn.02950707" } -{ "doi": "10.2215/cjn.01820309" } -{ "doi": "10.2215/cjn.01820407" } -{ "doi": "10.2215/cjn.02960906" } -{ "doi": "10.1088/2058-7058/28/8/35" } -{ "doi": "10.1007/978-3-030-20024-4" } -{ "doi": "10.2172/7118817" } -{ "doi": "10.22201/fi.25940732e.2003.04n3.010" } -{ "doi": "10.21500/16578031.2172" } -{ "doi": "10.2215/cjn.01830309" } -{ "doi": "10.2172/7218931" } -{ "doi": "10.22201/fi.25940732e.2006.07n2.008" } -{ "doi": "10.1130/gsatg373gw.1" } -{ "doi": "10.2172/750108" } -{ "doi": "10.13005/bbra/1757" } -{ "doi": "10.2215/cjn.01841105" } -{ "doi": "10.2215/cjn.01860309" } -{ "doi": "10.5614/bull.geol.2021.5.1.1" } -{ "doi": "10.2215/cjn.01880216" } -{ "doi": "10.1038/aja.2013.76" } -{ "doi": "10.1051/matecconf/201711403011" } -{ "doi": "10.1371/journal.pmed.1001036" } -{ "doi": "10.22201/fi.25940732e.2006.07n3.013" } -{ "doi": "10.1130/gsatg392gw.1" } -{ "doi": "10.22201/fi.25940732e.2006.07n4.017" } -{ "doi": "10.2215/cjn.01881105" } -{ "doi": "10.32635/2176-9745.rbc.2017v63n1.188" } -{ "doi": "10.1088/0264-9381/20/11/330" } -{ "doi": "10.1001/jama.1899.02450730063034" } -{ "doi": "10.1175/bams_1001_cover2" } -{ "doi": "10.1128/jcm.41.11.5337-5339.2003" } -{ "doi": "10.1371/journal.pntd.0002094" } -{ "doi": "10.22271/ortho.2017.v3.i3e.56" } -{ "doi": "10.1371/journal.pone.0009444" } -{ "doi": "10.1073/pnas.12.3.162" } -{ "doi": "10.1080/09537287.2018.1555341" } -{ "doi": "10.1111/gove.12046" } -{ "doi": "10.2215/cjn.01890408" } -{ "doi": "10.1055/s-0038-1639376" } -{ "doi": "10.18502/kss.v3i13.4265" } -{ "doi": "10.22201/fi.25940732e.2007.08n1.001" } -{ "doi": "10.2215/cjn.01910213" } -{ "doi": "10.1371/journal.pone.0038316" } -{ "doi": "10.3390/en9070479" } -{ "doi": "10.22201/fi.25940732e.2007.08n3.014" } -{ "doi": "10.1002/asna.201612337" } -{ "doi": "10.1130/l1005.1" } -{ "doi": "10.1130/l1030.1" } -{ "doi": "10.1371/journal.pone.0041097" } -{ "doi": "10.2147/jbm.s52783" } -{ "doi": "10.20546/ijcmas.2017.612.078" } -{ "doi": "10.22271/ortho.2017.v3.i3f.61" } -{ "doi": "10.4414/cvm.2006.01205" } -{ "doi": "10.2307/1274161" } -{ "doi": "10.12962/j23546026.y2017i4.3076" } -{ "doi": "10.1371/journal.pone.0041119" } -{ "doi": "10.5216/ree.v15i1.16503" } -{ "doi": "10.1109/tie.2014.2331020" } -{ "doi": "10.22271/ortho.2017.v3.i3f.65" } -{ "doi": "10.1063/1.3625957" } -{ "doi": "10.1108/ijmf-07-2017-0144" } -{ "doi": "10.1161/hypertensionaha.108.117218" } -{ "doi": "10.1371/journal.pone.0041122" } -{ "doi": "10.1093/jxb/erz265" } -{ "doi": "10.2172/764628" } -{ "doi": "10.1371/journal.pone.0041136" } -{ "doi": "10.1371/journal.pone.0049162" } -{ "doi": "10.1051/0004-6361/201730693" } -{ "doi": "10.1051/0004-6361/201730700" } -{ "doi": "10.1051/0004-6361/201730701" } -{ "doi": "10.3390/f9100605" } -{ "doi": "10.2215/cjn.01960309" } -{ "doi": "10.2215/cjn.01990211" } -{ "doi": "10.22201/fi.25940732e.2007.08n4.021" } -{ "doi": "10.7181/acfs.2016.17.4.225" } -{ "doi": "10.5511/plantbiotechnology.14.0917a" } -{ "doi": "10.1371/journal.pone.0051799" } -{ "doi": "10.1371/journal.pone.0052753" } -{ "doi": "10.1371/journal.pone.0052762" } -{ "doi": "10.1051/0004-6361/201730704" } -{ "doi": "10.22201/fi.25940732e.2008.09n1.005" } -{ "doi": "10.1289/ehp.94102286" } -{ "doi": "10.2215/cjn.02000309" } -{ "doi": "10.2215/cjn.0200608" } -{ "doi": "10.1128/aac.00383-17" } -{ "doi": "10.1128/aac.00399-09" } -{ "doi": "10.1504/ijahuc.2014.064422" } -{ "doi": "10.22201/fi.25940732e.2008.09n2.012" } -{ "doi": "10.13031/2013.35254" } -{ "doi": "10.22201/fi.25940732e.2008.09n4.024" } -{ "doi": "10.2215/cjn.02020408" } -{ "doi": "10.1128/aac.02462-15" } -{ "doi": "10.2215/cjn.02030311" } -{ "doi": "10.1128/aac.02485-15" } -{ "doi": "10.1128/aac.03126-15" } -{ "doi": "10.1128/aac.48.4.1335-1343.2004" } -{ "doi": "10.1128/aem.01757-08" } -{ "doi": "10.1371/journal.pgen.1007102" } -{ "doi": "10.1128/jvi.01462-06" } -{ "doi": "10.3130/jaabe.7.233" } -{ "doi": "10.1109/apusncursinrsm.2019.8888598" } -{ "doi": "10.3346/jkms.2013.28.4.614" } -{ "doi": "10.22201/fi.25940732e.2008.09n4.026" } -{ "doi": "10.1128/jvi.01465-06" } -{ "doi": "10.1371/journal.pmed.0030006" } -{ "doi": "10.2215/cjn.02030408" } -{ "doi": "10.3390/mi8060177" } -{ "doi": "10.4314/ejesm.v5i2.6" } -{ "doi": "10.1128/aem.62.9.3304-3312.1996" } -{ "doi": "10.1089/cpb.2007.9993" } -{ "doi": "10.1093/molbev/msn137" } -{ "doi": "10.1055/s-0040-1722275" } -{ "doi": "10.1016/s0011-393x(96)80113-5" } -{ "doi": "10.1128/aem.66.11.4870-4876.2000" } -{ "doi": "10.4324/9781849772389" } -{ "doi": "10.1371/journal.pntd.0004265" } -{ "doi": "10.1074/jbc.m110.154575" } -{ "doi": "10.1371/journal.pntd.0004272" } -{ "doi": "10.1128/aem.66.11.4935-4939.2000" } -{ "doi": "10.1128/aem.66.11.4945-4953.2000" } -{ "doi": "10.22201/fi.25940732e.2009.10n2.016" } -{ "doi": "10.1128/aem.66.11.4992-4997.2000" } -{ "doi": "10.1016/j.chest.2018.12.027" } -{ "doi": "10.1128/aem.66.12.5206-5212.2000" } -{ "doi": "10.1128/aem.66.12.5406-5409.2000" } -{ "doi": "10.1128/aem.71.7.3512-3518.2005" } -{ "doi": "10.1093/mnras/stu1318" } -{ "doi": "10.1128/ec.1.3.391-400.2002" } -{ "doi": "10.1128/ec.2.1.9-18.2003" } -{ "doi": "10.1128/ec.2.3.510-520.2003" } -{ "doi": "10.1128/ec.3.2.393-405.2004" } -{ "doi": "10.18632/oncotarget.7125" } -{ "doi": "10.1128/ec.3.6.1619-1626.2004" } -{ "doi": "10.1128/ec.4.1.166-177.2005" } -{ "doi": "10.22271/tpi.2017.v6.i11l.02" } -{ "doi": "10.1128/ec.4.12.2066-2077.2005" } -{ "doi": "10.1128/ec.4.2.346-355.2005" } -{ "doi": "10.1128/ec.4.3.516-525.2005" } -{ "doi": "10.1128/ec.4.5.849-860.2005" } -{ "doi": "10.1128/genomea.00108-16" } -{ "doi": "10.1128/genomea.00379-18" } -{ "doi": "10.1128/genomea.00402-15" } -{ "doi": "10.1128/genomea.00406-17" } -{ "doi": "10.1128/genomea.00410-17" } -{ "doi": "10.1128/genomea.00620-15" } -{ "doi": "10.1021/om100149w" } -{ "doi": "10.1021/ic702471d" } -{ "doi": "10.2215/cjn.02050309" } -{ "doi": "10.1128/jvi.01470-07" } -{ "doi": "10.1128/jvi.02476-06" } -{ "doi": "10.1051/e3sconf/20130104004" } -{ "doi": "10.2514/6.2006-7734" } -{ "doi": "10.1128/jvi.03358-14" } -{ "doi": "10.22201/fi.25940732e.2009.10n3.024" } -{ "doi": "10.3389/fmars.2021.619190.s002" } -{ "doi": "10.5860/choice.191948" } -{ "doi": "10.1128/jcm.01682-06" } -{ "doi": "10.1001/archpsyc.58.8.737" } -{ "doi": "10.22201/fi.25940732e.2009.10n4.027" } -{ "doi": "10.1128/jvi.00578-15" } -{ "doi": "10.21926/obm.transplant.1901044" } -{ "doi": "10.1128/jvi.02155-09" } -{ "doi": "10.1128/jvi.65.3.1638-1643.1991" } -{ "doi": "10.3362/0262-8104.1999.021" } -{ "doi": "10.1515/rebs-2019-0080" } -{ "doi": "10.5020/23590777.14.1.141-151" } -{ "doi": "10.2215/cjn.02110508" } -{ "doi": "10.22201/fi.25940732e.2009.10n4.029" } -{ "doi": "10.1136/gut.33.6.759" } -{ "doi": "10.22201/fi.25940732e.2009.10n4.034" } -{ "doi": "10.3389/fpsyg.2019.00528" } -{ "doi": "10.1177/1555412005281819" } -{ "doi": "10.1371/journal.pone.0159550" } -{ "doi": "10.22201/fi.25940732e.2010.11n1.003" } -{ "doi": "10.1085/jgp.106.5.863" } -{ "doi": "10.2215/cjn.02180311" } -{ "doi": "10.2501/ija-30-4-641-663" } -{ "doi": "10.2215/cjn.02190312" } -{ "doi": "10.1029/2018gl079093" } -{ "doi": "10.22201/fi.25940732e.2010.11n3.029" } -{ "doi": "10.1175/2009mwr2843.1" } -{ "doi": "10.1051/0004-6361/201118429" } -{ "doi": "10.1099/00221287-98-1-147" } -{ "doi": "10.1006/ffta.2000.0291" } -{ "doi": "10.1038/leu.2014.285" } -{ "doi": "10.1073/pnas.73.3.867" } -{ "doi": "10.22201/fi.25940732e.2010.11n4.035" } -{ "doi": "10.2215/cjn.02220507" } -{ "doi": "10.2215/cjn.02230606" } -{ "doi": "10.1002/pola.10449" } -{ "doi": "10.22201/fi.25940732e.2011.12n3.027" } -{ "doi": "10.2215/cjn.02250508" } -{ "doi": "10.1371/journal.pgen.1003534" } -{ "doi": "10.1501/tipfak_0000000416" } -{ "doi": "10.1371/journal.pmed.1001664" } -{ "doi": "10.2215/cjn.02260507" } -{ "doi": "10.1002/wcm.321" } -{ "doi": "10.22201/fi.25940732e.2011.12n4.043" } -{ "doi": "10.18632/oncotarget.26699" } -{ "doi": "10.2215/cjn.02300311" } -{ "doi": "10.22201/fi.25940732e.2011.12n4.046" } -{ "doi": "10.22201/fi.25940732e.2012.13n1.007" } -{ "doi": "10.2172/782400" } -{ "doi": "10.2172/782408" } -{ "doi": "10.2215/cjn.02320311" } -{ "doi": "10.1371/journal.pone.0033056" } -{ "doi": "10.1371/journal.pone.0033067" } -{ "doi": "10.1371/journal.pone.0034420" } -{ "doi": "10.5788/25-1-1299" } -{ "doi": "10.1051/0004-6361/201423936" } -{ "doi": "10.1007/s40257-020-00504-4" } -{ "doi": "10.1007/978-3-642-16405-7_23" } -{ "doi": "10.22201/fi.25940732e.2012.13n1.012" } -{ "doi": "10.2215/cjn.02330508" } -{ "doi": "10.1016/j.mri.2010.06.028" } -{ "doi": "10.21236/ada389569" } -{ "doi": "10.1088/0305-4470/39/26/l03" } -{ "doi": "10.22201/fi.25940732e.2012.13n2.014" } -{ "doi": "10.1161/01.str.17.5.1019" } -{ "doi": "10.2215/cjn.02340409" } -{ "doi": "10.22201/fi.25940732e.2012.13n2.017" } -{ "doi": "10.1137/1.9781611972825.87" } -{ "doi": "10.2215/cjn.02340706" } -{ "doi": "10.1038/s41433-020-01221-2" } -{ "doi": "10.22201/fi.25940732e.2012.13n2.021" } -{ "doi": "10.22201/fi.25940732e.2012.13n3.029" } -{ "doi": "10.2172/7825" } -{ "doi": "10.23865/noasp.103" } -{ "doi": "10.2215/cjn.02360706" } -{ "doi": "10.2215/cjn.02400310" } -{ "doi": "10.22201/fi.25940732e.2012.13n3.034" } -{ "doi": "10.1090/tran/7688" } -{ "doi": "10.1155/1883/87478" } -{ "doi": "10.1155/2018/8614073" } -{ "doi": "10.1039/c0cc00078g" } -{ "doi": "10.2215/cjn.02400706" } -{ "doi": "10.1103/physrevd.57.6406" } -{ "doi": "10.1109/euma.2000.338815" } -{ "doi": "10.2172/782526" } -{ "doi": "10.17533/udea.redin.n76a03" } -{ "doi": "10.22201/fi.25940732e.2012.13n4.048" } -{ "doi": "10.1109/tnse.2018.2871726" } -{ "doi": "10.1371/journal.pone.0075442" } -{ "doi": "10.2215/cjn.02430607" } -{ "doi": "10.3390/80700556" } -{ "doi": "10.1017/cbo9781107415836.002" } -{ "doi": "10.1097/01.xme.0000395563.70945.52" } -{ "doi": "10.2353/ajpath.2010.090735" } -{ "doi": "10.1080/16878507.2021.1873901" } -{ "doi": "10.22201/fi.25940732e.2017.18n1.006" } -{ "doi": "10.1248/bpb.32.1500" } -{ "doi": "10.2215/cjn.02440607" } -{ "doi": "10.33582/2637-4927/1006" } -{ "doi": "10.1016/j.envpol.2007.12.019" } -{ "doi": "10.3390/admsci8030036" } -{ "doi": "10.1038/nrurol.2015.76" } -{ "doi": "10.25070/rea.v3i4.69" } -{ "doi": "10.2215/cjn.02450607" } -{ "doi": "10.1538/expanim.50.67" } -{ "doi": "10.22201/fi.25940732e.2017.18n3.025" } -{ "doi": "10.1371/journal.pone.0034706" } -{ "doi": "10.1016/s0167-6911(97)00081-9" } -{ "doi": "10.1186/s12877-019-1158-3" } -{ "doi": "10.1371/journal.pone.0051303" } -{ "doi": "10.1371/journal.pone.0060930" } -{ "doi": "10.1371/journal.pone.0060937" } -{ "doi": "10.25070/rea.v4i2.80" } -{ "doi": "10.2215/cjn.02480314" } -{ "doi": "10.22201/fi.25940732e.2017.18n3.028" } -{ "doi": "10.1371/journal.pone.0060964" } -{ "doi": "10.1371/journal.pone.0061231" } -{ "doi": "10.2215/cjn.02490409" } -{ "doi": "10.22201/fi.25940732e.2018.19n1.003" } -{ "doi": "10.1371/journal.pone.0062271" } -{ "doi": "10.31235/osf.io/p82fk" } -{ "doi": "10.2215/cjn.02510213" } -{ "doi": "10.2215/cjn.02510409" } -{ "doi": "10.1071/bi9750133" } -{ "doi": "10.3389/fonc.2021.605853" } -{ "doi": "10.1128/jb.171.12.6455-6467.1989" } -{ "doi": "10.1371/journal.pone.0063242" } -{ "doi": "10.3389/fonc.2021.636591" } -{ "doi": "10.3389/fonc.2021.646499" } -{ "doi": "10.22201/fpsi.20074719e.2012.3.234" } \ No newline at end of file diff --git a/src/update/lib/history.js b/src/update/lib/history.js index 163941b0..8e2060fb 100644 --- a/src/update/lib/history.js +++ b/src/update/lib/history.js @@ -7,6 +7,7 @@ const fs = require('fs-extra'); const readline = require('readline'); const config = require('config'); const zlib = require('zlib'); +const { format } = require('date-fns'); const logger = require('./logger'); const unpaywallEnrichedMapping = require('../mapping/unpaywall.json'); @@ -325,201 +326,115 @@ async function insertHistoryDataUnpaywall(insertConfig) { return true; } -/** - * Resets the unpaywall index according to a date and deletes the data in the history. - * @param {string} startDate - The date you wish to return to - */ -async function rollBack(startDate) { - // STEP 2 +async function step1(startDate) { + logger.debug('----------------------------------'); + logger.debug('STEP 1'); + logger.debug(`startDate: ${format(new Date(startDate), 'yyyy-MM-dd/hh:mm:ss')}`); + logger.debug(`Should DELETE greater or equal than ${format(new Date(startDate), 'yyyy-MM-dd/hh:mm:ss')}`); + const toRecentDataInHistoryBulk = []; + const toRecentDataInHistory = await searchWithRange(startDate, 'updated', 'gte', 'unpaywall_history'); + + toRecentDataInHistory.forEach((data) => { + toRecentDataInHistoryBulk.push({ delete: { _index: 'unpaywall_history', _id: data._id } }); + logger.debug(`HISTORY: DELETE: doi: ${data._source.doi}, genre: ${data._source.genre}, updated: ${data._source.updated}`); + }); + + await bulk(toRecentDataInHistoryBulk, true); + logger.debug(`DELETE ${toRecentDataInHistoryBulk.length} lines`); + logger.debug('UNPAYWALL: REFRESH'); + await refreshIndex('unpaywall_enriched'); +} + +async function step2(startDate) { + logger.debug('----------------------------------'); + logger.debug('STEP 2'); + logger.debug(`startDate: ${format(new Date(startDate), 'yyyy-MM-dd/hh:mm:ss')}`); + logger.debug(`Should UPDATE data less than ${format(new Date(startDate), 'yyyy-MM-dd/hh:mm:ss')}`); const nearestDataBulk = []; const nearestDataDeleteBulk = []; - const oldData = await searchWithRange(startDate, 'endValidity', 'lt', 'unpaywall_history'); + const oldData = await searchWithRange(startDate, 'updated', 'lt', 'unpaywall_history'); const listOfDoi = oldData.map((e) => e._source.doi); const listOfDoiFiltered = new Set(listOfDoi); for (const doi of listOfDoiFiltered) { - console.log(doi); const docs = oldData.filter((e) => doi === e._source.doi); let nearestData; - let nearestID; docs.forEach((doc) => { - const unpaywallData = doc._source; if (!nearestData) { - nearestID = doc._id; - nearestData = unpaywallData; + nearestData = doc; } - const date1 = new Date(startDate) - new Date(unpaywallData.endValidity); - const date2 = new Date(startDate) - new Date(nearestData.endValidity); + const date1 = new Date(startDate) - new Date(doc._source.updated); + const date2 = new Date(startDate) - new Date(nearestData._source.updated); if (date1 < date2) { - nearestID = doc._id; - nearestData = unpaywallData; + nearestData = doc; } }); - const oldDoc = await getDataByListOfDOI([nearestData.doi], 'unpaywall_enriched'); + const oldDoc = await getDataByListOfDOI([nearestData._source.doi], 'unpaywall_enriched'); + logger.info(`oldData: ${format(new Date(oldDoc[0].updated), 'yyyy-MM-dd/hh:mm:ss')} < startDate: ${format(new Date(startDate), 'yyyy-MM-dd/hh:mm:ss')}`); + + if (new Date(startDate) < new Date(oldDoc[0].updated)) { + // UPDATE + delete nearestData._source.endValidity; + nearestDataBulk.push({ index: { _index: 'unpaywall_enriched', _id: nearestData._source.doi } }); + nearestDataBulk.push(nearestData._source); + logger.debug(`UNPAYWALL: UPDATE: doi: ${nearestData._source.doi}, genre: ${nearestData._source.genre}, updated: ${nearestData._source.updated}`); - if (new Date(oldDoc[0].updated) >= new Date(startDate)) { - nearestDataDeleteBulk.push({ delete: { _index: 'unpaywall_history', _id: nearestID } }); - nearestDataBulk.push({ index: { _index: 'unpaywall_enriched', _id: nearestData.doi } }); - nearestDataBulk.push(nearestData); + // DELETE + nearestDataDeleteBulk.push({ delete: { _index: 'unpaywall_history', _id: nearestData._id } }); + logger.debug(`HISTORY: DELETE: doi: ${nearestData._source.doi}, genre: ${nearestData._source.genre}, updated: ${nearestData._source.updated}`); } } - // STEP 2 - // STEP 3 + // UPDATE await bulk(nearestDataBulk, true); + logger.debug(`UPDATE ${nearestDataBulk.length / 2} lines`); + logger.debug('UNPAYWALL: REFRESH'); await refreshIndex('unpaywall_enriched'); - console.log('Doc mit à jour dans l\'index courant', nearestDataBulk.length / 2); + + // DELETE await bulk(nearestDataDeleteBulk, true); + logger.debug(`DELETE ${nearestDataDeleteBulk.length} lines`); + logger.debug('HISTORY: REFRESH'); await refreshIndex('unpaywall_history'); - console.log('Doc supprimé dans l\'historique', nearestDataDeleteBulk.length); +} + +async function step3(startDate) { + logger.debug('----------------------------------'); + logger.debug('STEP 3'); - // STEP 4 const toRecentDataBulk = []; const toRecentData = await searchWithRange(startDate, 'updated', 'gte', 'unpaywall_enriched'); - - const listOfIDToRecentData = toRecentData.map((e) => e._id); - - listOfIDToRecentData.forEach((id) => { - toRecentDataBulk.push({ delete: { _index: 'unpaywall_enriched', _id: id } }); + logger.debug(`startDate: ${format(new Date(startDate), 'yyyy-MM-dd/hh:mm:ss')}`); + logger.debug(`Should DELETE data greater or equal than ${format(new Date(startDate), 'yyyy-MM-dd/hh:mm:ss')}`); + toRecentData.forEach((data) => { + toRecentDataBulk.push({ delete: { _index: 'unpaywall_enriched', _id: data._id } }); + logger.debug(`UNPAYWALL: DELETE: doi: ${data._source.doi}, genre: ${data._source.genre}, updated: ${format(new Date(data._source.updated), 'yyyy-MM-dd/hh:mm:ss')}`); }); await bulk(toRecentDataBulk, true); - await refreshIndex('unpaywall_enriched'); - - // STEP 1 - const toRecentDataInHistoryBulk = []; - const toRecentDataInHistory = await searchWithRange(startDate, 'endValidity', 'gte', 'unpaywall_history'); - - toRecentDataInHistory.forEach((e) => { - console.log(e._source.updated); - }); - - const listOfIDToRecentDataInHistory = toRecentDataInHistory.map((e) => e._id); - - listOfIDToRecentDataInHistory.forEach((id) => { - toRecentDataInHistoryBulk.push({ delete: { _index: 'unpaywall_history', _id: id } }); - }); - - await bulk(toRecentDataInHistoryBulk, true); - console.log('valeur trop recente', toRecentDataInHistoryBulk.length); + logger.debug(`DELETE ${toRecentDataBulk.length} lines`); + logger.debug('UNPAYWALL: REFRESH'); await refreshIndex('unpaywall_enriched'); } -// /** -// * Resets the unpaywall index according to a date and deletes the data in the history. -// * @param {string} startDate - The date you wish to return to -// */ -// async function rollBack(startDate) { -// const bulkUpdate = []; -// const bulkDelete = []; -// const bulkHistoryDelete = []; -// const bulkHistoryDelete2 = []; - -// let data = await searchWithRange(startDate, 'endValidity', 'lte', 'unpaywall_history'); -// logger.debug(`unpaywall_history.length: ${data.length}`); -// logger.info(format(startDate, 'yyyy-MM-dd-HH')); - -// const listOfDoi = data.map((e) => e._source.doi); -// const listOfDoiFiltered = new Set(listOfDoi); - -// for (const doi of listOfDoiFiltered) { -// const doc = data.filter((e) => doi === e._source.doi); -// let latestDoc; -// doc.forEach((e) => { -// const unpaywallData = e._source; -// if (!latestDoc) { -// latestDoc = unpaywallData; -// } -// const date1 = new Date(startDate) - new Date(unpaywallData.endValidity); -// const date2 = new Date(startDate) - new Date(latestDoc.endValidity); - -// if (date1 < date2) { -// latestDoc = unpaywallData; -// } -// }); - -// const tt = await getDataByListOfDOI([latestDoc.doi], 'unpaywall_enriched'); - -// logger.debug(`oldData : ${new Date(tt[0].updated)}`); -// logger.debug(`Request : ${new Date(startDate)}`); -// logger.debug(`doi : ${doi}`); -// if (new Date(tt[0].updated) >= new Date(startDate)) { -// bulkUpdate.push({ index: { _index: 'unpaywall_enriched', _id: latestDoc.doi } }); -// bulkUpdate.push(latestDoc); -// } -// logger.debug('$$$$$$$$$$$$$$$'); -// logger.debug(latestDoc.genre); -// } - -// logger.debug(`[unpaywall] update : ${bulkUpdate.length / 2}`); - -// // UPDATE in unpaywall index -// try { -// await bulk(bulkUpdate, true); -// } catch (err) { -// logger.error('[elastic] Cannot bulk', err); -// return false; -// } - -// if (bulkUpdate.length > 0) { -// const dataThatWillBeDeleted = -// await searchWithRange(startDate, 'endValidity', 'gte', 'unpaywall_history'); - -// const listOfID = dataThatWillBeDeleted.map((e) => e._id); - -// listOfID.forEach((id) => { -// bulkHistoryDelete.push({ delete: { _index: 'unpaywall_history', _id: id } }); -// }); -// } - -// logger.debug(`[unpaywall_history] delete: ${bulkHistoryDelete.length}`); - -// // DELETE in history index -// try { -// await bulk(bulkHistoryDelete, true); -// } catch (err) { -// logger.error('[elastic] Cannot bulk', err); -// return false; -// } - -// data = await searchWithRange(startDate, 'updated', 'gt', 'unpaywall_enriched'); - -// data.forEach((e) => { -// bulkDelete.push({ delete: { _index: 'unpaywall_enriched', _id: e._source.doi } }); -// }); - -// logger.debug(`[unpaywall] delete: ${bulkDelete.length}`); - -// // DELETE in unpaywall index -// try { -// await bulk(bulkDelete, true); -// } catch (err) { -// logger.error('[elastic] Cannot bulk', err); -// return false; -// } - -// logger.debug(`[unpaywall_history] delete: ${bulkHistoryDelete.length}`); - -// data = await searchWithRange(startDate, 'updated', 'gt', 'unpaywall_history'); -// data.forEach((e) => { -// bulkHistoryDelete2.push({ delete: { _index: 'unpaywall_history', _id: e._id } }); -// }); - -// logger.debug(`[unpaywall_history] second delete: ${bulkHistoryDelete2.length}`); - -// // DELETE in unpaywall history index -// try { -// await bulk(bulkHistoryDelete2, true); -// } catch (err) { -// logger.error('[elastic] Cannot bulk', err); -// return false; -// } -// } +/** + * Resets the unpaywall index according to a date and deletes the data in the history. + * @param {string} startDate - The date you wish to return to + */ +async function rollBack(startDate) { + logger.debug('================================'); + await step1(startDate); + await step2(startDate); + await step3(startDate); +} module.exports = { insertHistoryDataUnpaywall, rollBack, + step1, + step2, + step3, }; diff --git a/src/update/lib/routers/job.js b/src/update/lib/routers/job.js index 1ae478d6..c9010e91 100644 --- a/src/update/lib/routers/job.js +++ b/src/update/lib/routers/job.js @@ -8,6 +8,12 @@ const { historyRollBack, } = require('../controllers/job'); +const { + step1, + step2, + step3, +} = require('../history'); + const { validateSnapshotJob, validateJobChangefilesConfig, @@ -66,4 +72,40 @@ router.post('/job/history', checkStatus, checkAuth, validateHistoryJob, insertWi */ router.post('/job/history/reset', checkStatus, checkAuth, validateHistoryReset, historyRollBack); +/** + * Route that roll back the current and the history index according to a date. + * Auth required. + * No update process should be in progress. + * + * This route need a body that contains a config of job. + */ +router.post('/job/history/reset', checkStatus, checkAuth, validateHistoryReset, historyRollBack); + +router.post('/job/history/reset/step1', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { + const { + startDate, + } = req.data; + + await step1(startDate); + return res.status(202).json(); +}); + +router.post('/job/history/reset/step2', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { + const { + startDate, + } = req.data; + + await step2(startDate); + return res.status(202).json(); +}); + +router.post('/job/history/reset/step3', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { + const { + startDate, + } = req.data; + + await step3(startDate); + return res.status(202).json(); +}); + module.exports = router; diff --git a/src/update/lib/services/elastic.js b/src/update/lib/services/elastic.js index bd453e41..f1f10800 100644 --- a/src/update/lib/services/elastic.js +++ b/src/update/lib/services/elastic.js @@ -119,6 +119,7 @@ async function initAlias(indexName, mapping, aliasName) { } async function getDataByListOfDOI(dois, index) { + if (!dois) { return []; } // Normalize request const normalizeDOI = dois.map((doi) => doi.toLowerCase()); diff --git a/src/update/test/history.js b/src/update/test/history.js index 10f1a9ae..c338e78d 100644 --- a/src/update/test/history.js +++ b/src/update/test/history.js @@ -26,15 +26,8 @@ const updateURL = process.env.UPDATE_HOST || 'http://localhost:59702'; chai.use(chaiHttp); describe('Test: daily update route test with history', () => { - const now = Date.now(); - const oneDay = 1 * 24 * 60 * 60 * 1000; - const oneYear = 1 * 24 * 60 * 60 * 1000 * 365; - - // create date in a format (YYYY-mm-dd) to be use by ezunpaywall - // 2 years - one day - const date1 = new Date(now - (2 * oneYear) - (1 * oneDay)).toISOString().slice(0, 10); - // 2 years - three days - const date3 = new Date(now - (2 * oneYear) - (3 * oneDay)).toISOString().slice(0, 10); + const date1 = '2020-01-03'; + const date3 = '2020-01-01'; before(async function () { this.timeout(30000); @@ -174,6 +167,6 @@ describe('Test: daily update route test with history', () => { }); after(async () => { - await reset(); + // await reset(); }); }); diff --git a/src/update/test/rollback.js b/src/update/test/rollback.js index 70374f0d..db30d0ed 100644 --- a/src/update/test/rollback.js +++ b/src/update/test/rollback.js @@ -1,7 +1,6 @@ const { expect } = require('chai'); const chai = require('chai'); const chaiHttp = require('chai-http'); -const { setTimeout } = require('node:timers/promises'); const unpaywallEnrichedMapping = require('../mapping/unpaywall.json'); const unpaywallHistoryMapping = require('../mapping/unpaywall_history.json'); @@ -19,10 +18,10 @@ const updateURL = process.env.UPDATE_HOST || 'http://localhost:59702'; chai.use(chaiHttp); describe('Test: rollback history test', () => { - const date1 = '2020-01-04T13:00:00.000'; - const date2 = '2020-01-03T13:00:00.000'; - const date3 = '2020-01-02T13:00:00.000'; - // const date4 = '2020-01-01T13:00:00.000'; + const date1 = '2020-01-04T01:00:00.000'; + const date2 = '2020-01-03T01:00:00.000'; + const date3 = '2020-01-02T01:00:00.000'; + const date4 = '2020-01-01T01:00:00.000'; describe(`Rollback: history rollback at ${date1}`, () => { before(async () => { @@ -46,9 +45,9 @@ describe('Test: rollback history test', () => { it('Should get unpaywall data', async () => { const count1 = await countDocuments('unpaywall_enriched'); - const count2 = await countDocuments('unpaywall_history'); - expect(count1).to.equal(2); + + const count2 = await countDocuments('unpaywall_history'); expect(count2).to.equal(4); const data = await getAllData('unpaywall_enriched'); @@ -56,16 +55,11 @@ describe('Test: rollback history test', () => { data.forEach((e) => { expect(e.genre).to.equal('v3'); }); - - const historyData = await getAllData('unpaywall_history'); - historyData.forEach((e) => { - expect(e.genre).not.equal('v3'); - }); }); after(async () => { - // await deleteIndex('unpaywall_enriched'); - // await deleteIndex('unpaywall_history'); + await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_history'); }); }); @@ -90,23 +84,16 @@ describe('Test: rollback history test', () => { }); it('Should get unpaywall data', async () => { - await setTimeout(1000); const count1 = await countDocuments('unpaywall_enriched'); - const count2 = await countDocuments('unpaywall_history'); - expect(count1).to.equal(2); + + const count2 = await countDocuments('unpaywall_history'); expect(count2).to.equal(2); const data = await getAllData('unpaywall_enriched'); data.forEach((e) => { - expect(e.genre).to.equal('v1'); - }); - - const historyData = await getAllData('unpaywall_history'); - historyData.forEach((e) => { - expect(e.genre).not.equal('v2'); - expect(e.genre).not.equal('v3'); + expect(e.genre).to.equal('v2'); }); }); @@ -138,9 +125,9 @@ describe('Test: rollback history test', () => { it('Should get unpaywall data', async () => { const count1 = await countDocuments('unpaywall_enriched'); - const count2 = await countDocuments('unpaywall_history'); + expect(count1).to.equal(2); - expect(count1).to.equal(0); + const count2 = await countDocuments('unpaywall_history'); expect(count2).to.equal(0); const data = await getAllData('unpaywall_enriched'); @@ -155,4 +142,38 @@ describe('Test: rollback history test', () => { await deleteIndex('unpaywall_history'); }); }); + + describe(`Rollback: history rollback at ${date4}`, () => { + before(async () => { + await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_history'); + await createIndex('unpaywall_enriched', unpaywallEnrichedMapping); + await createIndex('unpaywall_history', unpaywallHistoryMapping); + await insertHistoryDataUnpaywall(); + }); + + it('Should return a status code 202', async () => { + const res = await chai.request(updateURL) + .post('/job/history/reset') + .send({ + startDate: date4, + }) + .set('x-api-key', 'changeme'); + + expect(res).have.status(202); + }); + + it('Should get unpaywall data', async () => { + const count1 = await countDocuments('unpaywall_enriched'); + expect(count1).to.equal(0); + + const count2 = await countDocuments('unpaywall_history'); + expect(count2).to.equal(0); + }); + + after(async () => { + await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_history'); + }); + }); }); diff --git a/src/update/test/utils/elastic.js b/src/update/test/utils/elastic.js index fdc92ab3..abea057a 100644 --- a/src/update/test/utils/elastic.js +++ b/src/update/test/utils/elastic.js @@ -77,7 +77,7 @@ async function insertHistoryDataUnpaywall() { console.error(err); } - const filepath2 = path.resolve(__dirname, '..', 'sources', 'unpaywall_history.jsonl'); + const filepath2 = path.resolve(__dirname, '..', 'sources', 'unpaywall.jsonl'); let readStream2; try { readStream2 = await fs.createReadStream(filepath2); From 5f71caa079753b184c2f1e013a4fa91b8ec00b5c Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Fri, 3 Nov 2023 15:43:42 +0100 Subject: [PATCH 006/114] fix: move history functionnality --- src/update/app.js | 4 ++ src/update/lib/routers/history/job.js | 76 +++++++++++++++++++++++++++ src/update/lib/routers/job.js | 65 ----------------------- 3 files changed, 80 insertions(+), 65 deletions(-) create mode 100644 src/update/lib/routers/history/job.js diff --git a/src/update/app.js b/src/update/app.js index d5b35653..816a08ef 100644 --- a/src/update/app.js +++ b/src/update/app.js @@ -18,6 +18,8 @@ const routerCron = require('./lib/routers/cron'); const routerElastic = require('./lib/routers/elastic'); const routerOpenapi = require('./lib/routers/openapi'); +const routerHistoryJob = require('./lib/routers/history/job'); + require('./lib/cron/file'); const dataDir = path.resolve(__dirname, 'data'); @@ -48,6 +50,8 @@ app.use(routerElastic); app.use(routerOpenapi); app.use(routerPing); +app.use(routerHistoryJob); + /* Errors and unknown routes */ app.use((req, res, next) => res.status(404).json({ message: `Cannot ${req.method} ${req.originalUrl}` })); diff --git a/src/update/lib/routers/history/job.js b/src/update/lib/routers/history/job.js new file mode 100644 index 00000000..c54798cb --- /dev/null +++ b/src/update/lib/routers/history/job.js @@ -0,0 +1,76 @@ +const router = require('express').Router(); + +const { + insertWithOaHistory, + historyRollBack, +} = require('../../controllers/job'); + +const { + step1, + step2, + step3, +} = require('../../history'); + +const { + validateHistoryJob, + validateHistoryReset, +} = require('../../middlewares/job'); + +const checkStatus = require('../../middlewares/status'); +const checkAuth = require('../../middlewares/auth'); + +/** + * Route that download and insert on elastic the changefiles from unpaywall between a period. + * Auth required. + * No update process should be in progress. + * + * This route need a body that contains a config of job. + */ +router.post('/history/job', checkStatus, checkAuth, validateHistoryJob, insertWithOaHistory); + +/** + * Route that roll back the current and the history index according to a date. + * Auth required. + * No update process should be in progress. + * + * This route need a body that contains a config of job. + */ +router.post('/history/job/reset', checkStatus, checkAuth, validateHistoryReset, historyRollBack); + +/** + * Route that roll back the current and the history index according to a date. + * Auth required. + * No update process should be in progress. + * + * This route need a body that contains a config of job. + */ +router.post('/history/job/reset', checkStatus, checkAuth, validateHistoryReset, historyRollBack); + +router.post('/history/job/reset/step1', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { + const { + startDate, + } = req.data; + + await step1(startDate); + return res.status(202).json(); +}); + +router.post('/history/job/reset/step2', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { + const { + startDate, + } = req.data; + + await step2(startDate); + return res.status(202).json(); +}); + +router.post('/job/history/reset/step3', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { + const { + startDate, + } = req.data; + + await step3(startDate); + return res.status(202).json(); +}); + +module.exports = router; diff --git a/src/update/lib/routers/job.js b/src/update/lib/routers/job.js index c9010e91..e8820ccb 100644 --- a/src/update/lib/routers/job.js +++ b/src/update/lib/routers/job.js @@ -3,27 +3,16 @@ const router = require('express').Router(); const { downloadAndInsertSnapshotJob, insertChangefilesOnPeriodJob, - insertWithOaHistory, insertChangefileJob, - historyRollBack, } = require('../controllers/job'); -const { - step1, - step2, - step3, -} = require('../history'); - const { validateSnapshotJob, validateJobChangefilesConfig, validateInsertFile, - validateHistoryJob, - validateHistoryReset, } = require('../middlewares/job'); const checkStatus = require('../middlewares/status'); - const checkAuth = require('../middlewares/auth'); /** @@ -54,58 +43,4 @@ router.post('/job/period', checkStatus, checkAuth, validateJobChangefilesConfig, */ router.post('/job/changefile/:filename', checkStatus, checkAuth, validateInsertFile, insertChangefileJob); -/** - * Route that download and insert on elastic the changefiles from unpaywall between a period. - * Auth required. - * No update process should be in progress. - * - * This route need a body that contains a config of job. - */ -router.post('/job/history', checkStatus, checkAuth, validateHistoryJob, insertWithOaHistory); - -/** - * Route that roll back the current and the history index according to a date. - * Auth required. - * No update process should be in progress. - * - * This route need a body that contains a config of job. - */ -router.post('/job/history/reset', checkStatus, checkAuth, validateHistoryReset, historyRollBack); - -/** - * Route that roll back the current and the history index according to a date. - * Auth required. - * No update process should be in progress. - * - * This route need a body that contains a config of job. - */ -router.post('/job/history/reset', checkStatus, checkAuth, validateHistoryReset, historyRollBack); - -router.post('/job/history/reset/step1', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { - const { - startDate, - } = req.data; - - await step1(startDate); - return res.status(202).json(); -}); - -router.post('/job/history/reset/step2', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { - const { - startDate, - } = req.data; - - await step2(startDate); - return res.status(202).json(); -}); - -router.post('/job/history/reset/step3', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { - const { - startDate, - } = req.data; - - await step3(startDate); - return res.status(202).json(); -}); - module.exports = router; From cbad5e6232421c2ac7ede538a6ddd90d4e1c8506 Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Thu, 4 Jan 2024 13:57:20 +0100 Subject: [PATCH 007/114] refactor: update architecture --- src/update/app.js | 46 +++--- .../lib/{ => classic}/controllers/cron.js | 0 .../lib/{ => classic}/controllers/job.js | 75 +-------- .../lib/{ => classic}/controllers/report.js | 4 +- .../lib/{ => classic}/controllers/state.js | 2 +- src/update/lib/classic/cron.js | 59 ++++++++ src/update/lib/{ => classic}/cron/file.js | 12 +- src/update/lib/{ => classic}/cron/update.js | 6 +- src/update/lib/{ => classic}/insert.js | 10 +- src/update/lib/{ => classic}/job.js | 24 ++- src/update/lib/{ => classic}/report.js | 4 +- src/update/lib/{ => classic}/routers/cron.js | 4 +- src/update/lib/{ => classic}/routers/job.js | 6 +- .../lib/{ => classic}/routers/report.js | 4 +- src/update/lib/{ => classic}/routers/state.js | 0 src/update/lib/cron.js | 143 +++++++++++------- src/update/lib/download.js | 2 +- .../lib/{ => global}/controllers/elastic.js | 4 +- .../lib/{ => global}/controllers/file.js | 8 +- .../lib/{ => global}/controllers/health.js | 4 +- .../lib/{ => global}/controllers/status.js | 2 +- .../lib/{ => global}/controllers/unpaywall.js | 2 +- .../lib/{ => global}/routers/elastic.js | 2 +- .../lib/{ => global}/routers/openapi.js | 2 +- src/update/lib/{ => global}/routers/ping.js | 0 .../lib/{ => global}/routers/snapshot.js | 14 +- src/update/lib/{ => global}/routers/status.js | 2 +- .../lib/{ => global}/routers/unpaywall.js | 2 +- src/update/lib/history/controllers/job.js | 79 ++++++++++ src/update/lib/{ => history}/history.js | 14 +- src/update/lib/history/job.js | 92 +++++++++++ src/update/lib/history/report.js | 52 +++++++ .../history => history/routers}/job.js | 4 +- src/update/lib/lab/history-v1.js | 2 +- src/update/lib/models/cron.js | 96 ------------ src/update/lib/{models => }/state.js | 8 +- src/update/test/{ => classic}/cron.js | 4 +- .../test/{ => classic}/insertionError.js | 14 +- .../test/{ => classic}/insertionFile.js | 14 +- .../test/{ => classic}/insertionPeriodDay.js | 14 +- .../test/{ => classic}/insertionPeriodWeek.js | 14 +- .../test/{ => classic}/insertionSnapshot.js | 12 +- src/update/test/{ => classic}/updateDay.js | 14 +- src/update/test/{ => classic}/updateWeek.js | 14 +- src/update/test/{ => history}/history.js | 14 +- src/update/test/{ => history}/rollback.js | 36 ++++- 46 files changed, 553 insertions(+), 387 deletions(-) rename src/update/lib/{ => classic}/controllers/cron.js (100%) rename src/update/lib/{ => classic}/controllers/job.js (58%) rename src/update/lib/{ => classic}/controllers/report.js (93%) rename src/update/lib/{ => classic}/controllers/state.js (90%) create mode 100644 src/update/lib/classic/cron.js rename src/update/lib/{ => classic}/cron/file.js (70%) rename src/update/lib/{ => classic}/cron/update.js (93%) rename src/update/lib/{ => classic}/insert.js (95%) rename src/update/lib/{ => classic}/job.js (92%) rename src/update/lib/{ => classic}/report.js (91%) rename src/update/lib/{ => classic}/routers/cron.js (86%) rename src/update/lib/{ => classic}/routers/job.js (89%) rename src/update/lib/{ => classic}/routers/report.js (82%) rename src/update/lib/{ => classic}/routers/state.js (100%) rename src/update/lib/{ => global}/controllers/elastic.js (78%) rename src/update/lib/{ => global}/controllers/file.js (90%) rename src/update/lib/{ => global}/controllers/health.js (91%) rename src/update/lib/{ => global}/controllers/status.js (92%) rename src/update/lib/{ => global}/controllers/unpaywall.js (93%) rename src/update/lib/{ => global}/routers/elastic.js (83%) rename src/update/lib/{ => global}/routers/openapi.js (79%) rename src/update/lib/{ => global}/routers/ping.js (100%) rename src/update/lib/{ => global}/routers/snapshot.js (69%) rename src/update/lib/{ => global}/routers/status.js (87%) rename src/update/lib/{ => global}/routers/unpaywall.js (85%) create mode 100644 src/update/lib/history/controllers/job.js rename src/update/lib/{ => history}/history.js (97%) create mode 100644 src/update/lib/history/job.js create mode 100644 src/update/lib/history/report.js rename src/update/lib/{routers/history => history/routers}/job.js (96%) delete mode 100644 src/update/lib/models/cron.js rename src/update/lib/{models => }/state.js (96%) rename src/update/test/{ => classic}/cron.js (98%) rename src/update/test/{ => classic}/insertionError.js (93%) rename src/update/test/{ => classic}/insertionFile.js (98%) rename src/update/test/{ => classic}/insertionPeriodDay.js (98%) rename src/update/test/{ => classic}/insertionPeriodWeek.js (98%) rename src/update/test/{ => classic}/insertionSnapshot.js (95%) rename src/update/test/{ => classic}/updateDay.js (98%) rename src/update/test/{ => classic}/updateWeek.js (97%) rename src/update/test/{ => history}/history.js (96%) rename src/update/test/{ => history}/rollback.js (84%) diff --git a/src/update/app.js b/src/update/app.js index 816a08ef..86b4d007 100644 --- a/src/update/app.js +++ b/src/update/app.js @@ -7,20 +7,21 @@ const morgan = require('./lib/morgan'); const logger = require('./lib/logger'); const getConfig = require('./lib/config'); -const routerPing = require('./lib/routers/ping'); -const routerJob = require('./lib/routers/job'); -const routerReport = require('./lib/routers/report'); -const routerSnapshot = require('./lib/routers/snapshot'); -const routerState = require('./lib/routers/state'); -const routerStatus = require('./lib/routers/status'); -const routerUnpaywall = require('./lib/routers/unpaywall'); -const routerCron = require('./lib/routers/cron'); -const routerElastic = require('./lib/routers/elastic'); -const routerOpenapi = require('./lib/routers/openapi'); +const routerGlobalPing = require('./lib/global/routers/ping'); +const routerGlobalElastic = require('./lib/global/routers/elastic'); +const routerGlobalOpenapi = require('./lib/global/routers/openapi'); +const routerGlobalUnpaywall = require('./lib/global/routers/unpaywall'); +const routerGlobalSnapshot = require('./lib/global/routers/snapshot'); +const routerGlobalStatus = require('./lib/global/routers/status'); -const routerHistoryJob = require('./lib/routers/history/job'); +const routerClassicJob = require('./lib/classic/routers/job'); +const routerClassicReport = require('./lib/classic/routers/report'); +const routerClassicState = require('./lib/classic/routers/state'); +const routerClassicCron = require('./lib/classic/routers/cron'); -require('./lib/cron/file'); +const routerHistoryJob = require('./lib/history/routers/job'); + +require('./lib/classic/cron/file'); const dataDir = path.resolve(__dirname, 'data'); fs.ensureDir(path.resolve(dataDir)); @@ -39,16 +40,17 @@ app.use(express.json()); app.use(cors()); app.use(morgan); -app.use(routerJob); -app.use(routerReport); -app.use(routerSnapshot); -app.use(routerState); -app.use(routerStatus); -app.use(routerUnpaywall); -app.use(routerCron); -app.use(routerElastic); -app.use(routerOpenapi); -app.use(routerPing); +app.use(routerGlobalPing); +app.use(routerGlobalElastic); +app.use(routerGlobalOpenapi); +app.use(routerGlobalUnpaywall); +app.use(routerGlobalSnapshot); +app.use(routerGlobalStatus); + +app.use(routerClassicJob); +app.use(routerClassicReport); +app.use(routerClassicState); +app.use(routerClassicCron); app.use(routerHistoryJob); diff --git a/src/update/lib/controllers/cron.js b/src/update/lib/classic/controllers/cron.js similarity index 100% rename from src/update/lib/controllers/cron.js rename to src/update/lib/classic/controllers/cron.js diff --git a/src/update/lib/controllers/job.js b/src/update/lib/classic/controllers/job.js similarity index 58% rename from src/update/lib/controllers/job.js rename to src/update/lib/classic/controllers/job.js index d1a0af0d..7e54f7b2 100644 --- a/src/update/lib/controllers/job.js +++ b/src/update/lib/classic/controllers/job.js @@ -2,19 +2,14 @@ const fs = require('fs-extra'); const path = require('path'); const { format } = require('date-fns'); -const snapshotsDir = path.resolve(__dirname, '..', '..', 'data', 'snapshots'); +const snapshotsDir = path.resolve(__dirname, '..', '..', '..', 'data', 'snapshots'); const { downloadAndInsertSnapshot, - insertWithOaHistoryJob, insertChangefilesOnPeriod, insertChangefile, } = require('../job'); -const { - rollBack, -} = require('../history'); - /** * Controller to start job that download and insert snapshot. * @@ -112,76 +107,8 @@ async function insertChangefileJob(req, res, next) { return res.status(202).json(); } -/** - * Controller to start job that download ans insert changefiles on period. - * - * @param {import('express').Request} req - HTTP request. - * @param {import('express').Response} res - HTTP response. - * @param {import('express').NextFunction} next - Do the following. - */ -async function insertWithOaHistory(req, res, next) { - const { - startDate, - endDate, - index, - interval, - } = req.data; - - if (new Date(startDate).getTime() > Date.now()) { - return res.status(400).json({ message: 'startDate cannot be in the futur' }); - } - - if (startDate && endDate) { - if (new Date(endDate).getTime() < new Date(startDate).getTime()) { - return res.status(400).json({ message: 'endDate cannot be lower than startDate' }); - } - } - - const jobConfig = { - index, - interval, - startDate, - endDate, - offset: 0, - limit: -1, - }; - - if (!startDate && !endDate) { - jobConfig.endDate = format(new Date(), 'yyyy-MM-dd'); - if (interval === 'week') jobConfig.startDate = format(new Date() - (7 * 24 * 60 * 60 * 1000), 'yyyy-MM-dd'); - if (interval === 'day') jobConfig.startDate = format(new Date(), 'yyyy-MM-dd'); - - insertWithOaHistoryJob(jobConfig); - return res.status(202).json(); - } - - if (startDate && !endDate) jobConfig.endDate = format(new Date(), 'yyyy-MM-dd'); - - insertWithOaHistoryJob(jobConfig); - return res.status(202).json(); -} - -/** - * Controller to start job that download ans insert changefiles on period. - * - * @param {import('express').Request} req - HTTP request. - * @param {import('express').Response} res - HTTP response. - * @param {import('express').NextFunction} next - Do the following. - */ -async function historyRollBack(req, res, next) { - const { - startDate, - index, - } = req.data; - - await rollBack(startDate, index); - return res.status(202).json(); -} - module.exports = { downloadAndInsertSnapshotJob, insertChangefilesOnPeriodJob, insertChangefileJob, - insertWithOaHistory, - historyRollBack, }; diff --git a/src/update/lib/controllers/report.js b/src/update/lib/classic/controllers/report.js similarity index 93% rename from src/update/lib/controllers/report.js rename to src/update/lib/classic/controllers/report.js index c8a0f83e..69198a0d 100644 --- a/src/update/lib/controllers/report.js +++ b/src/update/lib/classic/controllers/report.js @@ -1,10 +1,10 @@ const path = require('path'); const fs = require('fs-extra'); -const { getMostRecentFile } = require('../file'); +const { getMostRecentFile } = require('../../file'); const { getReport } = require('../report'); -const reportsDir = path.resolve(__dirname, '..', '..', 'data', 'reports'); +const reportsDir = path.resolve(__dirname, '..', '..', '..', 'data', 'reports'); /** * Controller to get list of reports or latest report. diff --git a/src/update/lib/controllers/state.js b/src/update/lib/classic/controllers/state.js similarity index 90% rename from src/update/lib/controllers/state.js rename to src/update/lib/classic/controllers/state.js index 72b8ce75..e854c93a 100644 --- a/src/update/lib/controllers/state.js +++ b/src/update/lib/classic/controllers/state.js @@ -1,4 +1,4 @@ -const { getState } = require('../models/state'); +const { getState } = require('../../state'); /** * Controller to get the state of update. diff --git a/src/update/lib/classic/cron.js b/src/update/lib/classic/cron.js new file mode 100644 index 00000000..4b768840 --- /dev/null +++ b/src/update/lib/classic/cron.js @@ -0,0 +1,59 @@ +const { format, subDays } = require('date-fns'); + +const Cron = require('../cron'); + +const { insertChangefilesOnPeriod } = require('./job'); + +const updateConfig = { + index: 'unpaywall', + interval: 'day', +}; + +/** + * Starts an update daily process if no update process is started. + * + * @returns {Promise} + */ +async function task() { + const week = (updateConfig.interval === 'week'); + const startDate = format(subDays(new Date(), week ? 7 : 0), 'yyyy-MM-dd'); + const endDate = format(new Date(), 'yyyy-MM-dd'); + await insertChangefilesOnPeriod({ + index: updateConfig.index, + interval: updateConfig.interval, + startDate, + endDate, + offset: 0, + limit: -1, + }); +} + +const cron = new Cron('update', '0 0 0 * * *', task); + +/** + * Update config of update process and config of cron. + * + * @param {Object} config - Global config. + */ +function update(config) { + if (config.time) updateConfig.time = config.time; + if (config.index) updateConfig.index = config.index; + if (config.interval) updateConfig.interval = config.interval; + + cron.setTask(task); +} + +/** + * Get config of update process and config of cron. + * @returns {Object} Config of update process and config of cron. + */ +function getGlobalConfig() { + const config = cron.getConfig(); + return { ...config, ...updateConfig }; +} + +module.exports = { + getGlobalConfig, + update, + cron, +}; diff --git a/src/update/lib/cron/file.js b/src/update/lib/classic/cron/file.js similarity index 70% rename from src/update/lib/cron/file.js rename to src/update/lib/classic/cron/file.js index b802ec80..a3ed67d0 100644 --- a/src/update/lib/cron/file.js +++ b/src/update/lib/classic/cron/file.js @@ -1,12 +1,12 @@ const path = require('path'); -const logger = require('../logger'); -const Cron = require('../models/cron'); +const logger = require('../../logger'); +const Cron = require('../../cron'); -const { deleteFilesInDir } = require('../file'); +const { deleteFilesInDir } = require('../../file'); -const reportsDir = path.resolve(__dirname, '..', '..', 'data', 'reports'); -const snapshotDir = path.resolve(__dirname, '..', '..', 'data', 'snapshots'); -const states = path.resolve(__dirname, '..', '..', 'data', 'states'); +const reportsDir = path.resolve(__dirname, '..', '..', '..', 'data', 'reports'); +const snapshotDir = path.resolve(__dirname, '..', '..', '..', 'data', 'snapshots'); +const states = path.resolve(__dirname, '..', '..', '..', 'data', 'states'); /** * Removes files generated by an update process that are older than 30 days. diff --git a/src/update/lib/cron/update.js b/src/update/lib/classic/cron/update.js similarity index 93% rename from src/update/lib/cron/update.js rename to src/update/lib/classic/cron/update.js index 4c292f12..918d7964 100644 --- a/src/update/lib/cron/update.js +++ b/src/update/lib/classic/cron/update.js @@ -1,11 +1,11 @@ const { format, subDays } = require('date-fns'); const { cron } = require('config'); -const Cron = require('../models/cron'); -const { getStatus } = require('../status'); +const Cron = require('../../cron'); +const { getStatus } = require('../../status'); const { insertChangefilesOnPeriod } = require('../job'); -const logger = require('../logger'); +const logger = require('../../logger'); let { active } = cron; diff --git a/src/update/lib/insert.js b/src/update/lib/classic/insert.js similarity index 95% rename from src/update/lib/insert.js rename to src/update/lib/classic/insert.js index 5e9f8969..682389ad 100644 --- a/src/update/lib/insert.js +++ b/src/update/lib/classic/insert.js @@ -8,27 +8,27 @@ const readline = require('readline'); const config = require('config'); const zlib = require('zlib'); -const logger = require('./logger'); -const unpaywallMapping = require('../mapping/unpaywall.json'); +const logger = require('../logger'); +const unpaywallMapping = require('../../mapping/unpaywall.json'); const { addStepInsert, getLatestStep, updateLatestStep, fail, -} = require('./models/state'); +} = require('../state'); const { refreshIndex, bulk, initAlias, createIndex, -} = require('./services/elastic'); +} = require('../services/elastic'); const indexAlias = config.get('elasticsearch.indexAlias'); const maxBulkSize = config.get('elasticsearch.maxBulkSize'); -const snapshotsDir = path.resolve(__dirname, '..', 'data', 'snapshots'); +const snapshotsDir = path.resolve(__dirname, '..', '..', 'data', 'snapshots'); /** * Insert data on elastic with elastic bulk request. diff --git a/src/update/lib/job.js b/src/update/lib/classic/job.js similarity index 92% rename from src/update/lib/job.js rename to src/update/lib/classic/job.js index d5808074..0ea008ea 100644 --- a/src/update/lib/job.js +++ b/src/update/lib/classic/job.js @@ -1,38 +1,34 @@ /* eslint-disable no-param-reassign */ -const logger = require('./logger'); +const logger = require('../logger'); const { endState, fail, -} = require('./models/state'); +} = require('../state'); const { setInUpdate, -} = require('./status'); +} = require('../status'); const { downloadBigSnapshot, downloadChangefile, -} = require('./download'); +} = require('../download'); const { createState, addStepGetChangefiles, updateLatestStep, getLatestStep, -} = require('./models/state'); +} = require('../state'); const insertDataUnpaywall = require('./insert'); -const { insertHistoryDataUnpaywall } = require('./history'); +const { insertHistoryDataUnpaywall } = require('../history/history'); -const { - getChangefiles, -} = require('./services/unpaywall'); +const { getChangefiles } = require('../services/unpaywall'); -const { - sendMailNoChangefile, -} = require('./services/mail'); +const { sendMailNoChangefile } = require('../services/mail'); /** * Download the current snapshot of unpaywall and insert his content. @@ -145,9 +141,7 @@ async function insertChangefile(jobConfig) { */ async function insertWithOaHistoryJob(jobConfig) { setInUpdate(true); - const { - interval, startDate, endDate, - } = jobConfig; + const { interval, startDate, endDate } = jobConfig; await createState(); const start = new Date(); addStepGetChangefiles(); diff --git a/src/update/lib/report.js b/src/update/lib/classic/report.js similarity index 91% rename from src/update/lib/report.js rename to src/update/lib/classic/report.js index 5acd98f9..866e4e15 100644 --- a/src/update/lib/report.js +++ b/src/update/lib/classic/report.js @@ -1,9 +1,9 @@ const path = require('path'); const fs = require('fs-extra'); const { format } = require('date-fns'); -const logger = require('./logger'); +const logger = require('../logger'); -const reportsDir = path.resolve(__dirname, '..', 'data', 'reports'); +const reportsDir = path.resolve(__dirname, '..', '..', 'data', 'reports'); /** * Create report on the folder "data/update/report" on behalf of the date of treatment. diff --git a/src/update/lib/routers/cron.js b/src/update/lib/classic/routers/cron.js similarity index 86% rename from src/update/lib/routers/cron.js rename to src/update/lib/classic/routers/cron.js index 98af616c..9fec5fab 100644 --- a/src/update/lib/routers/cron.js +++ b/src/update/lib/classic/routers/cron.js @@ -1,7 +1,7 @@ const router = require('express').Router(); -const checkAuth = require('../middlewares/auth'); -const validateCronConfig = require('../middlewares/cron'); +const checkAuth = require('../../middlewares/auth'); +const validateCronConfig = require('../../middlewares/cron'); const { startUpdateCron, diff --git a/src/update/lib/routers/job.js b/src/update/lib/classic/routers/job.js similarity index 89% rename from src/update/lib/routers/job.js rename to src/update/lib/classic/routers/job.js index e8820ccb..557f8729 100644 --- a/src/update/lib/routers/job.js +++ b/src/update/lib/classic/routers/job.js @@ -10,10 +10,10 @@ const { validateSnapshotJob, validateJobChangefilesConfig, validateInsertFile, -} = require('../middlewares/job'); +} = require('../../middlewares/job'); -const checkStatus = require('../middlewares/status'); -const checkAuth = require('../middlewares/auth'); +const checkStatus = require('../../middlewares/status'); +const checkAuth = require('../../middlewares/auth'); /** * Route that download the current snapshot of unpaywall and insert his content. diff --git a/src/update/lib/routers/report.js b/src/update/lib/classic/routers/report.js similarity index 82% rename from src/update/lib/routers/report.js rename to src/update/lib/classic/routers/report.js index f24c13a1..c018f617 100644 --- a/src/update/lib/routers/report.js +++ b/src/update/lib/classic/routers/report.js @@ -5,8 +5,8 @@ const { getReportByFilename, } = require('../controllers/report'); -const validateLatest = require('../middlewares/latest'); -const validateFilename = require('../middlewares/filename'); +const validateLatest = require('../../middlewares/latest'); +const validateFilename = require('../../middlewares/filename'); /** * Route that give the list of reports or the content of most recent report in JSON format. diff --git a/src/update/lib/routers/state.js b/src/update/lib/classic/routers/state.js similarity index 100% rename from src/update/lib/routers/state.js rename to src/update/lib/classic/routers/state.js diff --git a/src/update/lib/cron.js b/src/update/lib/cron.js index 4bf600ed..edb00c81 100644 --- a/src/update/lib/cron.js +++ b/src/update/lib/cron.js @@ -1,59 +1,96 @@ -const { format, subDays } = require('date-fns'); - -const Cron = require('./models/cron'); - -const { insertChangefilesOnPeriod } = require('./job'); - -const updateConfig = { - index: 'unpaywall', - interval: 'day', -}; - -/** - * Starts an update daily process if no update process is started. - * - * @returns {Promise} - */ -async function task() { - const week = (updateConfig.interval === 'week'); - const startDate = format(subDays(new Date(), week ? 7 : 0), 'yyyy-MM-dd'); - const endDate = format(new Date(), 'yyyy-MM-dd'); - await insertChangefilesOnPeriod({ - index: updateConfig.index, - interval: updateConfig.interval, - startDate, - endDate, - offset: 0, - limit: -1, - }); -} +const { CronJob } = require('cron'); -const cron = new Cron('update', '0 0 0 * * *', task); +const logger = require('./logger'); -/** - * Update config of update process and config of cron. - * - * @param {Object} config - Global config. - */ -function update(config) { - if (config.time) updateConfig.time = config.time; - if (config.index) updateConfig.index = config.index; - if (config.interval) updateConfig.interval = config.interval; +class Cron { + /** + * @constructor + * + * @param {string} name - Name of cron. + * @param {string} schedule - Schedule of cron. + * @param {function} task - Function that will be executed by the cron. + * @param {boolean} active - Indicates whether it is active or not. + */ + constructor(name, schedule, task, active) { + this.name = name; + this.schedule = schedule; + this.task = task; + this.active = active; + this.process = new CronJob(schedule, this.task, null, false, 'Europe/Paris'); + if (active) { + this.start(); + } + } - cron.setTask(task); -} + /** + * Getter of config of cron. + * + * @returns {Object} config of cron. + */ + getConfig() { + return { + name: this.name, + schedule: this.schedule, + active: this.active, + }; + } + + /** + * Set new task for cron. + * + * @param {function} task - Function that will be executed by the cron. + */ + setTask(task) { + this.process.stop(); + this.task = task; + logger.info(`[cron: ${this.name}] config - task updated`); + this.process = new CronJob(this.schedule, this.task, null, false, 'Europe/Paris'); + if (this.active) this.process.start(); + } + + /** + * Set new schedule for cron. + * + * @param {string} schedule - Schedule of cron. + */ + setSchedule(schedule) { + this.process.stop(); + this.schedule = schedule; + logger.info(`[cron: ${this.name}] config - schedule is updated [${this.schedule}]`); + this.process = new CronJob(this.schedule, async () => { + await this.task(); + }, null, false, 'Europe/Paris'); + if (this.active) this.process.start(); + } + + /** + * Make active to true. + */ + start() { + try { + this.process.start(); + logger.info(`[cron: ${this.name}] - started`); + logger.info(`[cron: ${this.name}] config - schedule: [${this.schedule}]`); + } catch (err) { + logger.error(`[cron ${this.name}] - error in start`, err); + return; + } + this.active = true; + } -/** - * Get config of update process and config of cron. - * @returns {Object} Config of update process and config of cron. - */ -function getGlobalConfig() { - const config = cron.getConfig(); - return { ...config, ...updateConfig }; + /** + * Make active to false. + */ + stop() { + try { + this.process.stop(); + logger.info(`[cron: ${this.name}] - stopped`); + } catch (err) { + logger.error(`[cron ${this.name}] - error in stop`, err); + return; + } + this.active = false; + } } -module.exports = { - getGlobalConfig, - update, - cron, -}; +module.exports = Cron; diff --git a/src/update/lib/download.js b/src/update/lib/download.js index a764f27b..efe41455 100644 --- a/src/update/lib/download.js +++ b/src/update/lib/download.js @@ -11,7 +11,7 @@ const { addStepDownload, fail, updateLatestStep, -} = require('./models/state'); +} = require('./state'); const { getSnapshot, diff --git a/src/update/lib/controllers/elastic.js b/src/update/lib/global/controllers/elastic.js similarity index 78% rename from src/update/lib/controllers/elastic.js rename to src/update/lib/global/controllers/elastic.js index 2a804b4c..690dc361 100644 --- a/src/update/lib/controllers/elastic.js +++ b/src/update/lib/global/controllers/elastic.js @@ -1,5 +1,5 @@ -const { initAlias } = require('../services/elastic'); -const unpaywallMapping = require('../../mapping/unpaywall.json'); +const { initAlias } = require('../../services/elastic'); +const unpaywallMapping = require('../../../mapping/unpaywall.json'); /** * Controller to create alias unpaywall on elastic. diff --git a/src/update/lib/controllers/file.js b/src/update/lib/global/controllers/file.js similarity index 90% rename from src/update/lib/controllers/file.js rename to src/update/lib/global/controllers/file.js index a6a04027..f0edc732 100644 --- a/src/update/lib/controllers/file.js +++ b/src/update/lib/global/controllers/file.js @@ -1,12 +1,12 @@ const fs = require('fs-extra'); const path = require('path'); -const snapshotsDir = path.resolve(__dirname, '..', '..', 'data', 'snapshots'); +const snapshotsDir = path.resolve(__dirname, '..', '..', '..', 'data', 'snapshots'); const { getMostRecentFile, deleteFile, -} = require('../file'); +} = require('../../file'); /** * Controller to start list of files or latest file downloaded on update service. @@ -50,7 +50,7 @@ async function uploadFile(req, res, next) { * @param {import('express').Response} res - HTTP response. * @param {import('express').NextFunction} next - Do the following. */ -async function deleteFileInstalled(req, res, next) { +async function deleteInstalledFile(req, res, next) { const filename = req.data; if (!await fs.pathExists(path.resolve(snapshotsDir, filename))) { @@ -69,5 +69,5 @@ async function deleteFileInstalled(req, res, next) { module.exports = { getFiles, uploadFile, - deleteFileInstalled, + deleteInstalledFile, }; diff --git a/src/update/lib/controllers/health.js b/src/update/lib/global/controllers/health.js similarity index 91% rename from src/update/lib/controllers/health.js rename to src/update/lib/global/controllers/health.js index 1960ff3f..bba58193 100644 --- a/src/update/lib/controllers/health.js +++ b/src/update/lib/global/controllers/health.js @@ -1,5 +1,5 @@ -const promiseWithTimeout = require('../ping'); -const { pingElastic } = require('../services/elastic'); +const promiseWithTimeout = require('../../ping'); +const { pingElastic } = require('../../services/elastic'); /** * Controller to get health of all services connected to update service. diff --git a/src/update/lib/controllers/status.js b/src/update/lib/global/controllers/status.js similarity index 92% rename from src/update/lib/controllers/status.js rename to src/update/lib/global/controllers/status.js index d129db5b..a99e4794 100644 --- a/src/update/lib/controllers/status.js +++ b/src/update/lib/global/controllers/status.js @@ -1,4 +1,4 @@ -const { getStatus, setInUpdate } = require('../status'); +const { getStatus, setInUpdate } = require('../../status'); /** * Controller to get status of update. diff --git a/src/update/lib/controllers/unpaywall.js b/src/update/lib/global/controllers/unpaywall.js similarity index 93% rename from src/update/lib/controllers/unpaywall.js rename to src/update/lib/global/controllers/unpaywall.js index 11357e92..ab1e7c57 100644 --- a/src/update/lib/controllers/unpaywall.js +++ b/src/update/lib/global/controllers/unpaywall.js @@ -1,4 +1,4 @@ -const { getChangefiles } = require('../services/unpaywall'); +const { getChangefiles } = require('../../services/unpaywall'); /** * Controller to get list of changefiles on unpaywall. diff --git a/src/update/lib/routers/elastic.js b/src/update/lib/global/routers/elastic.js similarity index 83% rename from src/update/lib/routers/elastic.js rename to src/update/lib/global/routers/elastic.js index c4219038..8fbcfeea 100644 --- a/src/update/lib/routers/elastic.js +++ b/src/update/lib/global/routers/elastic.js @@ -1,6 +1,6 @@ const router = require('express').Router(); -const checkAuth = require('../middlewares/auth'); +const checkAuth = require('../../middlewares/auth'); const createAlias = require('../controllers/elastic'); /** diff --git a/src/update/lib/routers/openapi.js b/src/update/lib/global/routers/openapi.js similarity index 79% rename from src/update/lib/routers/openapi.js rename to src/update/lib/global/routers/openapi.js index f68e1ca0..e80b5990 100644 --- a/src/update/lib/routers/openapi.js +++ b/src/update/lib/global/routers/openapi.js @@ -1,6 +1,6 @@ const router = require('express').Router(); -const openapi = require('../../openapi.json'); +const openapi = require('../../../openapi.json'); /** * Route that give the openapi.json file. diff --git a/src/update/lib/routers/ping.js b/src/update/lib/global/routers/ping.js similarity index 100% rename from src/update/lib/routers/ping.js rename to src/update/lib/global/routers/ping.js diff --git a/src/update/lib/routers/snapshot.js b/src/update/lib/global/routers/snapshot.js similarity index 69% rename from src/update/lib/routers/snapshot.js rename to src/update/lib/global/routers/snapshot.js index f9c45939..d496a848 100644 --- a/src/update/lib/routers/snapshot.js +++ b/src/update/lib/global/routers/snapshot.js @@ -1,16 +1,16 @@ const router = require('express').Router(); -const upload = require('../middlewares/multer'); -const checkAuth = require('../middlewares/auth'); -const dev = require('../middlewares/dev'); +const upload = require('../../middlewares/multer'); +const checkAuth = require('../../middlewares/auth'); +const dev = require('../../middlewares/dev'); -const validateLatest = require('../middlewares/latest'); -const validateFilename = require('../middlewares/filename'); +const validateLatest = require('../../middlewares/latest'); +const validateFilename = require('../../middlewares/filename'); const { getFiles, uploadFile, - deleteFileInstalled, + deleteInstalledFile, } = require('../controllers/file'); /** @@ -34,6 +34,6 @@ router.post('/snapshots', dev, checkAuth, upload.single('file'), uploadFile); * Route that delete a file on ezunpaywall. * Auth required. */ -router.delete('/snapshots/:filename', dev, checkAuth, validateFilename, deleteFileInstalled); +router.delete('/snapshots/:filename', dev, checkAuth, validateFilename, deleteInstalledFile); module.exports = router; diff --git a/src/update/lib/routers/status.js b/src/update/lib/global/routers/status.js similarity index 87% rename from src/update/lib/routers/status.js rename to src/update/lib/global/routers/status.js index 1e6b9aaa..a159b750 100644 --- a/src/update/lib/routers/status.js +++ b/src/update/lib/global/routers/status.js @@ -1,6 +1,6 @@ const router = require('express').Router(); -const checkAuth = require('../middlewares/auth'); +const checkAuth = require('../../middlewares/auth'); const { getUpdateStatus, patchUpdateStatus } = require('../controllers/status'); /** diff --git a/src/update/lib/routers/unpaywall.js b/src/update/lib/global/routers/unpaywall.js similarity index 85% rename from src/update/lib/routers/unpaywall.js rename to src/update/lib/global/routers/unpaywall.js index 9f39047b..e2a71e16 100644 --- a/src/update/lib/routers/unpaywall.js +++ b/src/update/lib/global/routers/unpaywall.js @@ -1,6 +1,6 @@ const router = require('express').Router(); -const validateIntervale = require('../middlewares/interval'); +const validateIntervale = require('../../middlewares/interval'); const getChangefilesOfUnpaywall = require('../controllers/unpaywall'); /** diff --git a/src/update/lib/history/controllers/job.js b/src/update/lib/history/controllers/job.js new file mode 100644 index 00000000..d4c91848 --- /dev/null +++ b/src/update/lib/history/controllers/job.js @@ -0,0 +1,79 @@ +const { format } = require('date-fns'); + +const { + insertWithOaHistoryJob, +} = require('../job'); + +const { + rollBack, +} = require('../history'); +/** + * Controller to start job that download ans insert changefiles on period. + * + * @param {import('express').Request} req - HTTP request. + * @param {import('express').Response} res - HTTP response. + * @param {import('express').NextFunction} next - Do the following. + */ +async function insertWithOaHistory(req, res, next) { + const { + startDate, + endDate, + index, + interval, + } = req.data; + + if (new Date(startDate).getTime() > Date.now()) { + return res.status(400).json({ message: 'startDate cannot be in the future' }); + } + + if (startDate && endDate) { + if (new Date(endDate).getTime() < new Date(startDate).getTime()) { + return res.status(400).json({ message: 'endDate cannot be lower than startDate' }); + } + } + + const jobConfig = { + index, + interval, + startDate, + endDate, + offset: 0, + limit: -1, + }; + + if (!startDate && !endDate) { + jobConfig.endDate = format(new Date(), 'yyyy-MM-dd'); + if (interval === 'week') jobConfig.startDate = format(new Date() - (7 * 24 * 60 * 60 * 1000), 'yyyy-MM-dd'); + if (interval === 'day') jobConfig.startDate = format(new Date(), 'yyyy-MM-dd'); + + insertWithOaHistoryJob(jobConfig); + return res.status(202).json(); + } + + if (startDate && !endDate) jobConfig.endDate = format(new Date(), 'yyyy-MM-dd'); + + insertWithOaHistoryJob(jobConfig); + return res.status(202).json(); +} + +/** + * Controller to start job that download ans insert changefiles on period. + * + * @param {import('express').Request} req - HTTP request. + * @param {import('express').Response} res - HTTP response. + * @param {import('express').NextFunction} next - Do the following. + */ +async function historyRollBack(req, res, next) { + const { + startDate, + index, + } = req.data; + + await rollBack(startDate, index); + return res.status(202).json(); +} + +module.exports = { + insertWithOaHistory, + historyRollBack, +}; diff --git a/src/update/lib/history.js b/src/update/lib/history/history.js similarity index 97% rename from src/update/lib/history.js rename to src/update/lib/history/history.js index 8e2060fb..2a64a54f 100644 --- a/src/update/lib/history.js +++ b/src/update/lib/history/history.js @@ -9,16 +9,16 @@ const config = require('config'); const zlib = require('zlib'); const { format } = require('date-fns'); -const logger = require('./logger'); -const unpaywallEnrichedMapping = require('../mapping/unpaywall.json'); -const unpaywallHistoryMapping = require('../mapping/unpaywall_history.json'); +const logger = require('../logger'); +const unpaywallEnrichedMapping = require('../../mapping/unpaywall.json'); +const unpaywallHistoryMapping = require('../../mapping/unpaywall_history.json'); const { addStepInsert, getLatestStep, updateLatestStep, fail, -} = require('./models/state'); +} = require('../state'); const { refreshIndex, @@ -26,11 +26,11 @@ const { bulk, createIndex, searchWithRange, -} = require('./services/elastic'); +} = require('../services/elastic'); const maxBulkSize = config.get('elasticsearch.maxBulkSize'); -const snapshotsDir = path.resolve(__dirname, '..', 'data', 'snapshots'); +const snapshotsDir = path.resolve(__dirname, '..', '..', 'data', 'snapshots'); /** * Insert data on elastic with elastic bulk request. @@ -103,7 +103,6 @@ async function insertData(listOfDoi, newData, date) { newData.forEach((data) => { const copyData = data; copyData.referencedAt = date; - // classic insertion // TODO not hardcode resData.push({ index: { _index: 'unpaywall_enriched', _id: data.doi } }); resData.push(copyData); @@ -139,7 +138,6 @@ async function insertData(listOfDoi, newData, date) { copyData.referencedAt = oldDataUnpaywall.referencedAt; } - // classic insertion // TODO not hardcode resData.push({ index: { _index: 'unpaywall_enriched', _id: data.doi } }); resData.push(copyData); diff --git a/src/update/lib/history/job.js b/src/update/lib/history/job.js new file mode 100644 index 00000000..56af843e --- /dev/null +++ b/src/update/lib/history/job.js @@ -0,0 +1,92 @@ +/* eslint-disable no-param-reassign */ + +const logger = require('../logger'); + +const { + endState, + fail, +} = require('../state'); + +const { + setInUpdate, +} = require('../status'); + +const { + downloadChangefile, +} = require('../download'); + +const { + createState, + addStepGetChangefiles, + updateLatestStep, + getLatestStep, +} = require('../state'); + +const { insertHistoryDataUnpaywall } = require('./history'); + +const { + getChangefiles, +} = require('../services/unpaywall'); + +const { + sendMailNoChangefile, +} = require('../services/mail'); + +/** + * Download and insert on elastic the changefiles from unpaywall between a period with history. + * + * @param {Object} jobConfig - Config of job. + * @param {string} jobConfig.index - Name of the index to which the data will be inserted. + * @param {string} jobConfig.interval - Interval of changefile, day or week are available. + * @param {string} jobConfig.startDate - Start date for the changefile period. + * @param {string} jobConfig.endDate - End date for the changefile period. + * @param {number} jobConfig.offset - Line of the snapshot at which the data insertion starts. + * @param {number} jobConfig.limit - Line in the file where the insertion stops. + * + * @returns {Promise} + */ +async function insertWithOaHistoryJob(jobConfig) { + setInUpdate(true); + const { + interval, startDate, endDate, + } = jobConfig; + await createState(); + const start = new Date(); + addStepGetChangefiles(); + const step = getLatestStep(); + const changefilesInfo = await getChangefiles(interval, startDate, endDate); + + if (!changefilesInfo) { + step.status = 'error'; + updateLatestStep(step); + await fail(); + return; + } + + step.took = (new Date() - start) / 1000; + step.status = 'success'; + updateLatestStep(step); + + if (changefilesInfo.length === 0) { + sendMailNoChangefile(startDate, endDate).catch((err) => { + logger.errorRequest(err); + }); + await endState(); + return; + } + + let success = true; + for (let i = 0; i < changefilesInfo.length; i += 1) { + success = await downloadChangefile(changefilesInfo[i], interval); + if (!success) return; + jobConfig.filename = changefilesInfo[i].filename; + jobConfig.date = changefilesInfo[i].date; + success = await insertHistoryDataUnpaywall(jobConfig); + if (!success) return; + } + await endState(); +} + +module.exports = { + insertWithOaHistoryJob, +}; diff --git a/src/update/lib/history/report.js b/src/update/lib/history/report.js new file mode 100644 index 00000000..866e4e15 --- /dev/null +++ b/src/update/lib/history/report.js @@ -0,0 +1,52 @@ +const path = require('path'); +const fs = require('fs-extra'); +const { format } = require('date-fns'); +const logger = require('../logger'); + +const reportsDir = path.resolve(__dirname, '..', '..', 'data', 'reports'); + +/** + * Create report on the folder "data/update/report" on behalf of the date of treatment. + * + * @param {string} state - State filename. + * + * @returns {Promise} + */ +async function createReport(state) { + const pathfile = path.resolve(reportsDir, `${format(new Date(), 'yyyy-MM-dd HH:mm:ss')}.json`); + try { + await fs.writeFile(pathfile, JSON.stringify(state, null, 2)); + } catch (err) { + logger.error(`[report] Cannot write [${JSON.stringify(state, null, 2)}] in ${pathfile}`, err); + throw err; + } +} + +/** + * Get report from the folder "data/update/report". + * + * @param {string} filename - Report filename. + * + * @returns {Promise} Report in json format. + */ +async function getReport(filename) { + let report; + try { + report = await fs.readFile(path.resolve(reportsDir, filename)); + } catch (err) { + logger.error(`[report] Cannot read [${path.resolve(reportsDir, filename)}]`, err); + return undefined; + } + try { + report = JSON.parse(report); + } catch (err) { + logger.error(`[report] Cannot parse [${report}] at json format`, err); + return undefined; + } + return report; +} + +module.exports = { + createReport, + getReport, +}; diff --git a/src/update/lib/routers/history/job.js b/src/update/lib/history/routers/job.js similarity index 96% rename from src/update/lib/routers/history/job.js rename to src/update/lib/history/routers/job.js index c54798cb..451ed7da 100644 --- a/src/update/lib/routers/history/job.js +++ b/src/update/lib/history/routers/job.js @@ -3,13 +3,13 @@ const router = require('express').Router(); const { insertWithOaHistory, historyRollBack, -} = require('../../controllers/job'); +} = require('../controllers/job'); const { step1, step2, step3, -} = require('../../history'); +} = require('../history'); const { validateHistoryJob, diff --git a/src/update/lib/lab/history-v1.js b/src/update/lib/lab/history-v1.js index 6cd71b48..d48b709f 100644 --- a/src/update/lib/lab/history-v1.js +++ b/src/update/lib/lab/history-v1.js @@ -17,7 +17,7 @@ const { getLatestStep, updateLatestStep, fail, -} = require('../models/state'); +} = require('../state'); const { refreshIndex, diff --git a/src/update/lib/models/cron.js b/src/update/lib/models/cron.js deleted file mode 100644 index 70436084..00000000 --- a/src/update/lib/models/cron.js +++ /dev/null @@ -1,96 +0,0 @@ -const { CronJob } = require('cron'); - -const logger = require('../logger'); - -class Cron { - /** - * @constructor - * - * @param {string} name - Name of cron. - * @param {string} schedule - Schedule of cron. - * @param {function} task - Function that will be executed by the cron. - * @param {boolean} active - Indicates whether it is active or not. - */ - constructor(name, schedule, task, active) { - this.name = name; - this.schedule = schedule; - this.task = task; - this.active = active; - this.process = new CronJob(schedule, this.task, null, false, 'Europe/Paris'); - if (active) { - this.start(); - } - } - - /** - * Getter of config of cron. - * - * @returns {Object} config of cron. - */ - getConfig() { - return { - name: this.name, - schedule: this.schedule, - active: this.active, - }; - } - - /** - * Set new task for cron. - * - * @param {function} task - Function that will be executed by the cron. - */ - setTask(task) { - this.process.stop(); - this.task = task; - logger.info(`[cron: ${this.name}] config - task updated`); - this.process = new CronJob(this.schedule, this.task, null, false, 'Europe/Paris'); - if (this.active) this.process.start(); - } - - /** - * Set new schedule for cron. - * - * @param {string} schedule - Schedule of cron. - */ - setSchedule(schedule) { - this.process.stop(); - this.schedule = schedule; - logger.info(`[cron: ${this.name}] config - schedule is updated [${this.schedule}]`); - this.process = new CronJob(this.schedule, async () => { - await this.task(); - }, null, false, 'Europe/Paris'); - if (this.active) this.process.start(); - } - - /** - * Make active to true. - */ - start() { - try { - this.process.start(); - logger.info(`[cron: ${this.name}] - started`); - logger.info(`[cron: ${this.name}] config - schedule: [${this.schedule}]`); - } catch (err) { - logger.error(`[cron ${this.name}] - error in start`, err); - return; - } - this.active = true; - } - - /** - * Make active to false. - */ - stop() { - try { - this.process.stop(); - logger.info(`[cron: ${this.name}] - stopped`); - } catch (err) { - logger.error(`[cron ${this.name}] - error in stop`, err); - return; - } - this.active = false; - } -} - -module.exports = Cron; diff --git a/src/update/lib/models/state.js b/src/update/lib/state.js similarity index 96% rename from src/update/lib/models/state.js rename to src/update/lib/state.js index 5326aa1e..47fc2dcb 100644 --- a/src/update/lib/models/state.js +++ b/src/update/lib/state.js @@ -1,17 +1,17 @@ /* eslint-disable no-restricted-syntax */ const { createReport, -} = require('../report'); +} = require('./history/report'); const { sendMailUpdateReport, -} = require('../services/mail'); +} = require('./services/mail'); const { setInUpdate, -} = require('../status'); +} = require('./status'); -const logger = require('../logger'); +const logger = require('./logger'); let state = {}; diff --git a/src/update/test/cron.js b/src/update/test/classic/cron.js similarity index 98% rename from src/update/test/cron.js rename to src/update/test/classic/cron.js index a785c27e..97c68251 100644 --- a/src/update/test/cron.js +++ b/src/update/test/classic/cron.js @@ -3,9 +3,9 @@ const { expect } = require('chai'); const chai = require('chai'); const chaiHttp = require('chai-http'); -const reset = require('./utils/reset'); +const reset = require('../utils/reset'); -const ping = require('./utils/ping'); +const ping = require('../utils/ping'); chai.use(chaiHttp); diff --git a/src/update/test/insertionError.js b/src/update/test/classic/insertionError.js similarity index 93% rename from src/update/test/insertionError.js rename to src/update/test/classic/insertionError.js index 275d00cc..504d49fe 100644 --- a/src/update/test/insertionError.js +++ b/src/update/test/classic/insertionError.js @@ -5,26 +5,26 @@ const chaiHttp = require('chai-http'); const { countDocuments, -} = require('./utils/elastic'); +} = require('../utils/elastic'); const { addSnapshot, updateChangeFile, -} = require('./utils/snapshot'); +} = require('../utils/snapshot'); const { getState, -} = require('./utils/state'); +} = require('../utils/state'); const { getReport, -} = require('./utils/report'); +} = require('../utils/report'); -const checkIfInUpdate = require('./utils/status'); +const checkIfInUpdate = require('../utils/status'); -const ping = require('./utils/ping'); +const ping = require('../utils/ping'); -const reset = require('./utils/reset'); +const reset = require('../utils/reset'); chai.use(chaiHttp); diff --git a/src/update/test/insertionFile.js b/src/update/test/classic/insertionFile.js similarity index 98% rename from src/update/test/insertionFile.js rename to src/update/test/classic/insertionFile.js index deb6bf0c..655b6c4e 100644 --- a/src/update/test/insertionFile.js +++ b/src/update/test/classic/insertionFile.js @@ -5,25 +5,25 @@ const chaiHttp = require('chai-http'); const { countDocuments, -} = require('./utils/elastic'); +} = require('../utils/elastic'); const { addSnapshot, -} = require('./utils/snapshot'); +} = require('../utils/snapshot'); const { getState, -} = require('./utils/state'); +} = require('../utils/state'); const { getReport, -} = require('./utils/report'); +} = require('../utils/report'); -const checkIfInUpdate = require('./utils/status'); +const checkIfInUpdate = require('../utils/status'); -const ping = require('./utils/ping'); +const ping = require('../utils/ping'); -const reset = require('./utils/reset'); +const reset = require('../utils/reset'); chai.use(chaiHttp); diff --git a/src/update/test/insertionPeriodDay.js b/src/update/test/classic/insertionPeriodDay.js similarity index 98% rename from src/update/test/insertionPeriodDay.js rename to src/update/test/classic/insertionPeriodDay.js index 397809ff..86408cf0 100644 --- a/src/update/test/insertionPeriodDay.js +++ b/src/update/test/classic/insertionPeriodDay.js @@ -6,26 +6,26 @@ const chaiHttp = require('chai-http'); const { deleteFile, updateChangeFile, -} = require('./utils/snapshot'); +} = require('../utils/snapshot'); const { getState, -} = require('./utils/state'); +} = require('../utils/state'); const { getReport, -} = require('./utils/report'); +} = require('../utils/report'); const { countDocuments, deleteIndex, -} = require('./utils/elastic'); +} = require('../utils/elastic'); -const checkIfInUpdate = require('./utils/status'); +const checkIfInUpdate = require('../utils/status'); -const ping = require('./utils/ping'); +const ping = require('../utils/ping'); -const reset = require('./utils/reset'); +const reset = require('../utils/reset'); const updateURL = process.env.UPDATE_HOST || 'http://localhost:59702'; diff --git a/src/update/test/insertionPeriodWeek.js b/src/update/test/classic/insertionPeriodWeek.js similarity index 98% rename from src/update/test/insertionPeriodWeek.js rename to src/update/test/classic/insertionPeriodWeek.js index 4d8c54ae..7c54dd94 100644 --- a/src/update/test/insertionPeriodWeek.js +++ b/src/update/test/classic/insertionPeriodWeek.js @@ -5,25 +5,25 @@ const chaiHttp = require('chai-http'); const { updateChangeFile, -} = require('./utils/snapshot'); +} = require('../utils/snapshot'); const { getState, -} = require('./utils/state'); +} = require('../utils/state'); const { getReport, -} = require('./utils/report'); +} = require('../utils/report'); const { countDocuments, -} = require('./utils/elastic'); +} = require('../utils/elastic'); -const checkIfInUpdate = require('./utils/status'); +const checkIfInUpdate = require('../utils/status'); -const ping = require('./utils/ping'); +const ping = require('../utils/ping'); -const reset = require('./utils/reset'); +const reset = require('../utils/reset'); const updateURL = process.env.UPDATE_HOST || 'http://localhost:59702'; diff --git a/src/update/test/insertionSnapshot.js b/src/update/test/classic/insertionSnapshot.js similarity index 95% rename from src/update/test/insertionSnapshot.js rename to src/update/test/classic/insertionSnapshot.js index 5118903a..dad864c2 100644 --- a/src/update/test/insertionSnapshot.js +++ b/src/update/test/classic/insertionSnapshot.js @@ -6,21 +6,21 @@ const { format } = require('date-fns'); const { countDocuments, -} = require('./utils/elastic'); +} = require('../utils/elastic'); const { getState, -} = require('./utils/state'); +} = require('../utils/state'); const { getReport, -} = require('./utils/report'); +} = require('../utils/report'); -const checkIfInUpdate = require('./utils/status'); +const checkIfInUpdate = require('../utils/status'); -const ping = require('./utils/ping'); +const ping = require('../utils/ping'); -const reset = require('./utils/reset'); +const reset = require('../utils/reset'); chai.use(chaiHttp); diff --git a/src/update/test/updateDay.js b/src/update/test/classic/updateDay.js similarity index 98% rename from src/update/test/updateDay.js rename to src/update/test/classic/updateDay.js index 14b000b0..b42f5199 100644 --- a/src/update/test/updateDay.js +++ b/src/update/test/classic/updateDay.js @@ -5,26 +5,26 @@ const chaiHttp = require('chai-http'); const { countDocuments, -} = require('./utils/elastic'); +} = require('../utils/elastic'); const { addSnapshot, updateChangeFile, -} = require('./utils/snapshot'); +} = require('../utils/snapshot'); const { getState, -} = require('./utils/state'); +} = require('../utils/state'); const { getReport, -} = require('./utils/report'); +} = require('../utils/report'); -const checkIfInUpdate = require('./utils/status'); +const checkIfInUpdate = require('../utils/status'); -const ping = require('./utils/ping'); +const ping = require('../utils/ping'); -const reset = require('./utils/reset'); +const reset = require('../utils/reset'); chai.use(chaiHttp); diff --git a/src/update/test/updateWeek.js b/src/update/test/classic/updateWeek.js similarity index 97% rename from src/update/test/updateWeek.js rename to src/update/test/classic/updateWeek.js index d99795a4..ebe057fe 100644 --- a/src/update/test/updateWeek.js +++ b/src/update/test/classic/updateWeek.js @@ -5,26 +5,26 @@ const chaiHttp = require('chai-http'); const { countDocuments, -} = require('./utils/elastic'); +} = require('../utils/elastic'); const { addSnapshot, updateChangeFile, -} = require('./utils/snapshot'); +} = require('../utils/snapshot'); const { getState, -} = require('./utils/state'); +} = require('../utils/state'); const { getReport, -} = require('./utils/report'); +} = require('../utils/report'); -const checkIfInUpdate = require('./utils/status'); +const checkIfInUpdate = require('../utils/status'); -const ping = require('./utils/ping'); +const ping = require('../utils/ping'); -const reset = require('./utils/reset'); +const reset = require('../utils/reset'); chai.use(chaiHttp); diff --git a/src/update/test/history.js b/src/update/test/history/history.js similarity index 96% rename from src/update/test/history.js rename to src/update/test/history/history.js index c338e78d..70bb5221 100644 --- a/src/update/test/history.js +++ b/src/update/test/history/history.js @@ -5,21 +5,21 @@ const chaiHttp = require('chai-http'); const { updateChangeFile, -} = require('./utils/snapshot'); +} = require('../utils/snapshot'); const { countDocuments, -} = require('./utils/elastic'); +} = require('../utils/elastic'); const { getState, -} = require('./utils/state'); +} = require('../utils/state'); -const checkIfInUpdate = require('./utils/status'); +const checkIfInUpdate = require('../utils/status'); -const ping = require('./utils/ping'); +const ping = require('../utils/ping'); -const reset = require('./utils/reset'); +const reset = require('../utils/reset'); const updateURL = process.env.UPDATE_HOST || 'http://localhost:59702'; @@ -43,7 +43,7 @@ describe('Test: daily update route test with history', () => { // test response it('Should return a status code 202', async () => { const res = await chai.request(updateURL) - .post('/job/history') + .post('/history/job') .send({ startDate: date3, endDate: date1, diff --git a/src/update/test/rollback.js b/src/update/test/history/rollback.js similarity index 84% rename from src/update/test/rollback.js rename to src/update/test/history/rollback.js index db30d0ed..685e2dc0 100644 --- a/src/update/test/rollback.js +++ b/src/update/test/history/rollback.js @@ -2,8 +2,8 @@ const { expect } = require('chai'); const chai = require('chai'); const chaiHttp = require('chai-http'); -const unpaywallEnrichedMapping = require('../mapping/unpaywall.json'); -const unpaywallHistoryMapping = require('../mapping/unpaywall_history.json'); +const unpaywallEnrichedMapping = require('../../mapping/unpaywall.json'); +const unpaywallHistoryMapping = require('../../mapping/unpaywall_history.json'); const { countDocuments, @@ -11,7 +11,7 @@ const { deleteIndex, getAllData, createIndex, -} = require('./utils/elastic'); +} = require('../utils/elastic'); const updateURL = process.env.UPDATE_HOST || 'http://localhost:59702'; @@ -34,7 +34,29 @@ describe('Test: rollback history test', () => { it('Should return a status code 202', async () => { const res = await chai.request(updateURL) - .post('/job/history/reset') + .post('/history/job/reset') + .send({ + startDate: date1, + }) + .set('x-api-key', 'changeme'); + + expect(res).have.status(202); + }); + + it('Should get state with all information', async () => { + const res = await chai.request(updateURL) + .post('/history/job/reset') + .send({ + startDate: date1, + }) + .set('x-api-key', 'changeme'); + + expect(res).have.status(202); + }); + + it('Should get report with all information', async () => { + const res = await chai.request(updateURL) + .post('/history/job/reset') .send({ startDate: date1, }) @@ -74,7 +96,7 @@ describe('Test: rollback history test', () => { it('Should return a status code 202', async () => { const res = await chai.request(updateURL) - .post('/job/history/reset') + .post('/history/job/reset') .send({ startDate: date2, }) @@ -114,7 +136,7 @@ describe('Test: rollback history test', () => { it('Should return a status code 202', async () => { const res = await chai.request(updateURL) - .post('/job/history/reset') + .post('/history/job/reset') .send({ startDate: date3, }) @@ -154,7 +176,7 @@ describe('Test: rollback history test', () => { it('Should return a status code 202', async () => { const res = await chai.request(updateURL) - .post('/job/history/reset') + .post('/history/job/reset') .send({ startDate: date4, }) From e4c36bf4b9694c71027ce793b268cb9b30a31b88 Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Fri, 5 Jan 2024 14:32:34 +0100 Subject: [PATCH 008/114] refactor: make independent unpaywallHistory and history --- src/update/app.js | 78 ++++++++----- src/update/config/default.json | 8 +- src/update/lib/classic/report.js | 52 --------- .../lib/{global => }/controllers/elastic.js | 4 +- .../lib/{global => }/controllers/health.js | 4 +- .../lib/{classic => }/controllers/state.js | 2 +- .../lib/{global => }/controllers/status.js | 2 +- .../lib/{global => }/controllers/unpaywall.js | 2 +- src/update/lib/download.js | 12 +- src/update/lib/file.js | 14 +++ src/update/lib/history/report.js | 52 --------- src/update/lib/lab/history-v1.js | 2 +- src/update/lib/middlewares/multer.js | 5 +- src/update/lib/path.js | 31 +++++ src/update/lib/report.js | 55 +++++++++ .../lib/{global => }/routers/elastic.js | 2 +- .../lib/{global => }/routers/openapi.js | 2 +- src/update/lib/{global => }/routers/ping.js | 0 src/update/lib/{classic => }/routers/state.js | 0 src/update/lib/{global => }/routers/status.js | 2 +- .../lib/{global => }/routers/unpaywall.js | 2 +- src/update/lib/state.js | 5 +- .../controllers/cron.js | 0 .../{global => unpaywall}/controllers/file.js | 4 +- .../{classic => unpaywall}/controllers/job.js | 3 +- .../controllers/report.js | 10 +- src/update/lib/{classic => unpaywall}/cron.js | 0 .../lib/{classic => unpaywall}/cron/file.js | 11 +- .../lib/{classic => unpaywall}/cron/update.js | 12 +- .../lib/{classic => unpaywall}/insert.js | 5 +- src/update/lib/{classic => unpaywall}/job.js | 81 ++----------- .../{classic => unpaywall}/routers/cron.js | 0 .../lib/{classic => unpaywall}/routers/job.js | 0 .../{classic => unpaywall}/routers/report.js | 0 .../{global => unpaywall}/routers/snapshot.js | 0 .../controllers/job.js | 0 .../{history => unpaywallHistory}/history.js | 5 +- .../lib/{history => unpaywallHistory}/job.js | 27 +---- .../routers/job.js | 0 .../lib/unpaywallHistory/routers/reports.js | 0 .../test/{classic => unpaywall}/cron.js | 0 .../{classic => unpaywall}/insertionError.js | 0 .../{classic => unpaywall}/insertionFile.js | 0 .../insertionPeriodDay.js | 1 + .../insertionPeriodWeek.js | 0 .../insertionSnapshot.js | 0 .../test/{classic => unpaywall}/updateDay.js | 0 .../test/{classic => unpaywall}/updateWeek.js | 0 .../{history => unpaywallHistory}/history.js | 106 ++++++++++++++++-- .../{history => unpaywallHistory}/rollback.js | 0 50 files changed, 321 insertions(+), 280 deletions(-) delete mode 100644 src/update/lib/classic/report.js rename src/update/lib/{global => }/controllers/elastic.js (78%) rename src/update/lib/{global => }/controllers/health.js (91%) rename src/update/lib/{classic => }/controllers/state.js (91%) rename src/update/lib/{global => }/controllers/status.js (92%) rename src/update/lib/{global => }/controllers/unpaywall.js (93%) delete mode 100644 src/update/lib/history/report.js create mode 100644 src/update/lib/path.js create mode 100644 src/update/lib/report.js rename src/update/lib/{global => }/routers/elastic.js (83%) rename src/update/lib/{global => }/routers/openapi.js (79%) rename src/update/lib/{global => }/routers/ping.js (100%) rename src/update/lib/{classic => }/routers/state.js (100%) rename src/update/lib/{global => }/routers/status.js (87%) rename src/update/lib/{global => }/routers/unpaywall.js (85%) rename src/update/lib/{classic => unpaywall}/controllers/cron.js (100%) rename src/update/lib/{global => unpaywall}/controllers/file.js (95%) rename src/update/lib/{classic => unpaywall}/controllers/job.js (97%) rename src/update/lib/{classic => unpaywall}/controllers/report.js (88%) rename src/update/lib/{classic => unpaywall}/cron.js (100%) rename src/update/lib/{classic => unpaywall}/cron/file.js (66%) rename src/update/lib/{classic => unpaywall}/cron/update.js (88%) rename src/update/lib/{classic => unpaywall}/insert.js (97%) rename src/update/lib/{classic => unpaywall}/job.js (59%) rename src/update/lib/{classic => unpaywall}/routers/cron.js (100%) rename src/update/lib/{classic => unpaywall}/routers/job.js (100%) rename src/update/lib/{classic => unpaywall}/routers/report.js (100%) rename src/update/lib/{global => unpaywall}/routers/snapshot.js (100%) rename src/update/lib/{history => unpaywallHistory}/controllers/job.js (100%) rename src/update/lib/{history => unpaywallHistory}/history.js (98%) rename src/update/lib/{history => unpaywallHistory}/job.js (84%) rename src/update/lib/{history => unpaywallHistory}/routers/job.js (100%) create mode 100644 src/update/lib/unpaywallHistory/routers/reports.js rename src/update/test/{classic => unpaywall}/cron.js (100%) rename src/update/test/{classic => unpaywall}/insertionError.js (100%) rename src/update/test/{classic => unpaywall}/insertionFile.js (100%) rename src/update/test/{classic => unpaywall}/insertionPeriodDay.js (99%) rename src/update/test/{classic => unpaywall}/insertionPeriodWeek.js (100%) rename src/update/test/{classic => unpaywall}/insertionSnapshot.js (100%) rename src/update/test/{classic => unpaywall}/updateDay.js (100%) rename src/update/test/{classic => unpaywall}/updateWeek.js (100%) rename src/update/test/{history => unpaywallHistory}/history.js (55%) rename src/update/test/{history => unpaywallHistory}/rollback.js (100%) diff --git a/src/update/app.js b/src/update/app.js index 86b4d007..71e3388d 100644 --- a/src/update/app.js +++ b/src/update/app.js @@ -7,32 +7,58 @@ const morgan = require('./lib/morgan'); const logger = require('./lib/logger'); const getConfig = require('./lib/config'); -const routerGlobalPing = require('./lib/global/routers/ping'); -const routerGlobalElastic = require('./lib/global/routers/elastic'); -const routerGlobalOpenapi = require('./lib/global/routers/openapi'); -const routerGlobalUnpaywall = require('./lib/global/routers/unpaywall'); -const routerGlobalSnapshot = require('./lib/global/routers/snapshot'); -const routerGlobalStatus = require('./lib/global/routers/status'); +const routerPing = require('./lib/routers/ping'); +const routerElastic = require('./lib/routers/elastic'); +const routerOpenapi = require('./lib/routers/openapi'); +const routerUnpaywall = require('./lib/routers/unpaywall'); +const routerStatus = require('./lib/routers/status'); +const routerState = require('./lib/routers/state'); -const routerClassicJob = require('./lib/classic/routers/job'); -const routerClassicReport = require('./lib/classic/routers/report'); -const routerClassicState = require('./lib/classic/routers/state'); -const routerClassicCron = require('./lib/classic/routers/cron'); +const routerUnpaywallJob = require('./lib/unpaywall/routers/job'); +const routerUnpaywallReport = require('./lib/unpaywall/routers/report'); -const routerHistoryJob = require('./lib/history/routers/job'); +const routerUnpaywallCron = require('./lib/unpaywall/routers/cron'); +const routerUnpaywallSnapshot = require('./lib/unpaywall/routers/snapshot'); -require('./lib/classic/cron/file'); +const routerUnpaywallHistoryJob = require('./lib/unpaywallHistory/routers/job'); +require('./lib/unpaywall/cron/file'); + +// create data directory const dataDir = path.resolve(__dirname, 'data'); fs.ensureDir(path.resolve(dataDir)); -fs.ensureDir(path.resolve(dataDir, 'reports')); -fs.ensureDir(path.resolve(dataDir, 'states')); -fs.ensureDir(path.resolve(dataDir, 'snapshots')); +// create all directories for unpaywall +const unpaywallDir = path.resolve(dataDir, 'unpaywall'); + +fs.ensureDir(path.resolve(unpaywallDir)); + +const unpaywallReportsDir = path.resolve(unpaywallDir, 'reports'); +const unpaywallStatesDir = path.resolve(unpaywallDir, 'states'); +const unpaywallSnapshotsDir = path.resolve(unpaywallDir, 'snapshots'); + +fs.ensureDir(path.resolve(unpaywallReportsDir)); +fs.ensureDir(path.resolve(unpaywallStatesDir)); +fs.ensureDir(path.resolve(unpaywallSnapshotsDir)); + +// create all directories for history unpaywall +const unpaywallHistoryDir = path.resolve(dataDir, 'unpaywallHistory'); +fs.ensureDir(path.resolve(unpaywallHistoryDir)); + +const unpaywallHistoryReportsDir = path.resolve(unpaywallHistoryDir, 'reports'); +const unpaywallHistoryStatesDir = path.resolve(unpaywallHistoryDir, 'states'); +const unpaywallHistorySnapshotsDir = path.resolve(unpaywallHistoryDir, 'snapshots'); + +fs.ensureDir(path.resolve(unpaywallHistoryReportsDir)); +fs.ensureDir(path.resolve(unpaywallHistoryStatesDir)); +fs.ensureDir(path.resolve(unpaywallHistorySnapshotsDir)); + +// create all directories for logs unpaywall const logDir = path.resolve(__dirname, 'log'); fs.ensureDir(path.resolve(logDir)); fs.ensureDir(path.resolve(logDir, 'application')); fs.ensureDir(path.resolve(logDir, 'access')); +fs.ensureDir(path.resolve(logDir, 'healthcheck')); const app = express(); @@ -40,19 +66,19 @@ app.use(express.json()); app.use(cors()); app.use(morgan); -app.use(routerGlobalPing); -app.use(routerGlobalElastic); -app.use(routerGlobalOpenapi); -app.use(routerGlobalUnpaywall); -app.use(routerGlobalSnapshot); -app.use(routerGlobalStatus); +app.use(routerPing); +app.use(routerElastic); +app.use(routerOpenapi); +app.use(routerUnpaywall); +app.use(routerStatus); +app.use(routerState); -app.use(routerClassicJob); -app.use(routerClassicReport); -app.use(routerClassicState); -app.use(routerClassicCron); +app.use(routerUnpaywallJob); +app.use(routerUnpaywallReport); +app.use(routerUnpaywallCron); +app.use(routerUnpaywallSnapshot); -app.use(routerHistoryJob); +app.use(routerUnpaywallHistoryJob); /* Errors and unknown routes */ app.use((req, res, next) => res.status(404).json({ message: `Cannot ${req.method} ${req.originalUrl}` })); diff --git a/src/update/config/default.json b/src/update/config/default.json index 93d3872a..6e056e3c 100644 --- a/src/update/config/default.json +++ b/src/update/config/default.json @@ -18,7 +18,13 @@ "indexAlias": "upw", "timeout": 20000 }, - "cron": { + "unpaywallCron": { + "schedule": "0 0 0 * * *", + "active": false, + "index": "unpaywall", + "interval": "day" + }, + "unpaywallHistoryCron": { "schedule": "0 0 0 * * *", "active": false, "index": "unpaywall", diff --git a/src/update/lib/classic/report.js b/src/update/lib/classic/report.js deleted file mode 100644 index 866e4e15..00000000 --- a/src/update/lib/classic/report.js +++ /dev/null @@ -1,52 +0,0 @@ -const path = require('path'); -const fs = require('fs-extra'); -const { format } = require('date-fns'); -const logger = require('../logger'); - -const reportsDir = path.resolve(__dirname, '..', '..', 'data', 'reports'); - -/** - * Create report on the folder "data/update/report" on behalf of the date of treatment. - * - * @param {string} state - State filename. - * - * @returns {Promise} - */ -async function createReport(state) { - const pathfile = path.resolve(reportsDir, `${format(new Date(), 'yyyy-MM-dd HH:mm:ss')}.json`); - try { - await fs.writeFile(pathfile, JSON.stringify(state, null, 2)); - } catch (err) { - logger.error(`[report] Cannot write [${JSON.stringify(state, null, 2)}] in ${pathfile}`, err); - throw err; - } -} - -/** - * Get report from the folder "data/update/report". - * - * @param {string} filename - Report filename. - * - * @returns {Promise} Report in json format. - */ -async function getReport(filename) { - let report; - try { - report = await fs.readFile(path.resolve(reportsDir, filename)); - } catch (err) { - logger.error(`[report] Cannot read [${path.resolve(reportsDir, filename)}]`, err); - return undefined; - } - try { - report = JSON.parse(report); - } catch (err) { - logger.error(`[report] Cannot parse [${report}] at json format`, err); - return undefined; - } - return report; -} - -module.exports = { - createReport, - getReport, -}; diff --git a/src/update/lib/global/controllers/elastic.js b/src/update/lib/controllers/elastic.js similarity index 78% rename from src/update/lib/global/controllers/elastic.js rename to src/update/lib/controllers/elastic.js index 690dc361..2a804b4c 100644 --- a/src/update/lib/global/controllers/elastic.js +++ b/src/update/lib/controllers/elastic.js @@ -1,5 +1,5 @@ -const { initAlias } = require('../../services/elastic'); -const unpaywallMapping = require('../../../mapping/unpaywall.json'); +const { initAlias } = require('../services/elastic'); +const unpaywallMapping = require('../../mapping/unpaywall.json'); /** * Controller to create alias unpaywall on elastic. diff --git a/src/update/lib/global/controllers/health.js b/src/update/lib/controllers/health.js similarity index 91% rename from src/update/lib/global/controllers/health.js rename to src/update/lib/controllers/health.js index bba58193..1960ff3f 100644 --- a/src/update/lib/global/controllers/health.js +++ b/src/update/lib/controllers/health.js @@ -1,5 +1,5 @@ -const promiseWithTimeout = require('../../ping'); -const { pingElastic } = require('../../services/elastic'); +const promiseWithTimeout = require('../ping'); +const { pingElastic } = require('../services/elastic'); /** * Controller to get health of all services connected to update service. diff --git a/src/update/lib/classic/controllers/state.js b/src/update/lib/controllers/state.js similarity index 91% rename from src/update/lib/classic/controllers/state.js rename to src/update/lib/controllers/state.js index e854c93a..fbe7d41a 100644 --- a/src/update/lib/classic/controllers/state.js +++ b/src/update/lib/controllers/state.js @@ -1,4 +1,4 @@ -const { getState } = require('../../state'); +const { getState } = require('../state'); /** * Controller to get the state of update. diff --git a/src/update/lib/global/controllers/status.js b/src/update/lib/controllers/status.js similarity index 92% rename from src/update/lib/global/controllers/status.js rename to src/update/lib/controllers/status.js index a99e4794..d129db5b 100644 --- a/src/update/lib/global/controllers/status.js +++ b/src/update/lib/controllers/status.js @@ -1,4 +1,4 @@ -const { getStatus, setInUpdate } = require('../../status'); +const { getStatus, setInUpdate } = require('../status'); /** * Controller to get status of update. diff --git a/src/update/lib/global/controllers/unpaywall.js b/src/update/lib/controllers/unpaywall.js similarity index 93% rename from src/update/lib/global/controllers/unpaywall.js rename to src/update/lib/controllers/unpaywall.js index ab1e7c57..11357e92 100644 --- a/src/update/lib/global/controllers/unpaywall.js +++ b/src/update/lib/controllers/unpaywall.js @@ -1,4 +1,4 @@ -const { getChangefiles } = require('../../services/unpaywall'); +const { getChangefiles } = require('../services/unpaywall'); /** * Controller to get list of changefiles on unpaywall. diff --git a/src/update/lib/download.js b/src/update/lib/download.js index efe41455..94c16468 100644 --- a/src/update/lib/download.js +++ b/src/update/lib/download.js @@ -18,7 +18,8 @@ const { getChangefile, } = require('./services/unpaywall'); -const snapshotsDir = path.resolve(__dirname, '..', 'data', 'snapshots'); +const { getPathOfDirectory } = require('./file'); +const pathDir = require('./path'); /** * Update the step the percentage in download regularly until the download is complete. @@ -100,15 +101,18 @@ async function download(file, filepath, size) { /** * Start the download of the changefile from unpaywall. * + * @param {string} type - Information of the file to download. * @param {string} info - Information of the file to download. * @param {string} interval - Type of changefile (day or week). * * @returns {Promise} success or not */ -async function downloadChangefile(info, interval) { +async function downloadChangefile(type, info, interval) { let stats; - const filepath = path.resolve(snapshotsDir, info.filename); + const pathOfDirectory = getPathOfDirectory(type, 'snapshots'); + + const filepath = path.resolve(pathOfDirectory, info.filename); const alreadyInstalled = await fs.pathExists(filepath); if (alreadyInstalled) stats = await fs.stat(filepath); @@ -145,7 +149,7 @@ async function downloadChangefile(info, interval) { */ async function downloadBigSnapshot() { const filename = `snapshot-${format(new Date(), 'yyyy-MM-dd')}.jsonl.gz`; - const filepath = path.resolve(snapshotsDir, filename); + const filepath = path.resolve(pathDir.unpaywall.snapshotsDir, filename); addStepDownload(); const step = getLatestStep(); diff --git a/src/update/lib/file.js b/src/update/lib/file.js index 471a3bae..65b591f3 100644 --- a/src/update/lib/file.js +++ b/src/update/lib/file.js @@ -2,6 +2,19 @@ const fs = require('fs-extra'); const path = require('path'); const logger = require('./logger'); +const dirPath = require('./path'); + +/** + * Get path of directory depending on type. + * + * @param {string} process - Type of process (unpaywall or unpaywallHistory). + * @param {string} type - Type of file (states, reports, snapshots). + * + * @returns {string} path of directory. + */ +function getPathOfDirectory(process, type) { + return dirPath[process][`${type}Dir`]; +} /** * Get the files in a directory in order by date. @@ -85,6 +98,7 @@ async function deleteFile(filepath) { } module.exports = { + getPathOfDirectory, getMostRecentFile, deleteFilesInDir, deleteFile, diff --git a/src/update/lib/history/report.js b/src/update/lib/history/report.js deleted file mode 100644 index 866e4e15..00000000 --- a/src/update/lib/history/report.js +++ /dev/null @@ -1,52 +0,0 @@ -const path = require('path'); -const fs = require('fs-extra'); -const { format } = require('date-fns'); -const logger = require('../logger'); - -const reportsDir = path.resolve(__dirname, '..', '..', 'data', 'reports'); - -/** - * Create report on the folder "data/update/report" on behalf of the date of treatment. - * - * @param {string} state - State filename. - * - * @returns {Promise} - */ -async function createReport(state) { - const pathfile = path.resolve(reportsDir, `${format(new Date(), 'yyyy-MM-dd HH:mm:ss')}.json`); - try { - await fs.writeFile(pathfile, JSON.stringify(state, null, 2)); - } catch (err) { - logger.error(`[report] Cannot write [${JSON.stringify(state, null, 2)}] in ${pathfile}`, err); - throw err; - } -} - -/** - * Get report from the folder "data/update/report". - * - * @param {string} filename - Report filename. - * - * @returns {Promise} Report in json format. - */ -async function getReport(filename) { - let report; - try { - report = await fs.readFile(path.resolve(reportsDir, filename)); - } catch (err) { - logger.error(`[report] Cannot read [${path.resolve(reportsDir, filename)}]`, err); - return undefined; - } - try { - report = JSON.parse(report); - } catch (err) { - logger.error(`[report] Cannot parse [${report}] at json format`, err); - return undefined; - } - return report; -} - -module.exports = { - createReport, - getReport, -}; diff --git a/src/update/lib/lab/history-v1.js b/src/update/lib/lab/history-v1.js index d48b709f..b317f77a 100644 --- a/src/update/lib/lab/history-v1.js +++ b/src/update/lib/lab/history-v1.js @@ -240,7 +240,7 @@ async function insertHistoryDataUnpaywall(insertConfig) { try { bytes = await fs.stat(filePath); } catch (err) { - logger.error(`[job: insert] Cannot stat [${filePath}]`, err); + logger.error(`[job: insert] Cannot get bytes [${filePath}]`, err); await fail(err); return false; } diff --git a/src/update/lib/middlewares/multer.js b/src/update/lib/middlewares/multer.js index 0a52fefe..fbb3eecd 100644 --- a/src/update/lib/middlewares/multer.js +++ b/src/update/lib/middlewares/multer.js @@ -1,11 +1,10 @@ -const path = require('path'); const multer = require('multer'); -const snapshotsDir = path.resolve(__dirname, '..', '..', 'data', 'snapshots'); +const dirPath = require('../path'); const storage = multer.diskStorage( { - destination: snapshotsDir, + destination: dirPath.unpaywall.snapshotsDir, filename: (req, file, cb) => { cb(null, file.originalname); }, diff --git a/src/update/lib/path.js b/src/update/lib/path.js new file mode 100644 index 00000000..f2c4a654 --- /dev/null +++ b/src/update/lib/path.js @@ -0,0 +1,31 @@ +const path = require('path'); + +const dataDir = path.resolve(__dirname, '..', 'data'); + +const unpaywallDir = path.resolve(dataDir, 'unpaywall'); + +const unpaywallReportsDir = path.resolve(unpaywallDir, 'reports'); +const unpaywallStatesDir = path.resolve(unpaywallDir, 'states'); +const unpaywallSnapshotsDir = path.resolve(unpaywallDir, 'snapshots'); + +const unpaywallHistoryDir = path.resolve(dataDir, 'unpaywallHistory'); + +const unpaywallHistoryReportsDir = path.resolve(unpaywallHistoryDir, 'reports'); +const unpaywallHistoryStatesDir = path.resolve(unpaywallHistoryDir, 'states'); +const unpaywallHistorySnapshotsDir = path.resolve(unpaywallHistoryDir, 'snapshots'); + +const dirPath = { + dataDir, + unpaywall: { + reportsDir: unpaywallReportsDir, + statesDir: unpaywallStatesDir, + snapshotsDir: unpaywallSnapshotsDir, + }, + unpaywallHistory: { + reportsDir: unpaywallHistoryReportsDir, + statesDir: unpaywallHistoryStatesDir, + snapshotsDir: unpaywallHistorySnapshotsDir, + }, +}; + +module.exports = dirPath; diff --git a/src/update/lib/report.js b/src/update/lib/report.js new file mode 100644 index 00000000..07a2a780 --- /dev/null +++ b/src/update/lib/report.js @@ -0,0 +1,55 @@ +const path = require('path'); +const fs = require('fs-extra'); +const { format } = require('date-fns'); +const logger = require('./logger'); +const { getPathOfDirectory } = require('./file'); + +/** + * Create report on the folder as name the date of process. + * + * @param {Object} state - State of process. + * + * @returns {Promise} + */ +async function createReport(state) { + const { type } = state; + const pathOfDirectory = getPathOfDirectory(type, 'reports'); + const filepath = path.resolve(pathOfDirectory, `${format(new Date(), 'yyyy-MM-dd HH:mm:ss')}.json`); + try { + await fs.writeFile(filepath, JSON.stringify(state, null, 2)); + } catch (err) { + logger.error(`[report] Cannot write [${JSON.stringify(state, null, 2)}] in ${filepath}`, err); + throw err; + } +} + +/** + * Get report from the folder data depending on type. + * + * @param {string} type - Type of report (unpaywall or unpaywallHistory). + * @param {string} filename - Report filename. + * + * @returns {Promise} Report in json format. + */ +async function getReport(type, filename) { + let report; + const pathOfDirectory = getPathOfDirectory(type, 'reports'); + try { + report = await fs.readFile(path.resolve(pathOfDirectory, filename)); + } catch (err) { + logger.error(`[report] Cannot read [${path.resolve(pathOfDirectory, filename)}]`, err); + return undefined; + } + try { + report = JSON.parse(report); + } catch (err) { + logger.error(`[report] Cannot parse [${report}] at json format`, err); + return undefined; + } + return report; +} + +module.exports = { + createReport, + getReport, +}; diff --git a/src/update/lib/global/routers/elastic.js b/src/update/lib/routers/elastic.js similarity index 83% rename from src/update/lib/global/routers/elastic.js rename to src/update/lib/routers/elastic.js index 8fbcfeea..c4219038 100644 --- a/src/update/lib/global/routers/elastic.js +++ b/src/update/lib/routers/elastic.js @@ -1,6 +1,6 @@ const router = require('express').Router(); -const checkAuth = require('../../middlewares/auth'); +const checkAuth = require('../middlewares/auth'); const createAlias = require('../controllers/elastic'); /** diff --git a/src/update/lib/global/routers/openapi.js b/src/update/lib/routers/openapi.js similarity index 79% rename from src/update/lib/global/routers/openapi.js rename to src/update/lib/routers/openapi.js index e80b5990..f68e1ca0 100644 --- a/src/update/lib/global/routers/openapi.js +++ b/src/update/lib/routers/openapi.js @@ -1,6 +1,6 @@ const router = require('express').Router(); -const openapi = require('../../../openapi.json'); +const openapi = require('../../openapi.json'); /** * Route that give the openapi.json file. diff --git a/src/update/lib/global/routers/ping.js b/src/update/lib/routers/ping.js similarity index 100% rename from src/update/lib/global/routers/ping.js rename to src/update/lib/routers/ping.js diff --git a/src/update/lib/classic/routers/state.js b/src/update/lib/routers/state.js similarity index 100% rename from src/update/lib/classic/routers/state.js rename to src/update/lib/routers/state.js diff --git a/src/update/lib/global/routers/status.js b/src/update/lib/routers/status.js similarity index 87% rename from src/update/lib/global/routers/status.js rename to src/update/lib/routers/status.js index a159b750..1e6b9aaa 100644 --- a/src/update/lib/global/routers/status.js +++ b/src/update/lib/routers/status.js @@ -1,6 +1,6 @@ const router = require('express').Router(); -const checkAuth = require('../../middlewares/auth'); +const checkAuth = require('../middlewares/auth'); const { getUpdateStatus, patchUpdateStatus } = require('../controllers/status'); /** diff --git a/src/update/lib/global/routers/unpaywall.js b/src/update/lib/routers/unpaywall.js similarity index 85% rename from src/update/lib/global/routers/unpaywall.js rename to src/update/lib/routers/unpaywall.js index e2a71e16..9f39047b 100644 --- a/src/update/lib/global/routers/unpaywall.js +++ b/src/update/lib/routers/unpaywall.js @@ -1,6 +1,6 @@ const router = require('express').Router(); -const validateIntervale = require('../../middlewares/interval'); +const validateIntervale = require('../middlewares/interval'); const getChangefilesOfUnpaywall = require('../controllers/unpaywall'); /** diff --git a/src/update/lib/state.js b/src/update/lib/state.js index 47fc2dcb..a883d918 100644 --- a/src/update/lib/state.js +++ b/src/update/lib/state.js @@ -1,7 +1,7 @@ /* eslint-disable no-restricted-syntax */ const { createReport, -} = require('./history/report'); +} = require('./report'); const { sendMailUpdateReport, @@ -28,13 +28,14 @@ function setState(key, value) { * * @return {Promise} name of the file where the state is saved. */ -async function createState() { +async function createState(type) { state = { done: false, createdAt: new Date(), endAt: null, steps: [], error: false, + type, }; } diff --git a/src/update/lib/classic/controllers/cron.js b/src/update/lib/unpaywall/controllers/cron.js similarity index 100% rename from src/update/lib/classic/controllers/cron.js rename to src/update/lib/unpaywall/controllers/cron.js diff --git a/src/update/lib/global/controllers/file.js b/src/update/lib/unpaywall/controllers/file.js similarity index 95% rename from src/update/lib/global/controllers/file.js rename to src/update/lib/unpaywall/controllers/file.js index f0edc732..9eb72c16 100644 --- a/src/update/lib/global/controllers/file.js +++ b/src/update/lib/unpaywall/controllers/file.js @@ -1,13 +1,15 @@ const fs = require('fs-extra'); const path = require('path'); -const snapshotsDir = path.resolve(__dirname, '..', '..', '..', 'data', 'snapshots'); +const dirPath = require('../../path'); const { getMostRecentFile, deleteFile, } = require('../../file'); +const { snapshotsDir } = dirPath.unpaywall; + /** * Controller to start list of files or latest file downloaded on update service. * diff --git a/src/update/lib/classic/controllers/job.js b/src/update/lib/unpaywall/controllers/job.js similarity index 97% rename from src/update/lib/classic/controllers/job.js rename to src/update/lib/unpaywall/controllers/job.js index 7e54f7b2..2c6353e3 100644 --- a/src/update/lib/classic/controllers/job.js +++ b/src/update/lib/unpaywall/controllers/job.js @@ -1,8 +1,9 @@ const fs = require('fs-extra'); const path = require('path'); const { format } = require('date-fns'); +const dirPath = require('../../path'); -const snapshotsDir = path.resolve(__dirname, '..', '..', '..', 'data', 'snapshots'); +const { snapshotsDir } = dirPath.unpaywall; const { downloadAndInsertSnapshot, diff --git a/src/update/lib/classic/controllers/report.js b/src/update/lib/unpaywall/controllers/report.js similarity index 88% rename from src/update/lib/classic/controllers/report.js rename to src/update/lib/unpaywall/controllers/report.js index 69198a0d..e12a1810 100644 --- a/src/update/lib/classic/controllers/report.js +++ b/src/update/lib/unpaywall/controllers/report.js @@ -2,10 +2,10 @@ const path = require('path'); const fs = require('fs-extra'); const { getMostRecentFile } = require('../../file'); -const { getReport } = require('../report'); - -const reportsDir = path.resolve(__dirname, '..', '..', '..', 'data', 'reports'); +const { getReport } = require('../../report'); +const dirPath = require('../../path'); +const { reportsDir } = dirPath.unpaywall; /** * Controller to get list of reports or latest report. * @@ -30,7 +30,7 @@ async function getReports(req, res, next) { let report; try { - report = await getReport(latestFile?.filename); + report = await getReport('unpaywall', latestFile?.filename); } catch (err) { return next({ message: `Cannot get [${latestFile?.filename}] latest report` }); } @@ -66,7 +66,7 @@ async function getReportByFilename(req, res, next) { let report; try { - report = await getReport(filename); + report = await getReport('unpaywall', filename); } catch (err) { return next({ message: `Cannot get ${filename} report` }); } diff --git a/src/update/lib/classic/cron.js b/src/update/lib/unpaywall/cron.js similarity index 100% rename from src/update/lib/classic/cron.js rename to src/update/lib/unpaywall/cron.js diff --git a/src/update/lib/classic/cron/file.js b/src/update/lib/unpaywall/cron/file.js similarity index 66% rename from src/update/lib/classic/cron/file.js rename to src/update/lib/unpaywall/cron/file.js index a3ed67d0..4c9fcec7 100644 --- a/src/update/lib/classic/cron/file.js +++ b/src/update/lib/unpaywall/cron/file.js @@ -1,12 +1,9 @@ -const path = require('path'); const logger = require('../../logger'); const Cron = require('../../cron'); - +const dirPath = require('../../path'); const { deleteFilesInDir } = require('../../file'); -const reportsDir = path.resolve(__dirname, '..', '..', '..', 'data', 'reports'); -const snapshotDir = path.resolve(__dirname, '..', '..', '..', 'data', 'snapshots'); -const states = path.resolve(__dirname, '..', '..', '..', 'data', 'states'); +const { reportsDir, snapshotsDir, statesDir } = dirPath; /** * Removes files generated by an update process that are older than 30 days. @@ -17,10 +14,10 @@ async function task() { const deletedReportFiles = await deleteFilesInDir(reportsDir, 30); logger.info(`[cron: delete out files] ${deletedReportFiles?.join(',')} (${deletedReportFiles.length}) reports are deleted`); - const deletedSnapshotFiles = await deleteFilesInDir(snapshotDir, 30); + const deletedSnapshotFiles = await deleteFilesInDir(snapshotsDir, 30); logger.info(`[cron: delete out files] ${deletedSnapshotFiles?.join(',')} (${deletedSnapshotFiles.length}) snapshots are deleted`); - const deletedStateFiles = await deleteFilesInDir(states, 30); + const deletedStateFiles = await deleteFilesInDir(statesDir, 30); logger.info(`[cron: delete out files] ${deletedStateFiles?.join(',')} (${deletedStateFiles.length}) states are deleted`); } diff --git a/src/update/lib/classic/cron/update.js b/src/update/lib/unpaywall/cron/update.js similarity index 88% rename from src/update/lib/classic/cron/update.js rename to src/update/lib/unpaywall/cron/update.js index 918d7964..bd961d63 100644 --- a/src/update/lib/classic/cron/update.js +++ b/src/update/lib/unpaywall/cron/update.js @@ -1,20 +1,20 @@ const { format, subDays } = require('date-fns'); -const { cron } = require('config'); +const { unpaywallCron } = require('config'); +const logger = require('../../logger'); const Cron = require('../../cron'); const { getStatus } = require('../../status'); const { insertChangefilesOnPeriod } = require('../job'); -const logger = require('../../logger'); -let { active } = cron; +let { active } = unpaywallCron; if (active === 'true' || active) active = true; else active = false; const updateConfig = { - index: cron.index, - interval: cron.interval, + index: unpaywallCron.index, + interval: unpaywallCron.interval, }; /** @@ -41,7 +41,7 @@ async function task() { }); } -const updateCron = new Cron('update', cron.schedule, task, active); +const updateCron = new Cron('update', unpaywallCron.schedule, task, active); /** * Update config of update process and config of cron. diff --git a/src/update/lib/classic/insert.js b/src/update/lib/unpaywall/insert.js similarity index 97% rename from src/update/lib/classic/insert.js rename to src/update/lib/unpaywall/insert.js index 682389ad..0b22540e 100644 --- a/src/update/lib/classic/insert.js +++ b/src/update/lib/unpaywall/insert.js @@ -8,6 +8,7 @@ const readline = require('readline'); const config = require('config'); const zlib = require('zlib'); +const dirPath = require('../path'); const logger = require('../logger'); const unpaywallMapping = require('../../mapping/unpaywall.json'); @@ -28,7 +29,7 @@ const { const indexAlias = config.get('elasticsearch.indexAlias'); const maxBulkSize = config.get('elasticsearch.maxBulkSize'); -const snapshotsDir = path.resolve(__dirname, '..', '..', 'data', 'snapshots'); +const { snapshotsDir } = dirPath.unpaywall; /** * Insert data on elastic with elastic bulk request. @@ -125,7 +126,7 @@ async function insertDataUnpaywall(insertConfig) { try { bytes = await fs.stat(filePath); } catch (err) { - logger.error(`[job: insert] Cannot stat [${filePath}]`, err); + logger.error(`[job: insert] Cannot get bytes [${filePath}]`, err); await fail(err); return false; } diff --git a/src/update/lib/classic/job.js b/src/update/lib/unpaywall/job.js similarity index 59% rename from src/update/lib/classic/job.js rename to src/update/lib/unpaywall/job.js index 0ea008ea..7818f361 100644 --- a/src/update/lib/classic/job.js +++ b/src/update/lib/unpaywall/job.js @@ -2,20 +2,13 @@ const logger = require('../logger'); -const { - endState, fail, -} = require('../state'); +const { setInUpdate } = require('../status'); -const { - setInUpdate, -} = require('../status'); - -const { - downloadBigSnapshot, - downloadChangefile, -} = require('../download'); +const { downloadBigSnapshot, downloadChangefile } = require('../download'); const { + endState, + fail, createState, addStepGetChangefiles, updateLatestStep, @@ -24,8 +17,6 @@ const { const insertDataUnpaywall = require('./insert'); -const { insertHistoryDataUnpaywall } = require('../history/history'); - const { getChangefiles } = require('../services/unpaywall'); const { sendMailNoChangefile } = require('../services/mail'); @@ -42,8 +33,8 @@ const { sendMailNoChangefile } = require('../services/mail'); */ async function downloadAndInsertSnapshot(jobConfig) { setInUpdate(true); - await createState(); - const filename = await downloadBigSnapshot(jobConfig); + await createState('unpaywall'); + const filename = await downloadBigSnapshot(); if (!filename) { await fail(); return; @@ -71,7 +62,7 @@ async function insertChangefilesOnPeriod(jobConfig) { const { interval, startDate, endDate, } = jobConfig; - await createState(); + await createState('unpaywall'); const start = new Date(); addStepGetChangefiles(); const step = getLatestStep(); @@ -98,7 +89,7 @@ async function insertChangefilesOnPeriod(jobConfig) { let success = true; for (let i = 0; i < changefilesInfo.length; i += 1) { - success = await downloadChangefile(changefilesInfo[i], interval); + success = await downloadChangefile('unpaywall', changefilesInfo[i], interval); if (!success) return; jobConfig.filename = changefilesInfo[i].filename; success = await insertDataUnpaywall(jobConfig); @@ -119,69 +110,15 @@ async function insertChangefilesOnPeriod(jobConfig) { */ async function insertChangefile(jobConfig) { setInUpdate(true); - await createState(); + await createState('unpaywall'); const success = await insertDataUnpaywall(jobConfig); if (success) { await endState(); } } -/** - * Download and insert on elastic the changefiles from unpaywall between a period with history. - * - * @param {Object} jobConfig - Config of job. - * @param {string} jobConfig.index - Name of the index to which the data will be inserted. - * @param {string} jobConfig.interval - Interval of changefile, day or week are available. - * @param {string} jobConfig.startDate - Start date for the changefile period. - * @param {string} jobConfig.endDate - End date for the changefile period. - * @param {number} jobConfig.offset - Line of the snapshot at which the data insertion starts. - * @param {number} jobConfig.limit - Line in the file where the insertion stops. - * - * @returns {Promise} - */ -async function insertWithOaHistoryJob(jobConfig) { - setInUpdate(true); - const { interval, startDate, endDate } = jobConfig; - await createState(); - const start = new Date(); - addStepGetChangefiles(); - const step = getLatestStep(); - const changefilesInfo = await getChangefiles(interval, startDate, endDate); - - if (!changefilesInfo) { - step.status = 'error'; - updateLatestStep(step); - await fail(); - return; - } - - step.took = (new Date() - start) / 1000; - step.status = 'success'; - updateLatestStep(step); - - if (changefilesInfo.length === 0) { - sendMailNoChangefile(startDate, endDate).catch((err) => { - logger.errorRequest(err); - }); - await endState(); - return; - } - - let success = true; - for (let i = 0; i < changefilesInfo.length; i += 1) { - success = await downloadChangefile(changefilesInfo[i], interval); - if (!success) return; - jobConfig.filename = changefilesInfo[i].filename; - jobConfig.date = changefilesInfo[i].date; - success = await insertHistoryDataUnpaywall(jobConfig); - if (!success) return; - } - await endState(); -} - module.exports = { downloadAndInsertSnapshot, insertChangefilesOnPeriod, - insertWithOaHistoryJob, insertChangefile, }; diff --git a/src/update/lib/classic/routers/cron.js b/src/update/lib/unpaywall/routers/cron.js similarity index 100% rename from src/update/lib/classic/routers/cron.js rename to src/update/lib/unpaywall/routers/cron.js diff --git a/src/update/lib/classic/routers/job.js b/src/update/lib/unpaywall/routers/job.js similarity index 100% rename from src/update/lib/classic/routers/job.js rename to src/update/lib/unpaywall/routers/job.js diff --git a/src/update/lib/classic/routers/report.js b/src/update/lib/unpaywall/routers/report.js similarity index 100% rename from src/update/lib/classic/routers/report.js rename to src/update/lib/unpaywall/routers/report.js diff --git a/src/update/lib/global/routers/snapshot.js b/src/update/lib/unpaywall/routers/snapshot.js similarity index 100% rename from src/update/lib/global/routers/snapshot.js rename to src/update/lib/unpaywall/routers/snapshot.js diff --git a/src/update/lib/history/controllers/job.js b/src/update/lib/unpaywallHistory/controllers/job.js similarity index 100% rename from src/update/lib/history/controllers/job.js rename to src/update/lib/unpaywallHistory/controllers/job.js diff --git a/src/update/lib/history/history.js b/src/update/lib/unpaywallHistory/history.js similarity index 98% rename from src/update/lib/history/history.js rename to src/update/lib/unpaywallHistory/history.js index 2a64a54f..1d477e0f 100644 --- a/src/update/lib/history/history.js +++ b/src/update/lib/unpaywallHistory/history.js @@ -9,6 +9,7 @@ const config = require('config'); const zlib = require('zlib'); const { format } = require('date-fns'); +const dirPath = require('../path'); const logger = require('../logger'); const unpaywallEnrichedMapping = require('../../mapping/unpaywall.json'); const unpaywallHistoryMapping = require('../../mapping/unpaywall_history.json'); @@ -30,7 +31,7 @@ const { const maxBulkSize = config.get('elasticsearch.maxBulkSize'); -const snapshotsDir = path.resolve(__dirname, '..', '..', 'data', 'snapshots'); +const { snapshotsDir } = dirPath.unpaywallHistory; /** * Insert data on elastic with elastic bulk request. @@ -217,7 +218,7 @@ async function insertHistoryDataUnpaywall(insertConfig) { try { bytes = await fs.stat(filePath); } catch (err) { - logger.error(`[job: insert] Cannot stat [${filePath}]`, err); + logger.error(`[job: insert] Cannot get bytes [${filePath}]`, err); await fail(err); return false; } diff --git a/src/update/lib/history/job.js b/src/update/lib/unpaywallHistory/job.js similarity index 84% rename from src/update/lib/history/job.js rename to src/update/lib/unpaywallHistory/job.js index 56af843e..dd46a953 100644 --- a/src/update/lib/history/job.js +++ b/src/update/lib/unpaywallHistory/job.js @@ -1,21 +1,12 @@ /* eslint-disable no-param-reassign */ const logger = require('../logger'); +const { setInUpdate } = require('../status'); +const { downloadChangefile } = require('../download'); const { endState, fail, -} = require('../state'); - -const { - setInUpdate, -} = require('../status'); - -const { - downloadChangefile, -} = require('../download'); - -const { createState, addStepGetChangefiles, updateLatestStep, @@ -23,14 +14,8 @@ const { } = require('../state'); const { insertHistoryDataUnpaywall } = require('./history'); - -const { - getChangefiles, -} = require('../services/unpaywall'); - -const { - sendMailNoChangefile, -} = require('../services/mail'); +const { getChangefiles } = require('../services/unpaywall'); +const { sendMailNoChangefile } = require('../services/mail'); /** * Download and insert on elastic the changefiles from unpaywall between a period with history. @@ -50,7 +35,7 @@ async function insertWithOaHistoryJob(jobConfig) { const { interval, startDate, endDate, } = jobConfig; - await createState(); + await createState('unpaywallHistory'); const start = new Date(); addStepGetChangefiles(); const step = getLatestStep(); @@ -77,7 +62,7 @@ async function insertWithOaHistoryJob(jobConfig) { let success = true; for (let i = 0; i < changefilesInfo.length; i += 1) { - success = await downloadChangefile(changefilesInfo[i], interval); + success = await downloadChangefile('unpaywallHistory', changefilesInfo[i], interval); if (!success) return; jobConfig.filename = changefilesInfo[i].filename; jobConfig.date = changefilesInfo[i].date; diff --git a/src/update/lib/history/routers/job.js b/src/update/lib/unpaywallHistory/routers/job.js similarity index 100% rename from src/update/lib/history/routers/job.js rename to src/update/lib/unpaywallHistory/routers/job.js diff --git a/src/update/lib/unpaywallHistory/routers/reports.js b/src/update/lib/unpaywallHistory/routers/reports.js new file mode 100644 index 00000000..e69de29b diff --git a/src/update/test/classic/cron.js b/src/update/test/unpaywall/cron.js similarity index 100% rename from src/update/test/classic/cron.js rename to src/update/test/unpaywall/cron.js diff --git a/src/update/test/classic/insertionError.js b/src/update/test/unpaywall/insertionError.js similarity index 100% rename from src/update/test/classic/insertionError.js rename to src/update/test/unpaywall/insertionError.js diff --git a/src/update/test/classic/insertionFile.js b/src/update/test/unpaywall/insertionFile.js similarity index 100% rename from src/update/test/classic/insertionFile.js rename to src/update/test/unpaywall/insertionFile.js diff --git a/src/update/test/classic/insertionPeriodDay.js b/src/update/test/unpaywall/insertionPeriodDay.js similarity index 99% rename from src/update/test/classic/insertionPeriodDay.js rename to src/update/test/unpaywall/insertionPeriodDay.js index 86408cf0..e7b34401 100644 --- a/src/update/test/classic/insertionPeriodDay.js +++ b/src/update/test/unpaywall/insertionPeriodDay.js @@ -78,6 +78,7 @@ describe('Test: download and insert file from unpaywall between a period', () => isUpdate = await checkIfInUpdate(); } const count = await countDocuments('unpaywall-test'); + expect(count).to.equal(150); }); diff --git a/src/update/test/classic/insertionPeriodWeek.js b/src/update/test/unpaywall/insertionPeriodWeek.js similarity index 100% rename from src/update/test/classic/insertionPeriodWeek.js rename to src/update/test/unpaywall/insertionPeriodWeek.js diff --git a/src/update/test/classic/insertionSnapshot.js b/src/update/test/unpaywall/insertionSnapshot.js similarity index 100% rename from src/update/test/classic/insertionSnapshot.js rename to src/update/test/unpaywall/insertionSnapshot.js diff --git a/src/update/test/classic/updateDay.js b/src/update/test/unpaywall/updateDay.js similarity index 100% rename from src/update/test/classic/updateDay.js rename to src/update/test/unpaywall/updateDay.js diff --git a/src/update/test/classic/updateWeek.js b/src/update/test/unpaywall/updateWeek.js similarity index 100% rename from src/update/test/classic/updateWeek.js rename to src/update/test/unpaywall/updateWeek.js diff --git a/src/update/test/history/history.js b/src/update/test/unpaywallHistory/history.js similarity index 55% rename from src/update/test/history/history.js rename to src/update/test/unpaywallHistory/history.js index 70bb5221..2c651fba 100644 --- a/src/update/test/history/history.js +++ b/src/update/test/unpaywallHistory/history.js @@ -3,17 +3,12 @@ const { expect } = require('chai'); const chai = require('chai'); const chaiHttp = require('chai-http'); -const { - updateChangeFile, -} = require('../utils/snapshot'); +const { updateChangeFile } = require('../utils/snapshot'); -const { - countDocuments, -} = require('../utils/elastic'); +const { countDocuments } = require('../utils/elastic'); -const { - getState, -} = require('../utils/state'); +const { getState } = require('../utils/state'); +const { getReport } = require('../utils/report'); const checkIfInUpdate = require('../utils/status'); @@ -54,7 +49,7 @@ describe('Test: daily update route test with history', () => { expect(res).have.status(202); }); - it('Should insert 2 data', async () => { + it('Should insert 2 data in enriched index', async () => { // wait for the update to finish let isUpdate = true; while (isUpdate) { @@ -65,7 +60,7 @@ describe('Test: daily update route test with history', () => { expect(count).to.equal(2); }); - it('Should insert 4 data', async () => { + it('Should insert 4 data in history index', async () => { // wait for the update to finish let isUpdate = true; while (isUpdate) { @@ -166,6 +161,95 @@ describe('Test: daily update route test with history', () => { }); }); + // it('Should get report with all informations from the download and insertion', async () => { + // const state = await getReport(); + + // expect(state).have.property('done').equal(true); + // expect(state).have.property('createdAt').to.not.equal(undefined); + // expect(state).have.property('endAt').to.not.equal(undefined); + // expect(state).have.property('steps').to.be.an('array'); + // expect(state).have.property('error').equal(false); + // expect(state).have.property('took').to.not.equal(undefined); + // expect(state).have.property('totalInsertedDocs').equal(6); + // expect(state).have.property('totalUpdatedDocs').equal(4); + + // expect(state.steps[0]).have.property('task').equal('getChangefiles'); + // expect(state.steps[0]).have.property('took').to.not.equal(undefined); + // expect(state.steps[0]).have.property('status').equal('success'); + + // expect(state.steps[1]).have.property('task').equal('download'); + // expect(state.steps[1]).have.property('file').equal('history3.jsonl.gz'); + // expect(state.steps[1]).have.property('percent').equal(100); + // expect(state.steps[1]).have.property('took').to.not.equal(undefined); + // expect(state.steps[1]).have.property('status').equal('success'); + + // expect(state.steps[2]).have.property('task').equal('insert'); + // expect(state.steps[2]).have.property('file').equal('history3.jsonl.gz'); + // expect(state.steps[2]).have.property('percent').equal(100); + // expect(state.steps[2]).have.property('linesRead').equal(2); + // expect(state.steps[2]).have.property('insertedDocs').equal(0); + // expect(state.steps[2]).have.property('updatedDocs').equal(0); + // expect(state.steps[2]).have.property('failedDocs').equal(0); + // expect(state.steps[2]).have.property('took').to.not.equal(undefined); + // expect(state.steps[2]).have.property('status').equal('success'); + + // const step2 = state.steps[2].index; + // expect(step2.unpaywall_enriched).have.property('insertedDocs').equal(2); + // expect(step2.unpaywall_enriched).have.property('updatedDocs').equal(0); + // expect(step2.unpaywall_enriched).have.property('failedDocs').equal(0); + // expect(step2.unpaywall_history).have.property('insertedDocs').equal(0); + // expect(step2.unpaywall_history).have.property('updatedDocs').equal(0); + // expect(step2.unpaywall_history).have.property('failedDocs').equal(0); + + // expect(state.steps[3]).have.property('task').equal('download'); + // expect(state.steps[3]).have.property('file').equal('history2.jsonl.gz'); + // expect(state.steps[3]).have.property('percent').equal(100); + // expect(state.steps[3]).have.property('took').to.not.equal(undefined); + // expect(state.steps[3]).have.property('status').equal('success'); + + // expect(state.steps[4]).have.property('task').equal('insert'); + // expect(state.steps[4]).have.property('file').equal('history2.jsonl.gz'); + // expect(state.steps[4]).have.property('percent').equal(100); + // expect(state.steps[4]).have.property('linesRead').equal(2); + // expect(state.steps[4]).have.property('insertedDocs').equal(0); + // expect(state.steps[4]).have.property('updatedDocs').equal(0); + // expect(state.steps[4]).have.property('failedDocs').equal(0); + // expect(state.steps[4]).have.property('took').to.not.equal(undefined); + // expect(state.steps[4]).have.property('status').equal('success'); + + // const step4 = state.steps[4].index; + // expect(step4.unpaywall_enriched).have.property('insertedDocs').equal(0); + // expect(step4.unpaywall_enriched).have.property('updatedDocs').equal(2); + // expect(step4.unpaywall_enriched).have.property('failedDocs').equal(0); + // expect(step4.unpaywall_history).have.property('insertedDocs').equal(2); + // expect(step4.unpaywall_history).have.property('updatedDocs').equal(0); + // expect(step4.unpaywall_history).have.property('failedDocs').equal(0); + + // expect(state.steps[5]).have.property('task').equal('download'); + // expect(state.steps[5]).have.property('file').equal('history1.jsonl.gz'); + // expect(state.steps[5]).have.property('percent').equal(100); + // expect(state.steps[5]).have.property('took').to.not.equal(undefined); + // expect(state.steps[5]).have.property('status').equal('success'); + + // expect(state.steps[6]).have.property('task').equal('insert'); + // expect(state.steps[6]).have.property('file').equal('history1.jsonl.gz'); + // expect(state.steps[6]).have.property('percent').equal(100); + // expect(state.steps[6]).have.property('linesRead').equal(2); + // expect(state.steps[6]).have.property('insertedDocs').equal(0); + // expect(state.steps[6]).have.property('updatedDocs').equal(0); + // expect(state.steps[6]).have.property('failedDocs').equal(0); + // expect(state.steps[6]).have.property('took').to.not.equal(undefined); + // expect(state.steps[6]).have.property('status').equal('success'); + + // const step6 = state.steps[6].index; + // expect(step6.unpaywall_enriched).have.property('insertedDocs').equal(0); + // expect(step6.unpaywall_enriched).have.property('updatedDocs').equal(2); + // expect(step6.unpaywall_enriched).have.property('failedDocs').equal(0); + // expect(step6.unpaywall_history).have.property('insertedDocs').equal(2); + // expect(step6.unpaywall_history).have.property('updatedDocs').equal(0); + // expect(step6.unpaywall_history).have.property('failedDocs').equal(0); + // }); + after(async () => { // await reset(); }); diff --git a/src/update/test/history/rollback.js b/src/update/test/unpaywallHistory/rollback.js similarity index 100% rename from src/update/test/history/rollback.js rename to src/update/test/unpaywallHistory/rollback.js From 19e992a5c673e8f75e35c5f60d8bd0956c633909 Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Fri, 5 Jan 2024 15:59:53 +0100 Subject: [PATCH 009/114] refactor: update middleware and update getReport function for history and unpaywall --- src/update/app.js | 2 + src/update/lib/controllers/report.js | 82 ++++++++ src/update/lib/controllers/unpaywall.js | 2 +- .../lib/middlewares/{ => format}/cron.js | 6 +- .../lib/middlewares/{ => format}/filename.js | 6 +- .../lib/middlewares/{ => format}/interval.js | 10 +- .../lib/middlewares/{ => format}/job.js | 31 ++- .../lib/middlewares/{ => format}/latest.js | 6 +- src/update/lib/middlewares/type.js | 24 +++ src/update/lib/routers/unpaywall.js | 4 +- src/update/lib/unpaywall/controllers/cron.js | 4 +- src/update/lib/unpaywall/controllers/file.js | 4 +- src/update/lib/unpaywall/controllers/job.js | 30 +-- .../lib/unpaywall/controllers/report.js | 4 +- src/update/lib/unpaywall/routers/cron.js | 2 +- src/update/lib/unpaywall/routers/job.js | 2 +- src/update/lib/unpaywall/routers/report.js | 14 +- src/update/lib/unpaywall/routers/snapshot.js | 4 +- .../lib/unpaywallHistory/controllers/job.js | 20 +- .../lib/unpaywallHistory/routers/job.js | 15 +- .../lib/unpaywallHistory/routers/report.js | 23 +++ .../lib/unpaywallHistory/routers/reports.js | 0 src/update/test/unpaywallHistory/history.js | 184 +++++++++--------- src/update/test/utils/report.js | 19 ++ 24 files changed, 329 insertions(+), 169 deletions(-) create mode 100644 src/update/lib/controllers/report.js rename src/update/lib/middlewares/{ => format}/cron.js (90%) rename src/update/lib/middlewares/{ => format}/filename.js (89%) rename src/update/lib/middlewares/{ => format}/interval.js (76%) rename src/update/lib/middlewares/{ => format}/job.js (91%) rename src/update/lib/middlewares/{ => format}/latest.js (88%) create mode 100644 src/update/lib/middlewares/type.js create mode 100644 src/update/lib/unpaywallHistory/routers/report.js delete mode 100644 src/update/lib/unpaywallHistory/routers/reports.js diff --git a/src/update/app.js b/src/update/app.js index 71e3388d..5d3ce392 100644 --- a/src/update/app.js +++ b/src/update/app.js @@ -21,6 +21,7 @@ const routerUnpaywallCron = require('./lib/unpaywall/routers/cron'); const routerUnpaywallSnapshot = require('./lib/unpaywall/routers/snapshot'); const routerUnpaywallHistoryJob = require('./lib/unpaywallHistory/routers/job'); +const routerUnpaywallHistoryReport = require('./lib/unpaywallHistory/routers/report'); require('./lib/unpaywall/cron/file'); @@ -79,6 +80,7 @@ app.use(routerUnpaywallCron); app.use(routerUnpaywallSnapshot); app.use(routerUnpaywallHistoryJob); +app.use(routerUnpaywallHistoryReport); /* Errors and unknown routes */ app.use((req, res, next) => res.status(404).json({ message: `Cannot ${req.method} ${req.originalUrl}` })); diff --git a/src/update/lib/controllers/report.js b/src/update/lib/controllers/report.js new file mode 100644 index 00000000..d15f4930 --- /dev/null +++ b/src/update/lib/controllers/report.js @@ -0,0 +1,82 @@ +const path = require('path'); +const fs = require('fs-extra'); + +const { getMostRecentFile } = require('../file'); +const { getReport } = require('../report'); +const { getPathOfDirectory } = require('../file'); + +/** + * Controller to get list of reports or latest report. + * + * @param {import('express').Request} req - HTTP request. + * @param {import('express').Response} res - HTTP response. + * @param {import('express').NextFunction} next - Do the following. + */ +async function getReports(req, res, next) { + const { latest, type } = req.data; + + const pathOfDirectory = getPathOfDirectory(type, 'reports'); + + if (latest) { + let latestFile; + try { + latestFile = await getMostRecentFile(pathOfDirectory); + } catch (err) { + return next({ message: err.message }); + } + + if (!latestFile) { + return next({ message: 'No reports are available' }); + } + + let report; + try { + report = await getReport(type, latestFile?.filename); + } catch (err) { + return next({ message: `Cannot get [${latestFile?.filename}] latest report` }); + } + return res.status(200).json(report); + } + + let reports = await fs.readdir(pathOfDirectory); + + reports = reports.sort((a, b) => { + const [date1] = a.split('.'); + const [date2] = b.split('.'); + return new Date(date2).getTime() - new Date(date1).getTime(); + }); + + return res.status(200).json(reports); +} + +/** + * Controller to get report by filename. + * + * @param {import('express').Request} req - HTTP request. + * @param {import('express').Response} res - HTTP response. + * @param {import('express').NextFunction} next - Do the following. + */ +async function getReportByFilename(req, res, next) { + const { filename, type } = req.data; + + const pathOfDirectory = getPathOfDirectory(type, 'reports'); + + try { + await fs.stat(path.resolve(pathOfDirectory, filename)); + } catch (err) { + return next({ message: `File [${filename}] not found` }); + } + + let report; + try { + report = await getReport(type, filename); + } catch (err) { + return next({ message: `Cannot get ${filename} report` }); + } + return res.status(200).json(report); +} + +module.exports = { + getReports, + getReportByFilename, +}; diff --git a/src/update/lib/controllers/unpaywall.js b/src/update/lib/controllers/unpaywall.js index 11357e92..250be05c 100644 --- a/src/update/lib/controllers/unpaywall.js +++ b/src/update/lib/controllers/unpaywall.js @@ -8,7 +8,7 @@ const { getChangefiles } = require('../services/unpaywall'); * @param {import('express').NextFunction} next - Do the following. */ async function getChangefilesOfUnpaywall(req, res, next) { - const interval = req.data; + const { interval } = req.data; let snapshotsInfo; diff --git a/src/update/lib/middlewares/cron.js b/src/update/lib/middlewares/format/cron.js similarity index 90% rename from src/update/lib/middlewares/cron.js rename to src/update/lib/middlewares/format/cron.js index 57f53e7a..7b64fda7 100644 --- a/src/update/lib/middlewares/cron.js +++ b/src/update/lib/middlewares/format/cron.js @@ -16,7 +16,11 @@ async function validateCronConfig(req, res, next) { if (error) return res.status(400).json({ message: error.details[0].message }); - req.data = value; + if (!req.data) { + req.data = {}; + } + + req.data.cronConfig = value; return next(); } diff --git a/src/update/lib/middlewares/filename.js b/src/update/lib/middlewares/format/filename.js similarity index 89% rename from src/update/lib/middlewares/filename.js rename to src/update/lib/middlewares/format/filename.js index 35b85c88..7effee60 100644 --- a/src/update/lib/middlewares/filename.js +++ b/src/update/lib/middlewares/format/filename.js @@ -13,7 +13,11 @@ async function validateFilename(req, res, next) { const { error, value } = joi.string().trim().required().validate(filename); if (error) return res.status(400).json({ message: error.details[0].message }); - req.data = value; + if (!req.data) { + req.data = {}; + } + + req.data.filename = value; return next(); } diff --git a/src/update/lib/middlewares/interval.js b/src/update/lib/middlewares/format/interval.js similarity index 76% rename from src/update/lib/middlewares/interval.js rename to src/update/lib/middlewares/format/interval.js index 435a6cb3..9511fa0c 100644 --- a/src/update/lib/middlewares/interval.js +++ b/src/update/lib/middlewares/format/interval.js @@ -7,15 +7,19 @@ const joi = require('joi'); * @param {import('express').Response} res - HTTP response. * @param {import('express').NextFunction} next - Do the following. */ -async function validateIntervale(req, res, next) { +async function validateInterval(req, res, next) { const { error, value } = joi.string().trim().valid('week', 'day').default('day') .validate(req.body.interval); if (error) return res.status(400).json({ message: error.details[0].message }); - req.data = value; + if (!req.data) { + req.data = {}; + } + + req.data.interval = value; return next(); } -module.exports = validateIntervale; +module.exports = validateInterval; diff --git a/src/update/lib/middlewares/job.js b/src/update/lib/middlewares/format/job.js similarity index 91% rename from src/update/lib/middlewares/job.js rename to src/update/lib/middlewares/format/job.js index db7e72ab..0053ffbc 100644 --- a/src/update/lib/middlewares/job.js +++ b/src/update/lib/middlewares/format/job.js @@ -12,7 +12,11 @@ function validateSnapshotJob(req, res, next) { if (error) return res.status(400).json({ message: error.details[0].message }); - req.data = value; + if (!req.data) { + req.data = {}; + } + + req.data.index = value; return next(); } @@ -34,7 +38,11 @@ async function validateJobChangefilesConfig(req, res, next) { if (error) return res.status(400).json({ message: error.details[0].message }); - req.data = value; + if (!req.data) { + req.data = {}; + } + + req.data.jobConfig = value; return next(); } @@ -65,7 +73,12 @@ async function validateInsertFile(req, res, next) { if (checkBody.error) return res.status(400).json({ message: checkBody.error.details[0].message }); const { index, offset, limit } = checkBody.value; - req.data = { + + if (!req.data) { + req.data = {}; + } + + req.data.jobConfig = { filename, index, offset, limit, }; @@ -89,7 +102,11 @@ async function validateHistoryJob(req, res, next) { if (error) return res.status(400).json({ message: error.details[0].message }); - req.data = value; + if (!req.data) { + req.data = {}; + } + + req.data.jobConfig = value; return next(); } @@ -109,7 +126,11 @@ async function validateHistoryReset(req, res, next) { if (error) return res.status(400).json({ message: error.details[0].message }); - req.data = value; + if (!req.data) { + req.data = {}; + } + + req.data.rollBackConfig = value; return next(); } diff --git a/src/update/lib/middlewares/latest.js b/src/update/lib/middlewares/format/latest.js similarity index 88% rename from src/update/lib/middlewares/latest.js rename to src/update/lib/middlewares/format/latest.js index 01aac6ab..dd68cf7a 100644 --- a/src/update/lib/middlewares/latest.js +++ b/src/update/lib/middlewares/format/latest.js @@ -11,7 +11,11 @@ async function validateLatest(req, res, next) { const { error, value } = joi.boolean().default(false).validate(req?.query?.latest); if (error) return res.status(400).json({ message: error.details[0].message }); - req.data = value; + if (!req.data) { + req.data = {}; + } + + req.data.latest = value; return next(); } diff --git a/src/update/lib/middlewares/type.js b/src/update/lib/middlewares/type.js new file mode 100644 index 00000000..ffc9d4a5 --- /dev/null +++ b/src/update/lib/middlewares/type.js @@ -0,0 +1,24 @@ +/** + * Joi middleware to check if interval in body is correct. + * + * @param {import('express').Request} req - HTTP request. + * @param {import('express').Response} res - HTTP response. + * @param {import('express').NextFunction} next - Do the following. + */ +async function updateType(req, res, next) { + let type = 'unpaywall'; + + if (req.originalUrl.split('/').includes('history')) { + type = 'unpaywallHistory'; + } + + if (!req.data) { + req.data = {}; + } + + req.data.type = type; + + return next(); +} + +module.exports = updateType; diff --git a/src/update/lib/routers/unpaywall.js b/src/update/lib/routers/unpaywall.js index 9f39047b..0ffe4798 100644 --- a/src/update/lib/routers/unpaywall.js +++ b/src/update/lib/routers/unpaywall.js @@ -1,6 +1,6 @@ const router = require('express').Router(); -const validateIntervale = require('../middlewares/interval'); +const validateInterval = require('../middlewares/format/interval'); const getChangefilesOfUnpaywall = require('../controllers/unpaywall'); /** @@ -8,6 +8,6 @@ const getChangefilesOfUnpaywall = require('../controllers/unpaywall'); * * This route can take a body which corresponds to the intervale of changefile. */ -router.get('/unpaywall/changefiles', validateIntervale, getChangefilesOfUnpaywall); +router.get('/unpaywall/changefiles', validateInterval, getChangefilesOfUnpaywall); module.exports = router; diff --git a/src/update/lib/unpaywall/controllers/cron.js b/src/update/lib/unpaywall/controllers/cron.js index e42fb369..2dfd7b53 100644 --- a/src/update/lib/unpaywall/controllers/cron.js +++ b/src/update/lib/unpaywall/controllers/cron.js @@ -43,8 +43,8 @@ function stopUpdateCron(req, res, next) { * @param {import('express').NextFunction} next - Do the following. */ function patchUpdateCron(req, res, next) { - const value = req.data; - const { time, index, interval } = value; + const { cronConfig } = req.data; + const { time, index, interval } = cronConfig; if (time) { const validCron = cronValidator.isValidCron(time, { seconds: true }); diff --git a/src/update/lib/unpaywall/controllers/file.js b/src/update/lib/unpaywall/controllers/file.js index 9eb72c16..ac6f7e42 100644 --- a/src/update/lib/unpaywall/controllers/file.js +++ b/src/update/lib/unpaywall/controllers/file.js @@ -18,7 +18,7 @@ const { snapshotsDir } = dirPath.unpaywall; * @param {import('express').NextFunction} next - Do the following. */ async function getFiles(req, res, next) { - const latest = req.data; + const { latest } = req.data; if (latest) { let latestSnapshot; @@ -53,7 +53,7 @@ async function uploadFile(req, res, next) { * @param {import('express').NextFunction} next - Do the following. */ async function deleteInstalledFile(req, res, next) { - const filename = req.data; + const { filename } = req.data; if (!await fs.pathExists(path.resolve(snapshotsDir, filename))) { return res.status(404).json({ message: `File [${filename}] not found` }); diff --git a/src/update/lib/unpaywall/controllers/job.js b/src/update/lib/unpaywall/controllers/job.js index 2c6353e3..959043cc 100644 --- a/src/update/lib/unpaywall/controllers/job.js +++ b/src/update/lib/unpaywall/controllers/job.js @@ -19,7 +19,7 @@ const { * @param {import('express').NextFunction} next - Do the following. */ async function downloadAndInsertSnapshotJob(req, res, next) { - const index = req.data; + const { index } = req.data; const jobConfig = { index, @@ -39,12 +39,13 @@ async function downloadAndInsertSnapshotJob(req, res, next) { * @param {import('express').NextFunction} next - Do the following. */ async function insertChangefilesOnPeriodJob(req, res, next) { + const { jobConfig } = req.data; + const { startDate, endDate, - index, interval, - } = req.data; + } = jobConfig; if (new Date(startDate).getTime() > Date.now()) { return res.status(400).json({ message: 'startDate cannot be in the futur' }); @@ -56,14 +57,8 @@ async function insertChangefilesOnPeriodJob(req, res, next) { } } - const jobConfig = { - index, - interval, - startDate, - endDate, - offset: 0, - limit: -1, - }; + jobConfig.offset = 0; + jobConfig.limit = -1; if (!startDate && !endDate) { jobConfig.endDate = format(new Date(), 'yyyy-MM-dd'); @@ -88,21 +83,14 @@ async function insertChangefilesOnPeriodJob(req, res, next) { * @param {import('express').NextFunction} next - Do the following. */ async function insertChangefileJob(req, res, next) { - const { - filename, index, offset, limit, - } = req.data; + const { jobConfig } = req.data; + + const { filename } = jobConfig; if (!await fs.pathExists(path.resolve(snapshotsDir, filename))) { return res.status(404).json({ message: `File [${filename}] not found` }); } - const jobConfig = { - filename, - index, - offset, - limit, - }; - insertChangefile(jobConfig); return res.status(202).json(); diff --git a/src/update/lib/unpaywall/controllers/report.js b/src/update/lib/unpaywall/controllers/report.js index e12a1810..008f7e52 100644 --- a/src/update/lib/unpaywall/controllers/report.js +++ b/src/update/lib/unpaywall/controllers/report.js @@ -14,7 +14,7 @@ const { reportsDir } = dirPath.unpaywall; * @param {import('express').NextFunction} next - Do the following. */ async function getReports(req, res, next) { - const latest = req.data; + const { latest } = req.data; if (latest) { let latestFile; @@ -56,7 +56,7 @@ async function getReports(req, res, next) { * @param {import('express').NextFunction} next - Do the following. */ async function getReportByFilename(req, res, next) { - const filename = req.data; + const { filename } = req.data; try { await fs.stat(path.resolve(reportsDir, filename)); diff --git a/src/update/lib/unpaywall/routers/cron.js b/src/update/lib/unpaywall/routers/cron.js index 9fec5fab..166f714d 100644 --- a/src/update/lib/unpaywall/routers/cron.js +++ b/src/update/lib/unpaywall/routers/cron.js @@ -1,7 +1,7 @@ const router = require('express').Router(); const checkAuth = require('../../middlewares/auth'); -const validateCronConfig = require('../../middlewares/cron'); +const validateCronConfig = require('../../middlewares/format/cron'); const { startUpdateCron, diff --git a/src/update/lib/unpaywall/routers/job.js b/src/update/lib/unpaywall/routers/job.js index 557f8729..543e5f87 100644 --- a/src/update/lib/unpaywall/routers/job.js +++ b/src/update/lib/unpaywall/routers/job.js @@ -10,7 +10,7 @@ const { validateSnapshotJob, validateJobChangefilesConfig, validateInsertFile, -} = require('../../middlewares/job'); +} = require('../../middlewares/format/job'); const checkStatus = require('../../middlewares/status'); const checkAuth = require('../../middlewares/auth'); diff --git a/src/update/lib/unpaywall/routers/report.js b/src/update/lib/unpaywall/routers/report.js index c018f617..5c2c266d 100644 --- a/src/update/lib/unpaywall/routers/report.js +++ b/src/update/lib/unpaywall/routers/report.js @@ -1,25 +1,23 @@ const router = require('express').Router(); -const { - getReports, - getReportByFilename, -} = require('../controllers/report'); +const { getReports, getReportByFilename } = require('../controllers/report'); -const validateLatest = require('../../middlewares/latest'); -const validateFilename = require('../../middlewares/filename'); +const validateLatest = require('../../middlewares/format/latest'); +const validateFilename = require('../../middlewares/format/filename'); +const updateType = require('../../middlewares/type'); /** * Route that give the list of reports or the content of most recent report in JSON format. * * This route can take in query latest. */ -router.get('/reports', validateLatest, getReports); +router.get('/reports', validateLatest, updateType, getReports); /** * Route that give the content of report in JSON format. * * This route takes a param which corresponds to the filename of report. */ -router.get('/reports/:filename', validateFilename, getReportByFilename); +router.get('/reports/:filename', validateFilename, updateType, getReportByFilename); module.exports = router; diff --git a/src/update/lib/unpaywall/routers/snapshot.js b/src/update/lib/unpaywall/routers/snapshot.js index d496a848..369c2e8c 100644 --- a/src/update/lib/unpaywall/routers/snapshot.js +++ b/src/update/lib/unpaywall/routers/snapshot.js @@ -4,8 +4,8 @@ const upload = require('../../middlewares/multer'); const checkAuth = require('../../middlewares/auth'); const dev = require('../../middlewares/dev'); -const validateLatest = require('../../middlewares/latest'); -const validateFilename = require('../../middlewares/filename'); +const validateLatest = require('../../middlewares/format/latest'); +const validateFilename = require('../../middlewares/format/filename'); const { getFiles, diff --git a/src/update/lib/unpaywallHistory/controllers/job.js b/src/update/lib/unpaywallHistory/controllers/job.js index d4c91848..f002bb9f 100644 --- a/src/update/lib/unpaywallHistory/controllers/job.js +++ b/src/update/lib/unpaywallHistory/controllers/job.js @@ -15,12 +15,13 @@ const { * @param {import('express').NextFunction} next - Do the following. */ async function insertWithOaHistory(req, res, next) { + const { jobConfig } = req.data; + const { startDate, endDate, - index, interval, - } = req.data; + } = jobConfig; if (new Date(startDate).getTime() > Date.now()) { return res.status(400).json({ message: 'startDate cannot be in the future' }); @@ -32,14 +33,8 @@ async function insertWithOaHistory(req, res, next) { } } - const jobConfig = { - index, - interval, - startDate, - endDate, - offset: 0, - limit: -1, - }; + jobConfig.offset = 0; + jobConfig.limit = -1; if (!startDate && !endDate) { jobConfig.endDate = format(new Date(), 'yyyy-MM-dd'); @@ -64,10 +59,7 @@ async function insertWithOaHistory(req, res, next) { * @param {import('express').NextFunction} next - Do the following. */ async function historyRollBack(req, res, next) { - const { - startDate, - index, - } = req.data; + const { startDate, index } = req.data.rollBackConfig; await rollBack(startDate, index); return res.status(202).json(); diff --git a/src/update/lib/unpaywallHistory/routers/job.js b/src/update/lib/unpaywallHistory/routers/job.js index 451ed7da..e4d54691 100644 --- a/src/update/lib/unpaywallHistory/routers/job.js +++ b/src/update/lib/unpaywallHistory/routers/job.js @@ -14,7 +14,7 @@ const { const { validateHistoryJob, validateHistoryReset, -} = require('../../middlewares/job'); +} = require('../../middlewares/format/job'); const checkStatus = require('../../middlewares/status'); const checkAuth = require('../../middlewares/auth'); @@ -47,27 +47,22 @@ router.post('/history/job/reset', checkStatus, checkAuth, validateHistoryReset, router.post('/history/job/reset', checkStatus, checkAuth, validateHistoryReset, historyRollBack); router.post('/history/job/reset/step1', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { - const { - startDate, - } = req.data; + const { startDate } = req.data; + console.log(req.data); await step1(startDate); return res.status(202).json(); }); router.post('/history/job/reset/step2', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { - const { - startDate, - } = req.data; + const { startDate } = req.data; await step2(startDate); return res.status(202).json(); }); router.post('/job/history/reset/step3', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { - const { - startDate, - } = req.data; + const { startDate } = req.data; await step3(startDate); return res.status(202).json(); diff --git a/src/update/lib/unpaywallHistory/routers/report.js b/src/update/lib/unpaywallHistory/routers/report.js new file mode 100644 index 00000000..4019db03 --- /dev/null +++ b/src/update/lib/unpaywallHistory/routers/report.js @@ -0,0 +1,23 @@ +const router = require('express').Router(); + +const { getReports, getReportByFilename } = require('../../controllers/report'); + +const validateLatest = require('../../middlewares/format/latest'); +const validateFilename = require('../../middlewares/format/filename'); +const updateType = require('../../middlewares/type'); + +/** + * Route that give the list of reports or the content of most recent report in JSON format. + * + * This route can take in query latest. + */ +router.get('/unpaywall/history/reports', validateLatest, updateType, getReports); + +/** + * Route that give the content of report in JSON format. + * + * This route takes a param which corresponds to the filename of report. + */ +router.get('/unpaywall/history/reports/:filename', validateFilename, updateType, getReportByFilename); + +module.exports = router; diff --git a/src/update/lib/unpaywallHistory/routers/reports.js b/src/update/lib/unpaywallHistory/routers/reports.js deleted file mode 100644 index e69de29b..00000000 diff --git a/src/update/test/unpaywallHistory/history.js b/src/update/test/unpaywallHistory/history.js index 2c651fba..b83d973a 100644 --- a/src/update/test/unpaywallHistory/history.js +++ b/src/update/test/unpaywallHistory/history.js @@ -8,7 +8,7 @@ const { updateChangeFile } = require('../utils/snapshot'); const { countDocuments } = require('../utils/elastic'); const { getState } = require('../utils/state'); -const { getReport } = require('../utils/report'); +const { getHistoryReport } = require('../utils/report'); const checkIfInUpdate = require('../utils/status'); @@ -71,7 +71,7 @@ describe('Test: daily update route test with history', () => { expect(count).to.equal(4); }); - it('Should get state with all informations from the download and insertion', async () => { + it('Should get state with all information from the download and insertion', async () => { const state = await getState(); expect(state).have.property('done').equal(true); @@ -159,98 +159,98 @@ describe('Test: daily update route test with history', () => { expect(step6.unpaywall_history).have.property('updatedDocs').equal(0); expect(step6.unpaywall_history).have.property('failedDocs').equal(0); }); - }); - // it('Should get report with all informations from the download and insertion', async () => { - // const state = await getReport(); - - // expect(state).have.property('done').equal(true); - // expect(state).have.property('createdAt').to.not.equal(undefined); - // expect(state).have.property('endAt').to.not.equal(undefined); - // expect(state).have.property('steps').to.be.an('array'); - // expect(state).have.property('error').equal(false); - // expect(state).have.property('took').to.not.equal(undefined); - // expect(state).have.property('totalInsertedDocs').equal(6); - // expect(state).have.property('totalUpdatedDocs').equal(4); - - // expect(state.steps[0]).have.property('task').equal('getChangefiles'); - // expect(state.steps[0]).have.property('took').to.not.equal(undefined); - // expect(state.steps[0]).have.property('status').equal('success'); - - // expect(state.steps[1]).have.property('task').equal('download'); - // expect(state.steps[1]).have.property('file').equal('history3.jsonl.gz'); - // expect(state.steps[1]).have.property('percent').equal(100); - // expect(state.steps[1]).have.property('took').to.not.equal(undefined); - // expect(state.steps[1]).have.property('status').equal('success'); - - // expect(state.steps[2]).have.property('task').equal('insert'); - // expect(state.steps[2]).have.property('file').equal('history3.jsonl.gz'); - // expect(state.steps[2]).have.property('percent').equal(100); - // expect(state.steps[2]).have.property('linesRead').equal(2); - // expect(state.steps[2]).have.property('insertedDocs').equal(0); - // expect(state.steps[2]).have.property('updatedDocs').equal(0); - // expect(state.steps[2]).have.property('failedDocs').equal(0); - // expect(state.steps[2]).have.property('took').to.not.equal(undefined); - // expect(state.steps[2]).have.property('status').equal('success'); - - // const step2 = state.steps[2].index; - // expect(step2.unpaywall_enriched).have.property('insertedDocs').equal(2); - // expect(step2.unpaywall_enriched).have.property('updatedDocs').equal(0); - // expect(step2.unpaywall_enriched).have.property('failedDocs').equal(0); - // expect(step2.unpaywall_history).have.property('insertedDocs').equal(0); - // expect(step2.unpaywall_history).have.property('updatedDocs').equal(0); - // expect(step2.unpaywall_history).have.property('failedDocs').equal(0); - - // expect(state.steps[3]).have.property('task').equal('download'); - // expect(state.steps[3]).have.property('file').equal('history2.jsonl.gz'); - // expect(state.steps[3]).have.property('percent').equal(100); - // expect(state.steps[3]).have.property('took').to.not.equal(undefined); - // expect(state.steps[3]).have.property('status').equal('success'); - - // expect(state.steps[4]).have.property('task').equal('insert'); - // expect(state.steps[4]).have.property('file').equal('history2.jsonl.gz'); - // expect(state.steps[4]).have.property('percent').equal(100); - // expect(state.steps[4]).have.property('linesRead').equal(2); - // expect(state.steps[4]).have.property('insertedDocs').equal(0); - // expect(state.steps[4]).have.property('updatedDocs').equal(0); - // expect(state.steps[4]).have.property('failedDocs').equal(0); - // expect(state.steps[4]).have.property('took').to.not.equal(undefined); - // expect(state.steps[4]).have.property('status').equal('success'); - - // const step4 = state.steps[4].index; - // expect(step4.unpaywall_enriched).have.property('insertedDocs').equal(0); - // expect(step4.unpaywall_enriched).have.property('updatedDocs').equal(2); - // expect(step4.unpaywall_enriched).have.property('failedDocs').equal(0); - // expect(step4.unpaywall_history).have.property('insertedDocs').equal(2); - // expect(step4.unpaywall_history).have.property('updatedDocs').equal(0); - // expect(step4.unpaywall_history).have.property('failedDocs').equal(0); - - // expect(state.steps[5]).have.property('task').equal('download'); - // expect(state.steps[5]).have.property('file').equal('history1.jsonl.gz'); - // expect(state.steps[5]).have.property('percent').equal(100); - // expect(state.steps[5]).have.property('took').to.not.equal(undefined); - // expect(state.steps[5]).have.property('status').equal('success'); - - // expect(state.steps[6]).have.property('task').equal('insert'); - // expect(state.steps[6]).have.property('file').equal('history1.jsonl.gz'); - // expect(state.steps[6]).have.property('percent').equal(100); - // expect(state.steps[6]).have.property('linesRead').equal(2); - // expect(state.steps[6]).have.property('insertedDocs').equal(0); - // expect(state.steps[6]).have.property('updatedDocs').equal(0); - // expect(state.steps[6]).have.property('failedDocs').equal(0); - // expect(state.steps[6]).have.property('took').to.not.equal(undefined); - // expect(state.steps[6]).have.property('status').equal('success'); - - // const step6 = state.steps[6].index; - // expect(step6.unpaywall_enriched).have.property('insertedDocs').equal(0); - // expect(step6.unpaywall_enriched).have.property('updatedDocs').equal(2); - // expect(step6.unpaywall_enriched).have.property('failedDocs').equal(0); - // expect(step6.unpaywall_history).have.property('insertedDocs').equal(2); - // expect(step6.unpaywall_history).have.property('updatedDocs').equal(0); - // expect(step6.unpaywall_history).have.property('failedDocs').equal(0); - // }); + it('Should get report with all information from the download and insertion', async () => { + const report = await getHistoryReport('unpaywallHistory'); + + expect(report).have.property('done').equal(true); + expect(report).have.property('createdAt').to.not.equal(undefined); + expect(report).have.property('endAt').to.not.equal(undefined); + expect(report).have.property('steps').to.be.an('array'); + expect(report).have.property('error').equal(false); + expect(report).have.property('took').to.not.equal(undefined); + expect(report).have.property('totalInsertedDocs').equal(6); + expect(report).have.property('totalUpdatedDocs').equal(4); + + expect(report.steps[0]).have.property('task').equal('getChangefiles'); + expect(report.steps[0]).have.property('took').to.not.equal(undefined); + expect(report.steps[0]).have.property('status').equal('success'); + + expect(report.steps[1]).have.property('task').equal('download'); + expect(report.steps[1]).have.property('file').equal('history3.jsonl.gz'); + expect(report.steps[1]).have.property('percent').equal(100); + expect(report.steps[1]).have.property('took').to.not.equal(undefined); + expect(report.steps[1]).have.property('status').equal('success'); + + expect(report.steps[2]).have.property('task').equal('insert'); + expect(report.steps[2]).have.property('file').equal('history3.jsonl.gz'); + expect(report.steps[2]).have.property('percent').equal(100); + expect(report.steps[2]).have.property('linesRead').equal(2); + expect(report.steps[2]).have.property('insertedDocs').equal(0); + expect(report.steps[2]).have.property('updatedDocs').equal(0); + expect(report.steps[2]).have.property('failedDocs').equal(0); + expect(report.steps[2]).have.property('took').to.not.equal(undefined); + expect(report.steps[2]).have.property('status').equal('success'); + + const step2 = report.steps[2].index; + expect(step2.unpaywall_enriched).have.property('insertedDocs').equal(2); + expect(step2.unpaywall_enriched).have.property('updatedDocs').equal(0); + expect(step2.unpaywall_enriched).have.property('failedDocs').equal(0); + expect(step2.unpaywall_history).have.property('insertedDocs').equal(0); + expect(step2.unpaywall_history).have.property('updatedDocs').equal(0); + expect(step2.unpaywall_history).have.property('failedDocs').equal(0); + + expect(report.steps[3]).have.property('task').equal('download'); + expect(report.steps[3]).have.property('file').equal('history2.jsonl.gz'); + expect(report.steps[3]).have.property('percent').equal(100); + expect(report.steps[3]).have.property('took').to.not.equal(undefined); + expect(report.steps[3]).have.property('status').equal('success'); + + expect(report.steps[4]).have.property('task').equal('insert'); + expect(report.steps[4]).have.property('file').equal('history2.jsonl.gz'); + expect(report.steps[4]).have.property('percent').equal(100); + expect(report.steps[4]).have.property('linesRead').equal(2); + expect(report.steps[4]).have.property('insertedDocs').equal(0); + expect(report.steps[4]).have.property('updatedDocs').equal(0); + expect(report.steps[4]).have.property('failedDocs').equal(0); + expect(report.steps[4]).have.property('took').to.not.equal(undefined); + expect(report.steps[4]).have.property('status').equal('success'); + + const step4 = report.steps[4].index; + expect(step4.unpaywall_enriched).have.property('insertedDocs').equal(0); + expect(step4.unpaywall_enriched).have.property('updatedDocs').equal(2); + expect(step4.unpaywall_enriched).have.property('failedDocs').equal(0); + expect(step4.unpaywall_history).have.property('insertedDocs').equal(2); + expect(step4.unpaywall_history).have.property('updatedDocs').equal(0); + expect(step4.unpaywall_history).have.property('failedDocs').equal(0); + + expect(report.steps[5]).have.property('task').equal('download'); + expect(report.steps[5]).have.property('file').equal('history1.jsonl.gz'); + expect(report.steps[5]).have.property('percent').equal(100); + expect(report.steps[5]).have.property('took').to.not.equal(undefined); + expect(report.steps[5]).have.property('status').equal('success'); + + expect(report.steps[6]).have.property('task').equal('insert'); + expect(report.steps[6]).have.property('file').equal('history1.jsonl.gz'); + expect(report.steps[6]).have.property('percent').equal(100); + expect(report.steps[6]).have.property('linesRead').equal(2); + expect(report.steps[6]).have.property('insertedDocs').equal(0); + expect(report.steps[6]).have.property('updatedDocs').equal(0); + expect(report.steps[6]).have.property('failedDocs').equal(0); + expect(report.steps[6]).have.property('took').to.not.equal(undefined); + expect(report.steps[6]).have.property('status').equal('success'); + + const step6 = report.steps[6].index; + expect(step6.unpaywall_enriched).have.property('insertedDocs').equal(0); + expect(step6.unpaywall_enriched).have.property('updatedDocs').equal(2); + expect(step6.unpaywall_enriched).have.property('failedDocs').equal(0); + expect(step6.unpaywall_history).have.property('insertedDocs').equal(2); + expect(step6.unpaywall_history).have.property('updatedDocs').equal(0); + expect(step6.unpaywall_history).have.property('failedDocs').equal(0); + }); + }); after(async () => { - // await reset(); + await reset(); }); }); diff --git a/src/update/test/utils/report.js b/src/update/test/utils/report.js index a06cdd51..ff461eaa 100644 --- a/src/update/test/utils/report.js +++ b/src/update/test/utils/report.js @@ -24,6 +24,25 @@ async function getReport() { return res?.body; } +/** + * get report of update + * + * @returns {Promise{Object}} report + */ +async function getHistoryReport() { + let res; + try { + res = await chai.request(updateURL) + .get('/unpaywall/history/reports') + .query({ latest: true }); + } catch (err) { + console.error(`Cannot GET ${updateURL}/report`); + process.exit(1); + } + return res?.body; +} + module.exports = { getReport, + getHistoryReport, }; From d32217e995717287c1d806499170f8b62f544308 Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Fri, 5 Jan 2024 16:00:35 +0100 Subject: [PATCH 010/114] fix: remove console.log --- src/update/lib/unpaywallHistory/routers/job.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/update/lib/unpaywallHistory/routers/job.js b/src/update/lib/unpaywallHistory/routers/job.js index e4d54691..03e4e7af 100644 --- a/src/update/lib/unpaywallHistory/routers/job.js +++ b/src/update/lib/unpaywallHistory/routers/job.js @@ -48,7 +48,6 @@ router.post('/history/job/reset', checkStatus, checkAuth, validateHistoryReset, router.post('/history/job/reset/step1', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { const { startDate } = req.data; - console.log(req.data); await step1(startDate); return res.status(202).json(); From 1514163254b349a06c9a1efefc9439c67e8d3653 Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Tue, 9 Jan 2024 10:57:11 +0100 Subject: [PATCH 011/114] refactor: use same router for both type of data --- src/update/app.js | 25 ++---- .../lib/{unpaywall => }/controllers/cron.js | 0 .../lib/{unpaywall => }/controllers/file.js | 25 +++--- .../lib/{unpaywall => }/controllers/job.js | 67 +++++++++++++- src/update/lib/controllers/report.js | 4 +- src/update/lib/{unpaywall => cron}/cron.js | 2 +- src/update/lib/{unpaywall => }/cron/file.js | 8 +- src/update/lib/{unpaywall => }/cron/update.js | 6 +- src/update/lib/download.js | 2 +- src/update/lib/{file.js => files.js} | 0 .../lib/{unpaywallHistory => }/history.js | 12 +-- src/update/lib/{unpaywall => }/insert.js | 10 +-- src/update/lib/{unpaywall => }/job.js | 70 +++++++++++++-- src/update/lib/middlewares/format/type.js | 25 ++++++ src/update/lib/middlewares/type.js | 24 ----- src/update/lib/report.js | 2 +- .../lib/{unpaywall => }/routers/cron.js | 4 +- src/update/lib/routers/job.js | 90 +++++++++++++++++++ .../lib/{unpaywall => }/routers/report.js | 10 +-- src/update/lib/routers/snapshot.js | 36 ++++++++ .../lib/unpaywall/controllers/report.js | 79 ---------------- src/update/lib/unpaywall/routers/job.js | 46 ---------- src/update/lib/unpaywall/routers/snapshot.js | 39 -------- .../lib/unpaywallHistory/controllers/job.js | 71 --------------- src/update/lib/unpaywallHistory/job.js | 77 ---------------- .../lib/unpaywallHistory/routers/job.js | 70 --------------- .../lib/unpaywallHistory/routers/report.js | 23 ----- src/update/test/auth.js | 12 +-- src/update/test/unpaywall/insertionError.js | 26 ++---- src/update/test/unpaywall/insertionFile.js | 39 +++----- .../test/unpaywall/insertionPeriodDay.js | 33 ++----- .../test/unpaywall/insertionPeriodWeek.js | 31 ++----- .../test/unpaywall/insertionSnapshot.js | 18 +--- src/update/test/unpaywall/updateDay.js | 32 ++----- src/update/test/unpaywall/updateWeek.js | 28 ++---- src/update/test/unpaywallHistory/history.js | 10 +-- src/update/test/unpaywallHistory/rollback.js | 12 +-- src/update/test/utils/report.js | 27 +----- src/update/test/utils/reset.js | 16 ++-- src/update/test/utils/snapshot.js | 8 +- 40 files changed, 419 insertions(+), 700 deletions(-) rename src/update/lib/{unpaywall => }/controllers/cron.js (100%) rename src/update/lib/{unpaywall => }/controllers/file.js (73%) rename src/update/lib/{unpaywall => }/controllers/job.js (59%) rename src/update/lib/{unpaywall => cron}/cron.js (95%) rename src/update/lib/{unpaywall => }/cron/file.js (84%) rename src/update/lib/{unpaywall => }/cron/update.js (93%) rename src/update/lib/{file.js => files.js} (100%) rename src/update/lib/{unpaywallHistory => }/history.js (97%) rename src/update/lib/{unpaywall => }/insert.js (96%) rename src/update/lib/{unpaywall => }/job.js (60%) create mode 100644 src/update/lib/middlewares/format/type.js delete mode 100644 src/update/lib/middlewares/type.js rename src/update/lib/{unpaywall => }/routers/cron.js (85%) create mode 100644 src/update/lib/routers/job.js rename src/update/lib/{unpaywall => }/routers/report.js (54%) create mode 100644 src/update/lib/routers/snapshot.js delete mode 100644 src/update/lib/unpaywall/controllers/report.js delete mode 100644 src/update/lib/unpaywall/routers/job.js delete mode 100644 src/update/lib/unpaywall/routers/snapshot.js delete mode 100644 src/update/lib/unpaywallHistory/controllers/job.js delete mode 100644 src/update/lib/unpaywallHistory/job.js delete mode 100644 src/update/lib/unpaywallHistory/routers/job.js delete mode 100644 src/update/lib/unpaywallHistory/routers/report.js diff --git a/src/update/app.js b/src/update/app.js index 5d3ce392..8739291b 100644 --- a/src/update/app.js +++ b/src/update/app.js @@ -14,16 +14,12 @@ const routerUnpaywall = require('./lib/routers/unpaywall'); const routerStatus = require('./lib/routers/status'); const routerState = require('./lib/routers/state'); -const routerUnpaywallJob = require('./lib/unpaywall/routers/job'); -const routerUnpaywallReport = require('./lib/unpaywall/routers/report'); +const routerJob = require('./lib/routers/job'); +const routerReport = require('./lib/routers/report'); +const routerCron = require('./lib/routers/cron'); +const routerSnapshot = require('./lib/routers/snapshot'); -const routerUnpaywallCron = require('./lib/unpaywall/routers/cron'); -const routerUnpaywallSnapshot = require('./lib/unpaywall/routers/snapshot'); - -const routerUnpaywallHistoryJob = require('./lib/unpaywallHistory/routers/job'); -const routerUnpaywallHistoryReport = require('./lib/unpaywallHistory/routers/report'); - -require('./lib/unpaywall/cron/file'); +require('./lib/cron/file'); // create data directory const dataDir = path.resolve(__dirname, 'data'); @@ -74,13 +70,10 @@ app.use(routerUnpaywall); app.use(routerStatus); app.use(routerState); -app.use(routerUnpaywallJob); -app.use(routerUnpaywallReport); -app.use(routerUnpaywallCron); -app.use(routerUnpaywallSnapshot); - -app.use(routerUnpaywallHistoryJob); -app.use(routerUnpaywallHistoryReport); +app.use(routerJob); +app.use(routerReport); +app.use(routerCron); +app.use(routerSnapshot); /* Errors and unknown routes */ app.use((req, res, next) => res.status(404).json({ message: `Cannot ${req.method} ${req.originalUrl}` })); diff --git a/src/update/lib/unpaywall/controllers/cron.js b/src/update/lib/controllers/cron.js similarity index 100% rename from src/update/lib/unpaywall/controllers/cron.js rename to src/update/lib/controllers/cron.js diff --git a/src/update/lib/unpaywall/controllers/file.js b/src/update/lib/controllers/file.js similarity index 73% rename from src/update/lib/unpaywall/controllers/file.js rename to src/update/lib/controllers/file.js index ac6f7e42..d63a0748 100644 --- a/src/update/lib/unpaywall/controllers/file.js +++ b/src/update/lib/controllers/file.js @@ -1,14 +1,9 @@ const fs = require('fs-extra'); const path = require('path'); -const dirPath = require('../../path'); +const { getMostRecentFile, deleteFile } = require('../files'); -const { - getMostRecentFile, - deleteFile, -} = require('../../file'); - -const { snapshotsDir } = dirPath.unpaywall; +const { getPathOfDirectory } = require('../files'); /** * Controller to start list of files or latest file downloaded on update service. @@ -18,18 +13,20 @@ const { snapshotsDir } = dirPath.unpaywall; * @param {import('express').NextFunction} next - Do the following. */ async function getFiles(req, res, next) { - const { latest } = req.data; + const { latest, type } = req.data; + + const pathOfDirectory = getPathOfDirectory(type, 'snapshots'); if (latest) { let latestSnapshot; try { - latestSnapshot = await getMostRecentFile(snapshotsDir); + latestSnapshot = await getMostRecentFile(pathOfDirectory); } catch (err) { return next({ message: 'Cannot get the lastest snapshot' }); } return res.status(200).json(latestSnapshot?.filename); } - const files = await fs.readdir(snapshotsDir); + const files = await fs.readdir(pathOfDirectory); return res.status(200).json(files); } @@ -53,14 +50,16 @@ async function uploadFile(req, res, next) { * @param {import('express').NextFunction} next - Do the following. */ async function deleteInstalledFile(req, res, next) { - const { filename } = req.data; + const { filename, type } = req.data; + + const pathOfDirectory = getPathOfDirectory(type, 'snapshots'); - if (!await fs.pathExists(path.resolve(snapshotsDir, filename))) { + if (!await fs.pathExists(path.resolve(pathOfDirectory, filename))) { return res.status(404).json({ message: `File [${filename}] not found` }); } try { - await deleteFile(path.resolve(snapshotsDir, filename)); + await deleteFile(path.resolve(pathOfDirectory, filename)); } catch (err) { return next({ message: err.message }); } diff --git a/src/update/lib/unpaywall/controllers/job.js b/src/update/lib/controllers/job.js similarity index 59% rename from src/update/lib/unpaywall/controllers/job.js rename to src/update/lib/controllers/job.js index 959043cc..04e845d7 100644 --- a/src/update/lib/unpaywall/controllers/job.js +++ b/src/update/lib/controllers/job.js @@ -1,7 +1,7 @@ const fs = require('fs-extra'); const path = require('path'); const { format } = require('date-fns'); -const dirPath = require('../../path'); +const dirPath = require('../path'); const { snapshotsDir } = dirPath.unpaywall; @@ -9,8 +9,13 @@ const { downloadAndInsertSnapshot, insertChangefilesOnPeriod, insertChangefile, + insertWithOaHistoryJob, } = require('../job'); +const { + rollBack, +} = require('../history'); + /** * Controller to start job that download and insert snapshot. * @@ -96,8 +101,68 @@ async function insertChangefileJob(req, res, next) { return res.status(202).json(); } +/** + * Controller to start job that download ans insert changefiles on period. + * + * @param {import('express').Request} req - HTTP request. + * @param {import('express').Response} res - HTTP response. + * @param {import('express').NextFunction} next - Do the following. + */ +async function insertWithOaHistory(req, res, next) { + const { jobConfig } = req.data; + + const { + startDate, + endDate, + interval, + } = jobConfig; + + if (new Date(startDate).getTime() > Date.now()) { + return res.status(400).json({ message: 'startDate cannot be in the future' }); + } + + if (startDate && endDate) { + if (new Date(endDate).getTime() < new Date(startDate).getTime()) { + return res.status(400).json({ message: 'endDate cannot be lower than startDate' }); + } + } + + jobConfig.offset = 0; + jobConfig.limit = -1; + + if (!startDate && !endDate) { + jobConfig.endDate = format(new Date(), 'yyyy-MM-dd'); + if (interval === 'week') jobConfig.startDate = format(new Date() - (7 * 24 * 60 * 60 * 1000), 'yyyy-MM-dd'); + if (interval === 'day') jobConfig.startDate = format(new Date(), 'yyyy-MM-dd'); + + insertWithOaHistoryJob(jobConfig); + return res.status(202).json(); + } + + if (startDate && !endDate) jobConfig.endDate = format(new Date(), 'yyyy-MM-dd'); + + insertWithOaHistoryJob(jobConfig); + return res.status(202).json(); +} + +/** + * Controller to start job that download ans insert changefiles on period. + * + * @param {import('express').Request} req - HTTP request. + * @param {import('express').Response} res - HTTP response. + * @param {import('express').NextFunction} next - Do the following. + */ +async function historyRollBack(req, res, next) { + const { startDate, index } = req.data.rollBackConfig; + + await rollBack(startDate, index); + return res.status(202).json(); +} + module.exports = { downloadAndInsertSnapshotJob, insertChangefilesOnPeriodJob, insertChangefileJob, + insertWithOaHistory, + historyRollBack, }; diff --git a/src/update/lib/controllers/report.js b/src/update/lib/controllers/report.js index d15f4930..94bb6030 100644 --- a/src/update/lib/controllers/report.js +++ b/src/update/lib/controllers/report.js @@ -1,9 +1,9 @@ const path = require('path'); const fs = require('fs-extra'); -const { getMostRecentFile } = require('../file'); +const { getMostRecentFile } = require('../files'); const { getReport } = require('../report'); -const { getPathOfDirectory } = require('../file'); +const { getPathOfDirectory } = require('../files'); /** * Controller to get list of reports or latest report. diff --git a/src/update/lib/unpaywall/cron.js b/src/update/lib/cron/cron.js similarity index 95% rename from src/update/lib/unpaywall/cron.js rename to src/update/lib/cron/cron.js index 4b768840..a006c486 100644 --- a/src/update/lib/unpaywall/cron.js +++ b/src/update/lib/cron/cron.js @@ -2,7 +2,7 @@ const { format, subDays } = require('date-fns'); const Cron = require('../cron'); -const { insertChangefilesOnPeriod } = require('./job'); +const { insertChangefilesOnPeriod } = require('../job'); const updateConfig = { index: 'unpaywall', diff --git a/src/update/lib/unpaywall/cron/file.js b/src/update/lib/cron/file.js similarity index 84% rename from src/update/lib/unpaywall/cron/file.js rename to src/update/lib/cron/file.js index 4c9fcec7..ed7c6be1 100644 --- a/src/update/lib/unpaywall/cron/file.js +++ b/src/update/lib/cron/file.js @@ -1,7 +1,7 @@ -const logger = require('../../logger'); -const Cron = require('../../cron'); -const dirPath = require('../../path'); -const { deleteFilesInDir } = require('../../file'); +const logger = require('../logger'); +const Cron = require('../cron'); +const dirPath = require('../path'); +const { deleteFilesInDir } = require('../files'); const { reportsDir, snapshotsDir, statesDir } = dirPath; diff --git a/src/update/lib/unpaywall/cron/update.js b/src/update/lib/cron/update.js similarity index 93% rename from src/update/lib/unpaywall/cron/update.js rename to src/update/lib/cron/update.js index bd961d63..df7a4c05 100644 --- a/src/update/lib/unpaywall/cron/update.js +++ b/src/update/lib/cron/update.js @@ -1,9 +1,9 @@ const { format, subDays } = require('date-fns'); const { unpaywallCron } = require('config'); -const logger = require('../../logger'); +const logger = require('../logger'); -const Cron = require('../../cron'); -const { getStatus } = require('../../status'); +const Cron = require('../cron'); +const { getStatus } = require('../status'); const { insertChangefilesOnPeriod } = require('../job'); diff --git a/src/update/lib/download.js b/src/update/lib/download.js index 94c16468..52458728 100644 --- a/src/update/lib/download.js +++ b/src/update/lib/download.js @@ -18,7 +18,7 @@ const { getChangefile, } = require('./services/unpaywall'); -const { getPathOfDirectory } = require('./file'); +const { getPathOfDirectory } = require('./files'); const pathDir = require('./path'); /** diff --git a/src/update/lib/file.js b/src/update/lib/files.js similarity index 100% rename from src/update/lib/file.js rename to src/update/lib/files.js diff --git a/src/update/lib/unpaywallHistory/history.js b/src/update/lib/history.js similarity index 97% rename from src/update/lib/unpaywallHistory/history.js rename to src/update/lib/history.js index 1d477e0f..4859aa8a 100644 --- a/src/update/lib/unpaywallHistory/history.js +++ b/src/update/lib/history.js @@ -9,17 +9,17 @@ const config = require('config'); const zlib = require('zlib'); const { format } = require('date-fns'); -const dirPath = require('../path'); -const logger = require('../logger'); -const unpaywallEnrichedMapping = require('../../mapping/unpaywall.json'); -const unpaywallHistoryMapping = require('../../mapping/unpaywall_history.json'); +const dirPath = require('./path'); +const logger = require('./logger'); +const unpaywallEnrichedMapping = require('../mapping/unpaywall.json'); +const unpaywallHistoryMapping = require('../mapping/unpaywall_history.json'); const { addStepInsert, getLatestStep, updateLatestStep, fail, -} = require('../state'); +} = require('./state'); const { refreshIndex, @@ -27,7 +27,7 @@ const { bulk, createIndex, searchWithRange, -} = require('../services/elastic'); +} = require('./services/elastic'); const maxBulkSize = config.get('elasticsearch.maxBulkSize'); diff --git a/src/update/lib/unpaywall/insert.js b/src/update/lib/insert.js similarity index 96% rename from src/update/lib/unpaywall/insert.js rename to src/update/lib/insert.js index 0b22540e..1854cbee 100644 --- a/src/update/lib/unpaywall/insert.js +++ b/src/update/lib/insert.js @@ -8,23 +8,23 @@ const readline = require('readline'); const config = require('config'); const zlib = require('zlib'); -const dirPath = require('../path'); -const logger = require('../logger'); -const unpaywallMapping = require('../../mapping/unpaywall.json'); +const dirPath = require('./path'); +const logger = require('./logger'); +const unpaywallMapping = require('../mapping/unpaywall.json'); const { addStepInsert, getLatestStep, updateLatestStep, fail, -} = require('../state'); +} = require('./state'); const { refreshIndex, bulk, initAlias, createIndex, -} = require('../services/elastic'); +} = require('./services/elastic'); const indexAlias = config.get('elasticsearch.indexAlias'); const maxBulkSize = config.get('elasticsearch.maxBulkSize'); diff --git a/src/update/lib/unpaywall/job.js b/src/update/lib/job.js similarity index 60% rename from src/update/lib/unpaywall/job.js rename to src/update/lib/job.js index 7818f361..e4456c4e 100644 --- a/src/update/lib/unpaywall/job.js +++ b/src/update/lib/job.js @@ -1,10 +1,10 @@ /* eslint-disable no-param-reassign */ -const logger = require('../logger'); +const logger = require('./logger'); -const { setInUpdate } = require('../status'); +const { setInUpdate } = require('./status'); -const { downloadBigSnapshot, downloadChangefile } = require('../download'); +const { downloadBigSnapshot, downloadChangefile } = require('./download'); const { endState, @@ -13,13 +13,15 @@ const { addStepGetChangefiles, updateLatestStep, getLatestStep, -} = require('../state'); +} = require('./state'); const insertDataUnpaywall = require('./insert'); -const { getChangefiles } = require('../services/unpaywall'); +const { insertHistoryDataUnpaywall } = require('./history'); -const { sendMailNoChangefile } = require('../services/mail'); +const { getChangefiles } = require('./services/unpaywall'); + +const { sendMailNoChangefile } = require('./services/mail'); /** * Download the current snapshot of unpaywall and insert his content. @@ -117,8 +119,64 @@ async function insertChangefile(jobConfig) { } } +/** + * Download and insert on elastic the changefiles from unpaywall between a period with history. + * + * @param {Object} jobConfig - Config of job. + * @param {string} jobConfig.index - Name of the index to which the data will be inserted. + * @param {string} jobConfig.interval - Interval of changefile, day or week are available. + * @param {string} jobConfig.startDate - Start date for the changefile period. + * @param {string} jobConfig.endDate - End date for the changefile period. + * @param {number} jobConfig.offset - Line of the snapshot at which the data insertion starts. + * @param {number} jobConfig.limit - Line in the file where the insertion stops. + * + * @returns {Promise} + */ +async function insertWithOaHistoryJob(jobConfig) { + setInUpdate(true); + const { + interval, startDate, endDate, + } = jobConfig; + await createState('unpaywallHistory'); + const start = new Date(); + addStepGetChangefiles(); + const step = getLatestStep(); + const changefilesInfo = await getChangefiles(interval, startDate, endDate); + + if (!changefilesInfo) { + step.status = 'error'; + updateLatestStep(step); + await fail(); + return; + } + + step.took = (new Date() - start) / 1000; + step.status = 'success'; + updateLatestStep(step); + + if (changefilesInfo.length === 0) { + sendMailNoChangefile(startDate, endDate).catch((err) => { + logger.errorRequest(err); + }); + await endState(); + return; + } + + let success = true; + for (let i = 0; i < changefilesInfo.length; i += 1) { + success = await downloadChangefile('unpaywallHistory', changefilesInfo[i], interval); + if (!success) return; + jobConfig.filename = changefilesInfo[i].filename; + jobConfig.date = changefilesInfo[i].date; + success = await insertHistoryDataUnpaywall(jobConfig); + if (!success) return; + } + await endState(); +} + module.exports = { downloadAndInsertSnapshot, insertChangefilesOnPeriod, insertChangefile, + insertWithOaHistoryJob, }; diff --git a/src/update/lib/middlewares/format/type.js b/src/update/lib/middlewares/format/type.js new file mode 100644 index 00000000..a673ef36 --- /dev/null +++ b/src/update/lib/middlewares/format/type.js @@ -0,0 +1,25 @@ +const joi = require('joi'); + +/** + * Joi middleware to check if type in param is correct. + * + * @param {import('express').Request} req - HTTP request. + * @param {import('express').Response} res - HTTP response. + * @param {import('express').NextFunction} next - Do the following. + */ +async function validateType(req, res, next) { + const { type } = req.params; + + const { error, value } = joi.string().trim().valid('unpaywall', 'unpaywallHistory').validate(type); + if (error) return res.status(400).json({ message: error.details[0].message }); + + if (!req.data) { + req.data = {}; + } + + req.data.type = value; + + return next(); +} + +module.exports = validateType; diff --git a/src/update/lib/middlewares/type.js b/src/update/lib/middlewares/type.js deleted file mode 100644 index ffc9d4a5..00000000 --- a/src/update/lib/middlewares/type.js +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Joi middleware to check if interval in body is correct. - * - * @param {import('express').Request} req - HTTP request. - * @param {import('express').Response} res - HTTP response. - * @param {import('express').NextFunction} next - Do the following. - */ -async function updateType(req, res, next) { - let type = 'unpaywall'; - - if (req.originalUrl.split('/').includes('history')) { - type = 'unpaywallHistory'; - } - - if (!req.data) { - req.data = {}; - } - - req.data.type = type; - - return next(); -} - -module.exports = updateType; diff --git a/src/update/lib/report.js b/src/update/lib/report.js index 07a2a780..af09c7d3 100644 --- a/src/update/lib/report.js +++ b/src/update/lib/report.js @@ -2,7 +2,7 @@ const path = require('path'); const fs = require('fs-extra'); const { format } = require('date-fns'); const logger = require('./logger'); -const { getPathOfDirectory } = require('./file'); +const { getPathOfDirectory } = require('./files'); /** * Create report on the folder as name the date of process. diff --git a/src/update/lib/unpaywall/routers/cron.js b/src/update/lib/routers/cron.js similarity index 85% rename from src/update/lib/unpaywall/routers/cron.js rename to src/update/lib/routers/cron.js index 166f714d..f844faa3 100644 --- a/src/update/lib/unpaywall/routers/cron.js +++ b/src/update/lib/routers/cron.js @@ -1,7 +1,7 @@ const router = require('express').Router(); -const checkAuth = require('../../middlewares/auth'); -const validateCronConfig = require('../../middlewares/format/cron'); +const checkAuth = require('../middlewares/auth'); +const validateCronConfig = require('../middlewares/format/cron'); const { startUpdateCron, diff --git a/src/update/lib/routers/job.js b/src/update/lib/routers/job.js new file mode 100644 index 00000000..4b469e03 --- /dev/null +++ b/src/update/lib/routers/job.js @@ -0,0 +1,90 @@ +const router = require('express').Router(); + +const { + downloadAndInsertSnapshotJob, + insertChangefilesOnPeriodJob, + insertChangefileJob, +} = require('../controllers/job'); + +const { + validateSnapshotJob, + validateJobChangefilesConfig, + validateInsertFile, +} = require('../middlewares/format/job'); + +const { insertWithOaHistory, historyRollBack } = require('../controllers/job'); + +const { step1, step2, step3 } = require('../history'); + +const { validateHistoryJob, validateHistoryReset } = require('../middlewares/format/job'); +const checkStatus = require('../middlewares/status'); +const checkAuth = require('../middlewares/auth'); + +/** + * Route that download the current snapshot of unpaywall and insert his content. + * Auth required. + * No update process should be in progress. + * + * This route need a body that contains a config of job. + */ +router.post('/job/snapshot', checkStatus, checkAuth, validateSnapshotJob, downloadAndInsertSnapshotJob); + +/** + * Route that download and insert on elastic the changefiles from unpaywall between a period. + * Auth required. + * No update process should be in progress. + * + * This route need a body that contains a config of job. + */ +router.post('/job/period', checkStatus, checkAuth, validateJobChangefilesConfig, insertChangefilesOnPeriodJob); + +/** + * Route that insert on elastic the content of file installed on ezunpaywall. + * Auth required. + * No update process should be in progress. + * + * This route need a body that contains a config of job + * and a param which corresponds to the filename. + */ +router.post('/job/changefile/:filename', checkStatus, checkAuth, validateInsertFile, insertChangefileJob); + +/** + * Route that download and insert on elastic the changefiles from unpaywall between a period. + * Auth required. + * No update process should be in progress. + * + * This route need a body that contains a config of job. + */ +router.post('/job/history', checkStatus, checkAuth, validateHistoryJob, insertWithOaHistory); + +/** + * Route that roll back the current and the history index according to a date. + * Auth required. + * No update process should be in progress. + * + * This route need a body that contains a config of job. + */ +router.post('/job/history/reset', checkStatus, checkAuth, validateHistoryReset, historyRollBack); + +router.post('/job/history/reset/step1', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { + const { startDate } = req.data; + + await step1(startDate); + return res.status(202).json(); +}); + +router.post('/job/history/reset/step2', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { + const { startDate } = req.data; + + await step2(startDate); + return res.status(202).json(); +}); + +router.post('/job/history/reset/step3', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { + const { startDate } = req.data; + + await step3(startDate); + return res.status(202).json(); +}); + +module.exports = router; diff --git a/src/update/lib/unpaywall/routers/report.js b/src/update/lib/routers/report.js similarity index 54% rename from src/update/lib/unpaywall/routers/report.js rename to src/update/lib/routers/report.js index 5c2c266d..35cd9b50 100644 --- a/src/update/lib/unpaywall/routers/report.js +++ b/src/update/lib/routers/report.js @@ -2,22 +2,22 @@ const router = require('express').Router(); const { getReports, getReportByFilename } = require('../controllers/report'); -const validateLatest = require('../../middlewares/format/latest'); -const validateFilename = require('../../middlewares/format/filename'); -const updateType = require('../../middlewares/type'); +const validateLatest = require('../middlewares/format/latest'); +const validateFilename = require('../middlewares/format/filename'); +const validateType = require('../middlewares/format/type'); /** * Route that give the list of reports or the content of most recent report in JSON format. * * This route can take in query latest. */ -router.get('/reports', validateLatest, updateType, getReports); +router.get('/reports/:type', validateType, validateLatest, getReports); /** * Route that give the content of report in JSON format. * * This route takes a param which corresponds to the filename of report. */ -router.get('/reports/:filename', validateFilename, updateType, getReportByFilename); +router.get('/reports/:type/:filename', validateType, validateFilename, getReportByFilename); module.exports = router; diff --git a/src/update/lib/routers/snapshot.js b/src/update/lib/routers/snapshot.js new file mode 100644 index 00000000..62ccd003 --- /dev/null +++ b/src/update/lib/routers/snapshot.js @@ -0,0 +1,36 @@ +const router = require('express').Router(); + +const upload = require('../middlewares/multer'); +const checkAuth = require('../middlewares/auth'); +const dev = require('../middlewares/dev'); + +const validateLatest = require('../middlewares/format/latest'); +const validateFilename = require('../middlewares/format/filename'); +const validateType = require('../middlewares/format/type'); + +const { getFiles, uploadFile, deleteInstalledFile } = require('../controllers/file'); + +/** + * Route that give the list of files installed on ezunpaywall of the most recent file. + * Auth required. + * + * This route can take in query latest. + */ +router.get('/snapshots/:type', checkAuth, validateType, validateLatest, getFiles); + +/** + * Route that upload a file on ezunpaywall. + * Auth required. + * Using for test. + * + * This route need a body that contains the file to upload. + */ +router.post('/snapshots/:type', dev, checkAuth, validateType, upload.single('file'), uploadFile); + +/** + * Route that delete a file on ezunpaywall. + * Auth required. + */ +router.delete('/snapshots/:type/:filename', dev, checkAuth, validateType, validateFilename, deleteInstalledFile); + +module.exports = router; diff --git a/src/update/lib/unpaywall/controllers/report.js b/src/update/lib/unpaywall/controllers/report.js deleted file mode 100644 index 008f7e52..00000000 --- a/src/update/lib/unpaywall/controllers/report.js +++ /dev/null @@ -1,79 +0,0 @@ -const path = require('path'); -const fs = require('fs-extra'); - -const { getMostRecentFile } = require('../../file'); -const { getReport } = require('../../report'); -const dirPath = require('../../path'); - -const { reportsDir } = dirPath.unpaywall; -/** - * Controller to get list of reports or latest report. - * - * @param {import('express').Request} req - HTTP request. - * @param {import('express').Response} res - HTTP response. - * @param {import('express').NextFunction} next - Do the following. - */ -async function getReports(req, res, next) { - const { latest } = req.data; - - if (latest) { - let latestFile; - try { - latestFile = await getMostRecentFile(reportsDir); - } catch (err) { - return next({ message: err.message }); - } - - if (!latestFile) { - return next({ message: 'No reports are available' }); - } - - let report; - try { - report = await getReport('unpaywall', latestFile?.filename); - } catch (err) { - return next({ message: `Cannot get [${latestFile?.filename}] latest report` }); - } - return res.status(200).json(report); - } - - let reports = await fs.readdir(reportsDir); - - reports = reports.sort((a, b) => { - const [date1] = a.split('.'); - const [date2] = b.split('.'); - return new Date(date2).getTime() - new Date(date1).getTime(); - }); - - return res.status(200).json(reports); -} - -/** - * Controller to get report by filename. - * - * @param {import('express').Request} req - HTTP request. - * @param {import('express').Response} res - HTTP response. - * @param {import('express').NextFunction} next - Do the following. - */ -async function getReportByFilename(req, res, next) { - const { filename } = req.data; - - try { - await fs.stat(path.resolve(reportsDir, filename)); - } catch (err) { - return next({ message: `File [${filename}] not found` }); - } - - let report; - try { - report = await getReport('unpaywall', filename); - } catch (err) { - return next({ message: `Cannot get ${filename} report` }); - } - return res.status(200).json(report); -} - -module.exports = { - getReports, - getReportByFilename, -}; diff --git a/src/update/lib/unpaywall/routers/job.js b/src/update/lib/unpaywall/routers/job.js deleted file mode 100644 index 543e5f87..00000000 --- a/src/update/lib/unpaywall/routers/job.js +++ /dev/null @@ -1,46 +0,0 @@ -const router = require('express').Router(); - -const { - downloadAndInsertSnapshotJob, - insertChangefilesOnPeriodJob, - insertChangefileJob, -} = require('../controllers/job'); - -const { - validateSnapshotJob, - validateJobChangefilesConfig, - validateInsertFile, -} = require('../../middlewares/format/job'); - -const checkStatus = require('../../middlewares/status'); -const checkAuth = require('../../middlewares/auth'); - -/** - * Route that download the current snapshot of unpaywall and insert his content. - * Auth required. - * No update process should be in progress. - * - * This route need a body that contains a config of job. - */ -router.post('/job/snapshot', checkStatus, checkAuth, validateSnapshotJob, downloadAndInsertSnapshotJob); - -/** - * Route that download and insert on elastic the changefiles from unpaywall between a period. - * Auth required. - * No update process should be in progress. - * - * This route need a body that contains a config of job. - */ -router.post('/job/period', checkStatus, checkAuth, validateJobChangefilesConfig, insertChangefilesOnPeriodJob); - -/** - * Route that insert on elastic the content of file installed on ezunpaywall. - * Auth required. - * No update process should be in progress. - * - * This route need a body that contains a config of job - * and a param which corresponds to the filename. - */ -router.post('/job/changefile/:filename', checkStatus, checkAuth, validateInsertFile, insertChangefileJob); - -module.exports = router; diff --git a/src/update/lib/unpaywall/routers/snapshot.js b/src/update/lib/unpaywall/routers/snapshot.js deleted file mode 100644 index 369c2e8c..00000000 --- a/src/update/lib/unpaywall/routers/snapshot.js +++ /dev/null @@ -1,39 +0,0 @@ -const router = require('express').Router(); - -const upload = require('../../middlewares/multer'); -const checkAuth = require('../../middlewares/auth'); -const dev = require('../../middlewares/dev'); - -const validateLatest = require('../../middlewares/format/latest'); -const validateFilename = require('../../middlewares/format/filename'); - -const { - getFiles, - uploadFile, - deleteInstalledFile, -} = require('../controllers/file'); - -/** - * Route that give the list of snapshots installed on ezunpaywall of the most recent file. - * Auth required. - * - * This route can take in query latest. - */ -router.get('/snapshots', checkAuth, validateLatest, getFiles); - -/** - * Route that upload a file on ezunpaywall. - * Auth required. - * Using for test. - * - * This route need a body that contains the file to upload. - */ -router.post('/snapshots', dev, checkAuth, upload.single('file'), uploadFile); - -/** - * Route that delete a file on ezunpaywall. - * Auth required. - */ -router.delete('/snapshots/:filename', dev, checkAuth, validateFilename, deleteInstalledFile); - -module.exports = router; diff --git a/src/update/lib/unpaywallHistory/controllers/job.js b/src/update/lib/unpaywallHistory/controllers/job.js deleted file mode 100644 index f002bb9f..00000000 --- a/src/update/lib/unpaywallHistory/controllers/job.js +++ /dev/null @@ -1,71 +0,0 @@ -const { format } = require('date-fns'); - -const { - insertWithOaHistoryJob, -} = require('../job'); - -const { - rollBack, -} = require('../history'); -/** - * Controller to start job that download ans insert changefiles on period. - * - * @param {import('express').Request} req - HTTP request. - * @param {import('express').Response} res - HTTP response. - * @param {import('express').NextFunction} next - Do the following. - */ -async function insertWithOaHistory(req, res, next) { - const { jobConfig } = req.data; - - const { - startDate, - endDate, - interval, - } = jobConfig; - - if (new Date(startDate).getTime() > Date.now()) { - return res.status(400).json({ message: 'startDate cannot be in the future' }); - } - - if (startDate && endDate) { - if (new Date(endDate).getTime() < new Date(startDate).getTime()) { - return res.status(400).json({ message: 'endDate cannot be lower than startDate' }); - } - } - - jobConfig.offset = 0; - jobConfig.limit = -1; - - if (!startDate && !endDate) { - jobConfig.endDate = format(new Date(), 'yyyy-MM-dd'); - if (interval === 'week') jobConfig.startDate = format(new Date() - (7 * 24 * 60 * 60 * 1000), 'yyyy-MM-dd'); - if (interval === 'day') jobConfig.startDate = format(new Date(), 'yyyy-MM-dd'); - - insertWithOaHistoryJob(jobConfig); - return res.status(202).json(); - } - - if (startDate && !endDate) jobConfig.endDate = format(new Date(), 'yyyy-MM-dd'); - - insertWithOaHistoryJob(jobConfig); - return res.status(202).json(); -} - -/** - * Controller to start job that download ans insert changefiles on period. - * - * @param {import('express').Request} req - HTTP request. - * @param {import('express').Response} res - HTTP response. - * @param {import('express').NextFunction} next - Do the following. - */ -async function historyRollBack(req, res, next) { - const { startDate, index } = req.data.rollBackConfig; - - await rollBack(startDate, index); - return res.status(202).json(); -} - -module.exports = { - insertWithOaHistory, - historyRollBack, -}; diff --git a/src/update/lib/unpaywallHistory/job.js b/src/update/lib/unpaywallHistory/job.js deleted file mode 100644 index dd46a953..00000000 --- a/src/update/lib/unpaywallHistory/job.js +++ /dev/null @@ -1,77 +0,0 @@ -/* eslint-disable no-param-reassign */ - -const logger = require('../logger'); -const { setInUpdate } = require('../status'); -const { downloadChangefile } = require('../download'); - -const { - endState, - fail, - createState, - addStepGetChangefiles, - updateLatestStep, - getLatestStep, -} = require('../state'); - -const { insertHistoryDataUnpaywall } = require('./history'); -const { getChangefiles } = require('../services/unpaywall'); -const { sendMailNoChangefile } = require('../services/mail'); - -/** - * Download and insert on elastic the changefiles from unpaywall between a period with history. - * - * @param {Object} jobConfig - Config of job. - * @param {string} jobConfig.index - Name of the index to which the data will be inserted. - * @param {string} jobConfig.interval - Interval of changefile, day or week are available. - * @param {string} jobConfig.startDate - Start date for the changefile period. - * @param {string} jobConfig.endDate - End date for the changefile period. - * @param {number} jobConfig.offset - Line of the snapshot at which the data insertion starts. - * @param {number} jobConfig.limit - Line in the file where the insertion stops. - * - * @returns {Promise} - */ -async function insertWithOaHistoryJob(jobConfig) { - setInUpdate(true); - const { - interval, startDate, endDate, - } = jobConfig; - await createState('unpaywallHistory'); - const start = new Date(); - addStepGetChangefiles(); - const step = getLatestStep(); - const changefilesInfo = await getChangefiles(interval, startDate, endDate); - - if (!changefilesInfo) { - step.status = 'error'; - updateLatestStep(step); - await fail(); - return; - } - - step.took = (new Date() - start) / 1000; - step.status = 'success'; - updateLatestStep(step); - - if (changefilesInfo.length === 0) { - sendMailNoChangefile(startDate, endDate).catch((err) => { - logger.errorRequest(err); - }); - await endState(); - return; - } - - let success = true; - for (let i = 0; i < changefilesInfo.length; i += 1) { - success = await downloadChangefile('unpaywallHistory', changefilesInfo[i], interval); - if (!success) return; - jobConfig.filename = changefilesInfo[i].filename; - jobConfig.date = changefilesInfo[i].date; - success = await insertHistoryDataUnpaywall(jobConfig); - if (!success) return; - } - await endState(); -} - -module.exports = { - insertWithOaHistoryJob, -}; diff --git a/src/update/lib/unpaywallHistory/routers/job.js b/src/update/lib/unpaywallHistory/routers/job.js deleted file mode 100644 index 03e4e7af..00000000 --- a/src/update/lib/unpaywallHistory/routers/job.js +++ /dev/null @@ -1,70 +0,0 @@ -const router = require('express').Router(); - -const { - insertWithOaHistory, - historyRollBack, -} = require('../controllers/job'); - -const { - step1, - step2, - step3, -} = require('../history'); - -const { - validateHistoryJob, - validateHistoryReset, -} = require('../../middlewares/format/job'); - -const checkStatus = require('../../middlewares/status'); -const checkAuth = require('../../middlewares/auth'); - -/** - * Route that download and insert on elastic the changefiles from unpaywall between a period. - * Auth required. - * No update process should be in progress. - * - * This route need a body that contains a config of job. - */ -router.post('/history/job', checkStatus, checkAuth, validateHistoryJob, insertWithOaHistory); - -/** - * Route that roll back the current and the history index according to a date. - * Auth required. - * No update process should be in progress. - * - * This route need a body that contains a config of job. - */ -router.post('/history/job/reset', checkStatus, checkAuth, validateHistoryReset, historyRollBack); - -/** - * Route that roll back the current and the history index according to a date. - * Auth required. - * No update process should be in progress. - * - * This route need a body that contains a config of job. - */ -router.post('/history/job/reset', checkStatus, checkAuth, validateHistoryReset, historyRollBack); - -router.post('/history/job/reset/step1', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { - const { startDate } = req.data; - - await step1(startDate); - return res.status(202).json(); -}); - -router.post('/history/job/reset/step2', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { - const { startDate } = req.data; - - await step2(startDate); - return res.status(202).json(); -}); - -router.post('/job/history/reset/step3', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { - const { startDate } = req.data; - - await step3(startDate); - return res.status(202).json(); -}); - -module.exports = router; diff --git a/src/update/lib/unpaywallHistory/routers/report.js b/src/update/lib/unpaywallHistory/routers/report.js deleted file mode 100644 index 4019db03..00000000 --- a/src/update/lib/unpaywallHistory/routers/report.js +++ /dev/null @@ -1,23 +0,0 @@ -const router = require('express').Router(); - -const { getReports, getReportByFilename } = require('../../controllers/report'); - -const validateLatest = require('../../middlewares/format/latest'); -const validateFilename = require('../../middlewares/format/filename'); -const updateType = require('../../middlewares/type'); - -/** - * Route that give the list of reports or the content of most recent report in JSON format. - * - * This route can take in query latest. - */ -router.get('/unpaywall/history/reports', validateLatest, updateType, getReports); - -/** - * Route that give the content of report in JSON format. - * - * This route takes a param which corresponds to the filename of report. - */ -router.get('/unpaywall/history/reports/:filename', validateFilename, updateType, getReportByFilename); - -module.exports = router; diff --git a/src/update/test/auth.js b/src/update/test/auth.js index d4e6e2d3..2bb00a46 100644 --- a/src/update/test/auth.js +++ b/src/update/test/auth.js @@ -24,9 +24,9 @@ describe('Test: auth service in update service', () => { this.timeout(30000); await ping(); await updateChangeFile('week'); - await deleteFile('fake1.jsonl.gz'); - await deleteFile('fake2.jsonl.gz'); - await deleteFile('fake3.jsonl.gz'); + await deleteFile('unpaywall', 'fake1.jsonl.gz'); + await deleteFile('unpaywall', 'fake2.jsonl.gz'); + await deleteFile('unpaywall', 'fake3.jsonl.gz'); await deleteIndex('unpaywall-test'); }); @@ -82,8 +82,8 @@ describe('Test: auth service in update service', () => { after(async () => { await deleteIndex('unpaywall-test'); - await deleteFile('fake1.jsonl.gz'); - await deleteFile('fake2.jsonl.gz'); - await deleteFile('fake3.jsonl.gz'); + await deleteFile('unpaywall', 'fake1.jsonl.gz'); + await deleteFile('unpaywall', 'fake2.jsonl.gz'); + await deleteFile('unpaywall', 'fake3.jsonl.gz'); }); }); diff --git a/src/update/test/unpaywall/insertionError.js b/src/update/test/unpaywall/insertionError.js index 504d49fe..c85aaf01 100644 --- a/src/update/test/unpaywall/insertionError.js +++ b/src/update/test/unpaywall/insertionError.js @@ -3,27 +3,13 @@ const { expect } = require('chai'); const chai = require('chai'); const chaiHttp = require('chai-http'); -const { - countDocuments, -} = require('../utils/elastic'); - -const { - addSnapshot, - updateChangeFile, -} = require('../utils/snapshot'); - -const { - getState, -} = require('../utils/state'); - -const { - getReport, -} = require('../utils/report'); - +const { countDocuments } = require('../utils/elastic'); +const { addSnapshot, updateChangeFile } = require('../utils/snapshot'); +const { getState } = require('../utils/state'); +const getReport = require('../utils/report'); const checkIfInUpdate = require('../utils/status'); const ping = require('../utils/ping'); - const reset = require('../utils/reset'); chai.use(chaiHttp); @@ -40,7 +26,7 @@ describe('Test: insert the content of a file already installed on ezunpaywall', describe('Do insertion of a corrupted file already installed', () => { before(async () => { await reset(); - await addSnapshot('fake1-error.jsonl.gz'); + await addSnapshot('unpaywall', 'fake1-error.jsonl.gz'); }); it('Should return a status code 202', async () => { @@ -90,7 +76,7 @@ describe('Test: insert the content of a file already installed on ezunpaywall', }); it('Should get report with all information from the insertion', async () => { - const report = await getReport(); + const report = await getReport('unpaywall'); expect(report).have.property('done').equal(true); expect(report).have.property('createdAt').to.not.equal(undefined); diff --git a/src/update/test/unpaywall/insertionFile.js b/src/update/test/unpaywall/insertionFile.js index 655b6c4e..8495df0f 100644 --- a/src/update/test/unpaywall/insertionFile.js +++ b/src/update/test/unpaywall/insertionFile.js @@ -3,26 +3,13 @@ const { expect } = require('chai'); const chai = require('chai'); const chaiHttp = require('chai-http'); -const { - countDocuments, -} = require('../utils/elastic'); - -const { - addSnapshot, -} = require('../utils/snapshot'); - -const { - getState, -} = require('../utils/state'); - -const { - getReport, -} = require('../utils/report'); - +const { countDocuments } = require('../utils/elastic'); +const { addSnapshot } = require('../utils/snapshot'); +const { getState } = require('../utils/state'); +const getReport = require('../utils/report'); const checkIfInUpdate = require('../utils/status'); const ping = require('../utils/ping'); - const reset = require('../utils/reset'); chai.use(chaiHttp); @@ -38,7 +25,7 @@ describe('Test: insert the content of a file already installed on ezunpaywall', describe('Do insertion of a file already installed', () => { before(async () => { await reset(); - await addSnapshot('fake1.jsonl.gz'); + await addSnapshot('unpaywall', 'fake1.jsonl.gz'); }); it('Should return a status code 202', async () => { @@ -88,7 +75,7 @@ describe('Test: insert the content of a file already installed on ezunpaywall', }); it('Should get report with all information from the insertion', async () => { - const report = await getReport(); + const report = await getReport('unpaywall'); expect(report).have.property('done').equal(true); expect(report).have.property('createdAt').to.not.equal(undefined); @@ -117,7 +104,7 @@ describe('Test: insert the content of a file already installed on ezunpaywall', describe('Do insertion of a file already installed with parameter limit=10', () => { before(async () => { await reset(); - await addSnapshot('fake1.jsonl.gz'); + await addSnapshot('unpaywall', 'fake1.jsonl.gz'); }); it('Should return a status code 202', async () => { @@ -167,7 +154,7 @@ describe('Test: insert the content of a file already installed on ezunpaywall', }); it('Should get report with all information from the insertion', async () => { - const report = await getReport(); + const report = await getReport('unpaywall'); expect(report).have.property('done').equal(true); expect(report).have.property('createdAt').to.not.equal(undefined); @@ -196,7 +183,7 @@ describe('Test: insert the content of a file already installed on ezunpaywall', describe('Do insertion of a file already installed with parameter offset=40', () => { before(async () => { await reset(); - await addSnapshot('fake1.jsonl.gz'); + await addSnapshot('unpaywall', 'fake1.jsonl.gz'); }); it('Should return a status code 202', async () => { @@ -246,7 +233,7 @@ describe('Test: insert the content of a file already installed on ezunpaywall', }); it('Should get report with all information from the insertion', async () => { - const report = await getReport(); + const report = await getReport('unpaywall'); expect(report).have.property('done').equal(true); expect(report).have.property('createdAt').to.not.equal(undefined); @@ -275,7 +262,7 @@ describe('Test: insert the content of a file already installed on ezunpaywall', describe('Do insertion of a file already installed with parameter offset=10 and limit=20', () => { before(async () => { await reset(); - await addSnapshot('fake1.jsonl.gz'); + await addSnapshot('unpaywall', 'fake1.jsonl.gz'); }); it('Should return a status code 202', async () => { @@ -326,7 +313,7 @@ describe('Test: insert the content of a file already installed on ezunpaywall', }); it('Should get report with all information from the insertion', async () => { - const report = await getReport(); + const report = await getReport('unpaywall'); expect(report).have.property('done').equal(true); expect(report).have.property('createdAt').to.not.equal(undefined); @@ -381,7 +368,7 @@ describe('Test: insert the content of a file already installed on ezunpaywall', describe('Don\'t do a insertion of a file already installed because the parameter limit can\t be lower than offset', () => { before(async () => { await reset(); - await addSnapshot('fake1.jsonl.gz'); + await addSnapshot('unpaywall', 'fake1.jsonl.gz'); }); it('Should return a status code 400', async () => { diff --git a/src/update/test/unpaywall/insertionPeriodDay.js b/src/update/test/unpaywall/insertionPeriodDay.js index e7b34401..8d293278 100644 --- a/src/update/test/unpaywall/insertionPeriodDay.js +++ b/src/update/test/unpaywall/insertionPeriodDay.js @@ -3,28 +3,13 @@ const { expect } = require('chai'); const chai = require('chai'); const chaiHttp = require('chai-http'); -const { - deleteFile, - updateChangeFile, -} = require('../utils/snapshot'); - -const { - getState, -} = require('../utils/state'); - -const { - getReport, -} = require('../utils/report'); - -const { - countDocuments, - deleteIndex, -} = require('../utils/elastic'); - +const { deleteFile, updateChangeFile } = require('../utils/snapshot'); +const { getState } = require('../utils/state'); +const getReport = require('../utils/report'); +const { countDocuments, deleteIndex } = require('../utils/elastic'); const checkIfInUpdate = require('../utils/status'); const ping = require('../utils/ping'); - const reset = require('../utils/reset'); const updateURL = process.env.UPDATE_HOST || 'http://localhost:59702'; @@ -134,7 +119,7 @@ describe('Test: download and insert file from unpaywall between a period', () => }); it('Should get report with all information from the download and insertion', async () => { - const report = await getReport(); + const report = await getReport('unpaywall'); expect(report).have.property('done').equal(true); expect(report).have.property('createdAt').to.not.equal(undefined); @@ -270,7 +255,7 @@ describe('Test: download and insert file from unpaywall between a period', () => }); it('Should get report with all information from the download and insertion', async () => { - const report = await getReport(); + const report = await getReport('unpaywall'); expect(report).have.property('done').equal(true); expect(report).have.property('createdAt').to.not.equal(undefined); @@ -466,8 +451,8 @@ describe('Test: download and insert file from unpaywall between a period', () => after(async () => { await deleteIndex('unpaywall-test'); - await deleteFile('fake1.jsonl.gz'); - await deleteFile('fake2.jsonl.gz'); - await deleteFile('fake3.jsonl.gz'); + await deleteFile('unpaywall', 'fake1.jsonl.gz'); + await deleteFile('unpaywall', 'fake2.jsonl.gz'); + await deleteFile('unpaywall', 'fake3.jsonl.gz'); }); }); diff --git a/src/update/test/unpaywall/insertionPeriodWeek.js b/src/update/test/unpaywall/insertionPeriodWeek.js index 7c54dd94..e98f809f 100644 --- a/src/update/test/unpaywall/insertionPeriodWeek.js +++ b/src/update/test/unpaywall/insertionPeriodWeek.js @@ -3,26 +3,13 @@ const { expect } = require('chai'); const chai = require('chai'); const chaiHttp = require('chai-http'); -const { - updateChangeFile, -} = require('../utils/snapshot'); - -const { - getState, -} = require('../utils/state'); - -const { - getReport, -} = require('../utils/report'); - -const { - countDocuments, -} = require('../utils/elastic'); - +const { updateChangeFile } = require('../utils/snapshot'); +const { getState } = require('../utils/state'); +const getReport = require('../utils/report'); +const { countDocuments } = require('../utils/elastic'); const checkIfInUpdate = require('../utils/status'); const ping = require('../utils/ping'); - const reset = require('../utils/reset'); const updateURL = process.env.UPDATE_HOST || 'http://localhost:59702'; @@ -34,11 +21,11 @@ describe('Week: Test: download and insert file from unpaywall between a period', const oneDay = (1 * 24 * 60 * 60 * 1000); // create date in a format (YYYY-mm-dd) to be use by ezunpaywall - // yersterday + // yesterday const date1 = new Date(now - (1 * oneDay)).toISOString().slice(0, 10); - // yersterday - one week + // yesterday - one week const date2 = new Date(now - (8 * oneDay)).toISOString().slice(0, 10); - // yersterday - two weeks + // yesterday - two weeks const date3 = new Date(now - (15 * oneDay)).toISOString().slice(0, 10); // these dates are for test between a short period const date4 = new Date(now - (4 * oneDay)).toISOString().slice(0, 10); @@ -132,7 +119,7 @@ describe('Week: Test: download and insert file from unpaywall between a period', }); it('Should get report with all information from the download and insertion', async () => { - const report = await getReport(); + const report = await getReport('unpaywall'); expect(report).have.property('done').equal(true); expect(report).have.property('createdAt').to.not.equal(undefined); @@ -266,7 +253,7 @@ describe('Week: Test: download and insert file from unpaywall between a period', }); it('Should get report with all information from the download and insertion', async () => { - const report = await getReport(); + const report = await getReport('unpaywall'); expect(report).have.property('done').equal(true); expect(report).have.property('createdAt').to.not.equal(undefined); diff --git a/src/update/test/unpaywall/insertionSnapshot.js b/src/update/test/unpaywall/insertionSnapshot.js index dad864c2..d0222e28 100644 --- a/src/update/test/unpaywall/insertionSnapshot.js +++ b/src/update/test/unpaywall/insertionSnapshot.js @@ -4,22 +4,12 @@ const chai = require('chai'); const chaiHttp = require('chai-http'); const { format } = require('date-fns'); -const { - countDocuments, -} = require('../utils/elastic'); - -const { - getState, -} = require('../utils/state'); - -const { - getReport, -} = require('../utils/report'); - +const { countDocuments } = require('../utils/elastic'); +const { getState } = require('../utils/state'); +const getReport = require('../utils/report'); const checkIfInUpdate = require('../utils/status'); const ping = require('../utils/ping'); - const reset = require('../utils/reset'); chai.use(chaiHttp); @@ -91,7 +81,7 @@ describe('Test: download and insert snapshot from unpaywall', () => { }); it('Should get report with all information from the insertion', async () => { - const report = await getReport(); + const report = await getReport('unpaywall'); expect(report).have.property('done').equal(true); expect(report).have.property('createdAt').to.not.equal(undefined); diff --git a/src/update/test/unpaywall/updateDay.js b/src/update/test/unpaywall/updateDay.js index b42f5199..25af3e7f 100644 --- a/src/update/test/unpaywall/updateDay.js +++ b/src/update/test/unpaywall/updateDay.js @@ -3,27 +3,13 @@ const { expect } = require('chai'); const chai = require('chai'); const chaiHttp = require('chai-http'); -const { - countDocuments, -} = require('../utils/elastic'); - -const { - addSnapshot, - updateChangeFile, -} = require('../utils/snapshot'); - -const { - getState, -} = require('../utils/state'); - -const { - getReport, -} = require('../utils/report'); - +const { countDocuments } = require('../utils/elastic'); +const { addSnapshot, updateChangeFile } = require('../utils/snapshot'); +const { getState } = require('../utils/state'); +const getReport = require('../utils/report'); const checkIfInUpdate = require('../utils/status'); const ping = require('../utils/ping'); - const reset = require('../utils/reset'); chai.use(chaiHttp); @@ -102,7 +88,7 @@ describe('Test: daily update route test', () => { }); it('Should get report with all information from the daily update', async () => { - const report = await getReport(); + const report = await getReport('unpaywall'); expect(report).have.property('done'); expect(report).have.property('steps').to.be.an('array'); @@ -204,7 +190,7 @@ describe('Test: daily update route test', () => { }); it('Should get report with all information from the weekly update', async () => { - const report = await getReport(); + const report = await getReport('unpaywall'); expect(report).have.property('done'); expect(report).have.property('steps').to.be.an('array'); @@ -290,7 +276,7 @@ describe('Test: daily update route test', () => { }); it('Should get report with all information from the weekly update', async () => { - const report = await getReport(); + const report = await getReport('unpaywall'); expect(report).have.property('done'); expect(report).have.property('steps').to.be.an('array'); @@ -325,7 +311,7 @@ describe('Test: daily update route test', () => { describe('Day: Do a daily update but the file is already installed', () => { before(async () => { await reset(); - await addSnapshot('fake1.jsonl.gz'); + await addSnapshot('unpaywall', 'fake1.jsonl.gz'); }); it('Should return a status code 202', async () => { @@ -379,7 +365,7 @@ describe('Test: daily update route test', () => { }); it('Should get report with all information from the daily update', async () => { - const report = await getReport(); + const report = await getReport('unpaywall'); expect(report).have.property('done').equal(true); expect(report).have.property('steps').to.be.an('array'); diff --git a/src/update/test/unpaywall/updateWeek.js b/src/update/test/unpaywall/updateWeek.js index ebe057fe..cc52492f 100644 --- a/src/update/test/unpaywall/updateWeek.js +++ b/src/update/test/unpaywall/updateWeek.js @@ -3,27 +3,13 @@ const { expect } = require('chai'); const chai = require('chai'); const chaiHttp = require('chai-http'); -const { - countDocuments, -} = require('../utils/elastic'); - -const { - addSnapshot, - updateChangeFile, -} = require('../utils/snapshot'); - -const { - getState, -} = require('../utils/state'); - -const { - getReport, -} = require('../utils/report'); - +const { countDocuments } = require('../utils/elastic'); +const { addSnapshot, updateChangeFile } = require('../utils/snapshot'); +const { getState } = require('../utils/state'); +const getReport = require('../utils/report'); const checkIfInUpdate = require('../utils/status'); const ping = require('../utils/ping'); - const reset = require('../utils/reset'); chai.use(chaiHttp); @@ -102,7 +88,7 @@ describe('Week: Test: weekly update route test', () => { }); it('Should get report with all information from the weekly update', async () => { - const report = await getReport(); + const report = await getReport('unpaywall'); expect(report).have.property('done'); expect(report).have.property('steps').to.be.an('array'); @@ -143,7 +129,7 @@ describe('Week: Test: weekly update route test', () => { describe('Week: Do a weekly update but the file is already installed', () => { before(async () => { await reset(); - await addSnapshot('fake1.jsonl.gz'); + await addSnapshot('unpaywall', 'fake1.jsonl.gz'); }); it('Should return a status code 202', async () => { @@ -196,7 +182,7 @@ describe('Week: Test: weekly update route test', () => { }); it('Should get report with all information from the weekly update', async () => { - const report = await getReport(); + const report = await getReport('unpaywall'); expect(report).have.property('done').equal(true); expect(report).have.property('steps').to.be.an('array'); diff --git a/src/update/test/unpaywallHistory/history.js b/src/update/test/unpaywallHistory/history.js index b83d973a..ba2d1782 100644 --- a/src/update/test/unpaywallHistory/history.js +++ b/src/update/test/unpaywallHistory/history.js @@ -4,16 +4,12 @@ const chai = require('chai'); const chaiHttp = require('chai-http'); const { updateChangeFile } = require('../utils/snapshot'); - const { countDocuments } = require('../utils/elastic'); - const { getState } = require('../utils/state'); -const { getHistoryReport } = require('../utils/report'); - +const getReport = require('../utils/report'); const checkIfInUpdate = require('../utils/status'); const ping = require('../utils/ping'); - const reset = require('../utils/reset'); const updateURL = process.env.UPDATE_HOST || 'http://localhost:59702'; @@ -38,7 +34,7 @@ describe('Test: daily update route test with history', () => { // test response it('Should return a status code 202', async () => { const res = await chai.request(updateURL) - .post('/history/job') + .post('/job/history') .send({ startDate: date3, endDate: date1, @@ -161,7 +157,7 @@ describe('Test: daily update route test with history', () => { }); it('Should get report with all information from the download and insertion', async () => { - const report = await getHistoryReport('unpaywallHistory'); + const report = await getReport('unpaywallHistory'); expect(report).have.property('done').equal(true); expect(report).have.property('createdAt').to.not.equal(undefined); diff --git a/src/update/test/unpaywallHistory/rollback.js b/src/update/test/unpaywallHistory/rollback.js index 685e2dc0..45333d10 100644 --- a/src/update/test/unpaywallHistory/rollback.js +++ b/src/update/test/unpaywallHistory/rollback.js @@ -34,7 +34,7 @@ describe('Test: rollback history test', () => { it('Should return a status code 202', async () => { const res = await chai.request(updateURL) - .post('/history/job/reset') + .post('/job/history/reset') .send({ startDate: date1, }) @@ -45,7 +45,7 @@ describe('Test: rollback history test', () => { it('Should get state with all information', async () => { const res = await chai.request(updateURL) - .post('/history/job/reset') + .post('/job/history/reset') .send({ startDate: date1, }) @@ -56,7 +56,7 @@ describe('Test: rollback history test', () => { it('Should get report with all information', async () => { const res = await chai.request(updateURL) - .post('/history/job/reset') + .post('/job/history/reset') .send({ startDate: date1, }) @@ -96,7 +96,7 @@ describe('Test: rollback history test', () => { it('Should return a status code 202', async () => { const res = await chai.request(updateURL) - .post('/history/job/reset') + .post('/job/history/reset') .send({ startDate: date2, }) @@ -136,7 +136,7 @@ describe('Test: rollback history test', () => { it('Should return a status code 202', async () => { const res = await chai.request(updateURL) - .post('/history/job/reset') + .post('/job/history/reset') .send({ startDate: date3, }) @@ -176,7 +176,7 @@ describe('Test: rollback history test', () => { it('Should return a status code 202', async () => { const res = await chai.request(updateURL) - .post('/history/job/reset') + .post('/job/history/reset') .send({ startDate: date4, }) diff --git a/src/update/test/utils/report.js b/src/update/test/utils/report.js index ff461eaa..70608e6e 100644 --- a/src/update/test/utils/report.js +++ b/src/update/test/utils/report.js @@ -11,11 +11,11 @@ const updateURL = process.env.UPDATE_HOST || 'http://localhost:59702'; * * @returns {Promise{Object}} report */ -async function getReport() { +async function getReport(type) { let res; try { res = await chai.request(updateURL) - .get('/reports') + .get(`/reports/${type}`) .query({ latest: true }); } catch (err) { console.error(`Cannot GET ${updateURL}/report`); @@ -24,25 +24,4 @@ async function getReport() { return res?.body; } -/** - * get report of update - * - * @returns {Promise{Object}} report - */ -async function getHistoryReport() { - let res; - try { - res = await chai.request(updateURL) - .get('/unpaywall/history/reports') - .query({ latest: true }); - } catch (err) { - console.error(`Cannot GET ${updateURL}/report`); - process.exit(1); - } - return res?.body; -} - -module.exports = { - getReport, - getHistoryReport, -}; +module.exports = getReport; diff --git a/src/update/test/utils/reset.js b/src/update/test/utils/reset.js index e0919134..b8642b56 100644 --- a/src/update/test/utils/reset.js +++ b/src/update/test/utils/reset.js @@ -14,14 +14,14 @@ const resetCronConfig = require('./cron'); * @returns {Promise} */ async function reset() { - await deleteFile('history1.jsonl.gz'); - await deleteFile('history2.jsonl.gz'); - await deleteFile('history3.jsonl.gz'); - await deleteFile('fake1.jsonl.gz'); - await deleteFile('fake2.jsonl.gz'); - await deleteFile('fake3.jsonl.gz'); - await deleteFile('fake1-error.jsonl.gz'); - await deleteFile('snapshot.jsonl.gz'); + await deleteFile('unpaywall', 'history1.jsonl.gz'); + await deleteFile('unpaywall', 'history2.jsonl.gz'); + await deleteFile('unpaywall', 'history3.jsonl.gz'); + await deleteFile('unpaywall', 'fake1.jsonl.gz'); + await deleteFile('unpaywall', 'fake2.jsonl.gz'); + await deleteFile('unpaywall', 'fake3.jsonl.gz'); + await deleteFile('unpaywall', 'fake1-error.jsonl.gz'); + await deleteFile('unpaywall', 'snapshot.jsonl.gz'); await deleteIndex('unpaywall-test'); await deleteIndex('unpaywall_enriched'); await deleteIndex('unpaywall_history'); diff --git a/src/update/test/utils/snapshot.js b/src/update/test/utils/snapshot.js index 09f89719..a068d22d 100644 --- a/src/update/test/utils/snapshot.js +++ b/src/update/test/utils/snapshot.js @@ -15,10 +15,10 @@ const fakeUnpaywall = process.env.FAKEUNPAYWALL_URL || 'http://localhost:59799'; * * @param {Promise} filename - Name of file needed to be delete on ezunpaywall. */ -async function deleteFile(filename) { +async function deleteFile(type, filename) { try { await chai.request(updateURL) - .delete(`/snapshots/${filename}`) + .delete(`/snapshots/${type}/${filename}`) .set('x-api-key', 'changeme'); } catch (err) { console.error(`Cannot DELETE ${updateURL}/snapshot/${filename}`); @@ -33,10 +33,10 @@ async function deleteFile(filename) { * * @returns {Promise} */ -async function addSnapshot(filename) { +async function addSnapshot(type, filename) { try { await chai.request(updateURL) - .post('/snapshots') + .post(`/snapshots/${type}`) .attach('file', path.resolve(snapshotsDir, filename), filename) .set('x-api-key', 'changeme'); } catch (err) { From f80358f69a53f321e9fb8ecf8bece08a9bdf9f4b Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Tue, 9 Jan 2024 15:43:32 +0100 Subject: [PATCH 012/114] doc: update openAPI --- src/update/openapi.json | 3 + src/update/openapi.yml | 885 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 888 insertions(+) create mode 100644 src/update/openapi.yml diff --git a/src/update/openapi.json b/src/update/openapi.json index 360b6741..dfd0d03d 100644 --- a/src/update/openapi.json +++ b/src/update/openapi.json @@ -13,6 +13,9 @@ "url": "http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html" } }, + "servers": [ + { "url": "https://unpaywall.inist.fr/api/update" } + ], "paths": { "/update": { "get": { diff --git a/src/update/openapi.yml b/src/update/openapi.yml new file mode 100644 index 00000000..518809be --- /dev/null +++ b/src/update/openapi.yml @@ -0,0 +1,885 @@ +openapi: 3.0.0 +info: + description: 'The update service allows to update the mirror data. This service queries unpaywall to get the snapshots and inserts their content in an elasticsearch index.' + version: 1.0.0 + title: Update service + contact: + email: ezteam@couperin.org + name: ezTeam + license: + name: CeCILL 2.1 + url: http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html +servers: + - url: https://unpaywall.inist.fr/api +tags: + - name: cron + - name: job + - name: ping +paths: + /update: + get: + tags: + - ping + operationId: get-update + summary: Name of service. + description: Get name of update service. + responses: + '200': + description: OK + content: + application/json: + examples: + response: + value: + message: update service + + /update/ping: + get: + operationId: get-update-ping + summary: Ping update service. + description: Ping update service. + tags: + - ping + responses: + '204': + description: No Content + + /update/health: + get: + tags: + - ping + operationId: get-update-health + summary: Health. + description: Health on all service connected to update service. + parameters: [] + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + redis: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + elastic: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + elapsedTime: + type: integer + status: + type: boolean + x-examples: + Example 1: + redis: + elapsedTime: 1 + status: true + elastic: + elapsedTime: 1 + status: true + elapsedTime: 1 + status: true + examples: + Success: + value: + redis: + elapsedTime: 1 + status: true + elastic: + elapsedTime: 1 + status: true + elapsedTime: 1 + status: true + Error redis: + value: + redis: + elapsedTime: 3001 + status: false + error: time out + elastic: + elapsedTime: 1 + status: true + elapsedTime: 3001 + status: false + + /update/health/redis: + get: + tags: + - ping + operationId: get-update-health-redis + summary: Health on redis service. + description: Health on redis. + parameters: [] + responses: + '200': + $ref: '#/components/responses/Health' + + /update/health/elastic: + get: + tags: + - ping + operationId: get-update-health-elastic + summary: Health on elastic service. + description: Health on elastic. + parameters: [] + responses: + '200': + $ref: '#/components/responses/Health' + + /update/job/snapshot: + post: + tags: + - job + operationId: post-update-job-snapshot + summary: Start process download and insert snapshot from unpaywall. + description: Download and insert the latest snapshot from unpaywall.org. + security: + - x-api-key: [] + requestBody: + content: + application/json: + schema: + type: object + required: + - userName + properties: + interval: + type: string + description: index where the unpaywall data will be inserted. + responses: + '202': + description: Process start + '401': + $ref: '#/components/responses/Not-authorized' + '409': + $ref: '#/components/responses/Conflict' + + /update/job/period: + post: + tags: + - job + operationId: post-update-job-period + summary: Start process download and insert changefiles from unpaywall. + description: Download and insert changefiles from unpaywall.org. + security: + - x-api-key: [] + requestBody: + description: config for process + content: + application/json: + schema: + type: object + required: + - userName + properties: + index: + type: string + interval: + type: string + startDate: + type: string + endDate: + type: string + responses: + '202': + description: Process start. + '400': + description: startDate cannot be in the futur or endDate cannot be lower than startDate. + '401': + $ref: '#/components/responses/Not-authorized' + '409': + $ref: '#/components/responses/Conflict' + + /update/job/changefile/{filename}: + post: + tags: + - job + operationId: post-update-job-changefile-$-filename + summary: Start process insert changesfiles downloaded on ezunpaywall. + description: Insert changefiles already downloaded on ezunpaywall. + security: + - x-api-key: [] + parameters: + - in: path + name: filename + description: filename + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + type: object + properties: + index: + type: string + offset: + type: integer + limit: + type: integer + description: config for process + responses: + '202': + description: Process start + '400': + description: startDate cannot be in the futur or endDate cannot be lower than startDate. + content: + application/json: + schema: + type: string + default: changeme + '401': + $ref: '#/components/responses/Not-authorized' + '404': + description: File not found + '409': + $ref: '#/components/responses/Conflict' + + /update/job/history: + post: + tags: + - job + operationId: post-update-job-history + summary: Start process download and insert changefiles from unpaywall and feed history. + description: Download and insert changefiles from unpaywall.org and feed history. + security: + - x-api-key: [] + requestBody: + description: config for process + content: + application/json: + schema: + type: object + required: + - userName + properties: + index: + type: string + interval: + type: string + startDate: + type: string + endDate: + type: string + responses: + '202': + description: Process start + '400': + description: startDate cannot be in the futur or endDate cannot be lower than startDate. + '401': + $ref: '#/components/responses/Not-authorized' + '409': + $ref: '#/components/responses/Conflict' + + /update/job/reset: + post: + tags: + - job + operationId: post-update-job-reset + summary: Start process that roll back the current and the history index according to a date. + description: roll back the current and the history index according to a date. + security: + - x-api-key: [] + requestBody: + description: config for process + content: + application/json: + schema: + type: object + required: + - userName + properties: + index: + type: string + startDate: + type: string + responses: + '202': + description: Process start + '400': + description: startDate required + '401': + $ref: '#/components/responses/Not-authorized' + '409': + $ref: '#/components/responses/Conflict' + + /update/reports: + get: + tags: + - job + operationId: get-update-reports + summary: Get reports. + description: Get list of reports or latest report. + parameters: + - in: query + name: latest + description: latest + required: false + schema: + type: boolean + responses: + '200': + description: report + content: + List of reports: + examples: + response: + value: + - 2022-10-10 13:10:51.json + - 2022-10-10 13:10:50.json + - 2022-10-10 13:10:49.json + - 2022-10-10 13:10:48.json + - 2022-10-10 13:10:47.json + - 2022-10-10 13:10:46.json + - 2022-10-10 13:10:45.json + - 2022-10-10 13:10:44.json + Latest report: + examples: + response: + value: + done: true + createdAt: '2022-10-10T13:10:46.828Z' + endAt: '2022-10-10T13:10:47.084Z' + steps: + - task: getChangefiles + took: 0.002 + status: success + - task: download + file: fake2.jsonl.gz + percent: 100 + took: 0 + status: success + - task: insert + index: unpaywall-test + file: fake2.jsonl.gz + linesRead: 100 + insertedDocs: 100 + updatedDocs: 0 + failedDocs: 0 + percent: 100 + took: 0.202 + status: success + error: false + took: 0.256 + + /update/reports/${filename}: + get: + tags: + - job + operationId: get-update-reports-$-filename + summary: Get report. + description: Get report with his filename. + parameters: + - in: path + name: filename + description: filename + required: true + schema: + type: string + responses: + '200': + description: report + content: + application/json: + schema: + type: object + x-examples: + example-1: + done: true + createdAt: '2022-10-10T13:10:46.828Z' + endAt: '2022-10-10T13:10:47.084Z' + steps: + - task: getChangefiles + took: 0.002 + status: success + - task: download + file: fake2.jsonl.gz + percent: 100 + took: 0 + status: success + - task: insert + index: unpaywall-test + file: fake2.jsonl.gz + linesRead: 100 + insertedDocs: 100 + updatedDocs: 0 + failedDocs: 0 + percent: 100 + took: 0.202 + status: success + error: false + took: 0.256 + Report: + examples: + response: + value: + done: true + createdAt: '2022-10-10T13:10:46.828Z' + endAt: '2022-10-10T13:10:47.084Z' + steps: + - task: getChangefiles + took: 0.002 + status: success + - task: download + file: fake2.jsonl.gz + percent: 100 + took: 0 + status: success + - task: insert + index: unpaywall-test + file: fake2.jsonl.gz + linesRead: 100 + insertedDocs: 100 + updatedDocs: 0 + failedDocs: 0 + percent: 100 + took: 0.202 + status: success + error: false + took: 0.256 + '404': + description: Not Found + content: + application/json: + schema: + type: object + properties: + message: + type: string + x-examples: + example-1: + message: File not found + File not found: + examples: + response: + value: + message: File not found + + /update/snapshots: + get: + tags: + - job + operationId: get-update-snapshots + summary: Get snapshots. + description: Get list of snapshot or latest snapshot. + security: + - x-api-key: [] + parameters: + - in: query + name: latest + description: latest + required: false + schema: + type: boolean + responses: + '200': + description: snapshot + content: + Lists of snapshots: + examples: + response: + value: + - snapshot-2022-10-10.jsonl.gz + '401': + $ref: '#/components/responses/Not-authorized' + + /update/snapshots/${filename}: + get: + tags: + - job + operationId: get-update-snapshots-$-filename + summary: Get snapshot. + description: Get snapshot with his filename. + security: + - x-api-key: [] + parameters: + - in: path + name: filename + description: filename + required: true + schema: + type: string + responses: + '200': + description: snapshot + '401': + $ref: '#/components/responses/Not-authorized' + '404': + description: File not found + content: + application/json: + schema: + type: object + properties: + message: + type: string + x-examples: + example-1: + message: File not found + examples: + example-1: + value: + message: string + File not found: + examples: + response: + value: + message: File not found + + /update/states: + get: + tags: + - job + operationId: get-update-states + summary: Get state. + description: Get latest state. + responses: + '200': + description: state + content: + Latest state: + examples: + response: + value: + done: true + createdAt: '2022-10-10T13:10:51.417Z' + endAt: '2022-10-10T13:10:51.584Z' + steps: + - task: getChangefiles + took: 0.002 + status: success + - task: insert + index: unpaywall-test + file: fake1.jsonl.gz + linesRead: 50 + insertedDocs: 50 + updatedDocs: 0 + failedDocs: 0 + percent: 100 + took: 0.165 + status: success + error: false + took: 0.167 + + /update/status: + get: + tags: + - job + summary: Get status + operationId: get-update-status + description: Get status + responses: + '200': + description: status + content: + No process in progress: + examples: + response: + value: false + Update in progress: + examples: + response: + value: true + + /update/unpaywall/changefiles: + get: + tags: + - job + operationId: get-update-unpaywall-changefiles + summary: Get changefiles from unpaywall. + description: Get changefiles from unpaywall. + security: + - x-api-key: [] + parameters: + - in: query + name: interval + description: interval of changefiles ("week" or "day") + required: false + schema: + type: string + responses: + '200': + description: status + content: + List of daily changefiles of unpaywall: + examples: + response: + value: + - date: '2022-10-10T00:00:00.000Z' + filename: fake1.jsonl.gz + filetype: jsonl + last_modified: '2021-10-09T13:10:50.000Z' + lines: 50 + size: 19896 + url: http://fakeUnpaywall:3000/daily-feed/changefiles/fake1.jsonl.gz + - date: '2022-10-09T00:00:00.000Z' + filename: fake2.jsonl.gz + filetype: jsonl + last_modified: '2022-10-09T13:10:50.000Z' + lines: 100 + size: 44137 + url: http://fakeUnpaywall:3000/daily-feed/changefiles/fake2.jsonl.gz + - date: '2022-10-08T00:00:00.000Z' + filename: fake3.jsonl.gz + filetype: jsonl + last_modified: '2022-10-08T13:10:50.000Z' + lines: 2000 + size: 877494 + url: http://fakeUnpaywall:3000/daily-feed/changefiles/fake3.jsonl.gz + List of weekly changefiles of unpaywall: + examples: + response: + value: + - date: '2022-10-10T00:00:00.000Z' + filename: fake1.jsonl.gz + filetype: jsonl + last_modified: '2021-10-09T13:10:50.000Z' + lines: 50 + size: 19896 + url: http://fakeUnpaywall:3000/feed/changefiles/fake1.jsonl.gz + - date: '2022-10-09T00:00:00.000Z' + filename: fake2.jsonl.gz + filetype: jsonl + last_modified: '2022-10-09T13:10:50.000Z' + lines: 100 + size: 44137 + url: http://fakeUnpaywall:3000/feed/changefiles/fake2.jsonl.gz + - date: '2022-10-08T00:00:00.000Z' + filename: fake3.jsonl.gz + filetype: jsonl + last_modified: '2022-10-08T13:10:50.000Z' + lines: 2000 + size: 877494 + url: http://fakeUnpaywall:3000/feed/changefiles/fake3.jsonl.gz + '401': + $ref: '#/components/responses/Not-authorized' + + /update/cron/start: + post: + tags: + - cron + operationId: post-update-cron-start + summary: Start update cron. + description: Start update cron. + responses: + '202': + description: Accepted + '401': + $ref: '#/components/responses/Not-authorized' + + security: + - x-api-key: [] + + /update/cron/stop: + post: + tags: + - cron + operationId: post-update-cron-stop + description: Stop update cron. + summary: Stop update cron. + security: + - x-api-key: [] + parameters: [] + responses: + '202': + description: Accepted + '401': + $ref: '#/components/responses/Not-authorized' + + /update/cron: + patch: + tags: + - cron + operationId: patch-update-cron + summary: update update cron. + description: update update cron. + security: + - x-api-key: [] + parameters: [] + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + config: + type: object + properties: + time: + type: string + status: + type: boolean + index: + type: string + interval: + type: string + x-examples: + example-1: + config: + time: '* * * * * *' + status: false + index: unpaywall + interval: day + examples: + updated: + value: + config: + time: '* 30 12-15 * * *' + status: true + index: unpaywall + interval: day + '401': + $ref: '#/components/responses/Not-authorized' + requestBody: + content: + application/json: + schema: + type: object + properties: + time: + type: string + index: + type: string + interval: + type: string + x-examples: + example-1: + time: '' + index: '' + interval: '' + examples: + daily: + value: + time: 0 0 0 * * * + index: unpaywall + interval: day + weekly: + value: + time: 0 0 0 * * 0 + index: unpaywall + interval: weekly + + get: + tags: + - cron + operationId: get-update-cron + summary: Get config of update cron. + description: Get config of update cron. + parameters: [] + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + config: + type: object + properties: + time: + type: string + status: + type: boolean + index: + type: string + interval: + type: string + x-examples: + example-1: + config: + time: '* * * * * *' + status: false + index: unpaywall + interval: day + examples: + day: + value: + config: + time: 0 0 0 * * * + status: true + index: unpaywall + interval: day + week: + value: + config: + time: 0 0 0 * * * + status: true + index: unpaywall + interval: day + 'off': + value: + config: + time: 0 0 0 * * * + status: false + index: unpaywall + interval: day + +components: + responses: + Not-authorized: + description: Not authorized + headers: {} + content: + application/json: + examples: + response: + value: + message: Not authorized + Conflict: + description: Conflict + headers: {} + content: + application/json: + examples: + response: + value: + message: Update in progress. + Health: + description: Example response + content: + application/json: + schema: + type: object + properties: + name: + type: string + status: + type: boolean + elapsedTime: + type: integer + x-examples: + Example 1: + name: redis + status: true + elapsedTime: 1 + examples: + Success: + value: + name: name of service + status: true + elapsedTime: 1 + Error redis: + value: + name: redis + elapsedTime: 3002 + error: time out + status: false + securitySchemes: + x-api-key: + type: apiKey + in: header + name: API Key + schemas: {} From 9f89451414d7e0ce63d2ceca49948dcdae4076fc Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Tue, 9 Jan 2024 15:44:41 +0100 Subject: [PATCH 013/114] feat: add parameter to cron controller --- src/update/lib/controllers/cron.js | 95 ++++++++++++++----- src/update/lib/cron.js | 7 +- src/update/lib/cron/cron.js | 2 +- .../lib/cron/{update.js => unpaywall.js} | 24 ++--- src/update/lib/cron/unpaywallHistory.js | 61 ++++++++++++ src/update/lib/middlewares/format/interval.js | 18 +++- src/update/lib/routers/cron.js | 17 ++-- src/update/lib/routers/job.js | 1 + src/update/test/unpaywall/cron.js | 20 ++-- src/update/test/utils/cron.js | 4 +- src/update/test/utils/reset.js | 2 +- 11 files changed, 185 insertions(+), 66 deletions(-) rename src/update/lib/cron/{update.js => unpaywall.js} (69%) create mode 100644 src/update/lib/cron/unpaywallHistory.js diff --git a/src/update/lib/controllers/cron.js b/src/update/lib/controllers/cron.js index 2dfd7b53..442860be 100644 --- a/src/update/lib/controllers/cron.js +++ b/src/update/lib/controllers/cron.js @@ -1,5 +1,6 @@ const cronValidator = require('cron-validator'); -const cron = require('../cron/update'); +const unpaywallCron = require('../cron/unpaywall'); +const unpaywallHistoryCron = require('../cron/unpaywallHistory'); /** * Controller to start update cron. @@ -8,11 +9,23 @@ const cron = require('../cron/update'); * @param {import('express').Response} res - HTTP response. * @param {import('express').NextFunction} next - Do the following. */ -function startUpdateCron(req, res, next) { - try { - cron.updateCron.start(); - } catch (err) { - return next(err); +function startCron(req, res, next) { + const { type } = req.data; + + if (type === 'unpaywall') { + try { + unpaywallCron.cron.start(); + } catch (err) { + return next(err); + } + } + + if (type === 'unpaywallHistory') { + try { + unpaywallHistoryCron.cron.start(); + } catch (err) { + return next(err); + } } return res.status(202).json(); @@ -25,11 +38,23 @@ function startUpdateCron(req, res, next) { * @param {import('express').Response} res - HTTP response. * @param {import('express').NextFunction} next - Do the following. */ -function stopUpdateCron(req, res, next) { - try { - cron.updateCron.stop(); - } catch (err) { - return next(err); +function stopCron(req, res, next) { + const { type } = req.data; + + if (type === 'unpaywall') { + try { + unpaywallCron.cron.stop(); + } catch (err) { + return next(err); + } + } + + if (type === 'unpaywallHistory') { + try { + unpaywallHistoryCron.cron.stop(); + } catch (err) { + return next(err); + } } return res.status(202).json(); @@ -42,8 +67,8 @@ function stopUpdateCron(req, res, next) { * @param {import('express').Response} res - HTTP response. * @param {import('express').NextFunction} next - Do the following. */ -function patchUpdateCron(req, res, next) { - const { cronConfig } = req.data; +function patchCron(req, res, next) { + const { cronConfig, type } = req.data; const { time, index, interval } = cronConfig; if (time) { @@ -51,13 +76,25 @@ function patchUpdateCron(req, res, next) { if (!validCron) { return res.status(400).json('schedule is invalid'); } } - try { - cron.update({ time, index, interval }); - } catch (err) { - return next(err); + let config; + + if (type === 'unpaywall') { + try { + unpaywallCron.update({ time, index, interval }); + } catch (err) { + return next(err); + } + config = unpaywallCron.getGlobalConfig(); } - const config = cron.getGlobalConfig(); + if (type === 'unpaywallHistory') { + try { + unpaywallHistoryCron.update({ time, index, interval }); + } catch (err) { + return next(err); + } + config = unpaywallHistoryCron.getGlobalConfig(); + } return res.status(200).json(config); } @@ -69,14 +106,24 @@ function patchUpdateCron(req, res, next) { * @param {import('express').Response} res - HTTP response. * @param {import('express').NextFunction} next - Do the following. */ -function getConfigOfUpdateCron(req, res) { - const config = cron.getGlobalConfig(); +function getConfigCron(req, res) { + const { type } = req.data; + + let config; + + if (type === 'unpaywall') { + config = unpaywallCron.getGlobalConfig(); + } + + if (type === 'unpaywallHistory') { + config = unpaywallHistoryCron.getGlobalConfig(); + } return res.status(200).json(config); } module.exports = { - startUpdateCron, - stopUpdateCron, - patchUpdateCron, - getConfigOfUpdateCron, + startCron, + stopCron, + patchCron, + getConfigCron, }; diff --git a/src/update/lib/cron.js b/src/update/lib/cron.js index edb00c81..9782622e 100644 --- a/src/update/lib/cron.js +++ b/src/update/lib/cron.js @@ -22,12 +22,7 @@ class Cron { } } - /** - * Getter of config of cron. - * - * @returns {Object} config of cron. - */ - getConfig() { + get config() { return { name: this.name, schedule: this.schedule, diff --git a/src/update/lib/cron/cron.js b/src/update/lib/cron/cron.js index a006c486..b984d4c0 100644 --- a/src/update/lib/cron/cron.js +++ b/src/update/lib/cron/cron.js @@ -48,7 +48,7 @@ function update(config) { * @returns {Object} Config of update process and config of cron. */ function getGlobalConfig() { - const config = cron.getConfig(); + const { config } = cron; return { ...config, ...updateConfig }; } diff --git a/src/update/lib/cron/update.js b/src/update/lib/cron/unpaywall.js similarity index 69% rename from src/update/lib/cron/update.js rename to src/update/lib/cron/unpaywall.js index df7a4c05..9b6c8436 100644 --- a/src/update/lib/cron/update.js +++ b/src/update/lib/cron/unpaywall.js @@ -12,7 +12,7 @@ let { active } = unpaywallCron; if (active === 'true' || active) active = true; else active = false; -const updateConfig = { +const cronConfig = { index: unpaywallCron.index, interval: unpaywallCron.interval, }; @@ -28,12 +28,12 @@ async function task() { logger.info(`[cron: ${this.name}] conflit: an update is already in progress`); return; } - const week = (updateConfig.interval === 'week'); + const week = (cronConfig.interval === 'week'); const startDate = format(subDays(new Date(), week ? 7 : 0), 'yyyy-MM-dd'); const endDate = format(new Date(), 'yyyy-MM-dd'); await insertChangefilesOnPeriod({ - index: updateConfig.index, - interval: updateConfig.interval, + index: cronConfig.index, + interval: cronConfig.interval, startDate, endDate, offset: 0, @@ -41,7 +41,7 @@ async function task() { }); } -const updateCron = new Cron('update', unpaywallCron.schedule, task, active); +const cron = new Cron('update', unpaywallCron.schedule, task, active); /** * Update config of update process and config of cron. @@ -49,12 +49,12 @@ const updateCron = new Cron('update', unpaywallCron.schedule, task, active); * @param {Object} newConfig - Global config. */ function update(newConfig) { - if (newConfig.time) updateCron.setSchedule(newConfig.time); + if (newConfig.time) cron.setSchedule(newConfig.time); - if (newConfig.index) updateConfig.index = newConfig.index; - if (newConfig.interval) updateConfig.interval = newConfig.interval; + if (newConfig.index) cronConfig.index = newConfig.index; + if (newConfig.interval) cronConfig.interval = newConfig.interval; - if (newConfig.index || newConfig.interval) updateCron.setTask(task); + if (newConfig.index || newConfig.interval) cron.setTask(task); } /** @@ -63,12 +63,12 @@ function update(newConfig) { * @returns {Object} Config of update process and config of cron. */ function getGlobalConfig() { - const cronConfig = updateCron.getConfig(); - return { ...cronConfig, ...updateConfig }; + const { config } = cron; + return { ...cronConfig, ...config }; } module.exports = { getGlobalConfig, update, - updateCron, + cron, }; diff --git a/src/update/lib/cron/unpaywallHistory.js b/src/update/lib/cron/unpaywallHistory.js new file mode 100644 index 00000000..df02752f --- /dev/null +++ b/src/update/lib/cron/unpaywallHistory.js @@ -0,0 +1,61 @@ +const { unpaywallHistoryCron } = require('config'); +const logger = require('../logger'); + +const Cron = require('../cron'); +const { getStatus } = require('../status'); + +let { active } = unpaywallHistoryCron; + +if (active === 'true' || active) active = true; +else active = false; + +const updateConfig = { + index: unpaywallHistoryCron.index, + interval: unpaywallHistoryCron.interval, +}; + +/** + * Starts an update daily process if no update process is started. + * + * @returns {Promise} + */ +async function task() { + const status = getStatus(); + if (status) { + logger.info(`[cron: ${this.name}] conflit: an update is already in progress`); + // return; + } + // TODO do update +} + +const cron = new Cron('updateHistory', unpaywallHistoryCron.schedule, task, active); + +/** + * Update config of update process and config of cron. + * + * @param {Object} newConfig - Global config. + */ +function update(newConfig) { + if (newConfig.time) cron.setSchedule(newConfig.time); + + if (newConfig.index) updateConfig.index = newConfig.index; + if (newConfig.interval) updateConfig.interval = newConfig.interval; + + if (newConfig.index || newConfig.interval) cron.setTask(task); +} + +/** + * Get config of update process and config of cron. + * + * @returns {Object} Config of update process and config of cron. + */ +function getGlobalConfig() { + const cronConfig = cron.config; + return { ...cronConfig, ...updateConfig }; +} + +module.exports = { + getGlobalConfig, + update, + cron, +}; diff --git a/src/update/lib/middlewares/format/interval.js b/src/update/lib/middlewares/format/interval.js index 9511fa0c..80700f44 100644 --- a/src/update/lib/middlewares/format/interval.js +++ b/src/update/lib/middlewares/format/interval.js @@ -8,8 +8,22 @@ const joi = require('joi'); * @param {import('express').NextFunction} next - Do the following. */ async function validateInterval(req, res, next) { - const { error, value } = joi.string().trim().valid('week', 'day').default('day') - .validate(req.body.interval); + let error; + let value; + + if (req.body.interval) { + const result = joi.string().trim().valid('week', 'day').default('day') + .validate(req.body.interval); + error = result?.error; + value = result?.value; + } + + if (req.query.interval) { + const result = joi.string().trim().valid('week', 'day').default('day') + .validate(req.query.interval); + error = result?.error; + value = result?.value; + } if (error) return res.status(400).json({ message: error.details[0].message }); diff --git a/src/update/lib/routers/cron.js b/src/update/lib/routers/cron.js index f844faa3..50612ae4 100644 --- a/src/update/lib/routers/cron.js +++ b/src/update/lib/routers/cron.js @@ -2,25 +2,26 @@ const router = require('express').Router(); const checkAuth = require('../middlewares/auth'); const validateCronConfig = require('../middlewares/format/cron'); +const validateType = require('../middlewares/format/type'); const { - startUpdateCron, - stopUpdateCron, - patchUpdateCron, - getConfigOfUpdateCron, + startCron, + stopCron, + patchCron, + getConfigCron, } = require('../controllers/cron'); /** * Route that start the update cron. * Auth required. */ -router.post('/cron/start', checkAuth, startUpdateCron); +router.post('/cron/:type/start', checkAuth, validateType, startCron); /** * Route that stop the update cron. * Auth required. */ -router.post('/cron/stop', checkAuth, stopUpdateCron); +router.post('/cron/:type/stop', checkAuth, validateType, stopCron); /** * Route that update the update cron. @@ -28,11 +29,11 @@ router.post('/cron/stop', checkAuth, stopUpdateCron); * * This route need a body that contains a config of cron. */ -router.patch('/cron', checkAuth, validateCronConfig, patchUpdateCron); +router.patch('/cron/:type', checkAuth, validateType, validateCronConfig, patchCron); /** * Route that get the config of update cron. */ -router.get('/cron', getConfigOfUpdateCron); +router.get('/cron/:type', validateType, getConfigCron); module.exports = router; diff --git a/src/update/lib/routers/job.js b/src/update/lib/routers/job.js index 4b469e03..98530a05 100644 --- a/src/update/lib/routers/job.js +++ b/src/update/lib/routers/job.js @@ -66,6 +66,7 @@ router.post('/job/history', checkStatus, checkAuth, validateHistoryJob, insertWi */ router.post('/job/history/reset', checkStatus, checkAuth, validateHistoryReset, historyRollBack); +// Dev router.post('/job/history/reset/step1', checkStatus, checkAuth, validateHistoryReset, async (req, res, next) => { const { startDate } = req.data; diff --git a/src/update/test/unpaywall/cron.js b/src/update/test/unpaywall/cron.js index 97c68251..32d008fa 100644 --- a/src/update/test/unpaywall/cron.js +++ b/src/update/test/unpaywall/cron.js @@ -24,7 +24,7 @@ describe('Test: cron manage', () => { it('Should return config of cron', async () => { const res = await chai.request(updateURL) - .get('/cron'); + .get('/cron/unpaywall'); expect(res).have.status(200); @@ -42,7 +42,7 @@ describe('Test: cron manage', () => { it('Should return updated config of cron', async () => { const res = await chai.request(updateURL) - .patch('/cron') + .patch('/cron/unpaywall') .send({ time: '0 0 0 1 * *' }) .set('x-api-key', 'changeme'); @@ -62,7 +62,7 @@ describe('Test: cron manage', () => { it('Should return updated config of cron', async () => { const res = await chai.request(updateURL) - .patch('/cron') + .patch('/cron/unpaywall') .send({ index: 'unpaywall2' }) .set('x-api-key', 'changeme'); @@ -82,7 +82,7 @@ describe('Test: cron manage', () => { it('Should return updated config of cron', async () => { const res = await chai.request(updateURL) - .patch('/cron') + .patch('/cron/unpaywall') .send({ interval: 'week' }) .set('x-api-key', 'changeme'); @@ -102,7 +102,7 @@ describe('Test: cron manage', () => { it('Should return a status code 401', async () => { const res = await chai.request(updateURL) - .patch('/cron') + .patch('/cron/unpaywall') .send({ interval: 'week' }); expect(res).have.status(401); @@ -117,7 +117,7 @@ describe('Test: cron manage', () => { it('Should return a status code 401', async () => { const res = await chai.request(updateURL) - .patch('/cron') + .patch('/cron/unpaywall') .send({ interval: 'week' }) .set('x-api-key', 'wrong apikey'); @@ -132,7 +132,7 @@ describe('Test: cron manage', () => { it('Should return a status code 202', async () => { const res = await chai.request(updateURL) - .post('/cron/start') + .post('/cron/unpaywall/start') .set('x-api-key', 'changeme'); expect(res).have.status(202); @@ -140,7 +140,7 @@ describe('Test: cron manage', () => { it('Should return updated config of cron', async () => { const res = await chai.request(updateURL) - .get('/cron'); + .get('/cron/unpaywall'); expect(res).have.status(200); @@ -158,7 +158,7 @@ describe('Test: cron manage', () => { it('Should return a status code 202', async () => { const res = await chai.request(updateURL) - .post('/cron/stop') + .post('/cron/unpaywall/stop') .set('x-api-key', 'changeme'); expect(res).have.status(202); @@ -166,7 +166,7 @@ describe('Test: cron manage', () => { it('Should return updated config of cron', async () => { const res = await chai.request(updateURL) - .get('/cron'); + .get('/cron/unpaywall'); expect(res).have.status(200); diff --git a/src/update/test/utils/cron.js b/src/update/test/utils/cron.js index b4fa06d3..86306c7d 100644 --- a/src/update/test/utils/cron.js +++ b/src/update/test/utils/cron.js @@ -10,10 +10,10 @@ chai.use(chaiHttp); * * @returns {Promise} */ -async function resetCronConfig() { +async function resetCronConfig(type) { try { await chai.request(updateURL) - .patch('/cron') + .patch(`/cron/${type}`) .send({ time: '0 0 0 * * *', index: 'unpaywall', interval: 'day' }) .set('x-api-key', 'changeme'); } catch (err) { diff --git a/src/update/test/utils/reset.js b/src/update/test/utils/reset.js index b8642b56..d1b23e93 100644 --- a/src/update/test/utils/reset.js +++ b/src/update/test/utils/reset.js @@ -25,7 +25,7 @@ async function reset() { await deleteIndex('unpaywall-test'); await deleteIndex('unpaywall_enriched'); await deleteIndex('unpaywall_history'); - await resetCronConfig(); + await resetCronConfig('unpaywall'); } module.exports = reset; From fdf59b56763bcb9f3737bae7310e0ff9d945bc57 Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Thu, 11 Jan 2024 10:37:52 +0100 Subject: [PATCH 014/114] feat: update history cron config --- src/update/README.md | 12 +- .../config/custom-environment-variables.json | 16 +- src/update/config/default.json | 3 +- src/update/lib/controllers/job.js | 4 +- src/update/lib/cron/cron.js | 59 -- src/update/lib/cron/unpaywall.js | 2 +- src/update/lib/cron/unpaywallHistory.js | 30 +- src/update/lib/history.js | 81 +- src/update/lib/job.js | 5 +- src/update/lib/middlewares/format/job.js | 6 +- src/update/lib/routers/job.js | 3 +- src/update/openapi.json | 791 +++++++++--------- src/update/openapi.yml | 171 ++-- src/update/test/{unpaywall => }/cron.js | 4 +- src/update/test/utils/reset.js | 1 + 15 files changed, 606 insertions(+), 582 deletions(-) delete mode 100644 src/update/lib/cron/cron.js rename src/update/test/{unpaywall => }/cron.js (98%) diff --git a/src/update/README.md b/src/update/README.md index 1f380f52..738b76e6 100644 --- a/src/update/README.md +++ b/src/update/README.md @@ -17,9 +17,13 @@ Health service indicating the status of the connection between each service. | ELASTICSEARCH_PASSWORD | changeme | password of elastic super user | | ELASTICSEARCH_MAX_BULK_SIZE | 4000 | max bulk size of update process | | ELASTICSEARCH_INDEX_ALIAS | upw | default alias of unpaywall data | -| UPDATE_CRON_SCHEDULE | 0 0 0 * * * | schedule of cron | -| UPDATE_CRON_ACTIVE | false | state of cron | -| UPDATE_CRON_INDEX | unpaywall | index of update process of cron | -| UPDATE_CRON_INTERVAL | day | interval of update process of cron | +| UNPAYWALL_CRON_SCHEDULE | 0 0 0 * * * | schedule of unpaywall cron | +| UNPAYWALL_CRON_ACTIVE | false | state of unpaywall cron | +| UNPAYWALL_CRON_INDEX | unpaywall | index of unpaywall process of cron | +| UNPAYWALL_CRON_INTERVAL | day | interval of unpaywall process of cron | +| UNPAYWALL_HISTORY_CRON_SCHEDULE | 0 0 0 * * * | schedule of unpaywall history cron | +| UNPAYWALL_HISTORY_CRON_ACTIVE | false | state of unpaywall history cron | +| UNPAYWALL_HISTORY_CRON_INDEX | unpaywall | index of unpaywall history process of cron | +| UNPAYWALL_HISTORY_CRON_INTERVAL | day | interval of unpaywall history process of cron | | UPDATE_APIKEY | changeme | update apikey to start update process | | HEALTH_TIMEOUT | 3000 | timeout to query the health route | \ No newline at end of file diff --git a/src/update/config/custom-environment-variables.json b/src/update/config/custom-environment-variables.json index 6845e35f..eee22755 100644 --- a/src/update/config/custom-environment-variables.json +++ b/src/update/config/custom-environment-variables.json @@ -18,11 +18,17 @@ "indexAlias": "ELASTICSEARCH_INDEX_ALIAS", "timeout": "ELASTICSEARCH_TIMEOUT" }, - "cron": { - "schedule": "UPDATE_CRON_SCHEDULE", - "active": "UPDATE_CRON_ACTIVE", - "index": "UPDATE_CRON_INDEX", - "interval": "UPDATE_CRON_INTERVAL" + "unpaywallCron": { + "schedule": "UNPAYWALL_CRON_SCHEDULE", + "active": "UNPAYWALL_CRON_ACTIVE", + "index": "UNPAYWALL_CRON_INDEX", + "interval": "UNPAYWALL_CRON_INTERVAL" + }, + "unpaywallHistoryCron": { + "schedule": "UNPAYWALL_HISTORY_CRON_SCHEDULE", + "active": "UNPAYWALL_HISTORY_CRON_ACTIVE", + "index": "UNPAYWALL_HISTORY_CRON_INDEX", + "interval": "UNPAYWALL_HISTORY_CRON_INTERVAL" }, "apikey": "UPDATE_APIKEY", "healthTimeout": "HEALTH_TIMEOUT" diff --git a/src/update/config/default.json b/src/update/config/default.json index 6e056e3c..8c03a7d2 100644 --- a/src/update/config/default.json +++ b/src/update/config/default.json @@ -27,7 +27,8 @@ "unpaywallHistoryCron": { "schedule": "0 0 0 * * *", "active": false, - "index": "unpaywall", + "indexBase": "unpaywall_enriched", + "indexHistory": "unpaywall_history", "interval": "day" }, "apikey": "changeme", diff --git a/src/update/lib/controllers/job.js b/src/update/lib/controllers/job.js index 04e845d7..0e8b4530 100644 --- a/src/update/lib/controllers/job.js +++ b/src/update/lib/controllers/job.js @@ -153,9 +153,9 @@ async function insertWithOaHistory(req, res, next) { * @param {import('express').NextFunction} next - Do the following. */ async function historyRollBack(req, res, next) { - const { startDate, index } = req.data.rollBackConfig; + const { startDate, indexBase, indexHistory } = req.data.rollBackConfig; - await rollBack(startDate, index); + await rollBack(startDate, indexBase, indexHistory); return res.status(202).json(); } diff --git a/src/update/lib/cron/cron.js b/src/update/lib/cron/cron.js deleted file mode 100644 index b984d4c0..00000000 --- a/src/update/lib/cron/cron.js +++ /dev/null @@ -1,59 +0,0 @@ -const { format, subDays } = require('date-fns'); - -const Cron = require('../cron'); - -const { insertChangefilesOnPeriod } = require('../job'); - -const updateConfig = { - index: 'unpaywall', - interval: 'day', -}; - -/** - * Starts an update daily process if no update process is started. - * - * @returns {Promise} - */ -async function task() { - const week = (updateConfig.interval === 'week'); - const startDate = format(subDays(new Date(), week ? 7 : 0), 'yyyy-MM-dd'); - const endDate = format(new Date(), 'yyyy-MM-dd'); - await insertChangefilesOnPeriod({ - index: updateConfig.index, - interval: updateConfig.interval, - startDate, - endDate, - offset: 0, - limit: -1, - }); -} - -const cron = new Cron('update', '0 0 0 * * *', task); - -/** - * Update config of update process and config of cron. - * - * @param {Object} config - Global config. - */ -function update(config) { - if (config.time) updateConfig.time = config.time; - if (config.index) updateConfig.index = config.index; - if (config.interval) updateConfig.interval = config.interval; - - cron.setTask(task); -} - -/** - * Get config of update process and config of cron. - * @returns {Object} Config of update process and config of cron. - */ -function getGlobalConfig() { - const { config } = cron; - return { ...config, ...updateConfig }; -} - -module.exports = { - getGlobalConfig, - update, - cron, -}; diff --git a/src/update/lib/cron/unpaywall.js b/src/update/lib/cron/unpaywall.js index 9b6c8436..1416d11d 100644 --- a/src/update/lib/cron/unpaywall.js +++ b/src/update/lib/cron/unpaywall.js @@ -25,7 +25,7 @@ const cronConfig = { async function task() { const status = getStatus(); if (status) { - logger.info(`[cron: ${this.name}] conflit: an update is already in progress`); + logger.info(`[cron: ${this.name}] conflict: an update is already in progress`); return; } const week = (cronConfig.interval === 'week'); diff --git a/src/update/lib/cron/unpaywallHistory.js b/src/update/lib/cron/unpaywallHistory.js index df02752f..98828887 100644 --- a/src/update/lib/cron/unpaywallHistory.js +++ b/src/update/lib/cron/unpaywallHistory.js @@ -1,16 +1,20 @@ +const { format, subDays } = require('date-fns'); const { unpaywallHistoryCron } = require('config'); const logger = require('../logger'); const Cron = require('../cron'); const { getStatus } = require('../status'); +const { insertWithOaHistoryJob } = require('../job'); + let { active } = unpaywallHistoryCron; if (active === 'true' || active) active = true; else active = false; -const updateConfig = { - index: unpaywallHistoryCron.index, +const cronConfig = { + indexBase: unpaywallHistoryCron.indexBase, + indexHistory: unpaywallHistoryCron.indexHistory, interval: unpaywallHistoryCron.interval, }; @@ -22,10 +26,16 @@ const updateConfig = { async function task() { const status = getStatus(); if (status) { - logger.info(`[cron: ${this.name}] conflit: an update is already in progress`); - // return; + logger.info(`[cron: ${this.name}] conflict: an update is already in progress`); + return; } - // TODO do update + const week = (cronConfig.interval === 'week'); + const startDate = format(subDays(new Date(), week ? 7 : 0), 'yyyy-MM-dd'); + await insertWithOaHistoryJob({ + index: cronConfig.index, + interval: cronConfig.interval, + startDate, + }); } const cron = new Cron('updateHistory', unpaywallHistoryCron.schedule, task, active); @@ -37,10 +47,8 @@ const cron = new Cron('updateHistory', unpaywallHistoryCron.schedule, task, acti */ function update(newConfig) { if (newConfig.time) cron.setSchedule(newConfig.time); - - if (newConfig.index) updateConfig.index = newConfig.index; - if (newConfig.interval) updateConfig.interval = newConfig.interval; - + if (newConfig.indexBase) cronConfig.indexBase = newConfig.indexBase; + if (newConfig.interval) cronConfig.indexHistory = newConfig.indexHistory; if (newConfig.index || newConfig.interval) cron.setTask(task); } @@ -50,8 +58,8 @@ function update(newConfig) { * @returns {Object} Config of update process and config of cron. */ function getGlobalConfig() { - const cronConfig = cron.config; - return { ...cronConfig, ...updateConfig }; + const { config } = cron; + return { ...cronConfig, ...config }; } module.exports = { diff --git a/src/update/lib/history.js b/src/update/lib/history.js index 4859aa8a..1f5f0ee2 100644 --- a/src/update/lib/history.js +++ b/src/update/lib/history.js @@ -93,9 +93,9 @@ async function insertUnpaywallDataInElastic(data, index) { * @param {*} step * @returns */ -async function insertData(listOfDoi, newData, date) { +async function insertData(listOfDoi, newData, indexBase, indexHistory, date) { // TODO not hardcode - const oldData = await getDataByListOfDOI(listOfDoi, 'unpaywall_enriched'); + const oldData = await getDataByListOfDOI(listOfDoi, indexBase); const resHistoryData = []; const resData = []; @@ -104,8 +104,7 @@ async function insertData(listOfDoi, newData, date) { newData.forEach((data) => { const copyData = data; copyData.referencedAt = date; - // TODO not hardcode - resData.push({ index: { _index: 'unpaywall_enriched', _id: data.doi } }); + resData.push({ index: { _index: indexBase, _id: data.doi } }); resData.push(copyData); }); return true; @@ -131,25 +130,21 @@ async function insertData(listOfDoi, newData, date) { }; newEntry.endValidity = data.updated; - // TODO not hardcode - resHistoryData.push({ index: { _index: 'unpaywall_history' } }); + resHistoryData.push({ index: { _index: indexHistory } }); resHistoryData.push(newEntry); // if data, get the old referencedAt copyData.referencedAt = oldDataUnpaywall.referencedAt; } - // TODO not hardcode - resData.push({ index: { _index: 'unpaywall_enriched', _id: data.doi } }); + resData.push({ index: { _index: indexBase, _id: data.doi } }); resData.push(copyData); }); - // TODO not hardcode - await insertUnpaywallDataInElastic(resData, 'unpaywall_enriched'); + await insertUnpaywallDataInElastic(resData, indexBase); if (resHistoryData.length > 0) { - // TODO not hardcode - await insertUnpaywallDataInElastic(resHistoryData, 'unpaywall_history'); + await insertUnpaywallDataInElastic(resHistoryData, indexHistory); } return true; @@ -170,23 +165,21 @@ async function insertData(listOfDoi, newData, date) { */ async function insertHistoryDataUnpaywall(insertConfig) { const { - filename, date, index, offset, limit, + filename, date, indexBase, indexHistory, offset, limit, } = insertConfig; try { - // TODO not hardcode index - await createIndex('unpaywall_enriched', unpaywallEnrichedMapping); + await createIndex(indexBase, unpaywallEnrichedMapping); } catch (err) { - logger.error(`[elastic] Cannot create index [${index}]`, err); + logger.error(`[elastic] Cannot create index [${indexBase}]`, err); await fail(err); return false; } try { - // TODO not hardcode index - await createIndex('unpaywall_history', unpaywallHistoryMapping); + await createIndex(indexHistory, unpaywallHistoryMapping); } catch (err) { - logger.error(`[elastic] Cannot create index [${index}]`, err); + logger.error(`[elastic] Cannot create index [${indexHistory}]`, err); await fail(err); return false; } @@ -197,13 +190,12 @@ async function insertHistoryDataUnpaywall(insertConfig) { const step = getLatestStep(); step.file = filename; step.index = { - // TODO not hardcode - unpaywall_enriched: { + [indexBase]: { insertedDocs: 0, updatedDocs: 0, failedDocs: 0, }, - unpaywall_history: { + [indexHistory]: { insertedDocs: 0, updatedDocs: 0, failedDocs: 0, @@ -275,7 +267,7 @@ async function insertHistoryDataUnpaywall(insertConfig) { // fill the array try { const doc = JSON.parse(line); - // [history] + // history listOfDoi.push(doc.doi); newData.push(doc); } catch (err) { @@ -302,8 +294,8 @@ async function insertHistoryDataUnpaywall(insertConfig) { } // last insertion if there is data left if (newData.length > 0) { - // [history] - success = await insertData(listOfDoi, newData, date); + // history + success = await insertData(listOfDoi, newData, indexBase, indexHistory, date); listOfDoi = []; newData = []; if (!success) return false; @@ -325,33 +317,33 @@ async function insertHistoryDataUnpaywall(insertConfig) { return true; } -async function step1(startDate) { +async function step1(startDate, indexBase, indexHistory) { logger.debug('----------------------------------'); logger.debug('STEP 1'); logger.debug(`startDate: ${format(new Date(startDate), 'yyyy-MM-dd/hh:mm:ss')}`); logger.debug(`Should DELETE greater or equal than ${format(new Date(startDate), 'yyyy-MM-dd/hh:mm:ss')}`); const toRecentDataInHistoryBulk = []; - const toRecentDataInHistory = await searchWithRange(startDate, 'updated', 'gte', 'unpaywall_history'); + const toRecentDataInHistory = await searchWithRange(startDate, 'updated', 'gte', indexHistory); toRecentDataInHistory.forEach((data) => { - toRecentDataInHistoryBulk.push({ delete: { _index: 'unpaywall_history', _id: data._id } }); + toRecentDataInHistoryBulk.push({ delete: { _index: indexHistory, _id: data._id } }); logger.debug(`HISTORY: DELETE: doi: ${data._source.doi}, genre: ${data._source.genre}, updated: ${data._source.updated}`); }); await bulk(toRecentDataInHistoryBulk, true); logger.debug(`DELETE ${toRecentDataInHistoryBulk.length} lines`); logger.debug('UNPAYWALL: REFRESH'); - await refreshIndex('unpaywall_enriched'); + await refreshIndex(indexBase); } -async function step2(startDate) { +async function step2(startDate, indexBase, indexHistory) { logger.debug('----------------------------------'); logger.debug('STEP 2'); logger.debug(`startDate: ${format(new Date(startDate), 'yyyy-MM-dd/hh:mm:ss')}`); logger.debug(`Should UPDATE data less than ${format(new Date(startDate), 'yyyy-MM-dd/hh:mm:ss')}`); const nearestDataBulk = []; const nearestDataDeleteBulk = []; - const oldData = await searchWithRange(startDate, 'updated', 'lt', 'unpaywall_history'); + const oldData = await searchWithRange(startDate, 'updated', 'lt', indexHistory); const listOfDoi = oldData.map((e) => e._source.doi); const listOfDoiFiltered = new Set(listOfDoi); @@ -371,18 +363,18 @@ async function step2(startDate) { } }); - const oldDoc = await getDataByListOfDOI([nearestData._source.doi], 'unpaywall_enriched'); + const oldDoc = await getDataByListOfDOI([nearestData._source.doi], indexBase); logger.info(`oldData: ${format(new Date(oldDoc[0].updated), 'yyyy-MM-dd/hh:mm:ss')} < startDate: ${format(new Date(startDate), 'yyyy-MM-dd/hh:mm:ss')}`); if (new Date(startDate) < new Date(oldDoc[0].updated)) { // UPDATE delete nearestData._source.endValidity; - nearestDataBulk.push({ index: { _index: 'unpaywall_enriched', _id: nearestData._source.doi } }); + nearestDataBulk.push({ index: { _index: indexBase, _id: nearestData._source.doi } }); nearestDataBulk.push(nearestData._source); logger.debug(`UNPAYWALL: UPDATE: doi: ${nearestData._source.doi}, genre: ${nearestData._source.genre}, updated: ${nearestData._source.updated}`); // DELETE - nearestDataDeleteBulk.push({ delete: { _index: 'unpaywall_history', _id: nearestData._id } }); + nearestDataDeleteBulk.push({ delete: { _index: indexHistory, _id: nearestData._id } }); logger.debug(`HISTORY: DELETE: doi: ${nearestData._source.doi}, genre: ${nearestData._source.genre}, updated: ${nearestData._source.updated}`); } } @@ -391,43 +383,42 @@ async function step2(startDate) { await bulk(nearestDataBulk, true); logger.debug(`UPDATE ${nearestDataBulk.length / 2} lines`); logger.debug('UNPAYWALL: REFRESH'); - await refreshIndex('unpaywall_enriched'); + await refreshIndex(indexBase); // DELETE await bulk(nearestDataDeleteBulk, true); logger.debug(`DELETE ${nearestDataDeleteBulk.length} lines`); logger.debug('HISTORY: REFRESH'); - await refreshIndex('unpaywall_history'); + await refreshIndex(indexHistory); } -async function step3(startDate) { +async function step3(startDate, indexBase) { logger.debug('----------------------------------'); logger.debug('STEP 3'); const toRecentDataBulk = []; - const toRecentData = await searchWithRange(startDate, 'updated', 'gte', 'unpaywall_enriched'); + const toRecentData = await searchWithRange(startDate, 'updated', 'gte', indexBase); logger.debug(`startDate: ${format(new Date(startDate), 'yyyy-MM-dd/hh:mm:ss')}`); logger.debug(`Should DELETE data greater or equal than ${format(new Date(startDate), 'yyyy-MM-dd/hh:mm:ss')}`); toRecentData.forEach((data) => { - toRecentDataBulk.push({ delete: { _index: 'unpaywall_enriched', _id: data._id } }); + toRecentDataBulk.push({ delete: { _index: indexBase, _id: data._id } }); logger.debug(`UNPAYWALL: DELETE: doi: ${data._source.doi}, genre: ${data._source.genre}, updated: ${format(new Date(data._source.updated), 'yyyy-MM-dd/hh:mm:ss')}`); }); await bulk(toRecentDataBulk, true); logger.debug(`DELETE ${toRecentDataBulk.length} lines`); logger.debug('UNPAYWALL: REFRESH'); - await refreshIndex('unpaywall_enriched'); + await refreshIndex(indexBase); } /** * Resets the unpaywall index according to a date and deletes the data in the history. * @param {string} startDate - The date you wish to return to */ -async function rollBack(startDate) { - logger.debug('================================'); - await step1(startDate); - await step2(startDate); - await step3(startDate); +async function rollBack(startDate, indexBase, indexHistory) { + await step1(startDate, indexBase, indexHistory); + await step2(startDate, indexBase, indexHistory); + await step3(startDate, indexBase); } module.exports = { diff --git a/src/update/lib/job.js b/src/update/lib/job.js index e4456c4e..6a39a87e 100644 --- a/src/update/lib/job.js +++ b/src/update/lib/job.js @@ -123,12 +123,11 @@ async function insertChangefile(jobConfig) { * Download and insert on elastic the changefiles from unpaywall between a period with history. * * @param {Object} jobConfig - Config of job. - * @param {string} jobConfig.index - Name of the index to which the data will be inserted. + * @param {string} jobConfig.indexBase - Name of the index to which the data will be inserted. + * @param {string} jobConfig.indexHistory - Name of the index to which the data will be inserted. * @param {string} jobConfig.interval - Interval of changefile, day or week are available. * @param {string} jobConfig.startDate - Start date for the changefile period. * @param {string} jobConfig.endDate - End date for the changefile period. - * @param {number} jobConfig.offset - Line of the snapshot at which the data insertion starts. - * @param {number} jobConfig.limit - Line in the file where the insertion stops. * * @returns {Promise} */ diff --git a/src/update/lib/middlewares/format/job.js b/src/update/lib/middlewares/format/job.js index 0053ffbc..d83e5a6d 100644 --- a/src/update/lib/middlewares/format/job.js +++ b/src/update/lib/middlewares/format/job.js @@ -94,7 +94,8 @@ async function validateInsertFile(req, res, next) { */ async function validateHistoryJob(req, res, next) { const { error, value } = joi.object({ - index: joi.string().trim().default('unpaywall_history'), + indexBase: joi.string().trim().default('unpaywall_enriched'), + indexHistory: joi.string().trim().default('unpaywall_history'), interval: joi.string().trim().valid('day', 'week').default('day'), startDate: joi.date().format('YYYY-MM-DD'), endDate: joi.date().format('YYYY-MM-DD').min(joi.ref('startDate')), @@ -120,7 +121,8 @@ async function validateHistoryJob(req, res, next) { */ async function validateHistoryReset(req, res, next) { const { error, value } = joi.object({ - index: joi.string().trim().default('unpaywall_history'), + indexBase: joi.string().trim().default('unpaywall_enriched'), + indexHistory: joi.string().trim().default('unpaywall_history'), startDate: joi.date(), }).with('endDate', 'startDate').validate(req.body); diff --git a/src/update/lib/routers/job.js b/src/update/lib/routers/job.js index 98530a05..455a2880 100644 --- a/src/update/lib/routers/job.js +++ b/src/update/lib/routers/job.js @@ -49,7 +49,8 @@ router.post('/job/period', checkStatus, checkAuth, validateJobChangefilesConfig, router.post('/job/changefile/:filename', checkStatus, checkAuth, validateInsertFile, insertChangefileJob); /** - * Route that download and insert on elastic the changefiles from unpaywall between a period. + * Route that download and insert on elastic the changefiles from unpaywall between a period + * and feed the history. * Auth required. * No update process should be in progress. * diff --git a/src/update/openapi.json b/src/update/openapi.json index dfd0d03d..334010a7 100644 --- a/src/update/openapi.json +++ b/src/update/openapi.json @@ -1,7 +1,7 @@ { "openapi": "3.0.0", "info": { - "description": "The update service allows to update the mirror data. This service queries unpaywall to get the snapshots and inserts their content in an elasticsearch index ", + "description": "The update service allows to update the mirror data. This service queries unpaywall to get the snapshots and inserts their content in an elasticsearch index.", "version": "1.0.0", "title": "Update service", "contact": { @@ -14,30 +14,35 @@ } }, "servers": [ - { "url": "https://unpaywall.inist.fr/api/update" } + { + "url": "https://unpaywall.inist.fr/api" + } + ], + "tags": [ + { + "name": "cron" + }, + { + "name": "job" + }, + { + "name": "ping" + } ], "paths": { "/update": { "get": { - "summary": "Name of service", + "tags": [ + "ping" + ], + "operationId": "get-update", + "summary": "Name of service.", + "description": "Get name of update service.", "responses": { "200": { "description": "OK", "content": { - "*/*": { - "schema": { - "type": "string", - "x-examples": { - "example-1": "update service" - } - }, - "examples": { - "service": { - "value": "update service" - } - } - }, - "Success": { + "application/json": { "examples": { "response": { "value": { @@ -48,19 +53,14 @@ } } } - }, - "operationId": "get-update", - "description": "Get name of update service", - "tags": [ - "ping" - ] + } } }, "/update/ping": { "get": { - "summary": "Ping update service", "operationId": "get-update-ping", - "description": "Ping update service", + "summary": "Ping update service.", + "description": "Ping update service.", "tags": [ "ping" ], @@ -73,12 +73,13 @@ }, "/update/health": { "get": { - "summary": "Health", - "operationId": "get-update-health", - "description": "Health on all service connected to update service", "tags": [ "ping" ], + "operationId": "get-update-health", + "summary": "Health.", + "description": "Health on all service connected to update service.", + "parameters": [], "responses": { "200": { "description": "OK", @@ -166,45 +167,53 @@ } } } - }, - "parameters": [] + } }, "/update/health/redis": { "get": { - "summary": "Health on redis service", + "tags": [ + "ping" + ], + "operationId": "get-update-health-redis", + "summary": "Health on redis service.", + "description": "Health on redis.", + "parameters": [], "responses": { "200": { "$ref": "#/components/responses/Health" } - }, - "operationId": "get-update-health-redis", - "description": "Health on redis", - "tags": [ - "ping" - ] - }, - "parameters": [] + } + } }, "/update/health/elastic": { "get": { - "summary": "Health on elastic service", + "tags": [ + "ping" + ], + "operationId": "get-update-health-elastic", + "summary": "Health on elastic service.", + "description": "Health on elastic.", + "parameters": [], "responses": { "200": { "$ref": "#/components/responses/Health" } - }, - "operationId": "get-update-health-elastic", - "description": "Health on elastic", - "tags": [ - "ping" - ] - }, - "parameters": [] + } + } }, "/update/job/snapshot": { "post": { - "summary": "Start process download and insert snapshot from unpaywall", + "tags": [ + "job" + ], "operationId": "post-update-job-snapshot", + "summary": "Start process download and insert snapshot from unpaywall.", + "description": "Download and insert the latest snapshot from unpaywall.org.", + "security": [ + { + "x-api-key": [] + } + ], "requestBody": { "content": { "application/json": { @@ -221,43 +230,36 @@ } } }, - "description": "index where the unpaywall data will be inserted" + "description": "index where the unpaywall data will be inserted." }, "responses": { "202": { - "description": "Process start" + "$ref": "#/components/responses/Accepted" }, "401": { "$ref": "#/components/responses/Not-authorized" }, "409": { - "description": "Update in progress", - "content": { - "application/json": { - "schema": { - "type": "string", - "default": "changeme" - } - } - } - } - }, - "security": [ - { - "x-api-key": [] + "$ref": "#/components/responses/Conflict" } - ], - "description": "Download and insert the latest snapshot from unpaywall.org", - "tags": [ - "job" - ] + } } }, "/update/job/period": { "post": { - "summary": "Start process download and insert changefiles from unpaywall", + "tags": [ + "job" + ], "operationId": "post-update-job-period", + "summary": "Start process download and insert changefiles from unpaywall.", + "description": "Download and insert changefiles from unpaywall.org.", + "security": [ + { + "x-api-key": [] + } + ], "requestBody": { + "description": "config for process", "content": { "application/json": { "schema": { @@ -281,68 +283,37 @@ } } } - }, - "description": "config for process" + } }, "responses": { "202": { - "description": "Process start" + "$ref": "#/components/responses/Accepted" }, "400": { - "description": "startDate cannot be in the futur or endDate cannot be lower than startDate", - "content": { - "application/json": { - "schema": { - "type": "string", - "default": "changeme" - } - }, - "Test1": { - "examples": { - "response": { - "value": "changeme" - } - } - }, - "Test2": { - "examples": { - "response": { - "value": "changeme" - } - } - } - } + "description": "startDate cannot be in the futur or endDate cannot be lower than startDate." }, "401": { "$ref": "#/components/responses/Not-authorized" }, "409": { - "description": "Update in progress", - "content": { - "application/json": { - "schema": { - "type": "string", - "default": "changeme" - } - } - } + "$ref": "#/components/responses/Conflict" } - }, - "security": [ - { - "x-api-key": [] - } - ], - "description": "Download and insert changefiles from unpaywall.org", - "tags": [ - "job" - ] + } } }, "/update/job/changefile/{filename}": { "post": { - "summary": "Start process insert changesfiles downloaded on ezunpaywall", + "tags": [ + "job" + ], "operationId": "post-update-job-changefile-$-filename", + "summary": "Start process insert changesfiles downloaded on ezunpaywall.", + "description": "Insert changefiles already downloaded on ezunpaywall.", + "security": [ + { + "x-api-key": [] + } + ], "parameters": [ { "in": "path", @@ -377,15 +348,18 @@ }, "responses": { "202": { - "description": "Process start" + "$ref": "#/components/responses/Accepted" }, "400": { - "description": "startDate cannot be in the futur or endDate cannot be lower than startDate", + "description": "startDate cannot be in the futur or endDate cannot be lower than startDate.", "content": { "application/json": { - "schema": { - "type": "string", - "default": "changeme" + "examples": { + "response": { + "value": { + "message": "startDate cannot be in the futur or endDate cannot be lower than startDate." + } + } } } } @@ -394,63 +368,138 @@ "$ref": "#/components/responses/Not-authorized" }, "404": { - "description": "File not found", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "message": { - "type": "string" - } + "$ref": "#/components/responses/File-not-found" + }, + "409": { + "$ref": "#/components/responses/Conflict" + } + } + } + }, + "/update/job/history": { + "post": { + "tags": [ + "job" + ], + "operationId": "post-update-job-history", + "summary": "Start process download and insert changefiles from unpaywall and feed history.", + "description": "Download and insert changefiles from unpaywall.org and feed history.", + "security": [ + { + "x-api-key": [] + } + ], + "requestBody": { + "description": "config for process", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "userName" + ], + "properties": { + "index": { + "type": "string" }, - "x-examples": { - "example-1": { - "message": "File not found" - } - } - } - }, - "File not found": { - "examples": { - "response": { - "value": { - "message": "File not found" - } + "interval": { + "type": "string" + }, + "startDate": { + "type": "string" + }, + "endDate": { + "type": "string" } } } } + } + }, + "responses": { + "202": { + "$ref": "#/components/responses/Accepted" + }, + "400": { + "description": "startDate cannot be in the futur or endDate cannot be lower than startDate." + }, + "401": { + "$ref": "#/components/responses/Not-authorized" }, "409": { - "description": "Update in progress", - "content": { - "application/json": { - "schema": { - "type": "string", - "default": "changeme" - } - } - } + "$ref": "#/components/responses/Conflict" } - }, + } + } + }, + "/update/job/reset": { + "post": { + "tags": [ + "job" + ], + "operationId": "post-update-job-reset", + "summary": "Start process that roll back the current and the history index according to a date.", + "description": "roll back the current and the history index according to a date.", "security": [ { "x-api-key": [] } ], - "description": "Insert changefiles already downloaded on ezunpaywall", - "tags": [ - "job" - ] + "requestBody": { + "description": "config for process", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "userName" + ], + "properties": { + "index": { + "type": "string" + }, + "startDate": { + "type": "string" + } + } + } + } + } + }, + "responses": { + "202": { + "$ref": "#/components/responses/Accepted" + }, + "400": { + "description": "startDate required" + }, + "401": { + "$ref": "#/components/responses/Not-authorized" + }, + "409": { + "$ref": "#/components/responses/Conflict" + } + } } }, - "/update/reports": { + "/update/reports/${type}": { "get": { - "summary": "Get reports", + "tags": [ + "job" + ], "operationId": "get-update-reports", - "description": "Get list of reports or latest report", + "summary": "Get reports.", + "description": "Get list of reports or latest report.", "parameters": [ + { + "in": "path", + "name": "type", + "description": "type of report (unpaywall or unpaywallHistory)", + "required": true, + "schema": { + "type": "string" + } + }, { "in": "query", "name": "latest", @@ -522,18 +571,27 @@ } } } - }, - "tags": [ - "job" - ] + } } }, - "/update/reports/${filename}": { + "/update/reports/${type}/${filename}": { "get": { - "summary": "Get report", + "tags": [ + "job" + ], "operationId": "get-update-reports-$-filename", - "description": "Get report with his filename", + "summary": "Get report.", + "description": "Get report with his filename.", "parameters": [ + { + "in": "path", + "name": "type", + "description": "type of report (unpaywall or unpaywallHistory)", + "required": true, + "schema": { + "type": "string" + } + }, { "in": "path", "name": "filename", @@ -629,47 +687,35 @@ } } }, - "404": { - "description": "Not Found", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "message": { - "type": "string" - } - }, - "x-examples": { - "example-1": { - "message": "File not found" - } - } - } - }, - "File not found": { - "examples": { - "response": { - "value": { - "message": "File not found" - } - } - } - } - } - } - }, - "tags": [ - "job" - ] + "404": { + "$ref": "#/components/responses/File-not-found" + } + } } }, - "/update/snapshots": { + "/update/snapshots/${type}": { "get": { - "summary": "Get snapshots", + "tags": [ + "job" + ], "operationId": "get-update-snapshots", - "description": "Get list of snapshot or latest snapshot", + "summary": "Get snapshots.", + "description": "Get list of snapshot or latest snapshot.", + "security": [ + { + "x-api-key": [] + } + ], "parameters": [ + { + "in": "path", + "name": "type", + "description": "type of report (unpaywall or unpaywallHistory)", + "required": true, + "schema": { + "type": "string" + } + }, { "in": "query", "name": "latest", @@ -698,23 +744,32 @@ "401": { "$ref": "#/components/responses/Not-authorized" } - }, + } + } + }, + "/update/snapshots/${type}/${filename}": { + "get": { + "tags": [ + "job" + ], + "operationId": "get-update-snapshots-$-filename", + "summary": "Get snapshot.", + "description": "Get snapshot with his filename.", "security": [ { "x-api-key": [] } ], - "tags": [ - "job" - ] - } - }, - "/update/snapshots/${filename}": { - "get": { - "summary": "Get snapshot", - "operationId": "get-update-snapshots-$-filename", - "description": "Get snapshot with his filename", "parameters": [ + { + "in": "path", + "name": "type", + "description": "type of report (unpaywall or unpaywallHistory)", + "required": true, + "schema": { + "type": "string" + } + }, { "in": "path", "name": "filename", @@ -733,57 +788,19 @@ "$ref": "#/components/responses/Not-authorized" }, "404": { - "description": "File not found", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "message": { - "type": "string" - } - }, - "x-examples": { - "example-1": { - "message": "File not found" - } - } - }, - "examples": { - "example-1": { - "value": { - "message": "string" - } - } - } - }, - "File not found": { - "examples": { - "response": { - "value": { - "message": "File not found" - } - } - } - } - } - } - }, - "security": [ - { - "x-api-key": [] + "$ref": "#/components/responses/File-not-found" } - ], - "tags": [ - "job" - ] + } } }, "/update/states": { "get": { - "summary": "Get state", + "tags": [ + "job" + ], "operationId": "get-update-states", - "description": "Get latest state", + "summary": "Get state.", + "description": "Get latest state.", "responses": { "200": { "description": "state", @@ -822,14 +839,14 @@ } } } - }, - "tags": [ - "job" - ] + } } }, "/update/status": { "get": { + "tags": [ + "job" + ], "summary": "Get status", "operationId": "get-update-status", "description": "Get status", @@ -853,39 +870,33 @@ } } } - }, - "tags": [ - "job" - ] + } } }, "/update/unpaywall/changefiles": { "get": { - "summary": "Get changefiles from unpaywall", + "tags": [ + "job" + ], "operationId": "get-update-unpaywall-changefiles", - "description": "Get changefiles from unpaywall", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "index": { - "type": "string" - } - } - }, - "examples": { - "example-1": { - "value": { - "index": "string" - } - } - } + "summary": "Get changefiles from unpaywall.", + "description": "Get changefiles from unpaywall.", + "security": [ + { + "x-api-key": [] + } + ], + "parameters": [ + { + "in": "query", + "name": "interval", + "description": "interval of changefiles (\"week\" or \"day\")", + "required": false, + "schema": { + "type": "string" } - }, - "description": "interval of changefiles (\"week\" or \"day\")" - }, + } + ], "responses": { "200": { "description": "status", @@ -965,69 +976,101 @@ "401": { "$ref": "#/components/responses/Not-authorized" } - }, + } + } + }, + "/update/cron/${type}/start": { + "post": { + "tags": [ + "cron" + ], + "operationId": "post-cron-start", + "summary": "Start cron.", + "description": "Start update cron.", "security": [ { "x-api-key": [] } ], - "tags": [ - "job" - ] - } - }, - "/update/cron/start": { - "post": { - "summary": "Start update cron", - "operationId": "post-update-cron-start", + "parameters": [ + { + "in": "path", + "name": "type", + "description": "type of report (unpaywall or unpaywallHistory)", + "required": true, + "schema": { + "type": "string" + } + } + ], "responses": { "202": { - "description": "Accepted" + "$ref": "#/components/responses/Accepted" }, "401": { "$ref": "#/components/responses/Not-authorized" } - }, - "description": "Start update cron", + } + } + }, + "/update/cron/${type}/stop": { + "post": { + "tags": [ + "cron" + ], + "operationId": "post-cron-stop", + "description": "Stop cron.", + "summary": "Stop cron.", "security": [ { "x-api-key": [] } ], - "tags": [ - "cron" - ] - } - }, - "/update/cron/stop": { - "post": { - "summary": "Stop update cron", - "operationId": "post-update-cron-stop", + "parameters": [ + { + "in": "path", + "name": "type", + "description": "type of report (unpaywall or unpaywallHistory)", + "required": true, + "schema": { + "type": "string" + } + } + ], "responses": { "202": { - "description": "Accepted" + "$ref": "#/components/responses/Accepted" }, "401": { "$ref": "#/components/responses/Not-authorized" } - }, - "description": "Stop update cron", + } + } + }, + "/update/cron/${type}": { + "patch": { + "tags": [ + "cron" + ], + "operationId": "patch-cron", + "summary": "update cron.", + "description": "update cron.", "security": [ { "x-api-key": [] } ], - "tags": [ - "cron" - ] - }, - "parameters": [] - }, - "/update/cron": { - "parameters": [], - "patch": { - "summary": "update update cron", - "operationId": "patch-update-cron", + "parameters": [ + { + "in": "path", + "name": "type", + "description": "type of report (unpaywall or unpaywallHistory)", + "required": true, + "schema": { + "type": "string" + } + } + ], "responses": { "200": { "description": "OK", @@ -1084,15 +1127,6 @@ "$ref": "#/components/responses/Not-authorized" } }, - "description": "update update cron", - "security": [ - { - "x-api-key": [] - } - ], - "tags": [ - "cron" - ], "requestBody": { "content": { "application/json": { @@ -1138,8 +1172,23 @@ } }, "get": { - "summary": "Get config of update cron", - "operationId": "get-update-cron", + "tags": [ + "cron" + ], + "operationId": "get-cron", + "summary": "Get config of cron.", + "description": "Get config of cron.", + "parameters": [ + { + "in": "path", + "name": "type", + "description": "type of report (unpaywall or unpaywallHistory)", + "required": true, + "schema": { + "type": "string" + } + } + ], "responses": { "200": { "description": "OK", @@ -1212,63 +1261,55 @@ } } } - }, - "description": "Get config of update cron", - "parameters": [], - "tags": [ - "cron" - ] + } } } }, - "externalDocs": { - "description": "Find out more about Swagger", - "url": "http://swagger.io" - }, - "tags": [ - { - "name": "cron" - }, - { - "name": "job" - }, - { - "name": "ping" - } - ], "components": { "responses": { + "Accepted": { + "description": "Accepted", + "headers": {} + }, "Not-authorized": { "description": "Not authorized", "headers": {}, "content": { - "*/*": { - "schema": { - "type": "object", - "properties": { - "message": { - "type": "string" - } - }, - "x-examples": { - "example-1": { + "application/json": { + "examples": { + "response": { + "value": { "message": "Not authorized" } } - }, + } + } + } + }, + "Conflict": { + "description": "Conflict", + "headers": {}, + "content": { + "application/json": { "examples": { - "example-1": { + "response": { "value": { - "message": "string" + "message": "Update in progress." } } } - }, - "Not authorized": { + } + } + }, + "File-not-found": { + "description": "File not found", + "headers": {}, + "content": { + "application/json": { "examples": { "response": { "value": { - "message": "Not authorized" + "message": "File not found." } } } diff --git a/src/update/openapi.yml b/src/update/openapi.yml index 518809be..7e81ff09 100644 --- a/src/update/openapi.yml +++ b/src/update/openapi.yml @@ -157,7 +157,7 @@ paths: description: index where the unpaywall data will be inserted. responses: '202': - description: Process start + $ref: '#/components/responses/Accepted' '401': $ref: '#/components/responses/Not-authorized' '409': @@ -191,7 +191,7 @@ paths: type: string responses: '202': - description: Process start. + $ref: '#/components/responses/Accepted' '400': description: startDate cannot be in the futur or endDate cannot be lower than startDate. '401': @@ -230,18 +230,19 @@ paths: description: config for process responses: '202': - description: Process start + $ref: '#/components/responses/Accepted' '400': description: startDate cannot be in the futur or endDate cannot be lower than startDate. content: application/json: - schema: - type: string - default: changeme + examples: + response: + value: + message: startDate cannot be in the futur or endDate cannot be lower than startDate. '401': $ref: '#/components/responses/Not-authorized' '404': - description: File not found + $ref: '#/components/responses/File-not-found' '409': $ref: '#/components/responses/Conflict' @@ -273,7 +274,7 @@ paths: type: string responses: '202': - description: Process start + $ref: '#/components/responses/Accepted' '400': description: startDate cannot be in the futur or endDate cannot be lower than startDate. '401': @@ -305,7 +306,7 @@ paths: type: string responses: '202': - description: Process start + $ref: '#/components/responses/Accepted' '400': description: startDate required '401': @@ -313,7 +314,7 @@ paths: '409': $ref: '#/components/responses/Conflict' - /update/reports: + /update/reports/${type}: get: tags: - job @@ -321,6 +322,12 @@ paths: summary: Get reports. description: Get list of reports or latest report. parameters: + - in: path + name: type + description: type of report (unpaywall or unpaywallHistory) + required: true + schema: + type: string - in: query name: latest description: latest @@ -372,7 +379,7 @@ paths: error: false took: 0.256 - /update/reports/${filename}: + /update/reports/${type}/${filename}: get: tags: - job @@ -380,6 +387,12 @@ paths: summary: Get report. description: Get report with his filename. parameters: + - in: path + name: type + description: type of report (unpaywall or unpaywallHistory) + required: true + schema: + type: string - in: path name: filename description: filename @@ -448,24 +461,9 @@ paths: error: false took: 0.256 '404': - description: Not Found - content: - application/json: - schema: - type: object - properties: - message: - type: string - x-examples: - example-1: - message: File not found - File not found: - examples: - response: - value: - message: File not found + $ref: '#/components/responses/File-not-found' - /update/snapshots: + /update/snapshots/${type}: get: tags: - job @@ -475,6 +473,12 @@ paths: security: - x-api-key: [] parameters: + - in: path + name: type + description: type of report (unpaywall or unpaywallHistory) + required: true + schema: + type: string - in: query name: latest description: latest @@ -493,7 +497,7 @@ paths: '401': $ref: '#/components/responses/Not-authorized' - /update/snapshots/${filename}: + /update/snapshots/${type}/${filename}: get: tags: - job @@ -503,6 +507,12 @@ paths: security: - x-api-key: [] parameters: + - in: path + name: type + description: type of report (unpaywall or unpaywallHistory) + required: true + schema: + type: string - in: path name: filename description: filename @@ -515,26 +525,7 @@ paths: '401': $ref: '#/components/responses/Not-authorized' '404': - description: File not found - content: - application/json: - schema: - type: object - properties: - message: - type: string - x-examples: - example-1: - message: File not found - examples: - example-1: - value: - message: string - File not found: - examples: - response: - value: - message: File not found + $ref: '#/components/responses/File-not-found' /update/states: get: @@ -664,48 +655,68 @@ paths: '401': $ref: '#/components/responses/Not-authorized' - /update/cron/start: + /update/cron/${type}/start: post: tags: - cron - operationId: post-update-cron-start - summary: Start update cron. + operationId: post-cron-start + summary: Start cron. description: Start update cron. + security: + - x-api-key: [] + parameters: + - in: path + name: type + description: type of report (unpaywall or unpaywallHistory) + required: true + schema: + type: string responses: '202': - description: Accepted + $ref: '#/components/responses/Accepted' '401': $ref: '#/components/responses/Not-authorized' - security: - - x-api-key: [] - /update/cron/stop: + + /update/cron/${type}/stop: post: tags: - cron - operationId: post-update-cron-stop - description: Stop update cron. - summary: Stop update cron. + operationId: post-cron-stop + description: Stop cron. + summary: Stop cron. security: - x-api-key: [] - parameters: [] + parameters: + - in: path + name: type + description: type of report (unpaywall or unpaywallHistory) + required: true + schema: + type: string responses: '202': - description: Accepted + $ref: '#/components/responses/Accepted' '401': $ref: '#/components/responses/Not-authorized' - /update/cron: + /update/cron/${type}: patch: tags: - cron - operationId: patch-update-cron - summary: update update cron. - description: update update cron. + operationId: patch-cron + summary: update cron. + description: update cron. security: - x-api-key: [] - parameters: [] + parameters: + - in: path + name: type + description: type of report (unpaywall or unpaywallHistory) + required: true + schema: + type: string responses: '200': description: OK @@ -774,10 +785,16 @@ paths: get: tags: - cron - operationId: get-update-cron - summary: Get config of update cron. - description: Get config of update cron. - parameters: [] + operationId: get-cron + summary: Get config of cron. + description: Get config of cron. + parameters: + - in: path + name: type + description: type of report (unpaywall or unpaywallHistory) + required: true + schema: + type: string responses: '200': description: OK @@ -819,7 +836,7 @@ paths: status: true index: unpaywall interval: day - 'off': + off: value: config: time: 0 0 0 * * * @@ -829,6 +846,9 @@ paths: components: responses: + Accepted: + description: Accepted + headers: {} Not-authorized: description: Not authorized headers: {} @@ -847,6 +867,15 @@ components: response: value: message: Update in progress. + File-not-found: + description: File not found + headers: {} + content: + application/json: + examples: + response: + value: + message: File not found. Health: description: Example response content: diff --git a/src/update/test/unpaywall/cron.js b/src/update/test/cron.js similarity index 98% rename from src/update/test/unpaywall/cron.js rename to src/update/test/cron.js index 32d008fa..12abad59 100644 --- a/src/update/test/unpaywall/cron.js +++ b/src/update/test/cron.js @@ -3,9 +3,9 @@ const { expect } = require('chai'); const chai = require('chai'); const chaiHttp = require('chai-http'); -const reset = require('../utils/reset'); +const reset = require('./utils/reset'); -const ping = require('../utils/ping'); +const ping = require('./utils/ping'); chai.use(chaiHttp); diff --git a/src/update/test/utils/reset.js b/src/update/test/utils/reset.js index d1b23e93..bc2b1a54 100644 --- a/src/update/test/utils/reset.js +++ b/src/update/test/utils/reset.js @@ -26,6 +26,7 @@ async function reset() { await deleteIndex('unpaywall_enriched'); await deleteIndex('unpaywall_history'); await resetCronConfig('unpaywall'); + await resetCronConfig('unpaywallHistory'); } module.exports = reset; From 55a9a872510334843ed7c2dea5aaaf0abdcb66a5 Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Thu, 11 Jan 2024 14:12:30 +0100 Subject: [PATCH 015/114] fix: replace .json by .yml for openapi.yml --- src/apikey/lib/routers/openapi.js | 5 +- src/apikey/openapi.json | 741 ---------- src/apikey/openapi.yml | 477 +++++++ src/enrich/lib/routers/openapi.js | 5 +- src/enrich/openapi.json | 633 --------- src/enrich/openapi.yml | 409 ++++++ src/frontend/components/openapi/OpenApi.vue | 4 +- src/graphql/lib/routers/openapi.js | 5 +- src/graphql/openapi.json | 306 ----- src/graphql/openapi.yml | 189 +++ src/health/lib/routers/openapi.js | 5 +- src/health/openapi.json | 477 ------- src/health/openapi.yml | 316 +++++ src/mail/lib/routers/openapi.js | 5 +- src/mail/openapi.json | 421 ------ src/mail/openapi.yml | 261 ++++ src/update/lib/routers/openapi.js | 5 +- src/update/openapi.json | 1374 ------------------- 18 files changed, 1666 insertions(+), 3972 deletions(-) delete mode 100644 src/apikey/openapi.json create mode 100644 src/apikey/openapi.yml delete mode 100644 src/enrich/openapi.json create mode 100644 src/enrich/openapi.yml delete mode 100644 src/graphql/openapi.json create mode 100644 src/graphql/openapi.yml delete mode 100644 src/health/openapi.json create mode 100644 src/health/openapi.yml delete mode 100644 src/mail/openapi.json create mode 100644 src/mail/openapi.yml delete mode 100644 src/update/openapi.json diff --git a/src/apikey/lib/routers/openapi.js b/src/apikey/lib/routers/openapi.js index f68e1ca0..bdf85c7d 100644 --- a/src/apikey/lib/routers/openapi.js +++ b/src/apikey/lib/routers/openapi.js @@ -1,10 +1,9 @@ const router = require('express').Router(); - -const openapi = require('../../openapi.json'); +const path = require('path'); /** * Route that give the openapi.json file. */ -router.get('/openapi.json', (req, res) => res.status(200).json(openapi)); +router.get('/openapi.yml', (req, res) => res.sendFile(path.resolve(__dirname, '..', '..', 'openapi.yml'))); module.exports = router; diff --git a/src/apikey/openapi.json b/src/apikey/openapi.json deleted file mode 100644 index 38303b0a..00000000 --- a/src/apikey/openapi.json +++ /dev/null @@ -1,741 +0,0 @@ -{ - "openapi": "3.0.0", - "info": { - "description": "The Apikey service is reserved for the administrator, it allows to manage the API keys", - "version": "1.0.0", - "title": "Apikey service", - "contact": { - "email": "ezteam@couperin.org", - "name": "ezTeam" - }, - "license": { - "name": "CeCILL 2.1", - "url": "http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html" - } - }, - "tags": [ - { - "name": "apikey" - }, - { - "name": "ping" - } - ], - "paths": { - "/apikey": { - "get": { - "summary": "Name of service", - "responses": { - "200": { - "description": "OK", - "content": { - "Success": { - "examples": { - "response": { - "value": "apikey service" - } - }, - "schema": { - "type": "string", - "x-examples": { - "example-1": "apikey service" - } - } - } - } - } - }, - "operationId": "get-apikey", - "description": "Name of service", - "tags": [ - "ping" - ] - }, - "parameters": [] - }, - "/apikey/ping": { - "get": { - "summary": "Ping apikey service", - "operationId": "get-apikey-ping", - "description": "Ping apikey service", - "responses": { - "204": { - "description": "No Content" - } - }, - "tags": [ - "ping" - ] - } - }, - "/apikey/health": { - "get": { - "summary": "Health", - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "examples": { - "Success": { - "value": { - "redis": { - "elapsedTime": 0, - "status": true - }, - "elapsedTime": 0, - "status": true - } - }, - "Error redis": { - "value": { - "redis": { - "elapsedTime": 3000, - "status": false, - "error": "time out" - }, - "elapsedTime": 3001, - "status": false - } - } - }, - "schema": { - "type": "object", - "properties": { - "redis": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - }, - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - }, - "x-examples": { - "Example 1": { - "redis": { - "elapsedTime": 0, - "status": true - }, - "elapsedTime": 0, - "status": true - } - } - } - } - } - } - }, - "operationId": "get-apikey-health", - "description": "Health on all service connected to apikey service", - "tags": [ - "ping" - ] - }, - "parameters": [] - }, - "/apikey/health/redis": { - "get": { - "summary": "Health on redis service", - "operationId": "get-apikey-health-redis", - "description": "Health on redis", - "tags": [ - "ping" - ], - "parameters": [], - "responses": { - "200": { - "$ref": "#/components/responses/Health" - } - } - }, - "parameters": [] - }, - "/apikey/keys/${apikey}": { - "get": { - "tags": [ - "apikey" - ], - "summary": "Get config of apikey", - "operationId": "get-apikey-keys-$-apikey", - "description": "Get config of apikey", - "parameters": [ - { - "in": "path", - "name": "apikey", - "description": "apikey", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "config of apikey", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "apikey": { - "type": "string", - "default": "azerty" - }, - "name": { - "type": "string", - "default": "john doe" - }, - "attributes": { - "type": "array", - "items": { - "type": "string", - "default": "*" - } - }, - "access": { - "type": "array", - "items": { - "type": "string", - "default": "graphql" - } - }, - "allowed": { - "type": "boolean" - } - } - } - } - } - } - } - }, - "put": { - "tags": [ - "apikey" - ], - "summary": "Update apikey", - "operationId": "put-apikeys-keys-$-apikey", - "description": "Update apikey", - "parameters": [ - { - "in": "path", - "name": "apikey", - "description": "apikey", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "update apikey", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "apikey": { - "type": "string", - "default": "azerty" - }, - "name": { - "type": "string", - "default": "john doe" - }, - "attributes": { - "type": "array", - "items": { - "type": "string", - "default": "*" - } - }, - "access": { - "type": "array", - "items": { - "type": "string", - "default": "graphql" - } - }, - "allowed": { - "type": "boolean" - } - } - }, - "examples": { - "John Doe apikey": { - "value": { - "apikey": "azerty", - "name": "john doe", - "attributes": [ - "*" - ], - "access": [ - "graphql" - ], - "allowed": true - } - } - } - } - } - }, - "400": { - "description": "Bad Request", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "message": { - "type": "string" - } - }, - "x-examples": { - "example-1": { - "message": "\"name\" is required" - } - } - }, - "examples": { - "Name is required": { - "value": { - "message": "\"name\" is required" - } - } - } - } - } - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - } - }, - "security": [ - { - "x-api-key": [] - } - ] - }, - "delete": { - "tags": [ - "apikey" - ], - "summary": "Delete apikey", - "operationId": "delete-apikey-keys-$-apikey", - "description": "Delete apikey", - "parameters": [ - { - "in": "path", - "name": "apikey", - "description": "apikey", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "204": { - "description": "apikey deleted" - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - } - }, - "security": [ - { - "x-api-key": [] - } - ] - } - }, - "/apikey/keys": { - "get": { - "tags": [ - "apikey" - ], - "summary": "Get all config of all apikey", - "operationId": "get-apikey-keys", - "description": "Get all config of all apikey", - "responses": { - "200": { - "description": "update apikey", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "type": "object", - "properties": { - "apikey": { - "type": "string", - "default": "azerty" - }, - "name": { - "type": "string", - "default": "john doe" - }, - "attributes": { - "type": "array", - "items": { - "type": "string", - "default": "*" - } - }, - "access": { - "type": "array", - "items": { - "type": "string", - "default": "graphql" - } - }, - "allowed": { - "type": "boolean" - } - } - } - }, - "examples": { - "John doe apikey": { - "value": [ - { - "apikey": "azerty", - "name": "john doe", - "attributes": [ - "*" - ], - "access": [ - "graphql" - ], - "allowed": true - } - ] - } - } - }, - "apikey of John Doe": { - "examples": { - "response": { - "value": [ - { - "apikey": "azerty123456", - "name": "John Doe", - "attributes": [ - "*" - ], - "access": [ - "graphql" - ], - "allowed": true - } - ] - } - } - } - } - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - } - }, - "security": [ - { - "x-api-key": [] - } - ] - }, - "delete": { - "tags": [ - "apikey" - ], - "summary": "Delete all apikey", - "operationId": "delete-apikey-keys", - "description": "Delete all apikey", - "responses": { - "204": { - "description": "all apikey are deleted" - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - } - }, - "security": [ - { - "x-api-key": [] - } - ] - }, - "post": { - "tags": [ - "apikey" - ], - "summary": "Create apikey", - "operationId": "post-apikey-keys", - "description": "Create new apikey", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "attributes": { - "type": "array", - "items": { - "type": "string" - } - }, - "access": { - "type": "array", - "items": { - "type": "string" - } - }, - "allowed": { - "type": "boolean" - } - } - } - } - }, - "description": "config for apikey", - "required": true - }, - "responses": { - "200": { - "description": "update apikey", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "apikey": { - "type": "string", - "default": "azerty" - }, - "name": { - "type": "string", - "default": "john doe" - }, - "attributes": { - "type": "array", - "items": { - "type": "string", - "default": "*" - } - }, - "access": { - "type": "array", - "items": { - "type": "string", - "default": "graphql" - } - }, - "allowed": { - "type": "boolean" - } - } - } - } - } - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - } - }, - "security": [ - { - "x-api-key": [] - } - ] - } - }, - "/apikey/keys/load": { - "post": { - "tags": [ - "apikey" - ], - "summary": "Load apikey", - "operationId": "post-apikeys-keys-load", - "description": "Load apikey", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "type": "object", - "properties": { - "apikey": { - "type": "string" - }, - "name": { - "type": "string" - }, - "attributes": { - "type": "array", - "items": { - "type": "string" - } - }, - "access": { - "type": "array", - "items": { - "type": "string" - } - }, - "allowed": { - "type": "boolean" - } - } - }, - "x-examples": { - "example-1": [ - { - "apikey": "string", - "name": "string", - "attributes": [ - "string" - ], - "access": [ - "string" - ], - "allowed": true - } - ] - } - }, - "examples": { - "example-1": { - "value": [ - { - "apikey": "string", - "name": "string", - "attributes": [ - "string" - ], - "access": [ - "string" - ], - "allowed": true - } - ] - } - } - } - }, - "description": "apikeys", - "required": true - }, - "responses": { - "204": { - "description": "all apikey are loaded" - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - } - }, - "security": [ - { - "x-api-key": [] - } - ] - } - } - }, - "externalDocs": { - "description": "Find out more about Swagger", - "url": "http://swagger.io" - }, - "components": { - "responses": { - "Not-authorized": { - "description": "Not authorized", - "content": { - "Not authorized": { - "examples": { - "response": { - "value": { - "message": "Not authorized" - } - } - } - } - }, - "headers": { - "x-api-key": { - "schema": { - "type": "string" - }, - "description": "redis password" - } - } - }, - "Health": { - "description": "Example response", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "status": { - "type": "boolean" - }, - "elapsedTime": { - "type": "integer" - } - }, - "x-examples": { - "Example 1": { - "name": "redis", - "status": true, - "elapsedTime": 1 - } - } - }, - "examples": { - "Success": { - "value": { - "name": "name of service", - "status": true, - "elapsedTime": 1 - } - }, - "Error redis": { - "value": { - "name": "redis", - "elapsedTime": 3002, - "error": "time out", - "status": false - } - } - } - } - } - } - }, - "securitySchemes": { - "x-api-key": { - "name": "API Key", - "type": "apiKey", - "in": "header" - } - } - } -} \ No newline at end of file diff --git a/src/apikey/openapi.yml b/src/apikey/openapi.yml new file mode 100644 index 00000000..9500e81e --- /dev/null +++ b/src/apikey/openapi.yml @@ -0,0 +1,477 @@ +openapi: 3.0.0 +info: + description: The Apikey service is reserved for the administrator, it allows to manage the API keys + version: 1.0.0 + title: Apikey service + contact: + email: ezteam@couperin.org + name: ezTeam + license: + name: CeCILL 2.1 + url: http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html +servers: + - url: https://unpaywall.inist.fr/api +tags: + - name: apikey + - name: ping +paths: + /apikey: + get: + tags: + - ping + summary: Name of service + operationId: get-apikey + description: Name of service + parameters: [] + responses: + '200': + description: OK + content: + Success: + examples: + response: + value: apikey service + schema: + type: string + x-examples: + example-1: apikey service + /apikey/ping: + get: + tags: + - ping + summary: Ping apikey service + operationId: get-apikey-ping + description: Ping apikey service + responses: + '204': + description: No Content + + /apikey/health: + get: + tags: + - ping + summary: Health + operationId: get-apikey-health + description: Health on all service connected to apikey service + parameters: [] + responses: + '200': + description: OK + content: + application/json: + examples: + Success: + value: + redis: + elapsedTime: 0 + status: true + elapsedTime: 0 + status: true + Error redis: + value: + redis: + elapsedTime: 3000 + status: false + error: time out + elapsedTime: 3001 + status: false + schema: + type: object + properties: + redis: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + elapsedTime: + type: integer + status: + type: boolean + x-examples: + Example 1: + redis: + elapsedTime: 0 + status: true + elapsedTime: 0 + status: true + + /apikey/health/redis: + get: + summary: Health on redis service + operationId: get-apikey-health-redis + description: Health on redis + tags: + - ping + parameters: [] + responses: + '200': + $ref: '#/components/responses/Health' + parameters: [] + /apikey/keys/${apikey}: + get: + tags: + - apikey + summary: Get config of apikey + operationId: get-apikey-keys-$-apikey + description: Get config of apikey + parameters: + - in: path + name: apikey + description: apikey + required: true + schema: + type: string + responses: + '200': + description: config of apikey + content: + application/json: + schema: + type: object + properties: + apikey: + type: string + default: azerty + name: + type: string + default: john doe + attributes: + type: array + items: + type: string + default: '*' + access: + type: array + items: + type: string + default: graphql + allowed: + type: boolean + put: + tags: + - apikey + summary: Update apikey + operationId: put-apikeys-keys-$-apikey + description: Update apikey + security: + - x-api-key: [] + parameters: + - in: path + name: apikey + description: apikey + required: true + schema: + type: string + responses: + '200': + description: update apikey + content: + application/json: + schema: + type: object + properties: + apikey: + type: string + default: azerty + name: + type: string + default: john doe + attributes: + type: array + items: + type: string + default: '*' + access: + type: array + items: + type: string + default: graphql + allowed: + type: boolean + examples: + John Doe apikey: + value: + apikey: azerty + name: john doe + attributes: + - '*' + access: + - graphql + allowed: true + '400': + description: Bad Request + content: + application/json: + schema: + type: object + properties: + message: + type: string + x-examples: + example-1: + message: '"name" is required' + examples: + Name is required: + value: + message: '"name" is required' + '401': + $ref: '#/components/responses/Not-authorized' + delete: + tags: + - apikey + summary: Delete apikey + operationId: delete-apikey-keys-$-apikey + description: Delete apikey + security: + - x-api-key: [] + parameters: + - in: path + name: apikey + description: apikey + required: true + schema: + type: string + responses: + '204': + description: apikey deleted + '401': + $ref: '#/components/responses/Not-authorized' + /apikey/keys: + get: + tags: + - apikey + summary: Get all config of all apikey + operationId: get-apikey-keys + description: Get all config of all apikey + security: + - x-api-key: [] + responses: + '200': + description: update apikey + content: + application/json: + schema: + type: array + items: + type: object + properties: + apikey: + type: string + default: azerty + name: + type: string + default: john doe + attributes: + type: array + items: + type: string + default: '*' + access: + type: array + items: + type: string + default: graphql + allowed: + type: boolean + examples: + John doe apikey: + value: + - apikey: azerty + name: john doe + attributes: + - '*' + access: + - graphql + allowed: true + apikey of John Doe: + examples: + response: + value: + - apikey: azerty123456 + name: John Doe + attributes: + - '*' + access: + - graphql + allowed: true + '401': + $ref: '#/components/responses/Not-authorized' + delete: + tags: + - apikey + summary: Delete all apikey + operationId: delete-apikey-keys + description: Delete all apikey + security: + - x-api-key: [] + responses: + '204': + description: all apikey are deleted + '401': + $ref: '#/components/responses/Not-authorized' + post: + tags: + - apikey + summary: Create apikey + operationId: post-apikey-keys + description: Create new apikey + security: + - x-api-key: [] + requestBody: + content: + application/json: + schema: + type: object + properties: + name: + type: string + attributes: + type: array + items: + type: string + access: + type: array + items: + type: string + allowed: + type: boolean + description: config for apikey + required: true + responses: + '200': + description: update apikey + content: + application/json: + schema: + type: object + properties: + apikey: + type: string + default: azerty + name: + type: string + default: john doe + attributes: + type: array + items: + type: string + default: '*' + access: + type: array + items: + type: string + default: graphql + allowed: + type: boolean + '401': + $ref: '#/components/responses/Not-authorized' + /apikey/keys/load: + post: + tags: + - apikey + summary: Load apikey + operationId: post-apikeys-keys-load + description: Load apikey + security: + - x-api-key: [] + requestBody: + content: + application/json: + schema: + type: array + items: + type: object + properties: + apikey: + type: string + name: + type: string + attributes: + type: array + items: + type: string + access: + type: array + items: + type: string + allowed: + type: boolean + x-examples: + example-1: + - apikey: string + name: string + attributes: + - string + access: + - string + allowed: true + examples: + example-1: + value: + - apikey: string + name: string + attributes: + - string + access: + - string + allowed: true + description: apikeys + required: true + responses: + '204': + description: all apikey are loaded + '401': + $ref: '#/components/responses/Not-authorized' +components: + responses: + Not-authorized: + description: Not authorized + headers: + x-api-key: + schema: + type: string + description: redis password + content: + application/json: + examples: + response: + value: + message: Not authorized + + Health: + description: Example response + content: + application/json: + schema: + type: object + properties: + name: + type: string + status: + type: boolean + elapsedTime: + type: integer + x-examples: + Example 1: + name: redis + status: true + elapsedTime: 1 + examples: + Success: + value: + name: name of service + status: true + elapsedTime: 1 + Error redis: + value: + name: redis + elapsedTime: 3002 + error: time out + status: false + securitySchemes: + x-api-key: + name: API Key + type: apiKey + in: header diff --git a/src/enrich/lib/routers/openapi.js b/src/enrich/lib/routers/openapi.js index f68e1ca0..bdf85c7d 100644 --- a/src/enrich/lib/routers/openapi.js +++ b/src/enrich/lib/routers/openapi.js @@ -1,10 +1,9 @@ const router = require('express').Router(); - -const openapi = require('../../openapi.json'); +const path = require('path'); /** * Route that give the openapi.json file. */ -router.get('/openapi.json', (req, res) => res.status(200).json(openapi)); +router.get('/openapi.yml', (req, res) => res.sendFile(path.resolve(__dirname, '..', '..', 'openapi.yml'))); module.exports = router; diff --git a/src/enrich/openapi.json b/src/enrich/openapi.json deleted file mode 100644 index 46a9b03f..00000000 --- a/src/enrich/openapi.json +++ /dev/null @@ -1,633 +0,0 @@ -{ - "openapi": "3.0.0", - "info": { - "description": "The enrichment service allows to enrich a csv or jsonl file which contains a \"doi\" field with the metadata of the ezunpaywall mirror", - "version": "1.0.0", - "title": "Enrich service", - "contact": { - "email": "ezteam@couperin.org", - "name": "ezTeam" - }, - "license": { - "name": "CeCILL 2.1", - "url": "http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html" - } - }, - "tags": [ - { - "name": "job" - }, - { - "name": "ping" - } - ], - "paths": { - "/enrich": { - "get": { - "summary": "Name of service", - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "type": "string", - "x-examples": { - "Example 1": "enrich service" - } - }, - "examples": { - "response": { - "value": "enrich service" - } - } - } - } - } - }, - "operationId": "get-enrich", - "description": "Get name of enrich service", - "tags": [ - "ping" - ] - }, - "parameters": [] - }, - "/enrich/ping": { - "get": { - "summary": "Ping enrich service", - "responses": { - "204": { - "description": "No Content" - } - }, - "operationId": "get-enrich-ping", - "description": "Ping enrich service", - "tags": [ - "ping" - ] - } - }, - "/enrich/health": { - "get": { - "summary": "Health", - "operationId": "get-enrich-health", - "description": "Health on all service connected to enrich service", - "tags": [ - "ping" - ], - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "redis": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - }, - "graphql": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - }, - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - }, - "x-examples": { - "Example 1": { - "redis": { - "elapsedTime": 1, - "status": true - }, - "graphql": { - "elapsedTime": 2, - "status": true - }, - "elapsedTime": 2, - "status": true - } - } - }, - "examples": { - "Success": { - "value": { - "redis": { - "elapsedTime": 4, - "status": true - }, - "graphql": { - "elapsedTime": 3, - "status": true - }, - "elapsedTime": 6, - "status": true - } - }, - "Error redis": { - "value": { - "redis": { - "elapsedTime": 3003, - "status": false, - "error": "time out" - }, - "graphql": { - "elapsedTime": 1, - "status": true - }, - "elapsedTime": 3003, - "status": false - } - } - } - } - } - } - } - }, - "parameters": [] - }, - "/enrich/health/redis": { - "get": { - "summary": "Health on redis service", - "responses": { - "200": { - "$ref": "#/components/responses/Health" - } - }, - "operationId": "get-enrich-health-redis", - "description": "Health on redis", - "tags": [ - "ping" - ] - }, - "parameters": [] - }, - "/enrich/health/elastic": { - "get": { - "summary": "Health on graphql service", - "operationId": "get-enrich-health-elastic", - "description": "Health on elastic", - "tags": [ - "ping" - ], - "responses": { - "200": { - "$ref": "#/components/responses/Health" - } - } - }, - "parameters": [] - }, - "/enrich/upload": { - "post": { - "summary": "Upload a file", - "operationId": "post-enrich-upload", - "description": "Upload a file to be enriched", - "responses": { - "200": { - "description": "filename of uploaded file", - "content": { - "application/json": { - "schema": { - "type": "string", - "default": "id" - } - } - } - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - } - }, - "security": [ - { - "x-api-key": [] - } - ], - "tags": [ - "job" - ] - }, - "parameters": [] - }, - "/enrich/job/{filename}": { - "post": { - "summary": "Enrich job", - "operationId": "post-enrich-job-$-filename", - "description": "Start a enrich job with uploaded file", - "parameters": [ - { - "in": "path", - "name": "filename", - "description": "filename", - "required": true, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "keys": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "type": "string" - }, - "args": { - "type": "string" - }, - "index": { - "type": "string" - }, - "separator": { - "type": "string" - } - } - } - } - } - } - } - }, - "description": "apikeys", - "required": true - }, - "responses": { - "200": { - "description": "filename of process", - "content": { - "application/json": { - "schema": { - "type": "string", - "default": "id" - } - } - } - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - } - }, - "security": [ - { - "x-api-key": [] - } - ], - "tags": [ - "job" - ] - } - }, - "/enrich/enriched": { - "get": { - "summary": "Get enriched files", - "operationId": "get-enrich-enriched", - "description": "Get the list of enriched files", - "responses": { - "200": { - "description": "list of enriched file", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "type": "string", - "default": "file.csv" - } - } - } - } - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - } - }, - "security": [ - { - "x-api-key": [] - } - ], - "tags": [ - "job" - ] - } - }, - "/enrich/enriched/${filename}": { - "get": { - "summary": "Get enriched file", - "operationId": "get-enrich-enriched-$-filename", - "description": "Get the enriched file, generated at the end of the enrichment process", - "parameters": [ - { - "in": "path", - "name": "filename", - "description": "filename", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "enriched file", - "content": { - "application/json": { - "schema": { - "type": "string", - "format": "binary" - } - } - } - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - } - }, - "security": [ - { - "x-api-key": [] - } - ], - "tags": [ - "job" - ] - } - }, - "/enrich/states": { - "get": { - "summary": "Get list of states of enrich job", - "operationId": "get-enrich-states", - "description": "Get list of filenames state of enrich process", - "parameters": [ - { - "in": "query", - "name": "latest", - "description": "latest", - "required": false, - "schema": { - "type": "boolean" - } - } - ], - "responses": { - "200": { - "description": "list of states file", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "type": "string", - "default": "file.json" - } - } - } - } - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - } - }, - "tags": [ - "job" - ] - } - }, - "/enrich/states/${filename}": { - "get": { - "summary": "Get State", - "operationId": "get-enrich-states-$-filename", - "description": "Get state of enrich process with his filename", - "parameters": [ - { - "in": "path", - "name": "filename", - "description": "filename", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "list of uploaded file", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "done": { - "type": "boolean", - "default": true - }, - "loaded": { - "type": "integer", - "default": 150 - }, - "linesRead": { - "type": "integer", - "default": 3 - }, - "enrichedLines": { - "type": "integer", - "default": 3 - }, - "createdAt": { - "type": "string", - "default": "2021-11-30T10:00:00.000Z" - }, - "endAt": { - "type": "string", - "default": "2021-11-30T10:01:00.000Z" - }, - "error": { - "type": "boolean", - "default": false - } - } - } - } - } - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - }, - "404": { - "description": "File not found", - "content": { - "File not found": { - "examples": { - "response": { - "value": { - "message": "File not found" - } - } - } - } - } - } - }, - "tags": [ - "job" - ] - } - }, - "/enrich/uploaded": { - "get": { - "summary": "Get uploaded files", - "operationId": "get-enriched-uploaded", - "description": "Get the lists of uploaded files", - "responses": { - "200": { - "description": "list of uploaded file", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "type": "string", - "default": "file.csv" - } - } - } - } - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - } - }, - "security": [ - { - "x-api-key": [] - } - ], - "tags": [ - "job" - ] - }, - "parameters": [] - } - }, - "externalDocs": { - "description": "Find out more about Swagger", - "url": "http://swagger.io" - }, - "components": { - "responses": { - "Not-authorized": { - "description": "Not authorized", - "headers": {}, - "content": { - "*/*": { - "schema": { - "type": "object", - "properties": { - "message": { - "type": "string" - } - }, - "x-examples": { - "example-1": { - "message": "Not authorized" - } - } - } - }, - "Not authorized": { - "examples": { - "response": { - "value": { - "message": "Not authorized" - } - } - } - } - } - }, - "Health": { - "description": "Example response", - "content": { - "Success": { - "schema": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "status": { - "type": "boolean" - }, - "elapsedTime": { - "type": "integer" - } - }, - "x-examples": { - "Example 1": { - "name": "redis", - "status": true, - "elapsedTime": 1 - } - } - }, - "examples": { - "Success": { - "value": { - "name": "name of service", - "status": true, - "elapsedTime": 1 - } - }, - "Error redis": { - "value": { - "name": "redis", - "elapsedTime": 3002, - "error": "time out", - "status": false - } - } - } - } - } - } - }, - "securitySchemes": { - "x-api-key": { - "name": "API Key", - "type": "apiKey", - "in": "header" - } - } - } -} \ No newline at end of file diff --git a/src/enrich/openapi.yml b/src/enrich/openapi.yml new file mode 100644 index 00000000..ceb2215d --- /dev/null +++ b/src/enrich/openapi.yml @@ -0,0 +1,409 @@ +openapi: 3.0.0 +info: + description: The enrichment service allows to enrich a csv or jsonl file which contains a "doi" field with the metadata of the ezunpaywall mirror + version: 1.0.0 + title: Enrich service + contact: + email: ezteam@couperin.org + name: ezTeam + license: + name: CeCILL 2.1 + url: http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html +servers: + - url: https://unpaywall.inist.fr/api +tags: + - name: job + - name: ping +paths: + /enrich: + get: + tags: + - ping + summary: Name of service + operationId: get-enrich + description: Get name of enrich service + parameters: [] + responses: + '200': + description: OK + content: + application/json: + schema: + type: string + x-examples: + Example 1: enrich service + examples: + response: + value: enrich service + + /enrich/ping: + get: + tags: + - ping + summary: Ping enrich service + operationId: get-enrich-ping + description: Ping enrich service + responses: + '204': + description: No Content + + /enrich/health: + get: + summary: Health + operationId: get-enrich-health + description: Health on all service connected to enrich service + parameters: [] + tags: + - ping + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + redis: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + graphql: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + elapsedTime: + type: integer + status: + type: boolean + x-examples: + Example 1: + redis: + elapsedTime: 1 + status: true + graphql: + elapsedTime: 2 + status: true + elapsedTime: 2 + status: true + examples: + Success: + value: + redis: + elapsedTime: 4 + status: true + graphql: + elapsedTime: 3 + status: true + elapsedTime: 6 + status: true + Error redis: + value: + redis: + elapsedTime: 3003 + status: false + error: time out + graphql: + elapsedTime: 1 + status: true + elapsedTime: 3003 + status: false + /enrich/health/redis: + get: + tags: + - ping + operationId: get-enrich-health-redis + description: Health on redis + summary: Health on redis service + parameters: [] + responses: + '200': + $ref: '#/components/responses/Health' + + /enrich/health/elastic: + get: + tags: + - ping + summary: Health on graphql service + operationId: get-enrich-health-elastic + description: Health on elastic + parameters: [] + responses: + '200': + $ref: '#/components/responses/Health' + + /enrich/upload: + post: + tags: + - job + summary: Upload a file + operationId: post-enrich-upload + description: Upload a file to be enriched + security: + - x-api-key: [] + parameters: [] + responses: + '200': + description: filename of uploaded file + content: + application/json: + schema: + type: string + default: id + '401': + $ref: '#/components/responses/Not-authorized' + + /enrich/job/{filename}: + post: + tags: + - job + summary: Enrich job + operationId: post-enrich-job-$-filename + description: Start a enrich job with uploaded file + security: + - x-api-key: [] + parameters: + - in: path + name: filename + description: filename + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + type: object + properties: + keys: + type: array + items: + type: object + properties: + id: + type: string + type: + type: string + args: + type: string + index: + type: string + separator: + type: string + description: apikeys + required: true + responses: + '200': + description: filename of process + content: + application/json: + schema: + type: string + default: id + '401': + $ref: '#/components/responses/Not-authorized' + + /enrich/enriched: + get: + tags: + - job + summary: Get enriched files + operationId: get-enrich-enriched + description: Get the list of enriched files + security: + - x-api-key: [] + responses: + '200': + description: list of enriched file + content: + application/json: + schema: + type: array + items: + type: string + default: file.csv + '401': + $ref: '#/components/responses/Not-authorized' + + /enrich/enriched/${filename}: + get: + tags: + - job + summary: Get enriched file + operationId: get-enrich-enriched-$-filename + description: Get the enriched file, generated at the end of the enrichment process + security: + - x-api-key: [] + parameters: + - in: path + name: filename + description: filename + required: true + schema: + type: string + responses: + '200': + description: enriched file + content: + application/json: + schema: + type: string + format: binary + '401': + $ref: '#/components/responses/Not-authorized' + + /enrich/states: + get: + tags: + - job + summary: Get list of states of enrich job + operationId: get-enrich-states + description: Get list of filenames state of enrich process + parameters: + - in: query + name: latest + description: latest + required: false + schema: + type: boolean + responses: + '200': + description: list of states file + content: + application/json: + schema: + type: array + items: + type: string + default: file.json + '401': + $ref: '#/components/responses/Not-authorized' + + /enrich/states/${filename}: + get: + tags: + - job + summary: Get State + operationId: get-enrich-states-$-filename + description: Get state of enrich process with his filename + parameters: + - in: path + name: filename + description: filename + required: true + schema: + type: string + responses: + '200': + description: list of uploaded file + content: + application/json: + schema: + type: object + properties: + done: + type: boolean + default: true + loaded: + type: integer + default: 150 + linesRead: + type: integer + default: 3 + enrichedLines: + type: integer + default: 3 + createdAt: + type: string + default: '2021-11-30T10:00:00.000Z' + endAt: + type: string + default: '2021-11-30T10:01:00.000Z' + error: + type: boolean + default: false + '401': + $ref: '#/components/responses/Not-authorized' + '404': + description: File not found + content: + File not found: + examples: + response: + value: + message: File not found + + /enrich/uploaded: + get: + tags: + - job + summary: Get uploaded files + operationId: get-enriched-uploaded + description: Get the lists of uploaded files + security: + - x-api-key: [] + parameters: [] + responses: + '200': + description: list of uploaded file + content: + application/json: + schema: + type: array + items: + type: string + default: file.csv + '401': + $ref: '#/components/responses/Not-authorized' + +components: + responses: + Not-authorized: + description: Not authorized + headers: {} + content: + application/json: + examples: + response: + value: + message: Not authorized + Health: + description: Example response + content: + Success: + schema: + type: object + properties: + name: + type: string + status: + type: boolean + elapsedTime: + type: integer + x-examples: + Example 1: + name: redis + status: true + elapsedTime: 1 + examples: + Success: + value: + name: name of service + status: true + elapsedTime: 1 + Error redis: + value: + name: redis + elapsedTime: 3002 + error: time out + status: false + securitySchemes: + x-api-key: + name: API Key + type: apiKey + in: header diff --git a/src/frontend/components/openapi/OpenApi.vue b/src/frontend/components/openapi/OpenApi.vue index 00dee022..94aa874e 100644 --- a/src/frontend/components/openapi/OpenApi.vue +++ b/src/frontend/components/openapi/OpenApi.vue @@ -14,7 +14,7 @@ watch( props, () => { SwaggerUI({ - url: `${props.host}/openapi.json`, + url: `${props.host}/openapi.yml`, dom_id: '#swagger', deepLinking: false, }); @@ -23,7 +23,7 @@ watch( onMounted(() => { SwaggerUI({ - url: `${props.host}/openapi.json`, + url: `${props.host}/openapi.yml`, dom_id: '#swagger', deepLinking: false, }); diff --git a/src/graphql/lib/routers/openapi.js b/src/graphql/lib/routers/openapi.js index f68e1ca0..bdf85c7d 100644 --- a/src/graphql/lib/routers/openapi.js +++ b/src/graphql/lib/routers/openapi.js @@ -1,10 +1,9 @@ const router = require('express').Router(); - -const openapi = require('../../openapi.json'); +const path = require('path'); /** * Route that give the openapi.json file. */ -router.get('/openapi.json', (req, res) => res.status(200).json(openapi)); +router.get('/openapi.yml', (req, res) => res.sendFile(path.resolve(__dirname, '..', '..', 'openapi.yml'))); module.exports = router; diff --git a/src/graphql/openapi.json b/src/graphql/openapi.json deleted file mode 100644 index 84c92ffa..00000000 --- a/src/graphql/openapi.json +++ /dev/null @@ -1,306 +0,0 @@ -{ - "openapi": "3.0.0", - "info": { - "description": "The graphql service allows access to unpaywall data stored in elastic.", - "version": "1.0.0", - "title": "Graphql service", - "contact": { - "email": "ezteam@couperin.org", - "name": "ezTeam" - }, - "license": { - "name": "CeCILL 2.1", - "url": "http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html" - } - }, - "paths": { - "/graphql": { - "get": { - "summary": "Name of service", - "responses": { - "200": { - "description": "OK", - "content": { - "*/*": { - "schema": { - "type": "string", - "x-examples": { - "example-1": "graphql service" - } - }, - "examples": { - "service": { - "value": "graphql service" - } - } - }, - "Success": { - "examples": { - "response": { - "value": { - "message": "graphql service" - } - } - } - } - } - } - }, - "operationId": "get-graphql", - "description": "Get name of graphql service", - "tags": [ - "ping" - ] - }, - "parameters": [] - }, - "/graphql/ping": { - "get": { - "summary": "Ping graphql service", - "operationId": "get-graphql-ping", - "description": "ping graphql service", - "tags": [ - "ping" - ], - "responses": { - "204": { - "description": "No Content" - } - } - }, - "parameters": [] - }, - "/graphql/health": { - "get": { - "summary": "Health", - "operationId": "get-graphql-health", - "description": "Health on all service connected to graphql service", - "tags": [ - "ping" - ], - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "redis": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - }, - "elastic": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - }, - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - }, - "x-examples": { - "Example 1": { - "redis": { - "elapsedTime": 0, - "status": true - }, - "elastic": { - "elapsedTime": 1, - "status": true - }, - "elapsedTime": 1, - "status": true - } - } - }, - "examples": { - "Success": { - "value": { - "redis": { - "elapsedTime": 1, - "status": true - }, - "elastic": { - "elapsedTime": 1, - "status": true - }, - "elapsedTime": 1, - "status": true - } - }, - "Error redis": { - "value": { - "redis": { - "elapsedTime": 3001, - "status": false, - "error": "time out" - }, - "elastic": { - "elapsedTime": 1, - "status": true - }, - "elapsedTime": 3001, - "status": false - } - } - } - } - } - } - } - }, - "parameters": [] - }, - "/graphql/health/redis": { - "get": { - "summary": "Health on redis service", - "responses": { - "200": { - "$ref": "#/components/responses/Health" - } - }, - "operationId": "get-graphql-health-redis", - "description": "Health on redis", - "tags": [ - "ping" - ] - }, - "parameters": [] - }, - "/graphql/health/elastic": { - "get": { - "summary": "Health on elastic service", - "responses": { - "200": { - "$ref": "#/components/responses/Health" - } - }, - "operationId": "get-graphql-health-elastic", - "description": "Health on elastic", - "tags": [ - "ping" - ] - }, - "parameters": [] - } - }, - "externalDocs": { - "description": "Find out more about Swagger", - "url": "http://swagger.io" - }, - "tags": [ - { - "name": "ping" - } - ], - "components": { - "responses": { - "Not-authorized": { - "description": "Not authorized", - "headers": {}, - "content": { - "*/*": { - "schema": { - "type": "object", - "properties": { - "message": { - "type": "string" - } - }, - "x-examples": { - "example-1": { - "message": "Not authorized" - } - } - }, - "examples": { - "example-1": { - "value": { - "message": "string" - } - } - } - }, - "Not authorized": { - "examples": { - "response": { - "value": { - "message": "Not authorized" - } - } - } - } - } - }, - "Health": { - "description": "Example response", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "status": { - "type": "boolean" - }, - "elapsedTime": { - "type": "integer" - } - }, - "x-examples": { - "Example 1": { - "name": "redis", - "status": true, - "elapsedTime": 1 - } - } - }, - "examples": { - "Success": { - "value": { - "name": "name of service", - "status": true, - "elapsedTime": 1 - } - }, - "Error redis": { - "value": { - "name": "redis", - "elapsedTime": 3002, - "error": "time out", - "status": false - } - } - } - } - } - } - }, - "securitySchemes": { - "x-api-key": { - "type": "apiKey", - "in": "header", - "name": "API Key" - } - } - } -} \ No newline at end of file diff --git a/src/graphql/openapi.yml b/src/graphql/openapi.yml new file mode 100644 index 00000000..8818adb5 --- /dev/null +++ b/src/graphql/openapi.yml @@ -0,0 +1,189 @@ +openapi: 3.0.0 +info: + description: The graphql service allows access to unpaywall data stored in elastic. + version: 1.0.0 + title: Graphql service + contact: + email: ezteam@couperin.org + name: ezTeam + license: + name: CeCILL 2.1 + url: http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html +servers: + - url: https://unpaywall.inist.fr/api +tags: + - name: ping +paths: + /graphql: + get: + tags: + - ping + operationId: get-graphql + summary: Name of service + description: Get name of graphql service + parameters: [] + responses: + '200': + description: OK + content: + '*/*': + schema: + type: string + x-examples: + example-1: graphql service + examples: + service: + value: graphql service + Success: + examples: + response: + value: + message: graphql service + + + /graphql/ping: + get: + tags: + - ping + operationId: get-graphql-ping + summary: Ping graphql service + description: ping graphql service + parameters: [] + responses: + '204': + description: No Content + + /graphql/health: + get: + tags: + - ping + operationId: get-graphql-health + summary: Health + description: Health on all service connected to graphql service + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + redis: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + elastic: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + elapsedTime: + type: integer + status: + type: boolean + x-examples: + Example 1: + redis: + elapsedTime: 0 + status: true + elastic: + elapsedTime: 1 + status: true + elapsedTime: 1 + status: true + examples: + Success: + value: + redis: + elapsedTime: 1 + status: true + elastic: + elapsedTime: 1 + status: true + elapsedTime: 1 + status: true + Error redis: + value: + redis: + elapsedTime: 3001 + status: false + error: time out + elastic: + elapsedTime: 1 + status: true + elapsedTime: 3001 + status: false + parameters: [] + /graphql/health/redis: + get: + tags: + - ping + operationId: get-graphql-health-redis + summary: Health on redis service + description: Health on redis + parameters: [] + responses: + '200': + $ref: '#/components/responses/Health' + + + /graphql/health/elastic: + get: + tags: + - ping + operationId: get-graphql-health-elastic + summary: Health on elastic service + description: Health on elastic + parameters: [] + responses: + '200': + $ref: '#/components/responses/Health' + + + + +externalDocs: + description: Find out more about Swagger + url: http://swagger.io +components: + responses: + Health: + description: Example response + content: + application/json: + schema: + type: object + properties: + name: + type: string + status: + type: boolean + elapsedTime: + type: integer + x-examples: + Example 1: + name: redis + status: true + elapsedTime: 1 + examples: + Success: + value: + name: name of service + status: true + elapsedTime: 1 + Error redis: + value: + name: redis + elapsedTime: 3002 + error: time out + status: false + securitySchemes: + x-api-key: + type: apiKey + in: header + name: API Key diff --git a/src/health/lib/routers/openapi.js b/src/health/lib/routers/openapi.js index f68e1ca0..bdf85c7d 100644 --- a/src/health/lib/routers/openapi.js +++ b/src/health/lib/routers/openapi.js @@ -1,10 +1,9 @@ const router = require('express').Router(); - -const openapi = require('../../openapi.json'); +const path = require('path'); /** * Route that give the openapi.json file. */ -router.get('/openapi.json', (req, res) => res.status(200).json(openapi)); +router.get('/openapi.yml', (req, res) => res.sendFile(path.resolve(__dirname, '..', '..', 'openapi.yml'))); module.exports = router; diff --git a/src/health/openapi.json b/src/health/openapi.json deleted file mode 100644 index 628951a1..00000000 --- a/src/health/openapi.json +++ /dev/null @@ -1,477 +0,0 @@ -{ - "openapi": "3.0.0", - "x-stoplight": { - "id": "i72lagp6c0vth" - }, - "info": { - "description": "The Health service allows to know if the services are healthy or not", - "version": "1.0.0", - "title": "Health service", - "contact": { - "email": "ezteam@couperin.org", - "name": "ezTeam" - }, - "license": { - "name": "CeCILL 2.1", - "url": "http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html" - } - }, - "paths": { - "/health": { - "get": { - "summary": "Name of service", - "responses": { - "200": { - "description": "OK", - "content": { - "*/*": { - "schema": { - "type": "string", - "x-examples": { - "Example 1": "health service" - } - }, - "examples": { - "service": { - "value": "health service" - } - } - }, - "Success": { - "examples": { - "response": { - "value": { - "message": "graphql service" - } - } - } - } - } - } - }, - "operationId": "get-graphql", - "description": "Get name of health service", - "tags": [ - "ping" - ] - }, - "parameters": [] - }, - "/health/ping": { - "get": { - "summary": "Your GET endpoint", - "tags": [ - "ping" - ], - "responses": { - "204": { - "description": "No Content" - } - }, - "operationId": "get-health-ping", - "description": "ping health service" - } - }, - "/health/health": { - "get": { - "summary": "Your GET endpoint", - "tags": [ - "ping" - ], - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "graphql": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "services": { - "type": "object", - "properties": { - "redis": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - }, - "elastic": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - } - } - }, - "status": { - "type": "boolean" - } - } - }, - "update": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "services": { - "type": "object", - "properties": { - "redis": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - }, - "elastic": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - } - } - }, - "status": { - "type": "boolean" - } - } - }, - "enrich": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "services": { - "type": "object", - "properties": { - "redis": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - }, - "graphql": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - } - } - }, - "status": { - "type": "boolean" - } - } - }, - "apikey": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "services": { - "type": "object", - "properties": { - "redis": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - } - } - }, - "status": { - "type": "boolean" - } - } - }, - "mail": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "services": { - "type": "object", - "properties": { - "smtp": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - }, - "error": { - "type": "string" - } - } - } - } - }, - "status": { - "type": "boolean" - } - } - }, - "elastic": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - }, - "unpaywall": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - }, - "redis": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - } - }, - "x-examples": { - "Example 1": { - "graphql": { - "elapsedTime": 12, - "services": { - "redis": { - "elapsedTime": 0, - "status": true - }, - "elastic": { - "elapsedTime": 1, - "status": true - } - }, - "status": true - }, - "update": { - "elapsedTime": 11, - "services": { - "redis": { - "elapsedTime": 0, - "status": true - }, - "elastic": { - "elapsedTime": 1, - "status": true - } - }, - "status": true - }, - "enrich": { - "elapsedTime": 11, - "services": { - "redis": { - "elapsedTime": 1, - "status": true - }, - "graphql": { - "elapsedTime": 1, - "status": true - } - }, - "status": true - }, - "apikey": { - "elapsedTime": 9, - "services": { - "redis": { - "elapsedTime": 0, - "status": true - } - }, - "status": true - }, - "mail": { - "elapsedTime": 109, - "services": { - "smtp": { - "elapsedTime": 104, - "status": false, - "error": "self signed certificate" - } - }, - "status": false - }, - "elastic": { - "elapsedTime": 11, - "status": true - }, - "unpaywall": { - "elapsedTime": 6, - "status": true - }, - "redis": { - "elapsedTime": 1, - "status": true - } - } - } - }, - "examples": { - "Success": { - "value": { - "graphql": { - "elapsedTime": 12, - "services": { - "redis": { - "elapsedTime": 0, - "status": true - }, - "elastic": { - "elapsedTime": 1, - "status": true - } - }, - "status": true - }, - "update": { - "elapsedTime": 11, - "services": { - "redis": { - "elapsedTime": 0, - "status": true - }, - "elastic": { - "elapsedTime": 1, - "status": true - } - }, - "status": true - }, - "enrich": { - "elapsedTime": 11, - "services": { - "redis": { - "elapsedTime": 1, - "status": true - }, - "graphql": { - "elapsedTime": 1, - "status": true - } - }, - "status": true - }, - "apikey": { - "elapsedTime": 9, - "services": { - "redis": { - "elapsedTime": 0, - "status": true - } - }, - "status": true - }, - "mail": { - "elapsedTime": 109, - "services": { - "smtp": { - "elapsedTime": 104, - "status": false, - "error": "self signed certificate" - } - }, - "status": false - }, - "elastic": { - "elapsedTime": 11, - "status": true - }, - "unpaywall": { - "elapsedTime": 6, - "status": true - }, - "redis": { - "elapsedTime": 1, - "status": true - } - } - } - } - } - } - } - }, - "operationId": "get-health-health", - "description": "Health on all service relied to ezunpaywall" - } - } - }, - "externalDocs": { - "description": "Find out more about Swagger", - "url": "http://swagger.io" - }, - "tags": [ - { - "name": "ping" - } - ], - "components": { - "responses": {}, - "securitySchemes": {} - } -} \ No newline at end of file diff --git a/src/health/openapi.yml b/src/health/openapi.yml new file mode 100644 index 00000000..b4b60df6 --- /dev/null +++ b/src/health/openapi.yml @@ -0,0 +1,316 @@ +openapi: 3.0.0 +x-stoplight: + id: i72lagp6c0vth +info: + description: The Health service allows to know if the services are healthy or not + version: 1.0.0 + title: Health service + contact: + email: ezteam@couperin.org + name: ezTeam + license: + name: CeCILL 2.1 + url: http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html +servers: + - url: https://unpaywall.inist.fr/api +tags: + - name: ping +paths: + /health: + get: + tags: + - ping + operationId: get-graphql + summary: Name of service + description: Get name of health service + parameters: [] + responses: + '200': + description: OK + content: + '*/*': + schema: + type: string + x-examples: + Example 1: health service + examples: + service: + value: health service + Success: + examples: + response: + value: + message: graphql service + + /health/ping: + get: + tags: + - ping + operationId: get-health-ping + summary: Your GET endpoint + description: ping health service + responses: + '204': + description: No Content + + /health/health: + get: + tags: + - ping + operationId: get-health-health + summary: Your GET endpoint + description: Health on all service relied to ezunpaywall + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + graphql: + type: object + properties: + elapsedTime: + type: integer + services: + type: object + properties: + redis: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + elastic: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + status: + type: boolean + update: + type: object + properties: + elapsedTime: + type: integer + services: + type: object + properties: + redis: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + elastic: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + status: + type: boolean + enrich: + type: object + properties: + elapsedTime: + type: integer + services: + type: object + properties: + redis: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + graphql: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + status: + type: boolean + apikey: + type: object + properties: + elapsedTime: + type: integer + services: + type: object + properties: + redis: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + status: + type: boolean + mail: + type: object + properties: + elapsedTime: + type: integer + services: + type: object + properties: + smtp: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + error: + type: string + status: + type: boolean + elastic: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + unpaywall: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + redis: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + x-examples: + Example 1: + graphql: + elapsedTime: 12 + services: + redis: + elapsedTime: 0 + status: true + elastic: + elapsedTime: 1 + status: true + status: true + update: + elapsedTime: 11 + services: + redis: + elapsedTime: 0 + status: true + elastic: + elapsedTime: 1 + status: true + status: true + enrich: + elapsedTime: 11 + services: + redis: + elapsedTime: 1 + status: true + graphql: + elapsedTime: 1 + status: true + status: true + apikey: + elapsedTime: 9 + services: + redis: + elapsedTime: 0 + status: true + status: true + mail: + elapsedTime: 109 + services: + smtp: + elapsedTime: 104 + status: false + error: self signed certificate + status: false + elastic: + elapsedTime: 11 + status: true + unpaywall: + elapsedTime: 6 + status: true + redis: + elapsedTime: 1 + status: true + examples: + Success: + value: + graphql: + elapsedTime: 12 + services: + redis: + elapsedTime: 0 + status: true + elastic: + elapsedTime: 1 + status: true + status: true + update: + elapsedTime: 11 + services: + redis: + elapsedTime: 0 + status: true + elastic: + elapsedTime: 1 + status: true + status: true + enrich: + elapsedTime: 11 + services: + redis: + elapsedTime: 1 + status: true + graphql: + elapsedTime: 1 + status: true + status: true + apikey: + elapsedTime: 9 + services: + redis: + elapsedTime: 0 + status: true + status: true + mail: + elapsedTime: 109 + services: + smtp: + elapsedTime: 104 + status: false + error: self signed certificate + status: false + elastic: + elapsedTime: 11 + status: true + unpaywall: + elapsedTime: 6 + status: true + redis: + elapsedTime: 1 + status: true + +components: + responses: {} + securitySchemes: {} diff --git a/src/mail/lib/routers/openapi.js b/src/mail/lib/routers/openapi.js index f68e1ca0..bdf85c7d 100644 --- a/src/mail/lib/routers/openapi.js +++ b/src/mail/lib/routers/openapi.js @@ -1,10 +1,9 @@ const router = require('express').Router(); - -const openapi = require('../../openapi.json'); +const path = require('path'); /** * Route that give the openapi.json file. */ -router.get('/openapi.json', (req, res) => res.status(200).json(openapi)); +router.get('/openapi.yml', (req, res) => res.sendFile(path.resolve(__dirname, '..', '..', 'openapi.yml'))); module.exports = router; diff --git a/src/mail/openapi.json b/src/mail/openapi.json deleted file mode 100644 index d82125fd..00000000 --- a/src/mail/openapi.json +++ /dev/null @@ -1,421 +0,0 @@ -{ - "openapi": "3.0.0", - "info": { - "description": "The mail service allows the sending of mail for the different services", - "version": "1.0.0", - "title": "Mail service", - "contact": { - "email": "ezteam@couperin.org", - "name": "ezTeam" - }, - "license": { - "name": "CeCILL 2.1", - "url": "http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html" - } - }, - "paths": { - "/mail": { - "get": { - "summary": "Name of service", - "responses": { - "200": { - "description": "OK", - "content": { - "*/*": { - "schema": { - "type": "string", - "x-examples": { - "example-1": "mail service" - } - }, - "examples": { - "service": { - "value": "mail service" - } - } - }, - "Success": { - "examples": { - "response": { - "value": { - "message": "mail service" - } - } - } - } - } - } - }, - "operationId": "get-mail", - "description": "Get name of mail service", - "tags": [ - "ping" - ] - } - }, - "/mail/ping": { - "get": { - "summary": "Ping mail service", - "operationId": "get-mail-ping", - "description": "Ping mail service", - "tags": [ - "ping" - ], - "responses": { - "204": { - "description": "No Content" - } - } - } - }, - "/mail/health": { - "get": { - "summary": "Health", - "operationId": "get-mail-health", - "description": "Health on all service connected to enrich service", - "tags": [ - "ping" - ], - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "smtp": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - } - }, - "x-examples": { - "Example 1": { - "smtp": { - "elapsedTime": 1, - "status": true - } - } - } - }, - "examples": { - "Success": { - "value": { - "smtp": { - "elapsedTime": 1, - "status": true - } - } - }, - "Error smtp": { - "value": { - "smtp": { - "elapsedTime": 133, - "status": false, - "error": "self signed certificate" - } - } - } - } - } - } - } - } - }, - "parameters": [] - }, - "/mail/health/smtp": { - "get": { - "summary": "Health on SMTP service", - "responses": { - "200": { - "$ref": "#/components/responses/Health" - } - }, - "operationId": "get-mail-health-smtp", - "description": "Health on smtp", - "tags": [ - "ping" - ] - }, - "parameters": [] - }, - "/mail/contact": { - "post": { - "summary": "Send mail contact", - "operationId": "post-mail-contact", - "responses": { - "202": { - "description": "Accepted" - }, - "400": { - "description": "Bad Request", - "content": { - "*/*": { - "schema": { - "type": "object", - "properties": {} - } - }, - "email expected": { - "examples": { - "response": { - "value": { - "message": "email are expected" - } - } - } - }, - "invalid email": { - "examples": { - "response": { - "value": { - "message": "[\"john\"] is invalid email" - } - } - } - }, - "subject expected": { - "examples": { - "response": { - "value": { - "message": "subject are expected" - } - } - } - }, - "message expected": { - "examples": { - "response": { - "value": { - "message": "message are expected" - } - } - } - } - } - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - }, - "403": { - "description": "Forbidden", - "content": { - "*/*": { - "schema": { - "type": "object", - "properties": {} - } - } - } - } - }, - "description": "Send contact mail", - "security": [ - { - "x-api-key": [] - } - ], - "tags": [ - "mail" - ] - } - }, - "/mail/update-start": { - "post": { - "summary": "Send mail update process start", - "operationId": "post-mail-update-start", - "responses": { - "202": { - "description": "Accepted" - }, - "400": { - "description": "Bad Request", - "content": { - "*/*": { - "schema": { - "type": "object", - "properties": {} - } - } - } - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - }, - "403": { - "description": "Forbidden", - "content": { - "*/*": { - "schema": { - "type": "object", - "properties": {} - } - } - } - } - }, - "description": "Sending of the email informing the launch of the update process", - "security": [ - { - "x-api-key": [] - } - ], - "tags": [ - "mail" - ] - }, - "parameters": [] - }, - "/mail/update-end": { - "post": { - "summary": "Send mail update process ended", - "operationId": "post-mail-update-end", - "responses": { - "202": { - "description": "Accepted" - }, - "400": { - "description": "Bad Request", - "content": { - "*/*": { - "schema": { - "type": "object", - "properties": {} - } - } - } - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - }, - "403": { - "description": "Forbidden", - "content": { - "*/*": { - "schema": { - "type": "object", - "properties": {} - } - } - } - } - }, - "description": "Sending of the email with the report of the end of the update", - "security": [ - { - "x-api-key": [] - } - ], - "tags": [ - "mail" - ] - }, - "parameters": [] - } - }, - "externalDocs": { - "description": "Find out more about Swagger", - "url": "http://swagger.io" - }, - "tags": [ - { - "name": "mail" - }, - { - "name": "ping" - } - ], - "components": { - "responses": { - "Not-authorized": { - "description": "Not authorized", - "headers": {}, - "content": { - "*/*": { - "schema": { - "type": "object", - "properties": { - "message": { - "type": "string" - } - }, - "x-examples": { - "example-1": { - "message": "Not authorized" - } - } - } - }, - "Not authorized": { - "examples": { - "response": { - "value": { - "message": "Not authorized" - } - } - } - } - } - }, - "Health": { - "description": "Example response", - "content": { - "Success": { - "schema": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "status": { - "type": "boolean" - }, - "elapsedTime": { - "type": "integer" - } - }, - "x-examples": { - "Example 1": { - "name": "redis", - "status": true, - "elapsedTime": 1 - } - } - }, - "examples": { - "Success": { - "value": { - "name": "name of service", - "status": true, - "elapsedTime": 1 - } - }, - "Error smtp": { - "value": { - "elapsedTime": 115, - "status": false, - "error": "self signed certificate" - } - } - } - } - } - } - }, - "securitySchemes": { - "x-api-key": { - "name": "API Key", - "type": "apiKey", - "in": "header" - } - } - } -} \ No newline at end of file diff --git a/src/mail/openapi.yml b/src/mail/openapi.yml new file mode 100644 index 00000000..53e1645c --- /dev/null +++ b/src/mail/openapi.yml @@ -0,0 +1,261 @@ +openapi: 3.0.0 +info: + description: The mail service allows the sending of mail for the different services + version: 1.0.0 + title: Mail service + contact: + email: ezteam@couperin.org + name: ezTeam + license: + name: CeCILL 2.1 + url: http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html +servers: + - url: https://unpaywall.inist.fr/api +tags: + - name: mail + - name: ping +paths: + /mail: + get: + tags: + - ping + operationId: get-mail + summary: Name of service + description: Get name of mail service + responses: + '200': + description: OK + content: + '*/*': + schema: + type: string + x-examples: + example-1: mail service + examples: + service: + value: mail service + Success: + examples: + response: + value: + message: mail service + + /mail/ping: + get: + tags: + - ping + operationId: get-mail-ping + summary: Ping mail service + description: Ping mail service + responses: + '204': + description: No Content + /mail/health: + get: + tags: + - ping + operationId: get-mail-health + summary: Health + description: Health on all service connected to enrich service + parameters: [] + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + smtp: + type: object + properties: + elapsedTime: + type: integer + status: + type: boolean + x-examples: + Example 1: + smtp: + elapsedTime: 1 + status: true + examples: + Success: + value: + smtp: + elapsedTime: 1 + status: true + Error smtp: + value: + smtp: + elapsedTime: 133 + status: false + error: self signed certificate + + /mail/health/smtp: + get: + tags: + - ping + operationId: get-mail-health-smtp + summary: Health on SMTP service + description: Health on smtp + parameters: [] + responses: + '200': + $ref: '#/components/responses/Health' + + /mail/contact: + post: + tags: + - mail + operationId: post-mail-contact + summary: Send mail contact + description: Send contact mail + security: + - x-api-key: [] + responses: + '202': + description: Accepted + '400': + description: Bad Request + content: + '*/*': + schema: + type: object + properties: {} + email expected: + examples: + response: + value: + message: email are expected + invalid email: + examples: + response: + value: + message: '["john"] is invalid email' + subject expected: + examples: + response: + value: + message: subject are expected + message expected: + examples: + response: + value: + message: message are expected + '401': + $ref: '#/components/responses/Not-authorized' + '403': + description: Forbidden + content: + '*/*': + schema: + type: object + properties: {} + + + /mail/update-start: + post: + tags: + - mail + operationId: post-mail-update-start + summary: Send mail update process start + description: Sending of the email informing the launch of the update process + security: + - x-api-key: [] + parameters: [] + responses: + '202': + description: Accepted + '400': + description: Bad Request + content: + '*/*': + schema: + type: object + properties: {} + '401': + $ref: '#/components/responses/Not-authorized' + '403': + description: Forbidden + content: + '*/*': + schema: + type: object + properties: {} + + /mail/update-end: + post: + tags: + - mail + operationId: post-mail-update-end + summary: Send mail update process ended + description: Sending of the email with the report of the end of the update + security: + - x-api-key: [] + parameters: [] + responses: + '202': + description: Accepted + '400': + description: Bad Request + content: + '*/*': + schema: + type: object + properties: {} + '401': + $ref: '#/components/responses/Not-authorized' + '403': + description: Forbidden + content: + '*/*': + schema: + type: object + properties: {} + + +components: + responses: + Not-authorized: + description: Not authorized + headers: {} + content: + application/json: + examples: + response: + value: + message: Not authorized + Health: + description: Example response + content: + Success: + schema: + type: object + properties: + name: + type: string + status: + type: boolean + elapsedTime: + type: integer + x-examples: + Example 1: + name: redis + status: true + elapsedTime: 1 + examples: + Success: + value: + name: name of service + status: true + elapsedTime: 1 + Error smtp: + value: + elapsedTime: 115 + status: false + error: self signed certificate + securitySchemes: + x-api-key: + name: API Key + type: apiKey + in: header diff --git a/src/update/lib/routers/openapi.js b/src/update/lib/routers/openapi.js index f68e1ca0..bdf85c7d 100644 --- a/src/update/lib/routers/openapi.js +++ b/src/update/lib/routers/openapi.js @@ -1,10 +1,9 @@ const router = require('express').Router(); - -const openapi = require('../../openapi.json'); +const path = require('path'); /** * Route that give the openapi.json file. */ -router.get('/openapi.json', (req, res) => res.status(200).json(openapi)); +router.get('/openapi.yml', (req, res) => res.sendFile(path.resolve(__dirname, '..', '..', 'openapi.yml'))); module.exports = router; diff --git a/src/update/openapi.json b/src/update/openapi.json deleted file mode 100644 index 334010a7..00000000 --- a/src/update/openapi.json +++ /dev/null @@ -1,1374 +0,0 @@ -{ - "openapi": "3.0.0", - "info": { - "description": "The update service allows to update the mirror data. This service queries unpaywall to get the snapshots and inserts their content in an elasticsearch index.", - "version": "1.0.0", - "title": "Update service", - "contact": { - "email": "ezteam@couperin.org", - "name": "ezTeam" - }, - "license": { - "name": "CeCILL 2.1", - "url": "http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html" - } - }, - "servers": [ - { - "url": "https://unpaywall.inist.fr/api" - } - ], - "tags": [ - { - "name": "cron" - }, - { - "name": "job" - }, - { - "name": "ping" - } - ], - "paths": { - "/update": { - "get": { - "tags": [ - "ping" - ], - "operationId": "get-update", - "summary": "Name of service.", - "description": "Get name of update service.", - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "examples": { - "response": { - "value": { - "message": "update service" - } - } - } - } - } - } - } - } - }, - "/update/ping": { - "get": { - "operationId": "get-update-ping", - "summary": "Ping update service.", - "description": "Ping update service.", - "tags": [ - "ping" - ], - "responses": { - "204": { - "description": "No Content" - } - } - } - }, - "/update/health": { - "get": { - "tags": [ - "ping" - ], - "operationId": "get-update-health", - "summary": "Health.", - "description": "Health on all service connected to update service.", - "parameters": [], - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "redis": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - }, - "elastic": { - "type": "object", - "properties": { - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - } - }, - "elapsedTime": { - "type": "integer" - }, - "status": { - "type": "boolean" - } - }, - "x-examples": { - "Example 1": { - "redis": { - "elapsedTime": 1, - "status": true - }, - "elastic": { - "elapsedTime": 1, - "status": true - }, - "elapsedTime": 1, - "status": true - } - } - }, - "examples": { - "Success": { - "value": { - "redis": { - "elapsedTime": 1, - "status": true - }, - "elastic": { - "elapsedTime": 1, - "status": true - }, - "elapsedTime": 1, - "status": true - } - }, - "Error redis": { - "value": { - "redis": { - "elapsedTime": 3001, - "status": false, - "error": "time out" - }, - "elastic": { - "elapsedTime": 1, - "status": true - }, - "elapsedTime": 3001, - "status": false - } - } - } - } - } - } - } - } - }, - "/update/health/redis": { - "get": { - "tags": [ - "ping" - ], - "operationId": "get-update-health-redis", - "summary": "Health on redis service.", - "description": "Health on redis.", - "parameters": [], - "responses": { - "200": { - "$ref": "#/components/responses/Health" - } - } - } - }, - "/update/health/elastic": { - "get": { - "tags": [ - "ping" - ], - "operationId": "get-update-health-elastic", - "summary": "Health on elastic service.", - "description": "Health on elastic.", - "parameters": [], - "responses": { - "200": { - "$ref": "#/components/responses/Health" - } - } - } - }, - "/update/job/snapshot": { - "post": { - "tags": [ - "job" - ], - "operationId": "post-update-job-snapshot", - "summary": "Start process download and insert snapshot from unpaywall.", - "description": "Download and insert the latest snapshot from unpaywall.org.", - "security": [ - { - "x-api-key": [] - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "object", - "required": [ - "userName" - ], - "properties": { - "interval": { - "type": "string" - } - } - } - } - }, - "description": "index where the unpaywall data will be inserted." - }, - "responses": { - "202": { - "$ref": "#/components/responses/Accepted" - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - }, - "409": { - "$ref": "#/components/responses/Conflict" - } - } - } - }, - "/update/job/period": { - "post": { - "tags": [ - "job" - ], - "operationId": "post-update-job-period", - "summary": "Start process download and insert changefiles from unpaywall.", - "description": "Download and insert changefiles from unpaywall.org.", - "security": [ - { - "x-api-key": [] - } - ], - "requestBody": { - "description": "config for process", - "content": { - "application/json": { - "schema": { - "type": "object", - "required": [ - "userName" - ], - "properties": { - "index": { - "type": "string" - }, - "interval": { - "type": "string" - }, - "startDate": { - "type": "string" - }, - "endDate": { - "type": "string" - } - } - } - } - } - }, - "responses": { - "202": { - "$ref": "#/components/responses/Accepted" - }, - "400": { - "description": "startDate cannot be in the futur or endDate cannot be lower than startDate." - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - }, - "409": { - "$ref": "#/components/responses/Conflict" - } - } - } - }, - "/update/job/changefile/{filename}": { - "post": { - "tags": [ - "job" - ], - "operationId": "post-update-job-changefile-$-filename", - "summary": "Start process insert changesfiles downloaded on ezunpaywall.", - "description": "Insert changefiles already downloaded on ezunpaywall.", - "security": [ - { - "x-api-key": [] - } - ], - "parameters": [ - { - "in": "path", - "name": "filename", - "description": "filename", - "required": true, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "index": { - "type": "string" - }, - "offset": { - "type": "integer" - }, - "limit": { - "type": "integer" - } - } - } - } - }, - "description": "config for process" - }, - "responses": { - "202": { - "$ref": "#/components/responses/Accepted" - }, - "400": { - "description": "startDate cannot be in the futur or endDate cannot be lower than startDate.", - "content": { - "application/json": { - "examples": { - "response": { - "value": { - "message": "startDate cannot be in the futur or endDate cannot be lower than startDate." - } - } - } - } - } - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - }, - "404": { - "$ref": "#/components/responses/File-not-found" - }, - "409": { - "$ref": "#/components/responses/Conflict" - } - } - } - }, - "/update/job/history": { - "post": { - "tags": [ - "job" - ], - "operationId": "post-update-job-history", - "summary": "Start process download and insert changefiles from unpaywall and feed history.", - "description": "Download and insert changefiles from unpaywall.org and feed history.", - "security": [ - { - "x-api-key": [] - } - ], - "requestBody": { - "description": "config for process", - "content": { - "application/json": { - "schema": { - "type": "object", - "required": [ - "userName" - ], - "properties": { - "index": { - "type": "string" - }, - "interval": { - "type": "string" - }, - "startDate": { - "type": "string" - }, - "endDate": { - "type": "string" - } - } - } - } - } - }, - "responses": { - "202": { - "$ref": "#/components/responses/Accepted" - }, - "400": { - "description": "startDate cannot be in the futur or endDate cannot be lower than startDate." - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - }, - "409": { - "$ref": "#/components/responses/Conflict" - } - } - } - }, - "/update/job/reset": { - "post": { - "tags": [ - "job" - ], - "operationId": "post-update-job-reset", - "summary": "Start process that roll back the current and the history index according to a date.", - "description": "roll back the current and the history index according to a date.", - "security": [ - { - "x-api-key": [] - } - ], - "requestBody": { - "description": "config for process", - "content": { - "application/json": { - "schema": { - "type": "object", - "required": [ - "userName" - ], - "properties": { - "index": { - "type": "string" - }, - "startDate": { - "type": "string" - } - } - } - } - } - }, - "responses": { - "202": { - "$ref": "#/components/responses/Accepted" - }, - "400": { - "description": "startDate required" - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - }, - "409": { - "$ref": "#/components/responses/Conflict" - } - } - } - }, - "/update/reports/${type}": { - "get": { - "tags": [ - "job" - ], - "operationId": "get-update-reports", - "summary": "Get reports.", - "description": "Get list of reports or latest report.", - "parameters": [ - { - "in": "path", - "name": "type", - "description": "type of report (unpaywall or unpaywallHistory)", - "required": true, - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "latest", - "description": "latest", - "required": false, - "schema": { - "type": "boolean" - } - } - ], - "responses": { - "200": { - "description": "report", - "content": { - "List of reports": { - "examples": { - "response": { - "value": [ - "2022-10-10 13:10:51.json", - "2022-10-10 13:10:50.json", - "2022-10-10 13:10:49.json", - "2022-10-10 13:10:48.json", - "2022-10-10 13:10:47.json", - "2022-10-10 13:10:46.json", - "2022-10-10 13:10:45.json", - "2022-10-10 13:10:44.json" - ] - } - } - }, - "Latest report": { - "examples": { - "response": { - "value": { - "done": true, - "createdAt": "2022-10-10T13:10:46.828Z", - "endAt": "2022-10-10T13:10:47.084Z", - "steps": [ - { - "task": "getChangefiles", - "took": 0.002, - "status": "success" - }, - { - "task": "download", - "file": "fake2.jsonl.gz", - "percent": 100, - "took": 0, - "status": "success" - }, - { - "task": "insert", - "index": "unpaywall-test", - "file": "fake2.jsonl.gz", - "linesRead": 100, - "insertedDocs": 100, - "updatedDocs": 0, - "failedDocs": 0, - "percent": 100, - "took": 0.202, - "status": "success" - } - ], - "error": false, - "took": 0.256 - } - } - } - } - } - } - } - } - }, - "/update/reports/${type}/${filename}": { - "get": { - "tags": [ - "job" - ], - "operationId": "get-update-reports-$-filename", - "summary": "Get report.", - "description": "Get report with his filename.", - "parameters": [ - { - "in": "path", - "name": "type", - "description": "type of report (unpaywall or unpaywallHistory)", - "required": true, - "schema": { - "type": "string" - } - }, - { - "in": "path", - "name": "filename", - "description": "filename", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "report", - "content": { - "application/json": { - "schema": { - "type": "object", - "x-examples": { - "example-1": { - "done": true, - "createdAt": "2022-10-10T13:10:46.828Z", - "endAt": "2022-10-10T13:10:47.084Z", - "steps": [ - { - "task": "getChangefiles", - "took": 0.002, - "status": "success" - }, - { - "task": "download", - "file": "fake2.jsonl.gz", - "percent": 100, - "took": 0, - "status": "success" - }, - { - "task": "insert", - "index": "unpaywall-test", - "file": "fake2.jsonl.gz", - "linesRead": 100, - "insertedDocs": 100, - "updatedDocs": 0, - "failedDocs": 0, - "percent": 100, - "took": 0.202, - "status": "success" - } - ], - "error": false, - "took": 0.256 - } - } - } - }, - "Report": { - "examples": { - "response": { - "value": { - "done": true, - "createdAt": "2022-10-10T13:10:46.828Z", - "endAt": "2022-10-10T13:10:47.084Z", - "steps": [ - { - "task": "getChangefiles", - "took": 0.002, - "status": "success" - }, - { - "task": "download", - "file": "fake2.jsonl.gz", - "percent": 100, - "took": 0, - "status": "success" - }, - { - "task": "insert", - "index": "unpaywall-test", - "file": "fake2.jsonl.gz", - "linesRead": 100, - "insertedDocs": 100, - "updatedDocs": 0, - "failedDocs": 0, - "percent": 100, - "took": 0.202, - "status": "success" - } - ], - "error": false, - "took": 0.256 - } - } - } - } - } - }, - "404": { - "$ref": "#/components/responses/File-not-found" - } - } - } - }, - "/update/snapshots/${type}": { - "get": { - "tags": [ - "job" - ], - "operationId": "get-update-snapshots", - "summary": "Get snapshots.", - "description": "Get list of snapshot or latest snapshot.", - "security": [ - { - "x-api-key": [] - } - ], - "parameters": [ - { - "in": "path", - "name": "type", - "description": "type of report (unpaywall or unpaywallHistory)", - "required": true, - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "latest", - "description": "latest", - "required": false, - "schema": { - "type": "boolean" - } - } - ], - "responses": { - "200": { - "description": "snapshot", - "content": { - "Lists of snapshots": { - "examples": { - "response": { - "value": [ - "snapshot-2022-10-10.jsonl.gz" - ] - } - } - } - } - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - } - } - } - }, - "/update/snapshots/${type}/${filename}": { - "get": { - "tags": [ - "job" - ], - "operationId": "get-update-snapshots-$-filename", - "summary": "Get snapshot.", - "description": "Get snapshot with his filename.", - "security": [ - { - "x-api-key": [] - } - ], - "parameters": [ - { - "in": "path", - "name": "type", - "description": "type of report (unpaywall or unpaywallHistory)", - "required": true, - "schema": { - "type": "string" - } - }, - { - "in": "path", - "name": "filename", - "description": "filename", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "snapshot" - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - }, - "404": { - "$ref": "#/components/responses/File-not-found" - } - } - } - }, - "/update/states": { - "get": { - "tags": [ - "job" - ], - "operationId": "get-update-states", - "summary": "Get state.", - "description": "Get latest state.", - "responses": { - "200": { - "description": "state", - "content": { - "Latest state": { - "examples": { - "response": { - "value": { - "done": true, - "createdAt": "2022-10-10T13:10:51.417Z", - "endAt": "2022-10-10T13:10:51.584Z", - "steps": [ - { - "task": "getChangefiles", - "took": 0.002, - "status": "success" - }, - { - "task": "insert", - "index": "unpaywall-test", - "file": "fake1.jsonl.gz", - "linesRead": 50, - "insertedDocs": 50, - "updatedDocs": 0, - "failedDocs": 0, - "percent": 100, - "took": 0.165, - "status": "success" - } - ], - "error": false, - "took": 0.167 - } - } - } - } - } - } - } - } - }, - "/update/status": { - "get": { - "tags": [ - "job" - ], - "summary": "Get status", - "operationId": "get-update-status", - "description": "Get status", - "responses": { - "200": { - "description": "status", - "content": { - "No process in progress": { - "examples": { - "response": { - "value": false - } - } - }, - "Update in progress": { - "examples": { - "response": { - "value": true - } - } - } - } - } - } - } - }, - "/update/unpaywall/changefiles": { - "get": { - "tags": [ - "job" - ], - "operationId": "get-update-unpaywall-changefiles", - "summary": "Get changefiles from unpaywall.", - "description": "Get changefiles from unpaywall.", - "security": [ - { - "x-api-key": [] - } - ], - "parameters": [ - { - "in": "query", - "name": "interval", - "description": "interval of changefiles (\"week\" or \"day\")", - "required": false, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "status", - "content": { - "List of daily changefiles of unpaywall": { - "examples": { - "response": { - "value": [ - { - "date": "2022-10-10T00:00:00.000Z", - "filename": "fake1.jsonl.gz", - "filetype": "jsonl", - "last_modified": "2021-10-09T13:10:50.000Z", - "lines": 50, - "size": 19896, - "url": "http://fakeUnpaywall:3000/daily-feed/changefiles/fake1.jsonl.gz" - }, - { - "date": "2022-10-09T00:00:00.000Z", - "filename": "fake2.jsonl.gz", - "filetype": "jsonl", - "last_modified": "2022-10-09T13:10:50.000Z", - "lines": 100, - "size": 44137, - "url": "http://fakeUnpaywall:3000/daily-feed/changefiles/fake2.jsonl.gz" - }, - { - "date": "2022-10-08T00:00:00.000Z", - "filename": "fake3.jsonl.gz", - "filetype": "jsonl", - "last_modified": "2022-10-08T13:10:50.000Z", - "lines": 2000, - "size": 877494, - "url": "http://fakeUnpaywall:3000/daily-feed/changefiles/fake3.jsonl.gz" - } - ] - } - } - }, - "List of weekly changefiles of unpaywall": { - "examples": { - "response": { - "value": [ - { - "date": "2022-10-10T00:00:00.000Z", - "filename": "fake1.jsonl.gz", - "filetype": "jsonl", - "last_modified": "2021-10-09T13:10:50.000Z", - "lines": 50, - "size": 19896, - "url": "http://fakeUnpaywall:3000/feed/changefiles/fake1.jsonl.gz" - }, - { - "date": "2022-10-09T00:00:00.000Z", - "filename": "fake2.jsonl.gz", - "filetype": "jsonl", - "last_modified": "2022-10-09T13:10:50.000Z", - "lines": 100, - "size": 44137, - "url": "http://fakeUnpaywall:3000/feed/changefiles/fake2.jsonl.gz" - }, - { - "date": "2022-10-08T00:00:00.000Z", - "filename": "fake3.jsonl.gz", - "filetype": "jsonl", - "last_modified": "2022-10-08T13:10:50.000Z", - "lines": 2000, - "size": 877494, - "url": "http://fakeUnpaywall:3000/feed/changefiles/fake3.jsonl.gz" - } - ] - } - } - } - } - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - } - } - } - }, - "/update/cron/${type}/start": { - "post": { - "tags": [ - "cron" - ], - "operationId": "post-cron-start", - "summary": "Start cron.", - "description": "Start update cron.", - "security": [ - { - "x-api-key": [] - } - ], - "parameters": [ - { - "in": "path", - "name": "type", - "description": "type of report (unpaywall or unpaywallHistory)", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "202": { - "$ref": "#/components/responses/Accepted" - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - } - } - } - }, - "/update/cron/${type}/stop": { - "post": { - "tags": [ - "cron" - ], - "operationId": "post-cron-stop", - "description": "Stop cron.", - "summary": "Stop cron.", - "security": [ - { - "x-api-key": [] - } - ], - "parameters": [ - { - "in": "path", - "name": "type", - "description": "type of report (unpaywall or unpaywallHistory)", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "202": { - "$ref": "#/components/responses/Accepted" - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - } - } - } - }, - "/update/cron/${type}": { - "patch": { - "tags": [ - "cron" - ], - "operationId": "patch-cron", - "summary": "update cron.", - "description": "update cron.", - "security": [ - { - "x-api-key": [] - } - ], - "parameters": [ - { - "in": "path", - "name": "type", - "description": "type of report (unpaywall or unpaywallHistory)", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "config": { - "type": "object", - "properties": { - "time": { - "type": "string" - }, - "status": { - "type": "boolean" - }, - "index": { - "type": "string" - }, - "interval": { - "type": "string" - } - } - } - }, - "x-examples": { - "example-1": { - "config": { - "time": "* * * * * *", - "status": false, - "index": "unpaywall", - "interval": "day" - } - } - } - }, - "examples": { - "updated": { - "value": { - "config": { - "time": "* 30 12-15 * * *", - "status": true, - "index": "unpaywall", - "interval": "day" - } - } - } - } - } - } - }, - "401": { - "$ref": "#/components/responses/Not-authorized" - } - }, - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "time": { - "type": "string" - }, - "index": { - "type": "string" - }, - "interval": { - "type": "string" - } - }, - "x-examples": { - "example-1": { - "time": "", - "index": "", - "interval": "" - } - } - }, - "examples": { - "daily": { - "value": { - "time": "0 0 0 * * *", - "index": "unpaywall", - "interval": "day" - } - }, - "weekly": { - "value": { - "time": "0 0 0 * * 0", - "index": "unpaywall", - "interval": "weekly" - } - } - } - } - } - } - }, - "get": { - "tags": [ - "cron" - ], - "operationId": "get-cron", - "summary": "Get config of cron.", - "description": "Get config of cron.", - "parameters": [ - { - "in": "path", - "name": "type", - "description": "type of report (unpaywall or unpaywallHistory)", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "config": { - "type": "object", - "properties": { - "time": { - "type": "string" - }, - "status": { - "type": "boolean" - }, - "index": { - "type": "string" - }, - "interval": { - "type": "string" - } - } - } - }, - "x-examples": { - "example-1": { - "config": { - "time": "* * * * * *", - "status": false, - "index": "unpaywall", - "interval": "day" - } - } - } - }, - "examples": { - "day": { - "value": { - "config": { - "time": "0 0 0 * * *", - "status": true, - "index": "unpaywall", - "interval": "day" - } - } - }, - "week": { - "value": { - "config": { - "time": "0 0 0 * * *", - "status": true, - "index": "unpaywall", - "interval": "day" - } - } - }, - "off": { - "value": { - "config": { - "time": "0 0 0 * * *", - "status": false, - "index": "unpaywall", - "interval": "day" - } - } - } - } - } - } - } - } - } - } - }, - "components": { - "responses": { - "Accepted": { - "description": "Accepted", - "headers": {} - }, - "Not-authorized": { - "description": "Not authorized", - "headers": {}, - "content": { - "application/json": { - "examples": { - "response": { - "value": { - "message": "Not authorized" - } - } - } - } - } - }, - "Conflict": { - "description": "Conflict", - "headers": {}, - "content": { - "application/json": { - "examples": { - "response": { - "value": { - "message": "Update in progress." - } - } - } - } - } - }, - "File-not-found": { - "description": "File not found", - "headers": {}, - "content": { - "application/json": { - "examples": { - "response": { - "value": { - "message": "File not found." - } - } - } - } - } - }, - "Health": { - "description": "Example response", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "status": { - "type": "boolean" - }, - "elapsedTime": { - "type": "integer" - } - }, - "x-examples": { - "Example 1": { - "name": "redis", - "status": true, - "elapsedTime": 1 - } - } - }, - "examples": { - "Success": { - "value": { - "name": "name of service", - "status": true, - "elapsedTime": 1 - } - }, - "Error redis": { - "value": { - "name": "redis", - "elapsedTime": 3002, - "error": "time out", - "status": false - } - } - } - } - } - } - }, - "securitySchemes": { - "x-api-key": { - "type": "apiKey", - "in": "header", - "name": "API Key" - } - }, - "schemas": {} - } -} \ No newline at end of file From d1a0fc9709746d42a586a8bca25b7c5eb86bc315 Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Thu, 11 Jan 2024 16:09:24 +0100 Subject: [PATCH 016/114] feat: update frontend to add new type of job --- .../administration/update/CronButton.vue | 13 ++- .../administration/update/CronDialog.vue | 86 +++++++++++++------ .../update/UpdateProcessButton.vue | 13 ++- .../update/UpdateProcessDialog.vue | 6 +- .../components/report/RefreshButton.vue | 6 +- .../components/report/ReportsCard.vue | 28 ++++-- .../components/report/ReportsDataTable.vue | 6 +- .../components/report/UnpaywallButton.vue | 2 +- src/frontend/locales/en.json | 12 ++- src/frontend/locales/fr.json | 12 ++- src/frontend/package.json | 2 +- src/frontend/pages/administration.vue | 8 +- src/frontend/pages/report-history.vue | 7 +- src/update/lib/insert.js | 2 + 14 files changed, 148 insertions(+), 55 deletions(-) diff --git a/src/frontend/components/administration/update/CronButton.vue b/src/frontend/components/administration/update/CronButton.vue index 9294ca32..dbbdfcee 100644 --- a/src/frontend/components/administration/update/CronButton.vue +++ b/src/frontend/components/administration/update/CronButton.vue @@ -1,15 +1,18 @@ - {{ t("administration.update.title") }} + {{ t("administration.job.title") }} - {{ t('administration.update.title') }} + {{ t('administration.job.title') }} @@ -21,22 +21,38 @@ @submit.prevent="startUpdate()" > + + + @@ -97,39 +113,87 @@ function formatDate(date) { const value = ref('false'); const valid = ref(true); const loading = ref(false); -const interval = ref('day'); const intervals = ref(['day', 'week']); -const startDate = ref(formatDate(new Date())); -const endDate = ref(formatDate(new Date())); const dateRegex = /^[0-9]{4}-[0-9]{2}-[0-9]{2}$/; -const dateFormatRule = ref((date) => dateRegex.test(date) || t('administration.update.invalidDate')); -const dateIsFutureRule = ref((date) => Date.now() > new Date(date) || t('administration.update.future')); +const dateFormatRule = ref((date) => dateRegex.test(date) || t('administration.job.invalidDate')); +const dateIsFutureRule = ref((date) => Date.now() > new Date(date) || t('administration.job.future')); const props = defineProps({ type: { type: String, default: 'unpaywall' }, }); +const config = computed(() => { + if (props.type === 'unpaywall') { + return { + interval: 'day', + startDate: formatDate(new Date()), + endDate: formatDate(new Date()), + }; + } + if (props.type === 'unpaywallHistory') { + return { + interval: 'day', + startDate: formatDate(new Date()), + indexBase: 'unpaywall_base', + indexHistory: 'unpaywall_history', + }; + } + return {}; +}); + +function startUpdatePeriod(data) { + return $update({ + method: 'POST', + url: '/job/period', + data, + headers: { + 'X-API-KEY': password.value, + }, + }); +} + +function startUpdateHistory(data) { + return $update({ + method: 'POST', + url: '/job/history', + data, + headers: { + 'X-API-KEY': password.value, + }, + }); +} + async function startUpdate() { loading.value = true; - try { - await $update({ - method: 'POST', - url: '/job/period', - data: { - interval: interval.value, - startDate: startDate.value, - endDate: endDate.value, - }, - headers: { - 'X-API-KEY': password.value, - }, - }); - } catch (err) { - snackStore.error(t('error.update.start')); - loading.value = false; - return; + let data; + if (props.type === 'unpaywall') { + data = { + index: config.index.value, + interval: config.interval.value, + time: config.schedule.value, + }; + try { + await startUpdatePeriod(data); + } catch (err) { + snackStore.error(t('error.update.start')); + loading.value = false; + } + } + if (props.type === 'unpaywallHistory') { + data = { + indexBase: config.indexBase.value, + indexHistory: config.indexHistory.value, + interval: config.interval.value, + time: config.schedule.value, + }; + try { + await startUpdateHistory(data); + } catch (err) { + snackStore.error(t('error.update.start')); + loading.value = false; + } } loading.value = false; snackStore.info(t('info.update.started')); diff --git a/src/frontend/components/report/ReportsCard.vue b/src/frontend/components/report/ReportsCard.vue index 44ee2b79..b248f0a0 100644 --- a/src/frontend/components/report/ReportsCard.vue +++ b/src/frontend/components/report/ReportsCard.vue @@ -36,6 +36,7 @@ /> @@ -122,17 +123,17 @@ async function getReports() { let report; reports.value = []; - const r = []; + const newReports = []; for (let i = 0; i < filenames.length; i += 1) { report = await getReport(filenames[i]); - r.push({ + newReports.push({ id: i, data: report, createdAt: formatDate(report.createdAt), }); } - reports.value = r; + reports.value = newReports; loading.value = false; } diff --git a/src/frontend/components/report/ReportsDataTable.vue b/src/frontend/components/report/ReportsDataTable.vue index b8d547de..ae8224a8 100644 --- a/src/frontend/components/report/ReportsDataTable.vue +++ b/src/frontend/components/report/ReportsDataTable.vue @@ -48,6 +48,7 @@ import DetailDialog from '@/components/report/DetailDialog.vue'; const { t } = useI18n(); const props = defineProps({ + type: { type: String, default: '' }, reports: { type: Array, default: () => [] }, loading: { type: Boolean, default: false }, }); @@ -55,28 +56,95 @@ const props = defineProps({ const dialogVisible = ref(false); const reportSelected = ref({}); -const tableHeaders = computed(() => [ - { - title: 'Date', - align: 'start', - sortable: false, - key: 'createdAt', - }, - { title: t('reports.status'), key: 'status', sortable: false }, - { - title: t('reports.updatedDocs'), - key: 'data.totalUpdatedDocs', - sortable: false, - }, - { - title: t('reports.insertedDocs'), - key: 'data.totalInsertedDocs', - sortable: false, - }, - { - title: t('detail'), key: 'details', sortable: false, align: 'right', - }, -]); +const tableHeaders = computed(() => { + if (props.type === 'unpaywall') { + return [ + { + title: 'Date', + align: 'start', + sortable: false, + key: 'createdAt', + }, + { + title: t('reports.status'), + key: 'status', + sortable: false, + }, + { + title: 'Index', + key: 'data.index', + sortable: false, + }, + { + title: t('reports.updatedDocs'), + key: 'data.totalUpdatedDocs', + sortable: false, + }, + { + title: t('reports.insertedDocs'), + key: 'data.totalInsertedDocs', + sortable: false, + }, + { + title: t('detail'), + key: 'details', + sortable: false, + align: 'right', + }, + ]; + } + if (props.type === 'unpaywallHistory') { + return [ + { + title: 'Date', + align: 'start', + sortable: false, + key: 'createdAt', + }, + { + title: t('reports.status'), + key: 'status', + sortable: false, + }, + { + title: 'IndexBase', + key: 'data.indexBase', + sortable: false, + }, + { + title: t('reports.updatedDocs'), + key: 'data.totalBaseInsertedDocs', + sortable: false, + }, + { + title: t('reports.insertedDocs'), + key: 'data.totalBaseUpdatedDocs', + sortable: false, + }, + { + title: 'IndexHistory', + key: 'data.indexHistory', + sortable: false, + }, + { + title: t('reports.updatedDocs'), + key: 'data.totalHistoryInsertedDocs', + sortable: false, + }, + { + title: t('reports.insertedDocs'), + key: 'data.totalHistoryUpdatedDocs', + sortable: false, + }, + { + title: t('detail'), + key: 'details', + sortable: false, + }, + ]; + } + return []; +}); function showDetails(item) { reportSelected.value = item; diff --git a/src/frontend/locales/en.json b/src/frontend/locales/en.json index 0f1d8dcb..34071e78 100644 --- a/src/frontend/locales/en.json +++ b/src/frontend/locales/en.json @@ -101,23 +101,24 @@ "delete": "Deleting the API key", "deleteMessage": "Are you sure you want to delete this API key (irreversible action)" }, - "update" : { + "job" : { "title": "Launch an update", "future": "The date is after today", "invalidDate": "The date is in wrong format", "interval": "Interval", "startDate": "Start date", - "endDate": "End date" + "endDate": "End date", + "index": "Elastic index for unpaywall data", + "indexBase": "Elastic index for basic unpaywall data", + "indexHistory": "Elastic index for history unpaywall data" }, "cron": { "title": "Config of the cron for update process", "interval": "Interval", "schedule": "Schedule (cron format)", - "index": { - "unpaywall": "Elastic index for unpaywall data", - "basic": "Elastic index for basic unpaywall data", - "history": "Elastic index for history unpaywall data" - }, + "index": "Elastic index for unpaywall data", + "indexBase": "Elastic index for basic unpaywall data", + "indexHistory": "Elastic index for history unpaywall data", "active": "Active" } }, diff --git a/src/frontend/locales/fr.json b/src/frontend/locales/fr.json index 9787abf4..9228d4e6 100644 --- a/src/frontend/locales/fr.json +++ b/src/frontend/locales/fr.json @@ -101,23 +101,24 @@ "delete": "Suppression de la clé d'API", "deleteMessage": "Êtes-vous sûr de vouloir supprimer cette clé d'API ? (action irréversible)" }, - "update" : { + "job" : { "title": "Lancement d'une mise à jour", "future": "La date est après aujourd'hui", "invalidDate": "La date est au mauvais format", "interval": "Intervalle", "startDate": "Date de début", - "endDate": "Date de fin" + "endDate": "Date de fin", + "index": "Index elastic pour les données unpaywall", + "indexBase": "Index elastic pour les données unpaywall basique", + "indexHistory": "Index elastic pour les données unpaywall historique" }, "cron": { "title": "Config de la cron de mise à jour des données", "interval": "Intervalle", "schedule": "Periode (format de cron)", - "index": { - "unpaywall": "Index elastic pour les données unpaywall", - "basic": "Index elastic pour les données unpaywall basique", - "history": "Index elastic pour les données unpaywall historique" - }, + "index": "Index elastic pour les données unpaywall", + "indexBase": "Index elastic pour les données unpaywall basique", + "indexHistory": "Index elastic pour les données unpaywall historique", "active": "Active" } }, diff --git a/src/update/config/default.json b/src/update/config/default.json index 8c03a7d2..a7e89cf3 100644 --- a/src/update/config/default.json +++ b/src/update/config/default.json @@ -27,7 +27,7 @@ "unpaywallHistoryCron": { "schedule": "0 0 0 * * *", "active": false, - "indexBase": "unpaywall_enriched", + "indexBase": "unpaywall_base", "indexHistory": "unpaywall_history", "interval": "day" }, diff --git a/src/update/lib/cron/unpaywallHistory.js b/src/update/lib/cron/unpaywallHistory.js index 98828887..3bdbbfff 100644 --- a/src/update/lib/cron/unpaywallHistory.js +++ b/src/update/lib/cron/unpaywallHistory.js @@ -48,7 +48,8 @@ const cron = new Cron('updateHistory', unpaywallHistoryCron.schedule, task, acti function update(newConfig) { if (newConfig.time) cron.setSchedule(newConfig.time); if (newConfig.indexBase) cronConfig.indexBase = newConfig.indexBase; - if (newConfig.interval) cronConfig.indexHistory = newConfig.indexHistory; + if (newConfig.indexHistory) cronConfig.indexHistory = newConfig.indexHistory; + if (newConfig.interval) cronConfig.interval = newConfig.interval; if (newConfig.index || newConfig.interval) cron.setTask(task); } diff --git a/src/update/lib/insert.js b/src/update/lib/insert.js index 2239bd73..1854cbee 100644 --- a/src/update/lib/insert.js +++ b/src/update/lib/insert.js @@ -101,8 +101,6 @@ async function insertDataUnpaywall(insertConfig) { filename, index, offset, limit, } = insertConfig; - console.log(index); - // step insertion in the state const start = new Date(); addStepInsert(filename); diff --git a/src/update/lib/job.js b/src/update/lib/job.js index 6a39a87e..b26f2f06 100644 --- a/src/update/lib/job.js +++ b/src/update/lib/job.js @@ -35,7 +35,7 @@ const { sendMailNoChangefile } = require('./services/mail'); */ async function downloadAndInsertSnapshot(jobConfig) { setInUpdate(true); - await createState('unpaywall'); + await createState({ type: 'unpaywall', index: jobConfig.index }); const filename = await downloadBigSnapshot(); if (!filename) { await fail(); @@ -64,7 +64,7 @@ async function insertChangefilesOnPeriod(jobConfig) { const { interval, startDate, endDate, } = jobConfig; - await createState('unpaywall'); + await createState({ type: 'unpaywall', index: jobConfig.index }); const start = new Date(); addStepGetChangefiles(); const step = getLatestStep(); @@ -112,7 +112,7 @@ async function insertChangefilesOnPeriod(jobConfig) { */ async function insertChangefile(jobConfig) { setInUpdate(true); - await createState('unpaywall'); + await createState({ type: 'unpaywall', index: jobConfig.index }); const success = await insertDataUnpaywall(jobConfig); if (success) { await endState(); @@ -136,7 +136,7 @@ async function insertWithOaHistoryJob(jobConfig) { const { interval, startDate, endDate, } = jobConfig; - await createState('unpaywallHistory'); + await createState({ type: 'unpaywallHistory', indexBase: jobConfig.indexBase, indexHistory: jobConfig.indexHistory }); const start = new Date(); addStepGetChangefiles(); const step = getLatestStep(); diff --git a/src/update/lib/middlewares/format/job.js b/src/update/lib/middlewares/format/job.js index d83e5a6d..d4452eff 100644 --- a/src/update/lib/middlewares/format/job.js +++ b/src/update/lib/middlewares/format/job.js @@ -94,7 +94,7 @@ async function validateInsertFile(req, res, next) { */ async function validateHistoryJob(req, res, next) { const { error, value } = joi.object({ - indexBase: joi.string().trim().default('unpaywall_enriched'), + indexBase: joi.string().trim().default('unpaywall_base'), indexHistory: joi.string().trim().default('unpaywall_history'), interval: joi.string().trim().valid('day', 'week').default('day'), startDate: joi.date().format('YYYY-MM-DD'), @@ -121,7 +121,7 @@ async function validateHistoryJob(req, res, next) { */ async function validateHistoryReset(req, res, next) { const { error, value } = joi.object({ - indexBase: joi.string().trim().default('unpaywall_enriched'), + indexBase: joi.string().trim().default('unpaywall_base'), indexHistory: joi.string().trim().default('unpaywall_history'), startDate: joi.date(), }).with('endDate', 'startDate').validate(req.body); diff --git a/src/update/lib/state.js b/src/update/lib/state.js index a883d918..aa5396a1 100644 --- a/src/update/lib/state.js +++ b/src/update/lib/state.js @@ -27,15 +27,23 @@ function setState(key, value) { * Create a new file on folder "data/update/state" containing the update state * * @return {Promise} name of the file where the state is saved. + * + * @param {string} config.type - Type of job process. + * @param {string} config.index - index where are inserted data. + * @param {string} config.indexBase - index where are inserted base data. + * @param {string} config.indexHistory - index where are inserted history data. */ -async function createState(type) { +async function createState(config) { state = { done: false, createdAt: new Date(), endAt: null, steps: [], error: false, - type, + type: config.type, + index: config?.index || '', + indexBase: config?.indexBase || '', + indexHistory: config?.indexHistory || '', }; } @@ -78,7 +86,6 @@ function addStepInsert(downloadFile) { logger.info('[job: state] add step of insert the content of update file from unpaywall'); const step = { task: 'insert', - index: 'unpaywall', file: downloadFile, linesRead: 0, insertedDocs: 0, @@ -115,23 +122,44 @@ function end() { state.endAt = new Date(); state.took = (new Date(state.endAt) - new Date(state.createdAt)) / 1000; - const insertSteps = state.steps.filter((e) => e.task === 'insert'); - let totalInsertedDocs = 0; - let totalUpdatedDocs = 0; - insertSteps.forEach((step) => { - totalInsertedDocs += step?.insertedDocs || 0; - totalUpdatedDocs += step?.updatedDocs || 0; - if (typeof step.index === 'object') { - const indices = Object.keys(step.index); - indices.forEach((i) => { - const data = step.index[i]; - totalInsertedDocs += data.insertedDocs || 0; - totalUpdatedDocs += data.updatedDocs || 0; - }); - } - }); - state.totalInsertedDocs = totalInsertedDocs; - state.totalUpdatedDocs = totalUpdatedDocs; + if (state.type === 'unpaywall') { + const insertSteps = state.steps.filter((e) => e.task === 'insert'); + let totalInsertedDocs = 0; + let totalUpdatedDocs = 0; + insertSteps.forEach((step) => { + totalInsertedDocs += step?.insertedDocs || 0; + totalUpdatedDocs += step?.updatedDocs || 0; + if (typeof step.index === 'object') { + const indices = Object.keys(step.index); + indices.forEach((i) => { + const data = step.index[i]; + totalInsertedDocs += data.insertedDocs || 0; + totalUpdatedDocs += data.updatedDocs || 0; + }); + } + }); + state.totalInsertedDocs = totalInsertedDocs; + state.totalUpdatedDocs = totalUpdatedDocs; + } + + if (state.type === 'unpaywallHistory') { + let totalInsertBase = 0; + let totalUpdateBase = 0; + let totalInsertHistory = 0; + let totalUpdateHistory = 0; + const insertSteps = state.steps.filter((e) => e.task === 'insert'); + insertSteps.forEach((step) => { + const keys = Object.keys(step.index); + totalInsertBase += step.index[keys[0]].insertedDocs; + totalUpdateBase += step.index[keys[0]].updatedDocs; + totalInsertHistory += step.index[keys[1]].insertedDocs; + totalUpdateHistory += step.index[keys[1]].updatedDocs; + }); + state.totalBaseInsertedDocs = totalInsertBase; + state.totalBaseUpdatedDocs = totalUpdateBase; + state.totalHistoryInsertedDocs = totalInsertHistory; + state.totalHistoryUpdatedDocs = totalUpdateHistory; + } } /** diff --git a/src/update/openapi.yml b/src/update/openapi.yml index 7e81ff09..25fd0bbf 100644 --- a/src/update/openapi.yml +++ b/src/update/openapi.yml @@ -199,7 +199,7 @@ paths: '409': $ref: '#/components/responses/Conflict' - /update/job/changefile/{filename}: + /update/job/changefile/${filename}: post: tags: - job diff --git a/src/update/test/unpaywallHistory/history.js b/src/update/test/unpaywallHistory/history.js index ba2d1782..00e60c1d 100644 --- a/src/update/test/unpaywallHistory/history.js +++ b/src/update/test/unpaywallHistory/history.js @@ -52,7 +52,7 @@ describe('Test: daily update route test with history', () => { await new Promise((resolve) => { setTimeout(resolve, 100); }); isUpdate = await checkIfInUpdate(); } - const count = await countDocuments('unpaywall_enriched'); + const count = await countDocuments('unpaywall_base'); expect(count).to.equal(2); }); @@ -100,9 +100,9 @@ describe('Test: daily update route test with history', () => { expect(state.steps[2]).have.property('status').equal('success'); const step2 = state.steps[2].index; - expect(step2.unpaywall_enriched).have.property('insertedDocs').equal(2); - expect(step2.unpaywall_enriched).have.property('updatedDocs').equal(0); - expect(step2.unpaywall_enriched).have.property('failedDocs').equal(0); + expect(step2.unpaywall_base).have.property('insertedDocs').equal(2); + expect(step2.unpaywall_base).have.property('updatedDocs').equal(0); + expect(step2.unpaywall_base).have.property('failedDocs').equal(0); expect(step2.unpaywall_history).have.property('insertedDocs').equal(0); expect(step2.unpaywall_history).have.property('updatedDocs').equal(0); expect(step2.unpaywall_history).have.property('failedDocs').equal(0); @@ -124,9 +124,9 @@ describe('Test: daily update route test with history', () => { expect(state.steps[4]).have.property('status').equal('success'); const step4 = state.steps[4].index; - expect(step4.unpaywall_enriched).have.property('insertedDocs').equal(0); - expect(step4.unpaywall_enriched).have.property('updatedDocs').equal(2); - expect(step4.unpaywall_enriched).have.property('failedDocs').equal(0); + expect(step4.unpaywall_base).have.property('insertedDocs').equal(0); + expect(step4.unpaywall_base).have.property('updatedDocs').equal(2); + expect(step4.unpaywall_base).have.property('failedDocs').equal(0); expect(step4.unpaywall_history).have.property('insertedDocs').equal(2); expect(step4.unpaywall_history).have.property('updatedDocs').equal(0); expect(step4.unpaywall_history).have.property('failedDocs').equal(0); @@ -148,9 +148,9 @@ describe('Test: daily update route test with history', () => { expect(state.steps[6]).have.property('status').equal('success'); const step6 = state.steps[6].index; - expect(step6.unpaywall_enriched).have.property('insertedDocs').equal(0); - expect(step6.unpaywall_enriched).have.property('updatedDocs').equal(2); - expect(step6.unpaywall_enriched).have.property('failedDocs').equal(0); + expect(step6.unpaywall_base).have.property('insertedDocs').equal(0); + expect(step6.unpaywall_base).have.property('updatedDocs').equal(2); + expect(step6.unpaywall_base).have.property('failedDocs').equal(0); expect(step6.unpaywall_history).have.property('insertedDocs').equal(2); expect(step6.unpaywall_history).have.property('updatedDocs').equal(0); expect(step6.unpaywall_history).have.property('failedDocs').equal(0); @@ -189,9 +189,9 @@ describe('Test: daily update route test with history', () => { expect(report.steps[2]).have.property('status').equal('success'); const step2 = report.steps[2].index; - expect(step2.unpaywall_enriched).have.property('insertedDocs').equal(2); - expect(step2.unpaywall_enriched).have.property('updatedDocs').equal(0); - expect(step2.unpaywall_enriched).have.property('failedDocs').equal(0); + expect(step2.unpaywall_base).have.property('insertedDocs').equal(2); + expect(step2.unpaywall_base).have.property('updatedDocs').equal(0); + expect(step2.unpaywall_base).have.property('failedDocs').equal(0); expect(step2.unpaywall_history).have.property('insertedDocs').equal(0); expect(step2.unpaywall_history).have.property('updatedDocs').equal(0); expect(step2.unpaywall_history).have.property('failedDocs').equal(0); @@ -213,9 +213,9 @@ describe('Test: daily update route test with history', () => { expect(report.steps[4]).have.property('status').equal('success'); const step4 = report.steps[4].index; - expect(step4.unpaywall_enriched).have.property('insertedDocs').equal(0); - expect(step4.unpaywall_enriched).have.property('updatedDocs').equal(2); - expect(step4.unpaywall_enriched).have.property('failedDocs').equal(0); + expect(step4.unpaywall_base).have.property('insertedDocs').equal(0); + expect(step4.unpaywall_base).have.property('updatedDocs').equal(2); + expect(step4.unpaywall_base).have.property('failedDocs').equal(0); expect(step4.unpaywall_history).have.property('insertedDocs').equal(2); expect(step4.unpaywall_history).have.property('updatedDocs').equal(0); expect(step4.unpaywall_history).have.property('failedDocs').equal(0); @@ -237,9 +237,9 @@ describe('Test: daily update route test with history', () => { expect(report.steps[6]).have.property('status').equal('success'); const step6 = report.steps[6].index; - expect(step6.unpaywall_enriched).have.property('insertedDocs').equal(0); - expect(step6.unpaywall_enriched).have.property('updatedDocs').equal(2); - expect(step6.unpaywall_enriched).have.property('failedDocs').equal(0); + expect(step6.unpaywall_base).have.property('insertedDocs').equal(0); + expect(step6.unpaywall_base).have.property('updatedDocs').equal(2); + expect(step6.unpaywall_base).have.property('failedDocs').equal(0); expect(step6.unpaywall_history).have.property('insertedDocs').equal(2); expect(step6.unpaywall_history).have.property('updatedDocs').equal(0); expect(step6.unpaywall_history).have.property('failedDocs').equal(0); diff --git a/src/update/test/unpaywallHistory/rollback.js b/src/update/test/unpaywallHistory/rollback.js index 45333d10..c4355bd9 100644 --- a/src/update/test/unpaywallHistory/rollback.js +++ b/src/update/test/unpaywallHistory/rollback.js @@ -25,9 +25,9 @@ describe('Test: rollback history test', () => { describe(`Rollback: history rollback at ${date1}`, () => { before(async () => { - await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_base'); await deleteIndex('unpaywall_history'); - await createIndex('unpaywall_enriched', unpaywallEnrichedMapping); + await createIndex('unpaywall_base', unpaywallEnrichedMapping); await createIndex('unpaywall_history', unpaywallHistoryMapping); await insertHistoryDataUnpaywall(); }); @@ -66,13 +66,13 @@ describe('Test: rollback history test', () => { }); it('Should get unpaywall data', async () => { - const count1 = await countDocuments('unpaywall_enriched'); + const count1 = await countDocuments('unpaywall_base'); expect(count1).to.equal(2); const count2 = await countDocuments('unpaywall_history'); expect(count2).to.equal(4); - const data = await getAllData('unpaywall_enriched'); + const data = await getAllData('unpaywall_base'); data.forEach((e) => { expect(e.genre).to.equal('v3'); @@ -80,16 +80,16 @@ describe('Test: rollback history test', () => { }); after(async () => { - await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_base'); await deleteIndex('unpaywall_history'); }); }); describe(`Rollback: history rollback at ${date2}`, () => { before(async () => { - await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_base'); await deleteIndex('unpaywall_history'); - await createIndex('unpaywall_enriched', unpaywallEnrichedMapping); + await createIndex('unpaywall_base', unpaywallEnrichedMapping); await createIndex('unpaywall_history', unpaywallHistoryMapping); await insertHistoryDataUnpaywall(); }); @@ -106,13 +106,13 @@ describe('Test: rollback history test', () => { }); it('Should get unpaywall data', async () => { - const count1 = await countDocuments('unpaywall_enriched'); + const count1 = await countDocuments('unpaywall_base'); expect(count1).to.equal(2); const count2 = await countDocuments('unpaywall_history'); expect(count2).to.equal(2); - const data = await getAllData('unpaywall_enriched'); + const data = await getAllData('unpaywall_base'); data.forEach((e) => { expect(e.genre).to.equal('v2'); @@ -120,16 +120,16 @@ describe('Test: rollback history test', () => { }); after(async () => { - await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_base'); await deleteIndex('unpaywall_history'); }); }); describe(`Rollback: history rollback at ${date3}`, () => { before(async () => { - await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_base'); await deleteIndex('unpaywall_history'); - await createIndex('unpaywall_enriched', unpaywallEnrichedMapping); + await createIndex('unpaywall_base', unpaywallEnrichedMapping); await createIndex('unpaywall_history', unpaywallHistoryMapping); await insertHistoryDataUnpaywall(); }); @@ -146,13 +146,13 @@ describe('Test: rollback history test', () => { }); it('Should get unpaywall data', async () => { - const count1 = await countDocuments('unpaywall_enriched'); + const count1 = await countDocuments('unpaywall_base'); expect(count1).to.equal(2); const count2 = await countDocuments('unpaywall_history'); expect(count2).to.equal(0); - const data = await getAllData('unpaywall_enriched'); + const data = await getAllData('unpaywall_base'); data.forEach((e) => { expect(e.genre).to.equal('v1'); @@ -160,16 +160,16 @@ describe('Test: rollback history test', () => { }); after(async () => { - await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_base'); await deleteIndex('unpaywall_history'); }); }); describe(`Rollback: history rollback at ${date4}`, () => { before(async () => { - await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_base'); await deleteIndex('unpaywall_history'); - await createIndex('unpaywall_enriched', unpaywallEnrichedMapping); + await createIndex('unpaywall_base', unpaywallEnrichedMapping); await createIndex('unpaywall_history', unpaywallHistoryMapping); await insertHistoryDataUnpaywall(); }); @@ -186,7 +186,7 @@ describe('Test: rollback history test', () => { }); it('Should get unpaywall data', async () => { - const count1 = await countDocuments('unpaywall_enriched'); + const count1 = await countDocuments('unpaywall_base'); expect(count1).to.equal(0); const count2 = await countDocuments('unpaywall_history'); @@ -194,7 +194,7 @@ describe('Test: rollback history test', () => { }); after(async () => { - await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_base'); await deleteIndex('unpaywall_history'); }); }); diff --git a/src/update/test/utils/elastic.js b/src/update/test/utils/elastic.js index abea057a..732be011 100644 --- a/src/update/test/utils/elastic.js +++ b/src/update/test/utils/elastic.js @@ -96,7 +96,7 @@ async function insertHistoryDataUnpaywall() { data2.push(JSON.parse(line)); } - const body2 = data2.flatMap((doc) => [{ index: { _index: 'unpaywall_enriched', _id: doc.doi } }, doc]); + const body2 = data2.flatMap((doc) => [{ index: { _index: 'unpaywall_base', _id: doc.doi } }, doc]); try { await elasticClient.bulk({ refresh: true, body: body2 }); diff --git a/src/update/test/utils/reset.js b/src/update/test/utils/reset.js index bc2b1a54..731494f2 100644 --- a/src/update/test/utils/reset.js +++ b/src/update/test/utils/reset.js @@ -23,7 +23,7 @@ async function reset() { await deleteFile('unpaywall', 'fake1-error.jsonl.gz'); await deleteFile('unpaywall', 'snapshot.jsonl.gz'); await deleteIndex('unpaywall-test'); - await deleteIndex('unpaywall_enriched'); + await deleteIndex('unpaywall_base'); await deleteIndex('unpaywall_history'); await resetCronConfig('unpaywall'); await resetCronConfig('unpaywallHistory'); From 2c1eb7007782efd14e8c6e93c4f9e5c95f4e0a6d Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Mon, 15 Jan 2024 14:06:46 +0100 Subject: [PATCH 018/114] feat(frontend): add elastic tab on administration --- .../administration/elastic/ElasticCard.vue | 164 ++++++++++++++++++ .../components/report/ReportsDataTable.vue | 41 ++++- src/frontend/components/skeleton/NoData.vue | 1 + src/frontend/locales/en.json | 17 +- src/frontend/locales/fr.json | 17 +- src/frontend/pages/administration.vue | 15 +- src/update/lib/controllers/elastic.js | 46 ++++- src/update/lib/routers/elastic.js | 16 +- src/update/lib/services/elastic.js | 17 ++ src/update/test/utils/reset.js | 1 + 10 files changed, 322 insertions(+), 13 deletions(-) create mode 100644 src/frontend/components/administration/elastic/ElasticCard.vue diff --git a/src/frontend/components/administration/elastic/ElasticCard.vue b/src/frontend/components/administration/elastic/ElasticCard.vue new file mode 100644 index 00000000..b08619ae --- /dev/null +++ b/src/frontend/components/administration/elastic/ElasticCard.vue @@ -0,0 +1,164 @@ + + + diff --git a/src/frontend/components/report/ReportsDataTable.vue b/src/frontend/components/report/ReportsDataTable.vue index ae8224a8..c278eb89 100644 --- a/src/frontend/components/report/ReportsDataTable.vue +++ b/src/frontend/components/report/ReportsDataTable.vue @@ -8,7 +8,46 @@ item-key="id" class="elevation-1" > - diff --git a/src/frontend/components/administration/apikey/ApikeyTab.vue b/src/frontend/components/administration/apikey/ApikeyTab.vue index 6e30d653..3395739c 100644 --- a/src/frontend/components/administration/apikey/ApikeyTab.vue +++ b/src/frontend/components/administration/apikey/ApikeyTab.vue @@ -49,11 +49,6 @@ {{ t("administration.apikey.create") }} - @@ -111,6 +129,8 @@ import { useSnacksStore } from '@/store/snacks'; import { useAdminStore } from '@/store/admin'; import CreateDialog from '@/components/administration/apikey/CreateDialog.vue'; +import DeleteDialog from '@/components/administration/apikey/DeleteDialog.vue'; +import UpdateDialog from '@/components/administration/apikey/UpdateDialog.vue'; import ImportDialog from '@/components/administration/apikey/ImportDialog.vue'; import ExportDialog from '@/components/administration/apikey/ExportDialog.vue'; import Loader from '@/components/skeleton/Loader.vue'; @@ -126,10 +146,20 @@ const { password } = storeToRefs(adminStore); const loading = ref(false); const createDialogVisible = ref(false); +const updateDialogVisible = ref(false); +const deleteDialogVisible = ref(false); const importDialogVisible = ref(false); const exportDialogVisible = ref(false); const apikeys = ref([]); +const selectedApikey = ref(''); +const selectedName = ref(''); +const selectedDescription = ref(''); +const selectedOwner = ref(''); +const selectedAccess = ref([]); +const selectedAttributes = ref([]); +const selectedAllowed = ref(false); + async function getApikeys() { let res; loading.value = true; @@ -150,6 +180,24 @@ async function getApikeys() { apikeys.value = res?.data; } +async function openUpdateDialog(id) { + const apikey = apikeys.value.find((e) => e.apikey === id); + selectedApikey.value = id; + selectedName.value = apikey.config.name; + selectedDescription.value = apikey.config.description; + selectedOwner.value = apikey.config.owner; + selectedAccess.value = apikey.config.access; + selectedAttributes.value = apikey.config.attributes; + selectedAllowed.value = apikey.config.allowed; + await nextTick(); + updateDialogVisible.value = true; +} + +async function openDeleteDialog(id) { + selectedApikey.value = id; + deleteDialogVisible.value = true; +} + onMounted(() => { getApikeys(); }); diff --git a/src/frontend/components/administration/apikey/CreateDialog.vue b/src/frontend/components/administration/apikey/CreateDialog.vue index b06499e1..8072070f 100644 --- a/src/frontend/components/administration/apikey/CreateDialog.vue +++ b/src/frontend/components/administration/apikey/CreateDialog.vue @@ -69,7 +69,7 @@ {{ t('administration.apikey.attributes') }} - - {{ apikey }} + {{ props.apikey }} diff --git a/src/frontend/components/administration/apikey/UpdateDialog.vue b/src/frontend/components/administration/apikey/UpdateDialog.vue index 243f7450..b19a240d 100644 --- a/src/frontend/components/administration/apikey/UpdateDialog.vue +++ b/src/frontend/components/administration/apikey/UpdateDialog.vue @@ -76,9 +76,13 @@ {{ t('administration.apikey.attributes') }} - @@ -114,7 +118,7 @@ diff --git a/src/frontend/components/enrich/SettingsTab.vue b/src/frontend/components/enrich/SettingsTab.vue index a8fbc1c8..8b0740b1 100644 --- a/src/frontend/components/enrich/SettingsTab.vue +++ b/src/frontend/components/enrich/SettingsTab.vue @@ -46,8 +46,12 @@ - {{ t('graphql.settings') }} - + From 54775cb3ee8f80d5c3dc1739a3dcbbcb5c5fc2ec Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Fri, 23 Feb 2024 15:37:22 +0100 Subject: [PATCH 050/114] fix: update ping of health --- src/health/README.md | 6 ----- .../config/custom-environment-variables.json | 4 --- src/health/config/default.json | 4 --- src/health/lib/config.js | 5 ---- src/health/lib/health.js | 25 ------------------- src/update/lib/controllers/health.js | 6 ++++- src/update/lib/services/mail.js | 23 +++++++++++++++++ src/update/lib/services/unpaywall.js | 21 ++++++++++++++++ 8 files changed, 49 insertions(+), 45 deletions(-) diff --git a/src/health/README.md b/src/health/README.md index b5deab68..e8e1d895 100644 --- a/src/health/README.md +++ b/src/health/README.md @@ -25,10 +25,6 @@ info: { "user": "elastic", "password": "********" }, - "unpaywall": { - "host": "http://fakeUnpaywall:3000", - "apikey": "********" - }, "redis": { "host": "redis", "port": "6379", @@ -52,8 +48,6 @@ info: { | ELASTICSEARCH_PORT | 9200 | elasticsearch port | | ELASTICSEARCH_USERNAME | elastic | elasticsearch admin username | | ELASTICSEARCH_PASSWORD | changeme | elasticsearch admin password | -| UNPAYWALL_HOST | http://fakeUnpaywall:3000 | unpaywall host | -| UNPAYWALL_APIKEY | default | unpaywall apikey | | REDIS_HOST | redis | redis host | | REDIS_PORT | 6379 | redis port | | REDIS_PASSWORD | changeme | redis password | diff --git a/src/health/config/custom-environment-variables.json b/src/health/config/custom-environment-variables.json index 056c5fc9..c7c84d6b 100644 --- a/src/health/config/custom-environment-variables.json +++ b/src/health/config/custom-environment-variables.json @@ -11,10 +11,6 @@ "user": "ELASTICSEARCH_USERNAME", "password": "ELASTICSEARCH_PASSWORD" }, - "unpaywall": { - "host": "UNPAYWALL_HOST", - "apikey": "UNPAYWALL_APIKEY" - }, "redis": { "host": "REDIS_HOST", "port": "REDIS_PORT", diff --git a/src/health/config/default.json b/src/health/config/default.json index 21a434f9..1949c53b 100644 --- a/src/health/config/default.json +++ b/src/health/config/default.json @@ -11,10 +11,6 @@ "user": "elastic", "password": "changeme" }, - "unpaywall": { - "host": "http://fakeUnpaywall:3000", - "apikey": "default" - }, "redis": { "host": "redis", "port": "6379", diff --git a/src/health/lib/config.js b/src/health/lib/config.js index dda44ea5..ed50dce4 100644 --- a/src/health/lib/config.js +++ b/src/health/lib/config.js @@ -17,14 +17,9 @@ function getConfig() { if (copyConfig.elasticsearch.password === defaultConfig.elasticsearch.password) { logger.warn('[config]: Elastic password has the default value'); } - if (copyConfig.unpaywall.apikey === defaultConfig.unpaywall.apikey) { - logger.warn('[config]: Unpaywall apikey has the default value'); - } copyConfig.redis.password = '********'; copyConfig.elasticsearch.password = '********'; - copyConfig.unpaywall.apikey = '********'; - logger.info(JSON.stringify(copyConfig, null, 2)); return copyConfig; diff --git a/src/health/lib/health.js b/src/health/lib/health.js index 95758b36..5f6fa27e 100644 --- a/src/health/lib/health.js +++ b/src/health/lib/health.js @@ -109,27 +109,6 @@ async function promiseWithTimeout(p1, name) { }; } -/** - * Ping service. - * - * @param {string} name - Name of service. - * @param {string} host - Host of service. - * - * @returns {Promise} ping - */ -async function ping(name, host) { - try { - await axios({ - method: 'GET', - url: host, - }); - } catch (err) { - logger.error(`[${name}] Cannot request ${host}`, err); - return false; - } - return true; -} - /** * Health and ping all service on ezunpaywall. * @@ -154,9 +133,6 @@ async function healthAll() { const pingElastic = promiseWithTimeout(pingElasticWithClient(), 'elastic'); - const unpaywallHost = config.get('unpaywall.host'); - const pingUnpaywall = promiseWithTimeout(ping('unpaywall', unpaywallHost), 'unpaywall'); - const pingRedis = promiseWithTimeout(pingRedisWithClient(), 'redis'); const result = await Promise.allSettled([ @@ -166,7 +142,6 @@ async function healthAll() { healthApikey, healthMail, pingElastic, - pingUnpaywall, pingRedis, ]); diff --git a/src/update/lib/controllers/health.js b/src/update/lib/controllers/health.js index 1960ff3f..09ce8c8b 100644 --- a/src/update/lib/controllers/health.js +++ b/src/update/lib/controllers/health.js @@ -1,5 +1,7 @@ const promiseWithTimeout = require('../ping'); const { pingElastic } = require('../services/elastic'); +const { pingMail } = require('../services/mail'); +const { pingUnpaywall } = require('../services/unpaywall'); /** * Controller to get health of all services connected to update service. @@ -12,8 +14,10 @@ async function health(req, res, next) { const start = Date.now(); const p1 = promiseWithTimeout(pingElastic(), 'elastic'); + const p2 = promiseWithTimeout(pingMail(), 'mail'); + const p3 = promiseWithTimeout(pingUnpaywall(), 'unpaywall'); - let resultPing = await Promise.allSettled([p1]); + let resultPing = await Promise.allSettled([p1, p2, p3]); resultPing = resultPing.map((e) => e.value); const result = {}; diff --git a/src/update/lib/services/mail.js b/src/update/lib/services/mail.js index 64a119f4..acb98b98 100644 --- a/src/update/lib/services/mail.js +++ b/src/update/lib/services/mail.js @@ -1,6 +1,8 @@ const axios = require('axios'); const config = require('config'); +const logger = require('../logger'); + const apikey = config.get('mail.apikey'); const mail = axios.create({ @@ -8,6 +10,26 @@ const mail = axios.create({ }); mail.host = config.get('mail.host'); +/** + * Ping mail service. + * + * @returns {Promise} healthy or not. + */ +async function pingMail() { + let res; + try { + res = await mail({ + method: 'GET', + url: '/ping', + }); + } catch (err) { + logger.error('[mail] Cannot ping mail', err); + return err?.message; + } + if (res?.status === 204) return true; + return false; +} + /** * Send update mail report. * @@ -67,6 +89,7 @@ function sendMailNoChangefile(startDate, endDate) { } module.exports = { + pingMail, sendMailUpdateReport, sendMailUpdateStarted, sendMailNoChangefile, diff --git a/src/update/lib/services/unpaywall.js b/src/update/lib/services/unpaywall.js index c793d72c..db1adc7a 100644 --- a/src/update/lib/services/unpaywall.js +++ b/src/update/lib/services/unpaywall.js @@ -9,6 +9,26 @@ unpaywall.baseURL = config.get('unpaywall.host'); const apikey = config.get('unpaywall.apikey'); +/** + * Ping unpaywall. + * + * @returns {Promise} healthy or not. + */ +async function pingUnpaywall() { + let res; + try { + res = await unpaywall({ + method: 'GET', + url: '/ping', + }); + } catch (err) { + logger.error('[unpaywall] Cannot ping unpaywall', err); + return err?.message; + } + if (res?.status === 204) return true; + return false; +} + /** * Get the current snapshot. * @@ -115,6 +135,7 @@ async function getChangefile(filename, interval) { } module.exports = { + pingUnpaywall, getSnapshot, getChangefiles, getChangefile, From 8e759659af1a1e6485f8dc66d85144eb9a16820b Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Fri, 23 Feb 2024 15:43:00 +0100 Subject: [PATCH 051/114] feat: add favicon --- src/frontend/public/favicon.ico | Bin 4286 -> 15406 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/src/frontend/public/favicon.ico b/src/frontend/public/favicon.ico index 18993ad91cfd43e03b074dd0b5cc3f37ab38e49c..df9b15fd17297d5e76c1004b8bbe0142de3715c8 100644 GIT binary patch literal 15406 zcmeHOd3;nw@_&98_0P%CIgn((=BwqU>y_C_&f;yI>Es9&dIp zgMq5qANF@($A-Qs!EU3B!LA#V+8FjH2f1uEg}RldMfpF;TB;o^eGH0$JyU;!k{agK zB`HWb)33Dx;@3tkCj}WQOu@CLCnk19w3hC~QDv=|B0Rezzk94-8w2#S^ka7_TYkn6 zxAlcxTX=F3#6By^I}qt#r8Y^d9q&(f{1F~b1Ixx>_YZh~y5sY^YCj8ZsjW;;E~kWh z7U1ZW>=@q$G&Vt>`XY~{)&|64;cubG`dhFh1-YI{H;2?Rg?MCB-`US)!;a6_z`RKV zVfxU1Fgz&?l7ro#?3Hn_bj~=Wr@&wGo`Egv{st@NPJ*Pi3Z#U3q@p}#=Oi?Q-CwO^ z?e~7Y0hYZw5ys_q0b__ee7*jC*tTIc!&H(Sfo`!gCuKxhGeYQ#+6TX`~peOvW`)kPS&=@foK1F?hIyS2Vte7{MVJzy_ z0^a`XC>WoE{gFTQ-%kBe7H^gghqYyMAUnQ3(+Aqv>f~UT`Du~faA@~dhUxJBT~IcA z9IRY46K0J}hmRJ&3Y&b7Dp-D0`3ElYGcE9!h(7}>VZqVGfU`ep3#q6MLaiRqxm827iR_eckJ(AV zE@jCA6x(LJ#=5y$i*>mj>c^&pd3>A{l~s^0RD-z(e)?RX*D-a}zO%$6*6XR#{{j|E4t>$&XM zx_KG~`z0%$x+Zx&0-Gm?xE<7J+Z%T^b%-rB+~cF-9x;BLw9HQ~v-ebCsmyBc-vWKW zNh|+sSMECgLLL_RU?21~Kj!s#s<|L;4-Buvdr?Budd6V4Bevt#Xwp39b6IU?s}H%2 zv90Tg+-^^>b&|8P89%-NPz-2ipV{)(jZ-HJYb+WzxgOWD!^1i?$EW}=d(5#qJ-YS~ zr+JgCg}kY($w#IkV=#qz{KJ~N1|$cup?_j)wcMYT*_v6s46OU8STx=mw#=A1B_sm< zl^#`ly4v1xUuzzON26;c3!Q%Ce#W*gw~9J83lvH~Ctq9~UDT9Mp-P|KxSOselkWXYqGN=cPsw7_xP^bzDU~`ojQ?eE2Y*NS(!xC_a{lNezUS-v^r1#5FTV%(?%akuw{F10hYw%@VxhigPyPlg zUYjEL6a6Xxm^s`8Yu=d+M-J@~@A5hXz`5U#Liyc0aQ4J+NarE!-i)zWh#PBT3aPc9 z_`g+L?2i6{#fhomUU2>DCD`-TdW=uJz!X{wc5nFl=T z{Pzm}g($;KYnDP*Y+a_0(dlvUpt2I?JfE+&2~w&?bPuXOpmXD3E|=25eHp!Hjp2gM z#Qxox9EM`-bjHE>Ulill_;Z@_{nEKpuyWyx!ag$E`SnN(_fFz%mSU_~aqkYrLV`bk zM>ejPf2w8tGJkG+{LbgbtdSY0t167~as}X_L|wwA2$De8Qi{c9evO(*tPj% z^lyPo9-G#d!TR@Jhu?nuj`^npyEa2{ucsM3@=?1seawFME6QQ-)(tSYXDir;-+krx z?!vW87Z7);kVQtjI@z3mUi${oEXK6RM%EEuYdAhGCvM;Ty`%Ntik3zHrm$N3N=r*! zIDd))a4s&eZL9MG4q^;}L1$mK^lv}5}eV!J1s3=C86TE00u-8Y7)d~7ma>DlX z%-i)^dfENdKZPFCufKyf&B%N?N9FKU$HmoeQwW5I-@VoC)cC7z1T;`7M zMO_2iAcoWW{+y>)FY87t^KV_P9eoFQZd79)M_i?P6vluqfz5F3`%a^w(bwu!Um|2+ zM>m+mxRBYdPJu*S4twNkwVo7VQAS~6qGx)9*URV|?yEcrgJT>rE@QCEHHs&n?bE7` zAZW$S`G?H72CeZMa0z}7-O-O#ex~`G5>{(hT9o&&B4Zn0nS(kc%fRFpk5Dd7it$FD z(bp8_IT`24;y<@mmg4H z)qJeXkS2#+rjF{ZIzRTqh`vEjrbqe?!gb?aw7LDp5Z9aXeB)*OwyMCK)^Ys4`7AZe za|YRKNp2j@LFuxKUwvQAD~W)!Mf|So@swY7e7zTN?jFaS=pBxe?(*Cf^^*U_`TPL; zT*Wc(%}WT#raYi?oSh$qqn4%S;9A-7bvvbmdG0}MRRSOPv3$MZ&(+@1X9gLn(!#yI z9N4*OoH?PXSM}p?gsS>p{4M1O(<5t7#Wn1T4Q_^0_Tg$-=t7de)>mtfdc2Yr;qy{q zVvEL{pEGx7M?^UgbCl0=5sy|QSAkiy(QsbN~N7@zW!}pf-i43+smz^1rZ5wk! zoqFb|+N-PM{YdZZ>Ovb2en+3q?a-)~*^HZDPF;?*P|&Su6SVb}4Bw;Lwvd0#@rZu> zrV#gIS@HGK9m98wnfED93iM71_j(QIm?cipWTfXOo@{bmX>R6t z>@5)0jKYu^TQ?W|w)*{L;rF<-{v!=L-Cypba29zEJ>v z^VcAINAh%3k7Lo{DL+85?ocRG@RR_{E{>gT3z96}DddTHz!yWGud@1JNZc}23T8@l+(O??lksj7*6J%EfM6^|p#%$V!32#Afi}n=fJxb06F5xN-F|{B~#` ztbS`24A992*X2`%iGlQq%r4BSWMTY@-+r2HjKO`4%eb#Vx#s)j<#6WsuW;|~ZI+9@ zjd|ReBT^U!8p~bGryf6ifIZ84&x)(Z+8_IQKjvDS;9gN@_Pp)0l`4LV?U|!}VdKih zY#*iaeg*FL>|?q8%8CkjaK93MIq)sWxpTyND=pk7&mw>IC@I{(dzL>bJa}?qoAnRSztCg4&#r+wii6U$uW;Pj`X82K$%$$Yq-I zTRvIAY+>i7zgzhS%EwSn$sFU^Ts~JpIy!me5bCU)oxS36km*vH*Asr)4|$aNyInVb z^|W&n8nOLA+Lt99w{KpD3Hgau`bB?v(HQK}Dl^8fr76^1#QGS&!Kq`1na)UedfNKu z>g1nYlmC(YC#U=Bxc+by_kr%-zQt^q{Ow;$($Mx@^jIz#TPm%L!AdLadx`fazkllJ zPf$^Qm+ejQdPn!_@Y}T?+!)vUPohoxh+nRKmtiI!N^8v*tCz6#&mM33zRHa8Xh}Li zn-Tb_E$yq)eq7mX%N`@v*`q;qkBlD?>ha^ZE<3A8`)N0?T}59x4QQW{@wB5OTBp$d zeB92^rrPYs#1$)~#58Yr48G$ludi^WMA{*q-rl+!w7Ge-+fUex*nGs`C`N zu3GdGof8%Fi}Tgr(dM_B!fQ85jq!N`%ajikGM+OwkKsSOeRr9 z&!)2Y^!%w~?3@j)1rHxQVE&7I$bulCA=Rhfu#sc02B9`z_=X%VTGL$9=kR)PeQ6bfm|}^7wa@ zoA7~}3?G+XR(yl$bRLr9mhYLd^``0P!Q=9&xu0$5 zqeiqRPVu!3|C;GLtt;6d+VF9pbM!Gsv*H?dCVwN#Tvs+w9s=#?jI(tAv+YzJnZmrT z6!mVEU_Xzm4o4nTQI}>$9K$t-*#3>WaL5DofO>j}x!522Cp3)WVwZ91WN-DD6JI|c z=lWSk^o0%lhsI*WeFw}lyoWi>@EUB!3O9c!j)`&4NoZ71tsd;ysu-7@clKmquRX>y z7xLRTEHw9M=~u0_bFjL*tRu{8m@qy)gXdpr&doWa3rB6R9%Xe4^DUHk_u=KG7qf(xaqj(Ond*T8-%2 z_DTG|mNLu-p2qKvM_#Wnj;X+Y=h2^Uulf9diore-D_E8X=USGQCc5D{lPGhP@0gU( zS|8xqjURD5x3xGylR4t8LO*ty@*OFmp373ByhastXp}%_5$qvgbqGU7?i?!nLy+m{ z`H^2M>e$8?&oDHlvol$-^}A=s`?Iq%c+RAd&Oa3 zOGiREbCw`xmFozJ^aNatJY>w+g ze6a2@u~m#^BZm@8wco9#Crlli0uLb^3E$t2-WIc^#(?t)*@`UpuofJ(Uyh@F>b3Ph z$D^m8Xq~pTkGJ4Q`Q2)te3mgkWYZ^Ijq|hkiP^9`De={bQQ%heZC$QU2UpP(-tbl8 zPWD2abEew;oat@w`uP3J^YpsgT%~jT(Dk%oU}sa$7|n6hBjDj`+I;RX(>)%lm_7N{+B7Mu%H?422lE%MBJH!!YTN2oT7xr>>N-8OF$C&qU^ z>vLsa{$0X%q1fjOe3P1mCv#lN{xQ4_*HCSAZjTb1`}mlc+9rl8$B3OP%VT@mch_~G z7Y+4b{r>9e=M+7vSI;BgB?ryZDY4m>&wcHSn81VH1N~`0gvwH{ z8dv#hG|OK`>1;j7tM#B)Z7zDN?{6=dUal}$e Date: Mon, 26 Feb 2024 16:23:19 +0100 Subject: [PATCH 052/114] fix: countDOI --- docker-compose.debug.yml | 10 +++++----- src/graphql/lib/middlewares/args.js | 24 ++++++++++++++---------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/docker-compose.debug.yml b/docker-compose.debug.yml index b8b8d635..f4c24700 100644 --- a/docker-compose.debug.yml +++ b/docker-compose.debug.yml @@ -5,7 +5,7 @@ services: image: node:18.17.1 container_name: ezunpaywall-graphql-dev environment: - - NODE_ENV=development + - NODE_ENV # elastic - ELASTICSEARCH_HOSTS - ELASTICSEARCH_PORT @@ -40,7 +40,7 @@ services: image: node:18.17.1 container_name: ezunpaywall-update-dev environment: - - NODE_ENV=development + - NODE_ENV - LOG_LEVEL=debug # unpaywall - UNPAYWALL_APIKEY @@ -84,7 +84,7 @@ services: image: node:18.17.1 container_name: ezunpaywall-enrich-dev environment: - - NODE_ENV=development + - NODE_ENV # redis - REDIS_HOST - REDIS_PORT @@ -117,7 +117,7 @@ services: image: node:18.17.1 container_name: ezunpaywall-apikey-dev environment: - - NODE_ENV=development + - NODE_ENV # redis - REDIS_HOST - REDIS_PORT @@ -182,7 +182,7 @@ services: - ${FRONTEND_PORT:-59706}:3000 - 24678:24678 environment: - - NODE_ENV=development + - NODE_ENV - NUXT_PUBLIC_ENVIRONMENT=development - NUXT_PUBLIC_ELASTIC_ENV=development - NUXT_PUBLIC_VERSION=development diff --git a/src/graphql/lib/middlewares/args.js b/src/graphql/lib/middlewares/args.js index dd856ba4..807c6ef0 100644 --- a/src/graphql/lib/middlewares/args.js +++ b/src/graphql/lib/middlewares/args.js @@ -2,35 +2,39 @@ const logger = require('../logger'); function getNumberOfDOI(req) { const patternBetweenBracket = /.*?(\[.*?\]).*?$/i; - const patternBetweenBracketQuery = /.*?(\[".*?"\]\)).*?$/i; + const patternBetweenBracketQuery = /.*?(\[".*?"\]).*?$/i; // BODY // { // query: 'query ($dois: [ID!]!) { GetByDOI(dois: $dois', // variables: { dois: [ '10.1186/s40510-015-0109-6' ] } // } - if (req?.body?.variables?.dois) { - return req.body.variables.dois.length; - } + if (req?.body?.variables?.dois) { return req.body.variables.dois.length; } // BODY // query: '{ GetByDOI(dois: ["10.1186/s40510-015-0109-6","Coin Coin"]) { doi, is_oa } }' if (req?.body?.query) { const match = patternBetweenBracketQuery.exec(req?.body?.query); + let parsedMatch; if (match?.length >= 1) { - let listOfDOI; try { - listOfDOI = match[1].split(',').length; + parsedMatch = JSON.parse(match[1]); + return parsedMatch.length; } catch (err) { - logger.error(`[Apollo]: Cannot parse [${match[1]}]`); + logger.error(`[express] Cannot parse [${match}]`); + return 0; } - return listOfDOI.length; } } // query: '{ GetByDOI(dois:["10.1186/s40510-015-0109-6"] if (req?.query?.query) { const match = patternBetweenBracket.exec(req.query.query); if (match?.length >= 1) { - const listOfDOI = JSON.parse(match[1]); - return listOfDOI.length; + try { + const parsedMatch = JSON.parse(match[1]); + return parsedMatch.length; + } catch (err) { + logger.error(`[express] Cannot parse [${match}]`); + return 0; + } } } From 9f67d89c5642e9da258a4a85c0fe0276e8df8f3a Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Tue, 27 Feb 2024 08:09:39 +0100 Subject: [PATCH 053/114] test: add test for countDOI --- src/graphql/README.md | 3 +- src/graphql/app.js | 2 +- .../lib/middlewares/{args.js => countDOI.js} | 6 +- src/graphql/test/countDOI.js | 107 ++++++++++++++++++ 4 files changed, 113 insertions(+), 5 deletions(-) rename src/graphql/lib/middlewares/{args.js => countDOI.js} (81%) create mode 100644 src/graphql/test/countDOI.js diff --git a/src/graphql/README.md b/src/graphql/README.md index 72668bc7..8f44017a 100644 --- a/src/graphql/README.md +++ b/src/graphql/README.md @@ -98,5 +98,6 @@ One cron automatically update metrics of unpaywall data. the elastic request tak # Functional tests npm run test # Unit test -# it's your turn to play +docker compose exec graphql npm run test ./test/countDOI.js +# For the rest, it's your turn to play ``` \ No newline at end of file diff --git a/src/graphql/app.js b/src/graphql/app.js index 44986e08..a0356fff 100644 --- a/src/graphql/app.js +++ b/src/graphql/app.js @@ -11,7 +11,7 @@ const { expressMiddleware } = require('@apollo/server/express4'); const { pingRedis, startConnectionRedis } = require('./lib/services/redis'); -const getNumberOfDOI = require('./lib/middlewares/args'); +const getNumberOfDOI = require('./lib/middlewares/countDOI'); const auth = require('./lib/middlewares/auth'); const logger = require('./lib/logger'); diff --git a/src/graphql/lib/middlewares/args.js b/src/graphql/lib/middlewares/countDOI.js similarity index 81% rename from src/graphql/lib/middlewares/args.js rename to src/graphql/lib/middlewares/countDOI.js index 807c6ef0..a8affcaf 100644 --- a/src/graphql/lib/middlewares/args.js +++ b/src/graphql/lib/middlewares/countDOI.js @@ -5,12 +5,12 @@ function getNumberOfDOI(req) { const patternBetweenBracketQuery = /.*?(\[".*?"\]).*?$/i; // BODY // { - // query: 'query ($dois: [ID!]!) { GetByDOI(dois: $dois', + // query: 'query ($dois: [ID!]!) { unpaywall(dois: $dois) { doi, is_oa } }', // variables: { dois: [ '10.1186/s40510-015-0109-6' ] } // } if (req?.body?.variables?.dois) { return req.body.variables.dois.length; } // BODY - // query: '{ GetByDOI(dois: ["10.1186/s40510-015-0109-6","Coin Coin"]) { doi, is_oa } }' + // query: '{ unpaywall(dois: ["10.1186/s40510-015-0109-6","Coin Coin"]) { doi, is_oa } }' if (req?.body?.query) { const match = patternBetweenBracketQuery.exec(req?.body?.query); let parsedMatch; @@ -24,7 +24,7 @@ function getNumberOfDOI(req) { } } } - // query: '{ GetByDOI(dois:["10.1186/s40510-015-0109-6"] + // query: '{ unpaywall(dois:["10.1186/s40510-015-0109-6"]) { doi, is_oa } }' if (req?.query?.query) { const match = patternBetweenBracket.exec(req.query.query); if (match?.length >= 1) { diff --git a/src/graphql/test/countDOI.js b/src/graphql/test/countDOI.js new file mode 100644 index 00000000..8cb79916 --- /dev/null +++ b/src/graphql/test/countDOI.js @@ -0,0 +1,107 @@ +const { expect } = require('chai'); + +const getNumberOfDOI = require('../lib/middlewares/countDOI'); + +describe('Test logger of graphql API', () => { + describe('GET: unpaywall graphql', () => { + const req1 = { query: {}, body: {} }; + const query1 = '{ unpaywall(dois:["1"]) { doi, is_oa } }'; + + const req2 = { query: {}, body: {} }; + const query2 = '{ unpaywall(dois:["1", "2"]) { doi, is_oa } }'; + + const req3 = { query: {}, body: {} }; + const query3 = '{ unpaywall(dois:[]) { doi, is_oa } }'; + + before(() => { + req1.query.query = query1; + req2.query.query = query2; + req3.query.query = query3; + }); + + it('Should get 1', async () => { + const count = getNumberOfDOI(req1); + expect(count).eql(1); + }); + + it('Should get 2', async () => { + const count = getNumberOfDOI(req2); + expect(count).eql(2); + }); + + it('Should get 0', async () => { + const count = getNumberOfDOI(req3); + expect(count).eql(0); + }); + }); + + describe('POST: unpaywall graphql', () => { + const req1 = { query: {}, body: {} }; + const query1 = '{ unpaywall(dois: ["1"]) { doi, is_oa } }'; + + const req2 = { query: {}, body: {} }; + const query2 = '{ unpaywall(dois: ["1", "2"]) { doi, is_oa } }'; + + const req3 = { query: {}, body: {} }; + const query3 = '{ unpaywall(dois: []) { doi, is_oa } }'; + + before(() => { + req1.body.query = query1; + req2.body.query = query2; + req3.body.query = query3; + }); + + it('Should get 1', async () => { + const count = getNumberOfDOI(req1); + expect(count).eql(1); + }); + + it('Should get 2', async () => { + const count = getNumberOfDOI(req2); + expect(count).eql(2); + }); + + it('Should get 0', async () => { + const count = getNumberOfDOI(req3); + expect(count).eql(0); + }); + }); + + describe('POST: unpaywall graphql other syntax', () => { + const req1 = { query: {}, body: {} }; + const query = 'query ($dois: [ID!]!) { unpaywall(dois: $dois) { doi, is_oa } }'; + const variables1 = { dois: ['1'] }; + + const req2 = { query: {}, body: {} }; + const variables2 = { dois: ['1', '2'] }; + + const req3 = { query: {}, body: {} }; + const variables3 = { dois: [] }; + + before(() => { + req1.body.query = query; + req1.body.variables = variables1; + + req2.body.query = query; + req2.body.variables = variables2; + + req3.body.query = query; + req3.body.variables = variables3; + }); + + it('Should get 1', async () => { + const count = getNumberOfDOI(req1); + expect(count).eql(1); + }); + + it('Should get 2', async () => { + const count = getNumberOfDOI(req2); + expect(count).eql(2); + }); + + it('Should get 0', async () => { + const count = getNumberOfDOI(req3); + expect(count).eql(0); + }); + }); +}); From 5e845db5cc5d6ef6a6ed6226c394ea6ef61ec740 Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Tue, 27 Feb 2024 13:33:07 +0100 Subject: [PATCH 054/114] chore: update version of node and redis --- docker-compose.debug.yml | 18 +++++++++--------- docker-compose.yml | 2 +- src/apikey/Dockerfile | 2 +- src/enrich/Dockerfile | 2 +- src/frontend/Dockerfile | 4 ++-- src/frontend/package-lock.json | 8 ++++---- src/graphql/Dockerfile | 2 +- src/health/Dockerfile | 2 +- src/mail/Dockerfile | 2 +- src/update/Dockerfile | 2 +- 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/docker-compose.debug.yml b/docker-compose.debug.yml index f4c24700..4c9e87cb 100644 --- a/docker-compose.debug.yml +++ b/docker-compose.debug.yml @@ -2,7 +2,7 @@ version: "3.3" services: graphql: - image: node:18.17.1 + image: node:18.19.1 container_name: ezunpaywall-graphql-dev environment: - NODE_ENV @@ -37,7 +37,7 @@ services: retries: 5 update: - image: node:18.17.1 + image: node:18.19.1 container_name: ezunpaywall-update-dev environment: - NODE_ENV @@ -81,7 +81,7 @@ services: retries: 5 enrich: - image: node:18.17.1 + image: node:18.19.1 container_name: ezunpaywall-enrich-dev environment: - NODE_ENV @@ -114,7 +114,7 @@ services: retries: 5 apikey: - image: node:18.17.1 + image: node:18.19.1 container_name: ezunpaywall-apikey-dev environment: - NODE_ENV @@ -143,7 +143,7 @@ services: retries: 5 mail: - image: node:18.17.1 + image: node:18.19.1 container_name: ezunpaywall-mail-dev environment: - NODE_CONFIG @@ -172,7 +172,7 @@ services: retries: 5 frontend: - image: node:18.17.1 + image: node:18.19.1 container_name: ezunpaywall-frontend-dev working_dir: /usr/src/app volumes: @@ -206,7 +206,7 @@ services: retries: 5 redis: - image: redis:7.2.1 + image: redis:7.2.4 container_name: ezunpaywall-redis-dev command: redis-server --requirepass "${REDIS_PASSWORD:-changeme}" volumes: @@ -225,7 +225,7 @@ services: retries: 5 health: - image: node:18.17-alpine3.18 + image: node:18.19.1-alpine3.19 container_name: ezunpaywall-health-dev working_dir: /usr/src/app volumes: @@ -305,7 +305,7 @@ services: fakeunpaywall: container_name: ezunpaywall-fakeunpaywall-dev - image: node:18.17.1 + image: node:18.19.1 working_dir: /usr/src/app volumes: - ./src/fakeUnpaywall:/usr/src/app diff --git a/docker-compose.yml b/docker-compose.yml index 2441fe25..314d30c2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -207,7 +207,7 @@ services: retries: 3 redis: - image: redis:7.2.1-alpine3.18 + image: redis:7.2.4-alpine3.19 container_name: ezunpaywall-redis-prod command: redis-server --requirepass "${REDIS_PASSWORD}" volumes: diff --git a/src/apikey/Dockerfile b/src/apikey/Dockerfile index bcf3f763..118b5d65 100644 --- a/src/apikey/Dockerfile +++ b/src/apikey/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18.17-alpine3.18 +FROM node:18.19.1-alpine3.19 LABEL maintainer="ezTeam " EXPOSE 3000 diff --git a/src/enrich/Dockerfile b/src/enrich/Dockerfile index bcf3f763..118b5d65 100644 --- a/src/enrich/Dockerfile +++ b/src/enrich/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18.17-alpine3.18 +FROM node:18.19.1-alpine3.19 LABEL maintainer="ezTeam " EXPOSE 3000 diff --git a/src/frontend/Dockerfile b/src/frontend/Dockerfile index 67423627..6f00ebb4 100644 --- a/src/frontend/Dockerfile +++ b/src/frontend/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18.17-alpine3.18 as build +FROM node:18.19.1-alpine3.19 as build LABEL maintainer="ezTeam " @@ -27,7 +27,7 @@ COPY . . RUN npm run build -FROM node:18.17-alpine3.18 +FROM node:18.19.1-alpine3.19 EXPOSE 3000 ENV NODE_ENV production COPY --from=build /usr/src/app/.output /usr/src/app diff --git a/src/frontend/package-lock.json b/src/frontend/package-lock.json index e38fc008..2e375b27 100644 --- a/src/frontend/package-lock.json +++ b/src/frontend/package-lock.json @@ -3888,8 +3888,8 @@ "dev": true }, "node_modules/@types/node": { - "version": "18.17.12", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.17.12.tgz", + "version": "18.19.12", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.12.tgz", "integrity": "sha512-d6xjC9fJ/nSnfDeU0AMDsaJyb1iHsqCSOdi84w4u+SlN/UgQdY5tRhpMzaFYsI4mnpvgTivEaQd0yOUhAtOnEQ==", "dev": true }, @@ -19480,8 +19480,8 @@ "dev": true }, "@types/node": { - "version": "18.17.12", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.17.12.tgz", + "version": "18.19.12", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.12.tgz", "integrity": "sha512-d6xjC9fJ/nSnfDeU0AMDsaJyb1iHsqCSOdi84w4u+SlN/UgQdY5tRhpMzaFYsI4mnpvgTivEaQd0yOUhAtOnEQ==", "dev": true }, diff --git a/src/graphql/Dockerfile b/src/graphql/Dockerfile index bcf3f763..118b5d65 100644 --- a/src/graphql/Dockerfile +++ b/src/graphql/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18.17-alpine3.18 +FROM node:18.19.1-alpine3.19 LABEL maintainer="ezTeam " EXPOSE 3000 diff --git a/src/health/Dockerfile b/src/health/Dockerfile index bcf3f763..118b5d65 100644 --- a/src/health/Dockerfile +++ b/src/health/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18.17-alpine3.18 +FROM node:18.19.1-alpine3.19 LABEL maintainer="ezTeam " EXPOSE 3000 diff --git a/src/mail/Dockerfile b/src/mail/Dockerfile index bcf3f763..118b5d65 100644 --- a/src/mail/Dockerfile +++ b/src/mail/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18.17-alpine3.18 +FROM node:18.19.1-alpine3.19 LABEL maintainer="ezTeam " EXPOSE 3000 diff --git a/src/update/Dockerfile b/src/update/Dockerfile index bcf3f763..118b5d65 100644 --- a/src/update/Dockerfile +++ b/src/update/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18.17-alpine3.18 +FROM node:18.19.1-alpine3.19 LABEL maintainer="ezTeam " EXPOSE 3000 From bd44ebea984d8f7426f54b7e22f8339e1fd9e984 Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Tue, 27 Feb 2024 14:07:09 +0100 Subject: [PATCH 055/114] chore: update package-lock.json --- src/frontend/Dockerfile | 2 +- src/frontend/package-lock.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/frontend/Dockerfile b/src/frontend/Dockerfile index 6f00ebb4..e4f6c6dd 100644 --- a/src/frontend/Dockerfile +++ b/src/frontend/Dockerfile @@ -21,7 +21,7 @@ WORKDIR /usr/src/app COPY package*.json ./ -RUN npm i +RUN npm ci COPY . . diff --git a/src/frontend/package-lock.json b/src/frontend/package-lock.json index 2e375b27..1abaecbd 100644 --- a/src/frontend/package-lock.json +++ b/src/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "client", - "version": "1.3.0", + "version": "1.3.5", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "client", - "version": "1.3.0", + "version": "1.3.5", "hasInstallScript": true, "dependencies": { "@highlightjs/vue-plugin": "^2.1.0", From cb16cd6cd94af40927b19b75a048b314a6f67574 Mon Sep 17 00:00:00 2001 From: felixleo22 Date: Wed, 28 Feb 2024 08:48:06 +0100 Subject: [PATCH 056/114] fix: update vuetify --- src/frontend/.eslintrc.json | 3 +- .../components/report/ReportsDataTable.vue | 16 +- src/frontend/package-lock.json | 33277 +++++----------- src/frontend/package.json | 4 +- src/frontend/plugins/vuetify.js | 6 - 5 files changed, 9193 insertions(+), 24113 deletions(-) diff --git a/src/frontend/.eslintrc.json b/src/frontend/.eslintrc.json index f5696740..96066e9f 100644 --- a/src/frontend/.eslintrc.json +++ b/src/frontend/.eslintrc.json @@ -25,6 +25,7 @@ ], "rules": { "import/extensions": "off", - "import/no-unresolved": "off" + "import/no-unresolved": "off", + "eslint-plugin-vue/valid-v-slot": "off" } } \ No newline at end of file diff --git a/src/frontend/components/report/ReportsDataTable.vue b/src/frontend/components/report/ReportsDataTable.vue index c278eb89..81f5392f 100644 --- a/src/frontend/components/report/ReportsDataTable.vue +++ b/src/frontend/components/report/ReportsDataTable.vue @@ -10,43 +10,43 @@ > -