diff --git a/README.md b/README.md index 39051af..c5089fa 100644 --- a/README.md +++ b/README.md @@ -34,14 +34,22 @@ Update config to fetch ez-unpyawall. | --- | --- | | -g --get | display the configuration | | -s --set | initialize the configuration file in $HOME/.config | -| -u --url | ezunpaywall url | -| -p --port | ezunpaywall port | +| --ezunpaywallURL | ezunpaywall url | +| --ezunpaywallPort | ezunpaywall port | +| --ezmetaURL | ezmeta url | +| --ezmetaPort | ezmeta port | +| --ezmetaUser | ezmeta url | +| --ezmetaPassword | ezmeta password | | -k --apikey | admin apikey | | -l --list | list of attributes required for configuration | #### Examples ```bash -$ ezunpaywall config --url http://localhost -$ ezunpaywall config --port 8080 +$ ezunpaywall config --ezunpaywallURL http://localhost +$ ezunpaywall config --ezunpaywallPort 8080 +$ ezunpaywall config --ezmetaURL http://localhost +$ ezunpaywall config --ezmetaPort 9200 +$ ezunpaywall config --ezmetaUser elastic +$ ezunpaywall config --ezmetaPassword changeme $ ezunpaywall config --apikey admin ``` ### ezu ping diff --git a/bin/cmds/config.js b/bin/cmds/config.js index a1c58aa..50cd355 100644 --- a/bin/cmds/config.js +++ b/bin/cmds/config.js @@ -8,67 +8,116 @@ const os = require('os'); */ const setConfig = async () => { const pathConfig = path.resolve(os.homedir(), '.config', '.ezunpaywallrc'); + const config = { - url: 'http://localhost', - port: 8080, + ezunpaywallURL: 'http://localhost', + ezunpaywallPort: 8080, + ezmetaURL: 'http://localhost', + ezmetaPort: 9200, + ezmetaUser: 'elastic', + ezmetaPassword: 'changeme', + apikey: 'admin', }; + try { await fs.writeFile(pathConfig, JSON.stringify(config, null, 2), 'utf8'); } catch (err) { console.error(err); } + console.log(`configuration has been initialized in ${pathConfig}`); console.log(JSON.stringify(config, null, 2)); }; /** * config management command to establish the connection between the command and ezunpaywall - * @param {Object} args commander arguments - * @param -g --get - display the configuration - * @param -s --set - initialize the configuration file in $HOME/.config - * @param --url - ezunpaywall url - * @param --port - ezunpaywall port - * @param --apikey - admin apikey - * @param -l --list - list of attributes required for configuration + * + * @param {boolean} args.get -g --get - display the configuration + * @param {boolean} args.set -s --set - initialize the configuration file in $HOME/.config + * @param {string} args.ezunpaywallURL --ezunpaywallURL - ezunpaywall url + * @param {number} args.ezunpaywallPort --ezunpaywallPort - ezunpaywall port + * @param {string} args.ezmetaURL --ezmetalURL - ezmetal url + * @param {number} args.ezmetaPort --ezmetalPort - ezmetal port + * @param {string} args.apikey --apikey - admin apikey + * @param {boolean} args.list -l --list - list of attributes required for configuration */ const manageConfig = async (args) => { if (args.list) { - console.log('--url ezunpaywall url'); - console.log('--port ezunpaywall port'); + console.log('--ezunpaywallURL ezunpaywall url'); + console.log('--ezunpaywallPort ezunpaywall port'); + console.log('--ezmetaURL ezmeta url'); + console.log('--ezmetaPort ezmeta port'); + console.log('--ezmetaUser ezmeta port'); + console.log('--ezmetaPassword ezmeta port'); + console.log('-k --apikey admin apikey'); process.exit(0); } + const configPath = path.resolve(os.homedir(), '.config', '.ezunpaywallrc'); + if (!await fs.pathExists(configPath)) { await setConfig(); } + if (args.set) { await setConfig(); process.exit(0); } const config = JSON.parse(await fs.readFile(configPath, 'utf-8')); + if (args.get) { console.log(JSON.stringify(config, null, 2)); console.log(`from ${configPath}`); process.exit(0); } - if (args.url) { - const regexURL = /^(ftp|http|https):\/\/[^ "]+$/; - const valideURL = regexURL.test(args.url); - if (valideURL) { - config.url = args.url; + + if (args.ezunpaywallURL) { + const regexURL = /^(http|https):\/\/[^ "]+$/; + const isValidURL = regexURL.test(args.url); + if (isValidURL) { + config.ezunpaywallURL = args.ezunpaywallURL; } else { - console.error(`'${args.url}' is not a valide URL`); + console.error(`'${args.ezunpaywallURL}' is not a valide URL`); process.exit(1); } } - if (args.port) { - if (Number.isNaN(args.port)) { - console.error(`${args.port} is not a number`); + + if (args.ezmetaURL) { + const regexURL = /^(http|https):\/\/[^ "]+$/; + const isValidURL = regexURL.test(args.url); + if (isValidURL) { + config.ezmetaURL = args.ezmetaURL; + } else { + console.error(`'${args.ezmetaURL}' is not a valide URL`); process.exit(1); } - config.port = args.port; } + + if (args.ezunpaywallPort) { + if (Number.isNaN(args.ezunpaywallPort)) { + console.error(`${args.ezunpaywallPort} is not a number`); + process.exit(1); + } + config.ezunpaywallPort = args.ezunpaywallPort; + } + + if (args.ezmetaPort) { + if (Number.isNaN(args.ezmetaPort)) { + console.error(`${args.ezmetaPort} is not a number`); + process.exit(1); + } + config.ezmetaPort = args.ezmetaPort; + } + + if (args.ezmetaUser) { + config.ezmetaUser = args.ezmetaUser; + } + + if (args.ezmetaPassword) { + config.ezmetaPassword = args.ezmetaPassword; + } + if (args.apikey) { config.apikey = args.apikey; } diff --git a/bin/cmds/enrich.js b/bin/cmds/enrich.js index b91df8f..8c5d93f 100644 --- a/bin/cmds/enrich.js +++ b/bin/cmds/enrich.js @@ -1,37 +1,42 @@ /* eslint-disable no-await-in-loop */ /* eslint-disable camelcase */ +const axios = require('axios'); const fs = require('fs-extra'); const path = require('path'); const uuid = require('uuid'); -const { connection } = require('../../lib/axios'); const { getConfig } = require('../../lib/config'); /** * start a csv file enrichment - * @param {Object} args commander arguments - * @param -f --file - file which must be enriched - * @param -a --attributes - attributes which must be enriched - * in graphql format. By default, all attributes are added * - * @param -s --separator - separator of csv out file - * @param -o --out - name of enriched file - * @param -v --verbose - logs how much lines are enriched - * @param -u --use - use a custom config + * @param {string} args.file -f --file - file which must be enriched + * @param {string} args.attributes -a --attributes - attributes which must be enriched + * in graphql format. By default, all attributes are added + * @param {string} args.separator -s --separator - separator of csv out file + * @param {string} args.out -o --out - name of enriched file + * @param {boolean} args.verbose -v --verbose - logs how much lines are enriched + * @param {boolean} args.use -u --use - pathfile of custom config */ const enrichCSV = async (args) => { - const axios = await connection(args.use); const config = await getConfig(args.use); + + const ezunpaywall = `${config.ezunpaywallURL}:${config.ezunpaywallPort}`; + if (!args.file) { console.error('file expected'); process.exit(1); } + const file = path.resolve(args.file); const fileExist = await fs.pathExists(file); + if (!fileExist) { console.error('file not found'); process.exit(1); } + const ext = path.extname(file).substring(1); + if (ext !== 'csv') { console.error(`${ext} is not suported for enrichCSV. Required .csv`); process.exit(1); @@ -42,16 +47,18 @@ const enrichCSV = async (args) => { const query = { separator: ',', }; + if (args.separator) query.separator = args.separator; if (args.attributes) query.args = args.attributes; if (args.index) query.index = args.index; const id = uuid.v4(); const stat = await fs.stat(args.file); + try { await axios({ method: 'POST', - url: `/enrich/csv/${id}`, + url: `${ezunpaywall}/enrich/csv/${id}`, params: query, data: fs.createReadStream(args.file), headers: { @@ -67,20 +74,22 @@ const enrichCSV = async (args) => { } let res2; + while (!res2?.data?.state?.done) { res2 = await axios({ method: 'GET', - url: `/enrich/state/${id}.json`, + url: `${ezunpaywall}/enrich/state/${id}.json`, responseType: 'json', }); await new Promise((resolve) => setTimeout(resolve, 1000)); } let res3; + try { res3 = await axios({ method: 'GET', - url: `/enrich/${id}.csv`, + url: `${ezunpaywall}/enrich/${id}.csv`, responseType: 'stream', }); } catch (err) { @@ -94,25 +103,27 @@ const enrichCSV = async (args) => { /** * start a jsonl file enrichment - * @param {Object} args commander arguments - * @param -f --file - file which must be enriched - * @param -a --attributes - attributes which must be enriched + * @param {string} args.file -f --file - file which must be enriched + * @param {string} args.attributes -a --attributes - attributes which must be enriched * in graphql format. By default, all attributes are added * - * @param -o --out - name of enriched file. By default, the output file is named: out.jsonl - * @param -v --verbose - logs how much lines are enriched - * @param -u --use - use a custom config + * @param {string} args.out -o --out - name of enriched file + * @param {boolean} args.verbose -v --verbose - logs how much lines are enriched + * @param {boolean} args.use -u --use - pathfile of custom config */ const enrichJSON = async (args) => { - const axios = await connection(args.use); const config = await getConfig(args.use); + const ezunpaywall = `${config.ezunpaywallURL}:${config.ezunpaywallPort}`; + if (!args.file) { console.error('file expected'); process.exit(1); } + const file = path.resolve(args.file); const fileExist = await fs.pathExists(file); + if (!fileExist) { console.error('file not found'); process.exit(1); @@ -130,10 +141,11 @@ const enrichJSON = async (args) => { const id = uuid.v4(); const stat = await fs.stat(args.file); + try { await axios({ method: 'POST', - url: `/enrich/json/${id}`, + url: `${ezunpaywall}/enrich/json/${id}`, params: query, data: fs.createReadStream(args.file), headers: { @@ -149,20 +161,22 @@ const enrichJSON = async (args) => { } let res2; + while (!res2?.data?.state?.done) { res2 = await axios({ method: 'GET', - url: `/enrich/state/${id}.json`, + url: `${ezunpaywall}/enrich/state/${id}.json`, responseType: 'json', }); await new Promise((resolve) => setTimeout(resolve, 1000)); } let res3; + try { res3 = await axios({ method: 'GET', - url: `/enrich/${id}.jsonl`, + url: `${ezunpaywall}/enrich/${id}.jsonl`, responseType: 'stream', }); } catch (err) { @@ -171,7 +185,6 @@ const enrichJSON = async (args) => { } const writer = fs.createWriteStream(out); - res3.data.pipe(writer); }; diff --git a/bin/cmds/ping.js b/bin/cmds/ping.js index 571d7c9..0014c02 100644 --- a/bin/cmds/ping.js +++ b/bin/cmds/ping.js @@ -1,24 +1,56 @@ -const { connection } = require('../../lib/axios'); +const axios = require('axios'); +const { Client } = require('@elastic/elasticsearch'); +const { URL } = require('url'); const { getConfig } = require('../../lib/config'); /** * check if service is available - * @param {*} args commander arguments - * @param -u --use - use a custom config + * + * @param {boolean} args.use -u --use - pathfile of custom config */ const ping = async (args) => { - const axios = await connection(args.use); const config = await getConfig(args.use); + + const ezunpaywall = `${config.ezunpaywallURL}:${config.ezunpaywallPort}`; + const ezmeta = `${config.ezmetaURL}:${config.ezmetaPort}`; + try { await axios({ method: 'GET', - url: '/ping', + url: `${ezunpaywall}/ping`, }); } catch (err) { - console.error(`service unavailable ${config.url}:${config.port}`); + console.error(`service unavailable ${ezunpaywall}`); + process.exit(1); + } + + console.log('ping ezunpaywall: OK'); + + const client = new Client({ + node: { + url: new URL(ezmeta), + auth: { + username: config.ezmetaUser, + password: config.ezmetaPassword, + }, + }, + }); + + let ezmetaping; + + try { + ezmetaping = await client.ping(); + } catch (err) { + console.error(`service unavailable ${ezmeta}`); process.exit(1); } - console.log(`service available ${config.url}:${config.port}`); + + if (!ezmetaping?.statusCode !== 200) { + console.error(`service unavailable ${ezmeta}`); + process.exit(1); + } + + console.log('ping ezmeta: OK'); process.exit(0); }; diff --git a/bin/cmds/reports.js b/bin/cmds/reports.js index 85b1a04..a6b24bc 100644 --- a/bin/cmds/reports.js +++ b/bin/cmds/reports.js @@ -1,40 +1,44 @@ const inquirer = require('inquirer'); +const axios = require('axios'); -const { connection } = require('../../lib/axios'); const { getConfig } = require('../../lib/config'); /** * get list of report in ezunpaywall - * @param {object} axios - axios - * @param {object} config - config + * @param {Stringng} ezunpaywall - ezunpaywallURL * @returns {array} array of name of report */ -const getReports = async (axios, config) => { +const getReports = async (ezunpaywall) => { let res; + try { res = await axios({ method: 'GET', - url: '/update/snapshot', + url: `${ezunpaywall}/update/snapshot`, }); } catch (err) { - console.error(`service unavailable ${config.url}:${config.port}`); + console.error(`service unavailable ${ezunpaywall}`); process.exit(1); } + return res?.data; }; /** * get the content of report - * @param {*} args commander arguments - * @param -f --file - report file installed on ezunpaywall - * @param -l --list - list of reports generated by ezunpaywall - * @param -la --latest [latest] - boolean - * @param -s --status - status of report, success and error only accepted - * @param -u --use - use a custom config + * + * @param {string} args.file -f --file - report file installed on ezunpaywall + * @param {boolean} args.list -l --list - list of reports generated by ezunpaywall + * @param {boolean} args.latest -la --latest [latest] + * @param {string} args.status -s --status - status of report, + * success and error only accepted + * @param {boolean} args.use -u --use - pathfile of custom config */ const report = async (args) => { - const axios = await connection(args.use); const config = await getConfig(args.use); + + const ezunpaywall = `${config.ezunpaywallURL}:${config.ezunpaywallPort}`; + // check list and latest, file, if (args.list && args.latest) { console.error('option --latest is impossible to use with --list'); @@ -44,6 +48,7 @@ const report = async (args) => { console.error('option --file is impossible to use with --list'); process.exit(1); } + // check file and latest, status if (args.file && args.latest) { console.error('option --latest is impossible to use with --file'); @@ -63,11 +68,12 @@ const report = async (args) => { let res1; let url = ''; let query = {}; + if (!args.status && !args.latest) query = null; if (args.status) query.status = args.status; - // if -l --list + if (args.list) { - const reports = await getReports(axios, config); + const reports = await getReports(ezunpaywall); if (!reports?.length) { console.log('no reports available'); process.exit(0); @@ -90,11 +96,11 @@ const report = async (args) => { if (args.file) url += `/${args.file}`; if (args.latest) query.latest = args.latest; - // if -f --file -l --latest + try { res1 = await axios({ method: 'get', - url: `/update/report${url}`, + url: `${ezunpaywall}/update/report${url}`, params: query, }); } catch (err) { diff --git a/bin/cmds/status.js b/bin/cmds/status.js index 4dbbe08..0797b4d 100644 --- a/bin/cmds/status.js +++ b/bin/cmds/status.js @@ -1,19 +1,22 @@ -const { connection } = require('../../lib/axios'); +const axios = require('axios'); + const { getConfig } = require('../../lib/config'); /** * Indicates if an update process is running - * @param {Object} args commander arguments - * @param -u --use - use a custom config + * + * @param {boolean} args.use -u --use - pathfile of custom config */ const getStatus = async (args) => { - const axios = await connection(args.use); const config = await getConfig(args.use); + + const ezunpaywall = `${config.ezunpaywallURL}:${config.ezunpaywallPort}`; + let res; try { res = await axios({ method: 'get', - url: '/update/status', + url: `${ezunpaywall}/update/status`, }); } catch (err) { console.error(`service unavailable ${config.url}:${config.port}`); diff --git a/bin/cmds/update.js b/bin/cmds/update.js index 09e2794..c7da200 100644 --- a/bin/cmds/update.js +++ b/bin/cmds/update.js @@ -1,6 +1,6 @@ /* eslint-disable no-param-reassign */ +const axios = require('axios'); const inquirer = require('inquirer'); -const { connection } = require('../../lib/axios'); const { getConfig } = require('../../lib/config'); /** @@ -9,15 +9,15 @@ const { getConfig } = require('../../lib/config'); * @param {object} config - config * @returns {array} array of name of snapshot */ -const getFiles = async (axios, config) => { +const getFiles = async (ezunpaywall) => { let res; try { res = await axios({ method: 'GET', - url: '/update/snapshot', + url: `${ezunpaywall}/update/snapshot`, }); } catch (err) { - console.error(`service unavailable ${config.url}:${config.port}`); + console.error(`service unavailable ${ezunpaywall}`); process.exit(1); } return res?.data; @@ -25,19 +25,22 @@ const getFiles = async (axios, config) => { /** * Starts an unpaywall data update process - * @param {Object} args commander arguments - * @param -f --file - snapshot\'s file installed on ezunpaywall - * @param -l --list - list of snapshot installed on ezunpaywall - * @param -sd --startDate - start date to download and insert updates from unpaywall - * @param -ed --endDate - end date to download and insert updates from unpaywall - * @param -of --offset - line where processing will start - * @param -li --limit - line where processing will end - * @param -u --use - use a custom config + * + * @param {string} args.file -f --file - snapshot's file installed on ezunpaywall + * @param {boolean} args.list -l --list - list of snapshot installed on ezunpaywall + * @param {string} args.startDate -sd --startDate - start date to download and insert + * updates from unpaywall + * @param {string} args.endDate -ed --endDate - end date to download + * and insert updates from unpaywall + * @param {string} args.offset -of --offset - line where processing will start + * @param {string} args.limit -li --limit - line where processing will end + * @param {string} args.use -u --use - pathfile of custom config */ const update = async (args) => { - const axios = await connection(args.use); const config = await getConfig(args.use); - // check date and list + + const ezunpaywall = `${config.ezunpaywallURL}:${config.ezunpaywallPort}`; + if (args.list) { if (args.startDate) { console.error('option --startDate is impossible to use with --list'); @@ -114,7 +117,7 @@ const update = async (args) => { const query = {}; if (args.list) { - const snapshots = await getFiles(axios, config); + const snapshots = await getFiles(ezunpaywall); const snapshot = await inquirer.prompt([{ type: 'list', pageSize: 5, @@ -141,7 +144,7 @@ const update = async (args) => { try { res = await axios({ method: 'post', - url: `/update${url}`, + url: `${ezunpaywall}/update${url}`, params: query, headers: { api_key: config.apikey, diff --git a/bin/ezunpaywall b/bin/ezunpaywall index 2bc039c..aef9bdf 100755 --- a/bin/ezunpaywall +++ b/bin/ezunpaywall @@ -22,8 +22,12 @@ program .description('config management command to establish the connection between the command and ezunpaywall') .option('-g --get', 'display the configuration') .option('-s --set', 'initialize the configuration file in $HOME/.config') - .option('-u --url ', 'ezunpaywall url') - .option('-p --port ', 'ezunpaywall port') + .option('--ezunpaywallURL ', 'ezunpaywall url') + .option('--ezunpaywallPort ', 'ezunpaywall port') + .option('--ezmetaURL ', 'ezmeta url') + .option('--ezmetaPort ', 'ezmeta port') + .option('--ezmetaUser ', 'ezmeta user') + .option('--ezmetaPassword ', 'ezmeta password') .option('-k -apikey ', 'admin apikey') .option('-L --list', 'list of attributes required for configuration') .action((args) => manageConfig(args)); diff --git a/lib/axios.js b/lib/axios.js index e1f3bc8..0b6b593 100644 --- a/lib/axios.js +++ b/lib/axios.js @@ -7,7 +7,6 @@ module.exports = { connection: async (customPath) => { let configPath = path.resolve(os.homedir(), '.config', '.ezunpaywallrc'); if (!await fs.pathExists(configPath)) { - console.log('you do not set the default configuration'); const config = { url: 'http://localhost', port: 8080, @@ -17,7 +16,7 @@ module.exports = { } catch (err) { console.error(err); } - console.log(`it's initialized automaticaly at ${configPath}`); + console.log(`fileconfig it's initialized automaticaly at ${configPath}`); console.log(JSON.stringify(config, null, 2)); }