From 98d8fe7baf9a63b7540ef75542556cd2fb6ece79 Mon Sep 17 00:00:00 2001 From: Antoine Hurard Date: Tue, 6 Feb 2024 09:59:31 +0100 Subject: [PATCH 1/4] fix: reference data resolvers could break schema --- src/utils/schema/resolvers/Entity/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/utils/schema/resolvers/Entity/index.ts b/src/utils/schema/resolvers/Entity/index.ts index d96db720d..ed261ce7f 100644 --- a/src/utils/schema/resolvers/Entity/index.ts +++ b/src/utils/schema/resolvers/Entity/index.ts @@ -305,6 +305,8 @@ export const getEntityResolver = ( return Object.assign(resolvers, { [field.name]: getReferenceDataResolver(field, referenceData), }); + } else { + return resolvers; } }, {}); From 1a31890fb715e5ddb895ecbf4be30e6919756d8e Mon Sep 17 00:00:00 2001 From: Antoine Hurard Date: Tue, 6 Feb 2024 15:33:51 +0100 Subject: [PATCH 2/4] add first version of the pipeline for sit --- .devops/sit-CD.yml | 61 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .devops/sit-CD.yml diff --git a/.devops/sit-CD.yml b/.devops/sit-CD.yml new file mode 100644 index 000000000..3f094952c --- /dev/null +++ b/.devops/sit-CD.yml @@ -0,0 +1,61 @@ +# Node.js +# Build a general Node.js project with npm. +# Add steps that analyze code, save build artifacts, deploy, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/javascript + +trigger: +- stage + +pool: + vmImage: ubuntu-latest + +steps: +- task: NodeTool@0 + inputs: + versionSpec: '18.x' + displayName: 'Install Node.js' + +- script: | + npm install + npm run build + displayName: 'npm install and build' + +- task: ArchiveFiles@2 + inputs: + rootFolderOrFile: '$(System.DefaultWorkingDirectory)/build' + includeRootFolder: false + archiveType: 'zip' + archiveFile: '$(Build.ArtifactStagingDirectory)/sit/api/$(Build.BuildId).zip' + replaceExistingArchive: true + +- task: ArchiveFiles@2 + inputs: + rootFolderOrFile: '$(System.DefaultWorkingDirectory)/node_modules' + includeRootFolder: false + archiveType: 'zip' + archiveFile: '$(Build.ArtifactStagingDirectory)/sit/api/$(Build.BuildId).zip' + replaceExistingArchive: false + +- task: ArchiveFiles@2 + inputs: + rootFolderOrFile: '$(System.DefaultWorkingDirectory)/config' + includeRootFolder: false + archiveType: 'zip' + archiveFile: '$(Build.ArtifactStagingDirectory)/sit/api/$(Build.BuildId).zip' + replaceExistingArchive: false + +- task: ArchiveFiles@2 + inputs: + rootFolderOrFile: '$(System.DefaultWorkingDirectory)/package.json' + includeRootFolder: false + archiveType: 'zip' + archiveFile: '$(Build.ArtifactStagingDirectory)/sit/api/$(Build.BuildId).zip' + replaceExistingArchive: false + +- task: ArchiveFiles@2 + inputs: + rootFolderOrFile: '$(System.DefaultWorkingDirectory)/package-lock.json' + includeRootFolder: false + archiveType: 'zip' + archiveFile: '$(Build.ArtifactStagingDirectory)/sit/api/$(Build.BuildId).zip' + replaceExistingArchive: false From 650ff216d775136579c428646edb1db1ffc30f7a Mon Sep 17 00:00:00 2001 From: Antoine Hurard Date: Tue, 6 Feb 2024 15:53:22 +0100 Subject: [PATCH 3/4] add dev devops cd --- .devops/dev-CD.yml | 61 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .devops/dev-CD.yml diff --git a/.devops/dev-CD.yml b/.devops/dev-CD.yml new file mode 100644 index 000000000..907e8d0e1 --- /dev/null +++ b/.devops/dev-CD.yml @@ -0,0 +1,61 @@ +# Node.js +# Build a general Node.js project with npm. +# Add steps that analyze code, save build artifacts, deploy, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/javascript + +trigger: +- dev + +pool: + vmImage: ubuntu-latest + +steps: +- task: NodeTool@0 + inputs: + versionSpec: '18.x' + displayName: 'Install Node.js' + +- script: | + npm install + npm run build + displayName: 'npm install and build' + +- task: ArchiveFiles@2 + inputs: + rootFolderOrFile: '$(System.DefaultWorkingDirectory)/build' + includeRootFolder: false + archiveType: 'zip' + archiveFile: '$(Build.ArtifactStagingDirectory)/sit/api/$(Build.BuildId).zip' + replaceExistingArchive: true + +- task: ArchiveFiles@2 + inputs: + rootFolderOrFile: '$(System.DefaultWorkingDirectory)/node_modules' + includeRootFolder: false + archiveType: 'zip' + archiveFile: '$(Build.ArtifactStagingDirectory)/sit/api/$(Build.BuildId).zip' + replaceExistingArchive: false + +- task: ArchiveFiles@2 + inputs: + rootFolderOrFile: '$(System.DefaultWorkingDirectory)/config' + includeRootFolder: false + archiveType: 'zip' + archiveFile: '$(Build.ArtifactStagingDirectory)/sit/api/$(Build.BuildId).zip' + replaceExistingArchive: false + +- task: ArchiveFiles@2 + inputs: + rootFolderOrFile: '$(System.DefaultWorkingDirectory)/package.json' + includeRootFolder: false + archiveType: 'zip' + archiveFile: '$(Build.ArtifactStagingDirectory)/sit/api/$(Build.BuildId).zip' + replaceExistingArchive: false + +- task: ArchiveFiles@2 + inputs: + rootFolderOrFile: '$(System.DefaultWorkingDirectory)/package-lock.json' + includeRootFolder: false + archiveType: 'zip' + archiveFile: '$(Build.ArtifactStagingDirectory)/sit/api/$(Build.BuildId).zip' + replaceExistingArchive: false From 45f1c898908b41e41326ee8c741a0ca4ed68833e Mon Sep 17 00:00:00 2001 From: Antoine Hurard <50317271+AntoineRelief@users.noreply.github.com> Date: Wed, 7 Feb 2024 13:21:18 +0100 Subject: [PATCH 4/4] refactor: cache ability for a few minutes in records query + add more elements to search index (#972) --------- Co-authored-by: AzureAD\DanielVedanayagam --- src/utils/schema/resolvers/Query/all.ts | 35 ++++++---- .../schema/resolvers/Query/getSearchFilter.ts | 65 +++++++++++-------- 2 files changed, 63 insertions(+), 37 deletions(-) diff --git a/src/utils/schema/resolvers/Query/all.ts b/src/utils/schema/resolvers/Query/all.ts index d42cca234..5fb8d6a24 100644 --- a/src/utils/schema/resolvers/Query/all.ts +++ b/src/utils/schema/resolvers/Query/all.ts @@ -1,5 +1,5 @@ import { GraphQLError } from 'graphql'; -import { Form, Record, ReferenceData, User } from '@models'; +import { Record, ReferenceData, User } from '@models'; import extendAbilityForRecords from '@security/extendAbilityForRecords'; import { decodeCursor, encodeCursor } from '@schema/types'; import getReversedFields from '../../introspection/getReversedFields'; @@ -19,10 +19,15 @@ import checkPageSize from '@utils/schema/errors/checkPageSize.util'; import { flatten, get, isArray, set } from 'lodash'; import { accessibleBy } from '@casl/mongoose'; import { graphQLAuthCheck } from '@schema/shared'; +import NodeCache from 'node-cache'; +import { AppAbility } from '@security/defineUserAbility'; /** Default number for items to get */ const DEFAULT_FIRST = 25; +/** Ability Cache, based on user id, time to live: 5min */ +const abilityCache = new NodeCache({ stdTTL: 60 * 5, checkperiod: 60 }); + // todo: improve by only keeping used fields in the $project stage /** * Project aggregation. @@ -233,6 +238,7 @@ export default (entityName: string, fieldsByName: any, idsByName: any) => checkPageSize(first); try { const user: User = context.user; + const userId = user._id.toString(); // Id of the form / resource const id = idsByName[entityName]; // List of form / resource fields @@ -407,16 +413,23 @@ export default (entityName: string, fieldsByName: any, idsByName: any) => }; // Additional filter from the user permissions - const form = await Form.findOne({ - $or: [{ _id: id }, { resource: id, core: true }], - }) - .select('_id permissions fields') - .populate({ path: 'resource', model: 'Resource' }); - const ability = await extendAbilityForRecords(user, form); - set(context, 'user.ability', ability); - const permissionFilters = Record.find( - accessibleBy(ability, 'read').Record - ).getFilter(); + let permissionFilters; + // Try to get ability from cache + let ability = abilityCache.get(userId); + if (!ability) { + // If not available, build ability + ability = await extendAbilityForRecords(user); + set(context, 'user.ability', ability); + permissionFilters = Record.find( + accessibleBy(ability, 'read').Record + ).getFilter(); + // And cache it + abilityCache.set(userId, ability); + } else { + permissionFilters = Record.find( + accessibleBy(ability, 'read').Record + ).getFilter(); + } // Finally putting all filters together const filters = { diff --git a/src/utils/schema/resolvers/Query/getSearchFilter.ts b/src/utils/schema/resolvers/Query/getSearchFilter.ts index 47eeb2c93..bcf5ca2f2 100644 --- a/src/utils/schema/resolvers/Query/getSearchFilter.ts +++ b/src/utils/schema/resolvers/Query/getSearchFilter.ts @@ -1,6 +1,4 @@ -// import { isNumber } from 'lodash'; import mongoose from 'mongoose'; -// import { MULTISELECT_TYPES, DATE_TYPES } from '@const/fieldTypes'; /** The default fields */ const DEFAULT_FIELDS = [ @@ -31,7 +29,7 @@ const DEFAULT_FIELDS = [ ]; /** - * + * Wild card search interface. */ interface WildcardSearch { wildcard: { @@ -46,7 +44,7 @@ interface WildcardSearch { } /** - * + * Search stage interface. */ interface SearchStage { $search: { @@ -57,18 +55,6 @@ interface SearchStage { }; } -/** - * - */ -let searchStage: SearchStage = { - $search: { - index: 'data_keyword_lowercase', - compound: { - must: [], - }, - }, -}; - /** Names of the default fields */ // eslint-disable-next-line @typescript-eslint/naming-convention export const FLAT_DEFAULT_FIELDS = DEFAULT_FIELDS.map((x) => x.name); @@ -93,8 +79,6 @@ export const extractFilterFields = (filter: any): string[] => { return fields; }; -let searchStageUsed = false; - /** * Transforms query filter into mongo filter. * @@ -102,17 +86,21 @@ let searchStageUsed = false; * @param fields list of structure fields * @param context request context * @param prefix prefix to access field + * @param searchStage Search stage being built * @returns Mongo filter. */ const buildMongoFilter = ( filter: any, fields: any[], context: any, - prefix = '' + prefix = '', + searchStage ): any => { if (filter.filters) { const filters = filter.filters - .map((x: any) => buildMongoFilter(x, fields, context, prefix)) + .map((x: any) => + buildMongoFilter(x, fields, context, prefix, searchStage) + ) .filter((x) => x); if (filters.length > 0) { switch (filter.logic) { @@ -253,7 +241,6 @@ const buildMongoFilter = ( allowAnalyzedField: true, }, }); - searchStageUsed = true; return; } } else { @@ -264,11 +251,36 @@ const buildMongoFilter = ( allowAnalyzedField: true, }, }); - searchStageUsed = true; return; } } } + case 'startswith': { + if (fieldName.includes('id')) { + return; + } + searchStage.$search.compound.must.unshift({ + wildcard: { + query: `${value}*`, + path: fieldName, + allowAnalyzedField: true, + }, + }); + return; + } + case 'endswith': { + if (fieldName.includes('id')) { + return; + } + searchStage.$search.compound.must.unshift({ + wildcard: { + query: `*${value}`, + path: fieldName, + allowAnalyzedField: true, + }, + }); + return; + } default: { return; } @@ -295,8 +307,8 @@ export default ( context?: any, prefix = 'data.' ) => { - searchStageUsed = false; - searchStage = { + // Default search stage + const searchStage: SearchStage = { $search: { index: 'keyword_lowercase', compound: { @@ -305,9 +317,10 @@ export default ( }, }; const expandedFields = fields.concat(DEFAULT_FIELDS); - buildMongoFilter(filter, expandedFields, context, prefix); + buildMongoFilter(filter, expandedFields, context, prefix, searchStage); - if (searchStageUsed) { + // If some rules are defined, return search stage, to be added to main pipeline + if (searchStage.$search.compound.must.length > 0) { return searchStage; } return;