Merge Prod Release into Master

.vscode/settings.json

@@ -3,7 +3,7 @@
   "editor.defaultFormatter": "esbenp.prettier-vscode",
   "eslint.packageManager": "npm",
   "editor.codeActionsOnSave": {
-    "source.fixAll.eslint": true
+    "source.fixAll.eslint": "explicit"
   },
   "[javascript]": {
     "editor.formatOnSave": true

package.json

@@ -34,9 +34,11 @@
     "pre-push": "npm run coverage && npm run build && npm run test-i"
   },
   "dependencies": {
+    "@aws-sdk/client-dynamodb": "3.567.0",
+    "@aws-sdk/client-secrets-manager": "3.567.0",
+    "@aws-sdk/lib-dynamodb": "3.567.0",
     "@azure/msal-node": "1.16.0",
     "aws-lambda": "1.0.7",
-    "aws-sdk": "2.1330.0",
     "axios": "1.3.4",
     "dateformat": "5.0.3",
     "knex": "2.4.2",
@@ -57,6 +59,7 @@
     "@types/jest": "^29.4.0",
     "@types/node": "^18.14.6",
     "@types/supertest": "^2.0.12",
+    "aws-sdk-client-mock": "4.0.0",
     "commitlint-plugin-function-rules": "^1.7.1",
     "concurrently": "^7.6.0",
     "cross-env": "^7.0.3",

src/aad/getMemberDetails.ts

@@ -6,6 +6,7 @@ import IMemberDetails, { MemberType } from './IMemberDetails';
 import getToken from './getToken';
 
 interface MemberList {
+  '@odata.nextLink': string;
   value: IMemberDetails[];
 }
 
@@ -27,7 +28,7 @@ export const getMemberDetails = async (): Promise<IMemberDetails[]> => {
 
   const promiseArray = groupIds.map(async (groupId) => {
     const requestUrl = new URL(
-      `/v1.0/groups/${groupId.trim()}/members?$count=true&$top=999&$filter=accountEnabled eq true`,
+      `/v1.0/groups/${groupId.trim()}/members?$count=true&$top=${config.aad.membersToRequest}&$filter=accountEnabled eq true`,
       aadBase,
     ).href;
 
@@ -36,10 +37,23 @@ export const getMemberDetails = async (): Promise<IMemberDetails[]> => {
     // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
     axios.defaults.headers.common.Authorization = `Bearer ${accessToken}`;
 
-    const response = await axios.get<MemberList>(requestUrl, {
+    let response = await axios.get<MemberList>(requestUrl, {
       headers: { Authorization: `Bearer ${accessToken}`, ConsistencyLevel: 'eventual' },
     });
-    return response.data.value;
+
+    let users = response.data.value
+
+    while (response.data['@odata.nextLink']) {
+      logger.info(`pagination needed for group ${groupId.trim()}. Fetching next ${config.aad.membersToRequest} users`)
+
+      const nextlink = response.data['@odata.nextLink'].replace('accountEnabled+eq+true', 'accountEnabled eq true')
+      response = await axios.get<MemberList>(nextlink, {
+        headers: { Authorization: `Bearer ${accessToken}`, ConsistencyLevel: 'eventual' },
+      });
+      users = users.concat(response.data.value)
+    }
+
+    return users;
   });
 
   const results = await Promise.allSettled(promiseArray);

src/dynamo/getDynamoRecords.ts

@@ -1,24 +1,25 @@
-import * as AWS from 'aws-sdk';
 import config from '../config';
 import IDynamoRecord, { ResourceType } from './IDynamoRecord';
+import { DynamoDBDocumentClient, QueryCommandInput, QueryCommand } from "@aws-sdk/lib-dynamodb"
+import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
 
-const dynamo = new AWS.DynamoDB.DocumentClient();
+const dynamo = DynamoDBDocumentClient.from(new DynamoDBClient());
 
 export const getDynamoMembers: () => Promise<IDynamoRecord[]> = async () => {
   const result = await dynamo
-    .query({
-      TableName: config.aws.dynamoTable,
-      KeyConditionExpression: 'resourceType = :type',
-      FilterExpression: 'attribute_not_exists(#ttl_key) or #ttl_key = :null',
-      ExpressionAttributeValues: {
-        ':type': ResourceType.User,
-        ':null': null,
-      },
-      ExpressionAttributeNames: {
-        '#ttl_key': 'ttl',
-      },
-    } as AWS.DynamoDB.DocumentClient.QueryInput)
-    .promise();
-
-  return result.Items as IDynamoRecord[];
+    .send(new QueryCommand(
+      {
+        TableName: config.aws.dynamoTable,
+        KeyConditionExpression: 'resourceType = :type',
+        FilterExpression: 'attribute_not_exists(#ttl_key) or #ttl_key = :null',
+        ExpressionAttributeValues: {
+          ':type': ResourceType.User,
+          ':null': null,
+        },
+        ExpressionAttributeNames: {
+          '#ttl_key': 'ttl',
+        },
+      } as QueryCommandInput
+    ));
+  return result.Items as unknown as IDynamoRecord[];
 };

src/handler.ts

@@ -1,19 +1,19 @@
-import * as AWS from 'aws-sdk';
 import 'source-map-support/register';
 import IMemberDetails from './aad/IMemberDetails';
 import { getMemberDetails } from './aad/getMemberDetails';
 import config from './config';
 import IDynamoRecord, { ResourceType } from './dynamo/IDynamoRecord';
 import { getDynamoMembers } from './dynamo/getDynamoRecords';
 import logger from './observability/logger';
+import { DynamoDBDocumentClient, PutCommand, PutCommandInput } from "@aws-sdk/lib-dynamodb"
+import { DynamoDB } from "@aws-sdk/client-dynamodb";
 
 const { NODE_ENV, SERVICE, AWS_PROVIDER_REGION, AWS_PROVIDER_STAGE } = process.env;
 
 logger.info(
   `\nRunning Service:\n '${SERVICE}'\n mode: ${NODE_ENV}\n stage: '${AWS_PROVIDER_STAGE}'\n region: '${AWS_PROVIDER_REGION}'\n\n`,
 );
-
-const client = new AWS.DynamoDB.DocumentClient();
+const client = DynamoDBDocumentClient.from(new DynamoDB());
 
 const handler = async (): Promise<void> => {
   logger.info('Function triggered, getting member details...');
@@ -24,7 +24,7 @@ const handler = async (): Promise<void> => {
 
   logger.info(`Found ${dynamoList.length} existing dynamo records, generating and executing...`);
   const stmts = await Promise.allSettled(
-    generateStatements(azureList, dynamoList).map((stmt) => client.put(stmt).promise()),
+    generateStatements(azureList, dynamoList).map((stmt) => client.send(new PutCommand(stmt))),
   );
 
   stmts.filter((r) => r.status === 'rejected').map((r) => logger.error((<PromiseRejectedResult>r).reason));
@@ -35,10 +35,10 @@ const handler = async (): Promise<void> => {
 function generateStatements(
   azureMembers: IMemberDetails[],
   dynamoRecords: IDynamoRecord[],
-): AWS.DynamoDB.DocumentClient.PutItemInput[] {
+): PutCommandInput[] {
   const memberMap = azureMembers.map(
     (am) =>
-      <AWS.DynamoDB.DocumentClient.PutItemInput>{
+      <PutCommandInput>{
         TableName: config.aws.dynamoTable,
         Item: <IDynamoRecord>{
           resourceType: ResourceType.User,
@@ -60,7 +60,7 @@ function generateStatements(
     .filter((dr) => !azureMembers.some((am) => am.id === dr.resourceKey))
     .map(
       (dr) =>
-        <AWS.DynamoDB.DocumentClient.PutItemInput>{
+        <PutCommandInput>{
           TableName: config.aws.dynamoTable,
           Item: <IDynamoRecord>{
             resourceType: dr.resourceType,

src/utils/index.ts

@@ -1,10 +1,10 @@
-import { SecretsManager } from 'aws-sdk';
+import { SecretsManager } from '@aws-sdk/client-secrets-manager';
 import logger from '../observability/logger';
 
 const getSecret = async (secretName: string): Promise<string> => {
   logger.debug('getSecret starting.');
   const secretsManager = new SecretsManager();
-  const secretValue = await secretsManager.getSecretValue({ SecretId: secretName }).promise();
+  const secretValue = await secretsManager.getSecretValue({ SecretId: secretName });
   logger.debug('getSecret finishing.');
   return secretValue.SecretString;
 };

tests/unit/getDynamoRecords.test.ts

@@ -1,56 +1,42 @@
+import { DynamoDBDocumentClient, QueryCommandOutput, QueryCommand } from '@aws-sdk/lib-dynamodb';
+import { mockClient } from 'aws-sdk-client-mock';
 import config from '../../src/config';
-import { getDynamoMembers } from '../../src/dynamo/getDynamoRecords';
 import { ResourceType } from '../../src/dynamo/IDynamoRecord';
+import { getDynamoMembers } from '../../src/dynamo/getDynamoRecords';
 
-// has to be 'var' as jest "hoists" execution behind the scenes and let/const cause errors
-/* tslint:disable */
-var mockDynamoQuery: jest.Mock;
-/* tslint:enable */
-
-jest.mock('aws-sdk', () => {
-  mockDynamoQuery = jest.fn().mockImplementation(() => ({
-    promise: jest.fn().mockResolvedValue({
-      Items: [
-        {
-          resourceType: ResourceType.User,
-          resourceKey: '6adbf131-c6c2-4bc6-b1e9-b62f812bed29',
-          name: 'test user',
-          email: 'testUser@example.com',
-        },
-        {
-          resourceType: ResourceType.User,
-          resourceKey: '7d9e8e38-78d5-46ad-9fd0-6adad882161b',
-          name: 'test user 2',
-          email: 'testUser2@example.com',
-        },
-      ],
-    } as AWS.DynamoDB.DocumentClient.QueryOutput),
-  }));
-  class FakeDynamoDb {
-    query = mockDynamoQuery;
-  }
-
-  const AWS = {
-    DynamoDB: {
-      DocumentClient: FakeDynamoDb,
+const mockItems = {
+  Items: [
+    {
+      resourceType: ResourceType.User,
+      resourceKey: '6adbf131-c6c2-4bc6-b1e9-b62f812bed29',
+      name: 'test user',
+      email: 'testUser@example.com',
+    },
+    {
+      resourceType: ResourceType.User,
+      resourceKey: '7d9e8e38-78d5-46ad-9fd0-6adad882161b',
+      name: 'test user 2',
+      email: 'testUser2@example.com',
     },
-  };
+  ],
+}
 
-  return AWS;
-});
+const client = mockClient(DynamoDBDocumentClient);
 
 describe('getDynamoMembers', () => {
   beforeEach(() => {
     config.aws.dynamoTable = 'testTable';
+    client.reset()
   });
 
   afterEach(() => {
     jest.clearAllMocks();
   });
 
   it('should call dynamo query', async () => {
+    client.on(QueryCommand).resolves(mockItems as unknown as QueryCommandOutput)
     await getDynamoMembers();
-    expect(mockDynamoQuery).toBeCalledWith({
+    expect(client.call(0).firstArg.input).toEqual({
       TableName: 'testTable',
       KeyConditionExpression: 'resourceType = :type',
       FilterExpression: 'attribute_not_exists(#ttl_key) or #ttl_key = :null',
@@ -61,6 +47,6 @@ describe('getDynamoMembers', () => {
       ExpressionAttributeNames: {
         '#ttl_key': 'ttl',
       },
-    } as AWS.DynamoDB.DocumentClient.QueryInput);
+    })
   });
 });

tests/unit/getMemberDetails.test.ts

@@ -132,4 +132,47 @@ describe('getMemberDetails', () => {
     const result = await getMemberDetails();
     expect(result).toHaveLength(2);
   });
+
+  it('should keep fetching users until no skip token is returned', async () => {
+    config.aad.groupId = 'testGroup1';
+    const members = [
+      generateUser('a62188fb-a6dc-4a8a-8882-c155130d6a56'),
+      { id: '07299098-5955-4de0-be5f-cba7337f30de', '@odata.type': '#microsoft.graph.device' },
+      generateUser('7a86586e-8eb5-46a0-b3a1-f957b03aa7af'),
+    ];
+    const members2 = [
+      generateUser('2e275dcc-54fe-488f-86ea-2dec2b6872a8'),
+      { id: '07299098-5955-4de0-be5f-cba7337f30de', '@odata.type': '#microsoft.graph.device' },
+      generateUser('15354861-0826-4fe9-8f42-e61c81fd79a3'),
+    ];
+
+    mockAxiosGet.mockResolvedValueOnce({
+      data: {
+        '@odata.nextLink': "https://test/v1.0/groups/testGroup/members?$count=true&$top=999&$filter=accountEnabled%20eq%20true$skipToken=token",
+        value: members },
+    });
+    mockAxiosGet.mockResolvedValueOnce({
+      data: { value: members2 },
+    });
+
+    const result = await getMemberDetails();
+    expect(result).toHaveLength(4);
+  });
+});
+
+it('should call the skip token url when one is provided', async () => {
+
+  mockAxiosGet.mockResolvedValueOnce({
+    data: {
+      '@odata.nextLink': "https://test/v1.0/groups/testGroup/members?$count=true&$top=999&$filter=accountEnabled+eq+true$skipToken=token",
+      value: [] },
+  });
+
+  await getMemberDetails();
+  expect(mockAxiosGet).toBeCalledWith(
+    'https://test/v1.0/groups/testGroup/members?$count=true&$top=999&$filter=accountEnabled eq true$skipToken=token',
+    {
+      headers: { Authorization: 'Bearer testToken', ConsistencyLevel: 'eventual' },
+    },
+  );
 });