From afe34c044392ed6db9b1fb2a2be373b68ca9f5d8 Mon Sep 17 00:00:00 2001 From: Riku Rouvila Date: Fri, 10 Jan 2025 19:51:56 +0900 Subject: [PATCH 1/2] make es only start up once in the beginning of events tests, use a new index for all test cases --- .../src/service/indexing/indexing.test.ts | 9 +++-- .../events/src/service/indexing/indexing.ts | 33 ++++++++++------- .../src/storage/__mocks__/elasticsearch.ts | 35 ++----------------- packages/events/src/storage/elasticsearch.ts | 2 ++ packages/events/src/tests/global-setup.ts | 23 +++++++++++- packages/events/src/tests/setup.ts | 24 +++++++------ packages/events/tsconfig.json | 2 +- 7 files changed, 69 insertions(+), 59 deletions(-) diff --git a/packages/events/src/service/indexing/indexing.test.ts b/packages/events/src/service/indexing/indexing.test.ts index 2ceaedfaad7..20d7622cce9 100644 --- a/packages/events/src/service/indexing/indexing.test.ts +++ b/packages/events/src/service/indexing/indexing.test.ts @@ -9,7 +9,10 @@ * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ -import { getOrCreateClient } from '@events/storage/__mocks__/elasticsearch' +import { + getEventIndexName, + getOrCreateClient +} from '@events/storage/elasticsearch' import { payloadGenerator } from '@events/tests/generators' import { createTestClient } from '@events/tests/utils' import { indexAllEvents } from './indexing' @@ -27,7 +30,7 @@ test('indexes all records from MongoDB with one function call', async () => { } const body = await esClient.search({ - index: 'events', + index: getEventIndexName(), body: { query: { match_all: {} @@ -43,7 +46,7 @@ test('records are automatically indexed when they are created', async () => { const esClient = getOrCreateClient() const body = await esClient.search({ - index: 'events', + index: getEventIndexName(), body: { query: { match_all: {} diff --git a/packages/events/src/service/indexing/indexing.ts b/packages/events/src/service/indexing/indexing.ts index ef8b733f23c..5af143a3d8d 100644 --- a/packages/events/src/service/indexing/indexing.ts +++ b/packages/events/src/service/indexing/indexing.ts @@ -9,8 +9,6 @@ * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ -const EVENTS_INDEX = 'events' - import { EventDocument, EventIndex, @@ -19,7 +17,10 @@ import { import { type estypes } from '@elastic/elasticsearch' import { getClient } from '@events/storage' -import { getOrCreateClient } from '@events/storage/elasticsearch' +import { + getEventIndexName, + getOrCreateClient +} from '@events/storage/elasticsearch' import { Transform } from 'stream' import { z } from 'zod' @@ -32,7 +33,7 @@ function eventToEventIndex(event: EventDocument): EventIndex { */ type EventIndexMapping = { [key in keyof EventIndex]: estypes.MappingProperty } -function createIndex(indexName: string) { +export function createIndex(indexName: string) { const client = getOrCreateClient() return client.indices.create({ index: indexName, @@ -58,9 +59,15 @@ function createIndex(indexName: string) { export async function indexAllEvents() { const mongoClient = await getClient() const esClient = getOrCreateClient() - await createIndex(EVENTS_INDEX) + const hasEventsIndex = await esClient.indices.exists({ + index: getEventIndexName() + }) - const stream = mongoClient.collection(EVENTS_INDEX).find().stream() + if (!hasEventsIndex) { + await createIndex(getEventIndexName()) + } + + const stream = mongoClient.collection(getEventIndexName()).find().stream() const transformedStreamData = new Transform({ readableObjectMode: true, @@ -76,7 +83,7 @@ export async function indexAllEvents() { datasource: stream.pipe(transformedStreamData), onDocument: (doc: EventIndex) => ({ index: { - _index: EVENTS_INDEX, + _index: getEventIndexName(), _id: doc.id } }), @@ -88,7 +95,7 @@ export async function indexEvent(event: EventDocument) { const esClient = getOrCreateClient() return esClient.update({ - index: EVENTS_INDEX, + index: getEventIndexName(), id: event.id, body: { doc: eventToEventIndex(event), @@ -102,7 +109,7 @@ export async function deleteEventIndex(eventId: string) { const esClient = getOrCreateClient() const response = await esClient.delete({ - index: EVENTS_INDEX, + index: getEventIndexName(), id: eventId, refresh: 'wait_for' }) @@ -113,19 +120,21 @@ export async function deleteEventIndex(eventId: string) { export async function getIndexedEvents() { const esClient = getOrCreateClient() - const hasEventsIndex = await esClient.indices.exists({ index: EVENTS_INDEX }) + const hasEventsIndex = await esClient.indices.exists({ + index: getEventIndexName() + }) if (!hasEventsIndex) { // @TODO: We probably want to create the index on startup or as part of the deployment process. // eslint-disable-next-line no-console console.error('Events index does not exist. Creating one.') - await createIndex(EVENTS_INDEX) + await createIndex(getEventIndexName()) return [] } const response = await esClient.search({ - index: EVENTS_INDEX, + index: getEventIndexName(), size: 10000, request_cache: false }) diff --git a/packages/events/src/storage/__mocks__/elasticsearch.ts b/packages/events/src/storage/__mocks__/elasticsearch.ts index 35cb420a744..9ebbaa1c6c3 100644 --- a/packages/events/src/storage/__mocks__/elasticsearch.ts +++ b/packages/events/src/storage/__mocks__/elasticsearch.ts @@ -8,42 +8,13 @@ * * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ -import { - ElasticsearchContainer, - StartedElasticsearchContainer -} from '@testcontainers/elasticsearch' import * as elasticsearch from '@elastic/elasticsearch' -let server: StartedElasticsearchContainer +import { inject, vi } from 'vitest' -export async function setupServer() { - if (server) { - return - } - - server = await new ElasticsearchContainer('elasticsearch:8.14.3') - .withExposedPorts(9200) - .withStartupTimeout(120_000) - .withEnvironment({ - 'discovery.type': 'single-node', - 'xpack.security.enabled': 'false', - 'action.destructive_requires_name': 'false' - }) - .start() -} - -export async function resetServer() { - const host = server.getHost() - const port = server.getMappedPort(9200) - return fetch(`http://${host}:${port}/*`, { - method: 'DELETE' - }) -} +export const getEventIndexName = vi.fn() export function getOrCreateClient() { - const host = server.getHost() - const port = server.getMappedPort(9200) - return new elasticsearch.Client({ - node: `http://${host}:${port}` + node: `http://${inject('ELASTICSEARCH_URI')}` }) } diff --git a/packages/events/src/storage/elasticsearch.ts b/packages/events/src/storage/elasticsearch.ts index 07e3df9be1d..b3f0ad44dfc 100644 --- a/packages/events/src/storage/elasticsearch.ts +++ b/packages/events/src/storage/elasticsearch.ts @@ -13,6 +13,8 @@ import { env } from '@events/environment' let client: elasticsearch.Client +export const getEventIndexName = () => 'events' + export const getOrCreateClient = () => { if (!client) { client = new elasticsearch.Client({ diff --git a/packages/events/src/tests/global-setup.ts b/packages/events/src/tests/global-setup.ts index bc5adfec26d..17697828a9a 100644 --- a/packages/events/src/tests/global-setup.ts +++ b/packages/events/src/tests/global-setup.ts @@ -9,22 +9,43 @@ * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ +import { ElasticsearchContainer } from '@testcontainers/elasticsearch' import { MongoMemoryServer } from 'mongodb-memory-server' export type { ProvidedContext } from 'vitest' declare module 'vitest' { export interface ProvidedContext { MONGO_URI: string + ELASTICSEARCH_URI: string } } +export async function teardown() {} + +async function setupServer() { + return new ElasticsearchContainer('elasticsearch:8.14.3') + .withExposedPorts(9200) + .withStartupTimeout(120_000) + .withEnvironment({ + 'discovery.type': 'single-node', + 'xpack.security.enabled': 'false', + 'action.destructive_requires_name': 'false' + }) + .start() +} + export default async function setup({ provide }: any) { - const mongod = await MongoMemoryServer.create() + const [mongod, es] = await Promise.all([ + await MongoMemoryServer.create(), + await setupServer() + ]) const uri = mongod.getUri() + provide('ELASTICSEARCH_URI', `${es.getHost()}:${es.getMappedPort(9200)}`) provide('MONGO_URI', uri) return async () => { + await es.stop() await mongod.stop() } } diff --git a/packages/events/src/tests/setup.ts b/packages/events/src/tests/setup.ts index 1b017fb7dd3..a79d60a86d2 100644 --- a/packages/events/src/tests/setup.ts +++ b/packages/events/src/tests/setup.ts @@ -8,14 +8,11 @@ * * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ -import { vi } from 'vitest' import { resetServer as resetMongoServer } from '@events/storage/__mocks__/mongodb' +import { inject, vi } from 'vitest' +import { createIndex } from '@events/service/indexing/indexing' import { tennisClubMembershipEvent } from '@opencrvs/commons/fixtures' -import { - resetServer as resetESServer, - setupServer as setupESServer -} from '@events/storage/__mocks__/elasticsearch' import { mswServer } from './msw' vi.mock('@events/storage/mongodb') @@ -28,16 +25,23 @@ vi.mock('@events/service/config/config', () => ({ ]) })) -beforeAll(() => Promise.all([setupESServer()]), 100000) -beforeEach(() => Promise.all([resetMongoServer(), resetESServer()])) +async function resetServer() { + // @ts-ignore "Cannot find module '@events/storage/elasticsearch' or its corresponding type declarations." + const { getEventIndexName } = await import('@events/storage/elasticsearch') + const index = 'events' + Date.now() + Math.random() + getEventIndexName.mockReturnValue(index) + await createIndex(index) +} + +beforeEach(async () => { + return Promise.all([resetMongoServer(), resetServer()]) +}) beforeAll(() => mswServer.listen({ onUnhandledRequest: (req) => { - const elasticRegex = /http:\/\/localhost:551\d{2}\/.*/ - const isElasticResetCall = - req.method === 'DELETE' && elasticRegex.test(req.url) + req.method === 'DELETE' && req.url.includes(inject('ELASTICSEARCH_URI')) if (!isElasticResetCall) { // eslint-disable-next-line no-console diff --git a/packages/events/tsconfig.json b/packages/events/tsconfig.json index 2c51ebffab2..241cda5dd4f 100644 --- a/packages/events/tsconfig.json +++ b/packages/events/tsconfig.json @@ -18,7 +18,7 @@ "noImplicitAny": true, "strictNullChecks": true, "noUnusedLocals": true, - "types": ["fhir", "jest"] + "types": ["vitest/globals"] }, "include": ["src/**/*.ts", "typings", "tslint-rules"], "exclude": ["node_modules", "build", "scripts", "acceptance-tests"] From 75b3ab0ba45fae3e4a453dd579cc6b19cebaaaca Mon Sep 17 00:00:00 2001 From: Riku Rouvila Date: Fri, 10 Jan 2025 19:54:39 +0900 Subject: [PATCH 2/2] remove unused function --- packages/events/src/tests/global-setup.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/events/src/tests/global-setup.ts b/packages/events/src/tests/global-setup.ts index 17697828a9a..28861472868 100644 --- a/packages/events/src/tests/global-setup.ts +++ b/packages/events/src/tests/global-setup.ts @@ -20,8 +20,6 @@ declare module 'vitest' { } } -export async function teardown() {} - async function setupServer() { return new ElasticsearchContainer('elasticsearch:8.14.3') .withExposedPorts(9200)