From 8e1e2f102515f9bec2d00f086152b4345cbb0efa Mon Sep 17 00:00:00 2001 From: lehuygiang28 Date: Fri, 28 Jun 2024 19:34:12 +0700 Subject: [PATCH] refactor(be): :truck: rename env --- apps/be/.env.example | 7 ++++--- apps/be/be.example.env | 7 ++++--- .../src/app/task-logs/task-log.processor.ts | 2 +- apps/be/src/app/task-logs/task-logs.module.ts | 12 +++++++++--- apps/be/src/app/tasks/tasks.module.ts | 19 ++++++++++++++++--- apps/be/src/app/tasks/tasks.service.ts | 15 +++++++++------ apps/be/worker.example.env | 7 ++++--- 7 files changed, 47 insertions(+), 22 deletions(-) diff --git a/apps/be/.env.example b/apps/be/.env.example index 99a6e24..f14559f 100644 --- a/apps/be/.env.example +++ b/apps/be/.env.example @@ -13,9 +13,10 @@ REDIS_CONNECT_TIMEOUT=5000 # OPTIONAL WORKER_MODE='' WORKER_NAME='tasktr' -BULL_TASK_CONCURRENCY=20 -BULL_BACKGROUND_CONCURRENCY=10 -BULL_SAVE_LOG_CONCURRENCY=10 +TASK_CONCURRENCY=100 +MAIL_CONCURRENCY=50 +SAVE_LOG_CONCURRENCY=100 +CLEAR_LOG_CONCURRENCY=100 BULLMQ_EVENTS_MAXLEN=100 MAX_LOGS_PER_TASK=10 diff --git a/apps/be/be.example.env b/apps/be/be.example.env index 74ef6b1..67fb3bf 100644 --- a/apps/be/be.example.env +++ b/apps/be/be.example.env @@ -13,9 +13,10 @@ MONGODB_URI='mongodb://user:password@localhost:27017/tasktr?authSource=admin&ret WORKER_MODE='false' WORKER_NAME='tasktr-be' -BULL_TASK_CONCURRENCY=1 -BULL_BACKGROUND_CONCURRENCY=1 -BULL_SAVE_LOG_CONCURRENCY=1 +TASK_CONCURRENCY=100 +MAIL_CONCURRENCY=50 +SAVE_LOG_CONCURRENCY=100 +CLEAR_LOG_CONCURRENCY=100 MAX_LOGS_PER_TASK=100 REDIS_HOST='localhost' diff --git a/apps/be/src/app/task-logs/task-log.processor.ts b/apps/be/src/app/task-logs/task-log.processor.ts index 900a436..251b3d3 100644 --- a/apps/be/src/app/task-logs/task-log.processor.ts +++ b/apps/be/src/app/task-logs/task-log.processor.ts @@ -11,7 +11,7 @@ export type TaskLogsJobName = 'saveTaskLog'; @Injectable() @Processor(BULLMQ_TASK_LOG_QUEUE, { - concurrency: Number(process.env['BULL_SAVE_LOG_CONCURRENCY']) || 10, + concurrency: Number(process.env['SAVE_LOG_CONCURRENCY']) || 10, }) export class TaskLogProcessor extends WorkerHost implements OnModuleInit { constructor( diff --git a/apps/be/src/app/task-logs/task-logs.module.ts b/apps/be/src/app/task-logs/task-logs.module.ts index 793cbc1..8355516 100644 --- a/apps/be/src/app/task-logs/task-logs.module.ts +++ b/apps/be/src/app/task-logs/task-logs.module.ts @@ -1,4 +1,4 @@ -import { Module } from '@nestjs/common'; +import { Module, Provider } from '@nestjs/common'; import { MongooseModule } from '@nestjs/mongoose'; import { BullModule } from '@nestjs/bullmq'; @@ -9,6 +9,12 @@ import { TaskLogsRepository } from './task-logs.repository'; import { TaskLogsService } from './task-logs.service'; import { TaskLogProcessor } from './task-log.processor'; +const providers: Provider[] = [TaskLogsRepository, TaskLogsService]; + +if (!(process.env['SAVE_LOG_CONCURRENCY'] && Number(process.env['SAVE_LOG_CONCURRENCY']) <= 0)) { + providers.push(TaskLogProcessor); +} + @Module({ imports: [ MongooseModule.forFeature([{ name: TaskLog.name, schema: TaskLogSchema }]), @@ -16,7 +22,7 @@ import { TaskLogProcessor } from './task-log.processor'; name: BULLMQ_TASK_LOG_QUEUE, }), ], - providers: [TaskLogsRepository, TaskLogsService, TaskLogProcessor], - exports: [TaskLogsService, TaskLogProcessor], + providers: providers, + exports: [TaskLogsService], }) export class TaskLogsModule {} diff --git a/apps/be/src/app/tasks/tasks.module.ts b/apps/be/src/app/tasks/tasks.module.ts index d0ae200..dafff73 100644 --- a/apps/be/src/app/tasks/tasks.module.ts +++ b/apps/be/src/app/tasks/tasks.module.ts @@ -1,7 +1,8 @@ -import { Module } from '@nestjs/common'; +import { Module, Provider } from '@nestjs/common'; import { MongooseModule } from '@nestjs/mongoose'; import { BullModule } from '@nestjs/bullmq'; import { HttpModule } from '@nestjs/axios'; +import { ConfigModule } from '@nestjs/config'; import { BULLMQ_TASK_QUEUE, @@ -15,10 +16,22 @@ import { TasksController } from './tasks.controller'; import { TasksService } from './tasks.service'; import { TasksRepository } from './tasks.repository'; import { TaskLogsModule } from '../task-logs'; -import { ClearTasksProcessor, TaskProcessor } from './processors'; +import { TaskProcessor, ClearTasksProcessor } from './processors'; +import tasksConfig from './config/tasks-config'; + +const providers: Provider[] = [TasksRepository, TasksService]; + +if (!(process.env['TASKS_CONCURRENCY'] && Number(process.env['TASKS_CONCURRENCY']) <= 0)) { + providers.push(TaskProcessor); +} + +if (!(process.env['CLEAR_LOG_CONCURRENCY'] && Number(process.env['CLEAR_LOG_CONCURRENCY']) <= 0)) { + providers.push(ClearTasksProcessor); +} @Module({ imports: [ + ConfigModule.forFeature(tasksConfig), HttpModule.register(axiosConfig), MongooseModule.forFeature([{ name: Task.name, schema: TaskSchema }]), TaskLogsModule, @@ -33,7 +46,7 @@ import { ClearTasksProcessor, TaskProcessor } from './processors'; }), ], controllers: [TasksController], - providers: [TasksRepository, TasksService, TaskProcessor, ClearTasksProcessor], + providers: providers, exports: [TasksService], }) export class TasksModule {} diff --git a/apps/be/src/app/tasks/tasks.service.ts b/apps/be/src/app/tasks/tasks.service.ts index 7d6405d..c7e4571 100644 --- a/apps/be/src/app/tasks/tasks.service.ts +++ b/apps/be/src/app/tasks/tasks.service.ts @@ -1,10 +1,12 @@ import { Injectable, OnModuleInit, UnprocessableEntityException } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; import { InjectQueue } from '@nestjs/bullmq'; import { Queue, JobsOptions } from 'bullmq'; import { FilterQuery, QueryOptions, Types, UpdateQuery } from 'mongoose'; +import { PinoLogger } from 'nestjs-pino'; import { BULLMQ_CLEAR_TASK_QUEUE, BULLMQ_TASK_QUEUE } from '~be/common/bullmq'; -import { convertToObjectId, MaybeType, validateCronFrequency } from '~be/common/utils'; +import { convertToObjectId, validateCronFrequency } from '~be/common/utils'; import { JwtPayloadType } from '~be/app/auth/strategies'; import { TasksRepository } from './tasks.repository'; @@ -13,9 +15,8 @@ import { CreateTaskDto, GetTasksResponseDto, TaskDto, UpdateTaskDto } from './dt import { GetTasksDto } from './dtos/get-tasks.dto'; import { GetLogsByTaskIdDto, GetLogsByTaskIdResponseDto, TaskLogDto } from '../task-logs/dtos'; import { TaskLogsService } from '../task-logs'; -import { ConfigService } from '@nestjs/config'; import { ClearTasksJobName } from './processors'; -import { PinoLogger } from 'nestjs-pino'; +import { AllConfig } from '../config'; @Injectable() export class TasksService implements OnModuleInit { @@ -27,7 +28,7 @@ export class TasksService implements OnModuleInit { @InjectQueue(BULLMQ_CLEAR_TASK_QUEUE) readonly clearTaskQueue: Queue, private readonly logger: PinoLogger, - private readonly configService: ConfigService, + private readonly configService: ConfigService, private readonly taskRepo: TasksRepository, private readonly taskLogsService: TaskLogsService, ) {} @@ -434,8 +435,10 @@ export class TasksService implements OnModuleInit { * Use in cronjob - background job to scan and delete tasks that have been soft deleted */ async scanToHardDeleteTasks(): Promise { - const deletionThreshold = - Number(this.configService.get>('SOFT_DELETE_THRESHOLD_DAYS')) || 30; + const deletionThreshold = this.configService.getOrThrow('tasks.softDeleteThresholdDays', { + infer: true, + }); + const thresholdDate = new Date(); thresholdDate.setDate(thresholdDate.getDate() - deletionThreshold); diff --git a/apps/be/worker.example.env b/apps/be/worker.example.env index ce92389..04a5187 100644 --- a/apps/be/worker.example.env +++ b/apps/be/worker.example.env @@ -10,9 +10,10 @@ MONGODB_URI='mongodb://user:password@localhost:27017/tasktr?authSource=admin&ret WORKER_MODE='true' WORKER_NAME='tasktr-worker-1' -BULL_TASK_CONCURRENCY=500 -BULL_BACKGROUND_CONCURRENCY=200 -BULL_SAVE_LOG_CONCURRENCY=200 +TASK_CONCURRENCY=100 +MAIL_CONCURRENCY=50 +SAVE_LOG_CONCURRENCY=100 +CLEAR_LOG_CONCURRENCY=100 MAX_LOGS_PER_TASK=100 REDIS_HOST='localhost'