Improving Test Reliability in RemoteTemporaryDirectory and TestJobManager

src/libsyncengine/jobs/abstractjob.h

@@ -49,7 +49,7 @@ class AbstractJob : public Poco::Runnable {
         inline void setProgressPercentCallback(const std::function<void(UniqueId, int)> &newCallback) {
             _progressPercentCallback = newCallback;
         }
-
+        inline ExitInfo exitInfo() const { return ExitInfo(_exitCode, _exitCause); }
         inline ExitCode exitCode() const { return _exitCode; }
         inline ExitCause exitCause() const { return _exitCause; }
 

test/libcommonserver/log/testlog.cpp

@@ -91,28 +91,37 @@ void TestLog::testExpiredLogFiles(void) {
     // This test checks that old archived log files are deleted after a certain time
     clearLogDirectory();
 
-    // Generate a fake log file
+    // Generate a fake old log file
     std::ofstream fakeLogFile(_logDir / APPLICATION_NAME "_fake.log.gz");
     fakeLogFile << "Fake old log file" << std::endl;
     fakeLogFile.close();
-    LOG_INFO(_logger, "Test log file expiration"); // Ensure the log file is created.
 
-    CPPUNIT_ASSERT_EQUAL(2, countFilesInDirectory(_logDir)); // The current log file and the fake archived log file.
+    // Ensure that a new log file is created
+    LOG_INFO(_logger, "Test log file expiration"); 
 
+    // Check that we got 2 log files (the current one and the fake old one)
+    CPPUNIT_ASSERT_EQUAL(2, countFilesInDirectory(_logDir)); 
+
+    // Set the expiration time to 2 seconds
     auto *appender = static_cast<CustomRollingFileAppender *>(_logger.getAppender(Log::rfName).get());
     appender->setExpire(2); // 2 seconds
-    Utility::msleep(1000);
-    appender->checkForExpiredFiles();
-
-    const auto now = std::chrono::system_clock::now();
-    KDC::testhelpers::setModificationDate(Log::instance()->getLogFilePath(), now);
 
-    CPPUNIT_ASSERT_EQUAL(2, countFilesInDirectory(_logDir)); // The fake log file should not be deleted yet (< 2 seconds).
-
-    Utility::msleep(1000);
-    appender->checkForExpiredFiles();
+    const auto start = std::chrono::system_clock::now();
+    auto now = std::chrono::system_clock::now();
+    while (now - start < std::chrono::seconds(3)) {
+        now = std::chrono::system_clock::now();
+        KDC::testhelpers::setModificationDate(Log::instance()->getLogFilePath(),
+                                              now); // Prevent the current log file from being deleted.
+        appender->checkForExpiredFiles();
+        if (now - start < std::chrono::milliseconds(1500)) { // The fake log file should not be deleted yet.
+            CPPUNIT_ASSERT_EQUAL(2, countFilesInDirectory(_logDir));
+        } else if (countFilesInDirectory(_logDir) == 1) { // The fake log file MIGHT be deleted now.
+            break;        
+        }
+        Utility::msleep(500);
+    }
 
-    CPPUNIT_ASSERT_EQUAL(1, countFilesInDirectory(_logDir)); // The fake log file should be deleted now.
+    CPPUNIT_ASSERT_EQUAL(1, countFilesInDirectory(_logDir)); // The fake log file SHOULD be deleted now.
     appender->setExpire(CommonUtility::logsPurgeRate * 24 * 3600);
 }
 
@@ -145,6 +154,8 @@ void TestLog::clearLogDirectory(void) const {
             continue;
         }
         IoHelper::deleteItem(entry.path(), ioError);
+        CPPUNIT_ASSERT_EQUAL(IoError::Success, ioError);
+
     }
     CPPUNIT_ASSERT_EQUAL(IoError::Success, ioError);
     CPPUNIT_ASSERT(endOfDirectory);

test/libsyncengine/jobs/testjobmanager.cpp

@@ -43,7 +43,6 @@ namespace KDC {
 static const SyncPath localTestDirPath(std::wstring(L"" TEST_DIR) + L"/test_ci");
 static const SyncPath localTestDirPath_manyFiles(std::wstring(L"" TEST_DIR) + L"/test_ci/many_files_dir");
 static const SyncPath localTestDirPath_pictures(std::wstring(L"" TEST_DIR) + L"/test_ci/test_pictures");
-static const SyncPath localTestDirPath_bigFiles(std::wstring(L"" TEST_DIR) + L"/test_ci/big_file_dir");
 static const int driveDbId = 1;
 void KDC::TestJobManager::setUp() {
     const testhelpers::TestVariables testVariables;
@@ -91,21 +90,38 @@ void KDC::TestJobManager::tearDown() {
 void TestJobManager::testWithoutCallback() {
     // Create temp remote directory
     const RemoteTemporaryDirectory remoteTmpDir(driveDbId, _testVariables.remoteDirId, "TestJobManager testWithoutCallback");
+    const LocalTemporaryDirectory localTmpDir("TestJobManager testWithoutCallback");
+    for (int i = 0; i < 100; i++) {
+        testhelpers::generateOrEditTestFile(localTmpDir.path() / ("file_" + std::to_string(i) + ".txt"));
+    }
 
     // Upload all files in testDir
     size_t counter = 0;
-    for (auto &dirEntry: std::filesystem::directory_iterator(localTestDirPath_manyFiles)) {
+    std::queue<UniqueId> jobIds;
+    for (auto &dirEntry: std::filesystem::directory_iterator(localTmpDir.path())) {
         if (dirEntry.path().filename() == ".DS_Store") {
             continue;
         }
 
-        std::shared_ptr<UploadJob> job = std::make_shared<UploadJob>(driveDbId, dirEntry.path(),
+        auto job = std::make_shared<UploadJob>(driveDbId, dirEntry.path(),
                                                                      dirEntry.path().filename().native(), remoteTmpDir.id(), 0);
         JobManager::instance()->queueAsyncJob(job);
+        jobIds.push(job->jobId());
         counter++;
     }
 
-    Utility::msleep(10000); // Wait 10sec
+    // Wait for all uploads to finish
+    const auto start = std::chrono::steady_clock::now();
+    while (!jobIds.empty()) {
+        const auto now = std::chrono::steady_clock::now();
+        CPPUNIT_ASSERT_MESSAGE("All uploads have not finished in 30 seconds",
+                               std::chrono::duration_cast<std::chrono::seconds>(now - start).count() < 30);
+
+        Utility::msleep(100); // Wait 100ms
+        while (!jobIds.empty() && JobManager::instance()->isJobFinished(jobIds.front())) {
+            jobIds.pop();
+        }
+    }
 
     GetFileListJob fileListJob(driveDbId, remoteTmpDir.id());
     fileListJob.runSynchronously();
@@ -140,7 +156,7 @@ void TestJobManager::testWithCallback() {
         _ongoingJobs.insert({job->jobId(), job});
     }
 
-    int waitCountMax = 100; // Wait max 10sec
+    int waitCountMax = 300; // Wait max 30sec
     while (ongoingJobsCount() > 0 && waitCountMax-- > 0 && !_jobErrorSocketsDefuncted && !_jobErrorOther) {
         Utility::msleep(100); // Wait 100ms
     }
@@ -175,25 +191,40 @@ void TestJobManager::testWithCallbackBigFiles() {
 }
 
 void TestJobManager::testCancelJobs() {
-    // Create temp remote directory
     const RemoteTemporaryDirectory remoteTmpDir(driveDbId, _testVariables.remoteDirId, "TestJobManager testCancelJobs");
+    const LocalTemporaryDirectory localTmpDir("testJobManager");
+    const int localFileCounter = 100;
+    for (int i = 0; i < localFileCounter; i++) {
+        testhelpers::generateOrEditTestFile(localTmpDir.path() / ("file_" + std::to_string(i) + ".txt"));
+    }
+
     // Upload all files in testDir
-    ulong jobCounter = 0;
-    for (auto &dirEntry: std::filesystem::directory_iterator(localTestDirPath_manyFiles)) {
+    for (auto &dirEntry: std::filesystem::directory_iterator(localTmpDir.path())) {
         auto job = std::make_shared<UploadJob>(driveDbId, dirEntry.path(), dirEntry.path().filename().native(), remoteTmpDir.id(),
                                                0);
         std::function<void(UniqueId)> callback = std::bind(&TestJobManager::callback, this, std::placeholders::_1);
         JobManager::instance()->queueAsyncJob(job, Poco::Thread::PRIO_NORMAL, callback);
-        jobCounter++;
         const std::scoped_lock lock(_mutex);
         _ongoingJobs.try_emplace(static_cast<uint64_t>(job->jobId()), job);
     }
-
-    Utility::msleep(1000); // Wait 1sec
+    while (_ongoingJobs.size() == localFileCounter) {
+        Utility::msleep(1); // Wait 1ms
+    }
 
     cancelAllOngoingJobs();
 
-    Utility::msleep(10000); // Wait 10sec
+    int retry = 1000; // Wait max 10sec
+    while ((!JobManager::_managedJobs.empty() || !JobManager::_queuedJobs.empty() || !JobManager::_runningJobs.empty() ||
+            !JobManager::_pendingJobs.empty()) &&
+           (retry > 0)) {
+        retry--;
+        Utility::msleep(10);
+    }
+
+    CPPUNIT_ASSERT(JobManager::instance()->_managedJobs.empty());
+    CPPUNIT_ASSERT(JobManager::instance()->_queuedJobs.empty());
+    CPPUNIT_ASSERT(JobManager::instance()->_runningJobs.empty());
+    CPPUNIT_ASSERT(JobManager::instance()->_pendingJobs.empty());
 
     GetFileListJob fileListJob(driveDbId, remoteTmpDir.id());
     fileListJob.runSynchronously();
@@ -202,14 +233,10 @@ void TestJobManager::testCancelJobs() {
     CPPUNIT_ASSERT(resObj);
 
     Poco::JSON::Array::Ptr data = resObj->getArray(dataKey);
-    size_t total = data->size();
-    CPPUNIT_ASSERT(jobCounter != total);
-    CPPUNIT_ASSERT(total > 0);
+    const size_t uploadedFileCounter = data->size();
+    CPPUNIT_ASSERT(localFileCounter != uploadedFileCounter);
+    CPPUNIT_ASSERT(uploadedFileCounter > 0);
     CPPUNIT_ASSERT(ongoingJobsCount() == 0);
-    CPPUNIT_ASSERT(JobManager::instance()->_managedJobs.empty());
-    CPPUNIT_ASSERT(JobManager::instance()->_queuedJobs.empty());
-    CPPUNIT_ASSERT(JobManager::instance()->_runningJobs.empty());
-    CPPUNIT_ASSERT(JobManager::instance()->_pendingJobs.empty());
 }
 
 std::queue<int64_t> finishedJobs;

test/test_utility/remotetemporarydirectory.cpp

@@ -23,28 +23,44 @@
 #include "../../src/libsyncengine/jobs/network/API_v2/deletejob.h"
 #include "libsyncengine/jobs/network/networkjobsparams.h"
 #include "libcommonserver/utility/utility.h"
+#include "libcommon/utility/utility.h"
 
 namespace KDC {
 RemoteTemporaryDirectory::RemoteTemporaryDirectory(int driveDbId, const NodeId& parentId,
-                                                   const std::string& testType /*= "undef"*/) : _driveDbId(driveDbId) {
-    // Generate directory name
-    const std::time_t now = std::time(nullptr);
-    const std::tm tm = *std::localtime(&now);
-    std::ostringstream woss;
-    woss << std::put_time(&tm, "%Y%m%d_%H%M");
-
-    _dirName = Str("kdrive_") + Str2SyncName(testType) + Str("_unit_tests_") + Str2SyncName(woss.str());
-
-    // Create remote test dir
-    CreateDirJob job(_driveDbId, parentId, _dirName);
-    CPPUNIT_ASSERT_EQUAL(ExitCode::Ok, job.runSynchronously());
-
-    // Extract file ID
-    CPPUNIT_ASSERT(job.jsonRes());
-    Poco::JSON::Object::Ptr dataObj = job.jsonRes()->getObject(dataKey);
-    CPPUNIT_ASSERT(dataObj);
-    _dirId = dataObj->get(idKey).toString();
+                                                   const std::string& testType /*= "undef"*/) :
+    _driveDbId(driveDbId) {
+    int retry = 5;
+    do {
+        std::string suffix = CommonUtility::generateRandomStringAlphaNum(5);
+        // Generate directory name
+        const std::time_t now = std::time(nullptr);
+        const std::tm tm = *std::localtime(&now);
+        std::ostringstream woss;
+        woss << std::put_time(&tm, "%Y%m%d_%H%M");
+        _dirName = Str("kdrive_") + Str2SyncName(testType) + Str("_unit_tests_") + Str2SyncName(woss.str() + "___" + suffix);
+
+        // Create remote test dir
+        CreateDirJob job(_driveDbId, parentId, _dirName);
+        job.runSynchronously();
+        if (job.exitInfo() == ExitInfo(ExitCode::BackError, ExitCause::FileAlreadyExist) && retry > 0) {
+            retry--;
+            continue;
+        }
+
+        CPPUNIT_ASSERT_EQUAL_MESSAGE("RemoteTemporaryDirectory() failed to create the directory on remote side.",
+                                     ExitInfo(ExitCode::Ok), job.exitInfo());
+
+        // Extract file ID
+        CPPUNIT_ASSERT_MESSAGE("RemoteTemporaryDirectory() Failed to extract the file id.", job.jsonRes());
+        Poco::JSON::Object::Ptr dataObj = job.jsonRes()->getObject(dataKey);
+        CPPUNIT_ASSERT_MESSAGE("RemoteTemporaryDirectory() Failed to extract the file id (2).", dataObj);
+        _dirId = dataObj->get(idKey).toString();
+        LOGW_INFO(Log::instance()->getLogger(), L"RemoteTemporaryDirectory created: " << Utility::formatSyncName(_dirName)
+                                                                                      << L" with ID: " << Utility::s2ws(_dirId));
+        break;
+    } while (true);
 }
+
 RemoteTemporaryDirectory::~RemoteTemporaryDirectory() {
     if (_isDeleted) return;