Merge pull request #32152 from vespa-engine/balder/1p-epsilon

- Only require similarity of 1 percent, or a minimum of 1_Mi. a fixed…

searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp

@@ -139,7 +139,7 @@ DiskMemUsageSampler::sampleDiskUsage()
 vespalib::ProcessMemoryStats
 DiskMemUsageSampler::sampleMemoryUsage()
 {
-    return vespalib::ProcessMemoryStats::create();
+    return vespalib::ProcessMemoryStats::create(0.01);
 }
 
 TransientResourceUsage

vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp

@@ -7,14 +7,12 @@
 #include <fstream>
 #include <sys/mman.h>
 #include <fcntl.h>
-#include <unistd.h>
-
 
 using namespace vespalib;
 
 namespace {
 
-constexpr uint64_t SIZE_EPSILON = 4095;
+constexpr double SIZE_EPSILON = 0.01;
 
 std::string toString(const ProcessMemoryStats &stats)
 {
@@ -46,7 +44,7 @@ TEST("grow anonymous memory")
     void *mapAddr = mmap(nullptr, mapLen, PROT_READ | PROT_WRITE,
                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
     EXPECT_NOT_EQUAL(reinterpret_cast<void *>(-1), mapAddr);
-    ProcessMemoryStats stats2(ProcessMemoryStats::create());
+    ProcessMemoryStats stats2(ProcessMemoryStats::create(0.01));
     std::cout << toString(stats2) << std::endl;
     EXPECT_LESS_EQUAL(stats1.getAnonymousVirt() + mapLen,
                       stats2.getAnonymousVirt());

vespalib/src/vespa/vespalib/util/process_memory_stats.cpp

@@ -5,6 +5,7 @@
 #include <algorithm>
 #include <vector>
 #include <cinttypes>
+#include <vespa/vespalib/util/size_literals.h>
 
 #include <vespa/log/log.h>
 
@@ -152,21 +153,22 @@ ProcessMemoryStats::ProcessMemoryStats(uint64_t mapped_virt,
 namespace {
 
 bool
-similar(uint64_t lhs, uint64_t rhs, uint64_t epsilon)
+similar(uint64_t lhs, uint64_t rhs, double epsilon)
 {
-    return (lhs < rhs) ? ((rhs - lhs) <= epsilon) : ((lhs - rhs) <= epsilon);
+    uint64_t maxDiff = std::max(1_Mi, uint64_t(epsilon * (lhs+rhs)/2.0));
+    return (lhs < rhs) ? ((rhs - lhs) <= maxDiff) : ((lhs - rhs) <= maxDiff);
 }
 
 }
 
 bool
-ProcessMemoryStats::similarTo(const ProcessMemoryStats &rhs, uint64_t sizeEpsilon) const
+ProcessMemoryStats::similarTo(const ProcessMemoryStats &rhs, double epsilon) const
 {
-    return similar(_mapped_virt, rhs._mapped_virt, sizeEpsilon) &&
-            similar(_mapped_rss, rhs._mapped_rss, sizeEpsilon) &&
-            similar(_anonymous_virt, rhs._anonymous_virt, sizeEpsilon) &&
-            similar(_anonymous_rss, rhs._anonymous_rss, sizeEpsilon) &&
-            (_mappings_count == rhs._mappings_count);
+    return similar(_mapped_virt, rhs._mapped_virt, epsilon) &&
+           similar(_mapped_rss, rhs._mapped_rss, epsilon) &&
+           similar(_anonymous_virt, rhs._anonymous_virt, epsilon) &&
+           similar(_anonymous_rss, rhs._anonymous_rss, epsilon) &&
+           (_mappings_count == rhs._mappings_count);
 }
 
 vespalib::string
@@ -182,23 +184,23 @@ ProcessMemoryStats::toString() const
 }
 
 ProcessMemoryStats
-ProcessMemoryStats::create(uint64_t sizeEpsilon)
+ProcessMemoryStats::create(double epsilon)
 {
     constexpr size_t NUM_TRIES = 3;
     std::vector<ProcessMemoryStats> samples;
-    samples.reserve(NUM_TRIES);
+    samples.reserve(NUM_TRIES + 1);
     samples.push_back(createStatsFromSmaps());
     for (size_t i = 0; i < NUM_TRIES; ++i) {
         samples.push_back(createStatsFromSmaps());
-        if (samples.back().similarTo(*(samples.rbegin()+1), sizeEpsilon)) {
+        if (samples.back().similarTo(*(samples.rbegin()+1), epsilon)) {
             return samples.back();
         }
         LOG(debug, "create(): Memory stats have changed, trying to read smaps file again: i=%zu, prevStats={%s}, currStats={%s}",
             i, (samples.rbegin()+1)->toString().c_str(), samples.back().toString().c_str());
     }
     std::sort(samples.begin(), samples.end());
-    LOG(debug, "We failed to find 2 consecutive samples that where similar with epsilon of %" PRIu64 ".\nSmallest is '%s',\n median is '%s',\n largest is '%s'",
-                 sizeEpsilon, samples.front().toString().c_str(), samples[samples.size()/2].toString().c_str(), samples.back().toString().c_str());
+    LOG(debug, "We failed to find 2 consecutive samples that where similar with epsilon of %d%%.\nSmallest is '%s',\n median is '%s',\n largest is '%s'",
+        int(epsilon*1000), samples.front().toString().c_str(), samples[samples.size()/2].toString().c_str(), samples.back().toString().c_str());
     return samples[samples.size()/2];
 }
 

vespalib/src/vespa/vespalib/util/process_memory_stats.h

@@ -28,13 +28,13 @@ class ProcessMemoryStats
      * Samples are taken until two consecutive memory stats are similar given the size epsilon.
      * This ensures a somewhat consistent memory stats snapshot.
      */
-    static ProcessMemoryStats create(uint64_t sizeEpsilon = 1 * 1024 * 1024);
+    static ProcessMemoryStats create(double epsilon);
     uint64_t getMappedVirt() const { return _mapped_virt; }
     uint64_t getMappedRss() const { return _mapped_rss; }
     uint64_t getAnonymousVirt() const { return _anonymous_virt; }
     uint64_t getAnonymousRss() const { return _anonymous_rss; }
     uint64_t getMappingsCount() const { return _mappings_count; }
-    bool similarTo(const ProcessMemoryStats &rhs, uint64_t sizeEpsilon) const;
+    bool similarTo(const ProcessMemoryStats &rhs, double epsilon) const;
     vespalib::string toString() const;
     bool operator < (const ProcessMemoryStats & rhs) const { return _anonymous_rss < rhs._anonymous_rss; }