Skip to content

Commit

Permalink
Prevent ghost containers after retries
Browse files Browse the repository at this point in the history
reportLeakedContainers adapted from
trinodb/trino#20297
trinodb/trino#21280

Co-authored-by: Piotr Findeisen <piotr.findeisen@gmail.com>
Co-authored-by: Jan Waś <jan.was@starburstdata.com>
  • Loading branch information
3 people committed Nov 5, 2024
1 parent 4aa1b6c commit 902cb03
Show file tree
Hide file tree
Showing 2 changed files with 62 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -550,6 +550,7 @@ private void tryStart() {
} else {
logger().error("There are no stdout/stderr logs available for the failed container");
}
stop();
}

throw new ContainerLaunchException("Could not create/start container", e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,12 @@
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.InspectContainerResponse;
import com.github.dockerjava.api.command.InspectContainerResponse.ContainerState;
import com.github.dockerjava.api.model.Container;
import com.github.dockerjava.api.model.ExposedPort;
import com.github.dockerjava.api.model.Info;
import com.github.dockerjava.api.model.Ports;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableList;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
Expand All @@ -28,9 +31,12 @@
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.MountableFile;

import java.time.Duration;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.stream.Collectors;
Expand All @@ -42,6 +48,21 @@

public class GenericContainerTest {

@Test
public void testStartupTimeoutWithAttemptsNotLeakingContainers() {
try (
GenericContainer<?> container = new GenericContainer<>(TestImages.TINY_IMAGE)
.withStartupAttempts(3)
.waitingFor(
Wait.forLogMessage("this text does not exist in logs", 1).withStartupTimeout(Duration.ofMillis(1))
)
.withCommand("tail", "-f", "/dev/null");
) {
assertThatThrownBy(container::start).hasStackTraceContaining("Retry limit hit with exception");
}
assertThat(reportLeakedContainers()).isEmpty();
}

@Test
public void shouldReportOOMAfterWait() {
Info info = DockerClientFactory.instance().client().infoCmd().exec();
Expand Down Expand Up @@ -273,6 +294,46 @@ public void shouldRespectWaitStrategy() {
}
}

private static Optional<String> reportLeakedContainers() {
@SuppressWarnings("resource") // Throws when close is attempted, as this is a global instance.
DockerClient dockerClient = DockerClientFactory.lazyClient();

List<Container> containers = dockerClient
.listContainersCmd()
.withLabelFilter(
Collections.singletonMap(
DockerClientFactory.TESTCONTAINERS_SESSION_ID_LABEL,
DockerClientFactory.SESSION_ID
)
)
// ignore status "exited" - for example, failed containers after using `withStartupAttempts()`
.withStatusFilter(Arrays.asList("created", "restarting", "running", "paused"))
.exec()
.stream()
.collect(ImmutableList.toImmutableList());

if (containers.isEmpty()) {
return Optional.empty();
}

return Optional.of(
String.format(
"Leaked containers: %s",
containers
.stream()
.map(container -> {
return MoreObjects
.toStringHelper("container")
.add("id", container.getId())
.add("image", container.getImage())
.add("imageId", container.getImageId())
.toString();
})
.collect(Collectors.joining(", ", "[", "]"))
)
);
}

static class NoopStartupCheckStrategy extends StartupCheckStrategy {

@Override
Expand Down

0 comments on commit 902cb03

Please sign in to comment.