From e1c4fda48c19d97898a6ce4a4e2c7517192d743f Mon Sep 17 00:00:00 2001 From: "Jonathan A. Sternberg" Date: Fri, 8 Mar 2024 09:39:21 -0600 Subject: [PATCH] hack: configure the otel-collector for hack/compose Configure the otel-collector for `hack/compose` to use it for processing traces and forwarding to jaeger. This simplifies the configuration and also opens up the ability to configure additional behavior around traces and metrics for debugging. This also removes the automatic override file and the gitignore file. It wasn't really working well for me and made it difficult to share extensions to the development environment. I'm going to try something different for that in a different change. Signed-off-by: Jonathan A. Sternberg --- docs/dev/remote-debugging.md | 12 ++-- hack/compose | 32 +--------- hack/composefiles/.gitignore | 8 --- hack/composefiles/README.md | 74 ++++++++++++++++++++++++ hack/composefiles/buildkitd.toml | 1 - hack/composefiles/compose.yaml | 19 +++++- hack/composefiles/extensions/buildx.yaml | 20 +++++++ hack/composefiles/otelcol.yaml | 14 +++++ 8 files changed, 132 insertions(+), 48 deletions(-) delete mode 100644 hack/composefiles/.gitignore create mode 100644 hack/composefiles/README.md create mode 100644 hack/composefiles/extensions/buildx.yaml create mode 100644 hack/composefiles/otelcol.yaml diff --git a/docs/dev/remote-debugging.md b/docs/dev/remote-debugging.md index c78ab07782aa..b2300b46280c 100644 --- a/docs/dev/remote-debugging.md +++ b/docs/dev/remote-debugging.md @@ -19,15 +19,13 @@ $ BUILDKIT_DEBUG=1 make images ## Running the buildkit image Delve runs on port 5000 but a different host address can be used if there's a port conflict. It's also recommended that -you limit the host port to the loopback interface (localhost) to prevent exposing buildkit or delve to external -computers. +you limit the host port to the loopback interface (localhost) to prevent exposing delve to external computers. ```bash -$ docker run --privileged -d \ +$ docker run --privileged -d --name=buildkit-dev \ -p 127.0.0.1:5000:5000 \ - -p 127.0.0.1:1234:1234 \ --restart always \ - moby/buildkit:local --addr tcp://0.0.0.0:1234 + moby/buildkit:local ``` It's also useful to use `--restart always` when debugging. Delve will always shutdown the program with `SIGTERM` when @@ -42,7 +40,7 @@ it's running through the debugger without optimizations. If using `docker` to run builds, you can inform docker about this new instance of buildkit by running the following: ```bash -$ docker buildx create --name=dev --driver=remote tcp://127.0.0.1:1234 +$ docker buildx create --name=dev --driver=remote docker-container://buildkit-dev ``` You can then set the builder in one of the following ways: @@ -82,4 +80,4 @@ client to connect. This mimics how buildkit runs as a release image and is useful enough for most debugging sessions, but may not be useful in others. If you need to debug startup issues, you can go into the Dockerfile and remove `--continue` from -the command line options. \ No newline at end of file +the command line options. diff --git a/hack/compose b/hack/compose index a65c2c2dddc1..5ca9e1e55b1b 100755 --- a/hack/compose +++ b/hack/compose @@ -1,40 +1,12 @@ #!/usr/bin/env bash -# Use to set up a dev environment for buildkit. -# Not meant for production deployments. -# Use the same way you would use docker compose. -# -# $ hack/compose up -d --build -# -# Can be extended for local development by adding -# a compose.override.yaml file to the same directory. -# -# Additional config files for extra services can -# also be placed in the same folder and they will -# be automatically ignored by git. -# -# To access the newly created development buildkit, -# use either: -# -# $ buildctl --addr tcp://127.0.0.1:1234 ... -# -# or alternatively configure a new builder with buildx. -# -# $ docker buildx create \ -# --bootstrap \ -# --name dev \ -# --driver remote \ -# tcp://127.0.0.1:1234 -# $ docker buildx --builder dev ... +# Use to set up a dev environment for buildkit using docker compose. +# See composefiles/README.md for more details. . $(dirname $0)/util set -eu -o pipefail filesDir=$(dirname $0)/composefiles - args=(compose '-f' "$filesDir/compose.yaml") -if [ -f "$filesDir/compose.override.yaml" ]; then - args+=('-f' "$filesDir/compose.override.yaml") -fi dockerCmd "${args[@]}" "$@" diff --git a/hack/composefiles/.gitignore b/hack/composefiles/.gitignore deleted file mode 100644 index ff805297024f..000000000000 --- a/hack/composefiles/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Exclude everything to allow this folder to be used for other service -# configurations. -* - -# Specifically allow certain configuration files. -!.gitignore -!buildkitd.toml -!compose.yaml diff --git a/hack/composefiles/README.md b/hack/composefiles/README.md new file mode 100644 index 000000000000..0815d8b4ab92 --- /dev/null +++ b/hack/composefiles/README.md @@ -0,0 +1,74 @@ +# Buildkit Docker Compose + +The `hack/compose` script provides a convenient development environment for building and testing buildkit with some +useful supporting services and configuration using docker compose to assemble the environment. +This configuration is not meant for production deployments. + +## Usage + +The primary way to use the script is as a substitute to `docker compose`. + +```bash +$ hack/compose up -d +``` + +All arguments to this script will be forwarded to `docker compose`. +This will use the `moby/buildkit:local` image to create a buildkit container. +The image can be either built with `make images` or the `--build` flag can be used. + +```bash +$ hack/compose up -d --build +``` + +To access the newly created development buildkit, use either: + +```bash +$ buildctl --addr docker-container://buildkit-dev ... +``` + +or alternatively configure a new builder with buildx. + +```bash +$ docker buildx create \ + --bootstrap \ + --name dev \ + --driver remote \ + docker-container://buildkit-dev +``` + +## Extending Configuration + +The compose definition can be extended by using `-f $COMPOSE_FILE` to extend the configuration. +These files are in the `extensions` folder. + +As an example, an extension can be used for aiding with buildx metrics development: + +```bash +$ hack/compose -f hack/composefiles/extensions/buildx.yaml up -d --build +``` + +This will modify the `otel-collector` configuration to emit buildx metrics to the debug exporter. + +Some of the files in this directory can be combined and some conflict with each other. In particular, +configurations that modify the `otel-collector` probably conflict with each other. + +## Running the debugger + +The debugger is exposed for the local buildkit when `--build` is used to build the local image. +This can be accessed using delve or any GUI client that uses delve (such as JetBrains GoLand). + +```bash +$ dlv connect localhost:5000 +``` + +More detailed documentation on remote debugging is [here](https://github.com/moby/buildkit/blob/master/docs/dev/remote-debugging.md#connecting-to-the-port-command-line). + +## OpenTelemetry + +OpenTelemetry is automatically configured for traces. +These can be visualized through Jaeger. + +### Traces + +Traces are sent to [Jaeger](http://localhost:16686). +Traces that happen in `buildctl` and `buildx` are automatically sent to `buildkit` which forwards them to Jaeger. diff --git a/hack/composefiles/buildkitd.toml b/hack/composefiles/buildkitd.toml index f32cc92ce194..6d1df14d1f9b 100644 --- a/hack/composefiles/buildkitd.toml +++ b/hack/composefiles/buildkitd.toml @@ -1,5 +1,4 @@ debug = true [grpc] - address = [ "tcp://0.0.0.0:1234" ] debugAddress = "0.0.0.0:6060" diff --git a/hack/composefiles/compose.yaml b/hack/composefiles/compose.yaml index 59686d03b44f..e7f6adbc8cee 100644 --- a/hack/composefiles/compose.yaml +++ b/hack/composefiles/compose.yaml @@ -8,28 +8,43 @@ services: BUILDKIT_DEBUG: 1 image: moby/buildkit:local ports: - - 127.0.0.1:1234:1234 - 127.0.0.1:5000:5000 - 127.0.0.1:6060:6060 restart: always privileged: true environment: DELVE_PORT: 5000 - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: http://jaeger:4317 + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 configs: - source: buildkit_config target: /etc/buildkit/buildkitd.toml volumes: - buildkit:/var/lib/buildkit + depends_on: + - otel-collector jaeger: image: jaegertracing/all-in-one:latest ports: - 127.0.0.1:16686:16686 + otel-collector: + image: otel/opentelemetry-collector-contrib:0.92.0 + restart: always + configs: + - source: otelcol_config + target: /etc/otelcol-contrib/config.yaml + ports: + - 127.0.0.1:4317:4317 + depends_on: + - jaeger + volumes: buildkit: configs: buildkit_config: file: ./buildkitd.toml + + otelcol_config: + file: ./otelcol.yaml diff --git a/hack/composefiles/extensions/buildx.yaml b/hack/composefiles/extensions/buildx.yaml new file mode 100644 index 000000000000..ffb08650afce --- /dev/null +++ b/hack/composefiles/extensions/buildx.yaml @@ -0,0 +1,20 @@ +services: + otel-collector: + command: >- + --config=file:/etc/otelcol-contrib/config.yaml + --config=file:/etc/otelcol-contrib/buildx.yaml + configs: + - source: otelcol_buildx_config + target: /etc/otelcol-contrib/buildx.yaml + +configs: + otelcol_buildx_config: + content: | + processors: + filter/buildx: + metrics::metric: + - 'instrumentation_scope.name != "github.com/docker/buildx"' + exporters::debug::verbosity: detailed + service::pipelines::metrics::receivers: [otlp] + service::pipelines::metrics::processors: [filter/buildx] + service::pipelines::metrics::exporters: [debug] diff --git a/hack/composefiles/otelcol.yaml b/hack/composefiles/otelcol.yaml new file mode 100644 index 000000000000..cb197bf41c2c --- /dev/null +++ b/hack/composefiles/otelcol.yaml @@ -0,0 +1,14 @@ +receivers: + otlp: + protocols::grpc::endpoint: 0.0.0.0:4317 + +exporters: + otlp/jaeger: + endpoint: jaeger:4317 + tls::insecure: true + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [otlp/jaeger]