Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add hostname to the labels of the metrics; pull out calls to metrics … #707

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions cachito/web/api_v1.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
from cachito.errors import MessageBrokerError, NoWorkers, RequestErrorOrigin, ValidationError
from cachito.web import db
from cachito.web.content_manifest import BASE_ICM
from cachito.web.metrics import cachito_metrics
from cachito.web.metrics import cachito_metrics, requests_dec, requests_inc
from cachito.web.models import (
ConfigFileBase64,
EnvironmentVariable,
Expand Down Expand Up @@ -337,8 +337,8 @@ def create_request():
db.session.add(request)
db.session.commit()

cachito_metrics["gauge_state"].labels(state="total").inc()
cachito_metrics["gauge_state"].labels(state=request.state.state_name).inc()
requests_inc("total")
requests_inc(request.state.state_name)

if current_user.is_authenticated:
flask.current_app.logger.info(
Expand Down Expand Up @@ -438,9 +438,9 @@ def create_request():
"Failed to schedule the task for request %d. Failing the request.", request.id
)
error = "Failed to schedule the task to the workers. Please try again."
cachito_metrics["gauge_state"].labels(state=request.state.state_name).dec()
requests_dec(request.state.state_name)
request.add_state("failed", error)
cachito_metrics["gauge_state"].labels(state=request.state.state_name).inc()
requests_inc(request.state.state_name)
db.session.commit()
raise MessageBrokerError(error)

Expand Down Expand Up @@ -525,8 +525,8 @@ def patch_request(request_id):
delete_logs = False

if "state" in payload and "state_reason" in payload:
cachito_metrics["gauge_state"].labels(state=payload["state"]).inc()
cachito_metrics["gauge_state"].labels(state=request.state.state_name).dec()
requests_inc(payload["state"])
requests_dec(request.state.state_name)
new_state = payload["state"]
delete_bundle = new_state == "stale" and request.state.state_name != "failed"
if new_state in ("stale", "failed"):
Expand Down
17 changes: 15 additions & 2 deletions cachito/web/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from prometheus_flask_exporter.multiprocess import GunicornInternalPrometheusMetrics

cachito_metrics = {}
hostname = socket.gethostname()


def init_metrics(app):
Expand All @@ -17,7 +18,6 @@ def init_metrics(app):
"""
registry = CollectorRegistry()
multiproc_temp_dir = app.config["PROMETHEUS_METRICS_TEMP_DIR"]
hostname = socket.gethostname()

if not os.path.isdir(multiproc_temp_dir):
os.makedirs(multiproc_temp_dir)
Expand All @@ -27,10 +27,23 @@ def init_metrics(app):
)
metrics.init_app(app)
gauge_state = Gauge(
"cachito_requests_count", "Requests in each state", ["state"], multiprocess_mode="livesum"
"cachito_requests_count",
"Requests in each state",
["state", "host"],
multiprocess_mode="livesum",
)
request_duration = Summary(
"cachito_request_duration_seconds", "Time spent in in_progress state"
)
cachito_metrics["gauge_state"] = gauge_state
cachito_metrics["request_duration"] = request_duration


def requests_inc(state: str) -> None:
"""Increase the number of requests in given state."""
cachito_metrics["gauge_state"].labels(state=state, host=hostname).inc()


def requests_dec(state: str) -> None:
"""Decrease the number of requests in the given state."""
cachito_metrics["gauge_state"].labels(state=state, host=hostname).dec()
2 changes: 1 addition & 1 deletion requirements-test.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# This file is autogenerated by pip-compile with python 3.10
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't we keep running pip-compile using python 3.10 since it's the newest version? (if there is any difference between using 3.9 and 3.10). Apart from that, LGTM

# This file is autogenerated by pip-compile with python 3.9
# To update, run:
#
# pip-compile --generate-hashes --output-file=requirements-test.txt requirements-test.in
Expand Down
Loading