Arvados

----------------------------------------------------------------------
FAIL: server_test.go:390: IntegrationSuite.TestMetrics

path: "/metrics.json" 
path: "/_health/ping" 
server_test.go:521:
    c.Check(string(allmetrics), check.Matches, `(?ms).*\narvados_keepweb_download_apparent_backend_speed_bucket{size_range="0",le="1e\+06"} 4\n.*`)
... value string = "" +
...     "# HELP arvados_config_load_timestamp_seconds Time when config file was loaded.\n" +
...     "# TYPE arvados_config_load_timestamp_seconds gauge\n" +
...     "arvados_config_load_timestamp_seconds{sha256=\"2d847d1a50ce8c82afe287ff1318b5252ff33242420cb2978fd69c2df3222e5c\"} 1.7120919906608348e+09\n" +
...     "# HELP arvados_config_source_timestamp_seconds Timestamp of config file when it was loaded.\n" +
...     "# TYPE arvados_config_source_timestamp_seconds gauge\n" +
...     "arvados_config_source_timestamp_seconds{sha256=\"2d847d1a50ce8c82afe287ff1318b5252ff33242420cb2978fd69c2df3222e5c\"} 1.7120919906608348e+09\n" +
...     "# HELP arvados_keepweb_download_apparent_backend_speed Apparent download speed from the backend (bytes per second) when serving file downloads, bucketed by transfer size range (see https://dev.arvados.org/projects/arvados/wiki/WebDAV_performance_metrics for explanation)\n" +
...     "# TYPE arvados_keepweb_download_apparent_backend_speed histogram\n" +
...     "arvados_keepweb_download_apparent_backend_speed_bucket{size_range=\"0\",le=\"10000\"} 0\n" +
...     "arvados_keepweb_download_apparent_backend_speed_bucket{size_range=\"0\",le=\"1e+06\"} 3\n" +
...     "arvados_keepweb_download_apparent_backend_speed_bucket{size_range=\"0\",le=\"1e+07\"} 4\n" +
...     "arvados_keepweb_download_apparent_backend_speed_bucket{size_range=\"0\",le=\"1e+08\"} 4\n" +
...     "arvados_keepweb_download_apparent_backend_speed_bucket{size_range=\"0\",le=\"1e+09\"} 4\n" +
...     "arvados_keepweb_download_apparent_backend_speed_bucket{size_range=\"0\",le=\"+Inf\"} 4\n" +
...     "arvados_keepweb_download_apparent_backend_speed_sum{size_range=\"0\"} 1.7675955431746405e+06\n" +
...     "arvados_keepweb_download_apparent_backend_speed_count{size_range=\"0\"} 4\n" +
...     "# HELP arvados_keepweb_download_speed Download speed (bytes per second) bucketed by transfer size range\n" +
...     "# TYPE arvados_keepweb_download_speed histogram\n" +
...     "arvados_keepweb_download_speed_bucket{size_range=\"0\",le=\"10000\"} 0\n" +
...     "arvados_keepweb_download_speed_bucket{size_range=\"0\",le=\"1e+06\"} 3\n" +
...     "arvados_keepweb_download_speed_bucket{size_range=\"0\",le=\"1e+07\"} 4\n" +
...     "arvados_keepweb_download_speed_bucket{size_range=\"0\",le=\"1e+08\"} 4\n" +
...     "arvados_keepweb_download_speed_bucket{size_range=\"0\",le=\"1e+09\"} 4\n" +
...     "arvados_keepweb_download_speed_bucket{size_range=\"0\",le=\"+Inf\"} 4\n" +
...     "arvados_keepweb_download_speed_sum{size_range=\"0\"} 1.7547408130565942e+06\n" +
...     "arvados_keepweb_download_speed_count{size_range=\"0\"} 4\n" +
...     "# HELP arvados_keepweb_sessions_active Number of active token sessions.\n" +
...     "# TYPE arvados_keepweb_sessions_active gauge\n" +
...     "arvados_keepweb_sessions_active 1\n" +
...     "# HELP arvados_keepweb_sessions_cached_session_bytes Total size of all cached sessions.\n" +
...     "# TYPE arvados_keepweb_sessions_cached_session_bytes gauge\n" +
...     "arvados_keepweb_sessions_cached_session_bytes 1208\n" +
...     "# HELP arvados_keepweb_sessions_hits Number of token session cache hits.\n" +
...     "# TYPE arvados_keepweb_sessions_hits counter\n" +
...     "arvados_keepweb_sessions_hits 6\n" +
...     "# HELP arvados_keepweb_sessions_misses Number of token session cache misses.\n" +
...     "# TYPE arvados_keepweb_sessions_misses counter\n" +
...     "arvados_keepweb_sessions_misses 1\n" +
...     "# HELP arvados_keepweb_upload_speed Upload speed (bytes per second) bucketed by transfer size range\n" +
...     "# TYPE arvados_keepweb_upload_speed histogram\n" +
...     "arvados_keepweb_upload_speed_bucket{size_range=\"0\",le=\"10000\"} 0\n" +
...     "arvados_keepweb_upload_speed_bucket{size_range=\"0\",le=\"1e+06\"} 1\n" +
...     "arvados_keepweb_upload_speed_bucket{size_range=\"0\",le=\"1e+07\"} 1\n" +
...     "arvados_keepweb_upload_speed_bucket{size_range=\"0\",le=\"1e+08\"} 1\n" +
...     "arvados_keepweb_upload_speed_bucket{size_range=\"0\",le=\"1e+09\"} 2\n" +
...     "arvados_keepweb_upload_speed_bucket{size_range=\"0\",le=\"+Inf\"} 2\n" +
...     "arvados_keepweb_upload_speed_sum{size_range=\"0\"} 1.6426248257784134e+08\n" +
...     "arvados_keepweb_upload_speed_count{size_range=\"0\"} 2\n" +
...     "# HELP arvados_keepweb_upload_sync_delay_seconds Upload sync delay (time from last byte received to HTTP response)\n" +
...     "# TYPE arvados_keepweb_upload_sync_delay_seconds histogram\n" +
...     "arvados_keepweb_upload_sync_delay_seconds_bucket{size_range=\"0\",le=\"0.005\"} 0\n" +
...     "arvados_keepweb_upload_sync_delay_seconds_bucket{size_range=\"0\",le=\"0.01\"} 0\n" +
...     "arvados_keepweb_upload_sync_delay_seconds_bucket{size_range=\"0\",le=\"0.025\"} 0\n" +
...     "arvados_keepweb_upload_sync_delay_seconds_bucket{size_range=\"0\",le=\"0.05\"} 2\n" +
...     "arvados_keepweb_upload_sync_delay_seconds_bucket{size_range=\"0\",le=\"0.1\"} 2\n" +
...     "arvados_keepweb_upload_sync_delay_seconds_bucket{size_range=\"0\",le=\"0.25\"} 2\n" +
...     "arvados_keepweb_upload_sync_delay_seconds_bucket{size_range=\"0\",le=\"0.5\"} 2\n" +
...     "arvados_keepweb_upload_sync_delay_seconds_bucket{size_range=\"0\",le=\"1\"} 2\n" +
...     "arvados_keepweb_upload_sync_delay_seconds_bucket{size_range=\"0\",le=\"2.5\"} 2\n" +
...     "arvados_keepweb_upload_sync_delay_seconds_bucket{size_range=\"0\",le=\"5\"} 2\n" +
...     "arvados_keepweb_upload_sync_delay_seconds_bucket{size_range=\"0\",le=\"10\"} 2\n" +
...     "arvados_keepweb_upload_sync_delay_seconds_bucket{size_range=\"0\",le=\"+Inf\"} 2\n" +
...     "arvados_keepweb_upload_sync_delay_seconds_sum{size_range=\"0\"} 0.057429762999999995\n" +
...     "arvados_keepweb_upload_sync_delay_seconds_count{size_range=\"0\"} 2\n" +
...     "# HELP arvados_max_abandoned_request_age_seconds Maximum time since client hung up on a request whose processing thread is still running\n" +
...     "# TYPE arvados_max_abandoned_request_age_seconds gauge\n" +
...     "arvados_max_abandoned_request_age_seconds 0\n" +
...     "# HELP arvados_max_active_request_age_seconds Age of oldest active request\n" +
...     "# TYPE arvados_max_active_request_age_seconds gauge\n" +
...     "arvados_max_active_request_age_seconds 0\n" +
...     "# HELP arvados_queue_delay_seconds Time spent in the incoming request queue before start of processing\n" +
...     "# TYPE arvados_queue_delay_seconds summary\n" +
...     "arvados_queue_delay_seconds{priority=\"normal\",queue=\"api\",quantile=\"0.5\"} 5.31e-06\n" +
...     "arvados_queue_delay_seconds{priority=\"normal\",queue=\"api\",quantile=\"0.9\"} 4.112e-05\n" +
...     "arvados_queue_delay_seconds{priority=\"normal\",queue=\"api\",quantile=\"0.95\"} 4.112e-05\n" +
...     "arvados_queue_delay_seconds{priority=\"normal\",queue=\"api\",quantile=\"0.99\"} 4.112e-05\n" +
...     "arvados_queue_delay_seconds_sum{priority=\"normal\",queue=\"api\"} 7.792000000000001e-05\n" +
...     "arvados_queue_delay_seconds_count{priority=\"normal\",queue=\"api\"} 8\n" +
...     "# HELP arvados_version_running Indicated version is running.\n" +
...     "# TYPE arvados_version_running gauge\n" +
...     "arvados_version_running{version=\"dev (go1.20.6)\"} 1\n" +
...     "# HELP request_duration_seconds Summary of request duration.\n" +
...     "# TYPE request_duration_seconds summary\n" +
...     "request_duration_seconds_sum{code=\"200\",method=\"get\"} 0.028951757\n" +
...     "request_duration_seconds_count{code=\"200\",method=\"get\"} 3\n" +
...     "request_duration_seconds_sum{code=\"201\",method=\"put\"} 0.09216466000000001\n" +
...     "request_duration_seconds_count{code=\"201\",method=\"put\"} 2\n" +
...     "request_duration_seconds_sum{code=\"404\",method=\"get\"} 0.011275539\n" +
...     "request_duration_seconds_count{code=\"404\",method=\"get\"} 3\n" +
...     "# HELP time_to_status_seconds Summary of request TTFB.\n" +
...     "# TYPE time_to_status_seconds summary\n" +
...     "time_to_status_seconds_sum{code=\"200\",method=\"get\"} 0.028625047\n" +
...     "time_to_status_seconds_count{code=\"200\",method=\"get\"} 3\n" +
...     "time_to_status_seconds_sum{code=\"201\",method=\"put\"} 0.09190528\n" +
...     "time_to_status_seconds_count{code=\"201\",method=\"put\"} 2\n" +
...     "time_to_status_seconds_sum{code=\"404\",method=\"get\"} 0.010989879\n" +
...     "time_to_status_seconds_count{code=\"404\",method=\"get\"} 3\n" 
... regex string = "(?ms).*\\narvados_keepweb_download_apparent_backend_speed_bucket{size_range=\"0\",le=\"1e\\+06\"} 4\\n.*"