Comment 2 Maximilian Janßen 2022-07-15 13:12:32 CEST

Version: 5.0-2 errata345

Error:

The container for 4.4/prometheus=2.35.0-3 could not be started!

docker logs 687bf3ff9cc1324ee73004b11c3cb175bc350ff7f6c1957741d52d228935218e:
ts=2022-07-10T08:47:34.525Z caller=main.go:525 level=info msg="Starting Prometheus" version="(version=2.35.0, branch=HEAD, revision=6656cd29fe6ac92bab91ecec0fe162ef0f187654)"
ts=2022-07-10T08:47:34.525Z caller=main.go:530 level=info build_context="(go=go1.18.1, user=root@cf6852b14d68, date=20220421-09:53:42)"
ts=2022-07-10T08:47:34.526Z caller=main.go:531 level=info host_details="(Linux 4.19.0-20-amd64 #1 SMP Debian 4.19.235-1 (2022-03-17) x86_64 prome-76194721 (none))"
ts=2022-07-10T08:47:34.526Z caller=main.go:532 level=info fd_limits="(soft=1048576, hard=1048576)"
ts=2022-07-10T08:47:34.526Z caller=main.go:533 level=info vm_limits="(soft=unlimited, hard=unlimited)"
ts=2022-07-10T08:47:34.535Z caller=web.go:541 level=info component=web msg="Start listening for connections" address=0.0.0.0:9090
ts=2022-07-10T08:47:34.537Z caller=main.go:957 level=info msg="Starting TSDB ..."
ts=2022-07-10T08:47:34.537Z caller=web.go:573 level=info component=web msg="Router prefix" prefix=/metrics-prometheus
ts=2022-07-10T08:47:34.541Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false
ts=2022-07-10T08:47:34.553Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any"
ts=2022-07-10T08:47:34.554Z caller=head.go:536 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=35.376µs
ts=2022-07-10T08:47:34.554Z caller=head.go:542 level=info component=tsdb msg="Replaying WAL, this may take a while"
ts=2022-07-10T08:47:34.555Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5
ts=2022-07-10T08:47:34.556Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5
ts=2022-07-10T08:47:34.557Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5
ts=2022-07-10T08:47:34.558Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5
ts=2022-07-10T08:47:34.559Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5
ts=2022-07-10T08:47:34.559Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5
ts=2022-07-10T08:47:34.559Z caller=head.go:619 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=558.756µs wal_replay_duration=5.022144ms
total_replay_duration=5.70312ms
ts=2022-07-10T08:47:34.561Z caller=main.go:978 level=info fs_type=EXT4_SUPER_MAGIC


dockerd logs:
-- Logs begin at Wed 2022-06-29 19:04:41 CEST, end at Sun 2022-07-10 10:47:37 CEST. --


docker inspect:
{'Status': 'exited', 'Running': False, 'Paused': False, 'Restarting': False, 'OOMKilled': False, 'Dead': False, 'Pid': 0, 'ExitCode': 1, 'Error': '', 'StartedAt':
'2022-07-10T08:47:34.319379596Z', 'FinishedAt': '2022-07-10T08:47:34.716931112Z'}
{'Data': {'LowerDir':
'/var/lib/docker/overlay2/2bfa74a5e80f72e5a91caade6a5072f45977e60ca8b70ac71d5621ed2faf6ae1-init/diff:/var/lib/docker/overlay2/e8fca60829954c442368bebff6ae903008ecede5696f105e55e4ffac5fc47a55/diff:/var/lib/docker/overlay2/f9754022b1aa8acb34e1371ca341c6b12493817313d5926dfb5923a67961ec66/diff:/var/lib/docker/overlay2/e359efb4d97921c00ed2f05d1e6dd04e3f1d5cc36c78410f70f8fd40fd24006c/diff:/var/lib/docker/overlay2/b4566495fba89444d1c8ce7cb8a46ff50db7a39adbb70c00887c795a0d39ef0c/diff:/var/lib/docker/overlay2/0a0041eb8b59aa8ba305501a149f8a1f2ea4408c72c4e9c4941ab2934369c8ca/diff:/var/lib/docker/overlay2/c84d06c6e67073868e1e93af7af5cda12fdf75ec1513713b6df0dc36ff3d7bfd/diff:/var/lib/docker/overlay2/679f590da5cddbdf16560cf9174a2db243ba305d4d2efc3f2cdcb72bab001345/diff:/var/lib/docker/overlay2/ac2d1689e5fcfe247f199bff4d9bafcfa30acb7a3fd21bbcf9667cea4ce68aaa/diff:/var/lib/docker/overlay2/7020fc33f29993c41a7e6ed3097709c99f61f684c9d3e0adcf9b4f1cd96f98ff/diff:/var/lib/docker/overlay2/8be29402d28b97764e814c9e9066aec5e61dcbc5a7fce6ea4028e0ab6eafd25e/diff:/var/lib/docker/overlay2/4bd3bdcadcd3d8e05b2a7d01a3db15e320397fd270747fffec10151086447299/diff:/var/lib/docker/overlay2/68d9a777260cc7f3a7f6aa9ff64c05133862917fcf896bbf6658f9a429b0d108/diff',
'MergedDir': '/var/lib/docker/overlay2/2bfa74a5e80f72e5a91caade6a5072f45977e60ca8b70ac71d5621ed2faf6ae1/merged', 'UpperDir':
'/var/lib/docker/overlay2/2bfa74a5e80f72e5a91caade6a5072f45977e60ca8b70ac71d5621ed2faf6ae1/diff', 'WorkDir':
'/var/lib/docker/overlay2/2bfa74a5e80f72e5a91caade6a5072f45977e60ca8b70ac71d5621ed2faf6ae1/work'}, 'Name': 'overlay2'}

Role: domaincontroller_master

Comment 3 Maximilian Janßen 2022-07-25 14:04:46 CEST

Version: 5.0-2 errata366

Error:

The container for 4.4/prometheus=2.35.0-3 could not be started!

docker logs ff7295efcd9ec6e32524950140a7249b4906637f25f5cc43c7baafe05b393863:
ts=2022-07-22T17:40:41.294Z caller=main.go:525 level=info msg="Starting Prometheus" version="(version=2.35.0, branch=HEAD, revision=6656cd29fe6ac92bab91ecec0fe162ef0f187654)"
ts=2022-07-22T17:40:41.294Z caller=main.go:530 level=info build_context="(go=go1.18.1, user=root@cf6852b14d68, date=20220421-09:53:42)"
ts=2022-07-22T17:40:41.294Z caller=main.go:531 level=info host_details="(Linux 4.19.0-21-amd64 #1 SMP Debian 4.19.249-2 (2022-06-30) x86_64 prome-04656187 (none))"
ts=2022-07-22T17:40:41.294Z caller=main.go:532 level=info fd_limits="(soft=1048576, hard=1048576)"
ts=2022-07-22T17:40:41.294Z caller=main.go:533 level=info vm_limits="(soft=unlimited, hard=unlimited)"
ts=2022-07-22T17:40:41.296Z caller=web.go:541 level=info component=web msg="Start listening for connections" address=0.0.0.0:9090
ts=2022-07-22T17:40:41.297Z caller=main.go:957 level=info msg="Starting TSDB ..."
ts=2022-07-22T17:40:41.297Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1657195201195 maxt=1657260000000 ulid=01G7EQEZ5QYCKCWVRMAYHBVDTT
ts=2022-07-22T17:40:41.297Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1657260001195 maxt=1657324800000 ulid=01G7GN8GCRC21NEKT87RA0XVZ5
ts=2022-07-22T17:40:41.297Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1657324801195 maxt=1657389600000 ulid=01G7JK21N3ZYVD6NBST8XBMK30
ts=2022-07-22T17:40:41.297Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1657389601195 maxt=1657454400000 ulid=01G7MGVJX4JDJ4E2HR2W9N8QSP
ts=2022-07-22T17:40:41.297Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1657454401195 maxt=1657519200000 ulid=01G7PEN4507FW6TWRW6Q5BBT3D
ts=2022-07-22T17:40:41.298Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1657519201195 maxt=1657584000000 ulid=01G7RCEND1YK8T14BH6ZWHDPZM
ts=2022-07-22T17:40:41.298Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1657584001195 maxt=1657648800000 ulid=01G7TA86MS1T2WDWRD4T1ARK0E
ts=2022-07-22T17:40:41.298Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1657648801195 maxt=1657713600000 ulid=01G7W81QWGD3HQPPT810BXM7BV
ts=2022-07-22T17:40:41.298Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1657713601195 maxt=1657778400000 ulid=01G7Y5V955HMT7E4096JPPQ8FJ
ts=2022-07-22T17:40:41.298Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1657778401195 maxt=1657843200000 ulid=01G803MTD38E5YB1J8CGPPMRPG
ts=2022-07-22T17:40:41.298Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1657843201195 maxt=1657893600000 ulid=01G86KPKTRS5M38WSEFNMVGFDT
ts=2022-07-22T17:40:41.298Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1657978806780 maxt=1657987200000 ulid=01G86ZC1VXPFJ3KMQ0VYRMD961
ts=2022-07-22T17:40:41.298Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1658075766780 maxt=1658102400000 ulid=01G87TTYXVNWNN8N111TFYA0MF
ts=2022-07-22T17:40:41.298Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1658102401195 maxt=1658167200000 ulid=01G89RMG5WKC4PA08FXTX4BNWY
ts=2022-07-22T17:40:41.298Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1658167201195 maxt=1658232000000 ulid=01G8C5TP3DSYGQG9A8KT8GFDB6
ts=2022-07-22T17:40:41.298Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1658232001195 maxt=1658296800000 ulid=01G8DM7JNX0M3QWJZKH4PHAXXR
ts=2022-07-22T17:40:41.298Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1658296801195 maxt=1658361600000 ulid=01G8FJ13XZXKSSZ8DHKG56V7WZ


dockerd logs:
-- Logs begin at Fri 2022-05-27 17:12:56 CEST, end at Fri 2022-07-22 19:40:42 CEST. --
Jul 19 23:28:05 HUP-UCS-01 dockerd[1329]: time="2022-07-19T23:28:05.929189800+02:00" level=info msg="parsed scheme: \"unix\"" module=grpc
Jul 19 23:28:05 HUP-UCS-01 dockerd[1329]: time="2022-07-19T23:28:05.929553013+02:00" level=info msg="scheme \"unix\" not registered, fallback to default scheme" module=grpc
Jul 19 23:28:05 HUP-UCS-01 dockerd[1329]: time="2022-07-19T23:28:05.929894447+02:00" level=info msg="ccResolverWrapper: sending update to cc:
{[{unix:///run/containerd/containerd.sock 0  <nil>}] <nil>}" module=grpc
Jul 19 23:28:05 HUP-UCS-01 dockerd[1329]: time="2022-07-19T23:28:05.930185059+02:00" level=info msg="ClientConn switching balancer to \"pick_first\"" module=grpc
Jul 19 23:28:05 HUP-UCS-01 dockerd[1329]: time="2022-07-19T23:28:05.945760506+02:00" level=info msg="parsed scheme: \"unix\"" module=grpc
Jul 19 23:28:05 HUP-UCS-01 dockerd[1329]: time="2022-07-19T23:28:05.945931279+02:00" level=info msg="scheme \"unix\" not registered, fallback to default scheme" module=grpc
Jul 19 23:28:05 HUP-UCS-01 dockerd[1329]: time="2022-07-19T23:28:05.946067860+02:00" level=info msg="ccResolverWrapper: sending update to cc:
{[{unix:///run/containerd/containerd.sock 0  <nil>}] <nil>}" module=grpc
Jul 19 23:28:05 HUP-UCS-01 dockerd[1329]: time="2022-07-19T23:28:05.946182933+02:00" level=info msg="ClientConn switching balancer to \"pick_first\"" module=grpc
Jul 19 23:28:06 HUP-UCS-01 dockerd[1329]: time="2022-07-19T23:28:06.025800228+02:00" level=info msg="[graphdriver] using prior storage driver: overlay2"


docker inspect:
{'Status': 'exited', 'Running': False, 'Paused': False, 'Restarting': False, 'OOMKilled': False, 'Dead': False, 'Pid': 0, 'ExitCode': 1, 'Error': '', 'StartedAt':
'2022-07-22T17:40:41.217108378Z', 'FinishedAt': '2022-07-22T17:40:41.375069663Z'}
{'Data': {'LowerDir':
'/var/lib/docker/overlay2/1929f343a34b3c74fa360d2e61d1b073e18f8ccad646037b98e7b75208b2e21a-init/diff:/var/lib/docker/overlay2/bf427548e8317a7239c149aa90dbb32ad7dded8f914d9983c014f5843faf24b6/diff:/var/lib/docker/overlay2/6e7f2ba8a48c12bfd1222a74623d53bb08ab0e94748ce71eaf3e4bf17010907e/diff:/var/lib/docker/overlay2/cd642fbed5c7ba1af28603594dc26b103c20ebc8609c1d8c63129f4d283eaf29/diff:/var/lib/docker/overlay2/7ed71a070410157cc66958b9889c63b1dff910f8c60e0e3963c3d71d759b85bb/diff:/var/lib/docker/overlay2/8e24d500cdb5653c2faf788ed4ca3bdc7fe2f48f34c8aa660e40697515ad290a/diff:/var/lib/docker/overlay2/f8b8cb1f18a1a9053b7f450d2ad17c34e2170ad84aee4ac5717a47c316c4aa9d/diff:/var/lib/docker/overlay2/960add7bca998bebad475dff72b205302a86b10a332eb3f7464f5bdbaa4b3c0b/diff:/var/lib/docker/overlay2/8686b202f4c58f21e76bc2d387a48f006a3238d6b6ac3e995128fe88a46320ae/diff:/var/lib/docker/overlay2/5b946bd7c01e0e0e476e94ee63b2704a2b9d2588c2638794fa39962bd47f992b/diff:/var/lib/docker/overlay2/2eb7e6e8f44dbd2714321bd0d066ffa9ff77e3b18606e176943619fa278f632a/diff:/var/lib/docker/overlay2/c7baa1002dc622076ee771ba98e3d239adaf16e6fef5b1ab6937b4c1635333a8/diff:/var/lib/docker/overlay2/383ebe99f0c2c93b5c76783640920ff3dcabc920e4f5916f38efbb551cd50a1c/diff',
'MergedDir': '/var/lib/docker/overlay2/1929f343a34b3c74fa360d2e61d1b073e18f8ccad646037b98e7b75208b2e21a/merged', 'UpperDir':
'/var/lib/docker/overlay2/1929f343a34b3c74fa360d2e61d1b073e18f8ccad646037b98e7b75208b2e21a/diff', 'WorkDir':
'/var/lib/docker/overlay2/1929f343a34b3c74fa360d2e61d1b073e18f8ccad646037b98e7b75208b2e21a/work'}, 'Name': 'overlay2'}

Role: domaincontroller_master

Comment 4 Maximilian Janßen 2022-07-25 14:08:26 CEST

Version: 5.0-2 errata366

Error:

The container for 4.4/prometheus=2.35.0-3 could not be started!

docker logs 3a87ca6c005a9787ee52144b59eabc8f09f05b2952fb2990da09d33ab10ae714:
ts=2022-07-23T17:18:23.729Z caller=main.go:525 level=info msg="Starting Prometheus" version="(version=2.35.0, branch=HEAD, revision=6656cd29fe6ac92bab91ecec0fe162ef0f187654)"
ts=2022-07-23T17:18:23.729Z caller=main.go:530 level=info build_context="(go=go1.18.1, user=root@cf6852b14d68, date=20220421-09:53:42)"
ts=2022-07-23T17:18:23.729Z caller=main.go:531 level=info host_details="(Linux 4.19.0-21-amd64 #1 SMP Debian 4.19.249-2 (2022-06-30) x86_64 prome-12375020 (none))"
ts=2022-07-23T17:18:23.729Z caller=main.go:532 level=info fd_limits="(soft=1048576, hard=1048576)"
ts=2022-07-23T17:18:23.730Z caller=main.go:533 level=info vm_limits="(soft=unlimited, hard=unlimited)"
ts=2022-07-23T17:18:23.733Z caller=web.go:541 level=info component=web msg="Start listening for connections" address=0.0.0.0:9090
ts=2022-07-23T17:18:23.739Z caller=main.go:957 level=info msg="Starting TSDB ..."
ts=2022-07-23T17:18:23.740Z caller=web.go:573 level=info component=web msg="Router prefix" prefix=/metrics-prometheus
ts=2022-07-23T17:18:23.745Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false
ts=2022-07-23T17:18:23.747Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any"
ts=2022-07-23T17:18:23.747Z caller=head.go:536 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.611µs
ts=2022-07-23T17:18:23.747Z caller=head.go:542 level=info component=tsdb msg="Replaying WAL, this may take a while"
ts=2022-07-23T17:18:23.752Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5
ts=2022-07-23T17:18:23.753Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5
ts=2022-07-23T17:18:23.753Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5
ts=2022-07-23T17:18:23.753Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5
ts=2022-07-23T17:18:23.754Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5
ts=2022-07-23T17:18:23.754Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5
ts=2022-07-23T17:18:23.754Z caller=head.go:619 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=185.113µs wal_replay_duration=7.143927ms
total_replay_duration=7.370906ms
ts=2022-07-23T17:18:23.756Z caller=main.go:978 level=info fs_type=EXT4_SUPER_MAGIC
ts=2022-07-23T17:18:23.756Z caller=main.go:981 level=info msg="TSDB started"
ts=2022-07-23T17:18:23.756Z caller=main.go:1162 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml


dockerd logs:
-- Logs begin at Thu 2022-07-07 11:07:00 CEST, end at Sat 2022-07-23 19:18:26 CEST. --


docker inspect:
{'Status': 'exited', 'Running': False, 'Paused': False, 'Restarting': False, 'OOMKilled': False, 'Dead': False, 'Pid': 0, 'ExitCode': 1, 'Error': '', 'StartedAt':
'2022-07-23T17:18:23.655524084Z', 'FinishedAt': '2022-07-23T17:18:23.833975776Z'}
{'Data': {'LowerDir':
'/var/lib/docker/overlay2/9f4691a246d7a93ac5d2151cfce568b3381fd059c2ec5d826d2edf73454f57ce-init/diff:/var/lib/docker/overlay2/e845b404fd3a3dc8e4ecd243edd75eaf12233254c97d593f6bf8ef96548b1834/diff:/var/lib/docker/overlay2/acf04eab39326e68f1e86ebce5825da86e7cc01dc4cae6e7e794d5d13a609a69/diff:/var/lib/docker/overlay2/48e59b5b75862d910bbf6a2a30c5724fb4f7c5462d68b58f7cd5225d69b20ea2/diff:/var/lib/docker/overlay2/a8af3cdd16ec1f415252bd4776546d807e0e4b5040150d2517dce7f58af879f1/diff:/var/lib/docker/overlay2/d9e16600d6fefb29f1a34865a94a93499b98ed44c8ba8b704ddb976be6d359c9/diff:/var/lib/docker/overlay2/48ef49c3482a4878ac80f7a96126239a2282b965711461115641daa849b51971/diff:/var/lib/docker/overlay2/b6bf6e54417d4bf92f0b101bc8772c04e4141aac676231578d4ffdd935cba77d/diff:/var/lib/docker/overlay2/68f2c9b75a06c3f963b4c690e484250533b3349518e88c83d035327c0710ee6d/diff:/var/lib/docker/overlay2/a07e3f2e08283277ce4a3ae2e00a03849468350d8a4af2760b335ae39713a83c/diff:/var/lib/docker/overlay2/90318640a4d10c23eb0780dbaa583a6c19fa23bf1f1f7a68d55ee8e29dacc664/diff:/var/lib/docker/overlay2/deda612dda667baf382815ffdcdf701aaee6e1a59923f1688dc4e3989ef009ad/diff:/var/lib/docker/overlay2/46970ca1328b5f3e880de487bc7505e238e0d78ed5e2d9466dd3f743f3281e4f/diff',
'MergedDir': '/var/lib/docker/overlay2/9f4691a246d7a93ac5d2151cfce568b3381fd059c2ec5d826d2edf73454f57ce/merged', 'UpperDir':
'/var/lib/docker/overlay2/9f4691a246d7a93ac5d2151cfce568b3381fd059c2ec5d826d2edf73454f57ce/diff', 'WorkDir':
'/var/lib/docker/overlay2/9f4691a246d7a93ac5d2151cfce568b3381fd059c2ec5d826d2edf73454f57ce/work'}, 'Name': 'overlay2'}

Role: domaincontroller_master

Comment 5 Maximilian Janßen 2022-08-01 12:46:23 CEST

The container for prometheus=1.1-1 could not be started!

docker logs 05d5f9c9568cb60185dd07324af2634c99ebdd982e07c1edd78d040822a72157:
level=info ts=2021-03-13T08:48:53.946115299Z caller=main.go:238 msg="Starting Prometheus" version="(version=2.4.3, branch=HEAD, revision=167a4b4e73a8eca8df648d2d2043e21bdb9a7449)"
level=info ts=2021-03-13T08:48:53.946393899Z caller=main.go:239 build_context="(go=go1.11.1, user=root@1e42b46043e9, date=20181004-08:42:02)"
level=info ts=2021-03-13T08:48:53.946597099Z caller=main.go:240 host_details="(Linux 4.9.0-14-amd64 #1 SMP Debian 4.9.246-2 (2020-12-17) x86_64 prome-49647772 (none))"
level=info ts=2021-03-13T08:48:53.946818699Z caller=main.go:241 fd_limits="(soft=1048576, hard=1048576)"
level=info ts=2021-03-13T08:48:53.947084099Z caller=main.go:242 vm_limits="(soft=unlimited, hard=unlimited)"
level=info ts=2021-03-13T08:48:53.948592899Z caller=main.go:554 msg="Starting TSDB ..."
level=info ts=2021-03-13T08:48:53.958201599Z caller=web.go:397 component=web msg="Start listening for connections" address=0.0.0.0:9090
level=info ts=2021-03-13T08:48:53.973367599Z caller=web.go:440 component=web msg="router prefix" prefix=/metrics-prometheus
level=info ts=2021-03-13T08:48:54.036948499Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1610971200000 maxt=1610992800000 ulid=01EWBMSEYAQ1AKQ33GJK7RRKA8
level=info ts=2021-03-13T08:48:54.044438599Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1610992800000 maxt=1611057600000 ulid=01EWDJK4CKS5VD7MP3DGSJ0QTZ
level=info ts=2021-03-13T08:48:54.055024899Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1611057600000 maxt=1611122400000 ulid=01EWFGCR7V062EP7S51VQQ2XQB
level=info ts=2021-03-13T08:48:54.062807699Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1611122400000 maxt=1611187200000 ulid=01EWHE56YT5J4PB3GV5JN0YQN2
level=info ts=2021-03-13T08:48:54.065979399Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1611187200000 maxt=1611252000000 ulid=01EWKBYV0FCMFK5X6YZGDWEV3Z
level=info ts=2021-03-13T08:48:54.070778499Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1611252000000 maxt=1611316800000 ulid=01EWN9RFHKN09WHTY0H60AS0Y1
level=info ts=2021-03-13T08:48:54.076250299Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1611316800000 maxt=1611381600000 ulid=01EWQ7J3VTDJS9V9S8TZQCHXZB
level=info ts=2021-03-13T08:48:54.079160499Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1611381600000 maxt=1611446400000 ulid=01EWS5BPSCVJGGY67PAC3DE2FQ
level=info ts=2021-03-13T08:48:54.083892999Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1611446400000 maxt=1611511200000 ulid=01EWV35BQ5E84YB3M53X2QKVT4
level=info ts=2021-03-13T08:48:54.084402799Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1611511200000 maxt=1611576000000 ulid=01EWX0Z027F6T5D79G1XMQWPNY
level=info ts=2021-03-13T08:48:54.084802199Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1611576000000 maxt=1611640800000 ulid=01EWYYQX121ES9W0MGZHVM4PRT
level=info ts=2021-03-13T08:48:54.085211599Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1611640800000 maxt=1611705600000 ulid=01EX0WHGX8MP3H3EFNJ16AQWEG
level=info ts=2021-03-13T08:48:54.085610599Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1611705600000 maxt=1611770400000 ulid=01EX2TB2HAFS9Y55AXBQ6NY4Q5
level=info ts=2021-03-13T08:48:54.086014799Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1611770400000 maxt=1611835200000 ulid=01EX4R4PKNCNAMBQAY21BWXSAW
level=info ts=2021-03-13T08:48:54.086404799Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1611835200000 maxt=1611900000000 ulid=01EX6NYB7WW20YR9RAT1BTCWS5
level=info ts=2021-03-13T08:48:54.086819599Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1611900000000 maxt=1611964800000 ulid=01EX8KQHFAJ07DKXZ2EANQBRBT
level=info ts=2021-03-13T08:48:54.087206799Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1611964800000 maxt=1612029600000 ulid=01EXAHH4ZXBWGGF5PQ24TPNV1E
level=info ts=2021-03-13T08:48:54.087590099Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1612029600000 maxt=1612094400000 ulid=01EXCFATDKMRPAW3DXQKXQ5AVA
level=info ts=2021-03-13T08:48:54.087963899Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1612094400000 maxt=1612116000000 ulid=01EXD3XXB2P9NF8DAKDVG5B5ZC
level=info ts=2021-03-13T08:48:54.100713999Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1612137600000 maxt=1612144800000 ulid=01EXDRH2QZ220RR2VY60SXQA66
level=info ts=2021-03-13T08:48:54.101299799Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1612116000000 maxt=1612137600000 ulid=01EXDRH3M47D6P1ED37K33ZMGD
level=info ts=2021-03-13T08:48:54.101432399Z caller=main.go:423 msg="Stopping scrape discovery manager..."
level=info ts=2021-03-13T08:48:54.101451999Z caller=main.go:437 msg="Stopping notify discovery manager..."
level=info ts=2021-03-13T08:48:54.101467299Z caller=main.go:459 msg="Stopping scrape manager..."
level=info ts=2021-03-13T08:48:54.101483599Z caller=main.go:433 msg="Notify discovery manager stopped"
level=info ts=2021-03-13T08:48:54.101515199Z caller=main.go:419 msg="Scrape discovery manager stopped"
level=info ts=2021-03-13T08:48:54.101539399Z caller=manager.go:638 component="rule manager" msg="Stopping rule manager..."
level=info ts=2021-03-13T08:48:54.101555499Z caller=manager.go:644 component="rule manager" msg="Rule manager stopped"
level=info ts=2021-03-13T08:48:54.101573799Z caller=notifier.go:512 component=notifier msg="Stopping notification manager..."
level=info ts=2021-03-13T08:48:54.101595899Z caller=main.go:608 msg="Notifier manager stopped"
level=info ts=2021-03-13T08:48:54.101615899Z caller=main.go:453 msg="Scrape manager stopped"
level=error ts=2021-03-13T08:48:54.101702099Z caller=main.go:617 err="opening storage failed: block dir: \"data/01EXDZCFBNVD53ATKWAFZ5ABZX\": unexpected end of JSON input"


dockerd logs:
-- Logs begin at Sat 2020-12-26 09:50:01 CET, end at Sat 2021-03-13 09:49:01 CET. --
mars 09 18:50:28 www dockerd[2029]: time="2021-03-09T18:50:28.759736199+01:00" level=warning msg="Error (Unable to complete atomic operation, key modified) deleting object [endpoint 7f928a8c3bc6270a594558d389e5c48c37fcea9cb52b5f43bf4aa5ae6100196d 025f60a6cc6fd8cc5c4f1a9155117da6ed4b4b272370bf3e44ab9aa6c968867e], retrying...."
mars 09 18:50:30 www dockerd[2029]: time="2021-03-09T18:50:30.201219699+01:00" level=info msg="Removing stale sandbox adbc432f46ea593169795152a4c7581aaaca5beeb9306768c86a8517ad9367e2 (24e9768882a0981dc2cf9eb0cf8d18ca19c6403632619bd01e47165e0203b5f1)"
mars 09 18:50:30 www dockerd[2029]: time="2021-03-09T18:50:30.734857599+01:00" level=warning msg="Error (Unable to complete atomic operation, key modified) deleting object [endpoint 7f928a8c3bc6270a594558d389e5c48c37fcea9cb52b5f43bf4aa5ae6100196d 599d9be4d2de66593a897d11f28e78fe09f89a79730076a423cb5ad600584eac], retrying...."
mars 09 18:50:32 www dockerd[2029]: time="2021-03-09T18:50:32.004871499+01:00" level=info msg="Removing stale sandbox 19898378dcef08a512942d08216f6ee3c49074a8d819de708fa9b1bc717d3214 (92976171beea08efb16f7d297213f609cec839aaddf0e234d0135e77f6038e54)"
mars 09 18:50:32 www dockerd[2029]: time="2021-03-09T18:50:32.645894999+01:00" level=warning msg="Error (Unable to complete atomic operation, key modified) deleting object [endpoint 6a978f4b9c5c563dd167a223f908080a767c5cc9e594e769a9e908fcede5d80c ec8486e06a538599160aec45dd1e2c87109acdea4f3c3d84bcc547cbc5c63382], retrying...."
mars 09 18:50:33 www dockerd[2029]: time="2021-03-09T18:50:33.866358899+01:00" level=info msg="Removing stale sandbox 3893fe45fd73f3a052592edfec60470e841024a1bb786095cef457936e3e7b65 (ffe2d7c2e06307297f0630d0d2cd158d0cdbdbbb71a099c5d1b17fcadd9debdc)"
mars 09 18:50:34 www dockerd[2029]: time="2021-03-09T18:50:34.321053899+01:00" level=warning msg="Error (Unable to complete atomic operation, key modified) deleting object [endpoint 6a978f4b9c5c563dd167a223f908080a767c5cc9e594e769a9e908fcede5d80c a28c0c19ee342ea37738f97a1f39b703e0924c70a32b807b808b436a6d2b77e8], retrying...."
mars 09 18:50:35 www dockerd[2029]: time="2021-03-09T18:50:35.369932499+01:00" level=info msg="Removing stale sandbox 62dabe5c603e4cac73fd1ddf126f546aec9d5561863a32aa334ad7b8de1e5dd8 (4705af8b79a50ff222f8767334f45d7a3e15b2be8863a321f742d9e784fe4a6a)"
mars 09 18:50:35 www dockerd[2029]: time="2021-03-09T18:50:35.809579599+01:00" level=warning msg="Error (Unable to complete atomic operation, key modified) deleting object [endpoint 7f928a8c3bc6270a594558d389e5c48c37fcea9cb52b5f43bf4aa5ae6100196d 9d9f6088410933237db6ce5710981246beb0c8fa9f0a13b4e198e4f021f17d1b], retrying...."
mars 09 18:50:53 www dockerd[2029]: time="2021-03-09T18:50:53.657925899+01:00" level=info msg="Loading containers: done."
mars 09 18:50:53 www dockerd[2029]: time="2021-03-09T18:50:53.886371899+01:00" level=warning msg="failed to retrieve runc version: unknown output format: runc version spec: 1.0.1-dev\n"
mars 09 18:50:54 www dockerd[2029]: time="2021-03-09T18:50:54.173239099+01:00" level=info msg="Docker daemon" commit=2d0083d graphdriver(s)=overlay2 version=18.09.7
mars 09 18:50:54 www dockerd[2029]: time="2021-03-09T18:50:54.420397599+01:00" level=info msg="Daemon has completed initialization"
mars 09 18:50:56 www dockerd[2029]: time="2021-03-09T18:50:56.653029499+01:00" level=info msg="API listen on /var/run/docker.sock"


docker inspect:
{u'Status': u'exited', u'Pid': 0, u'OOMKilled': False, u'Dead': False, u'Paused': False, u'Running': False, u'FinishedAt': u'2021-03-13T08:48:54.141417599Z', u'Restarting': False, u'Error': u'', u'StartedAt': u'2021-03-13T08:48:53.029335299Z', u'ExitCode': 1}
{u'Data': {u'MergedDir': u'/var/lib/docker/overlay2/abf0df788a0be220f248eb6321d6a5a0cea421545ca4b132dce16f7aca091375/merged', u'WorkDir': u'/var/lib/docker/overlay2/abf0df788a0be220f248eb6321d6a5a0cea421545ca4b132dce16f7aca091375/work', u'LowerDir': u'/var/lib/docker/overlay2/abf0df788a0be220f248eb6321d6a5a0cea421545ca4b132dce16f7aca091375-init/diff:/var/lib/docker/overlay2/fd9192043f8646e8114d164f35aebc9bd1ba46ae86c21ccfe3b855583a4b3a5b/diff:/var/lib/docker/overlay2/56ef906c4845bf70d7e86c9d7e2a9ffcc15eec3c772fc1f438c969b3ca46d35f/diff:/var/lib/docker/overlay2/74a7028de79722a1563180f2b4e539cd861ae784ffb8cb21dec40618690a33da/diff:/var/lib/docker/overlay2/132642d97282ce080a67d7491e5169035a75c40a9d96f4f672687679d3a35969/diff:/var/lib/docker/overlay2/8a86b05163270d73339b3bc0fa0e039d96758ab29d6ebb0f2d1ced79eaecf666/diff:/var/lib/docker/overlay2/b60422b0f1f353af74e59906715d9b8efff405b65854bb39ca76dde1f2febf21/diff:/var/lib/docker/overlay2/d25035137621ed8995fecbd2b8d18e105c0a34f0b7224896aafacd8a1cdca611/diff:/var/lib/docker/overlay2/cd85e72e7b83543727c86126b4d4e1fb18d8e355a02d58ce1bf0c69bcc8738d7/diff:/var/lib/docker/overlay2/505bfce6be969ceea0f4d5ff65537d5bfd644aca3ac3b0387f77873bada707dd/diff', u'UpperDir': u'/var/lib/docker/overlay2/abf0df788a0be220f248eb6321d6a5a0cea421545ca4b132dce16f7aca091375/diff'}, u'Name': u'overlay2'}

Comment 6 Riya Bhattacharjee 2022-08-03 15:37:45 CEST

This issue again reported in feedback ticket 2022072521000146

Comment 7 Maximilian Janßen 2022-08-10 12:59:48 CEST

Version: 5.0-2 errata382

Internal server error during "appcenter/progress".
Request: appcenter/progress

  File "%PY3%/univention/lib/umc.py", line 561, in send
    raise HTTPError(request, response, self.hostname)

Response Content:

location: https://***.***.**/univention/command
result: {'display_feedback': True, 'title': 'The docker container could not be started! We are sorry for the inconvenience. Please help us to improve the App Center and the Apps by sending us the information below.'}
status: 400
traceback:
None


message:

The container for 4.4/prometheus=2.35.0-4 could not be started!

docker logs 2d69d2f92f774e89f5a38328d9214761a0ab9b7eebd9b7e14c0dfff9c40c4c63:
ts=2022-08-09T21:31:01.424Z caller=main.go:525 level=info msg="Starting Prometheus" version="(version=2.35.0, branch=HEAD, revision=6656cd29fe6ac92bab91ecec0fe162ef0f187654)"
ts=2022-08-09T21:31:01.424Z caller=main.go:530 level=info build_context="(go=go1.18.1, user=root@cf6852b14d68, date=20220421-09:53:42)"
ts=2022-08-09T21:31:01.424Z caller=main.go:531 level=info host_details="(Linux 4.19.0-21-amd64 #1 SMP Debian 4.19.249-2 (2022-06-30) x86_64 prome-21763624 (none))"
ts=2022-08-09T21:31:01.424Z caller=main.go:532 level=info fd_limits="(soft=1048576, hard=1048576)"
ts=2022-08-09T21:31:01.424Z caller=main.go:533 level=info vm_limits="(soft=unlimited, hard=unlimited)"
ts=2022-08-09T21:31:01.426Z caller=web.go:541 level=info component=web msg="Start listening for connections" address=***.***.***.***:9090
ts=2022-08-09T21:31:01.426Z caller=main.go:957 level=info msg="Starting TSDB ..."
ts=2022-08-09T21:31:01.432Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any"
ts=2022-08-09T21:31:01.436Z caller=head.go:536 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=4.51µs
ts=2022-08-09T21:31:01.436Z caller=head.go:542 level=info component=tsdb msg="Replaying WAL, this may take a while"
ts=2022-08-09T21:31:01.444Z caller=web.go:573 level=info component=web msg="Router prefix" prefix=/metrics-prometheus
ts=2022-08-09T21:31:01.445Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false
ts=2022-08-09T21:31:01.445Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4
ts=2022-08-09T21:31:01.445Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4
ts=2022-08-09T21:31:01.446Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4
ts=2022-08-09T21:31:01.447Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4
ts=2022-08-09T21:31:01.448Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4
ts=2022-08-09T21:31:01.448Z caller=head.go:619 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=142.241µs wal_replay_duration=11.35427ms total_replay_duration=11.538312ms
ts=2022-08-09T21:31:01.449Z caller=main.go:978 level=info fs_type=EXT4_SUPER_MAGIC
ts=2022-08-09T21:31:01.449Z caller=main.go:981 level=info msg="TSDB started"
ts=2022-08-09T21:31:01.449Z caller=main.go:1162 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
ts=2022-08-09T21:31:01.463Z caller=manager.go:973 level=error component="rule manager" msg="loading groups failed" err="/etc/prometheus/alert_UNIVENTION_DISK_ROOT.yml: 14:11: group \"UNIVENTION_DISK_ROOT\", rule 1, \"UNIVENTION_DISK_ROOT\": could not parse expression: 1:50: parse error: unexpected character inside braces: '%'"
ts=2022-08-09T21:31:01.469Z caller=main.go:1188 level=error msg="Failed to apply configuration" err="error loading rules, previous rule set restored"
ts=2022-08-09T21:31:01.469Z caller=main.go:819 level=info msg="Stopping scrape discovery manager..."
ts=2022-08-09T21:31:01.469Z caller=main.go:833 level=info msg="Stopping notify discovery manager..."
ts=2022-08-09T21:31:01.469Z caller=main.go:855 level=info msg="Stopping scrape manager..."
ts=2022-08-09T21:31:01.469Z caller=main.go:815 level=info msg="Scrape discovery manager stopped"
ts=2022-08-09T21:31:01.484Z caller=main.go:829 level=info msg="Notify discovery manager stopped"
ts=2022-08-09T21:31:01.484Z caller=manager.go:950 level=info component="rule manager" msg="Stopping rule manager..."
ts=2022-08-09T21:31:01.484Z caller=manager.go:960 level=info component="rule manager" msg="Rule manager stopped"
ts=2022-08-09T21:31:01.484Z caller=main.go:849 level=info msg="Scrape manager stopped"
ts=2022-08-09T21:31:01.485Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..."
ts=2022-08-09T21:31:01.485Z caller=main.go:1088 level=info msg="Notifier manager stopped"
ts=2022-08-09T21:31:01.485Z caller=main.go:1097 level=error err="error loading config from \"/etc/prometheus/prometheus.yml\": one or more errors occurred while applying the new configuration (--config.file=\"/etc/prometheus/prometheus.yml\")"


dockerd logs:
-- Logs begin at Wed 2022-05-18 08:00:01 CEST, end at Tue 2022-08-09 23:31:02 CEST. --


docker inspect:
{'Status': 'exited', 'Running': False, 'Paused': False, 'Restarting': False, 'OOMKilled': False, 'Dead': False, 'Pid': 0, 'ExitCode': 1, 'Error': '', 'StartedAt': '2022-08-09T21:31:01.355037712Z', 'FinishedAt': '2022-08-09T21:31:01.502560441Z'}
{'Data': {'LowerDir': '/var/lib/docker/overlay2/94c887afc44cdf78dac9b3b2fbc0f56bcdf3c88ed3befd95445c8a2cd42de296-init/diff:/var/lib/docker/overlay2/125753ebb6715b25e5f8af34fb4cea79ea7ac6dee6f9af7a1d02acd50c1d544a/diff:/var/lib/docker/overlay2/a38b0086eacc40e0d4c00743ce22f9a26a510949ecf7fb1f1db555d43d6796f7/diff:/var/lib/docker/overlay2/09bb1d989a7babe7c8b9626f19bf1ba1450f544345d5015d08710055befbd956/diff:/var/lib/docker/overlay2/9df75f92cc7301b5afa667f2b89826203989c4a8ca34b5a1e17505a2ae2f5724/diff:/var/lib/docker/overlay2/6b22824a81ac97bbc00c21ebae25b6d51379b101110db42b8a4706ef1c1116f8/diff:/var/lib/docker/overlay2/a9c7b8ec3f697084a5db063092a55f38fa65d4d9b42f8bd944cc6b35a1174bae/diff:/var/lib/docker/overlay2/3e605fc107fc6c9b6b91f6121570d225fb581e9d4ef9d85b3148176e49bd372e/diff:/var/lib/docker/overlay2/b9d521bde068f292146965419538e58f63bf7e849a23a8a1b178978888c6633a/diff:/var/lib/docker/overlay2/6606a2e656b536a1420bdce49548582e5e269d56d128d3d3333e2d750c5a3c19/diff:/var/lib/docker/overlay2/04c5120c97e5a66c7b348e096dd7b4b56572e957a30eea346fdbeda7e45d290b/diff:/var/lib/docker/overlay2/67c85f241813623cd6858b3bdfd9fd903b5cf62ef75012cb34f43c3c8b92a566/diff:/var/lib/docker/overlay2/232a2ddc730db00f50c922be581f0aa5703cad439813449142c80e18977c605a/diff', 'MergedDir': '/var/lib/docker/overlay2/94c887afc44cdf78dac9b3b2fbc0f56bcdf3c88ed3befd95445c8a2cd42de296/merged', 'UpperDir': '/var/lib/docker/overlay2/94c887afc44cdf78dac9b3b2fbc0f56bcdf3c88ed3befd95445c8a2cd42de296/diff', 'WorkDir': '/var/lib/docker/overlay2/94c887afc44cdf78dac9b3b2fbc0f56bcdf3c88ed3befd95445c8a2cd42de296/work'}, 'Name': 'overlay2'}

Comment 8 Maximilian Janßen 2022-08-10 14:36:54 CEST

Version: 5.0-2 errata382

The container for 4.4/prometheus=2.35.0-4 could not be started!

docker logs 6528a59939cd2a8502f02f35a700c9bafde3edfde825fae0636e8e68b43c0f98:
ts=2022-08-10T12:22:49.499Z caller=main.go:525 level=info msg="Starting Prometheus" version="(version=2.35.0, branch=HEAD, revision=6656cd29fe6ac92bab91ecec0fe162ef0f187654)"
ts=2022-08-10T12:22:49.499Z caller=main.go:530 level=info build_context="(go=go1.18.1, user=root@cf6852b14d68, date=20220421-09:53:42)"
ts=2022-08-10T12:22:49.499Z caller=main.go:531 level=info host_details="(Linux 4.19.0-21-amd64 #1 SMP Debian 4.19.249-2 (2022-06-30) x86_64 prome-33611061 (none))"
ts=2022-08-10T12:22:49.499Z caller=main.go:532 level=info fd_limits="(soft=1048576, hard=1048576)"
ts=2022-08-10T12:22:49.499Z caller=main.go:533 level=info vm_limits="(soft=unlimited, hard=unlimited)"
ts=2022-08-10T12:22:49.513Z caller=web.go:541 level=info component=web msg="Start listening for connections" address=***.***.***.***:9090
ts=2022-08-10T12:22:49.514Z caller=main.go:957 level=info msg="Starting TSDB ..."
ts=2022-08-10T12:22:49.515Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1658815228620 maxt=1658880000000 ulid=01G8Z0E9FP3QXNCQJZPWZSKT7R
ts=2022-08-10T12:22:49.514Z caller=web.go:573 level=info component=web msg="Router prefix" prefix=/metrics-prometheus
ts=2022-08-10T12:22:49.517Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false
ts=2022-08-10T12:22:49.515Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1658880028620 maxt=1658944800000 ulid=01G90Y7TB3THFVDCEVZYBYDDQB
ts=2022-08-10T12:22:49.517Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1658944828620 maxt=1659009600000 ulid=01G92W1BHSYPYFE5KE6KRYMFCN
ts=2022-08-10T12:22:49.517Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1659009628620 maxt=1659074400000 ulid=01G94STWY4X3HFY4DZR9VTFCCF
ts=2022-08-10T12:22:49.518Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1659074428620 maxt=1659139200000 ulid=01G96QMDNT5J305ASF3GMFF4K4
ts=2022-08-10T12:22:49.518Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1659139228620 maxt=1659204000000 ulid=01G98NDYH9E10TM0MM8RE88X2Z
ts=2022-08-10T12:22:49.518Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1659204028620 maxt=1659268800000 ulid=01G9AK7GECQDNXQFHJFD2YW29H
ts=2022-08-10T12:22:49.518Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1659268828620 maxt=1659333600000 ulid=01G9CH11QN3P7PPGAY6RBQ0423
ts=2022-08-10T12:22:49.518Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1659333628620 maxt=1659398400000 ulid=01G9EETJM1YAQAQ32FYR6ZM4AF
ts=2022-08-10T12:22:49.518Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1659398428620 maxt=1659463200000 ulid=01G9GCM4T0Y84NMJ3Q1DYJ862Y
ts=2022-08-10T12:22:49.518Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1659463228620 maxt=1659528000000 ulid=01G9JADMWK2TFE37V979SCW4DE
ts=2022-08-10T12:22:49.519Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1659528028620 maxt=1659592800000 ulid=01G9M8764G84NDQ12WFDXKCBTQ
ts=2022-08-10T12:22:49.519Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1659592828620 maxt=1659657600000 ulid=01G9P60Q2ZD8HVB2W3AJVVWJCN
ts=2022-08-10T12:22:49.519Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1659657628620 maxt=1659722400000 ulid=01G9R3T8JZDEGBBQZQ6HEXA4QB
ts=2022-08-10T12:22:49.519Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1659722428620 maxt=1659787200000 ulid=01G9T1KSDYNM6HD1R4JCYJECEG
ts=2022-08-10T12:22:49.519Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1659787228620 maxt=1659852000000 ulid=01G9VZDBMY6VKTJW2JPR7AN5J9
ts=2022-08-10T12:22:49.519Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1659852028620 maxt=1659916800000 ulid=01G9XX6X6PRZVRP3ZMMVJ2ZQ1V
ts=2022-08-10T12:22:49.519Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1659916828620 maxt=1659981600000 ulid=01G9ZV0E5WSMCBSFFNX7A7YD87
ts=2022-08-10T12:22:49.520Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1659981628620 maxt=1660046400000 ulid=01GA1RSZFGXG2REFFFMWMV87RD
ts=2022-08-10T12:22:49.520Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1660111206090 maxt=1660118400000 ulid=01GA3FQ0JZAE0NFRF3JSJSQ9EW
ts=2022-08-10T12:22:49.520Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1660118406091 maxt=1660125600000 ulid=01GA3PKTR36R0TJN6RH4S6PHYN
ts=2022-08-10T12:22:49.520Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1660046428620 maxt=1660111200000 ulid=01GA3PKWPDJHCFFYRNPAZAXTPS
ts=2022-08-10T12:22:49.602Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any"
ts=2022-08-10T12:22:49.622Z caller=head.go:536 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=19.75638ms
ts=2022-08-10T12:22:49.622Z caller=head.go:542 level=info component=tsdb msg="Replaying WAL, this may take a while"
ts=2022-08-10T12:22:51.134Z caller=head.go:578 level=info component=tsdb msg="WAL checkpoint loaded"
ts=2022-08-10T12:22:51.209Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=925 maxSegment=929
ts=2022-08-10T12:22:51.305Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=926 maxSegment=929
ts=2022-08-10T12:22:51.518Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=927 maxSegment=929
ts=2022-08-10T12:22:51.518Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=928 maxSegment=929
ts=2022-08-10T12:22:51.519Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=929 maxSegment=929
ts=2022-08-10T12:22:51.519Z caller=head.go:619 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=1.512197664s wal_replay_duration=384.837358ms total_replay_duration=1.916897514s
ts=2022-08-10T12:22:51.528Z caller=main.go:978 level=info fs_type=EXT4_SUPER_MAGIC
ts=2022-08-10T12:22:51.529Z caller=main.go:981 level=info msg="TSDB started"
ts=2022-08-10T12:22:51.529Z caller=main.go:1162 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
ts=2022-08-10T12:22:51.546Z caller=manager.go:973 level=error component="rule manager" msg="loading groups failed" err="/etc/prometheus/alert_UNIVENTION_DISK_ROOT.yml: 14:11: group \"UNIVENTION_DISK_ROOT\", rule 1, \"UNIVENTION_DISK_ROOT\": could not parse expression: 1:50: parse error: unexpected character inside braces: '%'"
ts=2022-08-10T12:22:51.546Z caller=main.go:1188 level=error msg="Failed to apply configuration" err="error loading rules, previous rule set restored"
ts=2022-08-10T12:22:51.546Z caller=main.go:819 level=info msg="Stopping scrape discovery manager..."
ts=2022-08-10T12:22:51.546Z caller=main.go:833 level=info msg="Stopping notify discovery manager..."
ts=2022-08-10T12:22:51.546Z caller=main.go:855 level=info msg="Stopping scrape manager..."
ts=2022-08-10T12:22:51.547Z caller=main.go:815 level=info msg="Scrape discovery manager stopped"
ts=2022-08-10T12:22:51.547Z caller=main.go:829 level=info msg="Notify discovery manager stopped"
ts=2022-08-10T12:22:51.547Z caller=manager.go:950 level=info component="rule manager" msg="Stopping rule manager..."
ts=2022-08-10T12:22:51.547Z caller=manager.go:960 level=info component="rule manager" msg="Rule manager stopped"
ts=2022-08-10T12:22:51.547Z caller=main.go:849 level=info msg="Scrape manager stopped"
ts=2022-08-10T12:22:51.596Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..."
ts=2022-08-10T12:22:51.596Z caller=main.go:1088 level=info msg="Notifier manager stopped"
ts=2022-08-10T12:22:51.596Z caller=main.go:1097 level=error err="error loading config from \"/etc/prometheus/prometheus.yml\": one or more errors occurred while applying the new configuration (--config.file=\"/etc/prometheus/prometheus.yml\")"


dockerd logs:
-- Logs begin at Fri 2022-05-20 07:05:32 CEST, end at Wed 2022-08-10 14:22:53 CEST. --
Jul 29 08:05:28 UCS02 dockerd[5342]: time="2022-07-29T08:05:28.321279014+02:00" level=info msg="Daemon has completed initialization"
Jul 29 08:05:28 UCS02 dockerd[5342]: time="2022-07-29T08:05:28.394872678+02:00" level=info msg="API listen on /var/run/docker.sock"
Aug 09 21:58:58 UCS02 dockerd[5342]: time="2022-08-09T21:58:58.104676236+02:00" level=info msg="Container 88b5379f37c2754acf432e22f318db3dfb94f14529aceaf05781c07412f77866 failed to exit within 60 seconds of signal 15 - using the force"
Aug 09 21:58:58 UCS02 dockerd[5342]: time="2022-08-09T21:58:58.369473048+02:00" level=error msg="stream copy error: reading from a closed fifo"
Aug 09 21:58:58 UCS02 dockerd[5342]: time="2022-08-09T21:58:58.378107363+02:00" level=error msg="Error running exec 2da999d9d18916fe890ce495f7cfe8f95c82f6c272c11135f43b7a51ebe6a91c in container: OCI runtime exec failed: exec failed: container_linux.go:349: starting container process caused \"process_linux.go:101: executing setns process caused \\\"exit status 1\\\"\": unknown"
Aug 09 21:58:58 UCS02 dockerd[5342]: time="2022-08-09T21:58:58.450980078+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 09 21:59:00 UCS02 dockerd[5342]: time="2022-08-09T21:59:00.320094841+02:00" level=error msg="Error setting up exec command in container 88b5379f37c2: Container 88b5379f37c2754acf432e22f318db3dfb94f14529aceaf05781c07412f77866 is not running"
Aug 10 14:09:49 UCS02 dockerd[5342]: time="2022-08-10T14:09:49.720096918+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 10 14:10:44 UCS02 dockerd[5342]: time="2022-08-10T14:10:44.886737827+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 10 14:12:13 UCS02 dockerd[5342]: time="2022-08-10T14:12:13.202515557+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 10 14:12:14 UCS02 dockerd[5342]: time="2022-08-10T14:12:14.277564629+02:00" level=error msg="Error setting up exec command in container 71848de33e31: Container 71848de33e3151a2efd0f191db19eb7733d6bab15d5540c818ac8a6701beaa14 is not running"
Aug 10 14:14:34 UCS02 dockerd[5342]: time="2022-08-10T14:14:34.954451250+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 10 14:14:44 UCS02 dockerd[5342]: time="2022-08-10T14:14:44.625484372+02:00" level=info msg="Container 0db2bec7b5f5c796956a2513b868156c69c05b61c049e3c084116a948e447c81 failed to exit within 10 seconds of signal 15 - using the force"
Aug 10 14:14:45 UCS02 dockerd[5342]: time="2022-08-10T14:14:45.298963232+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 10 14:15:46 UCS02 dockerd[5342]: time="2022-08-10T14:15:46.822660213+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 10 14:15:48 UCS02 dockerd[5342]: time="2022-08-10T14:15:48.171794883+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 10 14:16:13 UCS02 dockerd[5342]: time="2022-08-10T14:16:13.473272971+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 10 14:22:29 UCS02 dockerd[5342]: time="2022-08-10T14:22:29.474661458+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 10 14:22:37 UCS02 dockerd[5342]: time="2022-08-10T14:22:37.953954586+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 10 14:22:51 UCS02 dockerd[5342]: time="2022-08-10T14:22:51.886986951+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"


docker inspect:
{'Status': 'exited', 'Running': False, 'Paused': False, 'Restarting': False, 'OOMKilled': False, 'Dead': False, 'Pid': 0, 'ExitCode': 1, 'Error': '', 'StartedAt': '2022-08-10T12:22:49.355016954Z', 'FinishedAt': '2022-08-10T12:22:51.643881338Z'}
{'Data': {'LowerDir': '/var/lib/docker/overlay2/0e1eb24cab725772d3ff50453236ecc58009bac69a07c6e05f90c64ce5ac39ad-init/diff:/var/lib/docker/overlay2/8f7a5b67a52d289d49593b219078db9faf1b635ba996a56acadb3d17861f9426/diff:/var/lib/docker/overlay2/f7229700c0b2a61b922d38ab5e2973c0511190ab9273ee9a49a48ca5df67f319/diff:/var/lib/docker/overlay2/6e2fe2d9efec2ef59779217cdd040d982c335f4b1b56ecf46bade55e5685943b/diff:/var/lib/docker/overlay2/d71c3fe1d014f9eaa8bd972fe840648f66d039c3cf9e091cf6732839dc20f76b/diff:/var/lib/docker/overlay2/ac60f3c2933e491084da864f3b831c1fc56e74dd081d5686e3aa08256ec11ed0/diff:/var/lib/docker/overlay2/fafd39f847c0db5d836b4fc492e9d7823c79e56903c09bc785d5cd30cbb0484a/diff:/var/lib/docker/overlay2/aac5a8664796706d82ca0ceee0d799afbe278ab04582c2de031d425c8f384788/diff:/var/lib/docker/overlay2/b7dfedd1d7ef1e7fd59bbd1dfbd9c0ae4c5f9736748a86a5725aa4ece388c20e/diff:/var/lib/docker/overlay2/2b5be5cd8da13f9b543e6428f5caf1e42ee79961a84681e974a018158e9ac62c/diff:/var/lib/docker/overlay2/36d77c4e2dd998bf6bf9ad1dd909442309a3d74af7ceb87b73dff32682ac812a/diff:/var/lib/docker/overlay2/7a9e66e34776151e4dedb5d2a71737e8cdb5d069fa2a0cd3cb9651c5a9cc18e0/diff:/var/lib/docker/overlay2/b9a6252e86da3b3a15132a20e1815c39447fd89fb6403d0b749b8d1ef501c376/diff', 'MergedDir': '/var/lib/docker/overlay2/0e1eb24cab725772d3ff50453236ecc58009bac69a07c6e05f90c64ce5ac39ad/merged', 'UpperDir': '/var/lib/docker/overlay2/0e1eb24cab725772d3ff50453236ecc58009bac69a07c6e05f90c64ce5ac39ad/diff', 'WorkDir': '/var/lib/docker/overlay2/0e1eb24cab725772d3ff50453236ecc58009bac69a07c6e05f90c64ce5ac39ad/work'}, 'Name': 'overlay2'}

Comment 9 Mika Westphal 2022-10-27 14:31:17 CEST

Reported again
Version: 5.0-2 errata441

Remark: Attempted to upgrade UCD Dashboard Database from 2.35.0-3 to 2.35.0-5

Error:

The container for 4.4/prometheus=2.35.0-5 could not be started!

docker logs 2665e3bfa53f3fe366a2b376c25367b753caef88bae5240ffbad68a77ee2c92f:
ts=2022-10-05T00:14:07.442Z caller=main.go:525 level=info msg="Starting Prometheus" version="(version=2.35.0, branch=HEAD, revision=6656cd29fe6ac92bab91ecec0fe162ef0f187654)"
ts=2022-10-05T00:14:07.442Z caller=main.go:530 level=info build_context="(go=go1.18.1, user=root@cf6852b14d68, date=20220421-09:53:42)"
ts=2022-10-05T00:14:07.442Z caller=main.go:531 level=info host_details="(Linux 4.19.0-20-amd64 #1 SMP Debian 4.19.235-1 (2022-03-17) x86_64 prome-38943729 (none))"
ts=2022-10-05T00:14:07.442Z caller=main.go:532 level=info fd_limits="(soft=1048576, hard=1048576)"
ts=2022-10-05T00:14:07.443Z caller=main.go:533 level=info vm_limits="(soft=unlimited, hard=unlimited)"
ts=2022-10-05T00:14:07.448Z caller=web.go:541 level=info component=web msg="Start listening for connections" address=0.0.0.0:9090
ts=2022-10-05T00:14:07.449Z caller=web.go:573 level=info component=web msg="Router prefix" prefix=/metrics-prometheus
ts=2022-10-05T00:14:07.449Z caller=main.go:957 level=info msg="Starting TSDB ..."
ts=2022-10-05T00:14:07.458Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false
ts=2022-10-05T00:14:07.462Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any"
ts=2022-10-05T00:14:07.462Z caller=head.go:536 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=6.973µs
ts=2022-10-05T00:14:07.462Z caller=head.go:542 level=info component=tsdb msg="Replaying WAL, this may take a while"
ts=2022-10-05T00:14:07.463Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3
ts=2022-10-05T00:14:07.463Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3


dockerd logs:
-- Logs begin at Tue 2022-10-04 20:59:12 AEDT, end at Wed 2022-10-05 12:10:31 AEDT. --


docker inspect:
{'Status': 'exited', 'Running': False, 'Paused': False, 'Restarting': False, 'OOMKilled': False, 'Dead': False, 'Pid': 0, 'ExitCode': 1, 'Error': '', 'StartedAt':
'2022-10-05T00:14:07.352173959Z', 'FinishedAt': '2022-10-05T00:14:07.547460466Z'}
{'Data': {'LowerDir':
'/var/lib/docker/overlay2/15ad382c920beca81e20d992212299a2b66ec41b603cee366cb5c1f46323b462-init/diff:/var/lib/docker/overlay2/9955384fe9bd19dd6b72f39f7ac11ffb74d72d2bb9afd77fd1dbe963e6f5f8ad/diff:/var/lib/docker/overlay2/81bfd25cb3010a255f0b9d0008d568824970f452021b7675fdacece43410bae7/diff:/var/lib/docker/overlay2/103c7854993e5a81f5fc681d69a8d6c95cb94f9b70a209cb7680661383dfec22/diff:/var/lib/docker/overlay2/a87fbb31769ec00d90daf477a33db29c5900856a4acb404dee8790669adfd21e/diff:/var/lib/docker/overlay2/9675aafa7c11e202f0e1a5ad6b7fb9f516574f90336f4f3809d69e6c91e9aa31/diff:/var/lib/docker/overlay2/1eb4d466b758990f16553650d15a0d6f12f446671afd3ff7b82d00fef10169e5/diff:/var/lib/docker/overlay2/afeaeffdc20248a46a067369e01e131dc10b41505e6e85d25f2b6706a2af6e8d/diff:/var/lib/docker/overlay2/a9f50b8d13e53c51d69a09a4e1f28152e2b03c932c73589753f6e8b69534536c/diff:/var/lib/docker/overlay2/c716f4c82153e8dd92f1362f0b4a9b638c38a9e0855e53664732522d37ddee77/diff:/var/lib/docker/overlay2/bd25ea565f2be0b53c35305e2d3d25a82f7aebf1febe989d5ab9a9f14fa17cb2/diff:/var/lib/docker/overlay2/c519e86ad3f2864868e001195d7d2737fed50f752d2411dca3f41440ce582847/diff:/var/lib/docker/overlay2/494fc5b97724b02806a7129713deaa38328e098153b21b98b76a9e9fb7c9f506/diff',
'MergedDir': '/var/lib/docker/overlay2/15ad382c920beca81e20d992212299a2b66ec41b603cee366cb5c1f46323b462/merged', 'UpperDir':
'/var/lib/docker/overlay2/15ad382c920beca81e20d992212299a2b66ec41b603cee366cb5c1f46323b462/diff', 'WorkDir':
'/var/lib/docker/overlay2/15ad382c920beca81e20d992212299a2b66ec41b603cee366cb5c1f46323b462/work'}, 'Name': 'overlay2'}

Role: domaincontroller_backup

Comment 10 Mika Westphal 2022-11-11 11:04:22 CET

Reported again: 2022092621000239
Version: 5.0-2 errata431

Remark: Nach Aktualisierung UCS und Aktualisierung Nextcloud Hub, das ohne Probleme durchlief.

Error:

The container for 4.4/prometheus=2.35.0-5 could not be started!

docker logs 729757ef17e65e5936794f811be0bed4eafc2c286491ea78e8708a88e7aea026:
ts=2022-09-26T08:38:03.647Z caller=main.go:525 level=info msg="Starting Prometheus" version="(version=2.35.0, branch=HEAD, revision=6656cd29fe6ac92bab91ecec0fe162ef0f187654)"
ts=2022-09-26T08:38:03.647Z caller=main.go:530 level=info build_context="(go=go1.18.1, user=root@cf6852b14d68, date=20220421-09:53:42)"
ts=2022-09-26T08:38:03.647Z caller=main.go:531 level=info host_details="(Linux 4.19.0-20-amd64 #1 SMP Debian 4.19.235-1 (2022-03-17) x86_64 prome-84887635 (none))"
ts=2022-09-26T08:38:03.647Z caller=main.go:532 level=info fd_limits="(soft=1048576, hard=1048576)"
ts=2022-09-26T08:38:03.647Z caller=main.go:533 level=info vm_limits="(soft=unlimited, hard=unlimited)"
ts=2022-09-26T08:38:03.649Z caller=web.go:541 level=info component=web msg="Start listening for connections" address=0.0.0.0:9090
ts=2022-09-26T08:38:03.649Z caller=web.go:573 level=info component=web msg="Router prefix" prefix=/metrics-prometheus
ts=2022-09-26T08:38:03.649Z caller=main.go:957 level=info msg="Starting TSDB ..."
ts=2022-09-26T08:38:03.651Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false
ts=2022-09-26T08:38:03.653Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any"
ts=2022-09-26T08:38:03.653Z caller=head.go:536 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.347µs
ts=2022-09-26T08:38:03.653Z caller=head.go:542 level=info component=tsdb msg="Replaying WAL, this may take a while"
ts=2022-09-26T08:38:03.654Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=20
ts=2022-09-26T08:38:03.654Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=20
ts=2022-09-26T08:38:03.654Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=20
ts=2022-09-26T08:38:03.654Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=20
ts=2022-09-26T08:38:03.655Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=20
ts=2022-09-26T08:38:03.655Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=20
ts=2022-09-26T08:38:03.655Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=20
ts=2022-09-26T08:38:03.655Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=7 maxSegment=20
ts=2022-09-26T08:38:03.655Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=8 maxSegment=20
ts=2022-09-26T08:38:03.655Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=9 maxSegment=20
ts=2022-09-26T08:38:03.655Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=10 maxSegment=20
ts=2022-09-26T08:38:03.655Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=11 maxSegment=20
ts=2022-09-26T08:38:03.655Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=12 maxSegment=20
ts=2022-09-26T08:38:03.655Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=13 maxSegment=20
ts=2022-09-26T08:38:03.655Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=14 maxSegment=20
ts=2022-09-26T08:38:03.655Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=15 maxSegment=20
ts=2022-09-26T08:38:03.656Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=16 maxSegment=20
ts=2022-09-26T08:38:03.656Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=17 maxSegment=20
ts=2022-09-26T08:38:03.657Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=18 maxSegment=20


dockerd logs:
-- Logs begin at Fri 2022-07-15 09:09:06 CEST, end at Mon 2022-09-26 10:38:06 CEST. --
Sep 26 10:03:57 ucs-4965 dockerd[515]: time="2022-09-26T10:03:57.425360310+02:00" level=info msg="ccResolverWrapper: sending update to cc: {[{unix:///run/containerd/containerd.sock
0  <nil>}] <nil>}" module=grpc
Sep 26 10:03:57 ucs-4965 dockerd[515]: time="2022-09-26T10:03:57.425367053+02:00" level=info msg="ClientConn switching balancer to \"pick_first\"" module=grpc
Sep 26 10:03:57 ucs-4965 dockerd[515]: time="2022-09-26T10:03:57.427071431+02:00" level=info msg="parsed scheme: \"unix\"" module=grpc
Sep 26 10:03:57 ucs-4965 dockerd[515]: time="2022-09-26T10:03:57.427084604+02:00" level=info msg="scheme \"unix\" not registered, fallback to default scheme" module=grpc
Sep 26 10:03:57 ucs-4965 dockerd[515]: time="2022-09-26T10:03:57.427094476+02:00" level=info msg="ccResolverWrapper: sending update to cc: {[{unix:///run/containerd/containerd.sock
0  <nil>}] <nil>}" module=grpc
Sep 26 10:03:57 ucs-4965 dockerd[515]: time="2022-09-26T10:03:57.427103837+02:00" level=info msg="ClientConn switching balancer to \"pick_first\"" module=grpc
Sep 26 10:03:57 ucs-4965 dockerd[515]: time="2022-09-26T10:03:57.528691660+02:00" level=info msg="[graphdriver] using prior storage driver: overlay2"
Sep 26 10:03:57 ucs-4965 dockerd[515]: time="2022-09-26T10:03:57.714961189+02:00" level=warning msg="Your kernel does not support swap memory limit"
Sep 26 10:03:57 ucs-4965 dockerd[515]: time="2022-09-26T10:03:57.714983162+02:00" level=warning msg="Your kernel does not support cgroup rt period"


docker inspect:
{'Status': 'exited', 'Running': False, 'Paused': False, 'Restarting': False, 'OOMKilled': False, 'Dead': False, 'Pid': 0, 'ExitCode': 1, 'Error': '', 'StartedAt':
'2022-09-26T08:38:03.459180584Z', 'FinishedAt': '2022-09-26T08:38:03.733220041Z'}
{'Data': {'LowerDir':
'/var/lib/docker/overlay2/837a44865f10a2a1397d56b596ffa5357f421b726378bf43222cf6d66390c073-init/diff:/var/lib/docker/overlay2/7ff942e5289d91435d733989c275dfaacf5176d6d1a689ba033de3513bd113e9/diff:/var/lib/docker/overlay2/7c1b780b5572a53a0e0261bbc095b95daf793bc8844a8eb9cdecc73b919146ef/diff:/var/lib/docker/overlay2/bc5c4ef560298bb3bac6227f549b4677d4c9f914210c4a76a82b9117880d763a/diff:/var/lib/docker/overlay2/6a4e980e23d51fa800e3d5002f662348c320f2590a3f0b15049a78fa0904960d/diff:/var/lib/docker/overlay2/5eb67100b59b39d61504462010005b21057e127ddd45d25efc28c44d99eab9c6/diff:/var/lib/docker/overlay2/5da83718ff5b58c2bba6267cef3b299f302580ba32d2ea92bf798aaaef5ab1c0/diff:/var/lib/docker/overlay2/2110d182a7fd8556db0f6d1659bfc82724c2a1e39a89cba3796f619644bca139/diff:/var/lib/docker/overlay2/16de1c07bb3b13bb1199459e4a68c022caef316d3bae17fd202270dca9bc921f/diff:/var/lib/docker/overlay2/1fba2ec45353023c2709855e37eb6b5fb4e2cd3a6c5692153ba34fa2a706f506/diff:/var/lib/docker/overlay2/e9cf6967c16a1097019919311278b129454f6e17f84088e5edc887a002172bcf/diff:/var/lib/docker/overlay2/1ccbbd881753af1e1d0de40bc415ed69a4c4439eecf9a92ead5b2ce4bfd084fd/diff:/var/lib/docker/overlay2/20aec94a300a6baee05bb1354633d0e250b34da349087099aa394dbf6af1d960/diff',
'MergedDir': '/var/lib/docker/overlay2/837a44865f10a2a1397d56b596ffa5357f421b726378bf43222cf6d66390c073/merged', 'UpperDir':
'/var/lib/docker/overlay2/837a44865f10a2a1397d56b596ffa5357f421b726378bf43222cf6d66390c073/diff', 'WorkDir':
'/var/lib/docker/overlay2/837a44865f10a2a1397d56b596ffa5357f421b726378bf43222cf6d66390c073/work'}, 'Name': 'overlay2'}

Role: domaincontroller_master

Comment 11 Mika Westphal 2023-08-07 09:57:38 CEST

5.0-2 errata425
Role: domaincontroller_master

The container for 4.4/prometheus=2.35.0-5 could not be started!

docker logs 755b8c8e323a2f13e3d88481d7d2251a19a1a53a0ad84ba1faa6bf250718d068:
ts=2022-09-17T09:57:34.948Z caller=main.go:447 level=error msg="Error loading config (--config.file=/etc/prometheus/prometheus.yml)" file=/etc/prometheus/prometheus.yml err="open /etc/prometheus/prometheus.yml: permission denied"


dockerd logs:
-- Logs begin at Sat 2022-09-17 11:00:30 CEST, end at Sat 2022-09-17 11:57:35 CEST. --
wrz 17 11:40:33 master dockerd[1438]: time="2022-09-17T11:40:33.012717337+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
wrz 17 11:45:09 master dockerd[1438]: time="2022-09-17T11:45:09.214640704+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
wrz 17 11:54:19 master dockerd[1438]: time="2022-09-17T11:54:19.104148193+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
wrz 17 11:57:35 master dockerd[1438]: time="2022-09-17T11:57:35.157257002+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"


docker inspect:
{'Status': 'exited', 'Running': False, 'Paused': False, 'Restarting': False, 'OOMKilled': False, 'Dead': False, 'Pid': 0, 'ExitCode': 2, 'Error': '', 'StartedAt': '2022-09-17T09:57:34.688818604Z', 'FinishedAt': '2022-09-17T09:57:35.010224255Z'}
{'Data': {'LowerDir': '/var/lib/docker/overlay2/2e9e81795450c132ac209f4bfb34a46042dbd9ce0fc7ef7c3fb49165b7730b18-init/diff:/var/lib/docker/overlay2/c9d26e161833b3366f91e43804e8d82d742c6cd76857d8394b0c419152aefbcc/diff:/var/lib/docker/overlay2/e32c401164389d724ee100492dc15ad6f8a1529871641b79022c2e8a19e3ff97/diff:/var/lib/docker/overlay2/b9adfad662c3e5680a17d1b6c31dd163db7d83ea634fef8b5e63d73bdc8bc45f/diff:/var/lib/docker/overlay2/4ecf9874e5498db5bd8e705213cd7290278288d317505da39a154c02fa09f0e0/diff:/var/lib/docker/overlay2/a4d159fe6debb32f430ca7d52e852556a7a96d99d1e133a9e22522e263c7a690/diff:/var/lib/docker/overlay2/5525fda6b2f9d1b3e3d568ded5f2c55e1b7c470593d26ca58b94869d5945be08/diff:/var/lib/docker/overlay2/b479c2d251a7c2183959ebaee9767965e44c309d0363c589e518e2849e2778d1/diff:/var/lib/docker/overlay2/1d2c66194772f9961e24f7d25681367d72204ab3afa0851dd9eeb3b66c60c64f/diff:/var/lib/docker/overlay2/f47e9a944f18b0c812b20e85c548c205af58218691e030828425209cf7f904ee/diff:/var/lib/docker/overlay2/e247de74763df2a0974bbd8981ede4c0d1acf3d994a3265b396c8fd6c9e906e8/diff:/var/lib/docker/overlay2/4756e6f002f8953f09f0d73e8276dde2a2570cf87ca05b82bde88fa704cf750f/diff:/var/lib/docker/overlay2/08bfaa561cf8c8371759d1b8e3e0c33028c7e4426e4f608ff377fb9cd5ec09f7/diff', 'MergedDir': '/var/lib/docker/overlay2/2e9e81795450c132ac209f4bfb34a46042dbd9ce0fc7ef7c3fb49165b7730b18/merged', 'UpperDir': '/var/lib/docker/overlay2/2e9e81795450c132ac209f4bfb34a46042dbd9ce0fc7ef7c3fb49165b7730b18/diff', 'WorkDir': '/var/lib/docker/overlay2/2e9e81795450c132ac209f4bfb34a46042dbd9ce0fc7ef7c3fb49165b7730b18/work'}, 'Name': 'overlay2'}

Comment 12 Mika Westphal 2023-08-07 09:59:18 CEST

5.0-2 errata411
Role: domaincontroller_master

The container for 4.4/prometheus=2.35.0-5 could not be started!

docker logs 3f11277eb9fb891a7752388c67ff3526a41a1b387d27822926a15ba99a7bd5d7:


dockerd logs:
-- Logs begin at Fri 2022-06-17 19:32:36 CEST, end at Fri 2022-09-09 10:41:51 CEST. --
Sep 09 10:21:57 ucs01 dockerd[2408]: time="2022-09-09T10:21:57.413728148+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:22:58 ucs01 dockerd[2408]: time="2022-09-09T10:22:58.434738137+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:23:59 ucs01 dockerd[2408]: time="2022-09-09T10:23:59.416887533+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:25:00 ucs01 dockerd[2408]: time="2022-09-09T10:25:00.506030612+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:26:01 ucs01 dockerd[2408]: time="2022-09-09T10:26:01.543954054+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:27:02 ucs01 dockerd[2408]: time="2022-09-09T10:27:02.580014872+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:28:03 ucs01 dockerd[2408]: time="2022-09-09T10:28:03.633339487+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:29:04 ucs01 dockerd[2408]: time="2022-09-09T10:29:04.665670272+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:30:05 ucs01 dockerd[2408]: time="2022-09-09T10:30:05.774468295+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:31:06 ucs01 dockerd[2408]: time="2022-09-09T10:31:06.777964134+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:32:07 ucs01 dockerd[2408]: time="2022-09-09T10:32:07.970049295+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:33:09 ucs01 dockerd[2408]: time="2022-09-09T10:33:09.039040044+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:34:10 ucs01 dockerd[2408]: time="2022-09-09T10:34:10.035265967+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:35:11 ucs01 dockerd[2408]: time="2022-09-09T10:35:11.211717485+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:36:12 ucs01 dockerd[2408]: time="2022-09-09T10:36:12.251363785+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:37:13 ucs01 dockerd[2408]: time="2022-09-09T10:37:13.290985890+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:38:14 ucs01 dockerd[2408]: time="2022-09-09T10:38:14.399057897+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:39:15 ucs01 dockerd[2408]: time="2022-09-09T10:39:15.481540311+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:40:16 ucs01 dockerd[2408]: time="2022-09-09T10:40:16.598243288+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 09 10:41:17 ucs01 dockerd[2408]: time="2022-09-09T10:41:17.643251054+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"


docker inspect:
{'Status': 'created', 'Running': False, 'Paused': False, 'Restarting': False, 'OOMKilled': False, 'Dead': False, 'Pid': 0, 'ExitCode': 0, 'Error': '', 'StartedAt': '0001-01-01T00:00:00Z', 'FinishedAt': '0001-01-01T00:00:00Z'}
{'Data': {'LowerDir': '/var/lib/docker/overlay2/84fc1378b41bbc93172234db41f4812db03e5d7c54fd82787f934b240553b0d8-init/diff:/var/lib/docker/overlay2/7614977373f402a666456a2533f021e2ed71d75b65ed2e261595c39213715478/diff:/var/lib/docker/overlay2/1187e5d8636d4bf55bc2dfd3ff72926d1450a57f72070231798429423583564d/diff:/var/lib/docker/overlay2/cadb9b4bc8252239a0eed8e419892534c54af245485f60ce8d92adf277ef0772/diff:/var/lib/docker/overlay2/c7c65d73ea7984ff7ebfd1ba9a6d185c53e3c36dabc6624993fb87963deab4d6/diff:/var/lib/docker/overlay2/da3336bcd03de3e24d7a63c123bbcb5b06c0e0e656c46ab72fe30ac83578b267/diff:/var/lib/docker/overlay2/454fc19ca9c08f632336163dbc623c5e86672a14fc1e1b3e8dca28c2a6b91247/diff:/var/lib/docker/overlay2/aa6dd4a14730cbd292775d55737b328672bc0a0b73ea001be69173e602ec2eb0/diff:/var/lib/docker/overlay2/259b115831f72edfc19a77cf35849c8fafe9a5df4f90cee518e2e670df21fc7a/diff:/var/lib/docker/overlay2/11055ef70fd818e50dee7a9b883045421eb891f06c6efa8384fb71ebb4bfeaf8/diff:/var/lib/docker/overlay2/191316be22d9a19b93aa701291cb915c6cca96c0fee7ded94b90fc61e05cefef/diff:/var/lib/docker/overlay2/11b83a5bf75ec490055ce48c9fec38bf39f520c61d746385dace44e9f588a266/diff:/var/lib/docker/overlay2/e0c94ef642d5d2895145804834fb495768ed1db26e72f022b87f63f9805d71de/diff', 'MergedDir': '/var/lib/docker/overlay2/84fc1378b41bbc93172234db41f4812db03e5d7c54fd82787f934b240553b0d8/merged', 'UpperDir': '/var/lib/docker/overlay2/84fc1378b41bbc93172234db41f4812db03e5d7c54fd82787f934b240553b0d8/diff', 'WorkDir': '/var/lib/docker/overlay2/84fc1378b41bbc93172234db41f4812db03e5d7c54fd82787f934b240553b0d8/work'}, 'Name': 'overlay2'}

Comment 13 Mika Westphal 2023-08-07 12:48:58 CEST

5.0-2 errata411

The container for 4.4/prometheus=2.35.0-5 could not be started!

docker logs 7bc49022d3ff9d83d55d08d0b7bed35f679728a41b669d535ddb68526907923f:
ts=2022-09-08T15:51:47.414Z caller=main.go:525 level=info msg="Starting Prometheus" version="(version=2.35.0, branch=HEAD, revision=6656cd29fe6ac92bab91ecec0fe162ef0f187654)"
ts=2022-09-08T15:51:47.414Z caller=main.go:530 level=info build_context="(go=go1.18.1, user=root@cf6852b14d68, date=20220421-09:53:42)"
ts=2022-09-08T15:51:47.414Z caller=main.go:531 level=info host_details="(Linux 4.19.0-21-amd64 #1 SMP Debian 4.19.249-2 (2022-06-30) x86_64 prome-33611061 (none))"
ts=2022-09-08T15:51:47.414Z caller=main.go:532 level=info fd_limits="(soft=1048576, hard=1048576)"
ts=2022-09-08T15:51:47.414Z caller=main.go:533 level=info vm_limits="(soft=unlimited, hard=unlimited)"
ts=2022-09-08T15:51:47.417Z caller=web.go:541 level=info component=web msg="Start listening for connections" address=0.0.0.0:9090
ts=2022-09-08T15:51:47.419Z caller=main.go:957 level=info msg="Starting TSDB ..."
ts=2022-09-08T15:51:47.419Z caller=web.go:573 level=info component=web msg="Router prefix" prefix=/metrics-prometheus
ts=2022-09-08T15:51:47.419Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1660301841552 maxt=1660305600000 ulid=01GA95H0WMK8391CQ4V6C2YTJ9
ts=2022-09-08T15:51:47.420Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1660046428620 maxt=1660082400000 ulid=01GA993TAMAPHEH1FHD2W7FTRM
ts=2022-09-08T15:51:47.420Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1660305606768 maxt=1660370400000 ulid=01GABDS385A11XCGFBKJYM8CTD
ts=2022-09-08T15:51:47.420Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1660370406768 maxt=1660435200000 ulid=01GADBJM37VGQWX4Y38S77ZXXN
ts=2022-09-08T15:51:47.420Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1660435206768 maxt=1660500000000 ulid=01GAF9C5TF7HZN7VQACVN3J4SA
ts=2022-09-08T15:51:47.420Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false
ts=2022-09-08T15:51:47.420Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1660500006769 maxt=1660564800000 ulid=01GAH75QQ2ZSJF031G8QE55W9X
ts=2022-09-08T15:51:47.420Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1660564806767 maxt=1660629600000 ulid=01GAK4Z7PZRHKS7GTJYCC2NNE2
ts=2022-09-08T15:51:47.421Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1660629606768 maxt=1660694400000 ulid=01GAN2RS1XNKWRV692XY1FBHFN
ts=2022-09-08T15:51:47.421Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1660694406768 maxt=1660759200000 ulid=01GAQ0JAFDTD0SZTYDT052R6VR
ts=2022-09-08T15:51:47.421Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1660759206768 maxt=1660824000000 ulid=01GARYBV5BCAK31GC75CQWYYTF
ts=2022-09-08T15:51:47.421Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1660824006768 maxt=1660888800000 ulid=01GATW59G36SR1CQF122N25F23
ts=2022-09-08T15:51:47.421Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1660888806768 maxt=1660953600000 ulid=01GAWSYXWJ3SSX5J4VJB0EHXBQ
ts=2022-09-08T15:51:47.421Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1660953606768 maxt=1661018400000 ulid=01GAYQRC6GPH93Z84M1B5J7ZVE
ts=2022-09-08T15:51:47.421Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1661018406768 maxt=1661083200000 ulid=01GB0NJ03QV9VZYA6BJRZN9NP8
ts=2022-09-08T15:51:47.421Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1661083206769 maxt=1661148000000 ulid=01GB2KBJV00SYP7G7X3C0HJY4K
ts=2022-09-08T15:51:47.421Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1661148006768 maxt=1661212800000 ulid=01GB4H53K8RNGN15G846VRY0WF
ts=2022-09-08T15:51:47.422Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1661212806768 maxt=1661277600000 ulid=01GB6EYNEWAA2JFVE86CCXPY85
ts=2022-09-08T15:51:47.422Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1661277606768 maxt=1661299200000 ulid=01GB73HNVB435EEKVX8BY58PVT
ts=2022-09-08T15:51:47.422Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1661320806769 maxt=1661328000000 ulid=01GB7H971VYQ31YJ5A0E8DG68K
ts=2022-09-08T15:51:47.422Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1661328006768 maxt=1661335200000 ulid=01GB7R4Y9S473F9TKQCA224MH7
ts=2022-09-08T15:51:47.422Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1661299206768 maxt=1661320800000 ulid=01GB7R4Z87HCBRC55HM8XSG9R5
ts=2022-09-08T15:51:47.422Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1661335206768 maxt=1661342400000 ulid=01GB7Z0NJJADZD4MVECRWZHFG4
ts=2022-09-08T15:51:47.422Z caller=repair.go:57 level=info component=tsdb msg="Found healthy block" mint=1661342406768 maxt=1661349600000 ulid=01GB85WCTH4VJSP607H2N70ZW7
ts=2022-09-08T15:51:47.454Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any"
ts=2022-09-08T15:51:47.455Z caller=head.go:536 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=162.653µs
ts=2022-09-08T15:51:47.455Z caller=head.go:542 level=info component=tsdb msg="Replaying WAL, this may take a while"
ts=2022-09-08T15:51:48.818Z caller=head.go:578 level=info component=tsdb msg="WAL checkpoint loaded"
ts=2022-09-08T15:51:48.860Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=1065 maxSegment=1071
ts=2022-09-08T15:51:48.905Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=1066 maxSegment=1071
ts=2022-09-08T15:51:48.931Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=1067 maxSegment=1071
ts=2022-09-08T15:51:48.932Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=1068 maxSegment=1071
ts=2022-09-08T15:51:48.932Z caller=head.go:613 level=info component=tsdb msg="WAL segment loaded" segment=1069 maxSegment=1071


dockerd logs:
-- Logs begin at Fri 2022-06-17 07:11:32 CEST, end at Thu 2022-09-08 17:51:49 CEST. --
Aug 24 17:57:49 UCS02 dockerd[1053]: time="2022-08-24T17:57:49.828121808+02:00" level=info msg="scheme \"unix\" not registered, fallback to default scheme" module=grpc
Aug 24 17:57:49 UCS02 dockerd[1053]: time="2022-08-24T17:57:49.828158958+02:00" level=info msg="ccResolverWrapper: sending update to cc: {[{unix:///run/containerd/containerd.sock 0  <nil>}] <nil>}" module=grpc
Aug 24 17:57:49 UCS02 dockerd[1053]: time="2022-08-24T17:57:49.828179841+02:00" level=info msg="ClientConn switching balancer to \"pick_first\"" module=grpc
Aug 24 17:57:51 UCS02 dockerd[1053]: time="2022-08-24T17:57:51.069729176+02:00" level=info msg="[graphdriver] using prior storage driver: overlay2"
Aug 24 17:57:53 UCS02 dockerd[1053]: time="2022-08-24T17:57:53.099717130+02:00" level=warning msg="Your kernel does not support swap memory limit"
Aug 24 17:57:53 UCS02 dockerd[1053]: time="2022-08-24T17:57:53.100756559+02:00" level=warning msg="Your kernel does not support cgroup rt period"
Aug 24 17:57:53 UCS02 dockerd[1053]: time="2022-08-24T17:57:53.101003573+02:00" level=warning msg="Your kernel does not support cgroup rt runtime"
Aug 24 17:57:53 UCS02 dockerd[1053]: time="2022-08-24T17:57:53.101858714+02:00" level=info msg="Loading containers: start."
Aug 24 17:57:58 UCS02 dockerd[1053]: time="2022-08-24T17:57:58.157378417+02:00" level=info msg="Removing stale sandbox 7ad2afa07fed72b7b17f5b2d37a37246f4edaba00c0005fe5ee66241aeeb1c0a (cb3238bb322b0c540aab67bce13aa927f32a1c9b6bd5fc4144dfead233d39fce)"
Aug 24 17:57:58 UCS02 dockerd[1053]: time="2022-08-24T17:57:58.639491935+02:00" level=warning msg="Error (Unable to complete atomic operation, key modified) deleting object [endpoint eb7ef100eff3a4e348be85c5486ef27656e94bc41d6bd925c4a9c53cc83b2ff7 73ea7ba874ea973fdaaaf47141d8f7c343d2b9979c00b04122339bc71b0e2348], retrying...."
Aug 24 17:57:59 UCS02 dockerd[1053]: time="2022-08-24T17:57:59.460749088+02:00" level=info msg="Removing stale sandbox bfcd9ce3454ff86b835659f4258807e9a6be278de3f8eab4e5ff7f4c5512a813 (d23394d5afa0830841f713a0e1f87f7936157f9a80b3ad295d86e7557f09f314)"
Aug 24 17:58:00 UCS02 dockerd[1053]: time="2022-08-24T17:58:00.642062642+02:00" level=warning msg="Error (Unable to complete atomic operation, key modified) deleting object [endpoint eb7ef100eff3a4e348be85c5486ef27656e94bc41d6bd925c4a9c53cc83b2ff7 c7fd5a5839aca894a4f4d16fb10195520596f269b4e00f54aab8989f75e51f32], retrying...."
Aug 24 17:58:04 UCS02 dockerd[1053]: time="2022-08-24T17:58:04.146263347+02:00" level=info msg="Loading containers: done."
Aug 24 17:58:05 UCS02 dockerd[1053]: time="2022-08-24T17:58:05.513862135+02:00" level=info msg="Docker daemon" commit=afacb8b7f0 graphdriver(s)=overlay2 version=19.03.8
Aug 24 17:58:05 UCS02 dockerd[1053]: time="2022-08-24T17:58:05.568552691+02:00" level=info msg="Daemon has completed initialization"
Aug 24 17:58:06 UCS02 dockerd[1053]: time="2022-08-24T17:58:06.113318391+02:00" level=info msg="API listen on /var/run/docker.sock"
Aug 24 17:58:26 UCS02 dockerd[1053]: time="2022-08-24T17:58:26.650247392+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 24 21:20:51 UCS02 dockerd[1053]: time="2022-08-24T21:20:51.906891219+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 24 21:21:03 UCS02 dockerd[1053]: time="2022-08-24T21:21:03.410964798+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 08 17:51:49 UCS02 dockerd[1053]: time="2022-09-08T17:51:49.207956213+02:00" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"


docker inspect:
{'Status': 'exited', 'Running': False, 'Paused': False, 'Restarting': False, 'OOMKilled': False, 'Dead': False, 'Pid': 0, 'ExitCode': 1, 'Error': '', 'StartedAt': '2022-09-08T15:51:47.312907686Z', 'FinishedAt': '2022-09-08T15:51:49.131801731Z'}
{'Data': {'LowerDir': '/var/lib/docker/overlay2/cc05a52221b6dfbe541bdab164d915b1fa6fc806dfe3dfe07f9382d402ced9ef-init/diff:/var/lib/docker/overlay2/8f7a5b67a52d289d49593b219078db9faf1b635ba996a56acadb3d17861f9426/diff:/var/lib/docker/overlay2/f7229700c0b2a61b922d38ab5e2973c0511190ab9273ee9a49a48ca5df67f319/diff:/var/lib/docker/overlay2/6e2fe2d9efec2ef59779217cdd040d982c335f4b1b56ecf46bade55e5685943b/diff:/var/lib/docker/overlay2/d71c3fe1d014f9eaa8bd972fe840648f66d039c3cf9e091cf6732839dc20f76b/diff:/var/lib/docker/overlay2/ac60f3c2933e491084da864f3b831c1fc56e74dd081d5686e3aa08256ec11ed0/diff:/var/lib/docker/overlay2/fafd39f847c0db5d836b4fc492e9d7823c79e56903c09bc785d5cd30cbb0484a/diff:/var/lib/docker/overlay2/aac5a8664796706d82ca0ceee0d799afbe278ab04582c2de031d425c8f384788/diff:/var/lib/docker/overlay2/b7dfedd1d7ef1e7fd59bbd1dfbd9c0ae4c5f9736748a86a5725aa4ece388c20e/diff:/var/lib/docker/overlay2/2b5be5cd8da13f9b543e6428f5caf1e42ee79961a84681e974a018158e9ac62c/diff:/var/lib/docker/overlay2/36d77c4e2dd998bf6bf9ad1dd909442309a3d74af7ceb87b73dff32682ac812a/diff:/var/lib/docker/overlay2/7a9e66e34776151e4dedb5d2a71737e8cdb5d069fa2a0cd3cb9651c5a9cc18e0/diff:/var/lib/docker/overlay2/b9a6252e86da3b3a15132a20e1815c39447fd89fb6403d0b749b8d1ef501c376/diff', 'MergedDir': '/var/lib/docker/overlay2/cc05a52221b6dfbe541bdab164d915b1fa6fc806dfe3dfe07f9382d402ced9ef/merged', 'UpperDir': '/var/lib/docker/overlay2/cc05a52221b6dfbe541bdab164d915b1fa6fc806dfe3dfe07f9382d402ced9ef/diff', 'WorkDir': '/var/lib/docker/overlay2/cc05a52221b6dfbe541bdab164d915b1fa6fc806dfe3dfe07f9382d402ced9ef/work'}, 'Name': 'overlay2'}


Role: domaincontroller_backup