level=info ts=2026-04-16T16:42:49.624231829Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-16T16:42:49.624261224Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-04-16T16:42:49.624560779Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-04-16T16:42:49.625210806Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-16T16:42:49.625611925Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-16T16:42:49.625680877Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.74 inf=eth0
level=info ts=2026-04-16T16:42:49.626128632Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-757f957d
ts=2026-04-16T16:42:49.626219026Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-16T16:42:49.626239212Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.74 advertisePort=7946
level=debug ts=2026-04-16T16:42:49.626259383Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.74 advertisePort=7946
level=info ts=2026-04-16T16:42:49.626371562Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=debug ts=2026-04-16T16:42:49.629096601Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.68:11211 count=1
level=info ts=2026-04-16T16:42:49.630060974Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-04-16T16:42:49.630126998Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
ts=2026-04-16T16:42:49.634322477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.67:7946"
level=info ts=2026-04-16T16:42:49.63523326Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-16T16:42:49.63524754Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-16T16:42:49.635246498Z
level=debug ts=2026-04-16T16:42:49.635262469Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-04-16T16:42:49.635274603Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-04-16T16:42:49.635287439Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=1
level=debug ts=2026-04-16T16:42:49.635291796Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=1 ts=2026-04-16T16:42:49.635291271Z
level=debug ts=2026-04-16T16:42:49.635295793Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-04-16T16:42:49.636134866Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-16T16:42:49.636222659Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.130.0.74 inf=eth0
ts=2026-04-16T16:42:49.636836051Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.66:7946: dial tcp 10.129.0.66:7946: connect: connection refused"
ts=2026-04-16T16:42:49.637747825Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.73:7946: dial tcp 10.130.0.73:7946: connect: connection refused"
ts=2026-04-16T16:42:49.639907021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.103:7946"
ts=2026-04-16T16:42:49.642694369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.104:7946"
ts=2026-04-16T16:42:49.645671891Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.101:7946: dial tcp 10.128.0.101:7946: connect: connection refused"
level=info ts=2026-04-16T16:42:49.645680654Z caller=memberlist_client.go:648 msg="memberlist fast-join finished" joined_nodes=3 elapsed_time=19.312058ms
level=info ts=2026-04-16T16:42:49.645695835Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-16T16:42:49.648743781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.67:7946"
ts=2026-04-16T16:42:49.64995716Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.66:7946: dial tcp 10.129.0.66:7946: connect: connection refused"
ts=2026-04-16T16:42:49.650082886Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.73:7946: dial tcp 10.130.0.73:7946: connect: connection refused"
level=info ts=2026-04-16T16:42:49.650129046Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-16T16:42:49.650177309Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-04-16T16:42:49.650196656Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-04-16T16:42:49.6502057Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-04-16T16:42:49.650214129Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-04-16T16:42:49.650342928Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-16T16:42:49.650300444Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-04-16T16:42:49.650522024Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-16T16:42:49.650554659Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-04-16T16:42:49.650564682Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-04-16T16:42:49.650719937Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-04-16T16:42:49.650728104Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-04-16T16:42:49.650726766Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2026-04-16T16:42:49.650734175Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-04-16T16:42:49.650735104Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=info ts=2026-04-16T16:42:49.650738778Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-04-16T16:42:49.650742964Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2026-04-16T16:42:49.650761787Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-04-16T16:42:49.65082842Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-04-16T16:42:49.650832715Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-04-16T16:42:49.650836188Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-16T16:42:49.650853642Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-04-16T16:42:49.650857068Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=info ts=2026-04-16T16:42:49.65086639Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-04-16T16:42:49.650879704Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
ts=2026-04-16T16:42:49.651009615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.103:7946"
ts=2026-04-16T16:42:49.653326471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.104:7946"
ts=2026-04-16T16:42:49.655457788Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.101:7946: dial tcp 10.128.0.101:7946: connect: connection refused"
level=info ts=2026-04-16T16:42:49.655469473Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=9.769283ms
level=info ts=2026-04-16T16:42:50.14459165Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=493.711672ms errors=false
level=info ts=2026-04-16T16:42:50.144625089Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
ts=2026-04-16T16:42:50.398667029Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-1233755e' from=10.129.0.67:7946"
level=info ts=2026-04-16T16:42:51.856867817Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=2.205987787s errors=false
level=info ts=2026-04-16T16:42:51.856911577Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-04-16T16:42:51.85694777Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=2.206068757s
level=info ts=2026-04-16T16:42:51.856966686Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-04-16T16:42:51.85701108Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-16T16:42:51.857006078Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=20.73200705s
level=info ts=2026-04-16T16:42:51.857054495Z caller=lifecycler.go:740 component=ingester msg="existing instance found in ring" state=ACTIVE tokens=128 ring=ingester readOnly=false readOnlyStateUpdate=0001-01-01T00:00:00Z
level=debug ts=2026-04-16T16:42:51.857085629Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-04-16T16:42:51.857115715Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-04-16T16:42:51.857139445Z caller=loki.go:599 msg="Loki started" startup_time=2.28807851s
ts=2026-04-16T16:42:52.399445519Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-1233755e' from=10.130.0.74:7946"
ts=2026-04-16T16:42:52.401098509Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-1233755e' from=10.128.0.103:7946"
ts=2026-04-16T16:42:52.401133152Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-1233755e' from=10.128.0.104:7946"
ts=2026-04-16T16:42:54.860332378Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-1233755e' from=10.128.0.104:7946"
level=warn ts=2026-04-16T16:42:56.857424853Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:42:56.860929108Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-1233755e' from=10.130.0.74:7946"
ts=2026-04-16T16:42:56.860959337Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-1233755e' from=10.128.0.103:7946"
ts=2026-04-16T16:42:56.861791412Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-1233755e' from=10.129.0.67:7946"
level=warn ts=2026-04-16T16:43:01.858135473Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:43:04.590949989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57798"
level=warn ts=2026-04-16T16:43:06.8580105Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:43:06.977719703Z caller=logging.go:137 msg="GET /ready (503) 126.558µs"
level=warn ts=2026-04-16T16:43:11.85817765Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-16T16:43:16.857218861Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:43:16.976550988Z caller=logging.go:137 msg="GET /ready (503) 99.658µs"
level=info ts=2026-04-16T16:43:21.857582738Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-16T16:43:21.857629186Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-04-16T16:43:21.857641071Z caller=recalculate_owned_streams.go:82 msg="fixed limit has been updated" tenant=kubearchive old=2147483647 new=10000
level=warn ts=2026-04-16T16:43:21.857650032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:43:21.857665714Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-16T16:43:26.857480295Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:43:28.821721922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.104:42910"
level=warn ts=2026-04-16T16:43:31.857367393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-16T16:43:36.858133247Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:43:38.911947732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="5.6 MB" avg_uncomp="5.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-16T16:43:38.911949892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="706 kB" avg_comp="706 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=debug ts=2026-04-16T16:43:38.911965033Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-16T16:43:38.911970933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.3 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-04-16T16:43:38.912019681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="431 kB" avg_comp="431 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-16T16:43:38.917621679Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d970aa8b6%3A19d970b420e%3Ad0782992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-16T16:43:38.917635903Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19d970aa2b3%3A19d971d81bf%3A78557ef7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-16T16:43:38.917644463Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19d970b421d%3A19d971abebf%3A588715ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-04-16T16:43:38.917656243Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d970aae0a%3A19d97137d21%3A1f674789?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-16T16:43:39.213100359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.0 MB" avg_comp="1.0 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-16T16:43:39.218212101Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19d970b420d%3A19d971d8699%3A6d08d35d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-16T16:43:39.51228569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-16T16:43:39.516418831Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d970aa448%3A19d9712c9d0%3A84cbcae8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-16T16:43:39.621591458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=warn ts=2026-04-16T16:43:41.857517921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-16T16:43:46.857566786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:43:47.618429076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-16T16:43:49.630163377Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.68:11211 count=1
level=info ts=2026-04-16T16:43:49.631177408Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-16T16:43:49.975447735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="5.6 MB" avg_uncomp="5.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-16T16:43:49.982048946Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19d970aa2b3%3A19d971d81bf%3A78557ef7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-16T16:43:51.857231661Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-16T16:43:51.857276621Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:43:51.857288925Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-16T16:43:51.857327762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:43:52.592037714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-16T16:43:52.598162349Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d970aa448%3A19d9712c9d0%3A84cbcae8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-16T16:43:54.364706819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="431 kB" avg_comp="431 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-16T16:43:54.368135381Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d970aae0a%3A19d97137d21%3A1f674789?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-16T16:43:56.549260745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.3 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-16T16:43:56.555492406Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d970aa8b6%3A19d970b420e%3Ad0782992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-16T16:43:56.857967862Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:43:57.885747988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="707 kB" avg_comp="707 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-04-16T16:43:57.890856349Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19d970b421d%3A19d971abebf%3A588715ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-04-16T16:43:58.206906628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.0 MB" avg_comp="1.0 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-16T16:43:58.211378471Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19d970b420d%3A19d971d8699%3A6d08d35d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-16T16:44:01.857743051Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-16T16:44:06.857568457Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:44:07.741558458Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T16:44:09.624779109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=warn ts=2026-04-16T16:44:11.857281677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:44:14.451568196Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-16T16:44:14.45165392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2a5ccb133cfae346 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-xejw\"}"
level=error ts=2026-04-16T16:44:14.458249526Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2a5ccb133cfae346/19d9712996b%3A19d972d6476%3Ab3c66bbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-xejw\"}"
level=info ts=2026-04-16T16:44:16.768787467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-16T16:44:16.773493952Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d970aa448%3A19d9712c9d0%3A84cbcae8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-16T16:44:16.857887207Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:44:19.452167568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="431 kB" avg_comp="431 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-16T16:44:19.455529008Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d970aae0a%3A19d97137d21%3A1f674789?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-16T16:44:21.8575652Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-16T16:44:21.857606689Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:44:21.857618188Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-16T16:44:21.857612097Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-16T16:44:26.85797979Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:44:27.588582865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="707 kB" avg_comp="707 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-04-16T16:44:27.592987582Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19d970b421d%3A19d971abebf%3A588715ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-04-16T16:44:28.286508343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2a5ccb133cfae346 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-xejw\"}"
level=error ts=2026-04-16T16:44:28.294311101Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2a5ccb133cfae346/19d9712996b%3A19d972d6476%3Ab3c66bbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-xejw\"}"
level=info ts=2026-04-16T16:44:28.850199084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.3 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-16T16:44:28.858701802Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d970aa8b6%3A19d970b420e%3Ad0782992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-16T16:44:29.608380707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="5.6 MB" avg_uncomp="5.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-16T16:44:29.612456004Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19d970aa2b3%3A19d971d81bf%3A78557ef7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-16T16:44:31.17747389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.0 MB" avg_comp="1.0 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-16T16:44:31.182505357Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19d970b420d%3A19d971d8699%3A6d08d35d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-16T16:44:31.857660019Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:44:33.793115637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-16T16:44:36.857446803Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:44:38.618899437Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-16T16:44:39.627781597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=warn ts=2026-04-16T16:44:41.857964857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-16T16:44:46.85803392Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:44:49.630235812Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-16T16:44:49.630577451Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.68:11211 count=1
level=info ts=2026-04-16T16:44:51.857541027Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-16T16:44:51.857574415Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:44:51.857582882Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-16T16:44:51.857644818Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-16T16:44:56.857977739Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:44:57.148286374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:39670"
ts=2026-04-16T16:44:58.830251482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.104:37578"
level=debug ts=2026-04-16T16:44:59.434869523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-16T16:45:01.83989706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2a5ccb133cfae346 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-xejw\"}"
level=error ts=2026-04-16T16:45:01.848798357Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2a5ccb133cfae346/19d9712996b%3A19d972d6476%3Ab3c66bbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-xejw\"}"
level=warn ts=2026-04-16T16:45:01.857859915Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:45:03.117046651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="431 kB" avg_comp="431 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-16T16:45:03.120381656Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d970aae0a%3A19d97137d21%3A1f674789?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-16T16:45:06.858027329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:45:09.630240968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=info ts=2026-04-16T16:45:10.334745155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.3 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-16T16:45:10.342102596Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d970aa8b6%3A19d970b420e%3Ad0782992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-16T16:45:10.597760655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="5.6 MB" avg_uncomp="5.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-16T16:45:10.601542614Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19d970aa2b3%3A19d971d81bf%3A78557ef7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-16T16:45:11.038693125Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-16T16:45:11.858179842Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:45:12.204355303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.0 MB" avg_comp="1.0 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-16T16:45:12.20885948Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19d970b420d%3A19d971d8699%3A6d08d35d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-16T16:45:13.967674528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-16T16:45:13.972085178Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d970aa448%3A19d9712c9d0%3A84cbcae8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-16T16:45:14.138700547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="707 kB" avg_comp="707 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-04-16T16:45:14.142299691Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19d970b421d%3A19d971abebf%3A588715ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-04-16T16:45:16.857529743Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:45:21.857222425Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-16T16:45:21.857250239Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:45:21.857259343Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-16T16:45:21.857293662Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:45:25.180541504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-16T16:45:25.182471359Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-16T16:45:26.857715325Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:45:27.150705044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:46628"
level=warn ts=2026-04-16T16:45:31.858019778Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-16T16:45:36.857577961Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:45:39.632419891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a04a5887 10.128.0.104:7946"
level=debug ts=2026-04-16T16:45:40.917918415Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-16T16:45:41.857902216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:45:44.919334689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2a5ccb133cfae346 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-xejw\"}"
level=error ts=2026-04-16T16:45:44.930709207Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2a5ccb133cfae346/19d9712996b%3A19d972d6476%3Ab3c66bbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-xejw\"}"
level=info ts=2026-04-16T16:45:45.384068714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="431 kB" avg_comp="431 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-16T16:45:45.388403057Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d970aae0a%3A19d97137d21%3A1f674789?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-16T16:45:46.857922203Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:45:49.630238461Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-16T16:45:49.631051926Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.68:11211 count=1
level=debug ts=2026-04-16T16:45:49.636391312Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-16T16:45:49.636438255Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1973730 n=1
level=debug ts=2026-04-16T16:45:49.636478811Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1973730 n=1
level=info ts=2026-04-16T16:45:51.857690348Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-16T16:45:51.857708569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:45:51.85772954Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:45:51.857742065Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-16T16:45:52.863481369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.0 MB" avg_comp="1.0 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-16T16:45:52.867586822Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19d970b420d%3A19d971d8699%3A6d08d35d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-16T16:45:55.025199607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-16T16:45:55.029922838Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d970aa448%3A19d9712c9d0%3A84cbcae8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-16T16:45:56.858077572Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:45:57.153512643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:60108"
level=info ts=2026-04-16T16:45:57.790203421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="707 kB" avg_comp="707 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-04-16T16:45:57.794190109Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19d970b421d%3A19d971abebf%3A588715ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-04-16T16:46:01.857967232Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:46:03.01467309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="5.6 MB" avg_uncomp="5.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-16T16:46:03.018755424Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19d970aa2b3%3A19d971d81bf%3A78557ef7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-16T16:46:05.694504809Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-16T16:46:05.95520103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.3 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-16T16:46:05.961375063Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d970aa8b6%3A19d970b420e%3Ad0782992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-16T16:46:06.8576689Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:46:07.626841508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T16:46:09.635132103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=warn ts=2026-04-16T16:46:11.857375975Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-16T16:46:16.857508416Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:46:21.857326411Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-16T16:46:21.857346335Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:46:21.857364665Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:46:21.857374431Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-16T16:46:26.857698833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:46:28.837527916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.104:53038"
level=warn ts=2026-04-16T16:46:31.857523801Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:46:33.668832971Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-16T16:46:36.693372789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="431 kB" avg_comp="431 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-16T16:46:36.698071708Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d970aae0a%3A19d97137d21%3A1f674789?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-16T16:46:36.857687474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:46:39.637705586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=warn ts=2026-04-16T16:46:41.857799256Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:46:43.877596166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2a5ccb133cfae346 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-xejw\"}"
level=error ts=2026-04-16T16:46:43.884630587Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2a5ccb133cfae346/19d9712996b%3A19d972d6476%3Ab3c66bbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-xejw\"}"
level=info ts=2026-04-16T16:46:45.197021857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="707 kB" avg_comp="707 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-04-16T16:46:45.201338946Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19d970b421d%3A19d971abebf%3A588715ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-04-16T16:46:46.688742671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.3 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-16T16:46:46.695237652Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d970aa8b6%3A19d970b420e%3Ad0782992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-16T16:46:46.857753606Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:46:49.367807503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="4.0 MB" avg_comp="2.0 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-16T16:46:49.372711594Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19d970b420d%3A19d971d8699%3A6d08d35d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-16T16:46:49.630313323Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-16T16:46:49.63057509Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.68:11211 count=1
level=info ts=2026-04-16T16:46:51.857890302Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-16T16:46:51.857926784Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-16T16:46:51.857914226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:46:51.857935678Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-16T16:46:51.874074736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-16T16:46:51.878522472Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d970aa448%3A19d9712c9d0%3A84cbcae8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-16T16:46:56.857549818Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:46:58.217188437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="5.6 MB" avg_uncomp="5.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-16T16:46:58.221081854Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19d970aa2b3%3A19d971d81bf%3A78557ef7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-16T16:47:01.151639151Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-16T16:47:01.857729947Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-16T16:47:06.857937407Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:47:09.640043338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=debug ts=2026-04-16T16:47:10.768080868Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-16T16:47:11.85742249Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-16T16:47:16.857429507Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:47:21.857317559Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-16T16:47:21.857330704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:47:21.857356852Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:47:21.857368757Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-16T16:47:26.857629355Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:47:28.840710135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.104:43698"
level=warn ts=2026-04-16T16:47:31.857988173Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:47:32.813864039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.3 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-16T16:47:32.822961372Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d970aa8b6%3A19d970b420e%3Ad0782992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-16T16:47:34.615931197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43566"
level=info ts=2026-04-16T16:47:34.870082562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="431 kB" avg_comp="431 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-16T16:47:34.87440793Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d970aae0a%3A19d97137d21%3A1f674789?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-16T16:47:36.857738449Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:47:36.951340056Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-16T16:47:39.511975313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="707 kB" avg_comp="707 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-04-16T16:47:39.516949561Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19d970b421d%3A19d971abebf%3A588715ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
ts=2026-04-16T16:47:39.642262909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-48j8q-f00cf162 10.129.0.67:7946"
level=info ts=2026-04-16T16:47:40.478864084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-16T16:47:40.484497726Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d970aa448%3A19d9712c9d0%3A84cbcae8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-16T16:47:40.591744415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2a5ccb133cfae346 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-xejw\"}"
level=error ts=2026-04-16T16:47:40.598528232Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2a5ccb133cfae346/19d9712996b%3A19d972d6476%3Ab3c66bbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-xejw\"}"
level=info ts=2026-04-16T16:47:41.602797854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="4.0 MB" avg_comp="2.0 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-16T16:47:41.608358766Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19d970b420d%3A19d971d8699%3A6d08d35d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-16T16:47:41.857463721Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-16T16:47:46.857950276Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:47:49.038118581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="5.6 MB" avg_uncomp="5.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-16T16:47:49.042929107Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19d970aa2b3%3A19d971d81bf%3A78557ef7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-16T16:47:49.630360017Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-16T16:47:49.630845472Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.68:11211 count=1
level=info ts=2026-04-16T16:47:51.857562491Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-16T16:47:51.857600253Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:47:51.857611589Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-16T16:47:51.857609017Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-16T16:47:51.857621548Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:47:51.857814991Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=debug ts=2026-04-16T16:47:51.858483907Z caller=checkpoint.go:378 msg="writing series" size="726 kB"
level=debug ts=2026-04-16T16:47:55.233548326Z caller=checkpoint.go:378 msg="writing series" size="144 kB"
level=warn ts=2026-04-16T16:47:56.8582702Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:47:57.161750519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:40540"
level=debug ts=2026-04-16T16:47:58.608633653Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=warn ts=2026-04-16T16:48:01.857497777Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:48:01.983840661Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=debug ts=2026-04-16T16:48:05.358356583Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-16T16:48:06.857636897Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:48:07.730022962Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-16T16:48:08.734206712Z caller=checkpoint.go:378 msg="writing series" size="611 kB"
level=debug ts=2026-04-16T16:48:08.734232859Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=6
ts=2026-04-16T16:48:09.645122849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a04a5887 10.128.0.104:7946"
level=warn ts=2026-04-16T16:48:11.857800671Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:48:12.05705054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-16T16:48:12.108489589Z caller=checkpoint.go:378 msg="writing series" size="152 kB"
level=debug ts=2026-04-16T16:48:15.483919344Z caller=checkpoint.go:378 msg="writing series" size="584 kB"
level=warn ts=2026-04-16T16:48:16.857764463Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:48:18.859001729Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=info ts=2026-04-16T16:48:21.85752289Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-16T16:48:21.85753696Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:48:21.857561364Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:48:21.857573517Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-16T16:48:22.240112251Z caller=checkpoint.go:378 msg="writing series" size="5.5 MB"
level=debug ts=2026-04-16T16:48:22.240161114Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.2 MB" series=4
level=info ts=2026-04-16T16:48:25.100409079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="431 kB" avg_comp="431 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-16T16:48:25.106507444Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d970aae0a%3A19d97137d21%3A1f674789?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-16T16:48:25.608202622Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=info ts=2026-04-16T16:48:26.525175243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-16T16:48:26.529829292Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d970aa448%3A19d9712c9d0%3A84cbcae8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-16T16:48:26.857974251Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:48:26.939285827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="707 kB" avg_comp="707 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-04-16T16:48:26.943571011Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19d970b421d%3A19d971abebf%3A588715ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-04-16T16:48:28.983857113Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=info ts=2026-04-16T16:48:31.686719013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.3 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-16T16:48:31.696627241Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d970aa8b6%3A19d970b420e%3Ad0782992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-16T16:48:31.858138685Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:48:32.358836276Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
ts=2026-04-16T16:48:34.621181608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57284"
level=debug ts=2026-04-16T16:48:34.862289272Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-16T16:48:35.443343939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="5.6 MB" avg_uncomp="5.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-16T16:48:35.446956374Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19d970aa2b3%3A19d971d81bf%3A78557ef7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-16T16:48:35.733439388Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-16T16:48:36.857726585Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:48:37.372292475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="4.0 MB" avg_comp="2.0 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-16T16:48:37.377064004Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19d970b420d%3A19d971d8699%3A6d08d35d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-16T16:48:39.108973756Z caller=checkpoint.go:378 msg="writing series" size="940 B"
level=info ts=2026-04-16T16:48:39.564564315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2a5ccb133cfae346 immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="18 MB" avg_uncomp="9.0 MB" full=1 synced=1 labels="{stream=\"group-xejw\"}"
level=error ts=2026-04-16T16:48:39.572436838Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2a5ccb133cfae346/19d9712996b%3A19d972d6476%3Ab3c66bbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-xejw\"}"
ts=2026-04-16T16:48:39.647693728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a04a5887 10.128.0.104:7946"
level=warn ts=2026-04-16T16:48:41.857241198Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:48:42.484458743Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-16T16:48:42.484485405Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=6
level=debug ts=2026-04-16T16:48:45.858884079Z caller=checkpoint.go:378 msg="writing series" size="214 kB"
level=warn ts=2026-04-16T16:48:46.857753197Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:48:49.23399896Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-04-16T16:48:49.63022762Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-16T16:48:49.630614715Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.68:11211 count=1
level=info ts=2026-04-16T16:48:51.857607802Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-16T16:48:51.857626508Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:48:51.857640723Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:48:51.857649392Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-16T16:48:52.608530273Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-16T16:48:52.608562715Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=3
level=debug ts=2026-04-16T16:48:55.98356228Z caller=checkpoint.go:378 msg="writing series" size="120 kB"
level=warn ts=2026-04-16T16:48:56.858178596Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:48:58.848269965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.104:44730"
level=debug ts=2026-04-16T16:48:59.359166661Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
level=warn ts=2026-04-16T16:49:01.857271543Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:49:02.733437918Z caller=checkpoint.go:378 msg="writing series" size="93 kB"
ts=2026-04-16T16:49:04.623509995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47002"
level=debug ts=2026-04-16T16:49:04.732077591Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-16T16:49:06.108602819Z caller=checkpoint.go:378 msg="writing series" size="257 kB"
level=warn ts=2026-04-16T16:49:06.857959956Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:49:07.515379496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="431 kB" avg_comp="431 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-16T16:49:07.520039929Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d970aae0a%3A19d97137d21%3A1f674789?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-16T16:49:09.483863613Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
ts=2026-04-16T16:49:09.650126434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a04a5887 10.128.0.104:7946"
level=warn ts=2026-04-16T16:49:11.857997746Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:49:12.868192322Z caller=checkpoint.go:378 msg="writing series" size="6.9 MB"
level=debug ts=2026-04-16T16:49:12.868218376Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.6 MB" series=6
level=info ts=2026-04-16T16:49:13.494220465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.3 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-16T16:49:13.502554115Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d970aa8b6%3A19d970b420e%3Ad0782992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-16T16:49:14.644856409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-16T16:49:14.649282253Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d970aa448%3A19d9712c9d0%3A84cbcae8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-16T16:49:16.233898457Z caller=checkpoint.go:378 msg="writing series" size="116 kB"
level=warn ts=2026-04-16T16:49:16.858165713Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:49:16.921277121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-16T16:49:17.468496034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="707 kB" avg_comp="707 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-04-16T16:49:17.472313296Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19d970b421d%3A19d971abebf%3A588715ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-04-16T16:49:18.826251139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="5.6 MB" avg_uncomp="5.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-16T16:49:18.829887931Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19d970aa2b3%3A19d971d81bf%3A78557ef7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-16T16:49:19.608580441Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-04-16T16:49:21.857175481Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-16T16:49:21.857215584Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:49:21.857228059Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-16T16:49:21.857220697Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:49:22.693851935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="4.0 MB" avg_comp="2.0 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-16T16:49:22.698961876Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19d970b420d%3A19d971d8699%3A6d08d35d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-16T16:49:22.983758175Z caller=checkpoint.go:378 msg="writing series" size="104 kB"
level=info ts=2026-04-16T16:49:23.363855765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2a5ccb133cfae346 immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="18 MB" avg_uncomp="9.0 MB" full=1 synced=1 labels="{stream=\"group-xejw\"}"
level=error ts=2026-04-16T16:49:23.370406452Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2a5ccb133cfae346/19d9712996b%3A19d972d6476%3Ab3c66bbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-xejw\"}"
level=debug ts=2026-04-16T16:49:26.358594516Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-16T16:49:26.858188517Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:49:29.733552018Z caller=checkpoint.go:378 msg="writing series" size="104 kB"
level=warn ts=2026-04-16T16:49:31.858183548Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:49:33.108657553Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=debug ts=2026-04-16T16:49:35.844502973Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-16T16:49:36.483291432Z caller=checkpoint.go:378 msg="writing series" size="151 kB"
level=warn ts=2026-04-16T16:49:36.857600583Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:49:39.65245108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=debug ts=2026-04-16T16:49:39.858236138Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=warn ts=2026-04-16T16:49:41.857363866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:49:43.234258384Z caller=checkpoint.go:378 msg="writing series" size="470 kB"
level=debug ts=2026-04-16T16:49:43.234286328Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-04-16T16:49:46.608782065Z caller=checkpoint.go:378 msg="writing series" size="473 kB"
level=warn ts=2026-04-16T16:49:46.85772676Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:49:49.631081348Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-16T16:49:49.632205407Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.68:11211 count=1
level=debug ts=2026-04-16T16:49:49.983514894Z caller=checkpoint.go:378 msg="writing series" size="152 kB"
level=info ts=2026-04-16T16:49:51.857338908Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-16T16:49:51.857357328Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:49:51.857380053Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:49:51.857396433Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-16T16:49:53.358184637Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=debug ts=2026-04-16T16:49:56.73326705Z caller=checkpoint.go:378 msg="writing series" size="639 B"
level=warn ts=2026-04-16T16:49:56.85778204Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:49:57.170707459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:35784"
level=debug ts=2026-04-16T16:50:00.110521772Z caller=checkpoint.go:378 msg="writing series" size="2.7 MB"
level=debug ts=2026-04-16T16:50:00.11054571Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.4 MB" series=5
level=debug ts=2026-04-16T16:50:00.390099511Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-16T16:50:01.60727794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-16T16:50:01.61378111Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d970aa448%3A19d9712c9d0%3A84cbcae8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-16T16:50:01.61380409Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-16T16:50:01.613829314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-16T16:50:01.618273668Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d970aa448%3A19d9712c9d0%3A84cbcae8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-16T16:50:01.857508908Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:50:03.483813015Z caller=checkpoint.go:378 msg="writing series" size="148 kB"
level=info ts=2026-04-16T16:50:04.597012469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.3 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-16T16:50:04.603587219Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d970aa8b6%3A19d970b420e%3Ad0782992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-16T16:50:04.603610508Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-16T16:50:04.603637424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.3 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-16T16:50:04.609437668Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d970aa8b6%3A19d970b420e%3Ad0782992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-16T16:50:06.068008458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="431 kB" avg_comp="431 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-16T16:50:06.071505712Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d970aae0a%3A19d97137d21%3A1f674789?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-16T16:50:06.071525787Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-16T16:50:06.071549855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="431 kB" avg_comp="431 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-16T16:50:06.074532951Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d970aae0a%3A19d97137d21%3A1f674789?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-16T16:50:06.857430819Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:50:06.858546261Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
ts=2026-04-16T16:50:09.655231354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a04a5887 10.128.0.104:7946"
level=info ts=2026-04-16T16:50:09.873942327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="707 kB" avg_comp="707 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-04-16T16:50:09.877958439Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19d970b421d%3A19d971abebf%3A588715ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-04-16T16:50:09.877981568Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-16T16:50:09.878014727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="44 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-16T16:50:09.885121905Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d970aace6%3A19d9712e550%3Aedc7a17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-16T16:50:10.233746127Z caller=checkpoint.go:378 msg="writing series" size="99 kB"
level=warn ts=2026-04-16T16:50:11.858194727Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:50:12.403566536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="5.6 MB" avg_uncomp="5.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-16T16:50:12.407036478Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19d970aa2b3%3A19d971d81bf%3A78557ef7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-16T16:50:12.407058078Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-16T16:50:12.407080631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="5.6 MB" avg_uncomp="5.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-16T16:50:12.410368115Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19d970aa2b3%3A19d971d81bf%3A78557ef7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-16T16:50:12.537908879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="4.0 MB" avg_comp="2.0 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-16T16:50:12.542071317Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19d970b420d%3A19d971d8699%3A6d08d35d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-16T16:50:12.542093182Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-16T16:50:12.707657301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="4.0 MB" avg_comp="2.0 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-16T16:50:12.712808556Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19d970b420d%3A19d971d8699%3A6d08d35d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-16T16:50:13.467491082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-16T16:50:13.471955592Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d970aa448%3A19d9712c9d0%3A84cbcae8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-16T16:50:13.608384328Z caller=checkpoint.go:378 msg="writing series" size="332 kB"
level=warn ts=2026-04-16T16:50:16.8579996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:50:16.983329272Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=info ts=2026-04-16T16:50:19.537756509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="431 kB" avg_comp="431 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-16T16:50:19.541214339Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d970aae0a%3A19d97137d21%3A1f674789?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-16T16:50:20.358510987Z caller=checkpoint.go:378 msg="writing series" size="616 kB"
level=debug ts=2026-04-16T16:50:20.358535789Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-04-16T16:50:21.256907081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.3 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-16T16:50:21.263586551Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d970aa8b6%3A19d970b420e%3Ad0782992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-16T16:50:21.366774036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="44 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-16T16:50:21.374201875Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d970aace6%3A19d9712e550%3Aedc7a17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-16T16:50:21.857815432Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-16T16:50:21.857855023Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:50:21.857869448Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-16T16:50:21.857862721Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:50:21.919197715Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-16T16:50:21.921069615Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-16T16:50:22.844970273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="4.0 MB" avg_comp="2.0 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-16T16:50:22.849282282Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19d970b420d%3A19d971d8699%3A6d08d35d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-16T16:50:23.366604675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2a5ccb133cfae346 immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="18 MB" avg_uncomp="9.0 MB" full=1 synced=1 labels="{stream=\"group-xejw\"}"
level=error ts=2026-04-16T16:50:23.374041967Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2a5ccb133cfae346/19d9712996b%3A19d972d6476%3Ab3c66bbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-xejw\"}"
level=debug ts=2026-04-16T16:50:23.7332693Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-16T16:50:26.858028376Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:50:27.108884358Z caller=checkpoint.go:378 msg="writing series" size="42 kB"
level=debug ts=2026-04-16T16:50:30.484114594Z caller=checkpoint.go:378 msg="writing series" size="407 kB"
level=warn ts=2026-04-16T16:50:31.857243018Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:50:31.921407939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="5.6 MB" avg_uncomp="5.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-16T16:50:31.926866195Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19d970aa2b3%3A19d971d81bf%3A78557ef7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-16T16:50:33.858350674Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-16T16:50:33.860539328Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-16T16:50:36.857785984Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:50:37.233282912Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
ts=2026-04-16T16:50:39.65798036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a04a5887 10.128.0.104:7946"
level=debug ts=2026-04-16T16:50:40.608172639Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-16T16:50:41.858105962Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:50:42.178554989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="431 kB" avg_comp="431 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-16T16:50:42.184345986Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d970aae0a%3A19d97137d21%3A1f674789?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-16T16:50:43.983893106Z caller=checkpoint.go:378 msg="writing series" size="600 kB"
level=debug ts=2026-04-16T16:50:43.983917358Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-04-16T16:50:44.17127887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.3 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-16T16:50:44.18018792Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d970aa8b6%3A19d970b420e%3Ad0782992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-16T16:50:46.14249944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-16T16:50:46.14688363Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d970aa448%3A19d9712c9d0%3A84cbcae8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-16T16:50:46.858187019Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:50:47.358311672Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-04-16T16:50:49.630563225Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-16T16:50:49.631345628Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.68:11211 count=1
level=debug ts=2026-04-16T16:50:50.734124648Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
level=info ts=2026-04-16T16:50:51.857105341Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-16T16:50:51.857139308Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:50:51.857164002Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-16T16:50:51.857196963Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:50:54.108229809Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-16T16:50:56.857586397Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:50:57.484420629Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-16T16:50:57.484445653Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=4
level=info ts=2026-04-16T16:50:57.554009339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="4.0 MB" avg_comp="2.0 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-16T16:50:57.560024962Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19d970b420d%3A19d971d8699%3A6d08d35d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-16T16:50:57.791794721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="44 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-16T16:50:57.799672111Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d970aace6%3A19d9712e550%3Aedc7a17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-16T16:51:00.858217724Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-16T16:51:01.857495678Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:51:04.233484883Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-16T16:51:06.857665969Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:51:07.267261882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="5.6 MB" avg_uncomp="5.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-16T16:51:07.271777073Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19d970aa2b3%3A19d971d81bf%3A78557ef7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-16T16:51:07.608729065Z caller=checkpoint.go:378 msg="writing series" size="357 kB"
level=debug ts=2026-04-16T16:51:09.64446712Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-16T16:51:09.660608262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=debug ts=2026-04-16T16:51:10.983450909Z caller=checkpoint.go:378 msg="writing series" size="205 kB"
level=warn ts=2026-04-16T16:51:11.858203195Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:51:12.66862506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2a5ccb133cfae346 immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="18 MB" avg_uncomp="9.0 MB" full=1 synced=1 labels="{stream=\"group-xejw\"}"
level=error ts=2026-04-16T16:51:12.67805428Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2a5ccb133cfae346/19d9712996b%3A19d972d6476%3Ab3c66bbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-xejw\"}"
level=error ts=2026-04-16T16:51:12.678083528Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-16T16:51:12.678109418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2a5ccb133cfae346 immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="18 MB" avg_uncomp="9.0 MB" full=1 synced=1 labels="{stream=\"group-xejw\"}"
level=error ts=2026-04-16T16:51:12.685403467Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2a5ccb133cfae346/19d9712996b%3A19d972d6476%3Ab3c66bbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-xejw\"}"
level=debug ts=2026-04-16T16:51:14.358296937Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=debug ts=2026-04-16T16:51:15.614747673Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-16T16:51:16.857694952Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:51:17.733538413Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=debug ts=2026-04-16T16:51:21.139429958Z caller=checkpoint.go:378 msg="writing series" size="44 MB"
level=debug ts=2026-04-16T16:51:21.13945562Z caller=checkpoint.go:390 msg="flushing series" totalSize="45 MB" series=7
level=info ts=2026-04-16T16:51:21.857750403Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-16T16:51:21.857783123Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:51:21.857794405Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-16T16:51:21.857857409Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:51:24.483270276Z caller=checkpoint.go:378 msg="writing series" size="121 kB"
level=warn ts=2026-04-16T16:51:26.85721353Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:51:27.858531429Z caller=checkpoint.go:378 msg="writing series" size="128 kB"
level=info ts=2026-04-16T16:51:28.372231031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2a5ccb133cfae346 immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="18 MB" avg_uncomp="9.0 MB" full=1 synced=1 labels="{stream=\"group-xejw\"}"
level=error ts=2026-04-16T16:51:28.381506858Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2a5ccb133cfae346/19d9712996b%3A19d972d6476%3Ab3c66bbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-xejw\"}"
level=debug ts=2026-04-16T16:51:31.233813463Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
level=warn ts=2026-04-16T16:51:31.857974Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:51:34.608568748Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=info ts=2026-04-16T16:51:36.429137537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.3 MB" total_uncomp="39 MB" avg_uncomp="19 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-16T16:51:36.436710907Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d970aa8b6%3A19d970b420e%3Ad0782992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-16T16:51:36.858028686Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:51:37.983301669Z caller=checkpoint.go:378 msg="writing series" size="99 kB"
level=info ts=2026-04-16T16:51:39.077781735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="431 kB" avg_comp="431 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-16T16:51:39.083878899Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d970aae0a%3A19d97137d21%3A1f674789?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-16T16:51:39.663505431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a04a5887 10.128.0.104:7946"
level=debug ts=2026-04-16T16:51:41.362230247Z caller=checkpoint.go:378 msg="writing series" size="7.2 MB"
level=debug ts=2026-04-16T16:51:41.362255869Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.5 MB" series=6
level=info ts=2026-04-16T16:51:41.785359407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-16T16:51:41.791031366Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d970aa448%3A19d9712c9d0%3A84cbcae8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-16T16:51:41.857444938Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:51:41.924786256Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-16T16:51:44.733847425Z caller=checkpoint.go:378 msg="writing series" size="238 kB"
level=info ts=2026-04-16T16:51:45.386572919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="6.9 MB" avg_comp="2.3 MB" total_uncomp="43 MB" avg_uncomp="14 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-16T16:51:45.392438335Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19d970b420d%3A19d971d8699%3A6d08d35d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-16T16:51:46.858202573Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:51:48.108783634Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=info ts=2026-04-16T16:51:49.630229967Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-16T16:51:49.630960142Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.68:11211 count=1
level=debug ts=2026-04-16T16:51:51.484014733Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=info ts=2026-04-16T16:51:51.857138921Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-16T16:51:51.857187741Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:51:51.857196741Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-16T16:51:51.857216436Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:51:53.486206828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="44 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-16T16:51:53.495026421Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d970aace6%3A19d9712e550%3Aedc7a17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-16T16:51:54.859630589Z caller=checkpoint.go:378 msg="writing series" size="537 kB"
level=warn ts=2026-04-16T16:51:56.857980578Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:51:57.17981365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:55104"
level=debug ts=2026-04-16T16:51:58.233920227Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
level=debug ts=2026-04-16T16:52:01.608046315Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=warn ts=2026-04-16T16:52:01.857426051Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:52:03.058279635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-16T16:52:03.710163533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2a5ccb133cfae346 immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="18 MB" avg_uncomp="9.0 MB" full=1 synced=1 labels="{stream=\"group-xejw\"}"
level=error ts=2026-04-16T16:52:03.719298679Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2a5ccb133cfae346/19d9712996b%3A19d972d6476%3Ab3c66bbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-xejw\"}"
level=info ts=2026-04-16T16:52:04.694348615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="5.6 MB" avg_uncomp="5.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-16T16:52:04.697879053Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19d970aa2b3%3A19d971d81bf%3A78557ef7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-16T16:52:04.983836047Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-16T16:52:06.857412818Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:52:08.359283981Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
ts=2026-04-16T16:52:09.665864646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=debug ts=2026-04-16T16:52:11.733975536Z caller=checkpoint.go:378 msg="writing series" size="323 kB"
level=debug ts=2026-04-16T16:52:11.734011198Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
level=warn ts=2026-04-16T16:52:11.857989944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:52:15.109524047Z caller=checkpoint.go:378 msg="writing series" size="984 kB"
level=debug ts=2026-04-16T16:52:16.502278841Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-16T16:52:16.858165088Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:52:18.48357759Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=debug ts=2026-04-16T16:52:18.483605024Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-04-16T16:52:21.858094463Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-16T16:52:21.858129443Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:52:21.85813839Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-16T16:52:21.858166687Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:52:21.865491247Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-04-16T16:52:21.866963145Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008980088s
level=info ts=2026-04-16T16:52:23.572662747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="431 kB" avg_comp="431 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-16T16:52:23.578690667Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d970aae0a%3A19d97137d21%3A1f674789?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-16T16:52:26.857982797Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:52:28.788178087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-16T16:52:28.793062901Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d970aa448%3A19d9712c9d0%3A84cbcae8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-16T16:52:30.645328409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.1 MB" avg_comp="2.4 MB" total_uncomp="62 MB" avg_uncomp="21 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-16T16:52:30.655719868Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d970aa8b6%3A19d970b420e%3Ad0782992?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-16T16:52:31.858104186Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:52:36.626869853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="6.9 MB" avg_comp="2.3 MB" total_uncomp="43 MB" avg_uncomp="14 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-16T16:52:36.631699998Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19d970b420d%3A19d971d8699%3A6d08d35d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-16T16:52:36.857489017Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-16T16:52:39.668893887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a04a5887 10.128.0.104:7946"
level=debug ts=2026-04-16T16:52:41.132635098Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-16T16:52:41.764379661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="44 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-16T16:52:41.772239742Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d970aace6%3A19d9712e550%3Aedc7a17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-16T16:52:41.857600661Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:52:44.014856879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2a5ccb133cfae346 immediate=false num_chunks=2 total_comp="4.6 MB" avg_comp="2.3 MB" total_uncomp="18 MB" avg_uncomp="9.0 MB" full=1 synced=1 labels="{stream=\"group-xejw\"}"
level=error ts=2026-04-16T16:52:44.021368081Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2a5ccb133cfae346/19d9712996b%3A19d972d6476%3Ab3c66bbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-xejw\"}"
level=warn ts=2026-04-16T16:52:46.857237212Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:52:49.630219518Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-16T16:52:49.630534313Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.68:11211 count=1
level=info ts=2026-04-16T16:52:51.857748272Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-16T16:52:51.857767419Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-16T16:52:51.857793659Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-16T16:52:51.85780253Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-16T16:52:51.857824983Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-16T16:52:51.857931882Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=debug ts=2026-04-16T16:52:51.858513652Z caller=checkpoint.go:378 msg="writing series" size="920 kB"
level=info ts=2026-04-16T16:52:53.847560432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="5.6 MB" avg_uncomp="5.6 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-16T16:52:53.853308894Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19d970aa2b3%3A19d971d81bf%3A78557ef7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-16T16:52:55.234029583Z caller=checkpoint.go:378 msg="writing series" size="148 kB"
level=debug ts=2026-04-16T16:52:55.234050933Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-04-16T16:52:56.857213862Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:52:58.609090309Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=debug ts=2026-04-16T16:52:58.877303151Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-16T16:53:01.857590714Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:53:01.983895264Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=debug ts=2026-04-16T16:53:05.358837722Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=warn ts=2026-04-16T16:53:06.858296676Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:53:08.734080851Z caller=checkpoint.go:378 msg="writing series" size="566 kB"
ts=2026-04-16T16:53:09.67210448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c4ml-b0f9528b 10.128.0.103:7946"
level=warn ts=2026-04-16T16:53:11.857856877Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-16T16:53:12.10848714Z caller=checkpoint.go:378 msg="writing series" size="152 kB"