Open
Description
Please, answer some short questions which should help us to understand your problem / question better?
- Which image of the operator are you using? e.g. ghcr.io/zalando/postgres-operator:v1.12.2
- Which image of the operator are you using for upgrade? e.g. ghcr.io/zalando/postgres-operator:v1.14.0
- Where do you run it - cloud or metal? Kubernetes or OpenShift? AKS
- Are you running Postgres Operator in production? YES
- Type of issue? Bug report
Hello
I faced the issue after postgres-opreator upgrade from 1.12.2 to 1.14.0
- I am not able to connect to database with 'master' service. it failed with error:
psql: error: connection to server at "observe-sonarqube-db-cluster" (10.0.186.167), port 5432 failed: No route to host
but there is no issue with replica service
root@observe-sonarqube-db-cluster-1:/home/postgres# psql -U postgres -h observe-sonarqube-db-cluster-repl
Password for user postgres:
root@observe-sonarqube-db-cluster-1:/home/postgres# psql -U postgres -h observe-sonarqube-db-cluster
psql: error: connection to server at "observe-sonarqube-db-cluster" (10.0.186.167), port 5432 failed: No route to host
Is the server running on that host and accepting TCP/IP connections?
- postgres-operator is not able to communicate as well.
time="2025-05-20T14:33:53Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.186.167:5432: connect: no route to host" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
the error repeats every 30 min for each sync
Downgrade back to version 1.12.2 solved the issue.
Here are my custom values-env.yaml
# general configuration parameters
configGeneral:
etcd_host: "etcd-postgres.postgres:2379"
configKubernetes:
# list of additional capabilities for postgres container
additional_pod_capabilities:
- "SYS_NICE"
# key name for annotation that compares manifest value with current date
delete_annotation_date_key: "delete-date"
# key name for annotation that compares manifest value with cluster name
delete_annotation_name_key: "delete-clustername"
# toogles readiness probe for database pods
enable_readiness_probe: true
# namespaced name of the ConfigMap with environment variables to populate on every pod
pod_environment_configmap: "postgres/postgres-pod-config"
# name of the Secret (in cluster namespace) with environment variables to populate on every pod
pod_environment_secret: "postgres-pod-secrets"
# operator watches for postgres objects in the given namespace
watched_namespace: "postgres"
# allow user secrets in other namespaces than the Postgres cluster
enable_cross_namespace_secret: true
configLoadBalancer:
# annotations to apply to service when load balancing is enabled
custom_service_annotations:
"service.beta.kubernetes.io/azure-load-balancer-internal": "true"
# to set external IP from internal IP of AZURE
configPatroni:
# enable Patroni DCS failsafe_mode feature
# If enabled, Patroni copes with DCS outages by avoiding leader demotion.
enable_patroni_failsafe_mode: truePostgres-operator logs during this sync event:
time="2025-05-20T13:02:19Z" level=info msg="SYNC event has been queued" cluster-name=postgres/observe-sonarqube-db-cluster pkg=controller worker=0
time="2025-05-20T13:02:19Z" level=info msg="SYNC event has been queued" cluster-name=postgres/observe-zalando-db-cluster pkg=controller worker=1
time="2025-05-20T13:02:19Z" level=info msg="there are 2 clusters running" pkg=controller
time="2025-05-20T13:02:19Z" level=info msg="started working in background" pkg=controller
time="2025-05-20T13:02:19Z" level=info msg="listening on :8080" pkg=apiserver
time="2025-05-20T13:02:19Z" level=info msg="ADD event has been queued" cluster-name=postgres/observe-sonarqube-db-cluster pkg=controller worker=0
time="2025-05-20T13:02:19Z" level=info msg="ADD event has been queued" cluster-name=postgres/observe-zalando-db-cluster pkg=controller worker=1
time="2025-05-20T13:02:19Z" level=debug msg="new node has been added: /aks-default-23858168-vmss000000 (azure:///subscriptions/abd73b18-1bda-49cf-9292-fc533efe3de1/resourceGroups/enh-dev--aks-nodes/providers/Microsoft.Compute/virtualMachineScaleSets/aks-default-23858168-vmss/virtualMachines/0)" pkg=controller
time="2025-05-20T13:02:19Z" level=debug msg="new node has been added: /aks-default-23858168-vmss000007 (azure:///subscriptions/abd73b18-1bda-49cf-9292-fc533efe3de1/resourceGroups/enh-dev--aks-nodes/providers/Microsoft.Compute/virtualMachineScaleSets/aks-default-23858168-vmss/virtualMachines/7)" pkg=controller
time="2025-05-20T13:02:19Z" level=info msg="syncing of the cluster started" cluster-name=postgres/observe-sonarqube-db-cluster pkg=controller worker=0
time="2025-05-20T13:02:19Z" level=warning msg="enable_cross_namespace_secret is set. Database role name contains the respective namespace i.e. postgres.sonarqube_user is the created user" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:19Z" level=debug msg="team API is disabled" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:19Z" level=debug msg="team API is disabled" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:19Z" level=debug msg="syncing secrets" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:19Z" level=info msg="syncing of the cluster started" cluster-name=postgres/observe-zalando-db-cluster pkg=controller worker=1
time="2025-05-20T13:02:20Z" level=debug msg="team API is disabled" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:20Z" level=debug msg="team API is disabled" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:20Z" level=debug msg="syncing secrets" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:20Z" level=debug msg="syncing master service" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:20Z" level=debug msg="syncing master service" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:20Z" level=debug msg="syncing replica service" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:20Z" level=debug msg="syncing replica service" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:20Z" level=debug msg="syncing observe-sonarqube-db-cluster-config service" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:20Z" level=debug msg="syncing observe-sonarqube-db-cluster-leader endpoint" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:20Z" level=debug msg="syncing observe-zalando-db-cluster-config service" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:20Z" level=debug msg="syncing observe-sonarqube-db-cluster-config endpoint" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:21Z" level=debug msg="syncing observe-zalando-db-cluster-leader endpoint" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:21Z" level=debug msg="syncing observe-sonarqube-db-cluster-sync endpoint" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:21Z" level=debug msg="syncing observe-zalando-db-cluster-config endpoint" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:21Z" level=debug msg="syncing observe-sonarqube-db-cluster-failover endpoint" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:21Z" level=debug msg="syncing observe-zalando-db-cluster-sync endpoint" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:21Z" level=debug msg="syncing volumes using \"pvc\" storage resize mode" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:21Z" level=debug msg="syncing observe-zalando-db-cluster-failover endpoint" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:21Z" level=debug msg="volume claim for volume \"pgdata-observe-sonarqube-db-cluster-0\" do not require updates" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:21Z" level=debug msg="syncing volumes using \"pvc\" storage resize mode" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:21Z" level=debug msg="volume claim for volume \"pgdata-observe-sonarqube-db-cluster-1\" do not require updates" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:22Z" level=debug msg="volume claim for volume \"pgdata-observe-zalando-db-cluster-0\" do not require updates" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:22Z" level=debug msg="volume claim for volume \"pgdata-observe-sonarqube-db-cluster-2\" do not require updates" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:22Z" level=debug msg="volume claim for volume \"pgdata-observe-zalando-db-cluster-1\" do not require updates" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:22Z" level=debug msg="volume claims have been synced successfully" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:22Z" level=debug msg="volume claim for volume \"pgdata-observe-zalando-db-cluster-2\" do not require updates" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:22Z" level=debug msg="volume claims have been synced successfully" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="syncing statefulsets" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=info msg="mark rolling update annotation for observe-sonarqube-db-cluster-0: reason pod changes" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=info msg="mark rolling update annotation for observe-sonarqube-db-cluster-1: reason pod changes" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="syncing statefulsets" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=info msg="mark rolling update annotation for observe-zalando-db-cluster-0: reason pod changes" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=info msg="mark rolling update annotation for observe-sonarqube-db-cluster-2: reason pod changes" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=info msg="mark rolling update annotation for observe-zalando-db-cluster-1: reason pod changes" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=info msg="statefulset postgres/observe-sonarqube-db-cluster is not in the desired state and needs to be updated" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="- image: ghcr.io/zalando/spilo-16:3.2-p3," cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="+ image: ghcr.io/zalando/spilo-17:4.0-p2," cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="- terminationMessagePath: /dev/termination-log," cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="- terminationMessagePolicy: File," cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="- restartPolicy: Always," cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="- dnsPolicy: ClusterFirst," cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="- serviceAccount: postgres-pod," cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="- securityContext: {}," cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="- schedulerName: default-scheduler" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="+ securityContext: {}" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="- kind: PersistentVolumeClaim," cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="- apiVersion: v1," cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="- status: {" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="- phase: Pending" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="- }" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="+ status: {}" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="- revisionHistoryLimit: 10," cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=info msg="reason: new statefulset containers's postgres (index 0) image does not match the current one" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=debug msg="updating statefulset" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:23Z" level=info msg="mark rolling update annotation for observe-zalando-db-cluster-2: reason pod changes" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=info msg="statefulset postgres/observe-zalando-db-cluster is not in the desired state and needs to be updated" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="- image: ghcr.io/zalando/spilo-16:3.2-p3," cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="+ image: ghcr.io/zalando/spilo-17:4.0-p2," cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="- terminationMessagePath: /dev/termination-log," cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="- terminationMessagePolicy: File," cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="- restartPolicy: Always," cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="- dnsPolicy: ClusterFirst," cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="- serviceAccount: postgres-pod," cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="- securityContext: {}," cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="- schedulerName: default-scheduler" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="+ securityContext: {}" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="- kind: PersistentVolumeClaim," cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="- apiVersion: v1," cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="- status: {" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="- phase: Pending" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="- }" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="+ status: {}" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="- revisionHistoryLimit: 10," cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=info msg="reason: new statefulset containers's postgres (index 0) image does not match the current one" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="updating statefulset" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="syncing Patroni config" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="making GET http request: http://10.244.0.68:8008/config" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="making GET http request: http://10.244.0.247:8008/config" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="making GET http request: http://10.244.0.76:8008/config" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="making GET http request: http://10.244.0.247:8008/patroni" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="making GET http request: http://10.244.0.76:8008/patroni" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="making GET http request: http://10.244.0.68:8008/patroni" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=info msg="performing rolling update" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=info msg="there are 3 pods in the cluster to recreate" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="subscribing to pod \"postgres/observe-sonarqube-db-cluster-1\"" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="syncing Patroni config" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="making GET http request: http://10.244.0.185:8008/config" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="making GET http request: http://10.244.0.97:8008/config" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="making GET http request: http://10.244.1.98:8008/config" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="making GET http request: http://10.244.0.97:8008/patroni" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="making GET http request: http://10.244.1.98:8008/patroni" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="making GET http request: http://10.244.0.185:8008/patroni" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=info msg="performing rolling update" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=info msg="there are 3 pods in the cluster to recreate" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:02:24Z" level=debug msg="subscribing to pod \"postgres/observe-zalando-db-cluster-1\"" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:03:31Z" level=info msg="pod \"postgres/observe-zalando-db-cluster-1\" has been recreated" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:03:31Z" level=debug msg="unsubscribing from pod \"postgres/observe-zalando-db-cluster-1\" events" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:03:31Z" level=debug msg="subscribing to pod \"postgres/observe-zalando-db-cluster-2\"" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:03:31Z" level=info msg="pod \"postgres/observe-sonarqube-db-cluster-1\" has been recreated" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:03:31Z" level=debug msg="unsubscribing from pod \"postgres/observe-sonarqube-db-cluster-1\" events" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:03:31Z" level=debug msg="subscribing to pod \"postgres/observe-sonarqube-db-cluster-2\"" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:03:54Z" level=info msg="pod \"postgres/observe-zalando-db-cluster-2\" has been recreated" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:03:54Z" level=debug msg="unsubscribing from pod \"postgres/observe-zalando-db-cluster-2\" events" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:03:54Z" level=debug msg="making GET http request: http://10.244.0.185:8008/cluster" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:03:55Z" level=debug msg="switching over from \"observe-zalando-db-cluster-0\" to \"postgres/observe-zalando-db-cluster-1\"" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:03:55Z" level=debug msg="subscribing to pod \"postgres/observe-zalando-db-cluster-1\"" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:03:55Z" level=debug msg="making POST http request: http://10.244.0.185:8008/failover" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:03:57Z" level=debug msg="successfully switched over from \"observe-zalando-db-cluster-0\" to \"postgres/observe-zalando-db-cluster-1\"" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:04:16Z" level=info msg="pod \"postgres/observe-sonarqube-db-cluster-2\" has been recreated" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:04:16Z" level=debug msg="unsubscribing from pod \"postgres/observe-sonarqube-db-cluster-2\" events" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:04:16Z" level=debug msg="making GET http request: http://10.244.0.68:8008/cluster" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:04:16Z" level=debug msg="switching over from \"observe-sonarqube-db-cluster-0\" to \"postgres/observe-sonarqube-db-cluster-1\"" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:04:16Z" level=debug msg="subscribing to pod \"postgres/observe-sonarqube-db-cluster-1\"" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:04:16Z" level=debug msg="making POST http request: http://10.244.0.68:8008/failover" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:04:18Z" level=debug msg="successfully switched over from \"observe-sonarqube-db-cluster-0\" to \"postgres/observe-sonarqube-db-cluster-1\"" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:13:57Z" level=debug msg="unsubscribing from pod \"postgres/observe-zalando-db-cluster-1\" events" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:13:57Z" level=warning msg="error while syncing cluster state: could not sync statefulsets: could not recreate pods: could not perform switch over: could not get master pod label: pod label wait timeout" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:13:57Z" level=error msg="could not sync cluster: could not sync statefulsets: could not recreate pods: could not perform switch over: could not get master pod label: pod label wait timeout" cluster-name=postgres/observe-zalando-db-cluster pkg=controller worker=1
time="2025-05-20T13:13:57Z" level=info msg="received add event for already existing Postgres cluster" cluster-name=postgres/observe-zalando-db-cluster pkg=controller worker=1
time="2025-05-20T13:14:18Z" level=debug msg="unsubscribing from pod \"postgres/observe-sonarqube-db-cluster-1\" events" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:14:18Z" level=warning msg="error while syncing cluster state: could not sync statefulsets: could not recreate pods: could not perform switch over: could not get master pod label: pod label wait timeout" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:14:18Z" level=error msg="could not sync cluster: could not sync statefulsets: could not recreate pods: could not perform switch over: could not get master pod label: pod label wait timeout" cluster-name=postgres/observe-sonarqube-db-cluster pkg=controller worker=0
time="2025-05-20T13:14:18Z" level=info msg="received add event for already existing Postgres cluster" cluster-name=postgres/observe-sonarqube-db-cluster pkg=controller worker=0
time="2025-05-20T13:32:19Z" level=info msg="SYNC event has been queued" cluster-name=postgres/observe-sonarqube-db-cluster pkg=controller worker=0
time="2025-05-20T13:32:19Z" level=info msg="SYNC event has been queued" cluster-name=postgres/observe-zalando-db-cluster pkg=controller worker=1
time="2025-05-20T13:32:19Z" level=info msg="there are 2 clusters running" pkg=controller
time="2025-05-20T13:32:19Z" level=info msg="syncing of the cluster started" cluster-name=postgres/observe-zalando-db-cluster pkg=controller worker=1
time="2025-05-20T13:32:19Z" level=debug msg="team API is disabled" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:19Z" level=debug msg="team API is disabled" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:19Z" level=debug msg="syncing secrets" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:19Z" level=info msg="syncing of the cluster started" cluster-name=postgres/observe-sonarqube-db-cluster pkg=controller worker=0
time="2025-05-20T13:32:19Z" level=warning msg="enable_cross_namespace_secret is set. Database role name contains the respective namespace i.e. postgres.sonarqube_user is the created user" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:19Z" level=debug msg="team API is disabled" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:19Z" level=debug msg="team API is disabled" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:19Z" level=debug msg="syncing secrets" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:20Z" level=debug msg="syncing master service" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:20Z" level=debug msg="syncing master service" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:20Z" level=debug msg="syncing replica service" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:20Z" level=debug msg="syncing replica service" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:20Z" level=debug msg="syncing observe-sonarqube-db-cluster-config service" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:20Z" level=debug msg="syncing observe-zalando-db-cluster-config service" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:20Z" level=debug msg="syncing observe-sonarqube-db-cluster-leader endpoint" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:20Z" level=debug msg="syncing observe-zalando-db-cluster-leader endpoint" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:20Z" level=debug msg="syncing observe-sonarqube-db-cluster-config endpoint" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:20Z" level=debug msg="syncing observe-zalando-db-cluster-config endpoint" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:21Z" level=debug msg="syncing observe-sonarqube-db-cluster-sync endpoint" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:21Z" level=debug msg="syncing observe-zalando-db-cluster-sync endpoint" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:21Z" level=debug msg="syncing observe-sonarqube-db-cluster-failover endpoint" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:21Z" level=debug msg="syncing observe-zalando-db-cluster-failover endpoint" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:21Z" level=debug msg="syncing volumes using \"pvc\" storage resize mode" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:21Z" level=debug msg="syncing volumes using \"pvc\" storage resize mode" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:21Z" level=debug msg="volume claim for volume \"pgdata-observe-sonarqube-db-cluster-0\" do not require updates" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:21Z" level=debug msg="volume claim for volume \"pgdata-observe-zalando-db-cluster-0\" do not require updates" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:21Z" level=debug msg="volume claim for volume \"pgdata-observe-sonarqube-db-cluster-1\" do not require updates" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:22Z" level=debug msg="volume claim for volume \"pgdata-observe-zalando-db-cluster-1\" do not require updates" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:22Z" level=debug msg="volume claim for volume \"pgdata-observe-sonarqube-db-cluster-2\" do not require updates" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:22Z" level=debug msg="volume claim for volume \"pgdata-observe-zalando-db-cluster-2\" do not require updates" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:22Z" level=debug msg="volume claims have been synced successfully" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:22Z" level=debug msg="volume claims have been synced successfully" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="syncing statefulsets" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="found rolling update flag on pod \"observe-sonarqube-db-cluster-0\"" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=info msg="1 / 3 pod(s) still need to be rotated" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="syncing statefulsets" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="found rolling update flag on pod \"observe-zalando-db-cluster-0\"" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=info msg="1 / 3 pod(s) still need to be rotated" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="syncing Patroni config" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="making GET http request: http://10.244.0.68:8008/config" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="making GET http request: http://10.244.1.125:8008/config" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="making GET http request: http://10.244.1.205:8008/config" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="making GET http request: http://10.244.0.68:8008/patroni" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="making GET http request: http://10.244.1.125:8008/patroni" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="making GET http request: http://10.244.1.205:8008/patroni" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=info msg="performing rolling update" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=info msg="there are 1 pods in the cluster to recreate" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="subscribing to pod \"postgres/observe-sonarqube-db-cluster-0\"" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="syncing Patroni config" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="making GET http request: http://10.244.0.185:8008/config" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="making GET http request: http://10.244.1.132:8008/config" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="making GET http request: http://10.244.1.234:8008/config" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="making GET http request: http://10.244.0.185:8008/patroni" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="making GET http request: http://10.244.1.132:8008/patroni" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="making GET http request: http://10.244.1.234:8008/patroni" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=info msg="performing rolling update" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=info msg="there are 1 pods in the cluster to recreate" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:32:23Z" level=debug msg="subscribing to pod \"postgres/observe-zalando-db-cluster-0\"" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:33:02Z" level=info msg="pod \"postgres/observe-zalando-db-cluster-0\" has been recreated" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:33:02Z" level=debug msg="unsubscribing from pod \"postgres/observe-zalando-db-cluster-0\" events" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:33:02Z" level=debug msg="syncing pod disruption budgets" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:33:02Z" level=debug msg="syncing roles" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:33:02Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.43.107:5432: connect: no route to host" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:33:02Z" level=info msg="pod \"postgres/observe-sonarqube-db-cluster-0\" has been recreated" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:33:02Z" level=debug msg="unsubscribing from pod \"postgres/observe-sonarqube-db-cluster-0\" events" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:33:02Z" level=debug msg="syncing pod disruption budgets" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:33:02Z" level=debug msg="syncing roles" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:33:02Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.186.167:5432: connect: no route to host" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:33:17Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.43.107:5432: connect: no route to host" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:33:17Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.186.167:5432: connect: no route to host" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:33:32Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.43.107:5432: connect: no route to host" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:33:32Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.186.167:5432: connect: no route to host" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:33:47Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.43.107:5432: connect: no route to host" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:33:47Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.186.167:5432: connect: no route to host" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:34:02Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.43.107:5432: connect: no route to host" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:34:02Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.186.167:5432: connect: no route to host" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:34:17Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.43.107:5432: connect: no route to host" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:34:17Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.186.167:5432: connect: no route to host" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:34:32Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.43.107:5432: connect: no route to host" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:34:32Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.186.167:5432: connect: no route to host" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:34:47Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.43.107:5432: connect: no route to host" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:34:47Z" level=error msg="could not sync roles: could not init db connection: could not init db connection: still failing after 8 retries" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:34:47Z" level=debug msg="syncing databases" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:34:47Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.43.107:5432: connect: no route to host" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:34:47Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.186.167:5432: connect: no route to host" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:34:47Z" level=error msg="could not sync roles: could not init db connection: could not init db connection: still failing after 8 retries" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:34:47Z" level=debug msg="syncing databases" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:34:47Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.186.167:5432: connect: no route to host" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:35:02Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.43.107:5432: connect: no route to host" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:35:02Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.186.167:5432: connect: no route to host" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:35:17Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.43.107:5432: connect: no route to host" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:35:17Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.186.167:5432: connect: no route to host" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:35:32Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.43.107:5432: connect: no route to host" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:35:32Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.186.167:5432: connect: no route to host" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:35:47Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.43.107:5432: connect: no route to host" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:35:47Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.186.167:5432: connect: no route to host" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:36:02Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.43.107:5432: connect: no route to host" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:36:02Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.186.167:5432: connect: no route to host" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:36:17Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.43.107:5432: connect: no route to host" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:36:17Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.186.167:5432: connect: no route to host" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:36:32Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.43.107:5432: connect: no route to host" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:36:32Z" level=error msg="could not sync databases: could not init database connection" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:36:32Z" level=debug msg="syncing prepared databases with schemas" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:36:32Z" level=debug msg="syncing connection pooler (master, replica) from (false, nil) to (false, nil)" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:36:32Z" level=debug msg="making GET http request: http://10.244.0.6:8008/patroni" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:36:32Z" level=debug msg="making GET http request: http://10.244.1.132:8008/patroni" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:36:32Z" level=debug msg="making GET http request: http://10.244.1.234:8008/patroni" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:36:32Z" level=info msg="recheck cluster version is already up to date. current: 130018, min desired: 130000" cluster-name=postgres/observe-zalando-db-cluster pkg=cluster
time="2025-05-20T13:36:32Z" level=info msg="cluster has been synced" cluster-name=postgres/observe-zalando-db-cluster pkg=controller worker=1
time="2025-05-20T13:36:32Z" level=warning msg="could not connect to Postgres database: dial tcp 10.0.186.167:5432: connect: no route to host" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:36:32Z" level=error msg="could not sync databases: could not init database connection" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:36:32Z" level=debug msg="syncing prepared databases with schemas" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:36:32Z" level=debug msg="syncing connection pooler (master, replica) from (false, nil) to (false, nil)" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:36:32Z" level=debug msg="making GET http request: http://10.244.0.25:8008/patroni" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:36:32Z" level=debug msg="making GET http request: http://10.244.1.125:8008/patroni" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:36:32Z" level=debug msg="making GET http request: http://10.244.1.205:8008/patroni" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:36:32Z" level=info msg="recheck cluster version is already up to date. current: 130018, min desired: 130000" cluster-name=postgres/observe-sonarqube-db-cluster pkg=cluster
time="2025-05-20T13:36:32Z" level=info msg="cluster has been synced" cluster-name=postgres/observe-sonarqube-db-cluster pkg=controller worker=0kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
observe-sonarqube-db-cluster ClusterIP 10.0.186.167 <none> 5432/TCP 44h
observe-sonarqube-db-cluster-repl ClusterIP 10.0.51.168 <none> 5432/TCP 44h
observe-zalando-db-cluster ClusterIP 10.0.43.107 <none> 5432/TCP 23h
observe-zalando-db-cluster-repl ClusterIP 10.0.30.234 <none> 5432/TCP 23h
postgres-operator ClusterIP 10.0.77.159 <none> 8080/TCP 44hkubectl get pods -l application=spilo -L spilo-role
NAME READY STATUS RESTARTS AGE SPILO-ROLE
observe-sonarqube-db-cluster-0 1/1 Running 0 16h replica
observe-sonarqube-db-cluster-1 1/1 Running 0 17h primary
observe-sonarqube-db-cluster-2 1/1 Running 0 17h replica
observe-zalando-db-cluster-0 1/1 Running 0 16h replica
observe-zalando-db-cluster-1 1/1 Running 0 17h primary
observe-zalando-db-cluster-2 1/1 Running 0 17h replicaI have noticed that pods label SPILO-ROLE changed form 'master' to 'primary' after the upgrade.
Could you please have look?
Metadata
Metadata
Assignees
Labels
No labels