K8SPG-643: Fix stanza-upgrade after major upgrade

e2e-tests/tests/major-upgrade/10-assert.yaml

@@ -7,7 +7,7 @@ kind: PerconaPGCluster
 metadata:
   name: major-upgrade
 spec:
-  postgresVersion: 16
+  postgresVersion: 13
 status:
   pgbouncer:
     ready: 3

e2e-tests/tests/major-upgrade/11-assert.yaml

@@ -1,10 +1,31 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
-timeout: 30
+timeout: 660
 ---
-kind: ConfigMap
-apiVersion: v1
+kind: Job
+apiVersion: batch/v1
 metadata:
-  name: 11-read-from-primary
-data:
-  data: ' 100500'
+  annotations:
+    postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-12-to-13
+  labels:
+    postgres-operator.crunchydata.com/pgbackrest-backup: manual
+    postgres-operator.crunchydata.com/pgbackrest-repo: repo1
+  ownerReferences:
+    - apiVersion: pgv2.percona.com/v2
+      kind: PerconaPGBackup
+      controller: true
+      blockOwnerDeletion: true
+status:
+  succeeded: 1
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGBackup
+metadata:
+  name: backup-after-12-to-13
+spec:
+  pgCluster: major-upgrade
+  repoName: repo1
+  options:
+    - --type=full
+status:
+  state: Succeeded

e2e-tests/tests/major-upgrade/11-run-backup.yaml

@@ -0,0 +1,9 @@
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGBackup
+metadata:
+  name: backup-after-12-to-13
+spec:
+  pgCluster: major-upgrade
+  repoName: repo1
+  options:
+    - --type=full

e2e-tests/tests/major-upgrade/12-assert.yaml

@@ -0,0 +1,30 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 660
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGRestore
+metadata:
+  name: restore-after-12-to-13
+spec:
+  pgCluster: major-upgrade
+  repoName: repo1
+status:
+  state: Succeeded
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGCluster
+metadata:
+  name: major-upgrade
+status:
+  pgbouncer:
+    ready: 3
+    size: 3
+  postgres:
+    instances:
+    - name: instance1
+      ready: 3
+      size: 3
+    ready: 3
+    size: 3
+  state: ready

e2e-tests/tests/major-upgrade/12-run-restore.yaml

@@ -0,0 +1,7 @@
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGRestore
+metadata:
+  name: restore-after-12-to-13
+spec:
+  pgCluster: major-upgrade
+  repoName: repo1

e2e-tests/tests/major-upgrade/05-read-from-primary.yaml → e2e-tests/tests/major-upgrade/13-read-from-primary.yaml

@@ -9,4 +9,4 @@ commands:
       source ../../functions
       data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass major-upgrade-pguser-postgres)@$(get_psql_user_host major-upgrade-pguser-postgres)")
 
-      kubectl create configmap -n "${NAMESPACE}" 05-read-from-primary --from-literal=data="${data}"
+      kubectl create configmap -n "${NAMESPACE}" 05-read-from-primary --from-literal=data="${data}"

e2e-tests/tests/major-upgrade/21-assert.yaml

@@ -0,0 +1,31 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 660
+---
+kind: Job
+apiVersion: batch/v1
+metadata:
+  annotations:
+    postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-13-to-14
+  labels:
+    postgres-operator.crunchydata.com/pgbackrest-backup: manual
+    postgres-operator.crunchydata.com/pgbackrest-repo: repo1
+  ownerReferences:
+    - apiVersion: pgv2.percona.com/v2
+      kind: PerconaPGBackup
+      controller: true
+      blockOwnerDeletion: true
+status:
+  succeeded: 1
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGBackup
+metadata:
+  name: backup-after-13-to-14
+spec:
+  pgCluster: major-upgrade
+  repoName: repo1
+  options:
+    - --type=full
+status:
+  state: Succeeded

e2e-tests/tests/major-upgrade/21-run-backup.yaml

@@ -0,0 +1,9 @@
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGBackup
+metadata:
+  name: backup-after-13-to-14
+spec:
+  pgCluster: major-upgrade
+  repoName: repo1
+  options:
+    - --type=full

e2e-tests/tests/major-upgrade/22-assert.yaml

@@ -0,0 +1,30 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 660
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGRestore
+metadata:
+  name: restore-after-13-to-14
+spec:
+  pgCluster: major-upgrade
+  repoName: repo1
+status:
+  state: Succeeded
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGCluster
+metadata:
+  name: major-upgrade
+status:
+  pgbouncer:
+    ready: 3
+    size: 3
+  postgres:
+    instances:
+    - name: instance1
+      ready: 3
+      size: 3
+    ready: 3
+    size: 3
+  state: ready

e2e-tests/tests/major-upgrade/22-run-restore.yaml

@@ -0,0 +1,7 @@
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGRestore
+metadata:
+  name: restore-after-13-to-14
+spec:
+  pgCluster: major-upgrade
+  repoName: repo1

e2e-tests/tests/major-upgrade/31-assert.yaml

@@ -0,0 +1,31 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 660
+---
+kind: Job
+apiVersion: batch/v1
+metadata:
+  annotations:
+    postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-14-to-15
+  labels:
+    postgres-operator.crunchydata.com/pgbackrest-backup: manual
+    postgres-operator.crunchydata.com/pgbackrest-repo: repo1
+  ownerReferences:
+    - apiVersion: pgv2.percona.com/v2
+      kind: PerconaPGBackup
+      controller: true
+      blockOwnerDeletion: true
+status:
+  succeeded: 1
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGBackup
+metadata:
+  name: backup-after-14-to-15
+spec:
+  pgCluster: major-upgrade
+  repoName: repo1
+  options:
+    - --type=full
+status:
+  state: Succeeded

e2e-tests/tests/major-upgrade/31-run-backup.yaml

@@ -0,0 +1,9 @@
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGBackup
+metadata:
+  name: backup-after-14-to-15
+spec:
+  pgCluster: major-upgrade
+  repoName: repo1
+  options:
+    - --type=full

e2e-tests/tests/major-upgrade/32-assert.yaml

@@ -0,0 +1,30 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 660
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGRestore
+metadata:
+  name: restore-after-14-to-15
+spec:
+  pgCluster: major-upgrade
+  repoName: repo1
+status:
+  state: Succeeded
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGCluster
+metadata:
+  name: major-upgrade
+status:
+  pgbouncer:
+    ready: 3
+    size: 3
+  postgres:
+    instances:
+    - name: instance1
+      ready: 3
+      size: 3
+    ready: 3
+    size: 3
+  state: ready

e2e-tests/tests/major-upgrade/32-run-restore.yaml

@@ -0,0 +1,8 @@
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGRestore
+metadata:
+  name: restore-after-14-to-15
+spec:
+  pgCluster: major-upgrade
+  repoName: repo1
+

e2e-tests/tests/major-upgrade/04-assert.yaml → e2e-tests/tests/major-upgrade/40-assert.yaml

@@ -7,7 +7,7 @@ kind: PerconaPGCluster
 metadata:
   name: major-upgrade
 spec:
-  postgresVersion: 13
+  postgresVersion: 16
 status:
   pgbouncer:
     ready: 3

e2e-tests/tests/major-upgrade/41-assert.yaml

@@ -0,0 +1,31 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 660
+---
+kind: Job
+apiVersion: batch/v1
+metadata:
+  annotations:
+    postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-15-to-16
+  labels:
+    postgres-operator.crunchydata.com/pgbackrest-backup: manual
+    postgres-operator.crunchydata.com/pgbackrest-repo: repo1
+  ownerReferences:
+    - apiVersion: pgv2.percona.com/v2
+      kind: PerconaPGBackup
+      controller: true
+      blockOwnerDeletion: true
+status:
+  succeeded: 1
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGBackup
+metadata:
+  name: backup-after-15-to-16
+spec:
+  pgCluster: major-upgrade
+  repoName: repo1
+  options:
+    - --type=full
+status:
+  state: Succeeded

e2e-tests/tests/major-upgrade/41-run-backup.yaml

@@ -0,0 +1,9 @@
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGBackup
+metadata:
+  name: backup-after-15-to-16
+spec:
+  pgCluster: major-upgrade
+  repoName: repo1
+  options:
+    - --type=full

e2e-tests/tests/major-upgrade/42-assert.yaml

@@ -0,0 +1,30 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 660
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGRestore
+metadata:
+  name: restore-after-15-to-16
+spec:
+  pgCluster: major-upgrade
+  repoName: repo1
+status:
+  state: Succeeded
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGCluster
+metadata:
+  name: major-upgrade
+status:
+  pgbouncer:
+    ready: 3
+    size: 3
+  postgres:
+    instances:
+    - name: instance1
+      ready: 3
+      size: 3
+    ready: 3
+    size: 3
+  state: ready

e2e-tests/tests/major-upgrade/42-run-restore.yaml

@@ -0,0 +1,8 @@
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGRestore
+metadata:
+  name: restore-after-15-to-16
+spec:
+  pgCluster: major-upgrade
+  repoName: repo1
+

e2e-tests/tests/major-upgrade/43-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 30
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: 11-read-from-primary
+data:
+  data: ' 100500'

internal/pgbackrest/pgbackrest.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/pkg/errors"
 
+	"github.com/percona/percona-postgresql-operator/internal/logging"
 	"github.com/percona/percona-postgresql-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
 )
 
@@ -103,11 +104,14 @@ else
     pgbackrest "${cmd}" --stanza="${stanza}"
 fi
 `
+	log := logging.FromContext(ctx)
 	if err := exec(ctx, nil, &stdout, &stderr, "bash", "-ceu", "--",
 		script, "-", configHash, DefaultStanzaName, errMsgConfigHashMismatch, errMsgStaleReposWithVolumesConfig,
 		fmt.Sprintf("stanza-%s", stanzaCmd), checkRepoCmd); err != nil {
 
-		errReturn := stderr.String()
+		errReturn := stdout.String() + " " + stderr.String()
+
+		log.Error(err, "stanza command failed", "cmd", stanzaCmd, "err", errReturn)
 
 		// if the config hashes didn't match, return true and don't return an error since this is
 		// expected while waiting for config changes in ConfigMaps and Secrets to make it to the