From a89ca13e2edd0f238316c127e29a038114adab3e Mon Sep 17 00:00:00 2001 From: Polina Bungina Date: Thu, 25 Jan 2024 15:35:06 +0100 Subject: [PATCH 1/3] Ensure correct bin_dir for pg params configuration Make inplace upgrade code use the bin_dir of the proper version throught the whole process, so that ConfigHandler.write_postgresql_conf() validates provided params using the proper PG version. --- postgres-appliance/major_upgrade/inplace_upgrade.py | 3 --- postgres-appliance/major_upgrade/pg_upgrade.py | 12 ++++++++++-- postgres-appliance/tests/docker-compose.yml | 1 + 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/postgres-appliance/major_upgrade/inplace_upgrade.py b/postgres-appliance/major_upgrade/inplace_upgrade.py index 2d4ade477..6fa9e31f5 100644 --- a/postgres-appliance/major_upgrade/inplace_upgrade.py +++ b/postgres-appliance/major_upgrade/inplace_upgrade.py @@ -529,10 +529,7 @@ def do_upgrade(self): if self.replica_connections: from patroni.postgresql.misc import parse_lsn - # Make sure we use the pg_controldata from the correct major version - self.postgresql.set_bin_dir(self.cluster_version) controldata = self.postgresql.controldata() - self.postgresql.set_bin_dir(self.desired_version) checkpoint_lsn = controldata.get('Latest checkpoint location') if controldata.get('Database cluster state') != 'shut down' or not checkpoint_lsn: diff --git a/postgres-appliance/major_upgrade/pg_upgrade.py b/postgres-appliance/major_upgrade/pg_upgrade.py index dee894b99..354223794 100644 --- a/postgres-appliance/major_upgrade/pg_upgrade.py +++ b/postgres-appliance/major_upgrade/pg_upgrade.py @@ -35,7 +35,9 @@ def restore_shared_preload_libraries(self): return True def start_old_cluster(self, config, version): + self._new_bin_dir = self._bin_dir self.set_bin_dir(version) + self._old_bin_dir = self._bin_dir # make sure we don't archive wals from the old version self._old_config_values = {'archive_mode': self.config.get('parameters').get('archive_mode')} @@ -53,7 +55,6 @@ def get_cluster_version(self): def set_bin_dir(self, version): from spilo_commons import get_bin_dir - self._old_bin_dir = self._bin_dir self._bin_dir = get_bin_dir(version) @property @@ -168,7 +169,7 @@ def pg_upgrade(self, check=False): os.chdir(upgrade_dir) pg_upgrade_args = ['-k', '-j', str(psutil.cpu_count()), - '-b', self._old_bin_dir, '-B', self._bin_dir, + '-b', self._old_bin_dir, '-B', self._new_bin_dir, '-d', self._data_dir, '-D', self._new_data_dir, '-O', "-c timescaledb.restoring='on'", '-O', "-c archive_mode='off'"] @@ -180,8 +181,12 @@ def pg_upgrade(self, check=False): else: self.config.write_postgresql_conf() + self._bin_dir = self._new_bin_dir + logger.info('Executing pg_upgrade%s', (' --check' if check else '')) if subprocess.call([self.pgcommand('pg_upgrade')] + pg_upgrade_args) == 0: + if check: + self._bin_dir = self._old_bin_dir os.chdir(old_cwd) shutil.rmtree(upgrade_dir) return True @@ -206,7 +211,9 @@ def prepare_new_pgdata(self, version): old_version_file = self._version_file self._version_file = os.path.join(self._data_dir, 'PG_VERSION') + self._old_bin_dir = self._bin_dir self.set_bin_dir(version) + self._new_bin_dir = self._bin_dir # shared_preload_libraries for the old cluster, cleaned from incompatible/missing libs old_shared_preload_libraries = self.config.get('parameters').get('shared_preload_libraries') @@ -239,6 +246,7 @@ def prepare_new_pgdata(self, version): self._new_data_dir, self._data_dir = self._data_dir, self._new_data_dir self.config._postgresql_conf = old_postgresql_conf self._version_file = old_version_file + self._bin_dir = self._old_bin_dir if old_shared_preload_libraries: self.config.get('parameters')['shared_preload_libraries'] = old_shared_preload_libraries diff --git a/postgres-appliance/tests/docker-compose.yml b/postgres-appliance/tests/docker-compose.yml index a2b86e016..48b5ef277 100644 --- a/postgres-appliance/tests/docker-compose.yml +++ b/postgres-appliance/tests/docker-compose.yml @@ -46,6 +46,7 @@ services: postgresql: parameters: wal_keep_segments: 8 + jit: 'off' postgresql: parameters: shared_buffers: 32MB From 42e0ffdab18714ecfa7da8521854e2356b2aa2ae Mon Sep 17 00:00:00 2001 From: Polina Bungina Date: Fri, 26 Jan 2024 12:50:29 +0100 Subject: [PATCH 2/3] Adapt for the newer Patroni versions Patroni caches available GUCs since 3.2.1 --- postgres-appliance/major_upgrade/pg_upgrade.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/postgres-appliance/major_upgrade/pg_upgrade.py b/postgres-appliance/major_upgrade/pg_upgrade.py index 354223794..72c68f0ad 100644 --- a/postgres-appliance/major_upgrade/pg_upgrade.py +++ b/postgres-appliance/major_upgrade/pg_upgrade.py @@ -36,7 +36,7 @@ def restore_shared_preload_libraries(self): def start_old_cluster(self, config, version): self._new_bin_dir = self._bin_dir - self.set_bin_dir(version) + self.set_bin_dir_for_version(version) self._old_bin_dir = self._bin_dir # make sure we don't archive wals from the old version @@ -52,10 +52,15 @@ def get_cluster_version(self): with open(self._version_file) as f: return f.read().strip() - def set_bin_dir(self, version): + def set_bin_dir_for_version(self, version): from spilo_commons import get_bin_dir self._bin_dir = get_bin_dir(version) + self._available_gucs = None + + def set_bin_dir(self, bin_dir): + self._bin_dir = bin_dir + self._available_gucs = None @property def local_conn_kwargs(self): @@ -181,12 +186,12 @@ def pg_upgrade(self, check=False): else: self.config.write_postgresql_conf() - self._bin_dir = self._new_bin_dir + self.set_bin_dir(self._new_bin_dir) logger.info('Executing pg_upgrade%s', (' --check' if check else '')) if subprocess.call([self.pgcommand('pg_upgrade')] + pg_upgrade_args) == 0: if check: - self._bin_dir = self._old_bin_dir + self.set_bin_dir(self._old_bin_dir) os.chdir(old_cwd) shutil.rmtree(upgrade_dir) return True @@ -212,7 +217,7 @@ def prepare_new_pgdata(self, version): self._version_file = os.path.join(self._data_dir, 'PG_VERSION') self._old_bin_dir = self._bin_dir - self.set_bin_dir(version) + self.set_bin_dir_for_version(version) self._new_bin_dir = self._bin_dir # shared_preload_libraries for the old cluster, cleaned from incompatible/missing libs @@ -246,7 +251,7 @@ def prepare_new_pgdata(self, version): self._new_data_dir, self._data_dir = self._data_dir, self._new_data_dir self.config._postgresql_conf = old_postgresql_conf self._version_file = old_version_file - self._bin_dir = self._old_bin_dir + self.set_bin_dir(self._old_bin_dir) if old_shared_preload_libraries: self.config.get('parameters')['shared_preload_libraries'] = old_shared_preload_libraries From b780f0bfc51c0ee1a12bfcd287d04df643cce9c2 Mon Sep 17 00:00:00 2001 From: Polina Bungina Date: Thu, 15 Feb 2024 14:40:36 +0100 Subject: [PATCH 3/3] Reflect review comments --- postgres-appliance/major_upgrade/pg_upgrade.py | 4 +--- postgres-appliance/tests/docker-compose.yml | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/postgres-appliance/major_upgrade/pg_upgrade.py b/postgres-appliance/major_upgrade/pg_upgrade.py index 72c68f0ad..487fcd22d 100644 --- a/postgres-appliance/major_upgrade/pg_upgrade.py +++ b/postgres-appliance/major_upgrade/pg_upgrade.py @@ -54,9 +54,7 @@ def get_cluster_version(self): def set_bin_dir_for_version(self, version): from spilo_commons import get_bin_dir - - self._bin_dir = get_bin_dir(version) - self._available_gucs = None + self.set_bin_dir(get_bin_dir(version)) def set_bin_dir(self, bin_dir): self._bin_dir = bin_dir diff --git a/postgres-appliance/tests/docker-compose.yml b/postgres-appliance/tests/docker-compose.yml index 48b5ef277..90a89f9f9 100644 --- a/postgres-appliance/tests/docker-compose.yml +++ b/postgres-appliance/tests/docker-compose.yml @@ -45,6 +45,7 @@ services: loop_wait: 2 postgresql: parameters: + wal_decode_buffer_size: '521kB' wal_keep_segments: 8 jit: 'off' postgresql: