Skip to content

Commit ea4e423

Browse files
gsmolkgsmolk
committed
Merge branch 'master' into issue_449
2 parents bbc7aec + 758a32f commit ea4e423

18 files changed

+343
-91
lines changed

‎doc/pgprobackup.xml

+6-1
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ doc/src/sgml/pgprobackup.sgml
131131
<arg choice="plain"><option>archive-push</option></arg>
132132
<arg choice="plain"><option>-B</option> <replaceable>backup_dir</replaceable></arg>
133133
<arg choice="plain"><option>--instance</option> <replaceable>instance_name</replaceable></arg>
134+
<arg choice="plain"><option>--wal-file-path</option> <replaceable>wal_file_path</replaceable></arg>
134135
<arg choice="plain"><option>--wal-file-name</option> <replaceable>wal_file_name</replaceable></arg>
135136
<arg rep="repeat"><replaceable>option</replaceable></arg>
136137
</cmdsynopsis>
@@ -5367,7 +5368,9 @@ pg_probackup catchup -b <replaceable>catchup_mode</replaceable>
53675368
Provides the path to the WAL file in
53685369
<parameter>archive_command</parameter> and
53695370
<parameter>restore_command</parameter>. Use the <literal>%p</literal>
5370-
variable as the value for this option for correct processing.
5371+
variable as the value for this option or explicitly specify the path to a file
5372+
outside of the data directory. If you skip this option, the path
5373+
specified in <filename>pg_probackup.conf</filename> will be used.
53715374
</para>
53725375
</listitem>
53735376
</varlistentry>
@@ -5380,6 +5383,8 @@ pg_probackup catchup -b <replaceable>catchup_mode</replaceable>
53805383
<parameter>archive_command</parameter> and
53815384
<parameter>restore_command</parameter>. Use the <literal>%f</literal>
53825385
variable as the value for this option for correct processing.
5386+
If the value of <option>--wal-file-path</option> is a path
5387+
outside of the data directory, explicitly specify the filename.
53835388
</para>
53845389
</listitem>
53855390
</varlistentry>

‎src/archive.c

+14-39
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* archive.c: - pg_probackup specific archive commands for archive backups.
44
*
55
*
6-
* Portions Copyright (c) 2018-2019, Postgres Professional
6+
* Portions Copyright (c) 2018-2021, Postgres Professional
77
*
88
*-------------------------------------------------------------------------
99
*/
@@ -140,15 +140,13 @@ get_xlogFileType(const char *filename)
140140
* Where archlog_path is $BACKUP_PATH/wal/instance_name
141141
*/
142142
void
143-
do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wal_file_path,
143+
do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg_xlog_dir,
144144
char *wal_file_name, int batch_size, bool overwrite,
145145
bool no_sync, bool no_ready_rename)
146146
{
147147
uint64 i;
148-
char current_dir[MAXPGPATH];
149-
char pg_xlog_dir[MAXPGPATH];
150-
char archive_status_dir[MAXPGPATH];
151-
uint64 system_id;
148+
/* usually instance pgdata/pg_wal/archive_status, empty if no_ready_rename or batch_size == 1 */
149+
char archive_status_dir[MAXPGPATH] = "";
152150
bool is_compress = false;
153151

154152
/* arrays with meta info for multi threaded backup */
@@ -169,31 +167,8 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
169167
parray *archive_subdirs = NULL;
170168
int n_threads;
171169

172-
if (wal_file_name == NULL)
173-
elog(ERROR, "Required parameter is not specified: --wal-file-name %%f");
174-
175-
if (!getcwd(current_dir, sizeof(current_dir)))
176-
elog(ERROR, "getcwd() error");
177-
178-
/* verify that archive-push --instance parameter is valid */
179-
system_id = get_system_identifier(current_dir, FIO_DB_HOST);
180-
181-
if (instance->pgdata == NULL)
182-
elog(ERROR, "Cannot read pg_probackup.conf for this instance");
183-
184-
if (system_id != instance->system_identifier)
185-
elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch."
186-
"Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT,
187-
wal_file_name, instanceState->instance_name, instance->system_identifier, system_id);
188-
189-
if (instance->compress_alg == PGLZ_COMPRESS)
190-
elog(ERROR, "Cannot use pglz for WAL compression");
191-
192-
join_path_components(pg_xlog_dir, current_dir, XLOGDIR);
193-
join_path_components(archive_status_dir, pg_xlog_dir, "archive_status");
194-
195-
/* Create 'archlog_path' directory. Do nothing if it already exists. */
196-
//fio_mkdir(instanceState->instance_wal_subdir_path, DIR_PERMISSION, FIO_BACKUP_HOST);
170+
if (!no_ready_rename || batch_size > 1)
171+
join_path_components(archive_status_dir, pg_xlog_dir, "archive_status");
197172

198173
#ifdef HAVE_LIBZ
199174
if (instance->compress_alg == ZLIB_COMPRESS)
@@ -246,12 +221,13 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
246221
{
247222
int rc;
248223
WALSegno *xlogfile = (WALSegno *) parray_get(batch_files, i);
224+
bool first_wal = strcmp(xlogfile->name, wal_file_name) == 0;
249225

250-
rc = push_file(xlogfile, archive_status_dir,
226+
rc = push_file(xlogfile, first_wal ? NULL : archive_status_dir,
251227
pg_xlog_dir, instanceState->instance_wal_subdir_path,
252228
overwrite, no_sync,
253229
instance->archive_timeout,
254-
no_ready_rename || (strcmp(xlogfile->name, wal_file_name) == 0) ? true : false,
230+
no_ready_rename || first_wal,
255231
is_compress && IsXLogFileName(xlogfile->name) ? true : false,
256232
instance->compress_level);
257233
if (rc == 0)
@@ -275,7 +251,7 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
275251
arg->first_filename = wal_file_name;
276252
arg->archive_dir = instanceState->instance_wal_subdir_path;
277253
arg->pg_xlog_dir = pg_xlog_dir;
278-
arg->archive_status_dir = archive_status_dir;
254+
arg->archive_status_dir = (!no_ready_rename || batch_size > 1) ? archive_status_dir : NULL;
279255
arg->overwrite = overwrite;
280256
arg->compress = is_compress;
281257
arg->no_sync = no_sync;
@@ -318,7 +294,7 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
318294

319295
/* Note, that we are leaking memory here,
320296
* because pushing into archive is a very
321-
* time-sensetive operation, so we skip freeing stuff.
297+
* time-sensitive operation, so we skip freeing stuff.
322298
*/
323299

324300
push_done:
@@ -398,9 +374,6 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir,
398374
int compress_level)
399375
{
400376
int rc;
401-
char wal_file_dummy[MAXPGPATH];
402-
403-
join_path_components(wal_file_dummy, archive_status_dir, xlogfile->name);
404377

405378
elog(LOG, "pushing file \"%s\"", xlogfile->name);
406379

@@ -417,11 +390,13 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir,
417390
#endif
418391

419392
/* take '--no-ready-rename' flag into account */
420-
if (!no_ready_rename)
393+
if (!no_ready_rename && archive_status_dir != NULL)
421394
{
395+
char wal_file_dummy[MAXPGPATH];
422396
char wal_file_ready[MAXPGPATH];
423397
char wal_file_done[MAXPGPATH];
424398

399+
join_path_components(wal_file_dummy, archive_status_dir, xlogfile->name);
425400
snprintf(wal_file_ready, MAXPGPATH, "%s.%s", wal_file_dummy, "ready");
426401
snprintf(wal_file_done, MAXPGPATH, "%s.%s", wal_file_dummy, "done");
427402

‎src/backup.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -943,7 +943,7 @@ check_system_identifiers(PGconn *conn, const char *pgdata)
943943
uint64 system_id_conn;
944944
uint64 system_id_pgdata;
945945

946-
system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST);
946+
system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false);
947947
system_id_conn = get_remote_system_identifier(conn);
948948

949949
/* for checkdb check only system_id_pgdata and system_id_conn */

‎src/catchup.c

+3-3
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, cons
4848

4949
/* Get WAL segments size and system ID of source PG instance */
5050
instance_config.xlog_seg_size = get_xlog_seg_size(source_pgdata);
51-
instance_config.system_identifier = get_system_identifier(source_pgdata, FIO_DB_HOST);
51+
instance_config.system_identifier = get_system_identifier(source_pgdata, FIO_DB_HOST, false);
5252
current.start_time = time(NULL);
5353

5454
strlcpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version));
@@ -163,15 +163,15 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn,
163163
uint64 source_conn_id, source_id, dest_id;
164164

165165
source_conn_id = get_remote_system_identifier(source_conn);
166-
source_id = get_system_identifier(source_pgdata, FIO_DB_HOST); /* same as instance_config.system_identifier */
166+
source_id = get_system_identifier(source_pgdata, FIO_DB_HOST, false); /* same as instance_config.system_identifier */
167167

168168
if (source_conn_id != source_id)
169169
elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu",
170170
source_conn_id, source_pgdata, source_id);
171171

172172
if (current.backup_mode != BACKUP_MODE_FULL)
173173
{
174-
dest_id = get_system_identifier(dest_pgdata, FIO_LOCAL_HOST);
174+
dest_id = get_system_identifier(dest_pgdata, FIO_LOCAL_HOST, false);
175175
if (source_conn_id != dest_id)
176176
elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu",
177177
source_conn_id, dest_pgdata, dest_id);

‎src/help.c

+5-1
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@ help_pg_probackup(void)
227227

228228
printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
229229
printf(_(" --wal-file-name=wal-file-name\n"));
230+
printf(_(" [--wal-file-path=wal-file-path]\n"));
230231
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
231232
printf(_(" [--archive-timeout=timeout]\n"));
232233
printf(_(" [--no-ready-rename] [--no-sync]\n"));
@@ -937,6 +938,7 @@ help_archive_push(void)
937938
{
938939
printf(_("\n%s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
939940
printf(_(" --wal-file-name=wal-file-name\n"));
941+
printf(_(" [--wal-file-path=wal-file-path]\n"));
940942
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
941943
printf(_(" [--archive-timeout=timeout]\n"));
942944
printf(_(" [--no-ready-rename] [--no-sync]\n"));
@@ -951,6 +953,8 @@ help_archive_push(void)
951953
printf(_(" --instance=instance_name name of the instance to delete\n"));
952954
printf(_(" --wal-file-name=wal-file-name\n"));
953955
printf(_(" name of the file to copy into WAL archive\n"));
956+
printf(_(" --wal-file-path=wal-file-path\n"));
957+
printf(_(" relative destination path of the WAL archive\n"));
954958
printf(_(" -j, --threads=NUM number of parallel threads\n"));
955959
printf(_(" --batch-size=NUM number of files to be copied\n"));
956960
printf(_(" --archive-timeout=timeout wait timeout before discarding stale temp file(default: 5min)\n"));
@@ -981,8 +985,8 @@ static void
981985
help_archive_get(void)
982986
{
983987
printf(_("\n%s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
984-
printf(_(" --wal-file-path=wal-file-path\n"));
985988
printf(_(" --wal-file-name=wal-file-name\n"));
989+
printf(_(" [--wal-file-path=wal-file-path]\n"));
986990
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
987991
printf(_(" [--no-validate-wal]\n"));
988992
printf(_(" [--remote-proto] [--remote-host]\n"));

‎src/init.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance)
5757
"(-D, --pgdata)");
5858

5959
/* Read system_identifier from PGDATA */
60-
instance->system_identifier = get_system_identifier(instance->pgdata, FIO_DB_HOST);
60+
instance->system_identifier = get_system_identifier(instance->pgdata, FIO_DB_HOST, false);
6161
/* Starting from PostgreSQL 11 read WAL segment size from PGDATA */
6262
instance->xlog_seg_size = get_xlog_seg_size(instance->pgdata);
6363

‎src/pg_probackup.c

+95-3
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
* which includes info about pgdata directory and connection.
3636
*
3737
* Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION
38-
* Portions Copyright (c) 2015-2019, Postgres Professional
38+
* Portions Copyright (c) 2015-2021, Postgres Professional
3939
*
4040
*-------------------------------------------------------------------------
4141
*/
@@ -151,6 +151,7 @@ static char *wal_file_path;
151151
static char *wal_file_name;
152152
static bool file_overwrite = false;
153153
static bool no_ready_rename = false;
154+
static char archive_push_xlog_dir[MAXPGPATH] = "";
154155

155156
/* archive get options */
156157
static char *prefetch_dir;
@@ -789,14 +790,105 @@ main(int argc, char *argv[])
789790
current.stream = stream_wal = true;
790791
if (instance_config.external_dir_str)
791792
elog(ERROR, "external directories not supported fom \"%s\" command", get_subcmd_name(backup_subcmd));
792-
// TODO проверить instance_config.conn_opt
793+
// TODO check instance_config.conn_opt
793794
}
794795

795796
/* sanity */
796797
if (backup_subcmd == VALIDATE_CMD && restore_params->no_validate)
797798
elog(ERROR, "You cannot specify \"--no-validate\" option with the \"%s\" command",
798799
get_subcmd_name(backup_subcmd));
799800

801+
if (backup_subcmd == ARCHIVE_PUSH_CMD)
802+
{
803+
/* Check archive-push parameters and construct archive_push_xlog_dir
804+
*
805+
* There are 4 cases:
806+
* 1. no --wal-file-path specified -- use cwd, ./PG_XLOG_DIR for wal files
807+
* (and ./PG_XLOG_DIR/archive_status for .done files inside do_archive_push())
808+
* in this case we can use batches and threads
809+
* 2. --wal-file-path is specified and it is the same dir as stored in pg_probackup.conf (instance_config.pgdata)
810+
* in this case we can use this path, as well as batches and thread
811+
* 3. --wal-file-path is specified and it isn't same dir as stored in pg_probackup.conf but control file present with correct system_id
812+
* in this case we can use this path, as well as batches and thread
813+
* (replica for example, see test_archive_push_sanity)
814+
* 4. --wal-file-path is specified and it is different from instance_config.pgdata and no control file found
815+
* disable optimizations and work with user specified path
816+
*/
817+
bool check_system_id = true;
818+
uint64 system_id;
819+
char current_dir[MAXPGPATH];
820+
821+
if (wal_file_name == NULL)
822+
elog(ERROR, "Required parameter is not specified: --wal-file-name %%f");
823+
824+
if (instance_config.pgdata == NULL)
825+
elog(ERROR, "Cannot read pg_probackup.conf for this instance");
826+
827+
/* TODO may be remove in preference of checking inside compress_init()? */
828+
if (instance_config.compress_alg == PGLZ_COMPRESS)
829+
elog(ERROR, "Cannot use pglz for WAL compression");
830+
831+
if (!getcwd(current_dir, sizeof(current_dir)))
832+
elog(ERROR, "getcwd() error");
833+
834+
if (wal_file_path == NULL)
835+
{
836+
/* 1st case */
837+
system_id = get_system_identifier(current_dir, FIO_DB_HOST, false);
838+
join_path_components(archive_push_xlog_dir, current_dir, XLOGDIR);
839+
}
840+
else
841+
{
842+
/*
843+
* Usually we get something like
844+
* wal_file_path = "pg_wal/0000000100000000000000A1"
845+
* wal_file_name = "0000000100000000000000A1"
846+
* instance_config.pgdata = "/pgdata/.../node/data"
847+
* We need to strip wal_file_name from wal_file_path, add XLOGDIR to instance_config.pgdata
848+
* and compare this directories.
849+
* Note, that pg_wal can be symlink (see test_waldir_outside_pgdata_archiving)
850+
*/
851+
char *stripped_wal_file_path = pgut_str_strip_trailing_filename(wal_file_path, wal_file_name);
852+
join_path_components(archive_push_xlog_dir, instance_config.pgdata, XLOGDIR);
853+
if (fio_is_same_file(stripped_wal_file_path, archive_push_xlog_dir, true, FIO_DB_HOST))
854+
{
855+
/* 2nd case */
856+
system_id = get_system_identifier(instance_config.pgdata, FIO_DB_HOST, false);
857+
/* archive_push_xlog_dir already have right value */
858+
}
859+
else
860+
{
861+
if (strlen(stripped_wal_file_path) < MAXPGPATH)
862+
strncpy(archive_push_xlog_dir, stripped_wal_file_path, MAXPGPATH);
863+
else
864+
elog(ERROR, "Value specified to --wal_file_path is too long");
865+
866+
system_id = get_system_identifier(current_dir, FIO_DB_HOST, true);
867+
/* 3rd case if control file present -- i.e. system_id != 0 */
868+
869+
if (system_id == 0)
870+
{
871+
/* 4th case */
872+
check_system_id = false;
873+
874+
if (batch_size > 1 || num_threads > 1 || !no_ready_rename)
875+
{
876+
elog(WARNING, "Supplied --wal_file_path is outside pgdata, force safe values for options: --batch-size=1 -j 1 --no-ready-rename");
877+
batch_size = 1;
878+
num_threads = 1;
879+
no_ready_rename = true;
880+
}
881+
}
882+
}
883+
pfree(stripped_wal_file_path);
884+
}
885+
886+
if (check_system_id && system_id != instance_config.system_identifier)
887+
elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch."
888+
"Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT,
889+
wal_file_name, instanceState->instance_name, instance_config.system_identifier, system_id);
890+
}
891+
800892
#if PG_VERSION_NUM >= 100000
801893
if (temp_slot && perm_slot)
802894
elog(ERROR, "You cannot specify \"--perm-slot\" option with the \"--temp-slot\" option");
@@ -820,7 +912,7 @@ main(int argc, char *argv[])
820912
switch (backup_subcmd)
821913
{
822914
case ARCHIVE_PUSH_CMD:
823-
do_archive_push(instanceState, &instance_config, wal_file_path, wal_file_name,
915+
do_archive_push(instanceState, &instance_config, archive_push_xlog_dir, wal_file_name,
824916
batch_size, file_overwrite, no_sync, no_ready_rename);
825917
break;
826918
case ARCHIVE_GET_CMD:

‎src/pg_probackup.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -895,7 +895,7 @@ extern int do_init(CatalogState *catalogState);
895895
extern int do_add_instance(InstanceState *instanceState, InstanceConfig *instance);
896896

897897
/* in archive.c */
898-
extern void do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wal_file_path,
898+
extern void do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg_xlog_dir,
899899
char *wal_file_name, int batch_size, bool overwrite,
900900
bool no_sync, bool no_ready_rename);
901901
extern void do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const char *prefetch_dir_arg, char *wal_file_path,
@@ -1163,7 +1163,7 @@ extern XLogRecPtr get_next_record_lsn(const char *archivedir, XLogSegNo segno, T
11631163
extern TimeLineID get_current_timeline(PGconn *conn);
11641164
extern TimeLineID get_current_timeline_from_control(const char *pgdata_path, fio_location location, bool safe);
11651165
extern XLogRecPtr get_checkpoint_location(PGconn *conn);
1166-
extern uint64 get_system_identifier(const char *pgdata_path, fio_location location);
1166+
extern uint64 get_system_identifier(const char *pgdata_path, fio_location location, bool safe);
11671167
extern uint64 get_remote_system_identifier(PGconn *conn);
11681168
extern uint32 get_data_checksum_version(bool safe);
11691169
extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path);

‎src/restore.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -2186,7 +2186,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
21862186
*/
21872187
elog(INFO, "Trying to read pg_control file in destination directory");
21882188

2189-
system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST);
2189+
system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false);
21902190

21912191
if (system_id_pgdata == instance_config.system_identifier)
21922192
system_id_match = true;

0 commit comments

Comments
 (0)