diff --git a/criu/pie/restorer.c b/criu/pie/restorer.c
index 0d1360c52b..9d1facf8ad 100644
--- a/criu/pie/restorer.c
+++ b/criu/pie/restorer.c
@@ -191,10 +191,8 @@ static int restore_creds(struct thread_creds_args *args, int procfd, int lsm_typ
 	int b, i, ret;
 	struct cap_header hdr;
 	struct cap_data data[_LINUX_CAPABILITY_U32S_3];
-
-	/*
-	 * We're still root here and thus can do it without failures.
-	 */
+	int ruid, euid, suid, fsuid;
+	int rgid, egid, sgid, fsgid;
 
 	/*
 	 * Setup supplementary group IDs early.
@@ -207,6 +205,18 @@ static int restore_creds(struct thread_creds_args *args, int procfd, int lsm_typ
 		}
 	}
 
+	/*
+	 * Compare xids with current values. If all match then we can skip
+	 * setting them (which requires extra capabilities).
+	 */
+	fsuid = sys_setfsuid(-1);
+	fsgid = sys_setfsgid(-1);
+	if (sys_getresuid(&ruid, &euid, &suid) == 0 && sys_getresgid(&rgid, &egid, &sgid) == 0 && ruid == ce->uid &&
+	    euid == ce->euid && suid == ce->suid && rgid == ce->gid && egid == ce->egid && sgid == ce->sgid &&
+	    fsuid == ce->fsuid && fsgid == ce->fsgid) {
+		goto skip_xids;
+	}
+
 	/*
 	 * First -- set the SECURE_NO_SETUID_FIXUP bit not to
 	 * lose caps bits when changing xids.
@@ -250,12 +260,13 @@ static int restore_creds(struct thread_creds_args *args, int procfd, int lsm_typ
 		return -1;
 	}
 
+skip_xids:
 	/*
 	 * Third -- restore securebits. We don't need them in any
 	 * special state any longer.
 	 */
 
-	if (!uid) {
+	if (sys_prctl(PR_GET_SECUREBITS, 0, 0, 0, 0) != ce->secbits) {
 		ret = sys_prctl(PR_SET_SECUREBITS, ce->secbits, 0, 0, 0);
 		if (ret) {
 			pr_err("Unable to set PR_SET_SECUREBITS: %d\n", ret);