Skip to content

Commit 27129b7

Browse files
openshift-merge-bot[bot]openshift-merge-bot[bot]
authored
Merge pull request #1984 from tesshuflower/rsync-tls-symlink-munging
Rsync tls symlink munging
2 parents e6fedb5 + 2075a82 commit 27129b7

12 files changed

Lines changed: 442 additions & 15 deletions

File tree

‎Dockerfile‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -122,14 +122,14 @@ RUN microdnf --refresh update -y && \
122122
openssh `# rsync/ssh - ssh key generation in operator` \
123123
openssh-clients `# rsync/ssh - ssh client` \
124124
openssh-server `# rsync/ssh - ssh server` \
125-
python3 `# rsync/ssh - rrsync script` \
125+
python3 `# rsync/ssh - rrsync, munge-symlinks scripts` \
126126
stunnel `# rsync-tls` \
127127
openssl `# syncthing - server certs` \
128128
vim-minimal `# for mover debug` \
129129
tar `# for mover debug` \
130130
&& microdnf --setopt=install_weak_deps=0 install -y \
131131
`# docs are needed so rrsync gets installed for ssh variant` \
132-
rsync `# rsync/ssh, rsync-tls - rsync, rrsync` \
132+
rsync `# rsync/ssh, rsync-tls - rsync, rrsync, munge-symlinks` \
133133
&& microdnf clean all && \
134134
rm -rf /var/cache/yum
135135

@@ -158,6 +158,7 @@ RUN chmod a+rx /mover-rsync/*.sh
158158
RUN ln -s /keys/destination /etc/ssh/ssh_host_rsa_key && \
159159
ln -s /keys/destination.pub /etc/ssh/ssh_host_rsa_key.pub && \
160160
install /usr/share/doc/rsync/support/rrsync /usr/local/bin && \
161+
install /usr/share/doc/rsync/support/munge-symlinks /usr/local/bin && \
161162
\
162163
SSHD_CONFIG="/etc/ssh/sshd_config" && \
163164
sed -ir 's|^[#\s]*\(.*/etc/ssh/ssh_host_ecdsa_key\)$|#\1|' "$SSHD_CONFIG" && \

‎custom-scorecard-tests/config-downstream.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,16 @@ stages:
155155
storage:
156156
spec:
157157
mountPath: {}
158+
- entrypoint:
159+
- volsync-custom-scorecard-tests
160+
- test_rsync_tls_priv_manyfiles.yml
161+
image: quay.io/backube/volsync-custom-scorecard-tests:latest
162+
labels:
163+
suite: volsync-e2e
164+
test: test_rsync_tls_priv_manyfiles.yml
165+
storage:
166+
spec:
167+
mountPath: {}
158168
- entrypoint:
159169
- volsync-custom-scorecard-tests
160170
- test_simple_rsync.yml

‎custom-scorecard-tests/config.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,16 @@ stages:
155155
storage:
156156
spec:
157157
mountPath: {}
158+
- entrypoint:
159+
- volsync-custom-scorecard-tests
160+
- test_rsync_tls_priv_manyfiles.yml
161+
image: quay.io/backube/volsync-custom-scorecard-tests:latest
162+
labels:
163+
suite: volsync-e2e
164+
test: test_rsync_tls_priv_manyfiles.yml
165+
storage:
166+
spec:
167+
mountPath: {}
158168
- entrypoint:
159169
- volsync-custom-scorecard-tests
160170
- test_simple_rsync.yml

‎custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,16 @@
141141
storage:
142142
spec:
143143
mountPath: {}
144+
- entrypoint:
145+
- volsync-custom-scorecard-tests
146+
- test_rsync_tls_priv_manyfiles.yml
147+
image: quay.io/backube/volsync-custom-scorecard-tests:latest
148+
labels:
149+
suite: volsync-e2e
150+
test: test_rsync_tls_priv_manyfiles.yml
151+
storage:
152+
spec:
153+
mountPath: {}
144154
- entrypoint:
145155
- volsync-custom-scorecard-tests
146156
- test_simple_rsync.yml

‎mover-rsync-tls/client.sh‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,11 +153,23 @@ while [[ $rc -ne 0 && $RETRY -lt $MAX_RETRIES ]]; do
153153
find "${SOURCE}" -mindepth 1 -maxdepth 1 -printf '/%P\n' > /tmp/filelist.txt
154154
if [[ -s /tmp/filelist.txt ]]; then
155155
# 1st run preserves as much as possible, but excludes the root directory
156-
rsync -aAhHSxz -r --exclude=lost+found --itemize-changes --info=stats2,misc2 --files-from=/tmp/filelist.txt ${SOURCE}/ rsync://127.0.0.1:$STUNNEL_LISTEN_PORT/data
156+
rsync -aAhHSxz -r --exclude=lost+found --itemize-changes --info=stats2,misc2 --files-from=/tmp/filelist.txt ${SOURCE}/ rsync://127.0.0.1:$STUNNEL_LISTEN_PORT/data | tee /tmp/rsync-full.log
157+
rc_a=$?
158+
159+
if [[ $rc_a -eq 0 ]]; then
160+
# Symlinks - if any symlinks were created or modified, create a control file containing the list of symlinks
161+
# Symlinks in rsync --itemize-changes output are prefixed with 'cL' when sent to rsync daemon
162+
grep -E '^cL' /tmp/rsync-full.log | awk -F' -> ' '{print $1}' | sed 's/^[^ ]* //' >> /tmp/symlink-munging-file
163+
if [[ -s /tmp/symlink-munging-file ]]; then
164+
echo "Symlinks were created or modified, sending symlink munging file to destination..."
165+
rsync /tmp/symlink-munging-file rsync://127.0.0.1:$STUNNEL_LISTEN_PORT/control/symlink-munging-file
166+
rc_a=$?
167+
fi
168+
fi
157169
else
158170
echo "Skipping sync of empty source directory"
171+
rc_a=0
159172
fi
160-
rc_a=$?
161173

162174
# To delete extra files, must sync at the directory-level, but need to avoid
163175
# trying to modify the directory itself. This pass will only delete files

‎mover-rsync-tls/server.sh‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ set -e -o pipefail
44

55
RSYNC_PID_FILE=/tmp/rsyncd.pid
66
CONTROL_FILE=/tmp/control/complete
7+
CONTROL_FILE_SYMLINK_MUNGING_FILE=/tmp/control/symlink-munging-file
78
RSYNCD_CONF=/tmp/rsyncd.conf
89
STUNNEL_CONF=/tmp/stunnel.conf
910
STUNNEL_PID_FILE=/tmp/stunnel.pid
@@ -110,7 +111,7 @@ log file = $RSYNC_LOG
110111
max verbosity = 10
111112
use chroot = false
112113
numeric ids = true
113-
munge symlinks = false
114+
munge symlinks = true
114115
open noatime = true
115116
reverse lookup = false
116117
transfer logging = true
@@ -166,6 +167,7 @@ STUNNEL_CONF
166167
TAIL_PID="$!"
167168

168169
rm -f "$CONTROL_FILE"
170+
rm -f "$CONTROL_FILE_SYMLINK_MUNGING_FILE"
169171
fi
170172

171173
if test -b $BLOCK_TARGET; then
@@ -218,6 +220,29 @@ done
218220

219221
sleep 5 # Give time for the rsync connection to finish
220222

223+
# Before shutting down, read the symlink munging file if it exists, and process
224+
if [[ -s $CONTROL_FILE_SYMLINK_MUNGING_FILE ]]; then
225+
echo "Symlink munging file found, processing..."
226+
#while read -r symlink; do
227+
while IFS= read -r symlink; do
228+
symlinkfullpath="${TARGET}/${symlink}"
229+
echo "Processing symlink: ${symlinkfullpath}"
230+
231+
symlink_uid_gid=""
232+
if [[ $PRIVILEGED_MOVER -ne 0 ]]; then
233+
# If privileged mover, save uid/gid of the symlink for later
234+
symlink_uid_gid=$(stat -c '%u:%g' "${symlinkfullpath}")
235+
fi
236+
237+
munge-symlinks --unmunge "${symlinkfullpath}"
238+
239+
if [[ $PRIVILEGED_MOVER -ne 0 && -n "${symlink_uid_gid}" ]]; then
240+
# If privileged mover, restore uid/gid of the symlink after unmunging
241+
chown -h "${symlink_uid_gid}" "${symlinkfullpath}"
242+
fi
243+
done < "$CONTROL_FILE_SYMLINK_MUNGING_FILE"
244+
fi
245+
221246
##############################
222247
## Terminate stunnel
223248
echo "Shutting down..."

‎test-e2e/roles/compare_pvc_data/templates/job.yml.j2‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,21 +23,48 @@ spec:
2323
id
2424

2525
echo "Validating contents: PVC1 -> PVC2"
26-
find /mnt -type f | \
27-
xargs sha256sum | \
26+
find /mnt -type f -print0 | \
27+
xargs -0 sha256sum | \
2828
sed 's| /mnt/| /mnt2/|g' | \
2929
grep -v lost+found | \
3030
sha256sum -c
3131

3232
echo "Validating contents: PVC2 -> PVC1"
33-
find /mnt2 -type f | \
34-
xargs sha256sum | \
33+
find /mnt2 -type f -print0 | \
34+
xargs -0 sha256sum | \
3535
sed 's| /mnt2/| /mnt/|' | \
3636
grep -v lost+found | \
3737
sha256sum -c
3838

3939
echo "... all file contents matched"
4040

41+
echo "Validating symlinks: PVC1 -> PVC2"
42+
find /mnt -type l | { grep -v lost+found || true; } | sort | while IFS= read -r f; do
43+
f2=$(echo "$f" | sed 's|^/mnt/|/mnt2/|')
44+
if [ ! -L "$f2" ]; then
45+
echo "MISSING SYMLINK: $f exists but $f2 does not"
46+
exit 1
47+
fi
48+
link1=$(readlink "$f")
49+
link2=$(readlink "$f2")
50+
if [ "$link1" != "$link2" ]; then
51+
echo "MISMATCH: $f -> $link1, but $f2 -> $link2"
52+
exit 1
53+
fi
54+
echo "OK: $f -> $link1"
55+
done
56+
57+
echo "Validating symlinks: PVC2 -> PVC1"
58+
find /mnt2 -type l | { grep -v lost+found || true; } | sort | while IFS= read -r f; do
59+
f1=$(echo "$f" | sed 's|^/mnt2/|/mnt/|')
60+
if [ ! -L "$f1" ]; then
61+
echo "EXTRA SYMLINK: $f exists but $f1 does not"
62+
exit 1
63+
fi
64+
done
65+
66+
echo "... all symlinks matched"
67+
4168
echo "File attributes:"
4269
find /mnt -exec stat -c "{{ properties_to_verify }}" {} \; | \
4370
grep -v lost+found | \
@@ -47,6 +74,9 @@ spec:
4774
grep -v lost+found | \
4875
sort > /tmp/attributes-mnt2
4976

77+
{% if skip_symlink_ownership_check | default(false) %}
78+
sed -i 's/\(symbolic link\) uid:[0-9]*/\1/; s/\(symbolic link.*\) gid:[0-9]*/\1/' /tmp/attributes-mnt /tmp/attributes-mnt2
79+
{% endif %}
5080
echo "Looking for differences:"
5181
diff /tmp/attributes-mnt /tmp/attributes-mnt2
5282
echo "... none found"

‎test-e2e/roles/gather_cluster_info/tasks/main.yml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,12 +36,19 @@
3636
}
3737
}, recursive=True) }}
3838
39+
- name: Save kubernetes minor version as integer
40+
ansible.builtin.set_fact:
41+
cluster_info: >
42+
{{ cluster_info | combine({
43+
'kubernetes_minor_version': cluster_info.version.server.kubernetes.minor | regex_replace("[A-Za-z+]", "") | int
44+
}, recursive=True) }}
45+
3946
# AnyVolumeDataSource feature gate enabled by default in 1.24 and above
4047
- name: Determine if volume populator is supported
4148
ansible.builtin.set_fact:
4249
cluster_info: >
4350
{{ cluster_info | combine({
44-
'volumepopulator_supported': cluster_info.version.server.kubernetes.minor | regex_replace("[A-Za-z+]", "") | int >= 24
51+
'volumepopulator_supported': cluster_info.kubernetes_minor_version >= 24
4552
}, recursive=True) }}
4653
4754
- name: Print volumepopulator_supported

‎test-e2e/roles/write_to_pvc/templates/job.yml.j2‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ spec:
2121
set -x -e -o pipefail
2222

2323
id
24-
mkdir -p `dirname /mnt/{{ path }}`
24+
mkdir -p "$(dirname '/mnt/{{ path }}')"
2525
echo '{{ data }}' > '/mnt/{{ path }}'
2626
stat '/mnt/{{ path }}'
2727

@@ -31,6 +31,13 @@ spec:
3131
counter=$((counter+1))
3232
done
3333

34+
{% if symlinks is defined %}
35+
{% for symlink in symlinks %}
36+
mkdir -p "$(dirname '/mnt/{{ symlink.path }}')"
37+
ln -s '{{ symlink.target }}' '/mnt/{{ symlink.path }}'
38+
stat '/mnt/{{ symlink.path }}'
39+
{% endfor %}
40+
{% endif %}
3441
sync
3542
securityContext:
3643
allowPrivilegeEscalation: false

‎test-e2e/test_rsync_tls_normal_manyfiles.yml‎

Lines changed: 38 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
- e2e
55
- rsync_tls
66
- manyfiles
7+
- symlinks
78
- unprivileged
89
- volumepopulator
910
tasks:
@@ -113,24 +114,54 @@
113114
file_count: 21000
114115
pvc_name: 'data-source'
115116

116-
- name: Write file that starts with a hash char
117+
- name: Write file that has spaces in the name
117118
include_role:
118119
name: write_to_pvc
119120
vars:
120121
data: '000111'
121-
path: '/#filestartswithhash'
122+
path: '/file with spaces.txt'
122123
file_count: 1
123124
pvc_name: 'data-source'
124125

125-
- name: Write dir that starts with a hash char
126+
- name: Write dir that has spaces in the name
126127
include_role:
127128
name: write_to_pvc
128129
vars:
129130
data: '222333444555'
130-
path: '/#dirtartswithhash/#subfile'
131+
path: '/dir with spaces/subfile wspaces.txt'
131132
file_count: 1
132133
pvc_name: 'data-source'
133134

135+
- name: Write file that starts with a hash char
136+
include_role:
137+
name: write_to_pvc
138+
vars:
139+
data: '000111'
140+
path: '/#filestartswithhash'
141+
file_count: 1
142+
pvc_name: 'data-source'
143+
144+
- name: Write symlinks
145+
include_role:
146+
name: write_to_pvc
147+
vars:
148+
data: '000111'
149+
path: '/mysymlinktargetfile.txt'
150+
pvc_name: 'data-source'
151+
symlinks:
152+
# Relative symlinks
153+
- path: '/link-relative'
154+
target: 'mysymlinktargetfile.txt'
155+
- path: '/subdir/link-relative-parent'
156+
target: '../mysymlinktargetfile.txt'
157+
# Absolute symlinks that point outside of the PVC
158+
- path: '/link-absolute'
159+
target: '/opt/test/somefile.txt'
160+
- path: '/subdir/symlink-absolute-parent/another-symlink'
161+
target: '/tmp/testing/another-symlink-target'
162+
- path: '/dir with spaces/another simlink with spaces'
163+
target: '/tmp/testing/another symlink target with spaces'
164+
134165
- name: Wait for key and address to be ready
135166
kubernetes.core.k8s_info:
136167
api_version: volsync.backube/v1alpha1
@@ -268,3 +299,6 @@
268299
pvc1_name: data-source
269300
pvc2_name: data-dest
270301
timeout: 900
302+
# Skip comparing uids/gids on symlinks for older k8s versions - old host-path driver (<1.17.1) had an issue with
303+
# restoring symlinks in a snapshot, which meant symlinks were created with uid/gid of 0
304+
skip_symlink_ownership_check: "{{ cluster_info.kubernetes_minor_version < 24 }}"

0 commit comments

Comments
 (0)