Skip to content

Commit 0bfb0b5

Browse files
thewtexkwrobot
thewtex
authored and
kwrobot
committed
Merge topic 'binary-data-script'
d1bb90f ENH: Add UploadBinaryData.sh script
2 parents 5b854d8 + d1bb90f commit 0bfb0b5

2 files changed

Lines changed: 196 additions & 0 deletions

File tree

‎Utilities/GitSetup/setup-git-aliases‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,6 @@ git config alias.gerrit-push \
1414
# Useful alias to see what commits are on the current branch with respect to
1515
# upstream/master
1616
git config alias.prepush 'log --graph --stat upstream/master..'
17+
18+
# Alias to the script to upload testing data
19+
git config alias.data-upload "!bash Utilities/UploadBinaryData.sh"

‎Utilities/UploadBinaryData.sh‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
#!/usr/bin/env bash
2+
#==========================================================================
3+
#
4+
# Copyright Insight Software Consortium
5+
#
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0.txt
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
#
18+
#==========================================================================*/
19+
20+
# Utility functions
21+
print_help() {
22+
cat << EOF
23+
Usage: $0 [--folder-id <folder_id>] <binary/file/path/in/repo/1> [<binary/file/path/in/repo/2> ...]
24+
25+
26+
Use this script to upload binary testing data for ITK and related projects.
27+
28+
Binary data, e.g. test input images, are not stored in Git because they
29+
will bloat a Git repository's size.
30+
31+
32+
To use the script:
33+
34+
1. Sign up for an account at https://data.kitware.com
35+
2. Place the binary file at the desired location in the Git repository.
36+
3. Run this script, and pass in the binary file(s) as arguments to script.
37+
4. In test/CMakeLists.txt, use the itk_add_test macro and reference the file
38+
path with \`DATA\` and braces, e.g.: DATA{<Relative/Path/To/Source/Tree/File>}.
39+
5. Re-build ITK, and the testing data will be downloaded into the build tree.
40+
41+
42+
If a GIRDER_API_KEY environmental variable is not set, a prompt will appear
43+
for your username and password. The API key can be created in the
44+
data.kitware.com user account web browser interface.
45+
46+
47+
The script will authenticate to data.kitware.com, upload the file to your
48+
user account's Public folder, and create a *.sha512 CMake ExternalData
49+
content link file. To specify a different folder, use the --folder flag.
50+
After the content link has been created, add the *.sha512 file to your
51+
git commit. The binary file will be removed from the source tree following
52+
upload.
53+
EOF
54+
}
55+
56+
die() {
57+
echo "$@" 1>&2; exit 1
58+
}
59+
60+
json_field() {
61+
local key=$1
62+
local json=$2
63+
echo $json | awk 'BEGIN { FS="\""; RS="," }; { if ($2 == "'$key'") {print $4} }'
64+
}
65+
66+
67+
# Parse arguments
68+
help=false
69+
folder_id=""
70+
while test $# -gt 0;
71+
do
72+
opt="$1";
73+
case "$opt" in
74+
"-h"|"--help")
75+
shift;
76+
help=true
77+
break;;
78+
"-f"|"--folder-id")
79+
shift;
80+
folder_id="$1"
81+
shift;;
82+
*)
83+
break;;
84+
esac
85+
done
86+
binary_files="$@"
87+
88+
if test "${binary_files}" = "" || $help; then
89+
print_help
90+
exit 1
91+
fi
92+
93+
94+
# Check for required dependencies
95+
if ! type curl > /dev/null; then
96+
die "Please install the curl executable."
97+
fi
98+
if ! type wc > /dev/null; then
99+
die "Please install the wc executable."
100+
fi
101+
102+
103+
# Authenticate
104+
token=""
105+
if test -n "${GIRDER_API_KEY}"; then
106+
token_response=$(curl -s -X POST --header 'Content-Length: 0' --header 'Content-Type: application/json' --header 'Accept: application/json' "https://data.kitware.com/api/v1/api_key/token?key=${GIRDER_API_KEY}&duration=1" || die "Could not retrieve token from API key.")
107+
token=$(json_field "token" "${token_response}")
108+
fi
109+
if test -z "${token}"; then
110+
if ! type base64 > /dev/null; then
111+
die "Please install the base64 executable."
112+
fi
113+
echo "Please provide your"
114+
echo ""
115+
echo " https://data.kitware.com"
116+
echo ""
117+
read -p "username: " username
118+
read -p "password: " -s password
119+
basic_content=$(echo -n "${username}:${password}" | base64)
120+
token_response=$(curl -s -X GET --header "Girder-Authorization: Basic ${basic_content}" --header 'Accept: */*' --header 'Host: data.kitware.com' --header 'Referer: https://data.kitware.com' 'https://data.kitware.com/api/v1/user/authentication' || die "Could not retrieve token from username / password.")
121+
token=$(json_field "token" "${token_response}")
122+
fi
123+
if test -z "${token}"; then
124+
die "Could not authenticate to https://data.kitware.com"
125+
fi
126+
127+
128+
# Get user / folder
129+
user_id_response=$(curl -s -X GET --header 'Accept: application/json' --header "Girder-Token: ${token}" 'https://data.kitware.com/api/v1/user/me' || die 'Could not get user id.')
130+
user_id=$(json_field "_id" "${user_id_response}")
131+
132+
if test -z "$folder_id"; then
133+
folder_id_response=$(curl -s -X GET --header 'Accept: application/json' --header "Girder-Token: ${token}" "https://data.kitware.com/api/v1/folder?parentType=user&parentId=${user_id}&name=Public&limit=3&sort=lowerName&sortdir=1" || die 'Could not get folder id.')
134+
folder_id=$(json_field "_id" "${folder_id_response}")
135+
fi
136+
137+
138+
# Upload files and create content links
139+
generated_content_links=""
140+
md5_content_link_conflicts=""
141+
for binary_file in $binary_files; do
142+
if test ! -e $binary_file; then
143+
die "$binary_file does not exist."
144+
fi
145+
item_name=$(basename "$binary_file")
146+
147+
create_item_response=$(curl -s -X POST --header 'Content-Length: 0' --header 'Content-Type: application/x-www-form-urlencoded' --header 'Accept: application/json' --header "Girder-Token: ${token}" "https://data.kitware.com/api/v1/item?folderId=${folder_id}&name=${item_name}&description=ITK%20testing%20data%20uploaded%20by%20ITK%2FUtilities%2FUploadBinaryData.sh&reuseExisting=true" || die 'Could not create item.')
148+
item_id=$(json_field "_id" "${create_item_response}")
149+
150+
size=$(wc -c < "$binary_file" | sed 's/^[[:space:]]*//')
151+
echo "Uploading ${item_name}..."
152+
upload_file_response=$(curl -X POST --data-binary "@${binary_file}" --header 'Content-Type: application/json' --header 'Accept: application/json' --header "Girder-Token: ${token}" "https://data.kitware.com/api/v1/file?parentType=item&parentId=${item_id}&name=${item_name}&size=${size}" || die 'Could not upload file.')
153+
file_id=$(json_field "_id" "${upload_file_response}")
154+
155+
curl -s -X GET --output "${binary_file}.sha512" --header 'Accept: text/plain' --header "Girder-Token: ${token}" "https://data.kitware.com/api/v1/file/${file_id}/hashsum_file/sha512" || die 'Could not get file sha512sum.'
156+
generated_content_links="$generated_content_links ${binary_file}.sha512"
157+
if type cmake > /dev/null; then
158+
cmake_sha512sum=$(cmake -E sha512sum "$binary_file" 2> /dev/null)
159+
# If sufficient CMake version, ...
160+
if test $? -eq 0; then
161+
local_sha512=$(echo $cmake_sha512sum | awk '{print $1}')
162+
remote_sha512=$(cat "${binary_file}.sha512")
163+
if test "$local_sha512" != "$remote_sha512"; then
164+
die "Local file hash does not match uploaded file hash."
165+
fi
166+
fi
167+
fi
168+
169+
md5_content_link="${binary_file}.md5"
170+
if test -e "$md5_content_link"; then
171+
md5_content_link_conflicts="$md5_content_link_conflicts $md5_content_link"
172+
fi
173+
rm $binary_file
174+
done
175+
176+
177+
# Recommend next steps
178+
cat << EOF
179+
180+
Testing data upload complete.
181+
182+
Now run:
183+
184+
git add --$generated_content_links
185+
186+
EOF
187+
if test -n "$md5_content_link_conflicts"; then
188+
cat << EOF
189+
and:
190+
191+
git rm --$md5_content_link_conflicts
192+
EOF
193+
fi

0 commit comments

Comments
 (0)