Skip to content

Commit a10c4f8

Browse files
smgollersmgoller
authored
GEODE-10369: Calculate heavy lifter costs. (#7796)
* Change to e2 instances. * Adjust machine type based on cpu and ram requirements. * Test half cpu on performance. * remove half-cpu test. Add calculations for standard machine types. * fix machine type * Indicate machine type to be created. * Determine CPU Platform. * Abandon n2 types because they don't scale high enough cpuwise. * Remove min cpu platform. * Utilize highcpu instances for cpu==ram jobs. * Make build job cpu==ram. * Change windows unit tests cpu/ram from 10 to 8. * Calculate total cost in delete_instance.sh * Add coreutils to alpine-tools so we can get gnu date. * Dump the json file with cost data. * Add highcpu case and identify unknown case. * fix MACHINE_FAMILY references. * Fix jq syntax. Copy cost-data.json to infra bucket. * Change cpuCost and ramCost to cpuRate and ramRate * Divide ram by 1024 because it's megabytes. * Use activity audit log instead of syslog so we can audit windows. * Revert job cpu changes.
1 parent e0f20a5 commit a10c4f8

3 files changed

Lines changed: 96 additions & 3 deletions

File tree

‎ci/images/alpine-tools/Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ COPY --from=hashicorp/packer:1.4.5 /bin/packer /usr/local/bin/packer
2929

3030
RUN apk --no-cache add \
3131
bash \
32+
coreutils \
3233
curl \
3334
git \
3435
jq \

‎ci/scripts/create_instance.sh‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -130,15 +130,28 @@ echo "${ZONE}" > "instance-data/zone"
130130
echo 'StrictHostKeyChecking no' >> /etc/ssh/ssh_config
131131
RAM_MEGABYTES=$( expr ${RAM} \* 1024 )
132132

133+
MACHINE_PREFIX="e2"
134+
135+
if (( ${RAM} > 128 )) || (( ${CPUS} > 32 )); then
136+
MACHINE_PREFIX="n1"
137+
fi
138+
139+
if (( ${RAM} == ${CPUS} )); then
140+
MACHINE_TYPE="${MACHINE_PREFIX}-highcpu-${CPUS}"
141+
elif (( ${RAM} / ${CPUS} == 4 )) && (( ${CPUS} <= 96 )); then
142+
MACHINE_TYPE="${MACHINE_PREFIX}-standard-${CPUS}"
143+
else
144+
MACHINE_TYPE="${MACHINE_PREFIX}-custom-${CPUS}-${RAM_MEGABYTES}"
145+
fi
146+
133147
TTL=$(($(date +%s) + 60 * 60 * 12))
134148
LABELS="instance_type=heavy-lifter,time-to-live=${TTL},job-name=${SANITIZED_BUILD_JOB_NAME},pipeline-name=${SANITIZED_BUILD_PIPELINE_NAME},build-name=${SANITIZED_BUILD_NAME},sha=${GEODE_SHA}"
135149
echo "Applying the following labels to the instance: ${LABELS}"
136-
150+
echo "Creating the instance with the following type: ${MACHINE_TYPE}"
137151
set +e
138152
INSTANCE_INFORMATION=$(gcloud compute --project=${GCP_PROJECT} instances create ${INSTANCE_NAME} \
139153
--zone=${ZONE} \
140-
--machine-type=custom-${CPUS}-${RAM_MEGABYTES} \
141-
--min-cpu-platform=Intel\ Skylake \
154+
--machine-type=${MACHINE_TYPE} \
142155
--network="${GCP_NETWORK}" \
143156
--subnet="${GCP_SUBNETWORK}" \
144157
--image="${IMAGE_NAME}" \

‎ci/scripts/delete_instance.sh‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
3333
is_source_from_pr_testable "geode" "$(get_geode_pr_exclusion_dirs)" "$(get_geode_pr_exclusion_files)" || exit 0
3434

3535
INSTANCE_NAME="$(cat instance-data/instance-name)"
36+
INSTANCE_ID="$(cat instance-data/instance-information | jq -r '.[].id')"
37+
GCP_PROJECT="$(cat instance-data/project)"
38+
INSTANCE_PRICING_JSON="$(gsutil cat gs://${GCP_PROJECT}-infra/pricing/instance_pricing.json)"
3639
PERMITTED_ZONES=($(gcloud compute zones list --filter="name~'us-central.*'" --format=json | jq -r .[].name))
3740

3841
echo 'StrictHostKeyChecking no' >> /etc/ssh/ssh_config
@@ -41,3 +44,79 @@ echo 'StrictHostKeyChecking no' >> /etc/ssh/ssh_config
4144
for KILL_ZONE in $(echo ${PERMITTED_ZONES[*]}); do
4245
gcloud compute instances delete ${INSTANCE_NAME} --zone=${KILL_ZONE} --quiet &>/dev/null || true
4346
done
47+
48+
while read BLAH ; do
49+
if [[ -z "${END_TIMESTAMP}" ]]; then
50+
END_TIMESTAMP=${BLAH}
51+
else
52+
START_TIMESTAMP=${BLAH}
53+
fi
54+
done < <(gcloud logging read "resource.type=gce_instance AND resource.labels.instance_id=${INSTANCE_ID} AND logName=projects/apachegeode-ci/logs/cloudaudit.googleapis.com%2Factivity" --format=json | jq -r '.[].timestamp')
55+
56+
START_SECONDS=$(date -d "${START_TIMESTAMP}" +%s)
57+
END_SECONDS=$(date -d "${END_TIMESTAMP}" +%s)
58+
TOTAL_SECONDS=$(expr ${END_SECONDS} - ${START_SECONDS})
59+
FULL_MACHINE_TYPE="$(cat instance-data/instance-information | jq -r '.[].machineType')"
60+
MACHINE_TYPE="${FULL_MACHINE_TYPE##*/}"
61+
62+
case "${MACHINE_TYPE}" in
63+
custom*)
64+
CPUS="$(echo "${MACHINE_TYPE}" | rev | cut -d'-' -f 2 | rev)"
65+
RAM_MB="$(echo "${MACHINE_TYPE}" | rev | cut -d'-' -f 1 | rev)"
66+
RAM="$(expr ${RAM_MB} / 1024)"
67+
MACHINE_FAMILY="n1"
68+
CPU_COST="$(echo "${INSTANCE_PRICING_JSON}" | jq -r ".\"${MACHINE_FAMILY}\".custom.cpu")"
69+
RAM_COST="$(echo "${INSTANCE_PRICING_JSON}" | jq -r ".\"${MACHINE_FAMILY}\".custom.ram")"
70+
;;
71+
*-custom*)
72+
CPUS="$(echo "${MACHINE_TYPE}" | rev | cut -d'-' -f 2 | rev)"
73+
RAM_MB="$(echo "${MACHINE_TYPE}" | rev | cut -d'-' -f 1 | rev)"
74+
RAM="$(expr ${RAM_MB} / 1024)"
75+
MACHINE_FAMILY="$(echo "${MACHINE_TYPE}" | cut -d'-' -f 1)"
76+
CPU_COST="$(echo "${INSTANCE_PRICING_JSON}" | jq -r ".\"${MACHINE_FAMILY}\".custom.cpu")"
77+
RAM_COST="$(echo "${INSTANCE_PRICING_JSON}" | jq -r ".\"${MACHINE_FAMILY}\".custom.ram")"
78+
;;
79+
*-standard*)
80+
CPUS="$(echo "${MACHINE_TYPE}" | rev | cut -d'-' -f 1 | rev)"
81+
RAM=$(expr ${CPUS} \* 4 )
82+
MACHINE_FAMILY="$(echo "${MACHINE_TYPE}" | cut -d'-' -f 1)"
83+
CPU_COST="$(echo "${INSTANCE_PRICING_JSON}" | jq -r ".\"${MACHINE_FAMILY}\".predefined.cpu")"
84+
RAM_COST="$(echo "${INSTANCE_PRICING_JSON}" | jq -r ".\"${MACHINE_FAMILY}\".predefined.ram")"
85+
;;
86+
*-highcpu*)
87+
CPUS="$(echo "${MACHINE_TYPE}" | rev | cut -d'-' -f 1 | rev)"
88+
RAM=${CPUS}
89+
MACHINE_FAMILY="$(echo "${MACHINE_TYPE}" | cut -d'-' -f 1)"
90+
CPU_COST="$(echo "${INSTANCE_PRICING_JSON}" | jq -r ".\"${MACHINE_FAMILY}\".predefined.cpu")"
91+
RAM_COST="$(echo "${INSTANCE_PRICING_JSON}" | jq -r ".\"${MACHINE_FAMILY}\".predefined.ram")"
92+
;;
93+
*)
94+
CPUS=0
95+
RAM=0
96+
MACHINE_FAMILY="unknown"
97+
CPU_COST=0
98+
RAM_COST=0
99+
;;
100+
esac
101+
102+
BUILD_NUMBER="$(cat instance-data/instance-information | jq -r '.[].labels."build-name"')"
103+
JOB_NAME="$(cat instance-data/instance-information | jq -r '.[].labels."job-name"')"
104+
PIPELINE_NAME="$(cat instance-data/instance-information | jq -r '.[].labels."pipeline-name"')"
105+
TOTAL_COST=$(echo "scale = 6; ((${CPUS} * ${CPU_COST}) + (${RAM} * ${RAM_COST})) * ${TOTAL_SECONDS} / 3600" | bc)
106+
echo "Total heavy lifter cost for ${PIPELINE_NAME}/${JOB_NAME} #${BUILD_NUMBER}: $ ${TOTAL_COST}"
107+
cat <<EOF > instance-data/cost-data.json
108+
{
109+
"pipeline": "${PIPELINE_NAME}",
110+
"job": "${JOB_NAME}",
111+
"instanceName": "${INSTANCE_NAME}",
112+
"instanceId": "${INSTANCE_ID}",
113+
"machineType": "${MACHINE_TYPE}",
114+
"cpu": "${CPUS}",
115+
"ram": "${RAM}",
116+
"seconds": "${TOTAL_SECONDS}",
117+
"cpuRate": "${CPU_COST}",
118+
"ramRate": "${RAM_COST}",
119+
"totalCost": "${TOTAL_COST}",
120+
}
121+
EOF
122+
gsutil cp instance-data/cost-data.json gs://${GCP_PROJECT}-infra/costs/${PIPELINE_NAME}/${JOB_NAME}/${PIPELINE_NAME}-${JOB_NAME}-${BUILD_NUMBER}-${INSTANCE_ID}.json

0 commit comments

Comments
 (0)