Skip to content

Commit 65ab1f5

Browse files
serban-nicusor-toptalserban-nicusor-toptal
authored
Docker images for Flatiron Institute Jenkins migration (#17)
* Refactored Dockerfiles for Flatiron CI * Add stan2ftp tensorflow image * Refactored Jenkinsfile for Flatiron CI
1 parent 70d76b0 commit 65ab1f5

7 files changed

Lines changed: 106 additions & 28 deletions

File tree

‎Jenkinsfile‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ pipeline {
4444
stages {
4545

4646
stage('Set tags') {
47-
agent { label 'gg-linux' }
47+
agent { label 'linux' }
4848
steps {
4949
script {
5050
println "Setting tags for stanc3 branch ${params.stanc3_branch} or ci-scripts branch ${params.ciscripts_branch}"
@@ -81,7 +81,7 @@ pipeline {
8181
parallel {
8282

8383
stage("stanc3 multiarch") {
84-
agent { label 'gg-linux' }
84+
agent { label 'linux' }
8585
when {
8686
beforeAgent true
8787
allOf {
@@ -105,7 +105,7 @@ pipeline {
105105
}
106106

107107
stage("stanc3 static") {
108-
agent { label 'gg-linux' }
108+
agent { label 'linux' }
109109
when {
110110
beforeAgent true
111111
allOf {
@@ -128,7 +128,7 @@ pipeline {
128128
}
129129

130130
stage("stanc3 debian") {
131-
agent { label 'gg-linux' }
131+
agent { label 'linux' }
132132
when {
133133
beforeAgent true
134134
allOf {
@@ -151,7 +151,7 @@ pipeline {
151151
}
152152

153153
stage("stanc3 debian-windows") {
154-
agent { label 'gg-linux' }
154+
agent { label 'linux' }
155155
when {
156156
beforeAgent true
157157
allOf {
@@ -177,7 +177,7 @@ pipeline {
177177
}
178178

179179
stage("update DockerHub main tags") {
180-
agent { label 'gg-linux' }
180+
agent { label 'linux' }
181181
when {
182182
beforeAgent true
183183
expression { params.replaceMainTags }

‎docker/ci/Dockerfile-gpu‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
FROM nvidia/cuda:11.2.2-devel-ubuntu20.04
2+
3+
RUN apt-get update && \
4+
DEBIAN_FRONTEND=noninteractive apt-get install -y \
5+
libcudnn8 \
6+
python3-appdirs \
7+
python3-mako \
8+
python3-numpy \
9+
python3-pytest \
10+
python3-pytest-cov \
11+
python3-pytools \
12+
python3-pip \
13+
python3-venv \
14+
python3-yaml \
15+
curl \
16+
&& \
17+
apt-get autoremove --purge -y && \
18+
apt-get autoclean -y && \
19+
rm -rf /var/cache/apt/* /var/lib/apt/lists/*
20+
21+
RUN pip3 install \
22+
scipy \
23+
configparser \
24+
torchvision \
25+
scikit-cuda \
26+
cupy \
27+
'tensorflow-gpu>=2.0.0a0' \
28+
scikit-learn

‎docker/stan2ftp/tensorflow/Dockerfile‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
FROM tensorflow/tensorflow
2+
3+
ARG PUID
4+
ARG PGID
5+
6+
USER root
7+
8+
RUN addgroup -gid ${PGID} jenkins
9+
RUN adduser --disabled-password --gecos '' --ingroup jenkins --uid ${PUID} jenkins
10+
11+
USER jenkins
12+
WORKDIR /home/jenkins
13+
14+
RUN pip3 install tfp-nightly==0.11.0.dev20200516 --user

‎docker/stanc3/debian-windows/Dockerfile‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,24 @@
11
#Pull the ubuntu:bionic image
22
FROM ubuntu:bionic
33

4-
# This image is for cross compiling for windows on debian
4+
USER root
5+
6+
ARG PUID
7+
ARG PGID
58

69
# Set the opam root
710
ENV OPAMROOT /usr/local/opam
811

9-
#Install OS dependencies
10-
RUN apt-get update -qq && apt-get install -y --no-install-recommends \
11-
curl ca-certificates rsync git build-essential m4 \
12-
unzip pkg-config libpcre3-dev mingw-w64 gcc wget gawk \
13-
vim nano sudo time
12+
RUN apt-get update -qq && apt-get install -y --no-install-recommends sudo curl ca-certificates rsync git build-essential m4 unzip pkg-config libpcre3-dev mingw-w64 gcc wget gawk vim nano time
13+
14+
RUN addgroup -gid ${PGID} jenkins
15+
RUN adduser --disabled-password --gecos '' --ingroup jenkins --uid ${PUID} jenkins
16+
RUN usermod -a -G sudo jenkins
17+
RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
18+
RUN chown -R jenkins:sudo /usr/local
19+
20+
USER jenkins
21+
WORKDIR /home/jenkins
1422

1523
#Copy our script and install ocaml + init
1624
COPY ./scripts/install_ocaml_windows.sh ./
@@ -21,12 +29,16 @@ COPY ./scripts/install_build_deps_windows.sh ./
2129
RUN bash -x install_build_deps_windows.sh
2230

2331
#Copy our script and install dev dependencies
24-
# COPY ./scripts/install_dev_deps.sh ./
25-
# RUN bash -x install_dev_deps.sh
32+
COPY ./scripts/install_dev_deps.sh ./
33+
RUN bash -x install_dev_deps.sh
2634

2735
# Install Javascript dev environment
2836
COPY ./scripts/install_js_deps.sh ./
2937
RUN opam update; bash -x install_js_deps.sh
3038

3139
#Specify our entrypoint
3240
ENTRYPOINT [ "opam", "config", "exec", "--" ]
41+
42+
# Build with:
43+
# cd stanc3
44+
# docker build -t stanorg/stanc3:debian-windowsfi --build-arg PUID=990 --progress=plain --build-arg PGID=986 -f ../ci-scripts/docker/stanc3/debian-windows/Dockerfile .

‎docker/stanc3/debian/Dockerfile‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,25 @@
11
# Pull the ubuntu:bionic base image
22
FROM ubuntu:bionic
33

4+
USER root
5+
6+
ARG PUID
7+
ARG PGID
8+
9+
# Set the opam root
10+
ENV OPAMROOT /usr/local/opam
11+
412
# Install OS dependencies
513
RUN apt-get update -qq && apt-get install -y --no-install-recommends \
614
curl ca-certificates \
715
rsync git build-essential m4 unzip pkg-config libpcre3-dev \
8-
python3 python3-pip nodejs
16+
python3 python3-pip nodejs sudo
17+
18+
RUN addgroup -gid ${PGID} jenkins
19+
RUN adduser --disabled-password --gecos '' --ingroup jenkins --uid ${PUID} jenkins
20+
21+
USER jenkins
22+
WORKDIR /home/jenkins
923

1024
# Install and initialize ocaml
1125
COPY ./scripts/install_ocaml.sh ./
@@ -27,3 +41,7 @@ RUN opam update; bash -x install_js_deps.sh
2741

2842
# Specify our entrypoint
2943
ENTRYPOINT [ "opam", "config", "exec", "--" ]
44+
45+
# Build with:
46+
# cd stanc3
47+
# docker build -t stanorg/stanc3:debianfi --build-arg PUID=990 --progress=plain --build-arg PGID=986 -f ../ci-scripts/docker/stanc3/debian/Dockerfile .

‎docker/stanc3/multiarch/README.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,5 +44,7 @@ Platforms: linux/amd64, linux/arm64, linux/riscv64, linux/ppc64le, linux/s390x,
4444
Call `buildx` with the list of target architectures and target repository (note this requires calling `docker login` first)
4545

4646
```
47-
docker buildx build -t stanorg/stanc3:multiarch --platform linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/mips64le,linux/s390x --push .
47+
docker buildx build -t stanorg/stanc3:multiarchfi --platform linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/mips64le,linux/s390x --build-arg PUID=990 --progress=plain --build-arg PGID=986 --push .
48+
docker buildx build -t stanorg/stanc3:multiarchfi --platform linux/mips64le --build-arg PUID=990 --progress=plain --no-cache --build-arg PGID=986 --push .
49+
docker buildx build -t stanorg/stanc3:multiarchfi --platform linux/arm/v6 --build-arg PUID=990 --progress=plain --build-arg PGID=986 --push .
4850
```

‎docker/stanc3/static/Dockerfile‎

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,31 +5,29 @@ FROM ocaml/opam:alpine-ocaml-4.12
55
#Switch to root user so we can install apk packages
66
USER root
77

8-
#Add opam group
8+
ARG PUID
9+
ARG PGID
910

10-
RUN addgroup -g 1004 jenkins-slave
11-
12-
RUN delgroup opam nogroup
13-
14-
RUN addgroup opam opam
15-
RUN addgroup opam jenkins-slave
11+
RUN addgroup -g ${PGID} jenkins && \
12+
adduser -D -u ${PUID} -G jenkins jenkins
1613

1714
#Set our distro_style
1815
LABEL distro_style="apk"
1916

2017
#Install os dependencies
21-
RUN apk update && apk add build-base bzip2 git tar curl ca-certificates openssl m4 bash
18+
RUN apk update && apk --no-cache add build-base bzip2 git tar curl ca-certificates openssl m4 bash docker jq git-subtree
2219

2320
# Add a recent version of the Skopeo package, which is used for looking up the correct multiarch docker image
24-
RUN curl https://dl-cdn.alpinelinux.org/alpine/latest-stable/community/x86_64/containers-common-0.38.11-r0.apk -o cont.apk && \
25-
curl https://dl-cdn.alpinelinux.org/alpine/latest-stable/community/x86_64/skopeo-1.3.1-r1.apk -o skopeo.apk && \
21+
# http://dl-cdn.alpinelinux.org/alpine/v3.14/community/aarch64/containers-common-0.38.11-r0.apk
22+
RUN curl https://dl-cdn.alpinelinux.org/alpine/latest-stable/community/x86_64/containers-common-0.44.3-r0.apk -o cont.apk && \
23+
curl https://dl-cdn.alpinelinux.org/alpine/latest-stable/community/x86_64/skopeo-1.5.2-r0.apk -o skopeo.apk && \
2624
apk add cont.apk && \
2725
apk add skopeo.apk
2826

2927
#Switch back to the normal user
30-
USER opam
28+
USER jenkins
3129

32-
#Init opam, create and switch to 4.07.0, update shell environment
30+
#Init opam, create and switch to 4.12.0, update shell environment
3331
RUN opam init --disable-sandboxing -y
3432
RUN opam switch create 4.12.0
3533
RUN opam switch 4.12.0
@@ -40,5 +38,11 @@ RUN opam repo add internet https://opam.ocaml.org
4038
COPY ./scripts/install_build_deps.sh ./
4139
RUN opam update; bash -x install_build_deps.sh
4240

41+
RUN opam install odoc -y
42+
4343
#Specify our entrypoint
4444
ENTRYPOINT [ "opam", "config", "exec", "--" ]
45+
46+
# Build with:
47+
# cd stanc3
48+
# docker build -t stanorg/stanc3:staticfi --build-arg PUID=990 --progress=plain --build-arg PGID=986 -f ../ci-scripts/docker/stanc3/static/Dockerfile .

0 commit comments

Comments
 (0)