diff --git a/arkouda-docker/arkouda-ibv-server b/arkouda-docker/arkouda-ibv-server new file mode 100644 index 0000000..8732934 --- /dev/null +++ b/arkouda-docker/arkouda-ibv-server @@ -0,0 +1,81 @@ +ARG CHAPEL_IBV_IMAGE +FROM $CHAPEL_IBV_IMAGE AS base + +# Download Arkouda +ARG ARKOUDA_DOWNLOAD_URL=${ARKOUDA_DOWNLOAD_URL} +ENV ARKOUDA_DOWNLOAD_URL=${ARKOUDA_DOWNLOAD_URL} +ARG ARKOUDA_DISTRO_NAME=${ARKOUDA_DISTRO_NAME} +ENV ARKOUDA_DISTRO_NAME=${ARKOUDA_DISTRO_NAME} +ARG ARKOUDA_BRANCH_NAME=${ARKOUDA_BRANCH_NAME} +ENV ARKOUDA_BRANCH_NAME=${ARKOUDA_BRANCH_NAME} +ARG MAKE_THREADS=1 + +# Install dependencies +RUN apt-get update && apt upgrade -y && apt-get install -y --no-install-recommends \ + unzip \ + libcurl4-openssl-dev \ + cmake \ + ssh \ + procps \ + dnsutils \ + openssh-client \ + openssh-server \ + sshpass + +FROM base AS build + +WORKDIR /opt + +# Download desired Arkouda distro, move to commont /opt/arkouda dir +RUN chmod 777 /opt && \ + wget $ARKOUDA_DOWNLOAD_URL && \ + unzip $ARKOUDA_DISTRO_NAME.zip && \ + mv /opt/arkouda-$ARKOUDA_BRANCH_NAME /opt/arkouda + +WORKDIR /opt/arkouda + +RUN python3 -m pip install --upgrade pip && \ + python3 -m pip install -e .[dev] + +# insecure to get around my ssl issues from zscaler +RUN echo insecure >> $HOME/.curlrc + +RUN make -j${MAKE_THREADS} install-deps + +ENV SSH_SERVERS=0.0.0.0 +ENV GASNET_SSH_SERVERS=0.0.0.0 +ENV GASNET_MASTERIP=0.0.0.0 +# required for gasnet/ibv build, as we don't have ibv devices at build time +ENV ARKOUDA_SKIP_CHECK_DEPS=true +# make arkouda server build faster +ENV ARKOUDA_QUICK_COMPILE=1 +ENV ARKOUDA_DEVELOPER= +RUN ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa && \ + cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys + +RUN service ssh start && make -j${MAKE_THREADS} + +# Remove unneeded files +RUN rm -rf /opt/$ARKOUDA_DISTRO_NAME.zip + +WORKDIR /opt/arkouda +RUN rm -rf benchmarks converter examples *.md pictures pydoc resources runs src test tests toys + +FROM base AS final + +COPY --from=build /opt /opt +WORKDIR /opt + +RUN apt-get update && apt upgrade -y && apt-get install -y --no-install-recommends \ + sudo \ + libcap2-bin + +RUN echo "* soft memlock unlimited" >> /etc/security/limits.conf && \ + echo "* hard memlock unlimited" >> /etc/security/limits.conf && \ + echo "root soft memlock unlimited" >> /etc/security/limits.conf && \ + echo "root hard memlock unlimited" >> /etc/security/limits.conf + +EXPOSE 5555 +EXPOSE 5000-65000/udp + +EXPOSE 22 diff --git a/arkouda-docker/arkouda-udp-server b/arkouda-docker/arkouda-udp-server index cfac2d9..430ef2b 100644 --- a/arkouda-docker/arkouda-udp-server +++ b/arkouda-docker/arkouda-udp-server @@ -1,5 +1,5 @@ ARG CHAPEL_UDP_IMAGE -FROM $CHAPEL_UDP_IMAGE +FROM $CHAPEL_UDP_IMAGE AS base # Download Arkouda ARG ARKOUDA_DOWNLOAD_URL=${ARKOUDA_DOWNLOAD_URL} @@ -8,15 +8,26 @@ ARG ARKOUDA_DISTRO_NAME=${ARKOUDA_DISTRO_NAME} ENV ARKOUDA_DISTRO_NAME=${ARKOUDA_DISTRO_NAME} ARG ARKOUDA_BRANCH_NAME=${ARKOUDA_BRANCH_NAME} ENV ARKOUDA_BRANCH_NAME=${ARKOUDA_BRANCH_NAME} +ARG MAKE_THREADS=1 # Install dependencies -RUN sudo apt-get update && sudo apt upgrade -y && \ - sudo apt-get install unzip libcurl4-openssl-dev cmake -y +RUN apt-get update && apt upgrade -y && apt-get install -y --no-install-recommends \ + unzip \ + libcurl4-openssl-dev \ + cmake \ + ssh \ + procps \ + dnsutils \ + openssh-client \ + openssh-server \ + sshpass + +FROM base AS build WORKDIR /opt # Download desired Arkouda distro, move to commont /opt/arkouda dir -RUN sudo chmod 777 /opt && \ +RUN chmod 777 /opt && \ wget $ARKOUDA_DOWNLOAD_URL && \ unzip $ARKOUDA_DISTRO_NAME.zip && \ mv /opt/arkouda-$ARKOUDA_BRANCH_NAME /opt/arkouda @@ -26,46 +37,39 @@ WORKDIR /opt/arkouda RUN python3 -m pip install --upgrade pip && \ python3 -m pip install -e .[dev] +# insecure to get around my ssl issues from zscaler +RUN echo insecure >> $HOME/.curlrc + +RUN make -j${MAKE_THREADS} install-deps + ENV SSH_SERVERS=0.0.0.0 +ENV GASNET_SSH_SERVERS=0.0.0.0 ENV GASNET_MASTERIP=0.0.0.0 - +# make arkouda server build faster +ENV ARKOUDA_SKIP_CHECK_DEPS=true +ENV ARKOUDA_QUICK_COMPILE=1 +ENV ARKOUDA_DEVELOPER= RUN ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa && \ cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys -RUN make install-deps && \ - sudo chown -R ubuntu:ubuntu /opt - -RUN sudo service ssh start && make +RUN service ssh start && make -j${MAKE_THREADS} -EXPOSE 5555 -EXPOSE 5000-65000/udp +# Remove unneeded files +RUN rm -rf /opt/$ARKOUDA_DISTRO_NAME.zip -ARG ARKOUDA_INTEGRATION_DOWNLOAD_URL=${ARKOUDA_INTEGRATION_DOWNLOAD_URL} -ENV ARKOUDA_INTEGRATION_DOWNLOAD_URL=${ARKOUDA_INTEGRATION_DOWNLOAD_URL} -ARG ARKOUDA_INTEGRATION_DISTRO_NAME=${ARKOUDA_INTEGRATION_DISTRO_NAME} -ENV ARKOUDA_INTEGRATION_DISTRO_NAME=${ARKOUDA_INTEGRATION_DISTRO_NAME} +WORKDIR /opt/arkouda +RUN rm -rf benchmarks converter examples *.md pictures pydoc resources runs src test tests toys -WORKDIR /opt +FROM base AS final +COPY --from=build /opt /opt -# Install arkouda_integration from arkouda-contrib repo -RUN wget $ARKOUDA_INTEGRATION_DOWNLOAD_URL && \ - unzip $ARKOUDA_INTEGRATION_DISTRO_NAME.zip && \ - mv /opt/arkouda-contrib-$ARKOUDA_INTEGRATION_DISTRO_NAME /opt/arkouda-contrib +RUN apt-get update && apt upgrade -y && apt-get install -y --no-install-recommends \ + sudo -WORKDIR /opt/arkouda-contrib/arkouda_integration/client -RUN python3 -m pip install -e . WORKDIR /opt +RUN chmod -R 777 /opt -# Remove unneeded files -RUN rm -rf ~/.ssh/ && \ - sudo rm -rf /opt/$ARKOUDA_DISTRO_NAME.zip && \ - sudo rm -rf /opt/$ARKOUDA_INTEGRATION_DISTRO_NAME.zip - -WORKDIR /opt/arkouda -RUN sudo rm -rf benchmarks converter examples *.md pictures pydoc resources runs src test tests toys - -WORKDIR /opt -ADD scripts/start-arkouda-server.sh /opt/arkouda/start-arkouda-server.sh -ADD scripts/start-arkouda-locale.sh /opt/arkouda/start-arkouda-locale.sh -ENTRYPOINT sh /opt/arkouda/start-arkouda-server.sh +EXPOSE 5555 +EXPOSE 5000-65000/udp +EXPOSE 22 diff --git a/arkouda-docker/build_docker_image.py b/arkouda-docker/build_docker_image.py index 4b3e007..1cc4c1d 100755 --- a/arkouda-docker/build_docker_image.py +++ b/arkouda-docker/build_docker_image.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import argparse, enum, subprocess, sys from typing import Optional from enum import Enum @@ -9,8 +11,9 @@ class ImageType(Enum): ARKOUDA_FULL_STACK = 'arkouda-full-stack' ARKOUDA_SMP_SERVER = 'arkouda-smp-server' ARKOUDA_UDP_SERVER = 'arkouda-udp-server' + ARKOUDA_IBV_SERVER = 'arkouda-ibv-server' CHAPEL_GASNET_UDP = 'chapel-gasnet-udp' - CHAPEL_GASNET_SMP = 'chapel-gasnet-smp' + CHAPEL_GASNET_IBV = 'chapel-gasnet-ibv' PROMETHEUS_ARKOUDA_EXPORTER = 'prometheus-arkouda-exporter' ARKOUDA_SMP_DEVELOPER = 'arkouda-smp-developer' @@ -34,7 +37,7 @@ def getDistro(tag: str) -> str: ''' return tag.lstrip('v') -def buildImage(dockerRepo: str, chapelVersion: str, file: str, distro: str, tag: Optional[str]) -> None: +def buildImage(dockerRepo: str, chapelVersion: str, file: str, distro: str, tag: Optional[str], multiarch: bool, concurrency: int, push: bool) -> None: ''' Generates a build tag and then builds the desired docker image @@ -43,96 +46,100 @@ def buildImage(dockerRepo: str, chapelVersion: str, file: str, distro: str, tag: :param str file: Dockerfile name :param str distro: Arkouda distro (branch name) :param Optional[str] tag: Arkouda tag name, if applicable + :param bool multiarch: whether to build a multiarch image + :param int concurrency: number of make jobs to run in parallel + :param bool push: whether to push the image to dockerhub after building :return: None ''' - def buildArkoudaFullStack(dockerRepo: str, chapelVersion: str, file: str, dockerTag: str, distro: str, tag: Optional[str]) -> None: - result = subprocess.run(args=['docker','build', - '--build-arg', f'CHAPEL_SMP_IMAGE={generateChplSmpVersion(chapelVersion)}', - '--build-arg', f'ARKOUDA_DISTRO_NAME={getDistroName(distro=distro, tag=tag)}', - '--build-arg', f'ARKOUDA_DOWNLOAD_URL={generateArkoudaDownloadUrl(tag=tag,branch=distro)}', - '--build-arg', f'ARKOUDA_BRANCH_NAME={distro}', - '-f',file, - '-t', dockerTag, '.'], stdout=subprocess.DEVNULL) - print(result) - - def buildArkoudaSmpServer(dockerRepo: str, chapelVersion: str, file: str, dockerTag: str, distro: str, tag: Optional[str]) -> None: - result = subprocess.run(args=['docker','build', - '--build-arg', f'CHAPEL_SMP_IMAGE={generateChplSmpVersion(chapelVersion)}', - '--build-arg', f'ARKOUDA_DISTRO_NAME={getDistroName(distro=distro, tag=tag)}', - '--build-arg', f'ARKOUDA_DOWNLOAD_URL={generateArkoudaDownloadUrl(tag=tag,branch=distro)}', - '--build-arg', f'ARKOUDA_BRANCH_NAME={distro}', - '-f',file, - '-t', dockerTag, '.'], stdout=subprocess.DEVNULL) - print(result) - - def buildArkoudaUdpServer(dockerRepo: str, chapelVersion: str, file: str, dockerTag: str, distro: str, tag: Optional[str]) -> None: - result = subprocess.run(args=['docker','build', - '--build-arg', f'CHAPEL_UDP_IMAGE={generateChplUdpVersion(chapelVersion)}', - '--build-arg', f'ARKOUDA_DISTRO_NAME={getDistroName(distro=distro, tag=tag)}', - '--build-arg', f'ARKOUDA_DOWNLOAD_URL={generateArkoudaDownloadUrl(tag=tag,branch=distro)}', - '--build-arg', f'ARKOUDA_BRANCH_NAME={distro}', - '--build-arg', 'ARKOUDA_INTEGRATION_DOWNLOAD_URL=https://github.com/Bears-R-Us/arkouda-contrib/archive/refs/heads/main.zip', - '--build-arg', 'ARKOUDA_INTEGRATION_DISTRO_NAME=main', - '-f',file, - '-t', dockerTag, '.'], stdout=subprocess.DEVNULL) - print(result) - - def buildChapelUdp(dockerRepo: str, chapelVersion: str, file: str, dockerTag: str) -> None: - result = subprocess.run(args=['docker','build', - '--build-arg', f'CHPL_BASE_IMAGE=ubuntu:22.04', - '--build-arg', f'CHPL_VERSION={chapelVersion}', - '--build-arg', f'CHPL_UDP_IMAGE_REPO={dockerRepo}', - '-f',file, - '-t', f'{dockerRepo}/{file}:{chapelVersion}', '.'], stdout=subprocess.DEVNULL) - print(result) - - def buildChapelSmp(dockerRepo: str, chapelVersion: str, file: str, dockerTag: str) -> None: - result = subprocess.run(args=['docker','build', - '--build-arg', f'CHPL_BASE_IMAGE=ubuntu:22.04', - '--build-arg', f'CHPL_VERSION={chapelVersion}', - '--build-arg', f'CHPL_SMP_IMAGE_REPO={dockerRepo}', - '-f',file, - '-t', f'{dockerRepo}/{file}:{chapelVersion}', '.'], stdout=subprocess.DEVNULL) - print(result) - def buildPrometheusArkoudaExporter(dockerRepo: str, file: str, dockerTag: str, distro: str, tag: Optional[str]) -> None: - result = subprocess.run(args=['docker', 'build' , - '--build-arg', f'ARKOUDA_DISTRO_NAME={getDistroName(distro=distro, tag=tag)}', - '--build-arg', f'ARKOUDA_DOWNLOAD_URL={generateArkoudaDownloadUrl(tag=tag,branch=distro)}', - '--build-arg', f'ARKOUDA_BRANCH_NAME={distro}', - '--build-arg', 'ARKOUDA_CONTRIB_DOWNLOAD_URL=https://github.com/Bears-R-Us/arkouda-contrib/archive/refs/heads/main.zip', - '--build-arg', 'ARKOUDA_CONTRIB_DISTRO_NAME=main', - '-f', file, - '-t', f'{dockerTag}', '.'], stdout=subprocess.DEVNULL) - print(result) - def buildArkoudaSmpDeveloper(dockerRepo: str, chapelVersion: str, file: str, dockerTag: str, distro: str, tag: Optional[str]) -> None: - result = subprocess.run(args=['docker','build', - '--build-arg', f'CHAPEL_SMP_IMAGE={generateChplSmpVersion(chapelVersion)}', - '--build-arg', f'ARKOUDA_DISTRO_NAME={getDistroName(distro=distro, tag=tag)}', - '--build-arg', f'ARKOUDA_DOWNLOAD_URL={generateArkoudaDownloadUrl(tag=tag,branch=distro)}', - '--build-arg', f'ARKOUDA_BRANCH_NAME={distro}', - '-f',file, - '-t', dockerTag, '.'], stdout=subprocess.DEVNULL) + def buildImageHelper(build_args: dict) -> None: + use_tag = tag + if not tag and build_args['CHPL_VERSION']: + use_tag = build_args['CHPL_VERSION'] + docker_tag = generateBuildTag(dockerRepo=dockerRepo,file=file,tag=use_tag,distro=distro) + print(f'Building docker image {docker_tag} using Dockerfile {file}') + args = ['docker', 'build'] + for key, value in build_args.items(): + args.append('--build-arg') + args.append(f'{key}={value}') + args.extend(['--build-arg', f'MAKE_THREADS={concurrency}']) + if multiarch: + args.extend(['--platform', 'linux/amd64,linux/arm64']) + if push: + args.extend(['--push']) + args.extend(['-f', file, '-t', docker_tag]) + args.append('.') + print("Running docker build command: ", ' '.join(args)) + result = subprocess.run(args, stdout=subprocess.DEVNULL) print(result) - dockerTag = generateBuildTag(dockerRepo=dockerRepo, file=file, tag=tag,distro=distro) - if file == ImageType.ARKOUDA_FULL_STACK.value: - buildArkoudaFullStack(dockerRepo=dockerRepo,chapelVersion=chapelVersion,file=file,dockerTag=dockerTag,distro=distro,tag=tag) + buildImageHelper(build_args={ + 'CHAPEL_SMP_IMAGE': generateChplSmpVersion(dockerRepo, chapelVersion), + 'ARKOUDA_DISTRO_NAME': getDistroName(distro=distro, tag=tag), + 'ARKOUDA_DOWNLOAD_URL': generateArkoudaDownloadUrl(tag=tag, branch=distro), + 'ARKOUDA_BRANCH_NAME': distro, + } + ) elif file == ImageType.ARKOUDA_SMP_SERVER.value: - buildArkoudaSmpServer(dockerRepo=dockerRepo,chapelVersion=chapelVersion,file=file,dockerTag=dockerTag,distro=distro,tag=tag) + buildImageHelper(build_args={ + 'CHAPEL_SMP_IMAGE': generateChplSmpVersion(dockerRepo, chapelVersion), + 'ARKOUDA_DISTRO_NAME': getDistroName(distro=distro, tag=tag), + 'ARKOUDA_DOWNLOAD_URL': generateArkoudaDownloadUrl(tag=tag, branch=distro), + 'ARKOUDA_BRANCH_NAME': distro, + } + ) elif file == ImageType.ARKOUDA_UDP_SERVER.value: - buildArkoudaUdpServer(dockerRepo=dockerRepo,chapelVersion=chapelVersion,file=file,dockerTag=dockerTag,distro=distro,tag=tag) + buildImageHelper(build_args={ + 'CHAPEL_UDP_IMAGE': generateChplUdpVersion(dockerRepo, chapelVersion), + 'ARKOUDA_DISTRO_NAME': getDistroName(distro=distro, tag=tag), + 'ARKOUDA_DOWNLOAD_URL': generateArkoudaDownloadUrl(tag=tag, branch=distro), + 'ARKOUDA_BRANCH_NAME': distro, + 'ARKOUDA_INTEGRATION_DOWNLOAD_URL': 'https://github.com/Bears-R-Us/arkouda-contrib/archive/refs/heads/main.zip', + 'ARKOUDA_INTEGRATION_DISTRO_NAME': 'main', + } + ) + elif file == ImageType.ARKOUDA_IBV_SERVER.value: + buildImageHelper(build_args={ + 'CHAPEL_IBV_IMAGE': generateChplIbvVersion(dockerRepo, chapelVersion), + 'ARKOUDA_DISTRO_NAME': getDistroName(distro=distro, tag=tag), + 'ARKOUDA_DOWNLOAD_URL': generateArkoudaDownloadUrl(tag=tag, branch=distro), + 'ARKOUDA_BRANCH_NAME': distro, + 'ARKOUDA_INTEGRATION_DOWNLOAD_URL': 'https://github.com/Bears-R-Us/arkouda-contrib/archive/refs/heads/main.zip', + 'ARKOUDA_INTEGRATION_DISTRO_NAME': 'main', + } + ) elif file == ImageType.CHAPEL_GASNET_UDP.value: - buildChapelUdp(dockerRepo=dockerRepo,chapelVersion=chapelVersion,file=file,dockerTag=dockerTag) - elif file == ImageType.CHAPEL_GASNET_SMP.value: - buildChapelSmp(dockerRepo=dockerRepo,chapelVersion=chapelVersion,file=file,dockerTag=dockerTag) + buildImageHelper(build_args={ + 'CHPL_VERSION': chapelVersion, + 'CHPL_UDP_IMAGE_REPO': dockerRepo, + } + ) + elif file == ImageType.CHAPEL_GASNET_IBV.value: + buildImageHelper(build_args={ + 'CHPL_VERSION': chapelVersion, + 'CHPL_IBV_IMAGE_REPO': dockerRepo, + } + ) elif file == ImageType.PROMETHEUS_ARKOUDA_EXPORTER.value: - buildPrometheusArkoudaExporter(dockerRepo=dockerRepo,file=file,distro=distro,dockerTag=dockerTag,tag=tag) + buildImageHelper(build_args={ + 'ARKOUDA_DISTRO_NAME': getDistroName(distro=distro, tag=tag), + 'ARKOUDA_DOWNLOAD_URL': generateArkoudaDownloadUrl(tag=tag, branch=distro), + 'ARKOUDA_BRANCH_NAME': distro, + 'ARKOUDA_CONTRIB_DOWNLOAD_URL': 'https://github.com/Bears-R-Us/arkouda-contrib/archive/refs/heads/main.zip', + 'ARKOUDA_CONTRIB_DISTRO_NAME': 'main', + } + ) elif file == ImageType.ARKOUDA_SMP_DEVELOPER.value: - buildArkoudaSmpDeveloper(dockerRepo=dockerRepo,chapelVersion=chapelVersion,file=file,dockerTag=dockerTag,distro=distro,tag=tag) + buildImageHelper(build_args={ + 'CHAPEL_SMP_IMAGE': generateChplSmpVersion(dockerRepo, chapelVersion), + 'ARKOUDA_DISTRO_NAME': getDistroName(distro=distro, tag=tag), + 'ARKOUDA_DOWNLOAD_URL': generateArkoudaDownloadUrl(tag=tag, branch=distro), + 'ARKOUDA_BRANCH_NAME': distro, + } + ) else: raise ValueError(f'Dockerfile {file} is invalid, check command-line args') @@ -148,11 +155,14 @@ def getDistroName(distro: str, tag: Optional[str]) -> str: ''' return tag if tag else distro -def generateChplSmpVersion(chapelVersion: str) -> str: - return f'bearsrus/chapel-gasnet-smp:{chapelVersion}' +def generateChplSmpVersion(dockerRepo: str, chapelVersion: str) -> str: + return f'{dockerRepo}/chapel-gasnet-smp:{chapelVersion}' + +def generateChplUdpVersion(dockerRepo: str, chapelVersion: str) -> str: + return f'{dockerRepo}/chapel-gasnet-udp:{chapelVersion}' -def generateChplUdpVersion(chapelVersion: str) -> str: - return f'bearsrus/chapel-gasnet-udp:{chapelVersion}' +def generateChplIbvVersion(dockerRepo: str, chapelVersion: str) -> str: + return f'{dockerRepo}/chapel-gasnet-ibv:{chapelVersion}' def generateArkoudaDownloadUrl(tag: Optional[str], branch: Optional[str]) -> str: ''' @@ -181,10 +191,12 @@ def generateBuildTag(dockerRepo: str, file: str, tag: Optional[str], distro: Opt :param Optional[str] tag: Arkouda tag name, if applicable :param Optional[str] distro: Arkouda distro (branch name), if applicable - :return: docker build tag + :return: docker build tag :rtype: str ''' - return f'{dockerRepo}/{file}:{tag}' if tag else f'{dockerRepo}/{file}:{distro}' + base = f'{dockerRepo}/{file}' + suffix = tag if tag else distro if distro else 'latest' + return f'{base}:{suffix}' def buildArkoudaImage(dockerFile: str) -> bool: ''' @@ -199,8 +211,8 @@ def buildArkoudaImage(dockerFile: str) -> bool: if __name__=="__main__": parser = argparse.ArgumentParser(description='Build bearsrus docker images') - parser.add_argument('--image_type', type=ImageType, - help='possible image types are arkouda_full_stack, arkouda-smp-server, arkouda-udp-server and chapel_udp') + parser.add_argument('--image_type', type=ImageType, required=True, + help=f'possible image types are: {", ".join([imageType.value for imageType in ImageType])}') parser.add_argument('--arkouda_tag', type=str, help='if the desired arkouda version is a tag') parser.add_argument('--arkouda_branch', type=str, @@ -211,6 +223,12 @@ def buildArkoudaImage(dockerFile: str) -> bool: help='the arkouda repo containing the arkouda source code, defaults to Bears-R-Us') parser.add_argument('--chapel_version', type=str, help='Version of Chapel used to build image') + parser.add_argument('--multiarch', action='store_true', + help='Build multiarch image (default native arch only)') + parser.add_argument('--concurrency', type=int, default=1, + help='Number of make jobs to run in parallel (default 1)') + parser.add_argument('--push', action='store_true', + help='Push the image to dockerhub after building (default False)') args = parser.parse_args() @@ -220,6 +238,9 @@ def buildArkoudaImage(dockerFile: str) -> bool: arkoudaRepo = args.arkouda_repo chapelVersion = args.chapel_version distro = None + multiarch = args.multiarch + concurrency = args.concurrency + push = args.push if buildArkoudaImage(file): if tag: @@ -229,5 +250,5 @@ def buildArkoudaImage(dockerFile: str) -> bool: distro = args.arkouda_branch else: raise ValueError('Either --arkouda_tag or --arkouda_branch must be specified') - - buildImage(dockerRepo=dockerRepo,chapelVersion=chapelVersion,file=file,tag=tag,distro=distro) + + buildImage(dockerRepo=dockerRepo,chapelVersion=chapelVersion,file=file,tag=tag,distro=distro,multiarch=multiarch,concurrency=concurrency,push=push) diff --git a/arkouda-docker/chapel-gasnet-ibv b/arkouda-docker/chapel-gasnet-ibv new file mode 100644 index 0000000..b43295c --- /dev/null +++ b/arkouda-docker/chapel-gasnet-ibv @@ -0,0 +1,31 @@ +ARG CHPL_VERSION +FROM chapel/chapel:${CHPL_VERSION} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + rdma-core \ + libibmad5 \ + opensm \ + ibutils \ + infiniband-diags \ + perftest \ + mstflint \ + libibverbs1 \ + libibverbs-dev \ + librdmacm1 \ + libibumad3 \ + ibverbs-providers \ + rdmacm-utils \ + infiniband-diags \ + libfabric1 \ + ibverbs-utils + +ENV CHPL_COMM=gasnet +ENV CHPL_COMM_SUBSTRATE=ibv +ENV CHPL_LAUNCHER=gasnetrun_ibv +ENV GASNET_BACKTRACE=1 + +ARG MAKE_THREADS +RUN CHPL_TARGET_COMPILER=llvm make -j${MAKE_THREADS} -C $CHPL_HOME \ + && CHPL_TARGET_COMPILER=gnu make -j${MAKE_THREADS} -C $CHPL_HOME \ + && make -j${MAKE_THREADS} -C $CHPL_HOME chpldoc test-venv mason \ + && make -j${MAKE_THREADS} -C $CHPL_HOME cleanall diff --git a/arkouda-docker/chapel-gasnet-smp b/arkouda-docker/chapel-gasnet-smp deleted file mode 100644 index 79b6748..0000000 --- a/arkouda-docker/chapel-gasnet-smp +++ /dev/null @@ -1,78 +0,0 @@ -FROM ubuntu:22.04 - -ARG CHPL_VERSION=${CHPL_VERSION} -ENV CHPL_VERSION=${CHPL_VERSION} - -ARG DEBIAN_FRONTEND=noninteractive - -ENV CHPL_GASNET_MORE_CFG_OPTIONS "--disable-pshm-posix --enable-pshm-sysv" - -RUN apt-get update && apt-get install -y --no-install-recommends \ - wget \ - ca-certificates \ - cmake \ - curl \ - gcc \ - g++ \ - perl \ - python2.7 \ - python2.7-dev \ - python-setuptools \ - python3 \ - python3-pip \ - libgmp10 \ - libgmp-dev \ - locales \ - bash \ - make \ - mawk \ - file \ - pkg-config \ - git \ - openssh-client \ - openssh-server \ - sshpass \ - sudo \ - procps \ - dnsutils \ - vim \ - hdf5-tools \ - libcurl4-openssl-dev \ - clang \ - lldb \ - lld \ - llvm-dev \ - libclang-14-dev \ - libclang-cpp-dev \ - && rm -rf /var/lib/apt/lists/* - -RUN useradd -m ubuntu && echo "ubuntu:ubuntu" | chpasswd && adduser ubuntu sudo -RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -EXPOSE 22 - -ENV CHPL_HOME=/opt/chapel/$CHPL_VERSION - -ENV CHPL_TARGET_PLATFORM=linux64 -ENV CHPL_COMM_SUBSTRATE=smp -ENV CHPL_COMM=gasnet -ENV CHPL_LAUNCHER=smp -ENV CHPL_LLVM=system -ENV CHPL_GMP=system -ENV CHPL_RE2=bundled -RUN chmod 777 /opt - -USER ubuntu -RUN mkdir /home/ubuntu/.ssh - -RUN mkdir -p /opt/chapel \ - && wget -q -O - https://github.com/chapel-lang/chapel/releases/download/$CHPL_VERSION/chapel-$CHPL_VERSION.tar.gz | tar -xzC /opt/chapel --transform 's/chapel-//' \ - && make -C $CHPL_HOME \ - && make -C $CHPL_HOME cleanall - -USER root -RUN chmod -R 777 /opt/chapel/$CHPL_VERSION/bin - -USER ubuntu -ENV PATH=/opt/chapel/$CHPL_VERSION/bin/linux64-x86_64/:$PATH - diff --git a/arkouda-docker/chapel-gasnet-udp b/arkouda-docker/chapel-gasnet-udp index 17cfebd..95b73f1 100644 --- a/arkouda-docker/chapel-gasnet-udp +++ b/arkouda-docker/chapel-gasnet-udp @@ -1,76 +1,12 @@ -ARG CHPL_BASE_IMAGE -FROM $CHPL_BASE_IMAGE +ARG CHPL_VERSION +FROM chapel/chapel:${CHPL_VERSION} -ARG DEBIAN_FRONTEND=noninteractive - -RUN apt-get update && apt-get install -y --no-install-recommends \ - wget \ - ca-certificates \ - cmake \ - curl \ - gcc \ - g++ \ - perl \ - python2.7 \ - python2.7-dev \ - python-setuptools \ - python3 \ - python3-pip \ - libgmp10 \ - libgmp-dev \ - locales \ - bash \ - make \ - mawk \ - file \ - pkg-config \ - git \ - openssh-client \ - openssh-server \ - sshpass \ - sudo \ - procps \ - dnsutils \ - vim \ - hdf5-tools \ - libcurl4-openssl-dev \ - clang \ - lldb \ - lld \ - llvm-dev \ - libclang-14-dev \ - libclang-cpp-dev \ - && rm -rf /var/lib/apt/lists/* - -RUN useradd -m ubuntu && echo "ubuntu:ubuntu" | chpasswd && adduser ubuntu sudo -RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -EXPOSE 22 - -ARG CHPL_VERSION=${CHPL_VERSION} -ENV CHPL_VERSION=${CHPL_VERSION} -ENV CHPL_HOME=/opt/chapel/$CHPL_VERSION - -ENV CHPL_COMM_SUBSTRATE=udp ENV CHPL_COMM=gasnet +ENV CHPL_COMM_SUBSTRATE=udp ENV CHPL_LAUNCHER=amudprun -ENV CHPL_LLVM=system -ENV CHPL_RE2=bundled -ENV CHPL_GMP=system -RUN chmod 777 /opt - -USER ubuntu -RUN mkdir /home/ubuntu/.ssh - -RUN mkdir -p /opt/chapel \ - && wget -q -O - https://github.com/chapel-lang/chapel/releases/download/$CHPL_VERSION/chapel-$CHPL_VERSION.tar.gz | tar -xzC /opt/chapel --transform 's/chapel-//' \ - && make -C $CHPL_HOME \ - #&& make -C $CHPL_HOME chpldoc test-venv mason \ - && make -C $CHPL_HOME cleanall - -USER root -RUN chmod -R 777 /opt/chapel/$CHPL_VERSION/bin - -USER ubuntu -ENV PATH=/opt/chapel/$CHPL_VERSION/bin/linux64-x86_64/:$PATH +ARG MAKE_THREADS +RUN CHPL_TARGET_COMPILER=llvm make -j${MAKE_THREADS} -C $CHPL_HOME \ + && CHPL_TARGET_COMPILER=gnu make -j${MAKE_THREADS} -C $CHPL_HOME \ + && make -j${MAKE_THREADS} -C $CHPL_HOME chpldoc test-venv mason \ + && make -j${MAKE_THREADS} -C $CHPL_HOME cleanall diff --git a/arkouda-docker/hello.chpl b/arkouda-docker/hello.chpl new file mode 100644 index 0000000..267456f --- /dev/null +++ b/arkouda-docker/hello.chpl @@ -0,0 +1 @@ +writeln('hi'); diff --git a/arkouda-helm-charts/arkouda-udp-locale/templates/arkouda-locale-headless-service.yaml b/arkouda-helm-charts/arkouda-udp-locale/templates/arkouda-locale-headless-service.yaml deleted file mode 100644 index 00bc4e4..0000000 --- a/arkouda-helm-charts/arkouda-udp-locale/templates/arkouda-locale-headless-service.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.locale.name }}-locale-headless - namespace: arkouda -spec: - clusterIP: None - selector: - app: {{ .Values.locale.name }} diff --git a/arkouda-helm-charts/arkouda-udp-locale/templates/arkouda-locale-launch-script-config-map.yaml b/arkouda-helm-charts/arkouda-udp-locale/templates/arkouda-locale-launch-script-config-map.yaml index cb34d9c..b835e8d 100644 --- a/arkouda-helm-charts/arkouda-udp-locale/templates/arkouda-locale-launch-script-config-map.yaml +++ b/arkouda-helm-charts/arkouda-udp-locale/templates/arkouda-locale-launch-script-config-map.yaml @@ -11,34 +11,34 @@ data: sudo service ssh start {{- if .Values.group.enabled }} - # Add group corresponding to gid and user - sudo groupadd -g {{ .Values.group.gid | int64 | quote }} {{ .Values.group.name }} - sudo groupadd {{ .Values.user.name }} + GID={{ .Values.group.gid | int64 | quote }} + GROUP={{ .Values.group.name }} + {{- else }} + GID=1000 + GROUP=ubuntu {{- end }} {{- if .Values.user.enabled }} - # Add user with home directory - sudo useradd {{ .Values.user.name }} -m -u {{ .Values.user.uid | int64 | quote }} -g {{ .Values.group.gid | int64 | quote }} - - # Add user to user group - sudo adduser {{ .Values.user.name }} {{ .Values.user.name }} - - export USER={{ .Values.user.name }} + UID={{ .Values.user.uid | int64 | quote }} + USER={{ .Values.user.name }} {{- else }} - export USER=ubuntu + UID=1000 + USER=ubuntu {{- end }} - # Add SSH certificate - sudo -u $USER mkdir /home/$USER/.ssh - sudo cp /home/ubuntu/ssh-keys/id_rsa* /home/$USER/.ssh/ - sudo chown -R $USER:$USER /home/$USER/.ssh + sudo groupadd -g $GID $GROUP + sudo useradd $USER -m -u $UID -g $GID + sudo adduser $USER $USER + HOME=$(eval echo ~$USER) + sudo groupadd $USER + echo $USER - # Start arkouda_server as the configured user - sudo -u $USER bash < ~/.ssh/authorized_keys \ + chmod -R 600 $HOME/.ssh/* + cat $HOME/.ssh/id_rsa.pub > $HOME/.ssh/authorized_keys /bin/bash -c -- "while true; do sleep 600; done;" - diff --git a/arkouda-helm-charts/arkouda-udp-locale/templates/arkouda-locale-ssh-service.yaml b/arkouda-helm-charts/arkouda-udp-locale/templates/arkouda-locale-ssh-service.yaml index 8348e9b..c50d673 100644 --- a/arkouda-helm-charts/arkouda-udp-locale/templates/arkouda-locale-ssh-service.yaml +++ b/arkouda-helm-charts/arkouda-udp-locale/templates/arkouda-locale-ssh-service.yaml @@ -2,11 +2,15 @@ apiVersion: v1 kind: Service metadata: name: {{ .Values.locale.name}}-locale-ssh + labels: + app: {{ .Values.locale.name }}-locale-ssh spec: type: ClusterIP + clusterIP: None ports: - port: 22 targetPort: 22 protocol: TCP + name: ssh selector: app: {{ .Values.locale.name }} diff --git a/arkouda-helm-charts/arkouda-udp-locale/templates/deployment.yaml b/arkouda-helm-charts/arkouda-udp-locale/templates/deployment.yaml index 7ad0cc5..1f9f96f 100644 --- a/arkouda-helm-charts/arkouda-udp-locale/templates/deployment.yaml +++ b/arkouda-helm-charts/arkouda-udp-locale/templates/deployment.yaml @@ -1,5 +1,5 @@ apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: {{ .Values.locale.name }}-locale spec: @@ -13,9 +13,10 @@ spec: app: {{ .Values.locale.name }} name: {{ .Values.locale.name }} spec: + terminationGracePeriodSeconds: 5 containers: - name: arkouda-locale - image: bearsrus/arkouda-udp-server:{{ .Values.releaseVersion }} + image: {{ .Values.imageRepo }}/arkouda-udp-server:{{ .Values.releaseVersion }} imagePullPolicy: {{ .Values.imagePullPolicy }} resources: requests: @@ -27,6 +28,7 @@ spec: command: [ "sh", "/opt/arkouda/start-arkouda-locale.sh" ] ports: - containerPort: 22 + name: ssh volumeMounts: - name: ssh mountPath: "/home/ubuntu/ssh-keys" @@ -40,7 +42,7 @@ spec: env: - name: MY_IP valueFrom: - fieldRef: + fieldRef: fieldPath: status.podIP - name: NUM_LOCALES value: {{ .Values.locale.numLocales | quote }} diff --git a/arkouda-helm-charts/arkouda-udp-locale/values.yaml b/arkouda-helm-charts/arkouda-udp-locale/values.yaml index 637e428..87024f2 100644 --- a/arkouda-helm-charts/arkouda-udp-locale/values.yaml +++ b/arkouda-helm-charts/arkouda-udp-locale/values.yaml @@ -2,7 +2,8 @@ ######################## Pod Settings ######################## -releaseVersion: # bearsrus arkouda-udp-server image version +releaseVersion: v2024.04.19 +imageRepo: arifthpe imagePullPolicy: Always resources: @@ -15,10 +16,10 @@ resources: ################ Arkouda Locale Configuration ################ -locale: - numLocales: # number of arkouda-udp-locale pods - name: # Kubernetes app name used by arkouda-udp-server for pod discovery - threadsPerLocale: # number of cpu cores assigned to each Arkouda locale +locale: + numLocales: 2 # number of arkouda-udp-locale pods + name: asdfasdf # Kubernetes app name used by arkouda-udp-server for pod discovery + threadsPerLocale: 2 # number of cpu cores assigned to each Arkouda locale persistence: enabled: # indicates if persistence of files to host is enabled, defaults to false @@ -36,4 +37,4 @@ group: gid: # gid of group Arkouda user needs to be a member of to execute host commands secrets: - ssh: # name of ssh secret used to launch Arkouda locales + ssh: sshkey diff --git a/arkouda-helm-charts/arkouda-udp-server/serviceaccount-token.yaml b/arkouda-helm-charts/arkouda-udp-server/serviceaccount-token.yaml new file mode 100644 index 0000000..5aa090e --- /dev/null +++ b/arkouda-helm-charts/arkouda-udp-server/serviceaccount-token.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: arkouda-sa + annotations: + kubernetes.io/service-account.name: arkouda-sa # matches the ServiceAccount name defined in previous step +type: kubernetes.io/service-account-token diff --git a/arkouda-helm-charts/arkouda-udp-server/serviceaccount.yaml b/arkouda-helm-charts/arkouda-udp-server/serviceaccount.yaml new file mode 100644 index 0000000..da2bedc --- /dev/null +++ b/arkouda-helm-charts/arkouda-udp-server/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: arkouda-sa +automountServiceAccountToken: false diff --git a/arkouda-helm-charts/arkouda-udp-server/templates/arkouda-server-headless-service.yaml b/arkouda-helm-charts/arkouda-udp-server/templates/arkouda-server-headless-service.yaml index f85c865..dbd42f3 100644 --- a/arkouda-helm-charts/arkouda-udp-server/templates/arkouda-server-headless-service.yaml +++ b/arkouda-helm-charts/arkouda-udp-server/templates/arkouda-server-headless-service.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: name: {{ .Values.server.name }}-server-headless-service - namespace: arkouda + namespace: {{ .Values.external.namespace }} spec: clusterIP: None selector: diff --git a/arkouda-helm-charts/arkouda-udp-server/templates/arkouda-server-launch-script-config-map.yaml b/arkouda-helm-charts/arkouda-udp-server/templates/arkouda-server-launch-script-config-map.yaml index edb70bf..2fdea68 100644 --- a/arkouda-helm-charts/arkouda-udp-server/templates/arkouda-server-launch-script-config-map.yaml +++ b/arkouda-helm-charts/arkouda-udp-server/templates/arkouda-server-launch-script-config-map.yaml @@ -12,69 +12,61 @@ data: #!/bin/bash sudo service ssh start - # Set SSL parameters - export SSL_TOKEN=$(cat sa/token) - export CACERT_FILE=/opt/sa/ca.crt + set -x - # Set env variables need to get arkouda-locale pod IP addresses and start arkouda - sudo sh -c "echo 'K8S_HOST=$K8S_HOST' >> /etc/environment" - sudo sh -c "echo 'SSL_TOKEN=$SSL_TOKEN' >> /etc/environment" - sudo sh -c "echo 'CACERT_FILE=$CACERT_FILE' >> /etc/environment" + # Set env variables needed to start arkouda sudo sh -c "echo 'GASNET_MASTERIP=$MY_IP' >> /etc/environment" - sudo sh -c "echo 'NAMESPACE=$NAMESPACE' >> /etc/environment" - sudo sh -c "echo 'EXTERNAL_SERVICE_NAME=$EXTERNAL_SERVICE_NAME' >> /etc/environment" sudo sh -c "echo 'METRICS_SERVICE_NAME=$METRICS_SERVICE_NAME' >> /etc/environment" # Retrieve arkouda-locale pod IP addresses and set the corresponding env variable - LOCALE_IPS=$(python3 /opt/arkouda-contrib/arkouda_integration/client/scripts/pods.py \ - '-c=$CACERT_FILE' '-k=$TOKEN' '-kh=$K8S_HOST' '-i=GET_POD_IPS' '-n=$NAMESPACE' '-a=$APP_NAME') + LOCALE_IPS="$(dig +search +short {{ .Values.locale.name }}-locale-ssh | tr '\n' ' ')" sudo sh -c "echo 'LOCALE_IPS=$LOCALE_IPS' >> /etc/environment" {{- if .Values.group.enabled }} - # Add group corresponding to gid and user - sudo groupadd -g {{ .Values.group.gid | int64 | quote }} {{ .Values.group.name }} - sudo groupadd {{ .Values.user.name }} + GID={{ .Values.group.gid | int64 | quote }} + GROUP={{ .Values.group.name }} + {{- else }} + GID=1000 + GROUP=ubuntu {{- end }} - {{- if .Values.user.enabled }} - # Add user with home directory - sudo useradd {{ .Values.user.name }} -m -u {{ .Values.user.uid | int64 | quote }} -g {{ .Values.group.gid | int64 | quote }} - - # Add user to user group - sudo adduser {{ .Values.user.name }} {{ .Values.user.name }} - - export USER={{ .Values.user.name }} + {{- if .Values.user.enabled }} + UID={{ .Values.user.uid | int64 | quote }} + USER={{ .Values.user.name }} {{- else }} - export USER=ubuntu + UID=1000 + USER=ubuntu {{- end }} + + sudo groupadd -g $GID $GROUP + sudo useradd $USER -m -u $UID -g $GID + sudo adduser $USER $USER + HOME=$(eval echo ~$USER) + sudo groupadd $USER echo $USER - # Add SSH certificate - sudo -u $USER mkdir /home/$USER/.ssh - sudo cp /home/ubuntu/ssh-keys/id_rsa* /home/$USER/.ssh/ - sudo chown -R $USER:$USER /home/$USER/.ssh - sudo ls -ltr /home/$USER/.ssh - # Start arkouda_server as the configured user - sudo -u $USER bash < $HOME/.ssh/authorized_keys + + # Generate list of arkouda-server and arkouda-locale IP addresses + export SSH_SERVERS="$MY_IP $LOCALE_IPS" + export GASNET_SSH_SERVERS="$SSH_SERVERS" - # Add public cert to authorized keys to enable GASNET SSH spawner - cat /home/$USER/.ssh/id_rsa.pub > /home/$USER/.ssh/authorized_keys \ - # Generate list of arkouda-server and arkouda-locale IP addresses - export SSH_SERVERS="$MY_IP $LOCALE_IPS" \ + # Start arkouda_server as the configured user + sudo -E -u $USER bash <