Merge branch 'refactor--and-improve-dir-structure-NMO-794'

Author: RedLeader962

build_matrix_config/dev/.env.build_matrix.dev

@@ -11,9 +11,9 @@
 #NBS_MATRIX_ROS_DISTRO=('none')
 #NBS_MATRIX_ROS_PKG=('none')
 
-NBS_COMPOSE_DIR=dockerized-norlab-images/core-images/base-images/ros2-install
-NBS_EXECUTE_BUILD_MATRIX_OVER_COMPOSE_FILE="${NBS_COMPOSE_DIR}/docker-compose.ros2.build.yaml"
-#NBS_EXECUTE_BUILD_MATRIX_OVER_COMPOSE_FILE="${NBS_COMPOSE_DIR}/docker-compose.ros2-vaul.build.yaml"
+#NBS_COMPOSE_DIR=dockerized-norlab-images/core-images/base-images/ros2-install
+#NBS_EXECUTE_BUILD_MATRIX_OVER_COMPOSE_FILE="${NBS_COMPOSE_DIR}/docker-compose.ros2.build.yaml"
+##NBS_EXECUTE_BUILD_MATRIX_OVER_COMPOSE_FILE="${NBS_COMPOSE_DIR}/docker-compose.ros2-vaul.build.yaml"
 
 #NBS_COMPOSE_DIR=dockerized-norlab-images/core-images/dependencies
 #NBS_EXECUTE_BUILD_MATRIX_OVER_COMPOSE_FILE="${NBS_COMPOSE_DIR}/docker-compose.dn-dependencies.build.yaml"
@@ -26,8 +26,8 @@ NBS_EXECUTE_BUILD_MATRIX_OVER_COMPOSE_FILE="${NBS_COMPOSE_DIR}/docker-compose.ro
 #NBS_COMPOSE_DIR=dockerized-norlab-images/core-images/pkg-perception
 #NBS_EXECUTE_BUILD_MATRIX_OVER_COMPOSE_FILE="${NBS_COMPOSE_DIR}/docker-compose.dn-perception.build.yaml"
 
-#NBS_COMPOSE_DIR=dockerized-norlab-images/core-images/dn-project
-#NBS_EXECUTE_BUILD_MATRIX_OVER_COMPOSE_FILE="${NBS_COMPOSE_DIR}/docker-compose.dn-project.build.yaml"
+NBS_COMPOSE_DIR=dockerized-norlab-images/core-images/dn-project
+NBS_EXECUTE_BUILD_MATRIX_OVER_COMPOSE_FILE="${NBS_COMPOSE_DIR}/docker-compose.dn-project.build.yaml"
 
 #
 # Build Dockerized-NorLab for the following base images

build_matrix_config/dev/run_config_buidl_over_single_build_matrix.bash

@@ -26,9 +26,9 @@ function buidl_over_single_build_matrix() {
 #  docker_flag+=( l4t-base-image-amd64-tester )
 
   # ....ROS2-install base images...................................................................
-  docker_flag+=( base-image-ros2-clean-main )
-  docker_flag+=( base-image-ros2-clean-tester )
-  docker_flag+=( base-image-ros2-clean )
+#  docker_flag+=( base-image-ros2-clean-main )
+#  docker_flag+=( base-image-ros2-clean-tester )
+#  docker_flag+=( base-image-ros2-clean )
 
   # ....dependencies images........................................................................
 #  docker_flag+=( dependencies-core-main )
@@ -58,15 +58,15 @@ function buidl_over_single_build_matrix() {
 #  docker_flag+=( dn-control-deep-rl-f1tenth-gym ) # final
 
   # ....dn-project images............................................................................
-#  docker_flag+=( project-core-main )
-#  docker_flag+=( project-core )
-#  docker_flag+=( project-develop-main )
-#  docker_flag+=( project-develop-test-ssh-user )
-#  docker_flag+=( project-develop-test-project-user )
-#  docker_flag+=( project-develop )
-#  docker_flag+=( project-deploy-main )
-#  docker_flag+=( project-deploy-tester )
-#  docker_flag+=( project-deploy )
+  docker_flag+=( project-core-main )
+  docker_flag+=( project-core )
+  docker_flag+=( project-develop-main )
+  docker_flag+=( project-develop-test-ssh-user )
+  docker_flag+=( project-develop-test-project-user )
+  docker_flag+=( project-develop )
+  docker_flag+=( project-deploy-main )
+  docker_flag+=( project-deploy-tester )
+  docker_flag+=( project-deploy )
 
   # ....Script flag..................................................................................
   local script_flag=()

dockerized-norlab-images/container-tools/dn_install_debugging_tools.bash

@@ -110,7 +110,7 @@ function dn::setup_debugging_tools() {
 
   n2st::print_msg_warning "Be advised, the ssh daemon still need to be started.
   Curently its done by 'dn_entrypoint.global.init.callback.bash'.
-  Ref 'dockerized-norlab-images/core-images/dn-project/project-core/project_entrypoints'.
+  Ref 'dockerized-norlab-images/core-images/dn-project/project-core/entrypoints'.
   "
 
   # ....Set password for users.......................................................................

dockerized-norlab-images/core-images/base-images/l4t-images/Dockerfile.l4t.squash

@@ -30,7 +30,8 @@ RUN <<EOF
           --exclude torchaudio  \
           --exclude tensorrt \
           --exclude torch2trt  \
-          --exclude pycuda
+          --exclude pycuda \
+          --exclude nvidia-pyindex
     ) > /requirements.l4t-base-image.txt
 
     # ....Builder sanity check.....................................................................
@@ -103,37 +104,6 @@ COPY --from=base-image / /
 #COPY --from=base-image /opt /opt
 #COPY --from=base-image ${ROS_ROOT} ${ROS_ROOT}
 
-RUN <<EOF
-    echo "(deb) Install basic utilities..."
-
-    # ....Setup Sources............................................................................
-    {
-      apt-get update \
-      && apt-get install --assume-yes --no-install-recommends \
-            software-properties-common \
-      && add-apt-repository --yes universe \
-      && apt-get update;
-    } || exit 1
-
-    apt-get install --assume-yes --no-install-recommends \
-            curl \
-            pkg-config \
-            lsb-release \
-      || exit 1
-
-    echo "Log build time environment variable..."
-    printenv
-
-    # Update symlink to point to system python3 instead of system python2
-    update-alternatives --install /usr/bin/python python /usr/bin/python3 1 || exit 1
-    update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 || exit 1
-
-    # ....Teardown.................................................................................
-    apt-get autoremove --assume-yes
-    apt-get clean
-    rm -rf /var/lib/apt/lists/*
-EOF
-
 # ....Transfer environment variable from base-images...............................................
 # Env var specific to 'nvcr.io/nvidia/l4t-jetpack' base images
 ARG CUDA_HOME
@@ -192,6 +162,37 @@ ENV NVCC_PATH=${NVCC_PATH}
 ENV CUDA_NVCC_EXECUTABLE=${CUDA_NVCC_EXECUTABLE}
 ENV CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES}
 
+RUN <<EOF
+    echo "Log build time environment variable..."
+    printenv
+
+    echo "(deb) Install basic utilities..."
+
+    # ....Setup Sources............................................................................
+    {
+      apt-get update \
+      && apt-get install --assume-yes --no-install-recommends \
+            software-properties-common \
+      && add-apt-repository --yes universe \
+      && apt-get update;
+    } || exit 1
+
+    apt-get install --assume-yes --no-install-recommends \
+            curl \
+            pkg-config \
+            lsb-release \
+      || exit 1
+
+    # Update symlink to point to system python3 instead of system python2
+    update-alternatives --install /usr/bin/python python /usr/bin/python3 1 || exit 1
+    update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 || exit 1
+
+    # ....Teardown.................................................................................
+    apt-get autoremove --assume-yes
+    apt-get clean
+    rm -rf /var/lib/apt/lists/*
+EOF
+
 
 FROM --platform=linux/amd64 ${MIMIC_DEPENDENCIES_BASE_IMAGE:?err}:${MIMIC_DEPENDENCIES_BASE_IMAGE_TAG:?err} AS base-image-amd64
 ARG TARGETPLATFORM
@@ -209,6 +210,9 @@ SHELL ["/bin/bash", "-c"]
 ARG UBUNTU_VERSION_MAJOR
 
 RUN <<EOF
+    echo "Log build time environment variable..."
+    printenv
+
     # ....Check pre-conditions.......................................................................
     {
         test -n "${UBUNTU_VERSION_MAJOR:?'Env variable need to be set and non-empty.'}" && \
@@ -234,8 +238,6 @@ RUN <<EOF
         python3-dev \
         python3-pip
 
-    echo "Log build time environment variable..."
-    printenv
 
     IMG_RELEASE="$( source /etc/lsb-release && echo ${DISTRIB_RELEASE})"
     if [[ "${UBUNTU_VERSION_MAJOR:?err}.04" != "${IMG_RELEASE}" ]]; then
@@ -280,6 +282,9 @@ ENV IS_TEAMCITY_RUN=${IS_TEAMCITY_RUN:-false}
 COPY --from=base-image /requirements.l4t-base-image.txt /requirements.l4t-base-image.txt
 
 RUN <<EOF
+    echo "Log build time environment variable..."
+    printenv
+
     # ....Check pre-conditions.....................................................................
     {
         test -n "${UBUNTU_VERSION_MAJOR:?'Env variable need to be set and non-empty.'}" && \
@@ -290,9 +295,8 @@ RUN <<EOF
     } || { echo -e "\033[1;31m[DN error]\033[0m Failed pre-condition check" 1>&2 ; exit 1 ; }
 
     # ....Reinstall l4t image package in amd64 image...............................................
-    apt-get update
-
     if [[ ${TARGETPLATFORM} =~ "linux/amd64".* ]]; then
+        apt-get update
 
         echo "Install general utilities..."
         apt-get install --assume-yes --no-install-recommends \
@@ -327,23 +331,28 @@ RUN <<EOF
     fi
 
     # ....Install l4t image ubuntu apt requirement.................................................
-    echo "Install l4t image ubuntu apt requirement..."
-    # Install pycuda related
-    #   Build instruction: https://github.com/berlinguyinca/pycuda/blob/master/doc/source/install.rst
-    #
-    # Requirement for 'pycairo' which is required by 'pycuda'
-    apt-get install --assume-yes --no-install-recommends \
-      libcairo2-dev
+    if [[ ${TARGETPLATFORM} =~ "linux/amd64".* ]]; then
+        echo "Install l4t image ubuntu apt requirement..."
 
-    # ....Install l4t image python requirement from file...........................................
-    echo "Install l4t image python requirement from file..."
-    # Requirement for 'graphsurgeon' and 'uff', it give access to nvidia pip index
-    pip3 install --no-cache-dir nvidia-pyindex
+        echo "Log requirements.l4t-base-image.txt for debuging..."
+        cat /requirements.l4t-base-image.txt
+
+        # Install pycuda related
+        #   Build instruction: https://github.com/berlinguyinca/pycuda/blob/master/doc/source/install.rst
+        #
+        # Requirement for 'pycairo' which is required by 'pycuda'
+        apt-get install --assume-yes --no-install-recommends \
+          libcairo2-dev
+
+        # ....Install l4t image python requirement from file...........................................
+        echo "Install l4t image python requirement from file..."
+        # Requirement for 'graphsurgeon' and 'uff', it give access to nvidia pip index
+        python3 -m pip install onnx_graphsurgeon uff --extra-index-url https://pypi.ngc.nvidia.com
+        #|| pip3 install --no-cache-dir nvidia-pyindex
 
-    if [[ ${TARGETPLATFORM} =~ "linux/amd64".* ]]; then
         echo "Strip versions from requirement"
         sed -i.bak 's/==.*//g' /requirements.l4t-base-image.txt
-        pip3 install -r /requirements.l4t-base-image.txt || exit 1
+        pip3 install -r /requirements.l4t-base-image.txt --extra-index-url https://pypi.ngc.nvidia.com
 
         # Clean up sed tmp file
         rm /requirements.l4t-base-image.txt.bak
@@ -356,9 +365,8 @@ RUN <<EOF
     #   - https://github.com/NVIDIA-AI-IOT/torch2trt
     #   - https://github.com/dusty-nv/jetson-containers/blob/master/packages/pytorch/torch2trt/Dockerfile
     #
-
-    # pip3 install git+https://github.com/NVIDIA-AI-IOT/torch2trt
     if [[ ${TARGETPLATFORM} =~ "linux/amd64".* ]]; then
+        # pip3 install git+https://github.com/NVIDIA-AI-IOT/torch2trt
         cd /opt
         git clone https://github.com/NVIDIA-AI-IOT/torch2trt
         cd torch2trt
@@ -392,6 +400,8 @@ ARG TARGETARCH
 ARG BUILDPLATFORM
 
 RUN <<EOF
+    echo "Test final"
+
     # ....Check pre-conditions.......................................................................
     {
         test -n "${TARGETPLATFORM:?'Env variable need to be set and non-empty.'}" && \
@@ -422,11 +432,11 @@ RUN <<EOF
 
     if [[ ${TARGETPLATFORM} =~ "linux/arm64".* ]]; then
       {
-        test -n ${CUDA_HOME:?'Environment variable was not passed from base-image build stage to test stage'} && \
-        test -n ${NVIDIA_VISIBLE_DEVICES:?'Environment variable was not passed from base-image build stage to test stage'} && \
-        test -n ${NVIDIA_DRIVER_CAPABILITIES:?'Environment variable was not passed from base-image build stage to test stage'} && \
-        test -n ${PATH:?'Environment variable was not passed from base-image build stage to test stage'} && \
-        test -n ${LD_LIBRARY_PATH:?'Environment variable was not passed from base-image build stage to test stage'} ;
+        test -n ${CUDA_HOME:?'(Final) Environment variable was not passed from base-image build stage to test stage'} && \
+        test -n ${NVIDIA_VISIBLE_DEVICES:?'(Final) Environment variable was not passed from base-image build stage to test stage'} && \
+        test -n ${NVIDIA_DRIVER_CAPABILITIES:?'(Final) Environment variable was not passed from base-image build stage to test stage'} && \
+        test -n ${PATH:?'(Final) Environment variable was not passed from base-image build stage to test stage'} && \
+        test -n ${LD_LIBRARY_PATH:?'(Final) Environment variable was not passed from base-image build stage to test stage'} ;
       } || { echo "Missing required environment variables!" 1>&2 ; exit 1 ; }
     fi
 

dockerized-norlab-images/core-images/base-images/l4t-images/docker-compose.l4t-squash.build.yaml

@@ -88,6 +88,8 @@ services:
       service: l4t-base-image-amd64
     build:
       target: mimic-l4t-image
+      platforms: !override
+        - linux/amd64
     depends_on:
       - l4t-base-image-amd64
 
@@ -107,6 +109,8 @@ services:
     image: ${DN_HUB:?err}/dockerized-norlab-base-image:${DN_IMAGE_TAG:?err}-amd64
     build:
       target: final-amd64
+      platforms: !override
+        - linux/amd64
     depends_on:
       - l4t-base-image-mimic-l4t-image
 
@@ -124,6 +128,8 @@ services:
       service: l4t-base-image-amd64
     build:
       target: test
+      platforms: !override
+        - linux/amd64
     depends_on:
       - l4t-base-image-amd64-final
 

dockerized-norlab-images/core-images/base-images/ubuntu-images/Dockerfile.cuda.squash

@@ -24,6 +24,7 @@ RUN touch /requirements.l4t-base-image.txt \
               --exclude torch2trt  \
               --exclude torchaudio  \
               --exclude tensorrt \
+              --exclude nvidia-pyindex \
         > /requirements.l4t-base-image.txt
 
 # ===Remove Jetson-container unused install========================================================
@@ -210,7 +211,7 @@ RUN apt-get update  \
         ninja
 
 # Requirement for 'graphsurgeon' and 'uff', it give access to nvidia pip index
-RUN pip3 install --no-cache-dir nvidia-pyindex
+RUN python3 -m pip install onnx_graphsurgeon uff --extra-index-url https://pypi.ngc.nvidia.com
 
 # Requirement for 'pycairo' which is required by 'pycuda'
 RUN apt-get update  \
@@ -249,7 +250,7 @@ RUN apt-get update  \
 
 #    pycuda==2023.1 \
 #    PySoundFile==0.9.0.post1 \
-RUN sed -i.bak 's/==.*//g' /requirements.l4t-base-image.txt  \
+RUN sed -i.bak 's/==.*//g' /requirements.l4t-base-image.txt \
     && rm /requirements.l4t-base-image.txt.bak
 
 RUN pip3 install --no-cache-dir -r /requirements.l4t-base-image.txt

dockerized-norlab-images/core-images/dependencies/Dockerfile.ros2-dn-custom

@@ -44,7 +44,6 @@ RUN <<EOF
         osrf-pycommon \
         rosbags \
         rosbags-dataframe \
-        rosbag-tools[plot] \
         || exit 1
 
     apt-get install --assume-yes --no-install-recommends \

dockerized-norlab-images/core-images/dn-project/project-core/Dockerfile.template

@@ -41,8 +41,8 @@ FROM project-setup AS project-custom-docker-steps
 # ====Project custom python libraries==============================================================
 
 # (NICE TO HAVE) ToDo: refactor to dn_project_core_init.bash (ref task RLRP-213 and NMO-548)
-COPY ./project_requirements/python.requirements.txt /python.requirements.txt
-COPY ./project_requirements/shell.requirements.bash /shell.requirements.bash
+COPY ./build_stage/python.requirements.txt /python.requirements.txt
+COPY ./build_stage/shell.requirements.bash /shell.requirements.bash
 
 # (NICE TO HAVE) ToDo: refactor to dn_project_core_init.bash (ref task RLRP-213 and NMO-548)
 RUN <<EOF
@@ -60,18 +60,18 @@ EOF
 # ====DN-project final=============================================================================
 FROM project-custom-docker-steps AS final
 
-# (NICE TO HAVE) ToDo: refactor project_entrypoints logic a fct. Either in DN container-tools or in DN-project
+# (NICE TO HAVE) ToDo: refactor entrypoints logic a fct. Either in DN container-tools or in DN-project
 # Note: project-develop container mounth those directories as a volume to skip rebuilding
 #       each time they are modify (see docker-compose.project.run.*.yaml). However, project-deploy
 #       and project release container will need them copied in the image for portability.
-# Note: Files copied from project_entrypoints/ directory are required by Dockerized-NorLab
+# Note: Files copied from entrypoints/ directory are required by Dockerized-NorLab
 #       dn_entrypoint.init.bash and dn_entrypoint.attach.bash
-WORKDIR /project_entrypoints
-ENV MOCK_PROJECT_ENTRYPOINT_PATH=./project_entrypoints
+WORKDIR /entrypoints
+ENV MOCK_PROJECT_ENTRYPOINT_PATH=./entrypoints
 COPY ${MOCK_PROJECT_ENTRYPOINT_PATH} .
 COPY --from=context-dn-container-tools ./dn_project_core_finalize.bash /dn_project_core_finalize.bash
 RUN <<EOF
-    # Note: DN-project path would be ".dockerized_norlab_project/configuration/project_entrypoints"
+    # Note: DN-project path would be ".dockerized_norlab_project/configuration/entrypoints"
     for each_file in ./dn_entrypoint.*.bash; do
       chmod +x "${each_file}"
     done