Merge remote-tracking branch 'origin/4.19' into add_systemvmtemplate_arm64_support

Author: yadvr

.asf.yaml

@@ -27,15 +27,21 @@ github:
     - infrastructure
     - java
     - python
-    - hosting
     - kvm
+    - libvirt
     - vsphere
+    - vmware
     - xenserver
     - xcp-ng
+    - orchestration
+    - virtualization
+    - virtual-machine
+    - kubernetes
 
   features:
     wiki: true
     issues: true
+    discussions: true
     projects: true
 
   enabled_merge_buttons:
@@ -45,5 +51,19 @@ github:
 
   collaborators:
     - acs-robot
+    - kiranchavala
+    - rajujith
+    - alexandremattioli
+    - vishesh92
+    - GaOrtiga
+    - SadiJr
+    - winterhazel
+    - rp-
 
   protected_branches: ~
+
+notifications:
+  commits:      commits@cloudstack.apache.org
+  issues:       commits@cloudstack.apache.org
+  pullrequests: commits@cloudstack.apache.org
+  discussions:  users@cloudstack.apache.org

.github/linters/.flake8

@@ -15,11 +15,17 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# E112 Expected an indented block
+# E113 Unexpected indentation
+# E133 Closing bracket is missing indentation
 # E223 Tab before operator
 # E224 Tab after operator
+# E227 Missing whitespace around bitwise or shift operator
 # E242 Tab after ','
 # E273 Tab after keyword
 # E274 Tab before keyword
+# E742 Do not define classes named 'I', 'O', or 'l'
+# E743 Do not define functions named 'I', 'O', or 'l'
 # E901 SyntaxError or IndentationError
 # E902 IOError
 # W291 Trailing whitespace
@@ -28,4 +34,7 @@
 # W391 Blank line at end of file
 
 [flake8]
-select = E223,E224,E242,E273,E274,E901,E902,W291,W292,W293,W391
+exclude =
+    .git,
+    venv
+select = E112,E113,E133,E223,E224,E227,E242,E273,E274,E742,E743,E901,E902,W291,W292,W293,W391

.github/workflows/build.yml

@@ -23,14 +23,17 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up JDK 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '11'
           distribution: 'adopt'
@@ -63,4 +66,4 @@ jobs:
         run: |
           git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss && cd nonoss && bash -x install-non-oss.sh && cd ..
           rm -fr nonoss
-          mvn -B -P developer,systemvm -Dsimulator -Dnoredist clean install
+          mvn -B -P developer,systemvm -Dsimulator -Dnoredist clean install -T$(nproc)

.github/workflows/ci.yml

@@ -23,9 +23,13 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    if: github.repository == 'apache/cloudstack'
+    runs-on: ubuntu-22.04
 
     strategy:
       fail-fast: false
@@ -50,6 +54,7 @@ jobs:
                   smoke/test_deploy_vm_with_userdata
                   smoke/test_deploy_vms_in_parallel
                   smoke/test_deploy_vms_with_varied_deploymentplanners
+                  smoke/test_restore_vm
                   smoke/test_diagnostics
                   smoke/test_direct_download
                   smoke/test_disk_offerings
@@ -58,7 +63,8 @@ jobs:
                   smoke/test_domain_network_offerings
                   smoke/test_domain_service_offerings
                   smoke/test_domain_vpc_offerings",
-                "smoke/test_dynamicroles
+                "smoke/test_cluster_drs
+                  smoke/test_dynamicroles
                   smoke/test_enable_account_settings_for_domain
                   smoke/test_enable_role_based_users_in_projects
                   smoke/test_events_resource
@@ -75,17 +81,20 @@ jobs:
                   smoke/test_list_ids_parameter
                   smoke/test_loadbalance
                   smoke/test_login
+                  smoke/test_2fa
                   smoke/test_metrics_api
                   smoke/test_migration
                   smoke/test_multipleips_per_nic
-                  smoke/test_nested_virtualization",
+                  smoke/test_nested_virtualization
+                  smoke/test_set_sourcenat",
                 "smoke/test_network
                   smoke/test_network_acl
                   smoke/test_network_ipv6
                   smoke/test_network_permissions
                   smoke/test_nic
                   smoke/test_nic_adapter_type
                   smoke/test_non_contigiousvlan
+                  smoke/test_object_stores
                   smoke/test_outofbandmanagement
                   smoke/test_outofbandmanagement_nestedplugin
                   smoke/test_over_provisioning
@@ -104,7 +113,8 @@ jobs:
                   smoke/test_reset_configuration_settings
                   smoke/test_reset_vm_on_reboot
                   smoke/test_resource_accounting
-                  smoke/test_resource_detail",
+                  smoke/test_resource_detail
+                  smoke/test_global_acls",
                 "smoke/test_router_dhcphosts
                   smoke/test_router_dns
                   smoke/test_router_dnsservice
@@ -122,6 +132,7 @@ jobs:
                   smoke/test_usage
                   smoke/test_usage_events
                   smoke/test_vm_deployment_planner
+                  smoke/test_vm_schedule
                   smoke/test_vm_life_cycle
                   smoke/test_vm_lifecycle_unmanage_import
                   smoke/test_vm_snapshot_kvm
@@ -184,14 +195,23 @@ jobs:
                   component/test_vpc_network
                   component/test_vpc_offerings
                   component/test_vpc_routers
-                  component/test_vpn_users",
-                "component/test_vpc_network_lbrules" ]
+                  component/test_vpn_users
+                  component/test_vpc_network_lbrules",
+                "smoke/test_list_accounts
+                  smoke/test_list_disk_offerings
+                  smoke/test_list_domains
+                  smoke/test_list_hosts
+                  smoke/test_list_service_offerings
+                  smoke/test_list_storage_pools
+                  smoke/test_list_volumes"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Set up JDK
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '11'
           distribution: 'adopt'
@@ -213,6 +233,11 @@ jobs:
         run: |
           python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycrypto mock flask netaddr pylint pycodestyle six astroid
 
+      - name: Install jacoco dependencies
+        run: |
+          wget https://github.com/jacoco/jacoco/releases/download/v0.8.10/jacoco-0.8.10.zip
+          unzip jacoco-0.8.10.zip -d jacoco
+
       - name: Env details
         run: |
           uname -a
@@ -236,7 +261,7 @@ jobs:
 
       - name: Build with Maven
         run: |
-          mvn -B -P developer,systemvm -Dsimulator clean install -DskipTests=true
+          mvn -B -P developer,systemvm -Dsimulator clean install -DskipTests=true -T$(nproc)
 
       - name: Setup Simulator Prerequisites
         run: |
@@ -245,16 +270,23 @@ jobs:
           mvn -q -Pdeveloper -pl developer -Ddeploydb
           mvn -q -Pdeveloper -pl developer -Ddeploydb-simulator
 
+      - name: Generate jacoco-coverage.sh
+        run: |
+          echo "java -jar jacoco/lib/jacococli.jar report jacoco-it.exec \\" > jacoco-report.sh
+          find . | grep "target/classes" | sed 's/\/classes\//\/classes /g' | awk '{print "--classfiles", $1, "\\"}' | sort |uniq >> jacoco-report.sh
+          find . | grep "src/main/java" | sed 's/\/java\//\/java /g' | awk '{print "--sourcefiles", $1, "\\"}' | sort | uniq >> jacoco-report.sh
+          echo "--xml jacoco-coverage.xml" >> jacoco-report.sh
+
       - name: Start CloudStack Management Server with Simulator
         run: |
-          export MAVEN_OPTS="-Xmx4096m -XX:MaxPermSize=800m -Djava.security.egd=file:/dev/urandom"
+          export MAVEN_OPTS="-Xmx4096m -XX:MaxPermSize=800m -Djava.security.egd=file:/dev/urandom  -javaagent:jacoco/lib/jacocoagent.jar=address=*,port=36320,output=tcpserver"
           echo -e "\nStarting simulator"
           set +e
           mvn -Dsimulator -Dorg.eclipse.jetty.annotations.maxWait=120 -pl :cloud-client-ui jetty:run 2>&1 > /tmp/jetty-log || true &
           while ! nc -vzw 5 localhost 8096 2>&1 > /dev/null; do grep Exception /tmp/jetty-log; sleep 10; done
           set -e
           echo -e "\nStarting Advanced Zone DataCenter deployment"
-          python3 tools/marvin/marvin/deployDataCenter.py -i setup/dev/advanced.cfg 2>&1 || true
+          python3 tools/marvin/marvin/deployDataCenter.py -i setup/dev/advdualzone.cfg 2>&1 || true
 
       - name: Run Integration Tests with Simulator
         run: |
@@ -267,14 +299,25 @@ jobs:
           TESTS=($(echo $TESTS | tr -d '\n' | tr -s ' '))
           for suite in "${TESTS[@]}" ; do
             echo -e "Currently running test: $suite\n"
-            time nosetests-3.4 --with-xunit --xunit-file=integration-test-results/$suite.xml --with-marvin --marvin-config=setup/dev/advanced.cfg test/integration/$suite.py -s -a tags=advanced,required_hardware=false --zone=Sandbox-simulator --hypervisor=simulator || true ;
+            time nosetests-3.4 --with-xunit --xunit-file=integration-test-results/$suite.xml --with-marvin --marvin-config=setup/dev/advdualzone.cfg test/integration/$suite.py -s -a tags=advanced,required_hardware=false --zone=zim1 --hypervisor=simulator || true ;
           done
 
           echo -e "Stopping Simulator, integration tests run completed\n"
+          java -jar jacoco/lib/jacococli.jar dump --address localhost --port 36320 --destfile jacoco-it.exec
+          bash jacoco-report.sh
           mvn -Dsimulator -pl client jetty:stop 2>&1
           find /tmp//MarvinLogs -type f -exec echo -e "Printing marvin logs {} :\n" \; -exec cat {} \;
 
       - name: Integration Tests Result
         run: |
           echo -e "Simulator CI Test Results: (only failures listed)\n"
           python3 ./tools/marvin/xunit-reader.py integration-test-results/
+
+      - uses: codecov/codecov-action@v4
+        with:
+          files: jacoco-coverage.xml
+          fail_ci_if_error: true
+          flags: simulator-marvin-tests
+          verbose: true
+          name: codecov
+          token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/codecov.yml

@@ -28,13 +28,16 @@ concurrency:
 
 jobs:
   build:
+    if: github.repository == 'apache/cloudstack'
     name: codecov
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Set up JDK11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '11'
@@ -44,11 +47,13 @@ jobs:
         run: |
           git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss
           cd nonoss && bash -x install-non-oss.sh && cd ..
-          mvn -P quality -Dsimulator -Dnoredist clean install
+          mvn -P quality -Dsimulator -Dnoredist clean install -T$(nproc)
 
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v4
         with:
           files: ./client/target/site/jacoco-aggregate/jacoco.xml
           fail_ci_if_error: true
+          flags: unittests
           verbose: true
           name: codecov
+          token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/docker-cloudstack-simulator.yml

@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Docker Image Build
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - 4.*
+      - 5.*
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    if: github.repository == 'apache/cloudstack'
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Login to Docker Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ secrets.DOCKER_REGISTRY }}
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set Docker repository name
+        run: echo "DOCKER_REPOSITORY=apache" >> $GITHUB_ENV
+
+      - uses: actions/checkout@v4
+
+      - name: Set ACS version
+        run: echo "ACS_VERSION=$(grep '<version>' pom.xml | head -2 | tail -1 | cut -d'>' -f2 |cut -d'<' -f1)" >> $GITHUB_ENV
+
+      - name: Set Docker image TAG
+        run: echo "TAG=$(if [ "${{ github.ref_type }}" = "tag" ];then echo ${{ github.ref_name }}; elif [[ "${ACS_VERSION}" =~ .*-SNAPSHOT ]];then echo ${ACS_VERSION}-$(date +'%Y%m%d-%H%M%S');else echo ${ACS_VERSION};fi)" >> $GITHUB_ENV
+
+      - name: Set Docker image FULL TAG
+        run: echo "FULL_TAG=$(if [ "${{ secrets.DOCKER_REGISTRY }}" = "" ];then echo ${DOCKER_REPOSITORY}/cloudstack-simulator:${TAG};else echo ${{ secrets.DOCKER_REGISTRY }}/${DOCKER_REPOSITORY}/cloudstack-simulator:${TAG};fi)" >> $GITHUB_ENV
+
+      - name: Build the Docker image for cloudstack-simulator
+        run: cd tools/docker && docker build ../.. --file Dockerfile --tag ${FULL_TAG}
+
+      - name: Push Docker image to Docker Registry
+        run: docker push ${FULL_TAG}