Merge branch 'master' into rbirkner/revert-sev-update-relaxation

Author: r-birkner

.github/workflows/schedule-daily.yml

@@ -128,6 +128,7 @@ jobs:
 
           # Run SEV tests
           bazel test //rs/tests/nested:guestos_upgrade_from_latest_release_to_current_sev --test_env=BARE_METAL_HOST_SECRETS="$(realpath file1)"
+          bazel test //rs/tests/nested:hostos_upgrade_from_latest_release_to_current_sev --test_env=BARE_METAL_HOST_SECRETS="$(realpath file1)"
           bazel test //rs/tests/nested:sev_recovery --test_env=BARE_METAL_HOST_SECRETS="$(realpath file1)"
 
           bazel clean

Cargo.lock

@@ -4,8 +4,8 @@
 #
 # This script determines which Bazel targets should be built or tested and writes them separated by newlines to stdout.
 #
-# If --base is passed only include targets with modified inputs in `git diff --name-only --merge-base $BASE $HEAD`.
-# When --head is not provided defaults to HEAD.
+# If --base is passed only include targets with modified inputs in `git diff --name-only --merge-base $BASE [$HEAD]`.
+# where `$HEAD` is from --head if specified.
 #
 # If --skip_long_tests is passed, tests tagged with 'long_test' will be excluded.
 #
@@ -110,15 +110,20 @@ def load_explicit_targets() -> dict[str, Set[str]]:
     return explicit_targets_dict
 
 
-def diff_only_query(command: str, base: str, head: str, skip_long_tests: bool) -> str:
+def diff_only_query(command: str, base: str, head: str | None, skip_long_tests: bool) -> str:
     """
-    Return a bazel query for all targets that have modified inputs in the specified git commit range. Taking into account:
+    Return a bazel query for all targets that have modified inputs in the specified git commit range.
+    If `head` is not specified it diffs against the working tree which is useful for testing locally.
+    It takes into account:
     * To return all targets in case files matching ALL_TARGETS_BLOBS are modified.
     * To only include test targets in case the bazel command was 'test'.
     * To exclude long_tests if requested.
     """
     modified_files = subprocess.run(
-        ["git", "diff", "--name-only", "--merge-base", base, head], check=True, capture_output=True, text=True
+        ["git", "diff", "--name-only", "--merge-base", base] + ([head] if head is not None else []),
+        check=True,
+        capture_output=True,
+        text=True,
     ).stdout.splitlines()
 
     n = len(modified_files)
@@ -187,7 +192,7 @@ def targets(
     query = (
         ("//..." + (" except attr(tags, long_test, //...)" if skip_long_tests else ""))
         if base is None
-        else diff_only_query(command, base, "HEAD" if head is None else head, skip_long_tests)
+        else diff_only_query(command, base, head, skip_long_tests)
     )
 
     # Finally, exclude targets that have any of the excluded tags:
@@ -277,7 +282,7 @@ def main():
     )
     parser.add_argument(
         "--base",
-        help="Only include targets with modified inputs in `git diff --name-only --merge-base $BASE $HEAD`. When --head is not provided defaults to HEAD.",
+        help="Only include targets with modified inputs in `git diff --name-only --merge-base $BASE [$HEAD]` where $HEAD is from --head if specified.",
     )
     parser.add_argument("--head", help="See --base.")
     args = parser.parse_args()

ci/scripts/targets.py

@@ -3,7 +3,8 @@ tmpfs    /tmp    tmpfs   defaults 0   2
 /dev/disk/by-partuuid/b78084e2-3363-1346-8c25-d426f26b8928   /boot/efi   vfat    defaults    0   2
 /dev/disk/by-partuuid/6788e4cf-f456-104e-9a34-a2c58cfb0ee6   /boot/grub  vfat    defaults    0   2
 /dev/disk/by-partuuid/a5ba3816-beaa-d74d-993e-cfa5aa6ba1f6   /boot/config    ext4    defaults,sync    0   2
-/dev/mapper/var_crypt   /var  ext4    defaults    0   2
+# Remove the BindsTo on the underlying crypt device. We suspect the device flapping in udev can lead to the mount locking up.
+/dev/mapper/var_crypt   /var  ext4    defaults,x-systemd.device-bound=false    0   2
 # If you add a new mount that depends on /dev/mapper/store, don't forget to add it to maybe-disable-store-mounts.service
 /dev/mapper/store-shared--backup   /var/lib/ic/backup    ext4 defaults,context=system_u:object_r:ic_data_t:s0  0   2
 /dev/mapper/store-shared--crypto   /var/lib/ic/crypto    ext4 defaults   0   2

ic-os/components/early-boot/fstab/fstab-guestos

@@ -10,7 +10,6 @@ source /opt/ic/bin/config.sh
 
 function read_config_variables() {
     config_bitcoind_addr=$(get_config_value '.guestos_settings.guestos_dev_settings.bitcoind_addr')
-    config_socks_proxy=$(get_config_value '.guestos_settings.guestos_dev_settings.socks_proxy')
 }
 
 function usage() {
@@ -43,13 +42,6 @@ done
 
 read_config_variables
 
-# Production socks5 proxy url needs to include schema, host and port to be accepted by the adapters.
-# Testnets deploy with a development socks_proxy config value to overwrite the production socks proxy with the testnet proxy.
-SOCKS_PROXY="socks5://socks5.ic0.app:1080"
-if [ "${config_socks_proxy}" != "" ] && [ "${config_socks_proxy}" != "null" ]; then
-    SOCKS_PROXY="${config_socks_proxy}"
-fi
-
 BITCOIN_NETWORK='"testnet4"'
 DNS_SEEDS='"seed.testnet4.bitcoin.sprovoost.nl",
             "seed.testnet4.wiz.biz"'
@@ -72,7 +64,7 @@ if [ "${OUT_FILE}" == "" ]; then
     exit 1
 fi
 
-# config_bitcoind_addr indicates that we are in system test environment. No socks proxy needed.
+# config_bitcoind_addr indicates that we are in system test environment.
 if [ "${config_bitcoind_addr}" != "" ] && [ "${config_bitcoind_addr}" != "null" ]; then
     echo '{
         "network": "regtest",

ic-os/components/guestos/ic-btc-adapter/generate-btc-adapter-config.sh

@@ -10,7 +10,6 @@ source /opt/ic/bin/config.sh
 
 function read_config_variables() {
     config_dogecoind_addr=$(get_config_value '.guestos_settings.guestos_dev_settings.dogecoind_addr')
-    config_socks_proxy=$(get_config_value '.guestos_settings.guestos_dev_settings.socks_proxy')
 }
 
 function usage() {
@@ -43,13 +42,6 @@ done
 
 read_config_variables
 
-# Production socks5 proxy url needs to include schema, host and port to be accepted by the adapters.
-# Testnets deploy with a development socks_proxy config value to overwrite the production socks proxy with the testnet proxy.
-SOCKS_PROXY="socks5://socks5.ic0.app:1080"
-if [ "${config_socks_proxy}" != "" ] && [ "${config_socks_proxy}" != "null" ]; then
-    SOCKS_PROXY="${config_socks_proxy}"
-fi
-
 DOGECOIN_NETWORK='"dogecoin:testnet"'
 CACHE_NAME='dogecoin_testnet_cache'
 DNS_SEEDS='"jrn.me.uk",
@@ -67,7 +59,7 @@ if [ "${OUT_FILE}" == "" ]; then
     exit 1
 fi
 
-# config_dogecoind_addr indicates that we are in system test environment. No socks proxy needed.
+# config_dogecoind_addr indicates that we are in system test environment.
 if [ "${config_dogecoind_addr}" != "" ] && [ "${config_dogecoind_addr}" != "null" ]; then
     echo '{
         "network": "dogecoin:regtest",

ic-os/components/guestos/ic-btc-adapter/generate-doge-adapter-config.sh

@@ -1,11 +1,12 @@
-# This unit is overridden to remove the BindsTo on the underlying crypt device.
-# Otherwise, it is the same as the systemd generated default. We suspect the
-# device flapping in udev can lead to the mount locking up.
+# This unit is overridden to replace the BindsTo on the underlying crypt device
+# with Requires. Otherwise, it is the same as the systemd generated default. We
+# suspect the device flapping in udev can lead to the mount locking up.
 
 [Unit]
 Description=File System Check on /dev/mapper/var_crypt
 Documentation=man:systemd-fsck@.service(8)
 DefaultDependencies=no
+Requires=dev-mapper-var_crypt.device
 Conflicts=shutdown.target
 Wants=systemd-fsckd.socket
 After=dev-mapper-var_crypt.device systemd-fsck-root.service local-fs-pre.target systemd-fsckd.socket

ic-os/components/guestos/init/setup-encryption/systemd-fsck@dev-mapper-var_crypt.service

@@ -77,7 +77,7 @@ function check_num_cpus() {
     local cpu_json="$1"
     local required_sockets="$2"
 
-    local num_cpu_sockets=$(echo "${cpu_json}" | jq -r '.[].id' | wc -l)
+    local num_cpu_sockets=$(echo "${cpu_json}" | jq -r '.[] | select(has("disabled") | not) | .id' | wc -l)
     log_and_halt_installation_on_error "$?" "Unable to extract CPU sockets from CPU JSON."
 
     if [ "${num_cpu_sockets}" -lt "${required_sockets}" ]; then
@@ -89,7 +89,7 @@ function verify_capability_for_all_sockets() {
     local cpu_json="$1"
     local capability_name="$2"
 
-    for socket_id in $(echo "${cpu_json}" | jq -r '.[].id'); do
+    for socket_id in $(echo "${cpu_json}" | jq -r '.[] | select(has("disabled") | not) | .id'); do
         local capability=$(echo "${cpu_json}" | jq -r \
             --arg socket "${socket_id}" \
             --arg capability "${capability_name}" \
@@ -112,7 +112,7 @@ function verify_model_for_all_sockets() {
     local cpu_json="$1"
     local required_model="$2"
 
-    for socket_id in $(echo "${cpu_json}" | jq -r '.[].id'); do
+    for socket_id in $(echo "${cpu_json}" | jq -r '.[] | select(has("disabled") | not) | .id'); do
         local model=$(echo "${cpu_json}" | jq -r --arg socket "${socket_id}" '.[] | select(.id==$socket) | .product')
 
         echo -n "* CPU model '${model}' "

ic-os/components/setupos/check-hardware.sh

@@ -130,16 +130,26 @@ function test_verify_cpu() {
     ]' 64 1
 
     # Gen3 Success
-    test_verify_cpu_helper "verify_cpu Gen3 success" "3" '[
+
+    ## AMD SVM 2 sockets
+    test_verify_cpu_helper "verify_cpu Gen3 success AMD" "3" '[
       {"id": "cpu:0", "product": "Foobar CPU", "capabilities": {"svm": "true"}},
       {"id": "cpu:1", "product": "Foobaz CPU", "capabilities": {"svm": "true"}}
     ]' 1 0
 
-    test_verify_cpu_helper "verify_cpu Gen3 success" "3" '[
-      {"id": "cpu:0", "product": "Foobar CPU", "capabilities": {"vmx": "true"}},
+    ## Intel VMX 1 socket
+    test_verify_cpu_helper "verify_cpu Gen3 success Intel" "3" '[
+      {"id": "cpu:0", "product": "Foobar CPU", "capabilities": {"vmx": "true"}}
+    ]' 1 0
+
+    ## One unpopulated socket ignored
+    test_verify_cpu_helper "verify_cpu Gen3 success AMD 1 socket unpopulated" "3" '[
+      {"id": "cpu:0", "product": "Foobar CPU", "capabilities": {"svm": "true"}},
+      {"id": "cpu:1", "product": "Foobaz CPU", "disabled": true}
     ]' 1 0
 
     # Gen3 Failure
+    ## Missing caps
     test_verify_cpu_helper "verify_cpu Gen3 failure" "3" '[
       {"id": "cpu:0", "product": "Foobar CPU", "capabilities": {}},
       {"id": "cpu:1", "product": "Foobaz CPU", "capabilities": {}}

ic-os/components/setupos/test-setupos.sh

@@ -62,8 +62,10 @@ COPY packages.* /tmp/
 RUN apt-get -y update && \
     apt-get -y upgrade && \
     apt-get -y --no-install-recommends install $(for P in ${PACKAGE_FILES}; do cat /tmp/$P | sed -e "s/#.*//" ; done) \
-        ${_KERNEL_PACKAGE} \
-        linux-modules-extra-$(apt-cache depends ${_KERNEL_PACKAGE} | sed -n -e 's/  Depends: linux-image-\(.*\)-generic/\1/p')-generic && \
+        # TODO(NODE-1852): Temporarily pin the kernel here.
+        # ${_KERNEL_PACKAGE} \
+        # linux-modules-extra-$(apt-cache depends ${_KERNEL_PACKAGE} | sed -n -e 's/  Depends: linux-image-\(.*\)-generic/\1/p')-generic && \
+        linux-image-6.14.0-37-generic linux-modules-extra-6.14.0-37-generic && \
     rm /tmp/packages.*
 
 # Install node_exporter