xapi-project · liulinC · Apr 29, 2026 · Sep 30, 2024 · Nov 15, 2024 · Jan 31, 2025
diff --git a/doc/content/design/external-auth-ldaps.md b/doc/content/design/external-auth-ldaps.md
@@ -91,8 +91,9 @@ Given `ldaps` default to `false`, this feature is **NOT** enabled until explicit
 
 #### 3.1.2 Error code
 Following new error codes added to indicate ldaps enable related error
-- AUTH_NO_CERT,  no certs can be used for ldaps, refer to 4.1.2 for certs finding.
-- AUTH_INVALID_CERT, found certs, but none of the certs can be used to connect to DC
+- POOL_AUTH_ENABLE_FAILED_NO_CERTS,  no certs can be used for ldaps, refer to 4.1.2 for certs finding.
+- POOL_AUTH_ENABLE_FAILED_INVALID_CERTS, found certs, but none of the certs can be used to connect to DC
+**Note**: Current error code handing infrustrucure requires the error code prefix with POOL_AUTH_ENABLE_FAILED
 
 ### 3.2 Set/Get Pool LDAPS Status
 
@@ -134,10 +135,10 @@ xe pool-external-auth-set-ldaps uuid=<uuid> ldaps=<true|false>
 
 #### 3.2.1.2 Error code
 This API may raise following errors
-- AUTH_NO_CERT, no certs found to enable ldaps, refer to 4.1.2 for certs finding
-- AUTH_INVALID_CERT, found certs, but none of the certs can be used to connect to DC
+- AUTH_NO_CERTS, no certs found to enable ldaps, refer to 4.1.2 for certs finding
+- AUTH_INVALID_CERTS, found certs, but none of the certs can be used to connect to DC
 - AUTH_IS_DISABLED, AD is not enabled
-- AUTH_LDAPS_PING_FAILED,  failed to do ldaps query on all DCs with valid certs
+- AUTH_SET_LDAPS_FAILED,  Failed to set ldaps, the error message contains the details like ldap query on domain failed
 
 #### 3.2.2 Get Pool LDAPS Status
 
@@ -211,24 +212,6 @@ This design is following [trusted-certificates.md](https://github.com/xapi-proje
 - `pool.external_auth_set_ldaps` API
 - (Re)join domain
 
-### 4.2 Xapi Configuration
-
-#### 4.2.1 winbind-tls-verify-peer
-
-For security, xapi asks winbind to verify CA certificate. `ca_and_name_if_available` is the default.
-
-However, user may want to disable this verification for debug purpose.
-
-`winbind-tls-verify-peer` is introduced for xapi configuration, and the possible values are `no_check`, `ca_only`, `ca_and_name_if_available`, `ca_and_name` and `as_strict_as_possible`.
- The configured value will override  `tls verify peer` value in xapi generated samba configuration. Refer to [smb.conf](https://www.samba.org/samba/docs/current/man-html/smb.conf.5.html) for the details.
-
-
-**Note:** This item is not intended for public documentation. This is only for debug purpose, or system tuning for specific scenarios from engineering/support team.
-
-#### 4.2.2 ad-warning-message-interval
-
-xapi sends warning message to user with this interval on LDAP query failure. Default to 1 week. Refer to section "Session revalidate" for the details.
-
 ## 5. Session Revalidate
 
 xapi LDAP queries domain user status (if user has been added to manage XenServer) at configurable interval, and destroys the session created by domain user if user no longer in healthy status.
@@ -238,23 +221,11 @@ However, the LDAP query may fail due to various issues as follows:
 - Temporary network issues
 - CA certificate is not properly configured, or expired, etc.
 
-Instead of destroying user session for stability, a warning message will be sent to user with the details at configurable interval `ad-warning-message-interval`.
-
-- If no LDAP error, do nothing
-- If error happens, send the warning message if:
-  - first time see the error through xapi start up (so no need to persist last send time) or
-  - `current_time - last_sent_time > winbind_warning_message_interval`
-
-The message is defined as follows:
-- name: AD_DC_LDAP_CHECK
-- priority: Warning
-- cls: `Host
-- Body: LDAP(S) query check to `<DC>` of `<domain>` failed from `<host>` of `<pool>`
+Instead of destroying user session for stability, a warning will be printed in xensource.log
 
 Note:
 - The backend session revalidate check only performs on pool coordinator, thus the backend LDAP(S) query check only on coordinator
 - `external_auth_set_ldaps` perform LDAP(S) query check on every host
-- All previous AD_DC_LDAP_CHECK warning of a host will be cleaned on a successful LDAP(s) query from that host
 
 ## 6. Pool Join/Leave
 

diff --git a/doc/content/design/secureboot-certificate-expiry.md b/doc/content/design/secureboot-certificate-expiry.md
@@ -0,0 +1,169 @@
+---
+title: Handling Microsoft Secure Boot Certificate Expiry
+layout: default
+design_doc: true
+revision: 1
+status: draft
+---
+
+## 1. Background
+
+Microsoft Secure Boot certificates from 2011 are reaching end-of-life, and legacy VMs may still contain only the old certificate set. XenServer needs an out-of-band mechanism to update per-VM UEFI Secure Boot variables safely and at scale.
+
+Scope of this design:
+
+- Update certificate state tracking and update flow for VMs, snapshots, and templates
+- Provide API support for scheduling certificate updates on VM boot
+- Integrate xapi and varstored behavior for consistent state handling
+
+## 2. System Overview
+
+### 2.1 Out-of-band Update Mechanism
+
+Certificate update is implemented as a dedicated API-driven workflow (not a plugin), so that:
+
+- The interface is documented and SDK-generated
+- RBAC can be assigned precisely
+- xapi can route requests and coordinate host-side behavior consistently
+
+### 2.2 Certificate State Tracking
+
+A new VM field is introduced:
+
+- `VM.secureboot_certificates_state` (enum, readonly)
+
+States:
+
+- `ok`: No update required (including non-applicable VM types)
+- `update_available`: Update required
+- `update_on_boot`: Update scheduled for next boot
+
+~~~mermaid
+
+stateDiagram
+update_available --> update_on_boot : Admin marks VM for update
+update_on_boot --> ok : VM boots, update succeeds
+update_on_boot --> update_on_boot : VM boots, update fails(retain state)
+ok --> update_available : recompute state(e.g. legacy VM import)
+
+~~~
+
+### 2.3 RBAC
+
+The new update API follows VM-admin-level access, aligned with existing NVRAM-related VM operations.
+
+## 3. Design for Components
+
+### 3.1 VM Certificate State Model
+
+`VM.secureboot_certificates_state` applies to these VM-class objects,
+
+- VMs
+- Snapshots
+- Templates
+
+Transition intent:
+
+- Admin marks a VM for update: `update_available -> update_on_boot`
+- VM boots and update succeeds: `update_on_boot -> ok`
+- VM boots and update fails: remains `update_on_boot` or is reset to `update_available` based on update result handling
+
+### 3.2 API: Mark/Unmark Update-on-Boot
+
+New API:
+
+- `VM.update_secureboot_certificates_on_boot(session, vm, mark)`
+
+Behavior:
+
+- `mark=true`: require current state `update_available`, then set `update_on_boot`
+- `mark=false`: require current state `update_on_boot`, then set `update_available`
+
+Validation:
+
+- Reject invalid transitions with `OPERATION_NOT_ALLOWED`
+
+### 3.3 DB Upgrade and Import Handling
+
+On toolstack restart after upgrade:
+
+- Initialize `secureboot_certificates_state` for all VM records to `ok`
+- Re-evaluate NVRAM and set `update_available` where needed
+
+Applied to:
+
+- VMs
+- Snapshots
+- Non-default templates
+
+Default templates remain `ok`.
+
+For VM import and cross-pool migration:
+
+- If imported metadata lacks `secureboot_certificates_state`, determine state from NVRAM and set it during import
+- If imported metadata contains `secureboot_certificates_state`, reserve the state during import
+
+### 3.4 NVRAM and State Consistency
+
+The certificate state must stay consistent with actual NVRAM content.
+
+Key interface change:
+
+- Extend `VM.set_NVRAM_EFI_variables` with optional parameter `update`, we call it `VM.set_NVRAM_EFI_variables_V2`
+
+Rules:
+
+- `update=yes` -> set state `ok`
+- `update=no` -> do not update state
+- omitted -> xapi runs certificate check helper and derives state
+
+This ensures compatibility when old varstored instances are still running during rolling update windows.
+
+### 3.5 Certificate Check Helper
+
+A standalone program  will be introduced, which xapi calls to determine the SecureBoot cert state
+
+Inputs:
+
+- `temp file path` which contains NVRAM EFI-variables data
+
+Behavior:
+
+- This program comes to use some common functions shared with varstored.
+- This program is launched by xapi, it is executed in a sandboxed and reduced privileges environment.
+- Xapi retrieves VM's NVRAM content from database and passes it to this program via command-line arguments.
+- If this program outputs `update_required`, xapi sets `VM.secureboot_certificates_state` to be `update_available`.
+- If this program outputs `update_ok`, xapi sets `VM.secureboot_certificates_state` to be `ok`.
+- On toolstack restart, during DB upgrade, this program is invoked to compute `VM.secureboot_certificates_state`. Since xapi process has not completed initialization at that point, this program cannot call any services of xapi.
+
+### 3.6 Boot-time Automatic Update Path
+
+When varstored initializes a VM and sees `secureboot_certificates_state=update_on_boot`, varstored does,
+
+- Perform certificate update flow during boot-time initialization
+- Write updated NVRAM and synchronize state via `VM.set_NVRAM_EFI_variables_V2`
+
+The `VM.set_NVRAM_EFI_variables_V2` interface performs same as `VM.set_NVRAM_EFI_variables`, uses the existing varstored-guard process to make calls to xapi.
+
+If `VM.set_NVRAM_EFI_variables_V2` runs into error (e.g. there is something wrong with the communication with xapi),
+
+- xapi does not update VM NVRAM and `VM.secureboot_certificates_state`
+- VM boot gets stuck at the firmware initialization stage, if the issue is not fixed, rebooting the VM will still encounter the same problem
+- Once the issue is fixed, admin can continue the secureboot certificate upgrade by VM reboot
+
+### 3.7 End-to-end Workflow
+
+1. Upgrade packages (`xapi-core`, `varstored`, related components)
+2. Restart toolstack
+3. xapi DB upgrade initializes and recalculates `secureboot_certificates_state`
+4. Admin marks selected VMs via `VM.update_secureboot_certificates_on_boot`
+5. VM reboot triggers varstored certificate update
+6. xapi updates state to reflect post-update NVRAM content
+
+## 4. Out of Scope
+
+- User-notification mechanism for certificate expiry
+- Custom certificate workflow
+- Template/snapshot feature expansion beyond state tracking and conversion behavior
+- OS-specific test-process guidance
+- VM with Secure Boot PCR7 binding (e.g. Windows bitlocker), provide customer documentation to guide how to resolve such issues
diff --git a/doc/content/xapi/cli/_index.md b/doc/content/xapi/cli/_index.md
@@ -156,7 +156,7 @@ So each function receives a printer for sending text output to the xe client, an
       let mac = List.assoc_default "mac" params "" in
       let network = Client.Network.get_by_uuid rpc session_id network in
       let pifs = List.assoc "pif-uuids" params in
-      let uuids = String.split ',' pifs in
+      let uuids = String.split_on_char ',' pifs in
       let pifs = List.map (fun uuid -> Client.PIF.get_by_uuid rpc session_id uuid) uuids in
       let mode = Record_util.bond_mode_of_string (List.assoc_default "mode" params "") in
       let properties = read_map_params "properties" params in

diff --git a/dune-project b/dune-project
@@ -179,6 +179,8 @@
   (xapi-types
    (= :version))
   (xapi-stdext-zerocheck
+    (= :version))
+  (xapi-work-queues
     (= :version)))
  (synopsis "A CLI for xapi storage services")
  (description
@@ -191,7 +193,8 @@
  (name xapi-schema))
 
 (package
-  (name xapi-work-queues))
+ (name xapi-work-queues)
+ (depends ppx_deriving_rpc xapi-stdext-threads))
 
 (package
  (name rrdd-plugin)
@@ -327,6 +330,7 @@
   xapi-stdext-pervasives
   xapi-stdext-unix
   xapi-stdext-zerocheck
+  xapi-work-queues
   xen-api-client
   xen-api-client-lwt
   xenctrl
@@ -365,6 +369,7 @@
   rrdd-plugin
   xapi-stdext-std
   xapi-tracing-export
+  xapi-work-queues
   xen-api-client
   (alcotest :with-test)
   (ppx_deriving_rpc :with-test)
@@ -483,6 +488,8 @@
    (= :version))
   (xapi-types
    (= :version))
+  (xapi-work-queues
+   (= :version))
   (xen-api-client-lwt
    (= :version))
   xenctrl ; for quicktest

diff --git a/ocaml/database/parse_db_conf.ml b/ocaml/database/parse_db_conf.ml
@@ -13,7 +13,6 @@
  *)
 (* !!! This needs to be moved out of xapi and into the database directory; probably being merged with db_connections !!! *)
 
-open Xapi_stdext_std.Xstringext
 open Xapi_stdext_unix
 
 module D = Debug.Make (struct let name = "parse_db_conf" end)
@@ -110,7 +109,7 @@ let parse_db_conf s =
     let conf = Unixext.string_of_file s in
     let lines : string list ref = ref [] in
     let consume_line () = lines := List.tl !lines in
-    lines := String.split '\n' conf ;
+    lines := String.split_on_char '\n' conf ;
     List.iter (fun line -> debug "%s" line) !lines ;
     let read_block () =
       let path_line = List.hd !lines in
@@ -120,7 +119,7 @@ let parse_db_conf s =
       while !lines <> [] && List.hd !lines <> "" do
         let line = List.hd !lines in
         key_values :=
-          ( match String.split ':' line with
+          ( match String.split_on_char ':' line with
           | k :: vs ->
               ( String.lowercase_ascii k
               , String.lowercase_ascii (String.concat ":" vs)

diff --git a/ocaml/database/redo_log.ml b/ocaml/database/redo_log.ml
@@ -12,7 +12,6 @@
  * GNU Lesser General Public License for more details.
  *)
 open Xapi_stdext_pervasives.Pervasiveext
-open Xapi_stdext_std.Xstringext
 open Xapi_stdext_unix
 
 let with_lock = Xapi_stdext_threads.Threadext.Mutex.execute

diff --git a/ocaml/doc/dune b/ocaml/doc/dune
@@ -1,7 +1,7 @@
 (executable
   (modes exe)
   (name jsapi)
-  (libraries    
+  (libraries
     mustache
     rpclib.core
     rpclib.json
@@ -10,7 +10,6 @@
     xapi-consts
     xapi-datamodel
     xapi-stdext-pervasives
-    xapi-stdext-std
     xapi-stdext-unix
   )
   (preprocess (pps ppx_deriving_rpc))

diff --git a/ocaml/doc/jsapi.ml b/ocaml/doc/jsapi.ml
@@ -12,7 +12,6 @@
  * GNU Lesser General Public License for more details.
  *)
 
-open Xapi_stdext_std.Xstringext
 open Xapi_stdext_pervasives.Pervasiveext
 module Unixext = Xapi_stdext_unix.Unixext
 open Datamodel_types