[Merge lcm to lcm feature] For trusted certificates feature (#6940)

Author: minglumlu

ocaml/xapi/import.ml

@@ -2138,12 +2138,16 @@ let check_references ~__context (table : table) =
         (Diagnostic.show_path path)
         (Ref.string_of r)
   in
-  let rec go (clazz, _, r) =
-    match get_snapshot ~clazz ~r with
-    | Some record ->
-        Diagnostic.visit_references check_reference clazz record
-    | _ ->
-        debug "Could not find imported object %s" r
+  let go (clazz, _, r) =
+    (* Ref.null will lead to a get_record failure *)
+    if Ref.of_string r = Ref.null then
+      ()
+    else
+      match get_snapshot ~clazz ~r with
+      | Some record ->
+          Diagnostic.visit_references check_reference clazz record
+      | _ ->
+          debug "Could not find imported object %s" r
   in
   List.iter go table
 

ocaml/xapi/xapi_host.ml

@@ -2149,21 +2149,13 @@ let apply_edition_internal ~__context ~host ~edition ~additional =
     ~additional:new_ed.additional_params
 
 let apply_edition ~__context ~host ~edition ~force =
-  (* if HA is enabled do not allow the edition to be changed *)
-  let pool = Helpers.get_pool ~__context in
-  if
-    Db.Pool.get_ha_enabled ~__context ~self:pool
-    && edition <> Db.Host.get_edition ~__context ~self:host
-  then
-    raise (Api_errors.Server_error (Api_errors.ha_is_enabled, []))
-  else
-    let additional =
-      if force then
-        [("force", "true")]
-      else
-        []
-    in
-    apply_edition_internal ~__context ~host ~edition ~additional
+  let additional =
+    if force then
+      [("force", "true")]
+    else
+      []
+  in
+  apply_edition_internal ~__context ~host ~edition ~additional
 
 let license_add ~__context ~host ~contents =
   let license =

ocaml/xenopsd/lib/topology.ml

@@ -46,6 +46,9 @@ module NUMAResource = struct
 
   let min_memory r1 r2 = {r1 with memfree= min r1.memfree r2.memfree}
 
+  let shrink_memory r mem =
+    {r with memfree= Int64.max 0L (Int64.sub r.memfree mem)}
+
   let pp_dump =
     Fmt.(
       Dump.record

ocaml/xenopsd/lib/topology.mli

@@ -56,6 +56,9 @@ module NUMAResource : sig
   val min_memory : t -> t -> t
   (** [min_memory a b] is [a] with the memory set to the minimum of [a] and [b] *)
 
+  val shrink_memory : t -> int64 -> t
+  (** [shrink_memory a mem] is [a] with the memory set to [max 0 (a-mem)] *)
+
   val pp_dump : t Fmt.t
   (** [pp_dump ppf v] pretty-prints [v] on [ppf] *)
 end
@@ -89,6 +92,8 @@ module NUMA : sig
   (** A NUMA node index. Distinct from an int to avoid mixing with CPU numbers *)
   type node = private Node of int
 
+  module NodeMap : Map.S with type key = node
+
   val make :
        distances:int array array
     -> cpu_to_node:int array

ocaml/xenopsd/lib/xenopsd.ml

@@ -72,6 +72,8 @@ let numa_placement_compat = ref true
 
 let numa_best_effort_prio_mem_only = ref false
 
+let numa_reserve_node0_dmaheap_bits = ref 31
+
 (* O(N^2) operations, until we get a xenstore cache, so use a small number here *)
 let vm_guest_agent_xenstore_quota = ref 128
 
@@ -282,6 +284,12 @@ let options =
        filtered NUMA nodes based on available memory. Only use if there are \
        issues with the new best effort policy"
     )
+  ; ( "numa-reserve-node0-dmaheap-bits"
+    , Arg.Int (fun x -> numa_reserve_node0_dmaheap_bits := x)
+    , (fun () -> string_of_int !numa_reserve_node0_dmaheap_bits)
+    , "Reserve 2^N bytes on node0 in the NUMA planner, similar to how Xen would\n\
+      \  protect the low 4GiB of the memory for the DMA heap"
+    )
   ; ( "pci-quarantine"
     , Arg.Bool (fun b -> pci_quarantine := b)
     , (fun () -> string_of_bool !pci_quarantine)

ocaml/xenopsd/xc/domain.ml

@@ -984,9 +984,134 @@ let numa_hierarchy =
      NUMA.make ~distances ~cpu_to_node ~node_cores
     )
 
+module NUMAResources = struct
+  module DomidMap = Map.Make (Int)
+
+  type node_memusage = int64 Topology.NUMA.NodeMap.t
+
+  type t = {
+      host: Topology.NUMAResource.t Topology.NUMA.NodeMap.t
+    ; estimated: node_memusage DomidMap.t
+    ; outstanding_builds: node_memusage DomidMap.t
+  }
+
+  let node_memusage_of_domidmap node t =
+    DomidMap.fold
+      (fun _ nodemap acc ->
+        Topology.NUMA.NodeMap.find_opt node nodemap
+        |> Option.value ~default:0L
+        |> Int64.add acc
+      )
+      t 0L
+
+  let to_nodes t =
+    let open Topology in
+    t.host
+    |> NUMA.NodeMap.mapi @@ fun node r ->
+       NUMAResource.shrink_memory r
+         (Int64.add
+            (node_memusage_of_domidmap node t.estimated)
+            (node_memusage_of_domidmap node t.outstanding_builds)
+         )
+
+  let of_host nodes =
+    {
+      host= nodes |> Topology.NUMA.NodeMap.of_seq
+    ; estimated= DomidMap.empty
+    ; outstanding_builds= DomidMap.empty
+    }
+
+  let empty =
+    {
+      host= Topology.NUMA.NodeMap.empty
+    ; estimated= DomidMap.empty
+    ; outstanding_builds= DomidMap.empty
+    }
+
+  let pp_dump_nodemap pp_elt =
+    let open Topology.NUMA in
+    Fmt.(Dump.iter_bindings NodeMap.iter (any "nodemap") pp_dump_node pp_elt)
+
+  let pp_dump_domidmap pp_elt =
+    Fmt.(Dump.iter_bindings DomidMap.iter (any "domidmap") int pp_elt)
+
+  let pp_dump_numa_resources =
+    Fmt.Dump.(
+      record
+        [
+          field "host"
+            (fun t -> t.host)
+            (pp_dump_nodemap Topology.NUMAResource.pp_dump)
+        ; field "estimated"
+            (fun t -> t.estimated)
+            (pp_dump_domidmap @@ pp_dump_nodemap Fmt.int64)
+        ; field "outstanding_builds"
+            (fun t -> t.outstanding_builds)
+            (pp_dump_domidmap @@ pp_dump_nodemap Fmt.int64)
+        ]
+    )
+
+  let add_outstanding domid plan t =
+    D.debug "domid %u: adding outstanding NUMA memory usage estimate: %s" domid
+      ((Fmt.to_to_string @@ pp_dump_nodemap Fmt.int64) plan) ;
+    {t with outstanding_builds= DomidMap.add domid plan t.outstanding_builds}
+
+  let remove_outstanding domid t =
+    D.debug "domid %u: removing outstanding NUMA memory usage estimate" domid ;
+    {t with outstanding_builds= DomidMap.remove domid t.outstanding_builds}
+
+  let transfer_outstanding domid t =
+    D.debug "domid %u: transfering outstanding NUMA memory usage estimate" domid ;
+    DomidMap.find_opt domid t.outstanding_builds
+    |> Option.fold ~none:t ~some:(fun plan ->
+        {
+          t with
+          estimated= DomidMap.add domid plan t.estimated
+        ; outstanding_builds= DomidMap.remove domid t.outstanding_builds
+        }
+    )
+
+  let estimate_plan nodes mem plan =
+    let open Topology in
+    let _, _, nodemap =
+      List.fold_left
+        (fun (mem, remaining_nodes, acc) node ->
+          let per_node_usage =
+            Int64.div mem (remaining_nodes |> Int64.of_int)
+          in
+          let r = NUMA.NodeMap.find node nodes in
+          let allocated_from_node =
+            Int64.min per_node_usage r.NUMAResource.memfree
+          in
+          ( Int64.sub mem allocated_from_node
+          , remaining_nodes - 1
+          , NUMA.NodeMap.add node allocated_from_node acc
+          )
+        )
+        (mem, List.length plan, NUMA.NodeMap.empty)
+        plan
+    in
+    nodemap
+end
+
 let numa_mutex = Mutex.create ()
 
-let numa_resources = ref None
+(* protected by numa_mutex *)
+let numa_resources = ref NUMAResources.empty
+
+let numa_reserve_node0 mem =
+  let reserved = Int64.shift_left 1L !Xenopsd.numa_reserve_node0_dmaheap_bits in
+  let mem' = Int64.(max 0L (sub mem reserved)) in
+  (* Prefer nodes other than node0 when node0 is low on memory.
+     This is reserved for the DMA heap in Xen, and even if we try to make a VM
+     use it up, Xen will move some of its memory to other nodes to avoid
+     running out.
+     Without the claim API we have no way of forcing or querying this
+     (and even with the claim API it'd be desirable to avoid running out on
+     node0).
+   *)
+  D.debug "numa_reserve_node0, reducing free memory: %Lu -> %Lu" mem mem' ;
+  mem'
 
 let numa_init () =
   let xcext = Xenctrlext.get_handle () in
@@ -1007,28 +1132,48 @@ let set_affinity = function
   | Xenops_server.Soft ->
       Xenctrlext.vcpu_setaffinity_soft
 
-let numa_placement domid ~vcpus ~cores ~memory affinity =
+let numa_placement domid ~vcpus ~cores ~memory ~required_free affinity =
   let open Xenctrlext in
   let open Topology in
+  let open NUMAResources in
   with_lock numa_mutex (fun () ->
       let ( let* ) = Option.bind in
       let xcext = get_handle () in
       let* host = Lazy.force numa_hierarchy in
       let numa_meminfo = (numainfo xcext).memory |> Array.to_seq in
       let nodes =
         Seq.map2
-          (fun node m -> NUMA.resource host node ~memory:m.memfree)
+          (fun node m ->
+            ( node
+            , let (NUMA.Node nodeid) = node in
+              let memory = m.memfree in
+              let memory =
+                if nodeid = 0 then
+                  numa_reserve_node0 memory
+                else
+                  memory
+              in
+              NUMA.resource host node ~memory
+            )
+          )
           (NUMA.nodes host) numa_meminfo
       in
       let vm = NUMARequest.make ~memory ~vcpus ~cores in
+
+      let previous = !numa_resources in
+      if DomidMap.is_empty previous.outstanding_builds then begin
+        (* no outstanding domain builds: synchronize actual free node memory
+           with host *)
+          numa_resources := of_host nodes ;
+          D.debug "no outstanding domain builds, dropping NUMA usage estimates"
+      end ;
+      D.debug "numa_resources(in): %s"
+      @@ Fmt.to_to_string pp_dump_numa_resources !numa_resources ;
+
+      let estimated_nodes = to_nodes !numa_resources in
       let nodea =
-        match !numa_resources with
-        | None ->
-            Array.of_seq nodes
-        | Some a ->
-            Array.map2 NUMAResource.min_memory (Array.of_seq nodes) a
+        estimated_nodes |> NUMA.NodeMap.to_seq |> Seq.map snd |> Array.of_seq
       in
-      numa_resources := Some nodea ;
       let cpu_affinity, memory_plan =
         match Softaffinity.plan ~vm host nodea with
         | None ->
@@ -1037,6 +1182,12 @@ let numa_placement domid ~vcpus ~cores ~memory affinity =
         | Some (cpu_affinity, mem_plan) ->
             (Some cpu_affinity, mem_plan)
       in
+      numa_resources :=
+        add_outstanding domid
+          (estimate_plan estimated_nodes required_free memory_plan)
+          !numa_resources ;
+      D.debug "numa_resources(out): %s"
+      @@ Fmt.to_to_string pp_dump_numa_resources !numa_resources ;
       let set_vcpu_affinity = function
         | None ->
             D.debug "%s: not setting vcpu affinity for domain %d" __FUNCTION__
@@ -1177,6 +1328,7 @@ let build_pre ~xc ~xs ~vcpus ~memory ~hard_affinity domid =
               in
               numa_placement domid ~vcpus ~cores
                 ~memory:(Int64.mul memory.xen_max_mib 1048576L)
+                ~required_free:(Int64.mul memory.required_host_free_mib 1048576L)
                 affinity
               |> Option.map fst
         )
@@ -1358,6 +1510,11 @@ let build (task : Xenops_task.task_handle) ~xc ~xs ~store_domid ~console_domid
   let target_mib = Memory.mib_of_kib_used target_kib in
   (* Sanity check. *)
   assert (target_mib <= static_max_mib) ;
+  let finally () =
+    with_lock numa_mutex @@ fun () ->
+    numa_resources := NUMAResources.remove_outstanding domid !numa_resources
+  in
+  Fun.protect ~finally @@ fun () ->
   let store_mfn, store_port, console_mfn, console_port, vm_stuff, domain_type =
     match info.priv with
     | BuildHVM hvminfo ->
@@ -1444,6 +1601,10 @@ let build (task : Xenops_task.task_handle) ~xc ~xs ~store_domid ~console_domid
         )
   in
   let local_stuff = console_keys console_port console_mfn in
+  let () =
+    with_lock numa_mutex @@ fun () ->
+    numa_resources := NUMAResources.transfer_outstanding domid !numa_resources
+  in
   build_post ~xc ~xs ~target_mib ~static_max_mib domid domain_type store_mfn
     store_port local_stuff vm_stuff
 

python3/bin/xe-reset-networking

@@ -91,6 +91,7 @@ if __name__ == "__main__":
     parser.add_option("--gateway", help="Gateway for new management interface", dest="gateway", default='')
     parser.add_option("--gateway-v6", help="IPv6 Gateway for new management interface", dest="gateway_v6", default='')
     parser.add_option("--dns", help="DNS server for new management interface", dest="dns", default='')
+    parser.add_option("--keep-interface-name-rules", help="Keep (do NOT reset) the interface name rules", dest="keep_interface_name_rules", action="store_const", const=True, default=False)
     (options, args) = parser.parse_args()
 
     # Determine pool role
@@ -174,6 +175,10 @@ if __name__ == "__main__":
     if not os.access('/tmp/fist_network_reset_no_warning', os.F_OK):
         configuration = []
         configuration.append("Management interface:   " + device)
+        if not options.keep_interface_name_rules:
+            configuration.append("Reset interface name rules: Yes")
+        else:
+            configuration.append("Reset interface name rules: No")
         configuration.append("IP configuration mode:  " + options.mode)
         configuration.append("IPv6 configuration mode:" + options.mode_v6)
         if vlan != None:
@@ -288,8 +293,9 @@ Type 'no' to cancel.
     if rename_script_exists:
         # Reset the domain 0 network interface naming configuration
         # back to a fresh-install state for the currently-installed
-        # hardware.
-        os.system(f"{RENAME_SCRIPT} --reset-to-install")
+        # hardware (unless prevented by the user).
+        if not options.keep_interface_name_rules:
+            os.system(f"{RENAME_SCRIPT} --reset-to-install")
 
     # Reboot
     os.system("mount -o remount,rw / && reboot -f")