router: Save PlaceHolder nic for VR if network does not have source nat (#3902)

This PR aims to fix the issue below

Create a network offering for isolated network, services: Dns/Dhcp/Userdata, and enable it
create a isolated network with the new offering
create a vm
check the guest IP of virtual router,
restart network with cleanup
check the guest IP of new virtual router
The IP in step4 and step6 should be the same, but they are different actually.

Author: ustcweizhou

api/src/main/java/com/cloud/vm/NicProfile.java

@@ -423,6 +423,7 @@ public String toString() {
                 .append(iPv4Address)
                 .append("-")
                 .append(broadcastUri)
+                .append("]")
                 .toString();
     }
-}
+}

server/src/main/java/com/cloud/network/NetworkServiceImpl.java

@@ -2438,6 +2438,9 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
                                         // log assign usage events for new offering
                                         List<NicVO> nics = _nicDao.listByNetworkId(networkId);
                                         for (NicVO nic : nics) {
+                                            if (nic.getReservationStrategy() == Nic.ReservationStrategy.PlaceHolder) {
+                                                continue;
+                                            }
                                             long vmId = nic.getInstanceId();
                                             VMInstanceVO vm = _vmDao.findById(vmId);
                                             if (vm == null) {

server/src/main/java/com/cloud/network/guru/ExternalGuestNetworkGuru.java

@@ -30,6 +30,8 @@
 import com.cloud.network.IpAddressManager;
 import com.cloud.network.Network;
 import com.cloud.network.Network.GuestType;
+import com.cloud.network.Network.Provider;
+import com.cloud.network.Network.Service;
 import com.cloud.network.Network.State;
 import com.cloud.network.Networks.BroadcastDomainType;
 import com.cloud.network.PhysicalNetwork;
@@ -51,10 +53,12 @@
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.net.Ip;
 import com.cloud.utils.net.NetUtils;
+import com.cloud.vm.Nic;
 import com.cloud.vm.Nic.ReservationStrategy;
 import com.cloud.vm.NicProfile;
 import com.cloud.vm.NicVO;
 import com.cloud.vm.ReservationContext;
+import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachineProfile;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
@@ -261,6 +265,17 @@ public NicProfile allocate(Network config, NicProfile nic, VirtualMachineProfile
             profile.setIPv4Netmask(null);
         }
 
+        if (config.getVpcId() == null && vm.getType() == VirtualMachine.Type.DomainRouter) {
+            boolean isPublicNetwork = _networkModel.isProviderSupportServiceInNetwork(config.getId(), Service.SourceNat, Provider.VirtualRouter);
+            if (!isPublicNetwork) {
+                Nic placeholderNic = _networkModel.getPlaceholderNicForRouter(config, null);
+                if (placeholderNic == null) {
+                    s_logger.debug("Saving placeholder nic with ip4 address " + profile.getIPv4Address() +
+                            " and ipv6 address " + profile.getIPv6Address() + " for the network " + config);
+                    _networkMgr.savePlaceholderNic(config, profile.getIPv4Address(), profile.getIPv6Address(), VirtualMachine.Type.DomainRouter);
+                }
+            }
+        }
         return profile;
     }
 

server/src/main/java/com/cloud/network/guru/GuestNetworkGuru.java

@@ -75,6 +75,7 @@
 import com.cloud.utils.db.TransactionStatus;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.net.NetUtils;
+import com.cloud.vm.Nic;
 import com.cloud.vm.Nic.ReservationStrategy;
 import com.cloud.vm.NicProfile;
 import com.cloud.vm.ReservationContext;
@@ -372,15 +373,25 @@ public NicProfile allocate(final Network network, NicProfile nic, final VirtualM
 
                 if (isGateway) {
                     guestIp = network.getGateway();
-                } else if (vm.getVirtualMachine().getType() == VirtualMachine.Type.DomainRouter) {
-                    guestIp = _ipAddrMgr.acquireGuestIpAddressByPlacement(network, nic.getRequestedIPv4());
                 } else {
-                    guestIp = _ipAddrMgr.acquireGuestIpAddress(network, nic.getRequestedIPv4());
-                }
-
-                if (!isGateway && guestIp == null && network.getGuestType() != GuestType.L2 && !_networkModel.listNetworkOfferingServices(network.getNetworkOfferingId()).isEmpty()) {
-                    throw new InsufficientVirtualNetworkCapacityException("Unable to acquire Guest IP" + " address for network " + network, DataCenter.class,
-                            dc.getId());
+                    if (network.getGuestType() != GuestType.L2 && vm.getType() == VirtualMachine.Type.DomainRouter) {
+                        Nic placeholderNic = _networkModel.getPlaceholderNicForRouter(network, null);
+                        if (placeholderNic != null) {
+                            s_logger.debug("Nic got an ip address " + placeholderNic.getIPv4Address() + " stored in placeholder nic for the network " + network);
+                            guestIp = placeholderNic.getIPv4Address();
+                        }
+                    }
+                    if (guestIp == null) {
+                        if (vm.getVirtualMachine().getType() == VirtualMachine.Type.DomainRouter) {
+                            guestIp = _ipAddrMgr.acquireGuestIpAddressByPlacement(network, nic.getRequestedIPv4());
+                        } else {
+                            guestIp = _ipAddrMgr.acquireGuestIpAddress(network, nic.getRequestedIPv4());
+                        }
+                    }
+                    if (guestIp == null && network.getGuestType() != GuestType.L2 && !_networkModel.listNetworkOfferingServices(network.getNetworkOfferingId()).isEmpty()) {
+                        throw new InsufficientVirtualNetworkCapacityException("Unable to acquire Guest IP" + " address for network " + network, DataCenter.class,
+                                dc.getId());
+                    }
                 }
 
                 nic.setIPv4Address(guestIp);

server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java

@@ -710,8 +710,8 @@ public LinkedHashMap<Network, List<? extends NicProfile>> configureGuestNic(fina
         if (guestNetwork != null) {
             s_logger.debug("Adding nic for Virtual Router in Guest network " + guestNetwork);
             String defaultNetworkStartIp = null, defaultNetworkStartIpv6 = null;
+            final Nic placeholder = _networkModel.getPlaceholderNicForRouter(guestNetwork, routerDeploymentDefinition.getPodId());
             if (!routerDeploymentDefinition.isPublicNetwork()) {
-                final Nic placeholder = _networkModel.getPlaceholderNicForRouter(guestNetwork, routerDeploymentDefinition.getPodId());
                 if (guestNetwork.getCidr() != null) {
                     if (placeholder != null && placeholder.getIPv4Address() != null) {
                         s_logger.debug("Requesting ipv4 address " + placeholder.getIPv4Address() + " stored in placeholder nic for the network "
@@ -744,6 +744,9 @@ public LinkedHashMap<Network, List<? extends NicProfile>> configureGuestNic(fina
                         }
                     }
                 }
+            } else if (placeholder != null) {
+                // Remove placeholder nic if router has public network
+                _nicDao.remove(placeholder.getId());
             }
 
             final NicProfile gatewayNic = new NicProfile(defaultNetworkStartIp, defaultNetworkStartIpv6);