feat(NodeAllocator): Specify a container's host IP

Author: jimcadden

src/hosted/NodeAllocator.cc

@@ -80,15 +80,25 @@ ebbrt::NodeAllocator::DockerContainer::~DockerContainer() {
   }
 }
 
+std::string ebbrt::NodeAllocator::DockerContainer::GetIp() {
+  auto cip = RunCmd("docker inspect -f '{{range "
+                    ".NetworkSettings.Networks}}{{.IPAddress}}{{end}}' " +
+                    cid_ );
+  assert(cip != "");
+  return cip;
+}
+
 std::string ebbrt::NodeAllocator::DockerContainer::Start() {
   if (!cid_.empty() || img_.size() == 0) {
     throw std::runtime_error("Error: attempt to start unspecified container ");
   }
   std::stringstream cmd;
-  cmd << "docker run -d " << arg_ << " " << img_ << " " << cmd_;
+  cmd << base_ << " run -d " << arg_ << " " << img_ << " " << cmd_;
   cid_ = RunCmd(cmd.str());
   std::cerr << "Docker Container:" << std::endl;
-  std::cerr << "|  img:" << img_ << std::endl;
+  if( host_ != ""){
+    std::cerr << "| host:" << host_ << std::endl;
+  }
   std::cerr << "|  cid: " << cid_.substr(0, 12) << std::endl;
   std::cerr << "|  log: docker logs " << cid_.substr(0, 12) << std::endl;
   return cid_;
@@ -255,14 +265,11 @@ ebbrt::NodeAllocator::AllocateNode(std::string binary_path,
                         std::to_string(allocation_id);
   std::stringstream docker_args;
   std::stringstream qemu_args;
+
 #ifndef NDEBUG
   docker_args << " --expose 1234 -e DEBUG=true";
 #endif
 
-  if (!args.constraint_node.empty()) {
-    docker_args << " -e constraint:node==" << args.constraint_node << " ";
-  }
-
   if (CustomNetworkNodeArguments.empty()) {
     docker_args << " --net=" << network_id_ << " ";
   } else {
@@ -287,30 +294,27 @@ ebbrt::NodeAllocator::AllocateNode(std::string binary_path,
             << " -append \"" << cmdline_ << ";allocid=" << allocation_id
             << "\"";
 
-  auto c = DockerContainer(repo, docker_args.str(), qemu_args.str());
+  auto c = DockerContainer(repo, docker_args.str(), qemu_args.str(), args.constraint_node);
   auto id = c.Start();
+  auto cip = c.GetIp(); /* get IP of container */
 
   nodes_.insert(std::make_pair(std::string(id), std::move(c)));
   auto rfut = promise_map_[allocation_id].GetFuture();
 
-  /* get IP of container */
-  auto cip = RunCmd("docker inspect -f '{{range "
-                    ".NetworkSettings.Networks}}{{.IPAddress}}{{end}}' " +
-                    id);
-  assert(id != "");
-
   /* transfer image into container */
   RunCmd("ping -c 3 -w 30 " + cip);
   RunCmd("nc -z -w 30 " + cip + " 22");
-  RunCmd("scp -q -o UserKnownHostsFile=/dev/null -o "
+  RunCmd("scp -q -o UserKnownHostsFile=/dev/null -o " 
          "StrictHostKeyChecking=no " +
          binary_path + " root@" + cip + ":/root/img.elf");
   /* kick start vm */
-  RunCmd("docker exec -dt " + id + " touch /tmp/signal");
+  RunCmd("ssh -q -o UserKnownHostsFile=/dev/null -o "
+         "StrictHostKeyChecking=no root@"+cip+" 'touch /tmp/signal'");
 
   std::cerr << "Node Allocation Details: " << std::endl;
   std::cerr << "| img: " << binary_path << std::endl;
   std::cerr << "|  id: " << container_name << std::endl;
+  std::cerr << "|  ip: " << cip << std::endl;
 #ifndef NDEBUG
   std::cerr << "# debug w/ gdb: " << std::endl;
   std::cerr << "# gdb " << binary_path.substr(0, binary_path.size() - 2)

src/hosted/NodeAllocator.h

@@ -19,24 +19,33 @@
 namespace ebbrt {
 class NodeAllocator : public StaticSharedEbb<NodeAllocator> {
   struct DockerContainer {
-    DockerContainer() {}
-    DockerContainer(std::string repo,
-                    std::string container_args = std::string(),
-                    std::string run_cmd = std::string())
-        : img_{std::move(repo)}, arg_{std::move(container_args)},
-          cmd_{std::move(run_cmd)} {}
+    DockerContainer() = delete; 
+    DockerContainer(std::string img,
+                    std::string arg = std::string(),
+                    std::string cmd = std::string(), 
+                    std::string host = std::string())
+        : arg_{arg}, cmd_{cmd}, host_{host}, img_{img} {
+            base_ = "docker";
+            if(host_ != ""){
+              base_ += " -H "+host_;
+            }
+          }
     ~DockerContainer();
     DockerContainer(DockerContainer&& other) noexcept /* move constructor */
-        : img_{std::move(other.img_)},
-          arg_{std::move(other.arg_)},
+        : arg_{std::move(other.arg_)},
+          base_{std::move(other.base_)},
+          cid_{std::move(other.cid_)},
           cmd_{std::move(other.cmd_)},
-          cid_{std::move(other.cid_)} {}
+          host_{std::move(other.host_)},
+          img_{std::move(other.img_)} {}
     DockerContainer&
     operator=(DockerContainer&& other) noexcept /* move assignment */ {
-      img_ = std::move(other.img_);
       arg_ = std::move(other.arg_);
-      cmd_ = std::move(other.cmd_);
+      base_ = std::move(other.base_);
       cid_ = std::move(other.cid_);
+      cmd_ = std::move(other.cmd_);
+      host_ = std::move(other.host_);
+      img_ = std::move(other.img_);
       return *this;
     }
     DockerContainer(const DockerContainer& other) =
@@ -45,13 +54,16 @@ class NodeAllocator : public StaticSharedEbb<NodeAllocator> {
     operator=(const DockerContainer& other) = delete; /* copy assignment */
     std::string Start();
     std::string StdOut();
+    std::string GetIp();
     void Stop() {}
 
    private:
-    std::string img_ = std::string();
     std::string arg_ = std::string();
-    std::string cmd_ = std::string();
+    std::string base_ = std::string();
     std::string cid_ = std::string();
+    std::string cmd_ = std::string();
+    std::string host_ = std::string();
+    std::string img_ = std::string();
   };
 
   static const constexpr uint8_t kDefaultCpus = 2;

src/hosted/PoolAllocator.cc

@@ -46,7 +46,6 @@ void ebbrt::PoolAllocator::AllocateNode(int i) {
   std::string node = (specified_nodes) ? nodes_[i % nodes_.size()] 
     : std::string();
 
-  args.constraint_node = node;
   auto nd = ebbrt::node_allocator->AllocateNode(binary_path_, args);
 
   nd.NetworkId().Then(