Handle docker in docker side car container access

Signed-off-by: Dom Del Nano <ddelnano@gmail.com>

Author: ddelnano

src/shared/services/pgtest/pgtest.go

@@ -19,7 +19,11 @@
 package pgtest
 
 import (
+	"bufio"
 	"fmt"
+	"os"
+	"regexp"
+	"time"
 
 	"github.com/golang-migrate/migrate"
 	"github.com/golang-migrate/migrate/database/postgres"
@@ -33,6 +37,27 @@ import (
 	"px.dev/pixie/src/shared/services/pg"
 )
 
+// selfContainerID returns the current Docker container ID by parsing
+// /proc/self/mountinfo. Docker bind-mounts /etc/hostname from
+// /var/lib/docker/containers/<id>/hostname, exposing the container ID.
+// Returns empty string if not running inside a Docker container.
+func selfContainerID() string {
+	f, err := os.Open("/proc/self/mountinfo")
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+
+	re := regexp.MustCompile(`/containers/([a-f0-9]{64})/hostname`)
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		if m := re.FindStringSubmatch(scanner.Text()); m != nil {
+			return m[1]
+		}
+	}
+	return ""
+}
+
 // SetupTestDB sets up a test database instance and applies migrations.
 func SetupTestDB(schemaSource *bindata.AssetSource) (*sqlx.DB, func(), error) {
 	var db *sqlx.DB
@@ -69,18 +94,58 @@ func SetupTestDB(schemaSource *bindata.AssetSource) (*sqlx.DB, func(), error) {
 	if err != nil {
 		return nil, nil, fmt.Errorf("Failed to run docker pool: %w", err)
 	}
-	// Set a 5 minute expiration on resources.
-	err = resource.Expire(300)
+	// Set a 15 minute expiration on resources (extended for debugging).
+	err = resource.Expire(900)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	viper.Set("postgres_port", resource.GetPort("5432/tcp"))
-	viper.Set("postgres_hostname", resource.Container.NetworkSettings.Gateway)
+	// When running inside a container (e.g. CI), the postgres container is on
+	// a different Docker network and we can't reach it via host port mapping.
+	// Detect this and connect postgres to our network instead.
+	pgHost := resource.Container.NetworkSettings.Gateway
+	pgPort := resource.GetPort("5432/tcp")
+	selfID := selfContainerID()
+	log.Infof("selfContainerID: %q", selfID)
+	if selfID != "" {
+		selfContainer, err := pool.Client.InspectContainer(selfID)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to inspect self container %s: %w", selfID, err)
+		}
+		for netName, net := range selfContainer.NetworkSettings.Networks {
+			if netName == "host" {
+				continue
+			}
+			err := pool.Client.ConnectNetwork(net.NetworkID, docker.NetworkConnectionOptions{
+				Container: resource.Container.ID,
+			})
+			if err != nil {
+				return nil, nil, fmt.Errorf("failed to connect postgres to network %s: %w", netName, err)
+			}
+			// Re-inspect to get the postgres container's IP on our network.
+			updated, err := pool.Client.InspectContainer(resource.Container.ID)
+			if err != nil {
+				return nil, nil, fmt.Errorf("failed to re-inspect postgres container: %w", err)
+			}
+			resource.Container = updated
+			if pgNet, ok := updated.NetworkSettings.Networks[netName]; ok {
+				pgHost = pgNet.IPAddress
+				pgPort = "5432"
+				log.Infof("pgHost set to %s:%s via network %s", pgHost, pgPort, netName)
+			}
+			break
+		}
+	}
+	if pgHost == "" {
+		pgHost = "localhost"
+	}
+	viper.Set("postgres_port", pgPort)
+	viper.Set("postgres_hostname", pgHost)
 	viper.Set("postgres_db", dbName)
 	viper.Set("postgres_username", "postgres")
 	viper.Set("postgres_password", "secret")
 
+	pool.MaxWait = 10 * time.Minute
 	if err = pool.Retry(func() error {
 		log.Info("trying to connect")
 		db = pg.MustCreateDefaultPostgresDB()

src/utils/testingutils/docker/elastic.go

@@ -19,14 +19,39 @@
 package docker
 
 import (
+	"bufio"
 	"fmt"
+	"os"
+	"regexp"
+	"time"
 
 	"github.com/olivere/elastic/v7"
 	"github.com/ory/dockertest/v3"
 	"github.com/ory/dockertest/v3/docker"
 	log "github.com/sirupsen/logrus"
 )
 
+// selfContainerID returns the current Docker container ID by parsing
+// /proc/self/mountinfo. Docker bind-mounts /etc/hostname from
+// /var/lib/docker/containers/<id>/hostname, exposing the container ID.
+// Returns empty string if not running inside a Docker container.
+func selfContainerID() string {
+	f, err := os.Open("/proc/self/mountinfo")
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+
+	re := regexp.MustCompile(`/containers/([a-f0-9]{64})/hostname`)
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		if m := re.FindStringSubmatch(scanner.Text()); m != nil {
+			return m[1]
+		}
+	}
+	return ""
+}
+
 func connectElastic(esURL string, esUser string, esPass string) (*elastic.Client, error) {
 	es, err := elastic.NewClient(elastic.SetURL(esURL),
 		elastic.SetBasicAuth(esUser, esPass),
@@ -104,12 +129,52 @@ func SetupElastic() (*elastic.Client, func(), error) {
 		return nil, cleanup, err
 	}
 
-	clientPort := resource.GetPort("9200/tcp")
+	// When running inside a container (e.g. CI), the ES container is on
+	// a different Docker network and we can't reach it via host port mapping.
+	// Detect this and connect ES to our network instead.
+	esHost := resource.Container.NetworkSettings.Gateway
+	esPort := resource.GetPort("9200/tcp")
+	selfID := selfContainerID()
+	log.Infof("selfContainerID: %q", selfID)
+	if selfID != "" {
+		selfContainer, err := pool.Client.InspectContainer(selfID)
+		if err != nil {
+			return nil, cleanup, fmt.Errorf("failed to inspect self container %s: %w", selfID, err)
+		}
+		for netName, net := range selfContainer.NetworkSettings.Networks {
+			if netName == "host" {
+				continue
+			}
+			err := pool.Client.ConnectNetwork(net.NetworkID, docker.NetworkConnectionOptions{
+				Container: resource.Container.ID,
+			})
+			if err != nil {
+				return nil, cleanup, fmt.Errorf("failed to connect ES to network %s: %w", netName, err)
+			}
+			// Re-inspect to get the ES container's IP on our network.
+			updated, err := pool.Client.InspectContainer(resource.Container.ID)
+			if err != nil {
+				return nil, cleanup, fmt.Errorf("failed to re-inspect ES container: %w", err)
+			}
+			resource.Container = updated
+			if esNet, ok := updated.NetworkSettings.Networks[netName]; ok {
+				esHost = esNet.IPAddress
+				esPort = "9200"
+				log.Infof("esHost set to %s:%s via network %s", esHost, esPort, netName)
+			}
+			break
+		}
+	}
+	if esHost == "" {
+		esHost = "localhost"
+	}
+
+	pool.MaxWait = 10 * time.Minute
 	var client *elastic.Client
 	err = pool.Retry(func() error {
 		var err error
 		client, err = connectElastic(fmt.Sprintf("http://%s:%s",
-			resource.Container.NetworkSettings.Gateway, clientPort), "elastic", esPass)
+			esHost, esPort), "elastic", esPass)
 		if err != nil {
 			log.WithError(err).Errorf("Failed to connect to elasticsearch.")
 		}