Skip to content

[Bug]: Restarted container cannot fetch its port #2580

Closed
@artuross

Description

@artuross

Testcontainers version

0.31.0

Using the latest Testcontainers version?

Yes

Host OS

MacOS

Host arch

Apple Silicon

Go version

1.22.1

Docker version

Client:
 Cloud integration: v1.0.35+desktop.5
 Version:           24.0.6
 API version:       1.43
 Go version:        go1.20.7
 Git commit:        ed223bc
 Built:             Mon Sep  4 12:28:49 2023
 OS/Arch:           darwin/arm64
 Context:           desktop-linux

Server: Docker Desktop 4.25.2 (129061)
 Engine:
  Version:          24.0.6
  API version:      1.43 (minimum version 1.12)
  Go version:       go1.20.7
  Git commit:       1a79695
  Built:            Mon Sep  4 12:31:36 2023
  OS/Arch:          linux/arm64
  Experimental:     false
 containerd:
  Version:          1.6.22
  GitCommit:        8165feabfdfe38c65b599c4993d227328c231fca
 runc:
  Version:          1.1.8
  GitCommit:        v1.1.8-0-g82f18fe
 docker-init:
  Version:          0.19.0
  GitCommit:        de40ad0

Docker info

Client:
 Version:    24.0.6
 Context:    desktop-linux
 Debug Mode: false
 Plugins:
  buildx: Docker Buildx (Docker Inc.)
    Version:  v0.11.2-desktop.5
    Path:     /Users/artur/.docker/cli-plugins/docker-buildx
  compose: Docker Compose (Docker Inc.)
    Version:  v2.23.0-desktop.1
    Path:     /Users/artur/.docker/cli-plugins/docker-compose
  dev: Docker Dev Environments (Docker Inc.)
    Version:  v0.1.0
    Path:     /Users/artur/.docker/cli-plugins/docker-dev
  extension: Manages Docker extensions (Docker Inc.)
    Version:  v0.2.20
    Path:     /Users/artur/.docker/cli-plugins/docker-extension
  init: Creates Docker-related starter files for your project (Docker Inc.)
    Version:  v0.1.0-beta.9
    Path:     /Users/artur/.docker/cli-plugins/docker-init
  sbom: View the packaged-based Software Bill Of Materials (SBOM) for an image (Anchore Inc.)
    Version:  0.6.0
    Path:     /Users/artur/.docker/cli-plugins/docker-sbom
  scan: Docker Scan (Docker Inc.)
    Version:  v0.26.0
    Path:     /Users/artur/.docker/cli-plugins/docker-scan
  scout: Docker Scout (Docker Inc.)
    Version:  v1.0.9
    Path:     /Users/artur/.docker/cli-plugins/docker-scout

Server:
 Containers: 3
  Running: 3
  Paused: 0
  Stopped: 0
 Images: 73
 Server Version: 24.0.6
 Storage Driver: overlay2
  Backing Filesystem: extfs
  Supports d_type: true
  Using metacopy: false
  Native Overlay Diff: true
  userxattr: false
 Logging Driver: json-file
 Cgroup Driver: cgroupfs
 Cgroup Version: 2
 Plugins:
  Volume: local
  Network: bridge host ipvlan macvlan null overlay
  Log: awslogs fluentd gcplogs gelf journald json-file local logentries splunk syslog
 Swarm: inactive
 Runtimes: io.containerd.runc.v2 runc
 Default Runtime: runc
 Init Binary: docker-init
 containerd version: 8165feabfdfe38c65b599c4993d227328c231fca
 runc version: v1.1.8-0-g82f18fe
 init version: de40ad0
 Security Options:
  seccomp
   Profile: unconfined
  cgroupns
 Kernel Version: 6.4.16-linuxkit
 Operating System: Docker Desktop
 OSType: linux
 Architecture: aarch64
 CPUs: 2
 Total Memory: 2.853GiB
 Name: linuxkit-c6f47efeb708
 ID: f825f2b6-85df-4465-9bde-0d443fd771f3
 Docker Root Dir: /var/lib/docker
 Debug Mode: false
 HTTP Proxy: http.docker.internal:3128
 HTTPS Proxy: http.docker.internal:3128
 No Proxy: hubproxy.docker.internal
 Experimental: false
 Insecure Registries:
  hubproxy.docker.internal:5555
  127.0.0.0/8
 Live Restore Enabled: false

WARNING: daemon is not using the default seccomp profile

What happened?

I have a following code to create container and setup database for me

func createPostgresRepository(t *testing.T) *postgresContainer {
	t.Helper()

	ctx, cancelCtx := context.WithTimeout(context.Background(), 120*time.Second)

	dbName := "repository-test"
	dbUser := "postgres"
	dbPassword := "postgres"

	container, err := postgres.RunContainer(
		ctx,
		testcontainers.WithLogger(testcontainers.TestLogger(t)),
		testcontainers.WithImage("docker.io/postgres:16.1-alpine3.19"),
		postgres.WithPassword(dbPassword),
		postgres.WithUsername(dbUser),
		postgres.WithDatabase(dbName),
		postgres.WithInitScripts("../../../sql/schema.sql"),
		testcontainers.WithWaitStrategy(
			wait.ForLog("database system is ready to accept connections").
				WithOccurrence(2).
				WithStartupTimeout(60*time.Second),
		),
	)
	require.NoError(t, err, "failed to start container")

	err = container.Snapshot(ctx)
	require.NoError(t, err, "failed to snapshot container")

	dsn, err := container.ConnectionString(ctx)
	require.NoError(t, err, "failed to get connection string")

	dbConn, err := pgxpool.New(ctx, dsn)
	require.NoError(t, err, "failed to create new pgx conn")

	dbRepo := db.New(dbConn)

	conn := &dbConn
	repository := &dbRepo

	stopContainer := func(ctx context.Context) {
		dbConn.Reset()

		duration := 30 * time.Second
		err := container.Stop(ctx, &duration)
		require.NoError(t, err, "failed to stop container")
	}

	startContainer := func(ctx context.Context) {
		dbConn.Reset()

		err := container.Start(ctx)
		require.NoError(t, err, "failed to start container")

		var dsn string
		for i := 0; i < 30; i++ {
			dsn, err = container.ConnectionString(ctx)
			if err == nil {
				break
			}

			t.Logf("failed to get connection string: %v\n", err)
			time.Sleep(time.Second)
		}

		newConn, err := pgxpool.New(ctx, dsn)
		require.NoError(t, err, "failed to create new pgx conn")

		newRepository := db.New(newConn)

		*conn = newConn
		*repository = newRepository
	}

	terminateContainer := func(ctx context.Context) {
		dbConn.Reset()

		err := container.Terminate(ctx)
		require.NoError(t, err, "failed to terminate container")
	}

	resetState := func(ctx context.Context) {
		dbConn.Reset()

		err := container.Restore(ctx)
		require.NoError(t, err, "failed to restore container to initial state")
	}

	config := &postgresContainer{
		ctx:                ctx,
		cancelCtx:          cancelCtx,
		container:          container,
		resetState:         resetState,
		terminateContainer: terminateContainer,
		stopContainer:      stopContainer,
		startContainer:     startContainer,
		dsn:                dsn,
		conn:               conn,
		repository:         repository,
	}

	return config
}

In one test case, I stop the container and start it on cleanup (done with startContainer function). This behaviour is inconsistent, but about 50% executions fail on the call to container.ConnectionString(ctx) with error port not found.

Initially, I thought that the container was simply not up yet, but as you can see, I added a for loop to try few times. Additionally, I can see a port mapped when looking at container via Docker Desktop (also can check logs and can confirm that the container is ready).

Interestingly, I've never had this happen when I wasn't restarting container (creating new container on each test case).

Relevant log output

=== RUN   TestRepository_GetUser
2024/06/13 01:32:48 github.com/testcontainers/testcontainers-go - Connected to docker:
  Server Version: 24.0.6
  API Version: 1.43
  Operating System: Docker Desktop
  Total Memory: 2921 MB
  Resolved Docker Host: unix:///var/run/docker.sock
  Resolved Docker Socket Path: /var/run/docker.sock
  Test SessionID: 91cad614ad1661b177ab16ae205fb9c6ca6680c0eba68fec7196489bd1cbd0ce
  Test ProcessID: 316e6da6-c0aa-4efe-a2e8-ee62257dba29
    /local/path/db/lifecycle.go:60: 🐳 Creating container for image testcontainers/ryuk:0.7.0
    /local/path/db/lifecycle.go:66: ✅ Container created: 4aba2e7819b4
    /local/path/db/lifecycle.go:72: 🐳 Starting container: 4aba2e7819b4
    /local/path/db/lifecycle.go:78: ✅ Container started: 4aba2e7819b4
    /local/path/db/lifecycle.go:209: 🚧 Waiting for container id 4aba2e7819b4 image: testcontainers/ryuk:0.7.0. Waiting for: &{Port:8080/tcp timeout:<nil> PollInterval:100ms}
    /local/path/db/lifecycle.go:84: 🔔 Container is ready: 4aba2e7819b4
    /local/path/db/lifecycle.go:60: 🐳 Creating container for image docker.io/postgres:16.1-alpine3.19
    /local/path/db/lifecycle.go:66: ✅ Container created: 365dd194af94
    /local/path/db/lifecycle.go:72: 🐳 Starting container: 365dd194af94
    /local/path/db/lifecycle.go:78: ✅ Container started: 365dd194af94
    /local/path/db/lifecycle.go:209: 🚧 Waiting for container id 365dd194af94 image: docker.io/postgres:16.1-alpine3.19. Waiting for: &{timeout:<nil> deadline:0x140003fc298 Strategies:[0x140003f4540]}
    /local/path/db/lifecycle.go:84: 🔔 Container is ready: 365dd194af94
=== RUN   TestRepository_GetUser/errors
=== RUN   TestRepository_GetUser/errors/generic
    /local/path/db/lifecycle.go:90: 🐳 Stopping container: 365dd194af94
    /local/path/db/lifecycle.go:96: ✅ Container stopped: 365dd194af94
    /local/path/db/lifecycle.go:72: 🐳 Starting container: 365dd194af94
    /local/path/db/lifecycle.go:78: ✅ Container started: 365dd194af94
    /local/path/db/lifecycle.go:209: 🚧 Waiting for container id 365dd194af94 image: docker.io/postgres:16.1-alpine3.19. Waiting for: &{timeout:<nil> deadline:0x140003fc298 Strategies:[0x140003f4540]}
    /local/path/db/lifecycle.go:84: 🔔 Container is ready: 365dd194af94
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
    /local/path/db/user_test.go:312: failed to get connection string: port not found
panic: test timed out after 30s
running tests:

Additional information

No response

Metadata

Metadata

Assignees

No one assigned

    Labels

    bugAn issue with the library

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions