diff --git a/docs/api/create_docker_image.md b/docs/api/create_docker_image.md index aa2afb3c7..f22f1985e 100644 --- a/docs/api/create_docker_image.md +++ b/docs/api/create_docker_image.md @@ -16,11 +16,19 @@ await futureImage.CreateAsync() .ConfigureAwait(false); ``` +To build a Docker image with Testcontainers, it's important to understand the build context. Testcontainers needs three things: + +1. **Docker build context**: The directory containing files Docker can use during the build +2. **Dockerfile name**: The name of the Dockerfile to use +3. **Dockerfile directory**: Where the Dockerfile is located + !!!tip - The Dockerfile must be part of the build context, otherwise the build fails. + The build context is optional. If you don't specify one, it defaults to the Dockerfile directory. + +Testcontainers creates a tarball with all files and subdirectorys in the build context, incl. the Dockerfile. This tarball is sent to the Docker daemon to build the image. The build context acts as the root for all file operations in the Dockerfile, so all paths (like `COPY` commands) must be relative to it. -It is essential to take into account and comprehend the build context to enable Testcontainers to build the Docker image. Testcontainers generates a tarball that contains all the files and subdirectories within the build context. The tarball is passed to the Docker daemon to build the image. The tarball serves as the new root of the Dockerfile's content. Therefore, all paths must be relative to the new root. If your app or service follows to the following project structure, the build context is `/Users/testcontainers/WeatherForecast/`. +For example, if your project looks like this, the build context would be: `/Users/testcontainers/WeatherForecast/`. / └── Users/ @@ -61,6 +69,17 @@ RUN dotnet publish $SLN_FILE_PATH --configuration Release --framework net6.0 --o ENTRYPOINT ["dotnet", "/app/WeatherForecast.dll"] ``` +### Choosing a build context + +You can use `WithContextDirectory(string)` to set a build context separate from your Dockerfile. This is useful when the Dockerfile is in one directory but the files you want to include are in another. + +```csharp +_ = new ImageFromDockerfileBuilder() + .WithContextDirectory("/path/to/build/context") + .WithDockerfile("Dockerfile") + .WithDockerfileDirectory("/path/to/dockerfile/directory"); +``` + ## Delete multi-stage intermediate layers A multi-stage Docker image build generates intermediate layers that serve as caches. Testcontainers' Resource Reaper is unable to automatically delete these layers after the test execution. The necessary label is not forwarded by the Docker image build. Testcontainers is unable to track the intermediate layers during the test. To delete the intermediate layers after the test execution, pass the Resource Reaper session to each stage. @@ -92,8 +111,9 @@ _ = new ImageFromDockerfileBuilder() | `WithCleanUp` | Will remove the image automatically after all tests have been run. | | `WithLabel` | Applies metadata to the image e.g. `-l`, `--label "testcontainers=awesome"`. | | `WithName` | Sets the image name e.g. `-t`, `--tag "testcontainers:0.1.0"`. | +| `WithContextDirectory` | Sets the Docker build context directory. | | `WithDockerfile` | Sets the name of the `Dockerfile`. | -| `WithDockerfileDirectory` | Sets the build context (directory path that contains the `Dockerfile`). | +| `WithDockerfileDirectory` | Sets the directory path that contains the `Dockerfile`. | | `WithImageBuildPolicy` | Specifies an image build policy to determine when an image is built. | | `WithDeleteIfExists` | Will remove the image if it already exists. | | `WithBuildArgument` | Sets build-time variables e.g `--build-arg "MAGIC_NUMBER=42"`. | diff --git a/src/Testcontainers/Builders/IImageFromDockerfileBuilder`1.cs b/src/Testcontainers/Builders/IImageFromDockerfileBuilder`1.cs index eb6b39ec7..05374082e 100644 --- a/src/Testcontainers/Builders/IImageFromDockerfileBuilder`1.cs +++ b/src/Testcontainers/Builders/IImageFromDockerfileBuilder`1.cs @@ -29,26 +29,35 @@ public interface IImageFromDockerfileBuilder TBuilderEntity WithName(IImage image); /// - /// Sets the Dockerfile. + /// Sets the directory to use as the Docker build context. + /// This is the directory that Docker will use to resolve files referenced in the Dockerfile. /// - /// An absolute path or a name value within the Docker build context. + /// An absolute path or relative name of the directory to use as the Docker build context. + /// A configured instance of . + [PublicAPI] + TBuilderEntity WithContextDirectory(string contextDirectory); + + /// + /// Sets the path to the Dockerfile to use for the build. + /// + /// The filename or path of the Dockerfile. /// A configured instance of . [PublicAPI] TBuilderEntity WithDockerfile(string dockerfile); /// - /// Sets the Dockerfile directory. + /// Sets the directory containing the Dockerfile. /// - /// An absolute path or a name value to the Docker build context. + /// An absolute path or relative path to the directory containing the Dockerfile. /// A configured instance of . [PublicAPI] TBuilderEntity WithDockerfileDirectory(string dockerfileDirectory); /// - /// Sets the Dockerfile directory. + /// Sets the directory containing the Dockerfile. /// /// A common directory path that contains the Dockerfile directory. - /// A relative path or a name value to the Docker build context. + /// An absolute path or relative path to the directory containing the Dockerfile. /// A configured instance of . [PublicAPI] TBuilderEntity WithDockerfileDirectory(CommonDirectoryPath commonDirectoryPath, string dockerfileDirectory); diff --git a/src/Testcontainers/Builders/ImageFromDockerfileBuilder.cs b/src/Testcontainers/Builders/ImageFromDockerfileBuilder.cs index b341c114c..2a4f881e6 100644 --- a/src/Testcontainers/Builders/ImageFromDockerfileBuilder.cs +++ b/src/Testcontainers/Builders/ImageFromDockerfileBuilder.cs @@ -63,6 +63,12 @@ public ImageFromDockerfileBuilder WithName(IImage image) return Merge(DockerResourceConfiguration, new ImageFromDockerfileConfiguration(image: image.ApplyHubImageNamePrefix())); } + /// + public ImageFromDockerfileBuilder WithContextDirectory(string contextDirectory) + { + return Merge(DockerResourceConfiguration, new ImageFromDockerfileConfiguration(contextDirectory: contextDirectory)); + } + /// public ImageFromDockerfileBuilder WithDockerfile(string dockerfile) { diff --git a/src/Testcontainers/Clients/TestcontainersClient.cs b/src/Testcontainers/Clients/TestcontainersClient.cs index 4ad2bfaf0..cf93eac58 100644 --- a/src/Testcontainers/Clients/TestcontainersClient.cs +++ b/src/Testcontainers/Clients/TestcontainersClient.cs @@ -372,6 +372,7 @@ public async Task BuildAsync(IImageFromDockerfileConfiguration configura if (configuration.ImageBuildPolicy(cachedImage)) { var dockerfileArchive = new DockerfileArchive( + configuration.ContextDirectory, configuration.DockerfileDirectory, configuration.Dockerfile, configuration.Image, diff --git a/src/Testcontainers/Configurations/Images/IImageFromDockerfileConfiguration.cs b/src/Testcontainers/Configurations/Images/IImageFromDockerfileConfiguration.cs index d97da133e..706860f46 100644 --- a/src/Testcontainers/Configurations/Images/IImageFromDockerfileConfiguration.cs +++ b/src/Testcontainers/Configurations/Images/IImageFromDockerfileConfiguration.cs @@ -17,6 +17,11 @@ public interface IImageFromDockerfileConfiguration : IResourceConfiguration bool? DeleteIfExists { get; } + /// + /// Gets the context directory. + /// + string ContextDirectory { get; } + /// /// Gets the Dockerfile. /// diff --git a/src/Testcontainers/Configurations/Images/ImageFromDockerfileConfiguration.cs b/src/Testcontainers/Configurations/Images/ImageFromDockerfileConfiguration.cs index 145ed26c8..ee059ee9b 100644 --- a/src/Testcontainers/Configurations/Images/ImageFromDockerfileConfiguration.cs +++ b/src/Testcontainers/Configurations/Images/ImageFromDockerfileConfiguration.cs @@ -15,6 +15,7 @@ internal sealed class ImageFromDockerfileConfiguration : ResourceConfiguration /// Initializes a new instance of the class. /// + /// The context directory. /// The Dockerfile. /// The Dockerfile directory. /// The target. @@ -23,6 +24,7 @@ internal sealed class ImageFromDockerfileConfiguration : ResourceConfigurationA list of build arguments. /// A value indicating whether Testcontainers removes an existing image or not. public ImageFromDockerfileConfiguration( + string contextDirectory = null, string dockerfile = null, string dockerfileDirectory = null, string target = null, @@ -31,6 +33,7 @@ public ImageFromDockerfileConfiguration( IReadOnlyDictionary buildArguments = null, bool? deleteIfExists = null) { + ContextDirectory = contextDirectory; Dockerfile = dockerfile; DockerfileDirectory = dockerfileDirectory; Target = target; @@ -66,6 +69,7 @@ public ImageFromDockerfileConfiguration(IImageFromDockerfileConfiguration resour public ImageFromDockerfileConfiguration(IImageFromDockerfileConfiguration oldValue, IImageFromDockerfileConfiguration newValue) : base(oldValue, newValue) { + ContextDirectory = BuildConfiguration.Combine(oldValue.ContextDirectory, newValue.ContextDirectory); Dockerfile = BuildConfiguration.Combine(oldValue.Dockerfile, newValue.Dockerfile); DockerfileDirectory = BuildConfiguration.Combine(oldValue.DockerfileDirectory, newValue.DockerfileDirectory); Target = BuildConfiguration.Combine(oldValue.Target, newValue.Target); @@ -79,6 +83,10 @@ public ImageFromDockerfileConfiguration(IImageFromDockerfileConfiguration oldVal [JsonIgnore] public bool? DeleteIfExists { get; } + /// + [JsonIgnore] + public string ContextDirectory { get; } + /// [JsonIgnore] public string Dockerfile { get; } diff --git a/src/Testcontainers/Images/DockerfileArchive.cs b/src/Testcontainers/Images/DockerfileArchive.cs index 27189e3ca..5cbffed8c 100644 --- a/src/Testcontainers/Images/DockerfileArchive.cs +++ b/src/Testcontainers/Images/DockerfileArchive.cs @@ -11,6 +11,7 @@ namespace DotNet.Testcontainers.Images using DotNet.Testcontainers.Configurations; using ICSharpCode.SharpZipLib.Tar; using Microsoft.Extensions.Logging; + using JetBrains.Annotations; /// /// Generates a tar archive with Docker configuration files. The tar archive can be used to build a Docker image. @@ -23,10 +24,14 @@ internal sealed class DockerfileArchive : ITarArchive private static readonly Regex VariablePattern = new Regex("\\$(\\{(?[A-Za-z_][A-Za-z0-9_]*)\\}|(?[A-Za-z_][A-Za-z0-9_]*))", RegexOptions.None, TimeSpan.FromSeconds(1)); + private readonly DirectoryInfo _contextDirectory; + private readonly DirectoryInfo _dockerfileDirectory; private readonly FileInfo _dockerfile; + private readonly FileInfo _dockerignore; + private readonly IImage _image; private readonly IReadOnlyDictionary _buildArguments; @@ -36,6 +41,7 @@ internal sealed class DockerfileArchive : ITarArchive /// /// Initializes a new instance of the class. /// + /// Directory to Docker build context. /// Directory to Docker configuration files. /// Name of the Dockerfile, which is necessary to start the Docker build. /// Docker image information to create the tar archive for. @@ -43,14 +49,20 @@ internal sealed class DockerfileArchive : ITarArchive /// The logger. /// Thrown when the Dockerfile directory does not exist or the directory does not contain a Dockerfile. public DockerfileArchive( - string dockerfileDirectory, - string dockerfile, - IImage image, - IReadOnlyDictionary buildArguments, - ILogger logger) + [CanBeNull] string contextDirectory, + [NotNull] string dockerfileDirectory, + [NotNull] string dockerfile, + [NotNull] IImage image, + [NotNull] IReadOnlyDictionary buildArguments, + [NotNull] ILogger logger) : this( + // The Docker build context wasn't originally supported. To stay backwards + // compatible, the argument is optional and can be null. If it isn't set, + // fall back to the Dockerfile directory. + new DirectoryInfo(contextDirectory ?? dockerfileDirectory), new DirectoryInfo(dockerfileDirectory), - new FileInfo(dockerfile), + new FileInfo(Path.Combine(dockerfileDirectory, dockerfile)), + new FileInfo(Path.Combine(dockerfileDirectory, dockerfile + ".dockerignore")), image, buildArguments, logger) @@ -60,31 +72,37 @@ public DockerfileArchive( /// /// Initializes a new instance of the class. /// + /// Directory to Docker build context. /// Directory to Docker configuration files. /// Name of the Dockerfile, which is necessary to start the Docker build. + /// Name of the .dockerignore file. /// Docker image information to create the tar archive for. /// Docker build arguments. /// The logger. /// Thrown when the Dockerfile directory does not exist or the directory does not contain a Dockerfile. - public DockerfileArchive( - DirectoryInfo dockerfileDirectory, - FileInfo dockerfile, - IImage image, - IReadOnlyDictionary buildArguments, - ILogger logger) + private DockerfileArchive( + [NotNull] DirectoryInfo contextDirectory, + [NotNull] DirectoryInfo dockerfileDirectory, + [NotNull] FileInfo dockerfile, + [NotNull] FileInfo dockerignore, + [NotNull] IImage image, + [NotNull] IReadOnlyDictionary buildArguments, + [NotNull] ILogger logger) { if (!dockerfileDirectory.Exists) { throw new ArgumentException($"Directory '{dockerfileDirectory.FullName}' does not exist."); } - if (dockerfileDirectory.GetFiles(dockerfile.ToString(), SearchOption.TopDirectoryOnly).Length == 0) + if (!dockerfile.Exists) { - throw new ArgumentException($"{dockerfile} does not exist in '{dockerfileDirectory.FullName}'."); + throw new ArgumentException($"{dockerfile.Name} does not exist in '{dockerfileDirectory.FullName}'."); } + _contextDirectory = contextDirectory; _dockerfileDirectory = dockerfileDirectory; _dockerfile = dockerfile; + _dockerignore = dockerignore; _image = image; _buildArguments = buildArguments; _logger = logger; @@ -111,7 +129,7 @@ public IEnumerable GetBaseImages() const string valueGroup = "value"; - var lines = File.ReadAllLines(Path.Combine(_dockerfileDirectory.FullName, _dockerfile.ToString())) + var lines = File.ReadAllLines(_dockerfile.FullName) .Select(line => line.Trim()) .Where(line => !string.IsNullOrEmpty(line)) .Where(line => !line.StartsWith("#", StringComparison.Ordinal)) @@ -157,15 +175,15 @@ public IEnumerable GetBaseImages() /// public async Task Tar(CancellationToken ct = default) { - var dockerfileDirectoryPath = Unix.Instance.NormalizePath(_dockerfileDirectory.FullName); + var dockerIgnoreFileName = _dockerignore.Exists ? _dockerignore.Name : ".dockerignore"; - var dockerfileFilePath = Unix.Instance.NormalizePath(_dockerfile.ToString()); + var dockerIgnoreFile = new DockerIgnoreFile(_dockerfileDirectory, dockerIgnoreFileName, _dockerfile.Name, _logger); var dockerfileArchiveFileName = Regex.Replace(_image.FullName, "[^a-zA-Z0-9]", "-", RegexOptions.None, TimeSpan.FromSeconds(1)).ToLowerInvariant(); var dockerfileArchiveFilePath = Path.Combine(Path.GetTempPath(), $"{dockerfileArchiveFileName}.tar"); - var dockerIgnoreFile = new DockerIgnoreFile(dockerfileDirectoryPath, ".dockerignore", dockerfileFilePath, _logger); + var baseDirectoryLength = _contextDirectory.FullName.TrimEnd(Path.DirectorySeparatorChar).Length + 1; using (var tarOutputFileStream = new FileStream(dockerfileArchiveFilePath, FileMode.Create, FileAccess.Write)) { @@ -173,53 +191,81 @@ public async Task Tar(CancellationToken ct = default) { tarOutputStream.IsStreamOwner = false; - foreach (var absoluteFilePath in GetFiles(dockerfileDirectoryPath)) + foreach (var absoluteFilePath in GetFiles(_contextDirectory.FullName)) { // SharpZipLib drops the root path: https://github.com/icsharpcode/SharpZipLib/pull/582. - var relativeFilePath = absoluteFilePath.Substring(dockerfileDirectoryPath.TrimEnd(Path.AltDirectorySeparatorChar).Length + 1); + var relativeFilePath = absoluteFilePath.Substring(baseDirectoryLength); if (dockerIgnoreFile.Denies(relativeFilePath)) { continue; } - try + // If the build context already has a `Dockerfile`, we need to ignore it and + // instead use the one from the specified Dockerfile directory, which might be + // different. + if (_dockerfile.Name.Equals(relativeFilePath, StringComparison.Ordinal)) { - using (var inputStream = new FileStream(absoluteFilePath, FileMode.Open, FileAccess.Read)) - { - var entry = TarEntry.CreateTarEntry(relativeFilePath); - entry.TarHeader.Size = inputStream.Length; - entry.TarHeader.Mode = GetUnixFileMode(absoluteFilePath); - - await tarOutputStream.PutNextEntryAsync(entry, ct) - .ConfigureAwait(false); - - await inputStream.CopyToAsync(tarOutputStream, 81920, ct) - .ConfigureAwait(false); - - await tarOutputStream.CloseEntryAsync(ct) - .ConfigureAwait(false); - } - } - catch (IOException e) - { - throw new IOException("Cannot create Docker image tar archive.", e); + continue; } + + await AddAsync(absoluteFilePath, relativeFilePath, tarOutputStream) + .ConfigureAwait(false); } + + await AddAsync(_dockerfile.FullName, _dockerfile.Name, tarOutputStream) + .ConfigureAwait(false); } } return dockerfileArchiveFilePath; + + async Task AddAsync(string absoluteFilePath, string relativeFilePath, TarOutputStream tarOutputStream) + { + const int bufferSize = 4096; + + try + { + using (var stream = new FileStream( + absoluteFilePath, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize, + FileOptions.Asynchronous | FileOptions.SequentialScan)) + { + var fileMode = GetUnixFileMode(absoluteFilePath); + + var tarEntry = new TarEntry(new TarHeader()); + tarEntry.TarHeader.Name = relativeFilePath; + tarEntry.TarHeader.Mode = fileMode; + tarEntry.Size = stream.Length; + + await tarOutputStream.PutNextEntryAsync(tarEntry, ct) + .ConfigureAwait(false); + + await stream.CopyToAsync(tarOutputStream, bufferSize, ct) + .ConfigureAwait(false); + + await tarOutputStream.CloseEntryAsync(ct) + .ConfigureAwait(false); + } + } + catch (IOException e) + { + throw new IOException("Cannot create Docker image tar archive.", e); + } + } } /// /// Gets all accepted Docker archive files. /// - /// Directory to Docker configuration files. + /// Directory to Docker configuration files. /// Returns a list with all accepted Docker archive files. - private static IEnumerable GetFiles(string directory) + private static IEnumerable GetFiles(string path) { - return Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories) + return Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories) .AsParallel() .Select(Path.GetFullPath) .Select(Unix.Instance.NormalizePath) diff --git a/tests/Testcontainers.Tests/Assets/.dockerignore b/tests/Testcontainers.Tests/Assets/.dockerignore index 71b2c46ad..013108d08 100644 --- a/tests/Testcontainers.Tests/Assets/.dockerignore +++ b/tests/Testcontainers.Tests/Assets/.dockerignore @@ -1,6 +1,7 @@ Dockerfile credHelpers credsStore +context healthWaitStrategy pullBaseImages scratch diff --git a/tests/Testcontainers.Tests/Assets/context/Dockerfile b/tests/Testcontainers.Tests/Assets/context/Dockerfile new file mode 100644 index 000000000..b4449d24a --- /dev/null +++ b/tests/Testcontainers.Tests/Assets/context/Dockerfile @@ -0,0 +1,2 @@ +FROM scratch +COPY docker-entrypoint.sh docker-entrypoint.sh diff --git a/tests/Testcontainers.Tests/Unit/Images/BuildContextTest.cs b/tests/Testcontainers.Tests/Unit/Images/BuildContextTest.cs new file mode 100644 index 000000000..c9faf137f --- /dev/null +++ b/tests/Testcontainers.Tests/Unit/Images/BuildContextTest.cs @@ -0,0 +1,63 @@ +namespace DotNet.Testcontainers.Tests.Unit +{ + using System; + using System.IO; + using System.Threading.Tasks; + using DotNet.Testcontainers.Builders; + using DotNet.Testcontainers.Commons; + using Xunit; + + public sealed class BuildContextTest : IDisposable + { + private readonly string _contextDirectory = Path.Combine(TestSession.TempDirectoryPath, Guid.NewGuid().ToString("D")); + + public BuildContextTest() + { + Directory.CreateDirectory(_contextDirectory); + } + + public void Dispose() + { + Directory.Delete(_contextDirectory, true); + } + + [Fact] + public async Task CreateImageShouldSucceedWhenContextContainsRequiredFiles() + { + // Given + var imageFromDockerfileBuilder = new ImageFromDockerfileBuilder() + .WithContextDirectory(_contextDirectory) + .WithDockerfile("Dockerfile") + .WithDockerfileDirectory("Assets/context/") + .Build(); + + // When + await File.WriteAllBytesAsync(Path.Combine(_contextDirectory, "docker-entrypoint.sh"), Array.Empty(), TestContext.Current.CancellationToken) + .ConfigureAwait(true); + + var exception = await Record.ExceptionAsync(() => imageFromDockerfileBuilder.CreateAsync(TestContext.Current.CancellationToken)) + .ConfigureAwait(true); + + // Then + Assert.Null(exception); + } + + [Fact] + public async Task CreateImageShouldFailWhenContextDoesNotContainRequiredFiles() + { + // Given + var imageFromDockerfileBuilder = new ImageFromDockerfileBuilder() + .WithContextDirectory(_contextDirectory) + .WithDockerfile("Dockerfile") + .WithDockerfileDirectory("Assets/context/") + .Build(); + + // When + var exception = await Record.ExceptionAsync(() => imageFromDockerfileBuilder.CreateAsync(TestContext.Current.CancellationToken)) + .ConfigureAwait(true); + + // Then + Assert.NotNull(exception); + } + } +} diff --git a/tests/Testcontainers.Tests/Unit/Images/ImageFromDockerfileTest.cs b/tests/Testcontainers.Tests/Unit/Images/ImageFromDockerfileTest.cs index b450d88db..c68f0be6e 100644 --- a/tests/Testcontainers.Tests/Unit/Images/ImageFromDockerfileTest.cs +++ b/tests/Testcontainers.Tests/Unit/Images/ImageFromDockerfileTest.cs @@ -38,7 +38,7 @@ public void DockerfileArchiveGetBaseImages() var buildArguments = new Dictionary(); buildArguments.Add("SDK_VERSION_8_0", "8.0.414"); - var dockerfileArchive = new DockerfileArchive("Assets/pullBaseImages/", "Dockerfile", image, buildArguments, NullLogger.Instance); + var dockerfileArchive = new DockerfileArchive(null, "Assets/pullBaseImages/", "Dockerfile", image, buildArguments, NullLogger.Instance); // When var actual = dockerfileArchive.GetBaseImages(); @@ -59,7 +59,7 @@ public async Task DockerfileArchiveTar() var buildArguments = new ReadOnlyDictionary(new Dictionary()); - var dockerfileArchive = new DockerfileArchive("Assets/", "Dockerfile", image, buildArguments, NullLogger.Instance); + var dockerfileArchive = new DockerfileArchive(null, "Assets/", "Dockerfile", image, buildArguments, NullLogger.Instance); var dockerfileArchiveFilePath = await dockerfileArchive.Tar(TestContext.Current.CancellationToken) .ConfigureAwait(true); @@ -120,7 +120,7 @@ public async Task BuildsDockerScratchImage() { // Given var imageFromDockerfileBuilder = new ImageFromDockerfileBuilder() - .WithDockerfileDirectory("Assets/scratch") + .WithDockerfileDirectory("Assets/scratch/") .Build(); // When @@ -177,7 +177,7 @@ public async Task BuildTargetBuildsUpToExpectedTarget() var logger = new TestLogger(); var imageFromDockerfileBuilder = new ImageFromDockerfileBuilder() - .WithDockerfileDirectory("Assets/target") + .WithDockerfileDirectory("Assets/target/") .WithTarget("build") .WithLogger(logger) .Build();