Skip to content

Pull images used in volumes with type=image #13811

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Divyanshupandey007 wants to merge 1 commit into
base: main
Choose a base branch
from
+418 −69
Open

Pull images used in volumes with type=image #13811

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions pkg/api/api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -870,10 +870,9 @@ func GetImageNameOrDefault(service types.ServiceConfig, projectName string) stri
}

// GetDependentImages returns the additional images a service depends on beyond
// its main image. Currently this is the set of pre_start hook images, which run
// as ephemeral init containers with their own image. A hook without an explicit
// image, or one reusing the service image, is skipped as that image is already
// accounted for as the service image.
// its main image. This includes pre_start hook images and type=image volume
// sources. Images that match the service image are skipped as they are already
// accounted for.
func GetDependentImages(service types.ServiceConfig, projectName string) []string {
serviceImage := GetImageNameOrDefault(service, projectName)
var images []string
Expand All @@ -882,5 +881,10 @@ func GetDependentImages(service types.ServiceConfig, projectName string) []strin
images = append(images, hook.Image)
}
}
for _, vol := range service.Volumes {
if vol.Type == types.VolumeTypeImage && vol.Source != serviceImage {
images = append(images, vol.Source)
}
}
return images
}
52 changes: 52 additions & 0 deletions pkg/api/api_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,58 @@ func TestGetDependentImages(t *testing.T) {
},
expected: nil,
},
{
name: "type=image volume source is collected",
service: types.ServiceConfig{
ContainerSpec: types.ContainerSpec{
Image: "nginx:alpine",
Volumes: []types.ServiceVolumeConfig{
{Type: types.VolumeTypeImage, Source: "myorg/assets:latest", Target: "/srv/static"},
},
},
},
expected: []string{"myorg/assets:latest"},
},
{
name: "type=image volume reusing the service image is ignored",
service: types.ServiceConfig{
ContainerSpec: types.ContainerSpec{
Image: "myorg/shared:latest",
Volumes: []types.ServiceVolumeConfig{
{Type: types.VolumeTypeImage, Source: "myorg/shared:latest", Target: "/data"},
},
},
},
expected: nil,
},
{
name: "non-image volume types are ignored",
service: types.ServiceConfig{
ContainerSpec: types.ContainerSpec{
Image: "alpine:3.20",
Volumes: []types.ServiceVolumeConfig{
{Type: types.VolumeTypeBind, Source: "/host/path", Target: "/container"},
{Type: types.VolumeTypeVolume, Source: "myvolume", Target: "/data"},
},
},
},
expected: nil,
},
{
name: "hook and volume images are both collected",
service: types.ServiceConfig{
PreStart: []types.PreStartHook{
{ContainerSpec: types.ContainerSpec{Image: "init:latest", Command: types.ShellCommand{"echo", "init"}}},
},
ContainerSpec: types.ContainerSpec{
Image: "alpine:3.20",
Volumes: []types.ServiceVolumeConfig{
{Type: types.VolumeTypeImage, Source: "myorg/data:1.0", Target: "/data"},
},
},
},
expected: []string{"init:latest", "myorg/data:1.0"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
88 changes: 51 additions & 37 deletions pkg/compose/pull.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ func (s *composeService) pull(ctx context.Context, project *types.Project, opts

err = p.pullServiceImages(ctx)
if err == nil {
err = p.pullHookImages(ctx)
err = p.pullDependentImages(ctx)
}
if err != nil {
// join already-scheduled pulls before returning: bailing out with
Expand Down Expand Up @@ -135,10 +135,12 @@ func (p *imagePuller) pullServiceImages(ctx context.Context) error {
}
if !pullRequired {
p.eventSkippedPull("Image "+service.Image, skipReason)
p.scheduled[service.Image] = service.Name
continue
}
if service.Build != nil && p.opts.IgnoreBuildable {
p.eventSkippedPull("Image "+service.Image, "Image can be built")
p.scheduled[service.Image] = service.Name
continue
}
if _, ok := p.scheduled[service.Image]; ok {
Expand Down Expand Up @@ -179,19 +181,22 @@ func (p *imagePuller) runServicePull(ctx context.Context, idx int, service types
return nil
}

// pullHookImages schedules pulls for pre_start hook images, which run as
// ephemeral init containers with their own registry image. They have no pull
// policy of their own, so we inherit the parent service's policy for skip
// decisions — through the same shouldPullImage decision as the service image.
// Unlike the service image, a hook image can't be built, so `build` falls
// back to pull-if-missing instead of exempting it from pulling.
func (p *imagePuller) pullHookImages(ctx context.Context) error {
// pullDependentImages schedules pulls for dependent images (pre_start hooks
// and type=image volume sources). They have no pull policy of their own, so we
// inherit the parent service's policy for skip decisions — through the same
// shouldPullImage decision as the service image. A dependent image that is
// produced by a buildable service in the project is skipped when
// --ignore-buildable is set, consistent with how service images are handled.
func (p *imagePuller) pullDependentImages(ctx context.Context) error {
for name, service := range p.project.Services {
hookPolicy := service.PullPolicy
if hookPolicy == types.PullPolicyBuild {
hookPolicy = types.PullPolicyMissing
}
for _, img := range api.GetDependentImages(service, p.project.Name) {
if isServiceImage(img, p.project.Services) {
continue
}
pullRequired, skipReason, err := shouldPullImage(types.ServiceConfig{Name: name, ContainerSpec: types.ContainerSpec{Image: img, PullPolicy: hookPolicy}}, p.images)
if err != nil {
return err
Expand All @@ -202,15 +207,23 @@ func (p *imagePuller) pullHookImages(ctx context.Context) error {
}
continue
}
if p.opts.IgnoreBuildable && isServiceImageToBuild(types.ServiceConfig{ContainerSpec: types.ContainerSpec{Image: img}}, p.project.Services) {
p.eventSkippedPull("Image "+img, "Image can be built")
continue
}
if _, ok := p.scheduled[img]; ok {
continue
}
p.scheduled[img] = name
hookService := types.ServiceConfig{Name: name, ContainerSpec: types.ContainerSpec{Image: img}}
depService := types.ServiceConfig{Name: name, ContainerSpec: types.ContainerSpec{Image: img}}
p.eg.Go(func() error {
err := p.pullServiceImage(ctx, hookService, p.opts.Quiet, p.project.Environment["DOCKER_DEFAULT_PLATFORM"])
err := p.pullServiceImage(ctx, depService, p.opts.Quiet, p.project.Environment["DOCKER_DEFAULT_PLATFORM"])
if err != nil && !p.opts.IgnoreFailures {
// fail fast: a hook image can't be built as a fallback
if isServiceImageToBuild(depService, p.project.Services) {
// image can be built, so we can ignore pull failure
return nil
}
// fail fast: dependent image can't be built as a fallback
return err
}
return nil
Expand Down Expand Up @@ -420,24 +433,13 @@ func (s *composeService) pullRequiredImages(ctx context.Context, project *types.
}
if pull {
needPull[name] = service
scheduled[service.Image] = true
}
for i, vol := range service.Volumes {
if vol.Type == types.VolumeTypeImage {
if _, ok := images[vol.Source]; !ok {
// Hack: create a fake ServiceConfig so we pull missing volume image
n := fmt.Sprintf("%s:volume %d", name, i)
needPull[n] = types.ServiceConfig{
Name: n,
ContainerSpec: types.ContainerSpec{Image: vol.Source},
}
scheduled[vol.Source] = true
}
}
if service.Image != "" {
scheduled[service.Image] = true
}
}

if err := addPreStartHookPulls(project, images, needPull, scheduled); err != nil {
if err := addDependentImagePulls(project, images, needPull, scheduled); err != nil {
return err
}

Expand Down Expand Up @@ -504,16 +506,13 @@ func (s *composeService) resolvePulledImages(ctx context.Context, needPull map[s
return errs
}

// addPreStartHookPulls schedules pulls for pre_start hook images.
// pre_start hooks run as ephemeral init containers with their own registry
// image; they have no pull policy of their own, so they inherit the parent
// service's — interpreted by the same mustPull the service image goes
// through, so `up` and `compose pull` agree on hook images too. A hook image
// can't be built, so `build` falls back to pull-if-missing instead of
// exempting it from pulling; only `never` skips entirely. Running as a second
// pass over the services keeps the dedup against service/volume images (via
// scheduled) independent of service iteration order.
func addPreStartHookPulls(project *types.Project, images map[string]api.ImageSummary, needPull map[string]types.ServiceConfig, scheduled map[string]bool) error {
// addDependentImagePulls schedules pulls for dependent images (pre_start hooks
// and type=image volume sources) that are not yet present locally. Dependent
// images that are produced by a buildable service in the project are skipped,
// as they will be created during the build step. Running as a second pass over
// the services keeps the dedup against service images (via scheduled)
// independent of service iteration order.
func addDependentImagePulls(project *types.Project, images map[string]api.ImageSummary, needPull map[string]types.ServiceConfig, scheduled map[string]bool) error {
for name, service := range project.Services {
if service.PullPolicy == types.PullPolicyNever {
continue
Expand All @@ -523,6 +522,12 @@ func addPreStartHookPulls(project *types.Project, images map[string]api.ImageSum
hookPolicy = types.PullPolicyMissing
}
for i, img := range api.GetDependentImages(service, project.Name) {
if isServiceImage(img, project.Services) {
continue
}
if isServiceImageToBuild(types.ServiceConfig{ContainerSpec: types.ContainerSpec{Image: img}}, project.Services) {
continue
}
pull, err := mustPull(types.ServiceConfig{Name: name, ContainerSpec: types.ContainerSpec{Image: img, PullPolicy: hookPolicy}}, images)
if err != nil {
return err
Expand All @@ -531,8 +536,8 @@ func addPreStartHookPulls(project *types.Project, images map[string]api.ImageSum
continue
}
scheduled[img] = true
// Hack: create a fake ServiceConfig so we pull missing pre_start hook image
n := fmt.Sprintf("%s:pre_start %d", name, i)
// Hack: create a fake ServiceConfig so we pull the dependent image
n := fmt.Sprintf("%s:dependent %d", name, i)
needPull[n] = types.ServiceConfig{
Name: n,
ContainerSpec: types.ContainerSpec{Image: img},
Expand Down Expand Up @@ -571,6 +576,15 @@ func mustPull(service types.ServiceConfig, images map[string]api.ImageSummary) (
}
}

func isServiceImage(image string, services types.Services) bool {
for _, svc := range services {
if svc.Image == image {
return true
}
}
return false
}

func isServiceImageToBuild(service types.ServiceConfig, services types.Services) bool {
if service.Build != nil {
return true
Expand Down
Loading