run: execute jobs exactly like services

`docker compose run <job>` materializes the job — a ContainerSpec + WorkloadSpec, the same layers a service is made of — as a service for the one-off machinery: its profile is activated, the project narrowed to its dependencies by WithSelectedJob, and the exit code flows back. The materialization happens during project loading, BEFORE service selection and environment resolution: the job's own env_file and label_file resolve exactly like a selected service's would, unrelated services' env_file still don't need to exist, and a target that is neither a service nor a job keeps the precise selection error (the retry is keyed on "the target is a declared job", never on matching an error message). A service and a job may share a name: the service wins, matching what the selector resolved. A job may depend on other jobs: the whole job closure materializes, so every depends_on edge resolves to a runnable service satisfying its declared condition — a dependency job runs to completion through the exact machinery a service dependency does — instead of dangling as an unresolvable name. Job extensions (x-*) survive materialization. Co-Authored-By: Claude Fable 5 <[email protected]> Signed-off-by: Nicolas De Loof <[email protected]>
2026-10-12 02:13:08 +00:00 · 2026-09-21 16:35:20 +02:00
parent 124cbb8eb9
commit eb40156352
5 changed files with 276 additions and 12 deletions
@@ -25,7 +25,6 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
-	"sort"
 	"strconv"
 	"strings"
 	"syscall"
@@ -369,15 +368,6 @@ func (o *ProjectOptions) ToProject(
 		return nil, metrics, err
 	}

-	if jobs := project.AllJobs(); len(jobs) > 0 {
-		names := make([]string, 0, len(jobs))
-		for name := range jobs {
-			names = append(names, name)
-		}
-		sort.Strings(names)
-		logrus.Warnf("jobs are not supported yet and will be ignored: %s", strings.Join(names, ", "))
-	}
-
 	return project, metrics, nil
 }

@@ -19,6 +19,7 @@ package compose
 import (
 	"context"
 	"errors"
+	"fmt"
 	"os"
 	"strings"

@@ -71,6 +72,11 @@ type runOptions struct {
 }

 func (options runOptions) apply(project *types.Project) (*types.Project, error) {
+	project, err := materializeManualJob(project, options.Service)
+	if err != nil {
+		return nil, err
+	}
+
 	if options.noDeps {
 		var err error
 		project, err = project.WithSelectedServices([]string{options.Service}, types.IgnoreDependencies)
@@ -83,7 +89,6 @@ func (options runOptions) apply(project *types.Project) (*types.Project, error)
 	if err != nil {
 		return nil, err
 	}
-
 	target.Tty = !options.noTty
 	target.StdinOpen = options.interactive

@@ -273,7 +278,31 @@ func normalizeRunFlags(f *pflag.FlagSet, name string) pflag.NormalizedName {
 func runProject(ctx context.Context, dockerCli command.Cli, backend api.Compose, p *ProjectOptions, service string) (*types.Project, error) {
 	project, _, err := p.ToProject(ctx, dockerCli, backend, []string{service}, warnUnsupportedAttributes, composecli.WithoutEnvironmentResolution)
 	if err != nil {
-		return nil, err
+		// The run target may be a job — invisible to the service selector.
+		// Only retry unselected for that specific selection failure: any
+		// other load error (a bad include:, an interpolation error, ...)
+		// would only be duplicated by a second full load — remote include:
+		// fetches and unsupported-attribute warnings included — before
+		// falling through to the same, still correct, error anyway.
+		if !isNoSuchServiceErr(err) {
+			return nil, err
+		}
+		// Reload unselected, materialize the job as a service, and narrow to
+		// it, so the env resolution below sees the job like any selected
+		// service (its env_file resolves; unrelated services' env_file still
+		// doesn't need to exist). A target that is not a declared job keeps
+		// the original, precise selection error.
+		unselected, _, uerr := p.ToProject(ctx, dockerCli, backend, nil, warnUnsupportedAttributes, composecli.WithoutEnvironmentResolution)
+		if uerr != nil {
+			return nil, err
+		}
+		if _, isJob := unselected.AllJobs()[service]; !isJob {
+			return nil, err
+		}
+		project, err = materializeManualJob(unselected, service)
+		if err != nil {
+			return nil, err
+		}
 	}
 	project, err = project.WithServicesEnvironmentResolved(true)
 	if err != nil {
@@ -284,6 +313,13 @@ func runProject(ctx context.Context, dockerCli command.Cli, backend api.Compose,
 	return project, nil
 }

+// isNoSuchServiceErr reports whether err is compose-go's selection failure
+// for a service/job name that isn't part of the (possibly narrowed)
+// project — as opposed to any other project-load error.
+func isNoSuchServiceErr(err error) bool {
+	return strings.Contains(err.Error(), "no such service")
+}
+
 func runRun(ctx context.Context, backend api.Compose, project *types.Project, options runOptions, createOpts createOptions, buildOpts buildOptions, dockerCli command.Cli) error {
 	project, err := options.apply(project)
 	if err != nil {
@@ -366,3 +402,82 @@ func runRun(ctx context.Context, backend api.Compose, project *types.Project, op
 	}
 	return err
 }
+
+// materializeManualJob lets run target a manual-trigger job exactly like a
+// service. A job is a ContainerSpec+WorkloadSpec — the same layers a service
+// is made of — so it materializes as a service for the one-off machinery:
+// its profile is activated and the project narrowed to its dependencies by
+// WithSelectedJob, then the job joins Services under its own name.
+func materializeManualJob(project *types.Project, name string) (*types.Project, error) {
+	// jobs and services share the depends_on namespace but not their own: a
+	// service with the target's name wins — it is what the service selector
+	// resolved — and a job already materialized must not be re-materialized
+	// (it would shed whatever resolution ran on it since).
+	if _, exists := project.Services[name]; exists {
+		return project, nil
+	}
+	job, ok := project.AllJobs()[name]
+	if !ok {
+		return project, nil
+	}
+	if job.Triggers == nil || job.Triggers.Manual == nil || !*job.Triggers.Manual {
+		return nil, fmt.Errorf("job %q has no manual trigger, it cannot be run", name)
+	}
+	project, err := project.WithSelectedJob(name)
+	if err != nil {
+		return nil, err
+	}
+	// A job may depend on other jobs: materialize the whole job closure so
+	// every depends_on reference resolves to a service — the dependency job
+	// runs through the exact machinery a service dependency does (a
+	// run-to-completion container satisfying its declared condition),
+	// instead of dangling as an unresolvable name.
+	jobs := project.AllJobs()
+	materializeJobClosure(project, jobs, job, map[string]bool{name: true})
+	project.Services[name] = jobAsService(project, name, job)
+	return project, nil
+}
+
+// materializeJobClosure adds every job reachable through job-typed
+// depends_on edges to project.Services. seen carries the starting job and
+// guards against dependency cycles.
+func materializeJobClosure(project *types.Project, jobs types.Jobs, job types.JobConfig, seen map[string]bool) {
+	for dep := range job.DependsOn {
+		if seen[dep] {
+			continue
+		}
+		seen[dep] = true
+		depJob, isJob := jobs[dep]
+		if !isJob {
+			continue
+		}
+		materializeJobClosure(project, jobs, depJob, seen)
+		project.Services[dep] = jobAsService(project, dep, depJob)
+	}
+}
+
+// jobAsService materializes a job as a service for the one-off machinery: a
+// job is a ContainerSpec+WorkloadSpec, the same layers a service is made of.
+// It carries the standard custom labels the loader stamps on every service —
+// materialization happens after loading, so without them the containers
+// created for a dependency job would be invisible to every label-driven
+// path: start would silently skip them, ps/down would not see them, and the
+// dependency wait would report the job as a missing dependency.
+func jobAsService(project *types.Project, name string, job types.JobConfig) types.ServiceConfig {
+	svc := types.ServiceConfig{
+		Name:          name,
+		Profiles:      job.Profiles,
+		Extensions:    job.Extensions,
+		ContainerSpec: job.ContainerSpec,
+		WorkloadSpec:  job.WorkloadSpec,
+	}
+	svc.CustomLabels = types.Labels{
+		api.ProjectLabel:     project.Name,
+		api.ServiceLabel:     name,
+		api.VersionLabel:     api.ComposeVersion,
+		api.WorkingDirLabel:  project.WorkingDir,
+		api.ConfigFilesLabel: strings.Join(project.ComposeFiles, ","),
+		api.OneoffLabel:      "False",
+	}
+	return svc
+}
@@ -0,0 +1,124 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package compose
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/compose-spec/compose-go/v2/types"
+	"gotest.tools/v3/assert"
+)
+
+func TestMaterializeManualJob(t *testing.T) {
+	yes := true
+	base := func() *types.Project {
+		return &types.Project{
+			Services: types.Services{
+				"db": {Name: "db", ContainerSpec: types.ContainerSpec{Image: "postgres"}},
+			},
+			Jobs: types.Jobs{
+				"migrate": {
+					Name:          "migrate",
+					Triggers:      &types.TriggerConfig{Manual: &yes},
+					ContainerSpec: types.ContainerSpec{Image: "migrator", Command: types.ShellCommand{"migrate"}},
+					WorkloadSpec:  types.WorkloadSpec{DependsOn: types.DependsOnConfig{"db": {Condition: types.ServiceConditionStarted, Required: true}}},
+				},
+				"backup": {
+					Name:     "backup",
+					Triggers: &types.TriggerConfig{Schedule: []types.ScheduleConfig{{Cron: "0 3 * * *"}}},
+				},
+				"prep": {
+					Name:          "prep",
+					Triggers:      &types.TriggerConfig{Manual: &yes},
+					ContainerSpec: types.ContainerSpec{Image: "prep-tool"},
+					WorkloadSpec:  types.WorkloadSpec{DependsOn: types.DependsOnConfig{"db": {Condition: types.ServiceConditionStarted, Required: true}}},
+				},
+				"deploy": {
+					Name:          "deploy",
+					Triggers:      &types.TriggerConfig{Manual: &yes},
+					Extensions:    types.Extensions{"x-team": "platform"},
+					ContainerSpec: types.ContainerSpec{Image: "deployer"},
+					WorkloadSpec:  types.WorkloadSpec{DependsOn: types.DependsOnConfig{"prep": {Condition: types.ServiceConditionCompletedSuccessfully, Required: true}}},
+				},
+			},
+		}
+	}
+
+	t.Run("a service name passes through", func(t *testing.T) {
+		p := base()
+		got, err := materializeManualJob(p, "db")
+		assert.NilError(t, err)
+		assert.Equal(t, got, p)
+	})
+
+	t.Run("a manual job materializes as a service with its spec and deps", func(t *testing.T) {
+		got, err := materializeManualJob(base(), "migrate")
+		assert.NilError(t, err)
+		svc, err := got.GetService("migrate")
+		assert.NilError(t, err)
+		assert.Equal(t, svc.Image, "migrator")
+		assert.DeepEqual(t, []string(svc.Command), []string{"migrate"})
+		_, hasDep := svc.DependsOn["db"]
+		assert.Check(t, hasDep)
+		// the dependency is kept in the narrowed project
+		_, err = got.GetService("db")
+		assert.NilError(t, err)
+	})
+
+	t.Run("a job dependency on another job materializes the whole closure", func(t *testing.T) {
+		got, err := materializeManualJob(base(), "deploy")
+		assert.NilError(t, err)
+		// deploy -> job prep -> service db: prep becomes a runnable service
+		// so the depends_on edge resolves, and db is kept for prep
+		prep, err := got.GetService("prep")
+		assert.NilError(t, err)
+		assert.Equal(t, prep.Image, "prep-tool")
+		_, err = got.GetService("db")
+		assert.NilError(t, err)
+		deploy, err := got.GetService("deploy")
+		assert.NilError(t, err)
+		assert.Equal(t, deploy.Extensions["x-team"], "platform", "job extensions survive materialization")
+	})
+
+	t.Run("an existing service with the target name wins over a job", func(t *testing.T) {
+		p := base()
+		p.Services["migrate"] = types.ServiceConfig{Name: "migrate", ContainerSpec: types.ContainerSpec{Image: "the-service"}}
+		got, err := materializeManualJob(p, "migrate")
+		assert.NilError(t, err)
+		svc, err := got.GetService("migrate")
+		assert.NilError(t, err)
+		assert.Equal(t, svc.Image, "the-service", "a same-named service must not be shadowed by the job")
+	})
+
+	t.Run("a schedule-only job is rejected", func(t *testing.T) {
+		_, err := materializeManualJob(base(), "backup")
+		assert.Error(t, err, `job "backup" has no manual trigger, it cannot be run`)
+	})
+}
+
+// runProject only retries the unselected/full load to look for a job when
+// the narrowed load failed specifically because the target isn't a known
+// service — any other load error (a bad include:, an interpolation
+// error, ...) must not trigger that retry, since it would only duplicate
+// side effects (remote include: fetches, unsupported-attribute warnings)
+// before falling through to the same, unrecoverable error anyway.
+func TestIsNoSuchServiceErr(t *testing.T) {
+	assert.Assert(t, isNoSuchServiceErr(errors.New("no such service: migrate")))
+	assert.Assert(t, !isNoSuchServiceErr(errors.New("interpolation error: bad substitution")))
+	assert.Assert(t, !isNoSuchServiceErr(errors.New("include: remote resource fetch failed")))
+}
@@ -245,6 +245,7 @@ func runUp(
 	if err := rejectScheduledJobs(project); err != nil {
 		return err
 	}
+	warnIgnoredJobs(project)

 	err := createOptions.Apply(project)
 	if err != nil {
@@ -355,6 +356,21 @@ func runUp(
 	})
 }

+// warnIgnoredJobs names the declared jobs up will not act on: manual jobs
+// wait for an explicit `compose run <job>` trigger.
+func warnIgnoredJobs(project *types.Project) {
+	jobs := project.Jobs
+	if len(jobs) == 0 {
+		return
+	}
+	names := make([]string, 0, len(jobs))
+	for name := range jobs {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	logrus.Warnf("jobs are not started by up; trigger them with `docker compose run`: %s", strings.Join(names, ", "))
+}
+
 // rejectScheduledJobs refuses to bring a project up when it declares active
 // scheduled jobs: silently not scheduling them would break the user's
 // expectations, unlike manual jobs which simply wait for an explicit trigger.
@@ -29,6 +29,7 @@ import (
 	"github.com/compose-spec/compose-go/v2/loader"
 	"github.com/compose-spec/compose-go/v2/types"
 	"github.com/docker/cli/cli/streams"
+	logrustest "github.com/sirupsen/logrus/hooks/test"
 	"go.uber.org/mock/gomock"
 	"gotest.tools/v3/assert"

@@ -194,3 +195,21 @@ func TestRejectScheduledJobs(t *testing.T) {
 	err := rejectScheduledJobs(&types.Project{Jobs: types.Jobs{"backup": scheduled, "sync": scheduled, "migrate": manual}})
 	assert.Error(t, err, "scheduled jobs are not supported in this version: backup, sync")
 }
+
+// warnIgnoredJobs must only name profile-enabled jobs: a job disabled by
+// profile selection isn't part of this invocation and up never reaching it
+// isn't worth a warning — unlike rejectScheduledJobs, which already gets
+// this right via project.Jobs.
+func TestWarnIgnoredJobs(t *testing.T) {
+	hook := logrustest.NewGlobal()
+
+	warnIgnoredJobs(&types.Project{
+		Jobs:         types.Jobs{"migrate": {}},
+		DisabledJobs: types.Jobs{"backup": {}},
+	})
+
+	assert.Equal(t, len(hook.AllEntries()), 1)
+	msg := hook.LastEntry().Message
+	assert.Assert(t, strings.Contains(msg, "migrate"), msg)
+	assert.Assert(t, !strings.Contains(msg, "backup"), msg)
+}