diff --git a/cmd/compose/compose.go b/cmd/compose/compose.go index ef8062f14..14757f8dd 100644 --- a/cmd/compose/compose.go +++ b/cmd/compose/compose.go @@ -25,7 +25,6 @@ import ( "os" "os/signal" "path/filepath" - "sort" "strconv" "strings" "syscall" @@ -369,15 +368,6 @@ func (o *ProjectOptions) ToProject( return nil, metrics, err } - if jobs := project.AllJobs(); len(jobs) > 0 { - names := make([]string, 0, len(jobs)) - for name := range jobs { - names = append(names, name) - } - sort.Strings(names) - logrus.Warnf("jobs are not supported yet and will be ignored: %s", strings.Join(names, ", ")) - } - return project, metrics, nil } diff --git a/cmd/compose/run.go b/cmd/compose/run.go index b88f66cf5..33321aaa4 100644 --- a/cmd/compose/run.go +++ b/cmd/compose/run.go @@ -19,6 +19,7 @@ package compose import ( "context" "errors" + "fmt" "os" "strings" @@ -71,6 +72,11 @@ type runOptions struct { } func (options runOptions) apply(project *types.Project) (*types.Project, error) { + project, err := materializeManualJob(project, options.Service) + if err != nil { + return nil, err + } + if options.noDeps { var err error project, err = project.WithSelectedServices([]string{options.Service}, types.IgnoreDependencies) @@ -83,7 +89,6 @@ func (options runOptions) apply(project *types.Project) (*types.Project, error) if err != nil { return nil, err } - target.Tty = !options.noTty target.StdinOpen = options.interactive @@ -273,7 +278,31 @@ func normalizeRunFlags(f *pflag.FlagSet, name string) pflag.NormalizedName { func runProject(ctx context.Context, dockerCli command.Cli, backend api.Compose, p *ProjectOptions, service string) (*types.Project, error) { project, _, err := p.ToProject(ctx, dockerCli, backend, []string{service}, warnUnsupportedAttributes, composecli.WithoutEnvironmentResolution) if err != nil { - return nil, err + // The run target may be a job — invisible to the service selector. + // Only retry unselected for that specific selection failure: any + // other load error (a bad include:, an interpolation error, ...) + // would only be duplicated by a second full load — remote include: + // fetches and unsupported-attribute warnings included — before + // falling through to the same, still correct, error anyway. + if !isNoSuchServiceErr(err) { + return nil, err + } + // Reload unselected, materialize the job as a service, and narrow to + // it, so the env resolution below sees the job like any selected + // service (its env_file resolves; unrelated services' env_file still + // doesn't need to exist). A target that is not a declared job keeps + // the original, precise selection error. + unselected, _, uerr := p.ToProject(ctx, dockerCli, backend, nil, warnUnsupportedAttributes, composecli.WithoutEnvironmentResolution) + if uerr != nil { + return nil, err + } + if _, isJob := unselected.AllJobs()[service]; !isJob { + return nil, err + } + project, err = materializeManualJob(unselected, service) + if err != nil { + return nil, err + } } project, err = project.WithServicesEnvironmentResolved(true) if err != nil { @@ -284,6 +313,13 @@ func runProject(ctx context.Context, dockerCli command.Cli, backend api.Compose, return project, nil } +// isNoSuchServiceErr reports whether err is compose-go's selection failure +// for a service/job name that isn't part of the (possibly narrowed) +// project — as opposed to any other project-load error. +func isNoSuchServiceErr(err error) bool { + return strings.Contains(err.Error(), "no such service") +} + func runRun(ctx context.Context, backend api.Compose, project *types.Project, options runOptions, createOpts createOptions, buildOpts buildOptions, dockerCli command.Cli) error { project, err := options.apply(project) if err != nil { @@ -366,3 +402,82 @@ func runRun(ctx context.Context, backend api.Compose, project *types.Project, op } return err } + +// materializeManualJob lets run target a manual-trigger job exactly like a +// service. A job is a ContainerSpec+WorkloadSpec — the same layers a service +// is made of — so it materializes as a service for the one-off machinery: +// its profile is activated and the project narrowed to its dependencies by +// WithSelectedJob, then the job joins Services under its own name. +func materializeManualJob(project *types.Project, name string) (*types.Project, error) { + // jobs and services share the depends_on namespace but not their own: a + // service with the target's name wins — it is what the service selector + // resolved — and a job already materialized must not be re-materialized + // (it would shed whatever resolution ran on it since). + if _, exists := project.Services[name]; exists { + return project, nil + } + job, ok := project.AllJobs()[name] + if !ok { + return project, nil + } + if job.Triggers == nil || job.Triggers.Manual == nil || !*job.Triggers.Manual { + return nil, fmt.Errorf("job %q has no manual trigger, it cannot be run", name) + } + project, err := project.WithSelectedJob(name) + if err != nil { + return nil, err + } + // A job may depend on other jobs: materialize the whole job closure so + // every depends_on reference resolves to a service — the dependency job + // runs through the exact machinery a service dependency does (a + // run-to-completion container satisfying its declared condition), + // instead of dangling as an unresolvable name. + jobs := project.AllJobs() + materializeJobClosure(project, jobs, job, map[string]bool{name: true}) + project.Services[name] = jobAsService(project, name, job) + return project, nil +} + +// materializeJobClosure adds every job reachable through job-typed +// depends_on edges to project.Services. seen carries the starting job and +// guards against dependency cycles. +func materializeJobClosure(project *types.Project, jobs types.Jobs, job types.JobConfig, seen map[string]bool) { + for dep := range job.DependsOn { + if seen[dep] { + continue + } + seen[dep] = true + depJob, isJob := jobs[dep] + if !isJob { + continue + } + materializeJobClosure(project, jobs, depJob, seen) + project.Services[dep] = jobAsService(project, dep, depJob) + } +} + +// jobAsService materializes a job as a service for the one-off machinery: a +// job is a ContainerSpec+WorkloadSpec, the same layers a service is made of. +// It carries the standard custom labels the loader stamps on every service — +// materialization happens after loading, so without them the containers +// created for a dependency job would be invisible to every label-driven +// path: start would silently skip them, ps/down would not see them, and the +// dependency wait would report the job as a missing dependency. +func jobAsService(project *types.Project, name string, job types.JobConfig) types.ServiceConfig { + svc := types.ServiceConfig{ + Name: name, + Profiles: job.Profiles, + Extensions: job.Extensions, + ContainerSpec: job.ContainerSpec, + WorkloadSpec: job.WorkloadSpec, + } + svc.CustomLabels = types.Labels{ + api.ProjectLabel: project.Name, + api.ServiceLabel: name, + api.VersionLabel: api.ComposeVersion, + api.WorkingDirLabel: project.WorkingDir, + api.ConfigFilesLabel: strings.Join(project.ComposeFiles, ","), + api.OneoffLabel: "False", + } + return svc +} diff --git a/cmd/compose/run_test.go b/cmd/compose/run_test.go new file mode 100644 index 000000000..6fe87abd4 --- /dev/null +++ b/cmd/compose/run_test.go @@ -0,0 +1,124 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "errors" + "testing" + + "github.com/compose-spec/compose-go/v2/types" + "gotest.tools/v3/assert" +) + +func TestMaterializeManualJob(t *testing.T) { + yes := true + base := func() *types.Project { + return &types.Project{ + Services: types.Services{ + "db": {Name: "db", ContainerSpec: types.ContainerSpec{Image: "postgres"}}, + }, + Jobs: types.Jobs{ + "migrate": { + Name: "migrate", + Triggers: &types.TriggerConfig{Manual: &yes}, + ContainerSpec: types.ContainerSpec{Image: "migrator", Command: types.ShellCommand{"migrate"}}, + WorkloadSpec: types.WorkloadSpec{DependsOn: types.DependsOnConfig{"db": {Condition: types.ServiceConditionStarted, Required: true}}}, + }, + "backup": { + Name: "backup", + Triggers: &types.TriggerConfig{Schedule: []types.ScheduleConfig{{Cron: "0 3 * * *"}}}, + }, + "prep": { + Name: "prep", + Triggers: &types.TriggerConfig{Manual: &yes}, + ContainerSpec: types.ContainerSpec{Image: "prep-tool"}, + WorkloadSpec: types.WorkloadSpec{DependsOn: types.DependsOnConfig{"db": {Condition: types.ServiceConditionStarted, Required: true}}}, + }, + "deploy": { + Name: "deploy", + Triggers: &types.TriggerConfig{Manual: &yes}, + Extensions: types.Extensions{"x-team": "platform"}, + ContainerSpec: types.ContainerSpec{Image: "deployer"}, + WorkloadSpec: types.WorkloadSpec{DependsOn: types.DependsOnConfig{"prep": {Condition: types.ServiceConditionCompletedSuccessfully, Required: true}}}, + }, + }, + } + } + + t.Run("a service name passes through", func(t *testing.T) { + p := base() + got, err := materializeManualJob(p, "db") + assert.NilError(t, err) + assert.Equal(t, got, p) + }) + + t.Run("a manual job materializes as a service with its spec and deps", func(t *testing.T) { + got, err := materializeManualJob(base(), "migrate") + assert.NilError(t, err) + svc, err := got.GetService("migrate") + assert.NilError(t, err) + assert.Equal(t, svc.Image, "migrator") + assert.DeepEqual(t, []string(svc.Command), []string{"migrate"}) + _, hasDep := svc.DependsOn["db"] + assert.Check(t, hasDep) + // the dependency is kept in the narrowed project + _, err = got.GetService("db") + assert.NilError(t, err) + }) + + t.Run("a job dependency on another job materializes the whole closure", func(t *testing.T) { + got, err := materializeManualJob(base(), "deploy") + assert.NilError(t, err) + // deploy -> job prep -> service db: prep becomes a runnable service + // so the depends_on edge resolves, and db is kept for prep + prep, err := got.GetService("prep") + assert.NilError(t, err) + assert.Equal(t, prep.Image, "prep-tool") + _, err = got.GetService("db") + assert.NilError(t, err) + deploy, err := got.GetService("deploy") + assert.NilError(t, err) + assert.Equal(t, deploy.Extensions["x-team"], "platform", "job extensions survive materialization") + }) + + t.Run("an existing service with the target name wins over a job", func(t *testing.T) { + p := base() + p.Services["migrate"] = types.ServiceConfig{Name: "migrate", ContainerSpec: types.ContainerSpec{Image: "the-service"}} + got, err := materializeManualJob(p, "migrate") + assert.NilError(t, err) + svc, err := got.GetService("migrate") + assert.NilError(t, err) + assert.Equal(t, svc.Image, "the-service", "a same-named service must not be shadowed by the job") + }) + + t.Run("a schedule-only job is rejected", func(t *testing.T) { + _, err := materializeManualJob(base(), "backup") + assert.Error(t, err, `job "backup" has no manual trigger, it cannot be run`) + }) +} + +// runProject only retries the unselected/full load to look for a job when +// the narrowed load failed specifically because the target isn't a known +// service — any other load error (a bad include:, an interpolation +// error, ...) must not trigger that retry, since it would only duplicate +// side effects (remote include: fetches, unsupported-attribute warnings) +// before falling through to the same, unrecoverable error anyway. +func TestIsNoSuchServiceErr(t *testing.T) { + assert.Assert(t, isNoSuchServiceErr(errors.New("no such service: migrate"))) + assert.Assert(t, !isNoSuchServiceErr(errors.New("interpolation error: bad substitution"))) + assert.Assert(t, !isNoSuchServiceErr(errors.New("include: remote resource fetch failed"))) +} diff --git a/cmd/compose/up.go b/cmd/compose/up.go index 5295032fd..3716b5c8b 100644 --- a/cmd/compose/up.go +++ b/cmd/compose/up.go @@ -245,6 +245,7 @@ func runUp( if err := rejectScheduledJobs(project); err != nil { return err } + warnIgnoredJobs(project) err := createOptions.Apply(project) if err != nil { @@ -355,6 +356,21 @@ func runUp( }) } +// warnIgnoredJobs names the declared jobs up will not act on: manual jobs +// wait for an explicit `compose run ` trigger. +func warnIgnoredJobs(project *types.Project) { + jobs := project.Jobs + if len(jobs) == 0 { + return + } + names := make([]string, 0, len(jobs)) + for name := range jobs { + names = append(names, name) + } + sort.Strings(names) + logrus.Warnf("jobs are not started by up; trigger them with `docker compose run`: %s", strings.Join(names, ", ")) +} + // rejectScheduledJobs refuses to bring a project up when it declares active // scheduled jobs: silently not scheduling them would break the user's // expectations, unlike manual jobs which simply wait for an explicit trigger. diff --git a/cmd/compose/up_test.go b/cmd/compose/up_test.go index e3f28bb91..98ac8949d 100644 --- a/cmd/compose/up_test.go +++ b/cmd/compose/up_test.go @@ -29,6 +29,7 @@ import ( "github.com/compose-spec/compose-go/v2/loader" "github.com/compose-spec/compose-go/v2/types" "github.com/docker/cli/cli/streams" + logrustest "github.com/sirupsen/logrus/hooks/test" "go.uber.org/mock/gomock" "gotest.tools/v3/assert" @@ -194,3 +195,21 @@ func TestRejectScheduledJobs(t *testing.T) { err := rejectScheduledJobs(&types.Project{Jobs: types.Jobs{"backup": scheduled, "sync": scheduled, "migrate": manual}}) assert.Error(t, err, "scheduled jobs are not supported in this version: backup, sync") } + +// warnIgnoredJobs must only name profile-enabled jobs: a job disabled by +// profile selection isn't part of this invocation and up never reaching it +// isn't worth a warning — unlike rejectScheduledJobs, which already gets +// this right via project.Jobs. +func TestWarnIgnoredJobs(t *testing.T) { + hook := logrustest.NewGlobal() + + warnIgnoredJobs(&types.Project{ + Jobs: types.Jobs{"migrate": {}}, + DisabledJobs: types.Jobs{"backup": {}}, + }) + + assert.Equal(t, len(hook.AllEntries()), 1) + msg := hook.LastEntry().Message + assert.Assert(t, strings.Contains(msg, "migrate"), msg) + assert.Assert(t, !strings.Contains(msg, "backup"), msg) +}