package workflow import ( "errors" "os/exec" "strings" "testing" ) // The common case: a go repository whose whole CI is three run steps behind a checkout. func TestParseTheOrdinaryWorkflow(t *testing.T) { jobs, err := Parse(Dir+"/ci.yml", ` name: ci on: [push] env: CGO_ENABLED: "0" jobs: test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: build run: go build ./... - name: test run: | go vet ./... go test ./... `, Context{Repo: "john/bot", Ref: "refs/heads/master", SHA: "a3f9c2d"}) if err != nil { t.Fatal(err) } if len(jobs) != 1 { t.Fatalf("got %d jobs, want 1", len(jobs)) } j := jobs[0] if j.Workflow != "ci" || j.ID != "test" { t.Errorf("job identity is %+v", j) } if len(j.RunsOn) != 1 || j.RunsOn[0] != "ubuntu-latest" { t.Errorf("runs-on = %v", j.RunsOn) } for _, want := range []string{ "export CGO_ENABLED='0'", "go build ./...", "go vet ./...", "go test ./...", // Checkout is answered rather than skipped, because barerepo has already cloned. "barerepo cloned this repository", } { if !strings.Contains(j.Script, want) { t.Errorf("the script is missing %q\n%s", want, j.Script) } } if len(j.Skipped) != 0 { t.Errorf("nothing here is untranslatable, yet it skipped %+v", j.Skipped) } if !j.Runnable() { t.Error("a job with three run steps is not runnable") } } // Rule: never skip quietly. Every step barerepo does not run has to be named and explained. func TestEverySkipIsNamedAndExplained(t *testing.T) { jobs, err := Parse(Dir+"/ci.yml", ` jobs: build: runs-on: ubuntu-latest steps: - uses: actions/setup-node@v4 with: {node-version: 20} - name: only on main if: github.ref == 'refs/heads/main' run: ./deploy.sh - name: powershell shell: pwsh run: Write-Host hi - name: real work run: make ci `, Context{Repo: "john/bot", Ref: "refs/heads/master", SHA: "a3f9c2d"}) if err != nil { t.Fatal(err) } j := jobs[0] if !strings.Contains(j.Script, "make ci") { t.Error("the one runnable step did not survive the skipping") } // setup-node is answered with a check, not skipped, so it is not in the list below. if !strings.Contains(j.Script, "command -v node") { t.Errorf("setup-node did not become a check\n%s", j.Script) } want := map[string]string{ "only on main": "cannot evaluate", "powershell": "pwsh shell", } if len(j.Skipped) != len(want) { t.Fatalf("skipped %d steps, want %d: %+v", len(j.Skipped), len(want), j.Skipped) } for _, s := range j.Skipped { frag, ok := want[s.Step] if !ok { t.Errorf("unexpected skip of %q", s.Step) continue } if !strings.Contains(s.Reason, frag) { t.Errorf("%s was skipped because %q, want it to mention %q", s.Step, s.Reason, frag) } } } // sh reads ${{ as a bad substitution, so an expression barerepo knows is filled, not passed through. func TestKnownExpressionsAreFilledIn(t *testing.T) { jobs, err := Parse(Dir+"/ci.yml", ` jobs: b: runs-on: [self-hosted, linux] steps: - run: echo building ${{ github.sha }} of ${{ github.repository }} on ${{ runner.os }} `, Context{Repo: "john/bot", Ref: "refs/heads/master", SHA: "a3f9c2d"}) if err != nil { t.Fatal(err) } j := jobs[0] if len(j.RunsOn) != 2 || j.RunsOn[1] != "linux" { t.Errorf("a runs-on list did not survive: %v", j.RunsOn) } if len(j.Skipped) != 0 { t.Errorf("expressions barerepo knows were reported instead of filled: %+v", j.Skipped) } if strings.Contains(j.Script, "${{") { t.Errorf("an expression reached the script, where sh calls it a bad substitution\n%s", j.Script) } // Run it the way the runner does, and the values have to arrive. out, err := exec.Command("sh", "-c", j.Script).CombinedOutput() if err != nil { t.Fatalf("the script does not run: %v\n%s", err, out) } for _, want := range []string{"building a3f9c2d", "of john/bot"} { if !strings.Contains(string(out), want) { t.Errorf("the output is missing %q\n%s", want, out) } } // runner.os is the machine's own answer, so it is whatever this machine is. if !strings.Contains(string(out), "on ") { t.Errorf("runner.os produced nothing\n%s", out) } } // A workflow reading the environment directly must work too, because GitHub sets these. func TestGitHubEnvironmentIsSet(t *testing.T) { jobs, err := Parse(Dir+"/ci.yml", ` name: ci jobs: unit: steps: - run: echo "$GITHUB_REPOSITORY $GITHUB_REF_NAME $GITHUB_SHA $GITHUB_WORKFLOW $GITHUB_JOB $CI" `, Context{Repo: "john/bot", Ref: "refs/heads/master", SHA: "a3f9c2d"}) if err != nil { t.Fatal(err) } out, err := exec.Command("sh", "-c", jobs[0].Script).CombinedOutput() if err != nil { t.Fatalf("the script does not run: %v\n%s", err, out) } // ref_name is the branch alone, which is what a workflow means by it. want := "john/bot master a3f9c2d ci unit true" if !strings.Contains(string(out), want) { t.Errorf("the environment is wrong.\n got: %s\nwant: %s", out, want) } } // An expression barerepo cannot fill is left alone and reported, rather than guessed at. func TestUnknownExpressionsAreStillReported(t *testing.T) { jobs, err := Parse(Dir+"/ci.yml", ` jobs: b: steps: - run: deploy --key ${{ secrets.DEPLOY_KEY }} `, Context{Repo: "john/bot"}) if err != nil { t.Fatal(err) } j := jobs[0] if len(j.Skipped) != 1 || !strings.Contains(j.Skipped[0].Reason, "${{secrets.DEPLOY_KEY}}") { t.Fatalf("an unfillable expression was not reported: %+v", j.Skipped) } if !strings.Contains(j.Script, "${{ secrets.DEPLOY_KEY }}") { t.Error("an expression barerepo cannot fill was rewritten instead of left alone") } } // A matrix builds several times, so barerepo queues one job per combination. Chapter 15A. func TestAMatrixBecomesOneJobPerCombination(t *testing.T) { jobs, err := Parse(Dir+"/ci.yml", ` jobs: m: runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest, macos-latest] go: ["1.25", "1.26"] exclude: - os: macos-latest go: "1.25" steps: - run: echo testing go ${{ matrix.go }} on ${{ matrix.os }} `, Context{Repo: "john/bot", SHA: "a3f9c2d"}) if err != nil { t.Fatal(err) } // Two by two is four, less the one excluded. if len(jobs) != 3 { t.Fatalf("got %d jobs, want 3: %+v", len(jobs), jobs) } seen := map[string][]string{} for _, j := range jobs { if !j.Runnable() { t.Errorf("%s is not runnable", j.Name) } if len(j.Skipped) != 0 { t.Errorf("%s skipped something: %+v", j.Name, j.Skipped) } if strings.Contains(j.Script, "${{") { t.Errorf("%s kept an expression\n%s", j.Name, j.Script) } seen[j.Name] = j.RunsOn } // Each job asks for the machine its combination named, and sorted axes keep the names stable. want := map[string]string{ "m (go 1.25, os ubuntu-latest)": "ubuntu-latest", "m (go 1.26, os ubuntu-latest)": "ubuntu-latest", "m (go 1.26, os macos-latest)": "macos-latest", } for name, machine := range want { got, ok := seen[name] if !ok { t.Errorf("no job named %q, got %v", name, seen) continue } if len(got) != 1 || got[0] != machine { t.Errorf("%s asks for %v, want %s", name, got, machine) } } // The excluded pair must not have been built. if _, built := seen["m (go 1.25, os macos-latest)"]; built { t.Error("an excluded combination was queued") } } // The combination has to reach the command, or every job in the matrix runs the same build. func TestMatrixValuesReachTheScript(t *testing.T) { jobs, err := Parse(Dir+"/ci.yml", ` jobs: m: strategy: matrix: go-version: ["1.26"] steps: - run: echo "expression ${{ matrix.go-version }} env $MATRIX_GO_VERSION" `, Context{Repo: "john/bot"}) if err != nil { t.Fatal(err) } if len(jobs) != 1 { t.Fatalf("got %d jobs, want 1", len(jobs)) } out, err := exec.Command("sh", "-c", jobs[0].Script).CombinedOutput() if err != nil { t.Fatalf("the script does not run: %v\n%s", err, out) } // Both forms have to work: the expression, and the variable a workflow might read directly. if want := "expression 1.26 env 1.26"; !strings.Contains(string(out), want) { t.Errorf("got %q, want it to contain %q", out, want) } } // include can add keys and whole combinations, so barerepo says it did not apply it rather than guess. func TestMatrixIncludeIsReportedNotGuessedAt(t *testing.T) { jobs, err := Parse(Dir+"/ci.yml", ` jobs: m: strategy: matrix: os: [ubuntu-latest] include: - os: windows-latest experimental: true steps: - run: make `, Context{Repo: "john/bot"}) if err != nil { t.Fatal(err) } if len(jobs) != 1 { t.Fatalf("include added a combination barerepo cannot work out: %d jobs", len(jobs)) } if len(jobs[0].Skipped) != 1 || !strings.Contains(jobs[0].Skipped[0].Reason, "include") { t.Errorf("include was applied or ignored quietly: %+v", jobs[0].Skipped) } } // A condition is a program and barerepo has no evaluator, so a guarded job is not run blind. func TestAConditionalJobIsDeclinedWhole(t *testing.T) { jobs, err := Parse(Dir+"/ci.yml", ` jobs: m: if: github.ref == 'refs/heads/main' steps: - run: ./deploy.sh `, Context{Repo: "john/bot"}) if err != nil { t.Fatal(err) } if jobs[0].Runnable() { t.Error("a job behind a condition was run without the condition being checked") } } // container: names the image, which is what barerepo passes to the runner. Chapter 14. func TestContainerBecomesTheImage(t *testing.T) { for _, body := range []string{ "jobs:\n b:\n container: golang:1.26\n steps:\n - run: make\n", "jobs:\n b:\n container:\n image: golang:1.26\n steps:\n - run: make\n", } { jobs, err := Parse(Dir+"/ci.yml", body, Context{Repo: "john/bot", Ref: "refs/heads/master", SHA: "a3f9c2d"}) if err != nil { t.Fatal(err) } if jobs[0].Image != "golang:1.26" { t.Errorf("image = %q from\n%s", jobs[0].Image, body) } } } // Two reads of one file must give the same script, or a cache keyed by commit is worthless. func TestParseIsStable(t *testing.T) { body := ` jobs: b: env: {B: "2", A: "1", C: "3"} steps: - run: make a: steps: - run: make ` first, err := Parse(Dir+"/ci.yml", body, Context{Repo: "john/bot", Ref: "refs/heads/master", SHA: "a3f9c2d"}) if err != nil { t.Fatal(err) } for i := 0; i < 8; i++ { again, err := Parse(Dir+"/ci.yml", body, Context{Repo: "john/bot", Ref: "refs/heads/master", SHA: "a3f9c2d"}) if err != nil { t.Fatal(err) } if len(again) != len(first) { t.Fatalf("job count moved between reads") } for k := range first { if again[k].ID != first[k].ID || again[k].Script != first[k].Script { t.Fatalf("read %d differs:\n%q\n%q", i, first[k].Script, again[k].Script) } } } if first[0].ID != "a" { t.Errorf("jobs are not in a stable order: %s first", first[0].ID) } } // A file barerepo cannot read is an error, never a silently empty build. func TestBrokenYamlIsAnError(t *testing.T) { if _, err := Parse(Dir+"/ci.yml", "jobs:\n - this: [is not\n", Context{Repo: "john/bot", Ref: "refs/heads/master", SHA: "a3f9c2d"}); err == nil { t.Error("malformed yaml parsed without complaint") } } // A build must never install a toolchain onto a machine barerepo does not own, so setup checks instead. func TestSetupActionsCheckRatherThanInstall(t *testing.T) { jobs, err := Parse(Dir+"/ci.yml", ` jobs: b: steps: - uses: actions/setup-go@v5 with: go-version: "1.26" - uses: actions/setup-python@v5 - uses: actions/cache@v4 with: path: ~/.cache - run: make `, Context{Repo: "john/bot", Ref: "refs/heads/master", SHA: "a3f9c2d"}) if err != nil { t.Fatal(err) } j := jobs[0] if len(j.Skipped) != 0 { t.Errorf("a handled action was also reported as skipped: %+v", j.Skipped) } for _, want := range []string{ // The tool is checked for, and a missing one stops the build rather than failing oddly later. "command -v go >/dev/null 2>&1 || {", "this workflow needs go, and it is not on this runner", "exit 1", // The asked-for version is printed, not enforced, so a patch digit cannot fail a build. "this workflow asked for go 1.26", "go --version", "command -v python3", // The log has to say a toolchain was not installed, or a green build implies one was. "barerepo does not install toolchains", "barerepo does not cache between builds", "make", } { if !strings.Contains(j.Script, want) { t.Errorf("the script is missing %q\n%s", want, j.Script) } } // No command here may change the runner. Comments are prose and are not commands. for _, line := range strings.Split(j.Script, "\n") { if strings.HasPrefix(strings.TrimSpace(line), "#") { continue } for _, never := range []string{"apt-get", "brew install", "npm install", "pip install", "| sh"} { if strings.Contains(line, never) { t.Errorf("a command runs %q, and barerepo must not change the runner: %s", never, line) } } } } // A setup action with no version still checks, because the tool being absent is the real failure. func TestSetupWithNoVersionStillChecks(t *testing.T) { jobs, err := Parse(Dir+"/ci.yml", "jobs:\n b:\n steps:\n - uses: actions/setup-node@v4\n", Context{Repo: "john/bot", Ref: "refs/heads/master", SHA: "a3f9c2d"}) if err != nil { t.Fatal(err) } script := jobs[0].Script if !strings.Contains(script, "command -v node") { t.Errorf("no check was written\n%s", script) } if strings.Contains(script, "asked for node ") { t.Errorf("a version was claimed that the workflow never gave\n%s", script) } } // The runner passes the script to sh -c, which without this runs every step and reports the last. func TestAFailingStepFailsTheBuild(t *testing.T) { jobs, err := Parse(Dir+"/ci.yml", ` jobs: b: steps: - name: fails run: exit 3 - name: passes run: echo second step ran `, Context{Repo: "john/bot", Ref: "refs/heads/master", SHA: "a3f9c2d"}) if err != nil { t.Fatal(err) } script := jobs[0].Script if !strings.HasPrefix(script, "set -e\n") { t.Fatalf("the script does not stop at the first failure\n%s", script) } // Run it the way the runner does, and the build must fail without reaching the second step. cmd := exec.Command("sh", "-c", script) out, err := cmd.CombinedOutput() if err == nil { t.Errorf("a script whose first step exits 3 reported success\n%s", out) } if strings.Contains(string(out), "second step ran") { t.Errorf("a step after a failing one still ran\n%s", out) } var ee *exec.ExitError if errors.As(err, &ee) && ee.ExitCode() != 3 { t.Errorf("the build exited %d, losing the step's own code 3", ee.ExitCode()) } } // Chapter 15A: an expression barerepo cannot fill is declined out loud, and env is where secrets live. func TestAnExpressionInEnvIsSubstitutedOrDeclined(t *testing.T) { src := ` name: ci on: [push] jobs: test: runs-on: ubuntu-latest env: SHA: ${{ github.sha }} TOKEN: ${{ secrets.NPM_TOKEN }} steps: - run: echo "$SHA" ` jobs, err := Parse(Dir+"/ci.yml", src, Context{Repo: "john/bot", Ref: "refs/heads/master", SHA: "a3f9c2d"}) if err != nil { t.Fatal(err) } if len(jobs) != 1 { t.Fatalf("got %d jobs", len(jobs)) } j := jobs[0] // The ones barerepo knows are filled in, or the build reads the literal text of an expression. if !strings.Contains(j.Script, "export SHA=") || strings.Contains(j.Script, "export SHA='${{") { t.Errorf("github.sha was not substituted in env:\n%s", j.Script) } // The one it does not know is named, because a token that silently holds its own name is worse. said := false for _, s := range j.Skipped { if strings.Contains(s.Reason, "TOKEN") && strings.Contains(s.Reason, "secrets.NPM_TOKEN") { said = true } } if !said { t.Errorf("a secrets expression in env was neither filled nor reported: %v", j.Skipped) } } // A value that is not a plain scalar is left unset and named, since an empty export is a lie. func TestAnEnvValueThatIsNotPlainIsNamed(t *testing.T) { src := ` name: ci on: [push] jobs: test: runs-on: ubuntu-latest env: LIST: - one - two steps: - run: echo hi ` jobs, err := Parse(Dir+"/ci.yml", src, Context{Repo: "john/bot", Ref: "refs/heads/master", SHA: "a3f9c2d"}) if err != nil { t.Fatal(err) } if len(jobs) != 1 { t.Fatalf("got %d jobs", len(jobs)) } j := jobs[0] if strings.Contains(j.Script, "export LIST=") { t.Errorf("a list was exported as something:\n%s", j.Script) } said := false for _, s := range j.Skipped { if strings.Contains(s.Reason, "LIST is not a plain value") { said = true } } if !said { t.Errorf("an env value barerepo could not use was dropped silently: %v", j.Skipped) } }