// Package workflow reads .github/workflows and runs the parts barerepo can, naming the parts it cannot. package workflow import ( "fmt" "sort" "strings" "gopkg.in/yaml.v3" ) // Dir is where GitHub keeps them, and barerepo reads them where they already are. Chapter 15A. const Dir = ".github/workflows" // Context is what the push knew, which is what fills the github expressions a workflow uses. type Context struct { Repo string // owner/name Ref string SHA string } // RefName is the branch or tag alone, which is what github.ref_name means. func (c Context) RefName() string { for _, prefix := range []string{"refs/heads/", "refs/tags/"} { if name, ok := strings.CutPrefix(c.Ref, prefix); ok { return name } } return c.Ref } // vars are the environment GitHub sets, which barerepo sets too, so a workflow reading them works. func (c Context) vars() [][2]string { return [][2]string{ {"CI", "true"}, {"GITHUB_ACTIONS", "true"}, {"GITHUB_REPOSITORY", c.Repo}, {"GITHUB_REF", c.Ref}, {"GITHUB_REF_NAME", c.RefName()}, {"GITHUB_SHA", c.SHA}, {"GITHUB_EVENT_NAME", "push"}, {"GITHUB_WORKFLOW", ""}, {"GITHUB_JOB", ""}, } } // substitutions maps the expressions barerepo knows onto the shell that answers them. func substitutions() map[string]string { return map[string]string{ "github.repository": `"$GITHUB_REPOSITORY"`, "github.ref": `"$GITHUB_REF"`, "github.ref_name": `"$GITHUB_REF_NAME"`, "github.sha": `"$GITHUB_SHA"`, "github.event_name": `"$GITHUB_EVENT_NAME"`, "github.workspace": `"$PWD"`, "github.workflow": `"$GITHUB_WORKFLOW"`, "github.job": `"$GITHUB_JOB"`, "runner.os": `"$RUNNER_OS"`, "runner.arch": `"$RUNNER_ARCH"`, "runner.temp": `"$RUNNER_TEMP"`, } } // substitute fills the expressions barerepo knows and reports the ones it does not. func substitute(run string) (string, []string) { known := substitutions() var unknown []string var b strings.Builder rest := run for { before, after, found := strings.Cut(rest, "${{") if !found { b.WriteString(rest) return b.String(), unknown } b.WriteString(before) inner, tail, closed := strings.Cut(after, "}}") if !closed { // An unclosed expression is left exactly as written, since guessing at it is worse. b.WriteString("${{") b.WriteString(after) return b.String(), unknown } name := strings.TrimSpace(inner) if shell, ok := known[name]; ok { b.WriteString(shell) } else { b.WriteString("${{" + inner + "}}") unknown = append(unknown, "${{"+name+"}}") } rest = tail } } // Job is one workflow job reduced to what a barerepo runner needs: a machine, an image and a script. type Job struct { // Workflow is the name in the file, and Path is the file, so a run can say which one it is. Workflow string Path string Name string ID string // RunsOn is what the job asked for, kept verbatim, because it is matched against runner labels. RunsOn []string Image string Script string // Needs names jobs that must finish first, which barerepo does not order. Reported, never silent. Needs []string // Skipped is every step barerepo could not translate, so a green build never means less than it says. Skipped []Skip } // Skip is one thing barerepo did not do, and why, because a quiet omission is a lie about the build. type Skip struct { Step string Reason string } // Runnable reports whether anything is left to run after the skipping. func (j Job) Runnable() bool { return strings.TrimSpace(j.Script) != "" } // file is the shape of a workflow yaml, holding only the keys barerepo reads. type file struct { Name string `yaml:"name"` Env map[string]any `yaml:"env"` Jobs map[string]job `yaml:"jobs"` } type job struct { Name string `yaml:"name"` RunsOn any `yaml:"runs-on"` Needs any `yaml:"needs"` If string `yaml:"if"` Env map[string]any `yaml:"env"` Container any `yaml:"container"` Strategy *strategy `yaml:"strategy"` Steps []step `yaml:"steps"` } type strategy struct { Matrix map[string]any `yaml:"matrix"` } type step struct { Name string `yaml:"name"` Uses string `yaml:"uses"` Run string `yaml:"run"` If string `yaml:"if"` Env map[string]any `yaml:"env"` With map[string]any `yaml:"with"` // Shell is honoured only where it is a shell barerepo can start. Shell string `yaml:"shell"` WorkingDirectory string `yaml:"working-directory"` } // Parse reads one workflow file. A file barerepo cannot parse is reported, not guessed at. func Parse(path, body string, c Context) ([]Job, error) { var f file if err := yaml.Unmarshal([]byte(body), &f); err != nil { return nil, fmt.Errorf("%s is not yaml barerepo can read: %w", path, err) } if len(f.Jobs) == 0 { return nil, nil } // A map has no order, so the ids are sorted to keep two reads of one file identical. ids := make([]string, 0, len(f.Jobs)) for id := range f.Jobs { ids = append(ids, id) } sort.Strings(ids) out := make([]Job, 0, len(ids)) for _, id := range ids { j := f.Jobs[id] combos, notes := matrixOf(j) if len(combos) == 0 { // Nothing to build, and a job that runs with ${{ matrix.os }} still in it builds nonsense. skipped := make([]Skip, 0, len(notes)) for _, n := range notes { skipped = append(skipped, Skip{Step: "job " + id, Reason: n}) } out = append(out, Job{Workflow: workflowName(f, path), ID: id, Name: j.Name, Path: path, Skipped: skipped}) continue } for _, m := range combos { built := convert(path, f, id, j, c, m) for _, n := range notes { built.Skipped = append(built.Skipped, Skip{Step: "job " + id, Reason: n}) } out = append(out, built) } } return out, nil } // maxCombos is GitHub's own ceiling on one matrix, so a file that builds here builds there too. 15A. const maxCombos = 256 // matrixOf returns the combinations a job builds, which is one empty combination when it has no matrix. func matrixOf(j job) ([]combo, []string) { if j.Strategy == nil || len(j.Strategy.Matrix) == 0 { return []combo{{}}, nil } // Axes multiply, so eight of them name millions, and the file is read in a push. Counted, not built. if tooManyCombos(j.Strategy.Matrix) { return nil, []string{fmt.Sprintf("names more than %d combinations, and barerepo builds at most that many", maxCombos)} } combos, notes := expand(j.Strategy.Matrix) // expand answers nil when no axis widened anything, and an empty list when exclude emptied it. if combos == nil { return []combo{{}}, notes } return combos, notes } // tooManyCombos multiplies the axes the way expand will, and stops at the ceiling rather than at the end. func tooManyCombos(m map[string]any) bool { n := 1 for k, v := range m { // include and exclude are not axes, and exclude only ever removes, so neither widens this. if k == "include" || k == "exclude" { continue } values := len(stringList(v)) if values == 0 { continue } n *= values if n > maxCombos { return true } } return false } // convert turns one yaml job and one matrix combination into a barerepo job. Chapter 15A. func convert(path string, f file, id string, j job, c Context, m combo) Job { out := Job{Workflow: workflowName(f, path), ID: id, Name: j.Name, Path: path} if out.Name == "" { out.Name = id } // The combination is in the name, so two builds of one job are told apart in the runs list. out.Name += m.suffix() // The matrix is filled first, because runs-on and the image are usually what it decides. out.RunsOn = fillAll(m, stringList(j.RunsOn)) out.Needs = stringList(j.Needs) out.Image = m.fill(containerImage(j.Container)) // barerepo queues jobs and does not order them, so the wait this job asked for is named, not obeyed. 15A. if len(out.Needs) > 0 { // What is skipped is the waiting, not the job, and the hook reads Step as the thing skipped. out.Skipped = append(out.Skipped, Skip{Step: "the needs on job " + id, Reason: "names " + strings.Join(out.Needs, ", ") + ", and barerepo does not order jobs"}) } // An expression is a program, and barerepo has no evaluator, so a guarded job is not run blind. if j.If != "" { out.Skipped = append(out.Skipped, Skip{Step: "job " + id, Reason: "runs under a condition barerepo cannot evaluate: if " + j.If}) return out } var script strings.Builder // GitHub fails a job on its first failing step, and sh -c without this would run on and pass. script.WriteString("set -e\n") // The environment GitHub sets, so a workflow reading $GITHUB_SHA needs no change to work here. for _, kv := range c.vars() { value := kv[1] if kv[0] == "GITHUB_WORKFLOW" { value = out.Workflow } if kv[0] == "GITHUB_JOB" { value = id } fmt.Fprintf(&script, "export %s=%s\n", kv[0], shellQuote(value)) } // These two are the runner's own, so the script asks the machine rather than the server. script.WriteString("export RUNNER_OS=$(uname -s)\n") script.WriteString("export RUNNER_ARCH=$(uname -m)\n") script.WriteString("export RUNNER_TEMP=${TMPDIR:-/tmp}\n") script.WriteString(m.vars()) writeEnv(&script, f.Env, "the workflow env", &out) writeEnv(&script, j.Env, "the job env", &out) // Only a step that puts a command in the script is work, and the preamble above is not. 15A. commands := false for i, s := range j.Steps { name := stepName(s, i) switch { case s.If != "": out.Skipped = append(out.Skipped, Skip{Step: name, Reason: "runs under a condition barerepo cannot evaluate: if " + s.If}) case s.Uses != "": shim, note, ok := knownAction(s.Uses, s.With) if !ok { out.Skipped = append(out.Skipped, Skip{Step: name, Reason: "uses " + s.Uses + ", which barerepo does not run"}) continue } // A handled action still earns a line, so the log says what barerepo did about it. fmt.Fprintf(&script, "\n# %s: %s\n", name, note) script.WriteString(shim) commands = commands || shim != "" case strings.TrimSpace(s.Run) == "": out.Skipped = append(out.Skipped, Skip{Step: name, Reason: "has nothing to run"}) case !usableShell(s.Shell): out.Skipped = append(out.Skipped, Skip{Step: name, Reason: "asks for the " + s.Shell + " shell"}) default: s.Run = m.fill(s.Run) writeStep(&script, name, s, &out) commands = true } } if !commands { // A script of nothing but exports would pass, and a green build that ran none of the job is a lie. out.Skipped = append(out.Skipped, Skip{Step: "job " + id, Reason: "has no step barerepo can run, so barerepo did not queue it"}) return out } out.Script = script.String() return out } // writeStep puts one run step into the script, with its own environment and directory. func writeStep(script *strings.Builder, name string, s step, out *Job) { // An expression left in would reach sh as ${{, which is a bad substitution and fails the step. run, unknown := substitute(s.Run) s.Run = run fmt.Fprintf(script, "\n# %s\n", name) if len(s.Env) > 0 || s.WorkingDirectory != "" { // A subshell keeps a step's directory and environment from leaking into the next one. script.WriteString("(\n") writeEnv(script, s.Env, name, out) if s.WorkingDirectory != "" { fmt.Fprintf(script, "cd %s\n", shellQuote(s.WorkingDirectory)) } script.WriteString(s.Run) if !strings.HasSuffix(s.Run, "\n") { script.WriteString("\n") } script.WriteString(")\n") } else { script.WriteString(s.Run) if !strings.HasSuffix(s.Run, "\n") { script.WriteString("\n") } } // What barerepo could not fill is still reported, because it will fail the step when sh reads it. if len(unknown) > 0 { out.Skipped = append(out.Skipped, Skip{Step: name, Reason: "keeps expressions barerepo does not substitute: " + strings.Join(unknown, ", ")}) } } // writeEnv turns an env block into export lines, skipping what is not a plain scalar. func writeEnv(script *strings.Builder, env map[string]any, scope string, out *Job) { if len(env) == 0 { return } keys := make([]string, 0, len(env)) for k := range env { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { if !usableEnvName(k) { // The key is written into the script unquoted, so a key that is not a name is shell. 15A. out.Skipped = append(out.Skipped, Skip{Step: scope, Reason: k + " is not a name a shell can export, so barerepo did not set it"}) continue } raw, ok := env[k].(string) if !ok { if text := scalar(env[k]); text != "" { fmt.Fprintf(script, "export %s=%s\n", k, shellQuote(text)) continue } // An empty export is a lie about the value, so the variable is left unset and named. out.Skipped = append(out.Skipped, Skip{Step: scope, Reason: k + " is not a plain value, so barerepo did not set it"}) continue } // A quoted expression would export its own text, which is worse than failing. 15A. value, unknown := substitute(raw) if len(unknown) > 0 { out.Skipped = append(out.Skipped, Skip{Step: scope, Reason: k + " keeps expressions barerepo does not substitute: " + strings.Join(unknown, ", ")}) } fmt.Fprintf(script, "export %s=%s\n", k, shellQuote(value)) } } // usableEnvName reports whether a key can be exported, which is the shell's name and nothing wider. func usableEnvName(k string) bool { for i, r := range k { switch { case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r == '_': case i > 0 && r >= '0' && r <= '9': default: return false } } return k != "" } // tool is one thing a setup action promises, which barerepo checks for rather than installs. type tool struct { bin string version string // the with: key naming the version, if the action takes one name string } // setups are the actions barerepo answers, being the ones a repository reaches for after checkout. var setups = map[string]tool{ "actions/setup-go": {bin: "go", version: "go-version", name: "go"}, "actions/setup-node": {bin: "node", version: "node-version", name: "node"}, "actions/setup-python": {bin: "python3", version: "python-version", name: "python"}, "actions/setup-java": {bin: "java", version: "java-version", name: "java"}, "actions/setup-dotnet": {bin: "dotnet", version: "dotnet-version", name: "dotnet"}, } // knownAction answers an action with shell, or reports that barerepo does not run it. func knownAction(uses string, with map[string]any) (shim, note string, ok bool) { name, _, _ := strings.Cut(uses, "@") switch name { case "actions/checkout": return "", "barerepo cloned this repository at the commit already", true case "actions/cache": // Chapter 15: the build runs on a machine the user owns, which keeps its own state. return "", "barerepo does not cache between builds, so this build starts from the clone", true } if t, found := setups[name]; found { return setupShim(t, with), setupNote(t, with), true } return "", "", false } // setupShim checks the tool is there rather than installing one, because the runner is not barerepo's. func setupShim(t tool, with map[string]any) string { var b strings.Builder fmt.Fprintf(&b, "command -v %s >/dev/null 2>&1 || {\n", t.bin) fmt.Fprintf(&b, " echo 'this workflow needs %s, and it is not on this runner' >&2\n", t.name) b.WriteString(" exit 1\n}\n") if v := scalar(with[t.version]); v != "" { // The version is printed rather than enforced, since a build should not fail on a patch digit. fmt.Fprintf(&b, "echo 'this workflow asked for %s %s. this runner has:'\n", t.name, v) fmt.Fprintf(&b, "%s --version\n", t.bin) } return b.String() } // setupNote says in one line what barerepo did, so the log never implies a toolchain was installed. func setupNote(t tool, with map[string]any) string { if v := scalar(with[t.version]); v != "" { return "barerepo does not install toolchains, so this checks the runner has " + t.name + " " + v } return "barerepo does not install toolchains, so this checks the runner has " + t.name } // usableShell reports whether barerepo can start what the step asked for. Empty means the default. func usableShell(s string) bool { switch s { case "", "bash", "sh": return true } return false } // fillAll runs a combination through every string, which is how runs-on gets its value. func fillAll(m combo, in []string) []string { out := make([]string, 0, len(in)) for _, s := range in { out = append(out, m.fill(s)) } return out } // stringList takes a yaml value that is one string or a list of them. func stringList(v any) []string { switch t := v.(type) { case string: if t == "" { return nil } return []string{t} case []any: out := make([]string, 0, len(t)) for _, item := range t { if s := scalar(item); s != "" { out = append(out, s) } } return out } return nil } // containerImage reads container: as either a bare image or a map with one. func containerImage(v any) string { switch t := v.(type) { case string: return t case map[string]any: return scalar(t["image"]) } return "" } // scalar renders a yaml scalar as the shell sees it, and anything else as nothing. func scalar(v any) string { switch t := v.(type) { case string: return t case int: return fmt.Sprint(t) case bool: return fmt.Sprint(t) case float64: return strings.TrimSuffix(fmt.Sprintf("%v", t), ".0") } return "" } func stepName(s step, i int) string { switch { case s.Name != "": return oneLine(s.Name) case s.Uses != "": return oneLine(s.Uses) default: return fmt.Sprintf("step %d", i+1) } } // oneLine keeps a name from ending the comment it is written into, because the next line is a command and the name comes from the build file. 15A. func oneLine(s string) string { return strings.Map(func(r rune) rune { if r == '\n' || r == '\r' { return ' ' } return r }, s) } func workflowName(f file, path string) string { if f.Name != "" { return f.Name } return strings.TrimPrefix(path, Dir+"/") } // shellQuote wraps a value in single quotes, which is the only quoting sh does not look inside. func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" }