Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorLeitwolf112026-11-13 07:10:08 +0300
committerLeitwolf112026-11-13 07:10:08 +0300
commit8003318ad262d37f8c3fe29dd4f13937e7b2d309 (patch)
tree44ba4a81f89f90c6da2866f97085771c90560527
parent6bb2736f505c647d27f8d42bcb1f4b542c910e34 (diff)
downloadtazel-feature/tazel-test.tar.gz
Run test targets with tazel testfeature/tazel-test
tazel test [--release] [labels...] [-- arguments for every test] The test targets are built like any other, then run as the accepted DSL describes, close to Bazel's cc_test: - Each test runs in its output directory, beside the data files copied there, with its args, then the arguments after --, and its env, plus TEST_TARGET and a clean TEST_TMPDIR. - shards = N runs it N times with TEST_TOTAL_SHARDS and TEST_SHARD_INDEX; each shard has its own log (test.shard-K-of-N.log, else test.log). - The timeout is its own or the one of its size (1, 5, 15, 60 minutes); a test still running then is stopped with its children: TIMEOUT. - flaky = true runs a failing test up to three times; passing after a failure is FLAKY, which counts as passed. - EXCLUSIVE tests run alone, after the others run in parallel; MANUAL tests run only when their label is named, never through a pattern. - A test that passed is not run again while the digest of its program, its data files, its arguments, its env, its shards and its timeout are the same (CACHED), unless it is tagged NO_CACHE. - A failed test shows the end of its log and where the log is; the command exits with 1 when any test failed. Only this machine's triple is tested. Processes gets runLogged, which writes output to a file and enforces a time limit. Durations are printed without the locale. TestCommandTest builds C tests with the compiler of the machine and checks a passing test with data, args and env, a failing one, a flaky one, a sharded one, a test that hangs, MANUAL tests left out of patterns, cached results, and that other arguments are another run.
-rw-r--r--CHANGELOG.md4
-rw-r--r--sources/main/kotlin/com/progmasoft/tazel/cli/BuildCommands.kt72
-rw-r--r--sources/main/kotlin/com/progmasoft/tazel/cli/Main.kt13
-rw-r--r--sources/main/kotlin/com/progmasoft/tazel/driver/Processes.kt55
-rw-r--r--sources/main/kotlin/com/progmasoft/tazel/driver/Tests.kt167
-rw-r--r--sources/test/kotlin/com/progmasoft/tazel/cli/TestCommandTest.kt182
6 files changed, 479 insertions, 14 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d33f147..38c4a4b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,3 +25,7 @@ Versions follow [Semantic Versioning](https://semver.org). The earlier Tazel, 0.
output it wrote is gone or altered. Touching a file reruns nothing, and an object rebuilt to the same bytes reruns
neither its archive nor its program. Clang, GCC and
`zig cc` are found on PATH; Clang and `zig cc` build for other triples. Outputs go to `.tazel/out/<triple>/<mode>`.
+- `tazel test` builds and runs test targets: each in its output directory beside its data, with its args and env,
+ split into shards with `TEST_TOTAL_SHARDS` and `TEST_SHARD_INDEX`, stopped at its timeout (from `size` unless set),
+ run up to three times when `flaky`, with tests tagged `EXCLUSIVE` alone and `MANUAL` ones only when named. A test
+ that passed is not run again while its program, data, arguments and environment are the same, unless `NO_CACHE`.
diff --git a/sources/main/kotlin/com/progmasoft/tazel/cli/BuildCommands.kt b/sources/main/kotlin/com/progmasoft/tazel/cli/BuildCommands.kt
index 1567f18..bace4a2 100644
--- a/sources/main/kotlin/com/progmasoft/tazel/cli/BuildCommands.kt
+++ b/sources/main/kotlin/com/progmasoft/tazel/cli/BuildCommands.kt
@@ -9,10 +9,13 @@ import com.progmasoft.tazel.driver.BuildFailure
import com.progmasoft.tazel.driver.BuildPlan
import com.progmasoft.tazel.driver.Builder
import com.progmasoft.tazel.driver.CcToolchain
+import com.progmasoft.tazel.driver.FileSets
+import com.progmasoft.tazel.driver.TestRunner
import com.progmasoft.tazel.dsl.BuildMode
import com.progmasoft.tazel.dsl.ProjectException
import com.progmasoft.tazel.dsl.TargetKind
import com.progmasoft.tazel.dsl.TargetTriple
+import com.progmasoft.tazel.dsl.TestTag
import com.progmasoft.tazel.label.LabelException
import com.progmasoft.tazel.label.Labels
import com.progmasoft.tazel.label.PackageId
@@ -61,7 +64,14 @@ internal fun buildOptions(command: String, arguments: List<String>): BuildOption
/** A command line that is wrong; it exits with [ExitCode.USAGE]. */
internal class Usage(message: String) : Exception(message)
-/** `tazel build` and `tazel run`. */
+/** What a command selects targets for. */
+internal enum class Purpose {
+ BUILD,
+ RUN,
+ TEST,
+}
+
+/** `tazel build`, `tazel run` and `tazel test`. */
internal class BuildCommands(
private val environment: Environment,
private val out: Appendable,
@@ -70,20 +80,20 @@ internal class BuildCommands(
private val host = TargetTriple.host()
fun build(options: BuildOptions): Int {
- val (module, labels, triple) = select(options, run = false)
+ val (module, labels, triple) = select(options, Purpose.BUILD)
build(module, labels, triple, options.mode)
return ExitCode.SUCCESS
}
fun run(options: BuildOptions): Int {
- val (module, labels, triple) = select(options, run = true)
+ val (module, labels, triple) = select(options, Purpose.RUN)
if (triple.text != host.text) {
throw Stop("tazel run runs programs of this machine ($host), not of $triple")
}
val label = labels.single()
- val products = build(module, labels, triple, options.mode)
+ val plan = build(module, labels, triple, options.mode)
val settings = module.configure(label, triple, options.mode).settings
- val program = products.getValue(label)
+ val program = plan.targets.single { it.label == label }.product
return environment.processes.runWith(
listOf(program.toString()) + settings.args + options.programArguments,
environment.directory,
@@ -91,12 +101,37 @@ internal class BuildCommands(
)
}
+ /** Builds the selected tests, then runs them; see [TestRunner]. */
+ fun test(options: BuildOptions): Int {
+ val (module, labels, triple) = select(options, Purpose.TEST)
+ if (triple.text != host.text) {
+ throw Stop("tazel test runs tests of this machine ($host), not of $triple")
+ }
+ val plan = build(module, labels, triple, options.mode)
+ val execution = module.file.execution
+ val tests = plan.targets.filter { it.label in labels }
+ val results =
+ TestRunner(
+ environment.processes,
+ out,
+ execution.jobs ?: Runtime.getRuntime().availableProcessors(),
+ FileSets(ignoreCase = host.os == com.progmasoft.tazel.dsl.Os.WINDOWS),
+ )
+ .run(tests, options.programArguments)
+ val failed = results.count { !it.status.passed }
+ out.appendLine(
+ "%12s %d test(s): %d passed, %d failed"
+ .format("Tested", results.size, results.size - failed, failed)
+ )
+ return if (failed == 0) ExitCode.SUCCESS else ExitCode.FAILURE
+ }
+
private fun build(
module: Module,
labels: List<TargetLabel>,
triple: TargetTriple,
mode: BuildMode,
- ): Map<TargetLabel, java.nio.file.Path> {
+ ): BuildPlan {
val plan = BuildPlan(module, triple, mode)
labels.forEach(plan::add)
val execution = module.file.execution
@@ -121,7 +156,7 @@ internal class BuildCommands(
"%12s %d target(s) for %s, %s; %d command(s) run"
.format("Built", plan.targets.size, triple, mode.name.lowercase(), result.commandsRun)
)
- return result.products
+ return plan
}
private data class Selected(
@@ -130,7 +165,7 @@ internal class BuildCommands(
val triple: TargetTriple,
)
- private fun select(options: BuildOptions, run: Boolean): Selected {
+ private fun select(options: BuildOptions, purpose: Purpose): Selected {
val root = loader.findRoot(environment.directory)
val module = loader.load(root)
val triple = options.triple?.let(TargetTriple::parse) ?: host
@@ -157,11 +192,28 @@ internal class BuildCommands(
throw Stop("$selection is not built for $triple; name one of its triples with --triple")
}
// Without a label, only the package of the directory is meant, not the ones below it.
- labels +=
+ val meant =
if (options.selections.isEmpty()) forTriple.filter { it.packageId == current }
else forTriple
+ labels +=
+ if (purpose != Purpose.TEST) meant
+ else if (selection is TargetLabel) {
+ if (module.declaration(selection).second.kind != TargetKind.TEST) {
+ throw Stop("$selection is not a test target")
+ }
+ meant
+ } else {
+ // A pattern takes the tests, except those tagged MANUAL.
+ meant.filter { label ->
+ module.declaration(label).second.kind == TargetKind.TEST &&
+ TestTag.MANUAL !in module.configure(label, triple, options.mode).settings.tags
+ }
+ }
+ }
+ if (purpose == Purpose.TEST && labels.isEmpty()) {
+ throw Stop("no tests to run in ${written.joinToString()}")
}
- if (run) {
+ if (purpose == Purpose.RUN) {
val bins = labels.distinct().filter { module.declaration(it).second.kind == TargetKind.BIN }
return when {
options.selections.size > 1 -> throw Usage("tazel run takes one label")
diff --git a/sources/main/kotlin/com/progmasoft/tazel/cli/Main.kt b/sources/main/kotlin/com/progmasoft/tazel/cli/Main.kt
index 66fa08f..c9c4d61 100644
--- a/sources/main/kotlin/com/progmasoft/tazel/cli/Main.kt
+++ b/sources/main/kotlin/com/progmasoft/tazel/cli/Main.kt
@@ -44,19 +44,21 @@ internal val HELP =
Commands:
build builds targets, such as tazel build //app/:greeter or tazel build //...
run builds and runs one bin target; what follows -- goes to the program
+ test builds and runs test targets; what follows -- goes to every test
custom runs a command of tazel_custom.txt, or lists them
version prints the version of Tazel
help prints this help
- Options of build and run:
+ Options of build, run and test:
--release builds in release mode; without it, debug
--triple <triple> builds for that LLVM target triple; without it, this machine's
- Without a label, build and run take the package of the current directory.
+ Without a label, build, run and test take the package of the current directory.
+ A pattern such as //... takes no test tagged MANUAL.
"""
.trimIndent()
-private val COMMANDS = listOf("build", "run", "custom", "version", "help")
+private val COMMANDS = listOf("build", "run", "test", "custom", "version", "help")
/** Where a command runs and what it starts programs with; tests give their own. */
internal class Environment(
@@ -93,7 +95,8 @@ internal fun run(
ExitCode.SUCCESS
} else usage(err, "tazel version takes no arguments")
"build",
- "run" ->
+ "run",
+ "test" ->
BuildCommands.guarded(err) {
refuseWithoutSandbox(environment.directory)
val command = arguments.first()
@@ -101,6 +104,8 @@ internal fun run(
val commands = BuildCommands(environment, out)
if (command == "run") {
commands.run(options)
+ } else if (command == "test") {
+ commands.test(options)
} else {
if (options.programArguments.isNotEmpty()) {
throw Usage("tazel build takes nothing after --")
diff --git a/sources/main/kotlin/com/progmasoft/tazel/driver/Processes.kt b/sources/main/kotlin/com/progmasoft/tazel/driver/Processes.kt
index c592fc1..a543f70 100644
--- a/sources/main/kotlin/com/progmasoft/tazel/driver/Processes.kt
+++ b/sources/main/kotlin/com/progmasoft/tazel/driver/Processes.kt
@@ -8,6 +8,7 @@ package com.progmasoft.tazel.driver
import com.progmasoft.tazel.dsl.ProjectException
import java.nio.file.Path
import java.util.concurrent.TimeUnit
+import kotlin.time.Duration.Companion.nanoseconds
/** Starts the programs Tazel runs. A test replaces it to see the commands without running them. */
interface Processes {
@@ -34,6 +35,24 @@ interface Processes {
environment: Map<String, String>,
): GatheredRun = runGathered(command, directory)
+ /**
+ * Runs [command] in [directory] with [environment] added, for at most [timeout], writing its
+ * output and errors to [log]; a program still running then is stopped with its children. A
+ * stand-in may run it without a limit.
+ */
+ fun runLogged(
+ command: List<String>,
+ directory: Path,
+ environment: Map<String, String>,
+ log: Path,
+ timeout: kotlin.time.Duration,
+ ): TimedRun {
+ val started = System.nanoTime()
+ val result = runGatheredWith(command, directory, environment)
+ java.nio.file.Files.writeString(log, result.output)
+ return TimedRun(result.exitCode, (System.nanoTime() - started).nanoseconds)
+ }
+
/** Runs [command] and returns what it wrote to its output, or null when it failed. */
fun capture(command: List<String>): String?
@@ -57,6 +76,12 @@ interface Processes {
): Int = runGathered(command, directory).also { it.output.lines().forEach(line) }.exitCode
}
+/** How a program with a time limit ended: its exit code, or `null` when it ran out of time. */
+data class TimedRun(
+ val exitCode: Int?,
+ val elapsed: kotlin.time.Duration,
+)
+
/** The exit code of a program and everything it wrote, output and errors in the order written. */
data class GatheredRun(
val exitCode: Int,
@@ -127,6 +152,36 @@ object SystemProcesses : Processes {
throw ProjectException("${command.first()} could not be started: ${error.message}")
}
+ override fun runLogged(
+ command: List<String>,
+ directory: Path,
+ environment: Map<String, String>,
+ log: Path,
+ timeout: kotlin.time.Duration,
+ ): TimedRun {
+ val started = System.nanoTime()
+ val process =
+ try {
+ ProcessBuilder(command)
+ .directory(directory.toFile())
+ .also { it.environment().putAll(environment) }
+ .redirectErrorStream(true)
+ .redirectOutput(log.toFile())
+ .redirectInput(ProcessBuilder.Redirect.from(nullDevice()))
+ .start()
+ } catch (error: java.io.IOException) {
+ throw ProjectException("${command.first()} could not be started: ${error.message}")
+ }
+ val finished = process.waitFor(timeout.inWholeMilliseconds, TimeUnit.MILLISECONDS)
+ if (!finished) {
+ process.descendants().forEach { it.destroyForcibly() }
+ process.destroyForcibly()
+ process.waitFor()
+ }
+ val elapsed = (System.nanoTime() - started).nanoseconds
+ return TimedRun(if (finished) process.exitValue() else null, elapsed)
+ }
+
override fun capture(command: List<String>): String? =
try {
val process =
diff --git a/sources/main/kotlin/com/progmasoft/tazel/driver/Tests.kt b/sources/main/kotlin/com/progmasoft/tazel/driver/Tests.kt
new file mode 100644
index 0000000..8854717
--- /dev/null
+++ b/sources/main/kotlin/com/progmasoft/tazel/driver/Tests.kt
@@ -0,0 +1,167 @@
+/*
+ * SPDX-FileCopyrightText: 2026 Progmasoft <[email protected]>
+ * SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0
+ */
+
+package com.progmasoft.tazel.driver
+
+import com.progmasoft.tazel.dsl.TestTag
+import com.progmasoft.tazel.label.TargetLabel
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.time.Duration
+
+/** How a test ended. */
+enum class TestStatus(val passed: Boolean) {
+ PASSED(true),
+ /** Passed after it failed: only a flaky test is run again. */
+ FLAKY(true),
+ /** Passed before, and nothing it runs with changed since. */
+ CACHED(true),
+ FAILED(false),
+ TIMEOUT(false),
+}
+
+/** The result of one test target, all of its shards together. */
+data class TestResult(
+ val label: TargetLabel,
+ val status: TestStatus,
+ val elapsed: Duration,
+ /** One log per shard, of its last attempt. */
+ val logs: List<Path>,
+)
+
+/**
+ * Runs test targets as Bazel runs them: each in its output directory, beside its data files, with
+ * its args and env, split into [shards][com.progmasoft.tazel.dsl.TargetSettings.shards] that see
+ * `TEST_TOTAL_SHARDS` and `TEST_SHARD_INDEX`, stopped at its timeout, and run up to three times
+ * when it is flaky. A test that passed is not run again while its program, its data, its args and
+ * its env are the same, unless it is tagged NO_CACHE. Tests tagged EXCLUSIVE run alone, after the
+ * others.
+ */
+class TestRunner(
+ private val processes: Processes,
+ private val out: Appendable,
+ private val jobs: Int,
+ private val fileSets: FileSets,
+ private val digests: Digests = Digests(),
+) {
+ /** Runs [tests], each with [extraArguments] after its own args. */
+ fun run(
+ tests: List<PlannedTarget>,
+ extraArguments: List<String> = emptyList(),
+ ): List<TestResult> {
+ val (exclusive, shared) = tests.partition { TestTag.EXCLUSIVE in it.target.settings.tags }
+ val results =
+ JobPool(jobs, keepGoing = true).map(shared, { "test ${it.label}" }) {
+ test(it, extraArguments)
+ } + exclusive.map { test(it, extraArguments) }
+ return results.sortedBy { it.label.toString() }
+ }
+
+ private fun test(target: PlannedTarget, extraArguments: List<String>): TestResult {
+ val settings = target.target.settings
+ val directory = target.product.parent
+ val command = listOf(target.product.toString()) + settings.args + extraArguments
+ val key = cacheKey(target, command)
+ val record = directory.resolve(target.product.fileName.toString() + ".passed")
+ val cacheable = TestTag.NO_CACHE !in settings.tags
+ if (cacheable && Files.isRegularFile(record) && Files.readString(record) == key) {
+ val logs = (0 until settings.shards).map { logOf(target, it) }
+ return report(TestResult(target.label, TestStatus.CACHED, Duration.ZERO, logs))
+ }
+ Files.deleteIfExists(record)
+ val attempts = if (settings.flaky) 3 else 1
+ var elapsed = Duration.ZERO
+ var status = TestStatus.PASSED
+ val logs = mutableListOf<Path>()
+ for (shard in 0 until settings.shards) {
+ val environment = buildMap {
+ putAll(settings.env)
+ put("TEST_TARGET", target.label.toString())
+ put("TEST_TMPDIR", temporaryDirectory(target, shard).toString())
+ if (settings.shards > 1) {
+ put("TEST_TOTAL_SHARDS", settings.shards.toString())
+ put("TEST_SHARD_INDEX", shard.toString())
+ }
+ }
+ val log = logOf(target, shard)
+ var shardStatus = TestStatus.FAILED
+ for (attempt in 1..attempts) {
+ val run =
+ processes.runLogged(command, directory, environment, log, settings.effectiveTimeout)
+ elapsed += run.elapsed
+ shardStatus =
+ when (run.exitCode) {
+ null -> TestStatus.TIMEOUT
+ 0 -> if (attempt == 1) TestStatus.PASSED else TestStatus.FLAKY
+ else -> TestStatus.FAILED
+ }
+ if (shardStatus.passed) break
+ }
+ logs.add(log)
+ status = worse(status, shardStatus)
+ }
+ if (status.passed && cacheable) Files.writeString(record, key)
+ return report(TestResult(target.label, status, elapsed, logs))
+ }
+
+ /** What a passing result depends on: the program, its data files, its command line and env. */
+ private fun cacheKey(target: PlannedTarget, command: List<String>): String {
+ val settings = target.target.settings
+ val data = fileSets.expand(target.packageDirectory, settings.data)
+ val parts =
+ listOf(digests.of(target.product).orEmpty()) +
+ data.map { "${target.packageDirectory.relativize(it)}=${digests.of(it)}" } +
+ command.drop(1) +
+ settings.env.toSortedMap().map { (name, value) -> "$name=$value" } +
+ listOf("shards=${settings.shards}", "timeout=${settings.effectiveTimeout}")
+ return Digests.sha256(parts.joinToString("\u0000").toByteArray())
+ }
+
+ private fun worse(a: TestStatus, b: TestStatus): TestStatus {
+ val order =
+ listOf(
+ TestStatus.CACHED,
+ TestStatus.PASSED,
+ TestStatus.FLAKY,
+ TestStatus.FAILED,
+ TestStatus.TIMEOUT,
+ )
+ return if (order.indexOf(b) > order.indexOf(a)) b else a
+ }
+
+ private fun logOf(target: PlannedTarget, shard: Int): Path {
+ val shards = target.target.settings.shards
+ val name = if (shards > 1) "test.shard-${shard + 1}-of-$shards.log" else "test.log"
+ return target.outputDirectory.resolve(name)
+ }
+
+ private fun temporaryDirectory(target: PlannedTarget, shard: Int): Path {
+ val directory = target.outputDirectory.resolve("test-tmp").resolve(shard.toString())
+ if (Files.exists(directory)) directory.toFile().deleteRecursively()
+ return Files.createDirectories(directory)
+ }
+
+ private fun report(result: TestResult): TestResult {
+ val time =
+ if (result.status == TestStatus.CACHED) "(cached)"
+ else "in %.1fs".format(java.util.Locale.ROOT, result.elapsed.inWholeMilliseconds / 1000.0)
+ synchronized(out) {
+ out.appendLine("%12s %s %s".format(result.status.name, result.label, time))
+ if (!result.status.passed) {
+ for (log in result.logs) {
+ val lines = if (Files.isRegularFile(log)) Files.readAllLines(log) else emptyList()
+ lines.takeLast(LOG_LINES).forEach { out.appendLine(" $it") }
+ out.appendLine(" (log: $log)")
+ }
+ }
+ }
+ return result
+ }
+
+ private companion object {
+ /** How much of the log of a failed test is shown; the whole log stays in its file. */
+ const val LOG_LINES = 40
+ }
+}
diff --git a/sources/test/kotlin/com/progmasoft/tazel/cli/TestCommandTest.kt b/sources/test/kotlin/com/progmasoft/tazel/cli/TestCommandTest.kt
new file mode 100644
index 0000000..fcf43c5
--- /dev/null
+++ b/sources/test/kotlin/com/progmasoft/tazel/cli/TestCommandTest.kt
@@ -0,0 +1,182 @@
+/*
+ * SPDX-FileCopyrightText: 2026 Progmasoft <[email protected]>
+ * SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0
+ */
+
+package com.progmasoft.tazel.cli
+
+import com.progmasoft.tazel.driver.Programs
+import com.progmasoft.tazel.driver.SystemProcesses
+import com.progmasoft.tazel.dsl.TargetTriple
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.test.Test
+import kotlin.test.assertContains
+import kotlin.test.assertEquals
+import kotlin.test.assertFalse
+import kotlin.test.assertTrue
+
+/**
+ * Runs `tazel test` on C tests built with the compiler of this machine. Without Clang or GCC on
+ * PATH there is nothing to build with, and the tests return early.
+ */
+class TestCommandTest {
+ private val os = TargetTriple.host().os
+ private val programs = Programs(os)
+ private val compiler = programs.find("clang") ?: programs.find("gcc")
+
+ private fun module(): Path {
+ val root = Files.createTempDirectory("tazel-test-").toRealPath()
+ fun write(path: String, text: String) {
+ val file = root.resolve(path)
+ Files.createDirectories(file.parent)
+ Files.writeString(file, text.trimIndent() + "\n")
+ }
+ write("MODULE.kts", """module { name = "sample" }""")
+ write(
+ "tests/BUILD.kts",
+ """
+ c.test("passes") {
+ sources("passes.c")
+ data("fixture.txt")
+ args("--expected", "fixture")
+ env("SAMPLE_GREETING" to "hi")
+ }
+ c.test("fails") { sources("fails.c") }
+ c.test("flaky") {
+ sources("flaky.c")
+ flaky = true
+ }
+ c.test("sharded") {
+ sources("sharded.c")
+ shards = 2
+ }
+ c.test("hangs") {
+ sources("hangs.c")
+ timeout = 2.seconds
+ tags(TestTag.MANUAL)
+ }
+ """,
+ )
+ write("tests/fixture.txt", "fixture")
+ write(
+ "tests/passes.c",
+ """
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <string.h>
+ int main(int argc, char **argv) {
+ char text[32] = {0};
+ FILE *file = fopen("fixture.txt", "r");
+ if (!file || !fgets(text, sizeof text, file)) return 2;
+ text[strcspn(text, "\n")] = 0;
+ const char *greeting = getenv("SAMPLE_GREETING");
+ if (argc != 3 || strcmp(argv[2], text) != 0 || !greeting || strcmp(greeting, "hi") != 0) return 3;
+ puts("passed");
+ return 0;
+ }
+ """,
+ )
+ write(
+ "tests/fails.c",
+ """
+ #include <stdio.h>
+ int main(void) { puts("expected 2, got 3"); return 1; }
+ """,
+ )
+ write(
+ "tests/flaky.c",
+ """
+ #include <stdio.h>
+ int main(void) {
+ int runs = 0;
+ FILE *file = fopen("flaky.count", "r");
+ if (file) { if (fscanf(file, "%d", &runs) != 1) runs = 0; fclose(file); }
+ file = fopen("flaky.count", "w");
+ fprintf(file, "%d", runs + 1);
+ fclose(file);
+ return runs == 0 ? 1 : 0;
+ }
+ """,
+ )
+ write(
+ "tests/sharded.c",
+ """
+ #include <stdio.h>
+ #include <stdlib.h>
+ int main(void) {
+ printf("shard %s of %s\n", getenv("TEST_SHARD_INDEX"), getenv("TEST_TOTAL_SHARDS"));
+ return 0;
+ }
+ """,
+ )
+ write("tests/hangs.c", "int main(void) { for (;;) {} }")
+ return root
+ }
+
+ private fun tazel(root: Path, vararg arguments: String): Pair<Int, Recorder> {
+ val recorder = Recorder()
+ val code =
+ run(
+ arguments.toList(),
+ recorder.out,
+ recorder.err,
+ Environment(root, os, programs, SystemProcesses),
+ )
+ return code to recorder
+ }
+
+ @Test
+ fun testsPassFailRetryShardAndAreCached() {
+ if (compiler == null) return
+ val root = module()
+
+ val (code, output) = tazel(root, "test", "//tests/...")
+ val text = output.stdout()
+ assertEquals(ExitCode.FAILURE, code, text + output.stderr())
+ assertContains(text, "PASSED //tests/:passes")
+ assertContains(text, "FAILED //tests/:fails")
+ assertContains(text, "expected 2, got 3")
+ assertContains(text, "FLAKY //tests/:flaky")
+ assertContains(text, "PASSED //tests/:sharded")
+ assertContains(text, "4 test(s): 3 passed, 1 failed")
+ assertFalse("hangs" in text, "a MANUAL test runs only when it is named")
+
+ val out = root.resolve(".tazel/out").resolve(TargetTriple.host().text).resolve("debug/tests")
+ assertEquals(
+ "shard 0 of 2",
+ Files.readString(out.resolve("sharded/test.shard-1-of-2.log")).trim(),
+ )
+ assertEquals(
+ "shard 1 of 2",
+ Files.readString(out.resolve("sharded/test.shard-2-of-2.log")).trim(),
+ )
+
+ // Nothing changed: the tests that passed are not run again.
+ val (_, again) = tazel(root, "test", "//tests/:passes", "//tests/:sharded")
+ assertContains(again.stdout(), "CACHED //tests/:passes (cached)")
+ assertContains(again.stdout(), "CACHED //tests/:sharded (cached)")
+
+ // Other arguments are another run.
+ val (_, other) = tazel(root, "test", "//tests/:passes", "--", "--extra")
+ assertTrue("CACHED" !in other.stdout(), other.stdout())
+ }
+
+ @Test
+ fun aTestThatRunsTooLongIsStopped() {
+ if (compiler == null) return
+ val root = module()
+ val (code, output) = tazel(root, "test", "//tests/:hangs")
+ assertEquals(ExitCode.FAILURE, code)
+ assertContains(output.stdout(), "TIMEOUT //tests/:hangs")
+ }
+
+ @Test
+ fun onlyTestTargetsAreTested() {
+ val root = module()
+ Files.writeString(root.resolve("BUILD.kts"), """c.bin("app") { sources("app.c") }""")
+ val (code, output) = tazel(root, "test", "//:app")
+ assertEquals(ExitCode.FAILURE, code)
+ assertContains(output.stderr(), "//:app is not a test target")
+ }
+}