test: promote optional reference regressions

Author: martinfrancois

evals/45-workflow-validation-cleanup/capability.txt

@@ -0,0 +1 @@
+Implement Java Optional fallback logic without eager fallback work or presence-check value reads.

evals/45-workflow-validation-cleanup/criteria.json

@@ -0,0 +1,58 @@
+{
+  "context": "Transcript-derived maintainability cleanup from Symphony for Trello: preserve workflow validation behavior while replacing repeated Optional reopening in a real implementation.",
+  "type": "weighted_checklist",
+  "checklist": [
+    {
+      "name": "Creates revised validation class",
+      "category": "safety",
+      "max_score": 5,
+      "description": "Creates WorkflowConfigValidation.java with the requested class, imports, records, validate method, boardId method, and serverPort method."
+    },
+    {
+      "name": "Uses Optional as validation boundary",
+      "category": "optional_quality",
+      "max_score": 30,
+      "description": "Uses boardId(yaml).map(...).orElseGet(...) and serverPort(yaml).map(...).orElseGet(...) or an equivalent direct Optional boundary for validation."
+    },
+    {
+      "name": "Removes Optional reopening",
+      "category": "optional_quality",
+      "max_score": 30,
+      "description": "Does not keep isPresent()/isEmpty() followed by get() or orElseThrow() for configured board id or configured server port value reads."
+    },
+    {
+      "name": "Avoids null/list workaround",
+      "category": "optional_quality",
+      "max_score": 20,
+      "description": "Does not replace the Optionals with orElse(null), local null branching, Optional.stream().toList(), lists, or loops over a single Optional."
+    },
+    {
+      "name": "Keeps readable helpers",
+      "category": "maintainability",
+      "max_score": 5,
+      "description": "Extracts or uses clear helper methods for board-id validation and server-port validation, rather than embedding repeated warning construction and value reads in one long method."
+    },
+    {
+      "name": "Preserves board-id behavior",
+      "category": "safety",
+      "max_score": 4,
+      "description": "Accepts both board.boardId() and board.boardKey(), returns the missing tracker.board_id warning for absent/blank values, and returns the mismatch warning with expected id/key and found value."
+    },
+    {
+      "name": "Preserves server-port behavior",
+      "category": "safety",
+      "max_score": 4,
+      "description": "Accepts Number values and trimmed numeric String values, returns the missing server.port warning for absent/blank/malformed/unsupported values, and returns the mismatch warning with expected and found ports."
+    },
+    {
+      "name": "Preserves success result",
+      "category": "safety",
+      "max_score": 2,
+      "description": "Returns WorkflowValidation.valid() when both board id and server port match."
+    }
+  ],
+  "metadata": {
+    "invocation": "explicit",
+    "task_type": "implementation"
+  }
+}

evals/45-workflow-validation-cleanup/task.md

@@ -0,0 +1,90 @@
+# Finish workflow validation cleanup
+
+Assume Java 17.
+
+Use `$java-optionals` to create `WorkflowConfigValidation.java` with the revised class.
+
+This is adapted from a real AI-assisted maintainability cleanup. The original task was a broad
+readability/refactor pass that had to preserve the same behavior. The code below works, but the
+validation method should be easier to maintain.
+
+Current code:
+
+```java
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.Optional;
+
+final class WorkflowConfigValidation {
+    WorkflowValidation validate(ConnectedBoard board, Map<String, Object> yaml) {
+        Optional<String> configuredBoardId = boardId(yaml);
+        if (configuredBoardId.isEmpty()) {
+            return WorkflowValidation.warn("Workflow file is missing tracker.board_id for \""
+                    + board.boardName() + "\": " + board.workflowPath());
+        }
+        String boardId = configuredBoardId.orElseThrow();
+        if (!boardId.equals(board.boardId()) && !boardId.equals(board.boardKey())) {
+            return WorkflowValidation.warn("Workflow tracker.board_id does not match the connected board for \""
+                    + board.boardName() + "\": expected " + board.boardId() + " or " + board.boardKey()
+                    + " but found " + boardId);
+        }
+        Optional<Integer> configuredServerPort = serverPort(yaml);
+        if (configuredServerPort.isEmpty()) {
+            return WorkflowValidation.warn("Workflow file is missing server.port for \""
+                    + board.boardName() + "\": " + board.workflowPath());
+        }
+        if (configuredServerPort.orElseThrow() != board.serverPort()) {
+            return WorkflowValidation.warn("Workflow server.port does not match the connected board for \""
+                    + board.boardName() + "\": expected " + board.serverPort() + " but found "
+                    + configuredServerPort.orElseThrow());
+        }
+        return WorkflowValidation.valid();
+    }
+
+    Optional<String> boardId(Map<String, Object> yaml) {
+        Object trackerValue = yaml.get("tracker");
+        if (!(trackerValue instanceof Map<?, ?> tracker)) {
+            return Optional.empty();
+        }
+        Object value = tracker.get("board_id");
+        String text = value == null ? null : String.valueOf(value);
+        return text == null || text.isBlank() ? Optional.empty() : Optional.of(text);
+    }
+
+    Optional<Integer> serverPort(Map<String, Object> yaml) {
+        Object serverValue = yaml.get("server");
+        if (!(serverValue instanceof Map<?, ?> server)) {
+            return Optional.empty();
+        }
+        Object value = server.get("port");
+        if (value instanceof Number number) {
+            return Optional.of(number.intValue());
+        }
+        if (value instanceof String text && !text.isBlank()) {
+            try {
+                return Optional.of(Integer.parseInt(text.trim()));
+            } catch (NumberFormatException ignored) {
+                return Optional.empty();
+            }
+        }
+        return Optional.empty();
+    }
+
+    record ConnectedBoard(String boardName, String boardId, String boardKey, int serverPort, Path workflowPath) {}
+
+    record WorkflowValidation(boolean ok, String message) {
+        static WorkflowValidation valid() { return new WorkflowValidation(true, ""); }
+        static WorkflowValidation warn(String message) { return new WorkflowValidation(false, message); }
+    }
+}
+```
+
+Required changes:
+
+- Keep the same warning messages and successful `WorkflowValidation.valid()` result.
+- Keep accepting `board.boardId()` and `board.boardKey()` as matching tracker board ids.
+- Keep accepting numeric server ports and trimmed numeric string server ports.
+- Return the missing-board warning when `tracker.board_id` is absent or blank.
+- Return the missing-port warning when `server.port` is absent, blank, malformed, or unsupported.
+- Leave `boardId(...)` and `serverPort(...)` returning `Optional`.
+- You may extract private helpers if that makes the validation flow clearer.

evals/50-reference-command-sanitizer-feature/capability.txt

@@ -0,0 +1 @@
+Implement Java command sanitization while preserving exact output order and Optional match handling.

evals/50-reference-command-sanitizer-feature/criteria.json

@@ -0,0 +1,46 @@
+{
+  "context": "Representative real-collection lookup case: the option set is real data and matching should stay centralized while the outer loop handles redact-next state.",
+  "type": "weighted_checklist",
+  "checklist": [
+    {
+      "name": "Compiles and creates requested artifact",
+      "category": "safety",
+      "max_score": 5,
+      "description": "Creates a coherent CommandSanitizer.java with sanitize(List<String>) and the required secret option set, using APIs compatible with the stated Java 17 baseline."
+    },
+    {
+      "name": "Implements exact redaction behavior",
+      "category": "safety",
+      "max_score": 5,
+      "description": "Preserves exact-option behavior such as --token abc -> --token <redacted>, adds option=value behavior such as --key=abc -> --key=<redacted>, and preserves non-secret arguments exactly."
+    },
+    {
+      "name": "Preserves output order and sequence state",
+      "category": "safety",
+      "max_score": 5,
+      "description": "Keeps output order, uses String.join(\" \", sanitized), and keeps redact-next behavior only for exact secret options."
+    },
+    {
+      "name": "Keeps option matching centralized",
+      "category": "maintainability",
+      "max_score": 5,
+      "description": "Keeps exact and option=value matching in one readable helper or direct lookup; a real Set stream, loop, or contains-based shape is acceptable when behavior remains clear."
+    },
+    {
+      "name": "Consumes Optional result directly",
+      "category": "optional_quality",
+      "max_score": 45,
+      "description": "Handles the match result with map/orElse or another direct Optional boundary rather than isPresent()/get() or orElseThrow() value reads."
+    },
+    {
+      "name": "Avoids fake or noisy rewrites",
+      "category": "optional_quality",
+      "max_score": 35,
+      "description": "Does not force a fake Optional collection, duplicate matching branches throughout the loop, local null sentinels, labels, or unnecessary nested loops."
+    }
+  ],
+  "metadata": {
+    "invocation": "natural",
+    "task_type": "implementation"
+  }
+}

evals/50-reference-command-sanitizer-feature/task.md

@@ -0,0 +1,46 @@
+# Extend command sanitization
+
+Create `CommandSanitizer.java` with the revised class. Assume Java 17.
+
+Current code:
+
+```java
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+final class CommandSanitizer {
+    private static final Set<String> SECRET_OPTIONS = Set.of("--token", "--key", "--workflow", "--config-dir", "--state-home", "--output");
+
+    String sanitize(List<String> args) {
+        List<String> sanitized = new ArrayList<>();
+        boolean redactNext = false;
+        for (String arg : args) {
+            if (redactNext) {
+                sanitized.add("<redacted>");
+                redactNext = false;
+                continue;
+            }
+            Optional<String> option = SECRET_OPTIONS.stream()
+                    .filter(secret -> arg.equals(secret))
+                    .findFirst();
+            if (option.isPresent()) {
+                sanitized.add(option.orElseThrow());
+                redactNext = true;
+            } else {
+                sanitized.add(arg);
+            }
+        }
+        return String.join(" ", sanitized);
+    }
+}
+```
+
+Required changes:
+
+- Preserve exact-option behavior: `--token abc` becomes `--token <redacted>`.
+- Add `option=value` behavior: `--key=abc` becomes `--key=<redacted>`.
+- Preserve non-secret arguments.
+- Keep the real option-set lookup readable and centralized.
+- Return `String.join(" ", sanitized)`.