diff --git a/.claude/commands/mendix/check-script.md b/.claude/commands/mendix/check-script.md index 2ad951176..631a4e295 100644 --- a/.claude/commands/mendix/check-script.md +++ b/.claude/commands/mendix/check-script.md @@ -10,8 +10,8 @@ Validate an MDL script before executing it. ## Commands ```bash -# Syntax check only (no project needed) -mxcli check script.mdl +# Syntax check only (--no-project: do not auto-detect an .mpr in this directory) +mxcli check script.mdl --no-project # Syntax + reference validation (checks entities/modules exist) mxcli check script.mdl -p app.mpr --references diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-11-check-syntax-only-example-auto-detects-project.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-11-check-syntax-only-example-auto-detects-project.json new file mode 100644 index 000000000..b03f06d05 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-11-check-syntax-only-example-auto-detects-project.json @@ -0,0 +1 @@ +{"area":"cmd/mxcli","date":"2026-11-11","symptom":"`mxcli check --help` offered `# Check syntax only (no project needed) mxcli check script.mdl`, but run from a directory holding an .mpr it printed `Using project: ...\\project.mpr (auto-detected in the current directory; pass -p to use another)` and resolved references against it; there was no way to get the advertised syntax-only check there","cause":"`rootCmd.PersistentPreRun` auto-discovery sets `-p` before `check` runs, and `check` treats any project as an implicit `--references` (the fix for an unqualified `Check passed!`); the help example was only true outside a project directory","file":"`cmd/mxcli/main.go` (`PersistentPreRun`, `noProjectOptOut`/`skipsProjectDiscovery`), `cmd/mxcli/cmd_check.go` (help, `checkNoProjectConflicts`)","insight":"Two individually right decisions (auto-detect the sole .mpr, mendixlabs/mxcli#1444; a project implies reference checks) composed into a help example that is false exactly where users run it — a project directory, which is also where the generated CLAUDE.md template told agents `check script.mdl # Syntax only`. Changing the default would have silently weakened every `cd app && mxcli check` that relies on the auto-detected refs, so the fix is an opt-out (`--no-project`, honoured by PersistentPreRun for any command that registers it), a notice on the auto-detect line naming it, a refusal when combined with -p/--references/--post-migration, and help/docs whose syntax-only example carries the flag. Note `Flags().Set` by auto-detection marks `-p` Changed, so a conflict check on Changed only works because discovery is skipped first. Test runs real main() in a temp dir with a non-project Stray.mpr (opening it would error); control: discovery forced on under --no-project prints the reported `Using project:` line","refs":["ako/mxcli#2135","mendixlabs/mxcli#1817"]} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-12-describe-autodetect-skips-roles-navigation.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-12-describe-autodetect-skips-roles-navigation.json new file mode 100644 index 000000000..5ecc68645 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-12-describe-autodetect-skips-roles-navigation.json @@ -0,0 +1 @@ +{"area": "cmd/mxcli", "date": "2026-11-12", "symptom": "`describe Administration.Administrator` (no type) answered \"no describable document named ... found in the project\" although `describe module role Administration.Administrator` describes it; the same for a user role (`describe Administrator`) and a navigation profile (`describe Responsive`).", "cause": "resolveDescribeType resolved through the catalog's objects view and, on a miss, a live scan of top-level units plus domain-model entities and associations. Module roles, user roles and navigation profiles are members of the module-security, project-security and navigation documents, so neither path ever saw them.", "file": "cmd/mxcli/cmd_describe.go (describableMembers, called from resolveViaReader)", "insight": "Resolve members in the live fallback, not the objects view: the catalog has no complete module-role table (role_mappings lists only roles a user role maps), so a view built on it would answer 'not found' for an unmapped role, which is worse than a miss that falls back. The help text had codified the gap as a rule ('forms that have no single qualified name (... navigation, module role)'), which is why it read as by design for months.", "issue": "ako/mxcli#1262"} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-12-nightly-11-14-plus-runlifecycle-wants-reload-after-client-source-keep.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-12-nightly-11-14-plus-runlifecycle-wants-reload-after-client-source-keep.json new file mode 100644 index 000000000..821c58f83 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-12-nightly-11-14-plus-runlifecycle-wants-reload-after-client-source-keep.json @@ -0,0 +1 @@ +{"area":"cmd/mxcli","date":"2026-11-12","symptom":"The nightly goes red on the Mendix 11.14.0, 11.15.0 and 11.16.0 legs, in `TestRunLifecycle_DetachStatusExecWaitStop`: `wait (Mendix 11.14+): build #2 applied without mxcli restarting mxbuild --serve; if this mxbuild builds incrementally again, drop the 11.14+ branch`, with the run log showing `build #2 applied via reload in 3.7-8.5s`. Every older leg, push CI and the unit tests are green","cause":"Not a product defect: the improvement the test was written to notice. ako/mxcli#2150 (9156cf465, client_source_keep.go) copies aside the client source a serve build deletes and puts it back before the next, so --watch builds incrementally on the versions that bundle the client themselves. The test's 11.14+ branch still required serveRecovery's serve restart in the run log. The lifecycle test needs a cached mxbuild and a runtime, so it runs only in the nightly, never in the PR that changed the behaviour — the same path as 2026-10-23-nightly-11-14-11-15-runlifecycle-wants-serve-rebuild-hint","file":"`cmd/mxcli/cmd_run_lifecycle_integration_test.go` (`wantBuild2`, step 4); the behaviour is `cmd/mxcli/docker/client_source_keep.go` and `serve_recovery.go`","insight":"The branch was built to fail when the restart stopped being needed, and it did, a day after the change, which is the cost of a behaviour pinned only in a nightly-only test: whoever changes client_source_keep.go, serve_recovery.go or the watch loop should run the lifecycle test on 11.14 and 11.15 before merging. Rather than drop the 11.14+ branch, the test now pins the new contract on every version: a page change applies via reload and the run log never says `restarting mxbuild --serve`, the phrase every fallback that replaces the serve process prints — so a keep that stops holding the source fails it instead of degrading to a 20 s cold build per change unnoticed. The boot-race step keeps claiming only that build #2 applies. Reproduce in about 2 minutes: `mxcli setup mxbuild --version 11.14.0`, JAVA_HOME at a JDK 25, `MXCLI_RUNLIFE_VERSION=11.14.0 go test -tags integration -run TestRunLifecycle_DetachStatusExecWaitStop ./cmd/mxcli/` (the old test via `go test -overlay`). Controls: the old test against main fails at line 159 with the nightly's message; the new test passes on 11.14.0 and 11.15.0; with newClientSourceKeep stubbed to nil it fails on 11.14.0 naming the restart, the log showing `applied via restart in 22.9s`","refs":["mendixlabs/mxcli nightly run 43191062324","ako/mxcli nightly run 43190452893","ako/mxcli#2150"],"test":"cmd/mxcli/cmd_run_lifecycle_integration_test.go"} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-bb-action-override-pluggable-button.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-bb-action-override-pluggable-button.json new file mode 100644 index 000000000..16c0e7503 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-bb-action-override-pluggable-button.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-11", "symptom": "mendixlabs/mxcli#1821: `use building block PlatformDesignSystem.Button (action: call microflow M.Act) as b_` failed with 'action override supplied, but the block has no button to rebind', on every PlatformDesignSystem block (Marketplace 44.4.0, Mendix 11.12.6, mxcli v0.31.0).", "cause": "The action override's target predicate isButtonWidget matched only the built-in button types (actionbutton, linkbutton, button). The PDS blocks are built from the pluggable pdsbutton, so no widget matched, although the engine already writes a pluggable widget's on-click slot from the same Action property the override sets.", "file": "mdl/executor/cmd_pages_builder_v3.go (isPluggableButtonWidget, the ActionOverride branch of the building-block expansion)", "insight": "A pluggable widget is a button target when its definition names it a button AND maps a slot to OnClick. The click slot alone is not enough: a Gallery and a Data grid 2 take OnClick for a row click and a PDS Link for a link click, and on a PDS page header the first clickable widget is the 'Back to' link, not the primary button. With no project there is no registry, and only the built-in types match, as before.", "refs": ["mendixlabs/mxcli#1821"], "test": "mdl/executor/cmd_pages_builder_bb_rebind_test.go"} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-building-block-action-override-pluggable-button.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-building-block-action-override-pluggable-button.json new file mode 100644 index 000000000..6a1380752 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-building-block-action-override-pluggable-button.json @@ -0,0 +1 @@ +{"area":"mdl/executor","date":"2026-11-11","symptom":"`use building block PlatformDesignSystem.Button (action: call microflow M.F) as b_` fails exec with `action override supplied, but the block has no button to rebind`; the block is one pluggable `pdsbutton pDSButton1`. Same for any block whose only clickable widget is pluggable (Atlas Card_WithImage's Image)","cause":"The ActionOverride target predicate isButtonWidget matched the core keywords actionbutton/linkbutton/button only; a pluggable widget's action lives in a definition-declared property","file":"`mdl/executor/cmd_pages_builder_v3.go` (`pluggableClickActionSlot`, the `ActionOverride` branch of `expandBuildingBlockRef`)","insight":"Ask the widget definition, not a name list: GenerateDefJSON gives every click action (onClick, onClickAction, onClickEvent) Source \"OnClick\" via actionSourceForKey, and the engine reads that Source from Properties[\"Action\"] — so the override's write is unchanged, only the predicate widens. \"Has an action property\" alone is too wide: the pluggable Gallery and Data Grid 2 have onClick (an item action) and combobox/slider have onChange, so require Source OnClick and no datasource mapping. Try core buttons over the whole block FIRST and fall back to pluggable ones, or Atlas Card_ActionWithImage (image, then button) would silently move its override onto the image. Mock trap: a propertyless pluggable widget describes as the LAST SEGMENT of its widget id (extractCustomWidgetType), so a test id ending `.Button` renders as the core `button` keyword — use the real-shaped `…PDSButton`. PlatformDesignSystem is not fetchable here (no marketplace PAT, not in the cached catalog); measured instead on 11.14.0 with Card_WithImage: before, the same refusal; after, the Image gets OnClick and mx check reports 0 errors. Tests `use_building_block_pluggable_action_2145_test.go`; control = pre-fix run, the two pluggable tests failed with the reported message","refs":["ako/mxcli#2145","mendixlabs/mxcli#1821"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-building-block-prefix-intra-block-references.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-building-block-prefix-intra-block-references.json new file mode 100644 index 000000000..9eec652fe --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-building-block-prefix-intra-block-references.json @@ -0,0 +1 @@ +{"area":"mdl/executor","date":"2026-11-11","symptom":"`use building block Atlas_Web_Content.Master_Detail as md_` fails exec with `failed to build page: failed to build datasource: widget not found: gallery2` — the widgets became md_gallery2 / md_dataView7 but the data view's `selection gallery2` kept the old name; without a prefix the same block builds","cause":"prefixWidgetNames renamed WidgetV3.Name only. A block's widgets reference each other by name in property values, and those were never visited","file":"`mdl/executor/widget_prefix_rename.go` (`prefixWidgetNames`, `widgetRefRenamer`; shared by `expandBuildingBlockRef` and `expandFragmentRef`)","insight":"A rename is a rename of declarations AND references in one pass — the same rule as $ID renumbering. Widget names are referenced in two spellings: bare (selection Reference, association ContextVariable, `delete $w` SourceVariable, LinkVariable — listed by type because Reference means an entity or flow for other source types) and as a `$name` token in any text (flow args, `$dv.Attr` template params, visibility/editability expressions, where clauses). Rewrite the token form generically by reflecting over every property value rather than enumerating property keys, restricted to names DECLARED in the block so outside references survive. Rewrite copy-on-write: cloneWidget copies the property map, not the values, so mutating a *DataSourceV3 in place corrupts the fragment definition for its next `use`. Measured on 11.14.0: prefixed Master_Detail now gives exactly the mx check errors the unprefixed one does (CE0488/CE3637 from the unbound template gallery); with `(datasource: database from E)` those clear and only the template's unbound text boxes (CE7005) remain. Tests `use_building_block_prefix_refs_560_test.go`; control = pre-fix run, selection reference stayed `listView2`","refs":["ako/mxcli#560"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-domain-model-svg-ignores-stored-association-anchors.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-domain-model-svg-ignores-stored-association-anchors.json new file mode 100644 index 000000000..6efbfcc9b --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-domain-model-svg-ignores-stored-association-anchors.json @@ -0,0 +1 @@ +{"area":"mdl/executor","date":"2026-11-11","symptom":"`mxcli describe --format svg domainmodel ` draws every association centre to centre; the stored association anchors are ignored (tested with extreme anchor values), so the SVG cannot be used to verify \"corner to corner\" lines","cause":"renderDomainModelSVG computed both line ends from the box centres (edgePointFrom) and never read Association.ParentConnection/ChildConnection, although the backend had parsed them since #872","file":"`mdl/executor/cmd_domainmodel_svg.go` (`svgBox.anchorPoint`, `outward`, `selfLoop`, association loop in `renderDomainModelSVG`)","insight":"**The encoding was already measured — reuse it, do not re-derive it**: #872's two findings (search `anchor`) settled that ParentConnection (FROM) / ChildConnection (TO) are \"x;y\" integer PERCENTAGES of the entity box, 0..100, no -1/auto sentinel, and the loader does not range-check; a blank 11.14.0 app's raw unit still holds `0;54`/`100;54`. So the renderer's job is only `box.x + x/100*w`, applied to the box AS DRAWN (the SVG's widths are its own estimate, and after spread/clamp — a percentage survives that, a pixel offset would not). **nil is the only 'unset'**: every association mxcli writes stores 0;50/100;50, so after the fix almost every real diagram changes, and an mxcli-made FROM-left-of-TO pair now draws from FROM's left edge to TO's right edge, through both boxes — that is the stored data, not a bug in the renderer. **A self-association needs a route, not a line**: centre to centre it was a zero-length line; two anchors on one box joined straight cross the entity, so it walks an outset rectangle the shorter way. Out-of-range anchors are clamped onto the box — an unverified choice (no Studio Pro here), documented in the test. Test the endpoints numerically with a centre-line golden captured BEFORE the change as the unset-anchor control; the anchored cases fail on the old code with the exact reported symptom (every line (400,139)-(190,125) whatever the anchors). Tests `TestDomainModelSVG_AssociationUsesStoredAnchors`, `TestDomainModelSVG_UnsetAnchorsKeepCentreLine`, `TestDomainModelSVG_SelfAssociationUsesStoredAnchors`; example `mdl-examples/bug-tests/1841-domain-model-svg-stored-anchors.mdl`. ako/mxcli#1841, part 1 of upstream #1692","refs":["#1841","#1692","#872"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-measurer-if-join-rejoin-disagree-with-builder.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-measurer-if-join-rejoin-disagree-with-builder.json new file mode 100644 index 000000000..ce1595597 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-measurer-if-join-rejoin-disagree-with-builder.json @@ -0,0 +1 @@ +{"area":"mdl/executor","date":"2026-11-11","symptom":"Auto-layout put objects on or behind their neighbours and `layout flows` called the flow already laid out (ako/mxcli#1773): an if/else followed by an activity inside a THEN branch put the enclosing merge inside that activity (5px reported; 20px when the decision opens the branch); a handler row ending in `join` reserved about twice its drawn width; a falling-through handler of two or more statements rejoined at a merge left of the row's last activity, so the edge ran backwards. mx check 0 errors in every case.","cause":"Four measurer/builder disagreements. measureIfStatement measured from the split's centre as if it were the left edge (45px short) and gapBetween compensated after the IF only; measureStatements started every run at the left edge although the builder centres a first split where an activity's centre would be (15px short); measureBranch added an end event after any terminal statement, a join and a both-return IF included; and the rejoin merge was placed midway between the activity and the next one by mergeOnEdge while the row ran on past it. An IF/ELSE without a merge (one branch returns, or tailWithoutMerge at a loop body's end) was measured with a merge too.","file":"mdl/executor/layout.go (measureRun, measureIf, gapBetween, lastStmtDrawsEndEvent), mdl/executor/layout_lanes.go (ifHasMerge, ifJoinsNothingAtTail, ifMainWidth, rejoinsPastRow, rejoinAfterRow), mdl/executor/cmd_microflows_builder_flows.go (finishCustomErrorHandler, addErrorHandlerRejoinFlowForState)","insight":"The cheapest exact oracle for a measurer is a loop: fitContainerSize sizes the box from what was BUILT, measureLoopStatement from what was MEASURED, so wrapping any run in `loop … end loop` and comparing the two widths catches every disagreement to the pixel, including over-measures that never overlap anything (the join case). It found two defects the issue did not name (the first-element offset, and a merge counted for merge-less IF/ELSEs). Distinguish the main-line cursor from the drawn extent: an IF without a merge resumes at a point that is not an edge of anything, and folding that point into the extent re-introduced an over-measure at a body's tail. A `merge` label statement advances only half a pitch, so gapBetween's 20px floor over-measured it; MDL refuses join/merge in loops (MDL-FLOW04), but the oracle still applies because a run is measured the same way wherever it is. Moves: 10 of 376 doctype-example flows (02c CX_SYNC_FactoryTelemetry 5465->4315 wide, CX_VAL_Factory, CX_VAL_EmailTemplate, CX_ACT_BatchProcessFactories, CX_ACT_HandleAuthCode; 02 M008_NestedIf, M093_GetCountryLabel; 07 M020, M050, M053) and 4 of 29 fixture flows; no golden test changed. Not this class: vertical measurement (a Bounds height is not symmetric about the line, so an if-without-else inside nested loops pokes out of the outer box, MPR011) and a multi-value CASE outcome's own merge drawn on its first activity (documented at firstCaseGroupShift).","refs":["ako/mxcli#1773","ako/mxcli#1768","ako/mxcli#684"],"test":"mdl/executor/layout_measurer_1773_test.go, mdl/executor/layout_handler_lane_1768_test.go"} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-prefixed-building-block-false-ce0495.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-prefixed-building-block-false-ce0495.json new file mode 100644 index 000000000..2d92c3e14 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-prefixed-building-block-false-ce0495.json @@ -0,0 +1 @@ +{"area":"mdl/executor","date":"2026-11-11","symptom":"`check --references` refuses a building block placed several times with `as `: `duplicate widget name 'Atlas_Web_Content.Card' (used 4 times) — … (CE0495)`, while exec writes a_container2/b_container2… and mx check reports 0 errors; since a reference error blocks exec, the documented prefix form could not be run without --no-check","cause":"validatePageContextTree counted names on the PARSED tree, where `use building block X` / `use fragment X` is a sentinel WidgetV3 whose Name is the block's name. exec expands the sentinel first (expandFragments) and only then registers widget names","file":"`mdl/executor/validate_page_context.go` (`expandUseRefsForCheck`, used by `validatePageContextTree`)","insight":"A check that walks the parsed tree must walk the tree exec BUILDS: any pre-pass exec runs before naming (here sentinel expansion; ALTER PAGE's insert path already expanded before its duplicate check) has to run in check too. Do not just skip the sentinel — that loses the real collision: the same block twice with NO prefix makes exec refuse `duplicate widget name 'text22'`, and check now reports it by those expanded names. With no project (or a script-defined fragment) the reference contributes no names. Measured on 11.14.0: four prefixed Atlas Cards — check passes, exec writes, mx check 0 errors; two unprefixed — check reports container2/text22, exec refuses. Tests `validate_page_context_bb_559_test.go`; control = the pre-fix run, which failed all four with the reported message","refs":["ako/mxcli#559"],"ce":["CE0495"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-session-never-sees-another-process-write.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-session-never-sees-another-process-write.json new file mode 100644 index 000000000..414512c4b --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-11-session-never-sees-another-process-write.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-11", "symptom": "ako/mxcli#2121: a REPL session never saw a write by another process (a second mxcli exec, Studio Pro, git): `show microflows` kept the old set, `describe microflow` of a flow created elsewhere said `microflow not found`, and on MPR v2 the session's own `create microflow` of that name wrote a SECOND document with it; on v1 (#2115 already re-read the listing) `create entity` in a module made elsewhere still said `module not found`.", "cause": "Two layers cached with no external-change signal: the v2 reader's unit index (unitCacheValid, cleared only by this reader's own writes) and the executor's executorCache (modules, domain models, hierarchy, name maps), which nothing but a session write ever reset. A git checkout also replaces the .mpr by rename, which an open SQLite connection never sees: it goes on reading the old inode.", "file": "modelsdk/mpr/reader_disk_change.go (SyncWithDisk: PRAGMA data_version + os.SameFile on the .mpr), mdl/backend/modelsdk/backend.go (SyncWithDisk reopens on replacement, refuses while a deferred run holds writes), mdl/executor/disk_change.go (called at the top of Execute; dropProjectReads keeps created/dropped/modifiedDomainModels)", "insight": "Probe, don't re-read: PRAGMA data_version moves only for OTHER connections' commits (the reader holds one connection), so the session's own writes never trip it and #2094's caches survive; file size/mtime would fire on every own write and undo that perf work. data_version cannot see a file renamed into place, so add os.SameFile on the path. The v1 symptom survived #2115 because the stale cache was the executor's, not the reader's — test with two executors (two backends = two SQLite connections) on a copy of both the v1 and the v2 fixture, and assert at the executor, not the reader. Stubbing only the executor-cache drop leaves `module not found` failing on both formats, which is the control that shows both layers are needed. The write-from-stale-state case was a duplicate document (v2 create of a name taken elsewhere), not a lost update: domain-model writes re-read the unit and kept the other process's entity in every measured run. Not covered: a change between one statement's read and its write (the locking proposal), and a v2 change to mprcontents/ with the .mpr untouched.", "refs": ["ako/mxcli#2121", "ako/mxcli#2115", "ako/mxcli#2094"], "test": "mdl/executor/external_change_2121_test.go, modelsdk/mpr/reader_disk_change_test.go"} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-12-check-lists-project-per-statement.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-12-check-lists-project-per-statement.json new file mode 100644 index 000000000..3e947c9ec --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-12-check-lists-project-per-statement.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-12", "symptom": "with decoding and file reads cached (ako/mxcli#2160), the check before an exec was still a quarter of a 600-page script: per page statement it listed the domain models four times, the microflows three times and the nanoflows twice, and loaded each pluggable widget's template again", "cause": "each validator builds its own index from a backend listing - buildEntityIndex, buildEntityQualifiedNames, checkMemberTypeIndex and checkAssociationCatalog from ListDomainModels, buildFlowSignatures (reached from two validators) from ListMicroflows and ListNanoflows - and each is called per statement. templatePropertyTypes says 'once per widget for the run' but its cache is created inside the call, and the call is per statement. Below them every listing copied each unit's bytes out of the content cache (bytes.Clone) for a decoder that only reads them, and the decode cache then compared those bytes with its own (memeqbody): 12 s and 9 s of an 84 s stage, plus the collector's work to clear the copies", "file": "mdl/executor/validate_pass_memo.go (new: checkPassMemo, projectDomainModels/Microflows/Nanoflows/Pages), mdl/executor/validate.go (validateProgramWithWarnings), mdl/executor/validate_widget_attribute_scope.go (passTemplatePropertyTypes), modelsdk/mpr/reader_units.go (ListUnitsByTypeShared), modelsdk/mprread/decode_cache.go (sameBytes)", "insight": "A check reads the project and never writes it, so one pass may list each kind once; the memo hangs on the pass's ExecContext and is nil during execution, where a statement must see what the one before it wrote. That makes the change a mechanical substitution of ctx.Backend.ListX() in the validate*.go files, with no behaviour change outside a check - but a text substitution also rewrote s.ctx.Backend.ListX(), so compile and read the diff. A cache declared inside a function that is called per statement is not a cache 'for the run': look at where the closure is made, not at its comment. For the copies: count them (Reader.UnitContentCopies) rather than time them, and once the decoder is handed the cache's own slice an unchanged unit is recognised by slice identity before any byte is compared. Measured: listings per added page statement 4/3/2 -> 0; the 300/600/600 seed's page stage 84 s -> 64 s from the shared bytes alone. Still per statement after this: ListPages (one, from outside validate*.go), GetProjectSettings, the exec-side template load (augmentFromMPK per widget instance) and the gen -> sdk conversion of each listing at exec time."} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-12-domain-model-reconciled-per-entity-statement.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-12-domain-model-reconciled-per-entity-statement.json new file mode 100644 index 000000000..003aec86e --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-12-domain-model-reconciled-per-entity-statement.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-12", "symptom": "a script that builds a domain model spent half its time in Writer.updateUnit: seeding 300 entities and 299 associations took 16 s after the read-side fixes of ako/mxcli#2160, 48% of it rewriting and reconciling a module's whole domain model once per statement", "cause": "CREATE ENTITY and CREATE ASSOCIATION each rewrite the module's domain model unit, and every rewrite went through canon.Reconcile against the stored unit (identity carry, translation carry, $ID transplant, comparison) and was written. The cost of one statement is the size of the model it adds to, and the profile put it in the reconcile, not in the file write", "file": "mdl/executor/entity_run.go (new), mdl/executor/executor.go (stepRuns in both statement loops), modelsdk/mpr/writer_deferred.go (DeferUnitWritesJudged, judgeHeld), mdl/backend/modelsdk/backend.go", "insight": "The held-write run that access rules use (ako/mxcli#872) fits, but its reports are held to the end and these statements report as they go. The first experiment - just add the two statements to the run - cut the stage from 15 s to 6 s with identical first-run output and made every re-run statement say 'Modified' although nothing reached storage (#890 again). So the writer judges each held write as it is held and counts it in WriteStats like a write that landed: the first write of a unit against storage with a real reconcile, later ones against the write the run already holds. Two things made that later comparison wrong before it was right, each found by dumping both documents and diffing them with binaries masked: the two sides must be normalised for storage first (the writer completes what a builder leaves out - a validation rule's empty Message - so a unit rebuilt from storage has it and a freshly declared entity does not), and identity fields must be carried from the previous write. With both, the cheap comparison agrees with the reconcile on a 599-statement re-run. What moves is when a write can fail: the reconcile against storage, and the storage-GUID guard in it, now runs at the end of the run, after the statements have reported. Measured: reconciles 3 -> 2 for two entities and 11 -> 2 for six; the 300-entity seed 17 s -> 10 s on first run and 19 s -> 13 s on a settled re-run, same machine and session."} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-12-entity-lookups-list-every-domain-model.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-12-entity-lookups-list-every-domain-model.json new file mode 100644 index 000000000..69604b932 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-12-entity-lookups-list-every-domain-model.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-12", "symptom": "after the entity run of ako/mxcli#2160, a script that builds a domain model still spent a fifth of its time in Backend.ListDomainModels: each added entity and its association listed and converted every domain model of the project ten times", "cause": "lookups that ask about one named module listed the project to answer. findEntity listed every domain model and filtered by module name, and CREATE ASSOCIATION calls it four times (each end once for itself, once more through projectEntityPersistence). The dispatch's name-space guard (refuseNameClash) built the project-wide set of entity names and of association names for every create, though a name is only ever looked up qualified. The conversion to the semantic model is 70% of a listing, and the decode cache of ADR-0015 does not save it", "file": "mdl/executor/oql_type_inference.go (findEntity, moduleDomainModel), mdl/executor/validate_name_clash.go (domainModelNameSet, refuseNameClash), mdl/backend/modelsdk/domainmodel.go (DomainModelListings counter)", "insight": "The profile named two callers; a counter on the listing and a temporary print of its call stacks named eight, and the per-pair count was ten where reading the handler suggested seven. Count first. The narrowed reads keep the listing as their fallback for what a single read does not answer - the virtual System module, a module spelled in another case, a module that does not exist yet, a mock backend that serves only the listing - so the 140 mock tests that stub ListDomainModels alone say nothing about the new path; the behaviour is pinned on the PedApp fixture instead, and compared against the previous binary on six refusals. A first control (exact instead of case-folded module match) passed because of that same fallback: a mutation that lands in the fallback is not a control. Making the narrowed set forget associations, and the narrowed lookup miss, each fail the test. What remains is the member-name check (CE0069), one listing per statement: it follows generalizations and owner-both associations across modules, so one module's model cannot answer it. Measured: listings per added entity and association 10 -> 2; the 300-entity seed 9.7 s -> 8.4 s on first run and 14.5 s -> 9.6 s on a settled re-run, same machine, interleaved."} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-11-layout-flows-merge-overlaps-are-measurer-short.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-11-layout-flows-merge-overlaps-are-measurer-short.json new file mode 100644 index 000000000..6e0e3f2ea --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-11-layout-flows-merge-overlaps-are-measurer-short.json @@ -0,0 +1 @@ +{"area":"mdl/executor","date":"2026-11-11","symptom":"`mxcli layout flows` introduced MPR008 overlaps at merge nodes on the main path (ako/mxcli#684, after the reporter's correction 2, not 4): '(merge)' (2355,80) and '(merge)' (2360,80) 5px; '(merge)' (2815,1730) and '(unnamed)' (2740,1730) 75px.","cause":"Not a layout-flows defect: layout flows rebuilds with the CREATE builder, and the builder places an IF's closing merge from the measured width of its branch. A decision inside the branch was measured from its split's centre and from the branch's left edge although it starts 15px in (ako/mxcli#1773), so the closing merge landed in the activity — or the inner merge — before it.","file":"mdl/executor/layout.go (measureIf, measureRun)","insight":"Grade the layouter with the linter, as the reporter proposed, but do it where the positions are computed: assertNoSharedPlace (MPR008's box test with stored sizes) plus loopEscapes (MPR011) over a handful of shapes in five contexts (main line, loop body, THEN, ELSE, THEN-without-ELSE) reproduced the merge-into-activity overlap in 11 shape/context pairs with no project; the reporter's project was not needed. Reconstruct shapes by mechanism (what places a merge: a measured branch width), not by guessing the reporter's flows. The real-tool control on a fresh 11.14.0 app: flows exec'd by the pre-fix binary lint 5 MPR008; after `layout flows` with the fix, 3, all pre-existing and of other classes (a CASE's multi-value merge on its first activity; two vertical collisions from heights that are not symmetric about the line), dry-run then idempotent, mx check 0 errors. The note that lost `position:` after layout is not lost: layout flows re-places notes by design and describe omits a note's position when it is the default one above its activity.","refs":["ako/mxcli#684","ako/mxcli#1773"],"test":"mdl/executor/layout_merge_overlap_684_test.go"} diff --git a/.claude/skills/fix-issue/findings/modelsdk/2026-11-12-held-write-digested-again-per-statement.json b/.claude/skills/fix-issue/findings/modelsdk/2026-11-12-held-write-digested-again-per-statement.json new file mode 100644 index 000000000..c3db18707 --- /dev/null +++ b/.claude/skills/fix-issue/findings/modelsdk/2026-11-12-held-write-digested-again-per-statement.json @@ -0,0 +1 @@ +{"area": "modelsdk/mpr", "date": "2026-11-12", "symptom": "in a run of create entity / create association statements, judging each held write was 35% of the stage (ako/mxcli#2160): Writer.judgeHeld 2.9 s of 8.2 s sampled for 300 entities and 299 associations, 1.7 s of it in canon.Equal", "cause": "judgeHeld compared the new write with the write the run already held through canon.Equal, which digests both documents: the held domain model was unmarshalled, numbered and hashed again for every statement, although it had been the 'new' side of the statement before", "file": "modelsdk/mpr/writer_deferred.go (deferredWrite.judgedDigest, judgeHeld, judgedDigest), modelsdk/mpr/writer_core.go (JudgedDigests counter), mdl/backend/modelsdk/backend.go", "insight": "The digest that is free to keep is the one of the identity-carried form, not of the normalised form the run used to hold, so the run now holds the carried form. That changes no verdict - the next write takes the same top-level identity fields before it is compared, so both sides hold them either way - and the existing reporting test (a settled re-run reports nothing, one changed statement reports alone) is what says so. Two shortcuts were measured and dropped before writing anything: an unchanged statement's write is byte-identical to the held one in only 29 of 569 cases, because a re-declared entity gets fresh element $IDs that the digest numbers away and bytes do not; and comparing un-normalised digests cannot work for the reason normalisation was added (a rebuilt entity lacks what the writer completes). Count test: 2.0 digests per added statement before, 1.0 after. The seed: 7.1 s -> 6.4 s first run, 9.6 s -> 8.7 s settled re-run, output identical on four runs including one with a single changed statement. What is left in judgeHeld is normalizeForStorage, five passes that each unmarshal the document."} diff --git a/.claude/skills/fix-issue/findings/modelsdk/2026-11-12-listings-decode-every-unit-per-statement.json b/.claude/skills/fix-issue/findings/modelsdk/2026-11-12-listings-decode-every-unit-per-statement.json new file mode 100644 index 000000000..0e48c8d9c --- /dev/null +++ b/.claude/skills/fix-issue/findings/modelsdk/2026-11-12-listings-decode-every-unit-per-statement.json @@ -0,0 +1 @@ +{"area": "modelsdk/mprread", "date": "2026-11-12", "symptom": "mxcli exec of a script grows faster than its statement count on a larger project: seeding 300 entities, 600 microflows and 600 pages into PedApp took 45 s, 42 s and 286 s per stage where 120/240/240 took 10/10/46 s, and 1,600 pages had not finished after 15 minutes (ako/mxcli#2160)", "cause": "both the check before an exec (execPreflight -> validateProgramWithWarnings) and the exec itself list whole document types once per statement (buildFlowSignatures -> ListMicroflows from two validators per page statement; buildEntityIndex, checkMemberTypeIndex, checkAssociationCatalog, execCreateAssociation -> ListDomainModels; execCreatePageV3 -> ListPages), and every listing went through mprread.decodeUnits, which decoded each unit from its bytes on every call: about 1,000 full microflow listings for 600 page statements, at 0.11 s each", "file": "modelsdk/mprread/decode_cache.go (new), modelsdk/mprread/generic.go (decodeUnits), modelsdk/mpr/reader.go (Reader.Memo)", "insight": "Time one listing before blaming it: 616 microflows decode in 0.11 s, so 107 s under ListMicroflows in the profile is a call COUNT, and the fix is at the one function every listing shares rather than at the eight call sites the profile shows. The cache is keyed by a unit's stored bytes (kept and compared with bytes.Equal), so a write needs no invalidation. Two things make sharing a decoded element sound, and each has a test that fails without it: every lazily decoded scalar is loaded before the element is shared (42 data races under -race otherwise - the catalog builds with a worker pool), and an entry whose element IsDirty is decoded again instead of served (UpdateOqlQueriesForMovedEntity sets a property on a listed element). A unit is only remembered on its first decode and kept on its second, so a command that lists a type once pays nothing. Measured on the PedApp fixture: 240 decodes per added page statement before, 2 after; the 300/600/600 seed 45/42/286 s -> 25/22/110 s. What the second profile shows next is not decoding: every listing re-reads the unit files (readMprContents, a fifth of the page stage) because each write clears the reader's whole content cache, then gen -> sdk conversion and the GC that follows from both. MXCLI_CPUPROFILE= profiles the real command; MXCLI_DECODE_CACHE=off rules the cache out."} diff --git a/.claude/skills/fix-issue/findings/modelsdk/2026-11-12-write-empties-whole-content-cache.json b/.claude/skills/fix-issue/findings/modelsdk/2026-11-12-write-empties-whole-content-cache.json new file mode 100644 index 000000000..6c50ccaaf --- /dev/null +++ b/.claude/skills/fix-issue/findings/modelsdk/2026-11-12-write-empties-whole-content-cache.json @@ -0,0 +1 @@ +{"area": "modelsdk/mpr", "date": "2026-11-12", "symptom": "after the decode cache (ako/mxcli#2160), a script creating 600 pages still spent a fifth of its 110 s in Reader.readMprContents: every listing after every write read every unit file of its type again, through the persisted store (a SQLite query and a zstd decode per unit) or os.ReadFile", "cause": "two things. Reader.InvalidateCache, which every write calls, emptied the whole in-memory content cache, although a write changes one unit. And a script run held no CacheUnitReads scope at all - only check's passes and describe did - so execution never had the cache to begin with", "file": "modelsdk/mpr/reader_units.go (readMprContentsUnder, InvalidateCache, buildUnitCache), modelsdk/mpr/reader.go (cachedContent, rowHashes), mdl/executor/executor.go (ExecuteProgramReportingStop)", "insight": "The Unit table already holds each unit's ContentsHash and the writer already updates it, so an entry stored under the row hash it was read with retires itself when its unit is rewritten and survives every other write - the same content-keyed rule as the decode cache, with nothing new to keep in sync. What InvalidateCache drops instead is the hashes: between a write and the next buildUnitCache nothing is known to be current, so nothing is served or stored. buildUnitCache must cache its own reads under the hash it has just scanned (readMprContentsUnder), or the unit-type index's read and the listing's read are two. A second-order effect cost an hour: a run that ends inside a cache scope flushes what it read to the persisted unit store, so a test counting UnitFileReads for an 'uncached' control afterwards sees store hits instead and reads as if nothing were read - count UnitFileReads + UnitStoreHits when the question is 'not served from memory'. Measured on PedApp: 120 file reads per added page statement before, 1 after."} diff --git a/.claude/skills/fix-issue/findings/modelsdk/2026-11-12-write-rescans-unit-table-and-modules.json b/.claude/skills/fix-issue/findings/modelsdk/2026-11-12-write-rescans-unit-table-and-modules.json new file mode 100644 index 000000000..c00e3e8fd --- /dev/null +++ b/.claude/skills/fix-issue/findings/modelsdk/2026-11-12-write-rescans-unit-table-and-modules.json @@ -0,0 +1 @@ +{"area": "modelsdk/mpr", "date": "2026-11-12", "symptom": "after the #2159 caches (ako/mxcli#2160), a quarter of a 600-microflow script was Reader.ListModules and Reader.buildUnitCache: every statement that created a document rescanned the whole Unit table and re-read and re-decoded every module of the project", "cause": "three things under one symptom. Every write called Reader.InvalidateCache, which marked the list of units stale, so the next read scanned the Unit table again for a write that changed one row. Reader.ListModules read each module's contents and unmarshalled the whole document to take its Name, which the unit index already holds. And Backend.ListModules then read each module's bytes a second time and decoded them with a fresh decoder to fill in the Marketplace details, outside the decode cache. The executor asks for the modules to rebuild its container hierarchy, which it drops after every created document (82 call sites of invalidateHierarchy)", "file": "modelsdk/mpr/reader_units.go (noteUnitWritten), modelsdk/mpr/writer_core.go (insertUnit, updateUnit), modelsdk/mpr/types.go (ListModules), mdl/backend/modelsdk/backend.go (enrichModule), modelsdk/mprread/generic.go (UnitByIDAs)", "insight": "Fix the rebuild's cost, not its 82 triggers: with the listing cheap, the executor can go on dropping its hierarchy after every create. For the writer, patch the reader's list for the two writes that dominate (v2 insert and update) and let everything it cannot account for - delete, move, a transaction commit, a stale list, an insert of a row the list already has - fall back to the full invalidation, so the fast path can only be as wrong as its two cases. Then the trap that cost a test run: a Go type registered under two storage names. mprread.UnitByID[*genPr.Module] matches the ONE name TypeNameOf returns (Projects$Module) and a project's modules are Projects$ModuleImpl, so the lookup found nothing, returned no error, and every module silently lost FromAppStore and its version. The test that asserted Atlas_Core's Marketplace details caught it; a test that only counted reads and decodes would have passed. When a count test guards a rewrite of a read, assert a value the read returns as well. Measured on PedApp: Unit table scans per added create statement 1 -> 0; a module listing inside a script run reads 0 files and decodes 0 units where it read two files and decoded one unit per module."} diff --git a/.claude/skills/mendix/check-syntax/SKILL.md b/.claude/skills/mendix/check-syntax/SKILL.md index bc5159011..b6adcadbf 100644 --- a/.claude/skills/mendix/check-syntax/SKILL.md +++ b/.claude/skills/mendix/check-syntax/SKILL.md @@ -409,8 +409,8 @@ Run `mxcli syntax keywords` for the full list of 320+ reserved keywords. **Always run these checks:** ```bash -# Step 1: Syntax check (no project needed) -./bin/mxcli check script.mdl +# Step 1: Syntax check (--no-project: do not auto-detect an .mpr in this directory) +./bin/mxcli check script.mdl --no-project # Step 2: reference validation (needs project) # Validates microflow bodies, entity/enum references, and widget tree references diff --git a/.claude/skills/mendix/fragments/SKILL.md b/.claude/skills/mendix/fragments/SKILL.md index 829dbf66d..28f448e5d 100644 --- a/.claude/skills/mendix/fragments/SKILL.md +++ b/.claude/skills/mendix/fragments/SKILL.md @@ -12,11 +12,11 @@ description: "Define reusable widget groups with CREATE FRAGMENT and place them |---|---| | 24-32 | When to Use This Skill | | 33-40 | What Are Fragments? | -| 41-189 | Syntax Reference | -| 190-259 | Common Patterns | -| 260-306 | Common Mistakes | -| 307-316 | Validation Checklist | -| 317-322 | Related Documentation | +| 41-194 | Syntax Reference | +| 195-264 | Common Patterns | +| 265-311 | Common Mistakes | +| 312-321 | Validation Checklist | +| 322-327 | Related Documentation | # Mendix Fragments Skill @@ -170,7 +170,12 @@ use building block Atlas_Web_Content.List_Cards ``` Binding-point rule (prototype): datasource → the first widget carrying a -datasource; action → the first button widget. For anything more specific, copy +datasource; action → the first core button (`actionbutton`, `linkbutton`); in a +block with none, the first pluggable button (a widget whose definition names it +a button and takes `OnClick:`, such as PDS Button); failing that, the first +other pluggable widget with a click action, such as an Image. A PDS Link before +a PDS Button is skipped; a gallery or data grid row click never counts. +For anything more specific, copy the block in with `as prefix_` and use `alter page … set … on prefix_widget`. ### LIST FRAGMENTS diff --git a/.claude/skills/mendix/write-lint-rules/SKILL.md b/.claude/skills/mendix/write-lint-rules/SKILL.md index 05a9fcc1b..0b73dbabf 100644 --- a/.claude/skills/mendix/write-lint-rules/SKILL.md +++ b/.claude/skills/mendix/write-lint-rules/SKILL.md @@ -11,11 +11,11 @@ description: "Write custom Starlark lint rules in .claude/lint-rules/ that run b | lines | section | |---|---| | 25-67 | Rule File Structure | -| 68-167 | Available Query Functions | -| 168-574 | Object Properties | -| 575-588 | Helper Functions | -| 589-675 | Common Patterns | -| 676-684 | Validation | +| 68-170 | Available Query Functions | +| 171-577 | Object Properties | +| 578-591 | Helper Functions | +| 592-678 | Common Patterns | +| 679-687 | Validation | # Writing Custom Starlark Lint Rules @@ -111,10 +111,13 @@ Without this, a rule that reads a full-only table under a fast build gets | `strings(language = None)` | list of catalog_string | User-facing and documentary text, one row per text and language; pass `language` (`"nl_NL"`) to narrow. An untranslated language has **no row**. Needs a FULL catalog, which `mxcli lint` builds automatically for a rule that calls it | | `layouts()` | list of layout | All non-system layouts | | `published_rest_operations()` | list of published_rest_operation | Operations of published REST services, with the microflow behind each | +| `workflows()` | list of workflow | The user's workflows, with their parameter entity and activity counts | +| `workflow_activities()` | list of workflow_activity | Every activity of the user's workflows — boundary-event paths and event sub-processes included — with its kind and the microflow, page or workflow each slot names. Use it for "a system task's microflow is prefixed `WF_`": `refs_to()` says only that the workflow calls a microflow, not from which kind of activity | The fields of `module`, `folder`, `annotation`, `association`, `entity_event_handler`, -`navigation_menu_item`, `jar_dependency`, `jar_file`, `catalog_string`, `layout` and -`published_rest_operation` are in [catalog-tables.md](catalog-tables.md). +`navigation_menu_item`, `jar_dependency`, `jar_file`, `catalog_string`, `layout`, +`published_rest_operation`, `workflow` and `workflow_activity` are in +[catalog-tables.md](catalog-tables.md). ### Graph-analysis functions (architecture rules) diff --git a/.claude/skills/mendix/write-lint-rules/catalog-tables.md b/.claude/skills/mendix/write-lint-rules/catalog-tables.md index 4d4262381..0a3a686b8 100644 --- a/.claude/skills/mendix/write-lint-rules/catalog-tables.md +++ b/.claude/skills/mendix/write-lint-rules/catalog-tables.md @@ -177,6 +177,41 @@ Returned by `published_rest_operations()`. | `deprecated` | bool | Marked deprecated | | `module_name` | string | `"Sales"` | +### workflow +Returned by `workflows()`. + +| Property | Type | Example | +|----------|------|---------| +| `id` | string | Workflow ID | +| `qualified_name` | string | `"Sales.ApproveOrder"` | +| `name` | string | `"ApproveOrder"` | +| `module_name` | string | `"Sales"` | +| `description` | string | Documentation text | +| `parameter_entity` | string | Qualified name of the context entity: `"Sales.Order"` | +| `activity_count` | int | Activities in every flow of the workflow, boundary-event paths and event sub-processes included | +| `user_task_count` | int | User tasks and multi-user tasks | +| `microflow_call_count` | int | System tasks and call-microflow activities | +| `decision_count` | int | Decisions | + +### workflow_activity +Returned by `workflow_activities()`, each workflow's activities in walk order: the main flow, each activity followed by the paths of its boundary events, then the event sub-processes. A slot that names nothing is `""`. + +| Property | Type | Example | +|----------|------|---------| +| `id` | string | Activity ID | +| `workflow_qualified_name` | string | `"Sales.ApproveOrder"` | +| `module_name` | string | `"Sales"` | +| `name` | string | `"ReviewOrder"` | +| `caption` | string | `"Review the order"` | +| `activity_type` | string | `"UserTask"`, `"MultiUserTask"`, `"SystemTask"`, `"CallMicroflow"`, `"AIAgentTask"`, `"CallWorkflow"`, `"Decision"`, `"ParallelSplit"`, `"JumpTo"`, `"WaitForTimer"`, `"WaitForNotification"`, `"StartWorkflow"`, `"EndWorkflow"`, … | +| `microflow` | string | The microflow a `SystemTask`, `CallMicroflow` or `AIAgentTask` runs | +| `user_source_microflow` | string | A user task's targeting microflow (users or groups) | +| `on_created_microflow` | string | A user task's on-created microflow | +| `completion_microflow` | string | A multi-user task's completion microflow | +| `page` | string | A user task's page | +| `user_task_entity` | string | A user task's task entity | +| `called_workflow` | string | The workflow a `CallWorkflow` activity calls | + ### widget Returned by `widgets()` (full catalog — auto-detected). diff --git a/.claude/skills/mendix/xpath-constraints/SKILL.md b/.claude/skills/mendix/xpath-constraints/SKILL.md index a7bb82dea..002b3f897 100644 --- a/.claude/skills/mendix/xpath-constraints/SKILL.md +++ b/.claude/skills/mendix/xpath-constraints/SKILL.md @@ -460,8 +460,8 @@ grant read * on entity Module.Item to Module.User where [System.owner = '[%Curre Always validate XPath syntax before execution: ```bash -# Syntax check (no project needed) -./bin/mxcli check script.mdl +# Syntax check only (--no-project: do not auto-detect an .mpr in this directory) +./bin/mxcli check script.mdl --no-project # with reference validation (needs project) ./bin/mxcli check script.mdl -p app.mpr --references diff --git a/CHANGELOG.md b/CHANGELOG.md index 52aab8276..b585ac504 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added +- **`CATALOG.WORKFLOW_ACTIVITIES`, and `workflows()` / `workflow_activities()` for lint rules** (ako/mxcli#1295, mendixlabs/mxcli#1269) — one row per workflow activity, boundary-event paths and event sub-processes included, with its kind (user task, multi-user task, system task, call microflow, AI agent task, call workflow, …) and the microflow, page or workflow each of its slots names. A rule can now check "a system task's microflow is prefixed `WF_SYS_`": `refs` records only that the workflow calls the microflow, not from which activity. Catalog schema 48. - **`make bench`: a benchmark of what an agent waits for, and `MXCLI_STATS_FILE`** — a fixed session of mxcli commands (open, author entities, microflows and pages, describe, catalog refresh, check, a one-document edit, a bulk change, a re-run, a rename), one process per command, on PedApp and on PedApp with a generated app seeded into it. `make bench BENCH_RUNTIME=` adds a session that validates and builds the project with mx and then runs it — `docker check`, `docker build`, boot, a hot change, an entity change, a page check, an attached and a cold test run — on a project `mxcli new` creates, with mxbuild and the runtime installed by mxcli when they are not cached. With `BASELINE=` the two binaries run interleaved on the same machine and each step reports the ratio; the nightly `bench` job does that against the previous nightly and commits the record to the `benchmarks` branch. It also records counts that do not depend on the machine: unit files read and unit files changed. `MXCLI_STATS_FILE=` makes any mxcli command append one JSON line per closed project connection with its unit-file reads and its writes offered and landed. See "Agent Dev-Loop Benchmark" in the docs. - **`mxcli test --local --reuse-build` skips the injection and the build when nothing changed** (mendixlabs/mxcli#1783, closes mendixlabs/mxcli#1782) — a local test run injects its test microflows, builds and boots; when the project, the test files and mxcli are unchanged, the deployment the previous run left is already this run's build, so `--reuse-build` boots it directly (about 12 s against 60 s, measured by the author). The decision is a content hash of the project folder (outputs, caches, `theme-cache/`, and any `.git`/`node_modules` left out), the generated test MDL, the mxcli binary and the mxbuild override, pinned to the identity of `deployment/model/model.mdp`, so a deployment rebuilt by anything else is never taken for the test build. Not under `--watch` or `--skip-build`. - **A `genai-runtime` skill for the runtime side of Agents Kit 2** (ako/mxcli#2068) — registering a Mendix Cloud GenAI key or an OpenAI-compatible endpoint, seeding AgentCommons agents from JSON (matched by `Title` only, so a changed JSON never reaches a stored agent), resuming Conversational UI chats, loading PgVector and Mendix Cloud knowledge bases, and traces. It collects the gotchas that `check` and `mx check` cannot see: the encryption key both the OpenAI and PgVector setup need, the `vector` extension only the UI's Test button creates, the 60–120 s indexing delay, the chat that collapses outside its page, and `StoreTraces` defaulting to false. Its examples pass `check --references` and build with 0 errors against the Agent Builder Starter App's modules. @@ -24,10 +25,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Changed +- **A catalog build prints a line per table only on a terminal, and `-q` / `--quiet` works on every command** (ako/mxcli#1860, mendixlabs/mxcli#1702) — the first command after each model change rebuilt the catalog and printed about fifty `✓ Table: N` lines ahead of its answer, which an agent paid for in context every time. Off a terminal a quick build now prints none of them and a slow one a heartbeat every 15 s; the incremental summary and warnings still print. `-q` was honoured by the banner check and by `search` and rejected as an unknown flag by every other command. - **`docker build`, `docker run` and `docker reload` check and build the project as stored** (ako/mxcli#1794) — they ran `mx update-widgets` on their temporary copy first, so a project whose stored widget definitions are stale built, while `docker check`, `run --local` and Studio Pro all reported the CE0463s: the deployment was not built from what is committed. A stale widget now fails the build, each CE0463 labelled as one `mxcli fix widgets` clears or one that persists, as `docker check` labels them. `--update-widgets` builds a widget-normalised copy as before, leaving the project as it is; `--no-update-widgets` skips the labelling re-check. ### Fixed +- **A run of entity and association statements digests each domain model write once** (ako/mxcli#2160) — to tell whether a statement changed anything, its write was compared with the one the run already held, and both were digested each time: the whole domain model twice per statement. The held write's digest is now kept. What each statement reports is unchanged. Seeding 300 entities and 299 associations: 7.1 s before, 6.4 s after; re-running the settled script: 9.6 s before, 8.7 s after. +- **`describe ` resolves a module role, a user role and a navigation profile without a type** (ako/mxcli#1262, mendixlabs/mxcli#658) — the auto-detect looked at documents only, so `describe Administration.Administrator` answered "no describable document named …" for a role `describe module role` describes, and likewise for `describe Administrator` (a user role) and `describe Responsive` (a navigation profile). +- **`create entity` and `create association` read one module's domain model where they listed the project's** (ako/mxcli#2160) — finding an association's two entities, and checking that a new entity's or association's name is free in its module, each listed and converted every domain model of the project: ten listings for each added entity and its association. Both now read the named module's domain model; the refusals and their messages are unchanged. One listing per statement remains, for the member-name check (CE0069), which follows generalizations across modules. Seeding 300 entities and 299 associations: 9.7 s before, 8.4 s after; re-running the settled script: 14.5 s before, 9.6 s after. +- **A run of `create entity` and `create association` statements writes each domain model once** (ako/mxcli#2160) — every one of those statements rewrote its module's whole domain model and reconciled the rewrite against the stored unit, half the time of a script that builds a domain model and a cost per statement that grows with the model. Consecutive entity and association statements are now held and written when the run ends; any other statement ends the run first. Each statement still reports for itself (`Created`, `Modified`, or unchanged), and a statement that fails leaves the earlier ones written. What changes is when a write can be refused: the comparison with the stored domain model, and the storage-GUID guard in it, runs at the end of the run, so a refusal there is reported after the run's statements and names the domain model that was not written. Seeding 300 entities and 299 associations: 17 s before, 10 s after. +- **A statement that writes no longer rescans the project's unit table or re-reads its modules** (ako/mxcli#2160) — every write marked the reader's list of units stale, so the next read scanned the whole Unit table again, and the module listing that follows (to rebuild the container hierarchy) read every module's file twice and decoded it for its name and Marketplace details. A write now updates the list for the unit it wrote, a module's name comes from the unit index, and its details from the decode cache. Seeding 300 entities, 600 microflows and 600 pages into a 369-unit project: 20 s, 14 s and 44 s per stage before, 16 s, 9 s and 36 s after (45 s, 42 s and 286 s before ako/mxcli#2159). +- **A script no longer decodes the project once per statement** (ako/mxcli#2160) — checking and executing a script listed whole document types for every statement (every microflow to resolve one call, every domain model to resolve one entity), and each listing decoded every unit from its bytes: about a thousand full microflow listings for 600 page statements. A unit decoded twice is now kept and served until its stored bytes change. A script run also keeps the bytes of each unit file it has read, and a write retires only the unit it rewrote, where it used to empty the whole cache so that every listing read every file of its type again. The check before an exec lists the domain models, microflows and nanoflows once per run instead of several times per statement, and loads a pluggable widget's template once per run. Seeding 300 entities, 600 microflows and 600 pages into a 369-unit project went from 45 s, 42 s and 286 s per stage to 20 s, 14 s and 44 s; on that fixture each added page statement decodes 2 units where it decoded 240. `MXCLI_DECODE_CACHE=off` turns the cache off, and `MXCLI_CPUPROFILE=` writes a CPU profile of any command. +- **A REPL session sees what another process wrote to the project** (ako/mxcli#2121) — a session read the project once and kept answering from that state, so a write by a second `mxcli`, Studio Pro or a git checkout never showed up: `show microflows` kept the old set, `describe microflow` of a flow made elsewhere said `microflow not found`, `create entity` in a module made elsewhere said `module not found`, and on MPR v2 a `create microflow` of a name another process had taken wrote a second document with that name. Before each statement the session now asks whether another connection committed to the `.mpr` (SQLite's `data_version`) or the file was replaced (a git checkout), and when so drops what it kept and reads afresh; if the file was replaced it is reopened. The probe is one query and one stat, and the session's own writes do not trip it. Not covered: a change made while a statement runs, and on MPR v2 a change to `mprcontents/` that leaves the `.mpr` untouched. +- **`use building block … (action: …)` rebinds a pluggable button** (mendixlabs/mxcli#1821, mendixlabs/mxcli#1822, ako/mxcli#2145) — every block of the PlatformDesignSystem Marketplace module was refused with "action override supplied, but the block has no button to rebind", because the PDS blocks are built from the pluggable `pdsbutton` and the override looked only for the built-in button types. The override now takes, in this order, the block's first built-in button; its first pluggable button, a widget whose definition names it a button and has a click action; or its first other pluggable widget with a click action, such as the Image of Atlas's `Card_WithImage`. A data grid's or gallery's row click is never taken. The order matters on a PDS page header, whose first clickable widget is a "Back to" PDS Link: the action goes to the primary button after it. Without a project, only the built-in buttons match, as before. Reproduced on Mendix 11.12.6 with PlatformDesignSystem 44.4.0 (by Meisam Mahdian), and on 11.14.0 with `Card_WithImage`. - **`run --local --watch` builds a change as soon as `mxcli exec` or `mxcli -c` has finished writing it** (ako/mxcli#2151) — the watcher noticed a change within a second and then waited for the model to stay unchanged for two more, so a change was applied about 5 s after `exec` started although the build and reload took 2 s. `exec` and `-c` now tell a watching dev loop when they start and when the last write has landed: nothing is built while one is at work, however long the script, and the build starts within 100 ms of its end. Measured on a blank 11.16.0 app: 2.2-2.5 s from the start of `exec` to applied, for a microflow edit, a page edit and a new page. A change made by anything else (an editor, Studio Pro) is settled as before. - **A test can call a microflow that applies entity access** (ako/mxcli#2100, mendixlabs/mxcli#1805) — the generated test microflows, the legacy runner's `MxTest.TestRunner` and the wrappers `mxcli check` sees did not apply entity access, so a test (or its `@setup`) calling a production microflow that does failed with CE0114 at build time and MDL-SEC22 at check time, and the only workaround was weakening the production microflow. They now apply it, as `MxTest.RegisterEndpoint` already did. An applying microflow may not read `$latestError/Message` (CE0150), so the runner's error handlers read the message through a generated helper, `MxTest.Test_ErrorMessage`, that does not apply entity access and is removed with the test microflows; failure messages and `@throws` matching are unchanged. A test whose own error handler reads `$latestError/…` keeps a non-applying microflow, so it builds as before. Tests run in a system context, so nothing changes at run time. - **`run --local --watch` applies a change in about 5 s instead of 20-28 s on Mendix versions that bundle the client themselves** (ako/mxcli#2150) — measured on a blank 11.16.0 app: a microflow edit took 20-22 s and a page edit 25-28 s with a runtime restart that dropped every browser session, because each change cost a fresh `mxbuild --serve` and a cold build. mxbuild deletes the client source a build wrote, so its next build bundled a client without pages. That source is now kept while a build runs and put back before the next one: the incremental build takes about 2 s, a page edit is a reload, and an open browser tab shows it without a restart or a new login. The full build remains the fallback when the bundle still comes out incomplete. A deleted page's module stays in `dist/pages`, unused, until the next full build. On Windows the copies are read without blocking mxbuild's own deletes, and if a build still fails because `deployment/web` was in use, the kept source is given up for that run and builds go back to the full rebuild. diff --git a/cmd/mxcli/catalog_progress_1860_test.go b/cmd/mxcli/catalog_progress_1860_test.go new file mode 100644 index 000000000..928d89e30 --- /dev/null +++ b/cmd/mxcli/catalog_progress_1860_test.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// ako/mxcli#1860 (mendixlabs/mxcli#1702): after a model change the next +// command rebuilt the catalog and printed a line per table — about fifty — +// ahead of its answer, and an agent paid for them in context every time. +// There was no way to turn them off: `-q` was honoured by the banner check and +// by `search`, and rejected by every other command. + +// perTableLines are the build's "✓ Table: N" lines in out. +func perTableLines(out string) []string { + var got []string + for _, l := range strings.Split(out, "\n") { + if strings.HasPrefix(l, "✓ ") && !strings.HasPrefix(l, "✓ Catalog") { + got = append(got, l) + } + } + return got +} + +// Not a terminal (this test's pipes): a quick build prints no line per table. +func TestCatalogBuildOffTerminalPrintsNoLinePerTable_Issue1860(t *testing.T) { + if testing.Short() { + t.Skip("spawns mxcli against a project") + } + mpr := jsonPurityFixture(t) + r := runMainStreams(t, filepath.Dir(mpr), "-p", mpr, "-c", "select count(*) from CATALOG.ENTITIES") + if r.exitCode != 0 { + t.Fatalf("exit %d\nstderr:\n%s", r.exitCode, r.stderr) + } + if _, err := os.Stat(filepath.Join(filepath.Dir(mpr), ".mxcli", "catalog.db")); err != nil { + t.Fatalf("the command built no catalog, so this proves nothing: %v", err) + } + if !strings.Contains(r.stdout, "count(*)") { + t.Errorf("no answer on stdout:\n%s", r.stdout) + } + if lines := perTableLines(r.stdout + r.stderr); len(lines) > 0 { + t.Errorf("a build off a terminal printed %d per-table lines, e.g. %q", len(lines), lines[0]) + } +} + +// `-q` is a flag of every command, and `exec -q` says nothing about +// connecting. A run without it is the control: same script, same project. +func TestQuietFlagOnEveryCommand_Issue1860(t *testing.T) { + if testing.Short() { + t.Skip("spawns mxcli against a project") + } + mpr := jsonPurityFixture(t) + dir := filepath.Dir(mpr) + script := filepath.Join(dir, "q.mdl") + if err := os.WriteFile(script, []byte("show modules;\n"), 0o644); err != nil { + t.Fatal(err) + } + + loud := runMainStreams(t, dir, "exec", script, "-p", mpr) + if loud.exitCode != 0 || !strings.Contains(loud.stdout, "Connected to:") { + t.Fatalf("control: exec without -q should report its connection (exit %d)\nstdout:\n%s\nstderr:\n%s", + loud.exitCode, loud.stdout, loud.stderr) + } + for _, args := range [][]string{ + {"exec", script, "-p", mpr, "-q"}, + {"--quiet", "exec", script, "-p", mpr}, + {"-q", "-p", mpr, "-c", "show modules"}, + } { + r := runMainStreams(t, dir, args...) + if r.exitCode != 0 { + t.Errorf("mxcli %v: exit %d\nstderr:\n%s", args, r.exitCode, r.stderr) + continue + } + if strings.Contains(r.stdout+r.stderr, "Connected to:") { + t.Errorf("mxcli %v: reported its connection under -q:\n%s%s", args, r.stdout, r.stderr) + } + if !strings.Contains(r.stdout, "System") { + t.Errorf("mxcli %v: the answer went missing:\n%s", args, r.stdout) + } + } +} diff --git a/cmd/mxcli/check_no_project_test.go b/cmd/mxcli/check_no_project_test.go new file mode 100644 index 000000000..36246d57d --- /dev/null +++ b/cmd/mxcli/check_no_project_test.go @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// ako/mxcli#2135 (mendixlabs/mxcli#1817): `mxcli check --help` offered +// +// # Check syntax only (no project needed) +// mxcli check script.mdl +// +// but run from a directory holding an .mpr, that invocation printed +// "Using project: ...\project.mpr (auto-detected in the current directory; pass +// -p to use another)" and resolved every reference against it — the global +// auto-detection sets -p before check runs — and nothing said how to get the +// syntax-only check the help advertised. Auto-detection stays (it is the CLI's +// documented resolution, mendixlabs/mxcli#1444); --no-project is the opt-out. + +// noProjectFixture is a working directory holding exactly one .mpr, which +// auto-detection would pick, and a syntactically valid script. The .mpr is not +// a project, so any attempt to open it shows up as an error. +func noProjectFixture(t *testing.T) string { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "Stray.mpr"), []byte("not a project"), 0o644); err != nil { + t.Fatal(err) + } + script := "create persistent entity Shop.Customer (Name: string(100));\n" + if err := os.WriteFile(filepath.Join(dir, "script.mdl"), []byte(script), 0o644); err != nil { + t.Fatal(err) + } + return dir +} + +func TestCheckNoProject_SkipsAutoDetectedProject(t *testing.T) { + dir := noProjectFixture(t) + + r := runMainStreams(t, dir, "check", "script.mdl", "--no-project") + all := r.stdout + r.stderr + if strings.Contains(all, "Using project:") { + t.Errorf("--no-project still auto-detected the project:\n%s", all) + } + if strings.Contains(all, "Validating references against") { + t.Errorf("--no-project still resolved references:\n%s", all) + } + if r.exitCode != 0 { + t.Errorf("exit %d, want 0 for a syntactically valid script:\n%s", r.exitCode, all) + } + if !strings.Contains(r.stdout, "no project given") { + t.Errorf("verdict does not say no project was used:\n%s", r.stdout) + } +} + +// Control for the test above: the same directory without the flag is the +// reported behaviour, and it is kept — the auto-detected project is used and +// named. If this stops printing "Using project:", the fixture no longer +// exercises auto-detection and the test above proves nothing. +func TestCheckNoProject_DefaultStillAutoDetects(t *testing.T) { + dir := noProjectFixture(t) + + r := runMainStreams(t, dir, "check", "script.mdl") + if !strings.Contains(r.stderr, "Using project:") { + t.Fatalf("default check no longer auto-detects the project:\n%s%s", r.stdout, r.stderr) + } + // The notice is where the user meets the guess, so it names the way out. + if !strings.Contains(r.stderr, "--no-project") { + t.Errorf("auto-detection notice on check does not mention --no-project:\n%s", r.stderr) + } +} + +// --no-project with a flag that needs a project is a contradiction, refused +// before anything is checked rather than letting one silently win. +func TestCheckNoProject_ConflictsWithProjectFlags(t *testing.T) { + dir := noProjectFixture(t) + for _, extra := range [][]string{ + {"-p", "Stray.mpr"}, + {"--references"}, + {"--post-migration"}, + } { + args := append([]string{"check", "script.mdl", "--no-project"}, extra...) + r := runMainStreams(t, dir, args...) + if r.exitCode == 0 { + t.Errorf("%v: exit 0, want a refusal", extra) + } + if !strings.Contains(r.stderr, "--no-project") { + t.Errorf("%v: refusal does not name --no-project:\n%s", extra, r.stderr) + } + if strings.Contains(r.stdout, "Checking syntax") { + t.Errorf("%v: checked the script before refusing:\n%s", extra, r.stdout) + } + } +} + +// The help is the contract: its syntax-only example has to be the invocation +// that is syntax-only wherever it is run, and it has to say a project in the +// current directory is otherwise picked up. +func TestCheckHelp_SyntaxOnlyExampleMatchesBehaviour(t *testing.T) { + long := checkCmd.Long + if strings.Contains(long, "no project needed)\n mxcli check script.mdl\n") { + t.Errorf("help still offers a bare `mxcli check script.mdl` as syntax-only") + } + if !strings.Contains(long, "mxcli check script.mdl --no-project") { + t.Errorf("help has no --no-project example") + } + if !strings.Contains(long, "auto-detect") { + t.Errorf("help does not say a project in the current directory is auto-detected") + } + if checkCmd.Flags().Lookup("no-project") == nil { + t.Errorf("check has no --no-project flag") + } +} diff --git a/cmd/mxcli/cmd_check.go b/cmd/mxcli/cmd_check.go index b1377de35..64e7ce1bd 100644 --- a/cmd/mxcli/cmd_check.go +++ b/cmd/mxcli/cmd_check.go @@ -24,9 +24,15 @@ var checkCmd = &cobra.Command{ Long: `Check an MDL script file for syntax errors and optionally validate references. Without a project it checks syntax and the semantic rules that need no model. -Pass -p and it also resolves every reference — modules, entities, pages, -microflows and icons — against that project; --references is implied by -p and -is kept only for compatibility. +With a project it also resolves every reference — modules, entities, pages, +microflows and icons — against that project; --references is implied by a +project and is kept only for compatibility. + +The project is the one passed with -p or, without -p, the single .mpr in the +current directory, auto-detected as for every mxcli command (the "Using +project:" line says which). Run from a project directory, a plain +"mxcli check script.mdl" therefore checks references too. Pass --no-project to +check syntax and the no-model rules only, wherever it is run. It also resolves MEMBER names, not just the entity they belong to: an attribute named in a create/change activity is looked up on that entity and its @@ -122,12 +128,15 @@ for legacy native widgets that have pluggable replacements — Studio Pro does not auto-migrate these on a Mendix major-version upgrade. Examples: - # Check syntax only (no project needed) - mxcli check script.mdl + # Check syntax only, even in a directory holding an .mpr + mxcli check script.mdl --no-project # Check syntax, resolve references, and type-check expressions mxcli check script.mdl -p app.mpr + # The same, against the .mpr auto-detected in the current directory + mxcli check script.mdl + # Scan the project for legacy native widgets after a Mendix upgrade mxcli check script.mdl -p app.mpr --post-migration @@ -150,12 +159,32 @@ Examples: `, Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { + if err := checkNoProjectConflicts(cmd); err != nil { + fmt.Fprintln(os.Stderr, "Error:", err) + os.Exit(1) + } if code := runCheckFiles(cmd, args); code != 0 { os.Exit(code) } }, } +// checkNoProjectConflicts refuses --no-project together with a flag that asks +// for a project, rather than letting one of the two silently win. -p is only +// "changed" when the user gave it: auto-detection does not run under +// --no-project, so it cannot set it. +func checkNoProjectConflicts(cmd *cobra.Command) error { + if !skipsProjectDiscovery(cmd) { + return nil + } + for _, name := range []string{"project", "references", "post-migration"} { + if cmd.Flags().Changed(name) { + return fmt.Errorf("--%s asks for a project and --%s for none; pass one or the other", name, noProjectFlag) + } + } + return nil +} + // runCheckFile checks one script and returns the exit code: 0 when it passed. func runCheckFile(cmd *cobra.Command, filePath string) int { return runCheckFileAgainst(cmd, filePath, "") diff --git a/cmd/mxcli/cmd_describe.go b/cmd/mxcli/cmd_describe.go index 04391eb1d..99f6051c0 100644 --- a/cmd/mxcli/cmd_describe.go +++ b/cmd/mxcli/cmd_describe.go @@ -11,7 +11,9 @@ import ( modelsdk "github.com/mendixlabs/mxcli" "github.com/mendixlabs/mxcli/mdl/catalog" "github.com/mendixlabs/mxcli/mdl/executor" + "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/mdl/visitor" + "github.com/mendixlabs/mxcli/sdk/security" "github.com/spf13/cobra" ) @@ -26,9 +28,11 @@ repeat the header, which is allowed. --mdl 0 writes the alpha language instead, with no header (ako/mxcli#840). The is optional for a qualified document name: 'describe MyModule.Customer' -auto-detects the document type. Pass the type explicitly to disambiguate (a name -can match both an entity and a document, for example) or for forms that have no -single qualified name (module, settings, navigation, module role). +auto-detects the document type. So it is for a module role ('describe +Administration.Administrator'), a user role and a navigation profile +('describe Responsive'). Pass the type explicitly to disambiguate (a name can +match both an entity and a document, for example) or for forms that have no +single name (module, settings). Types: module Describe a module and its module roles (--all: everything in it) @@ -587,5 +591,52 @@ func resolveViaReader(projectPath, name string) ([]string, error) { } } } + out = append(out, describableMembers(reader, moduleName, name)...) return out, nil } + +// describableMembers is the `describe` type of each member named name that +// is not a document of its own: a module role (Module.Role), a user role and +// a navigation profile (both bare names). Each has a describe type, but the +// auto-detect saw only documents, so `describe Administration.Administrator` +// answered "no describable document named …" for a role `describe module +// role` describes (ako/mxcli#1262). +// +// The catalog's objects view does not list them, and has no complete table to +// list module roles from (role_mappings holds only the mapped ones), so these +// are resolved here, by the live scan a catalog miss falls back to. +func describableMembers(reader memberReader, moduleName map[string]string, name string) []string { + var out []string + if mss, err := reader.ListModuleSecurity(); err == nil { + for _, ms := range mss { + mn := moduleName[string(ms.ContainerID)] + for _, r := range ms.ModuleRoles { + if mn+"."+r.Name == name { + out = append(out, "modulerole") + } + } + } + } + if ps, err := reader.GetProjectSecurity(); err == nil && ps != nil { + for _, r := range ps.UserRoles { + if r.Name == name { + out = append(out, "userrole") + } + } + } + if nav, err := reader.GetNavigation(); err == nil && nav != nil { + for _, p := range nav.Profiles { + if p.Name == name { + out = append(out, "navigation") + } + } + } + return out +} + +// memberReader is the part of the backend describableMembers reads. +type memberReader interface { + ListModuleSecurity() ([]*security.ModuleSecurity, error) + GetProjectSecurity() (*security.ProjectSecurity, error) + GetNavigation() (*types.NavigationDocument, error) +} diff --git a/cmd/mxcli/cmd_run_lifecycle_integration_test.go b/cmd/mxcli/cmd_run_lifecycle_integration_test.go index f780c3368..78413ffe9 100644 --- a/cmd/mxcli/cmd_run_lifecycle_integration_test.go +++ b/cmd/mxcli/cmd_run_lifecycle_integration_test.go @@ -38,12 +38,15 @@ import ( // nightly runs this against each matrix leg's own version — which is how it // found that --watch could not start on 10.24 / 11.6. // -// On Mendix 11.14+ every build after the first in one mxbuild --serve process -// fails (an mxbuild regression, measured without mxcli: webclient_legacy_paths.go). -// mxcli recovers by restarting mxbuild --serve and building on the fresh process -// (serve_recovery.go), so build #2 still applies there; the steps that wait for it -// also assert that the run log shows that recovery. When an mxbuild fixes the -// regression, those assertions fail and this branch should go. +// On the Mendix versions that bundle the client themselves (11.14+ here), a +// build in an mxbuild --serve process deletes the client source it wrote, so the +// next build in that process failed (11.14) or bundled no pages (11.15+), and +// mxcli answered every change with a serve restart and a cold build. Since +// ako/mxcli#2150 it keeps that source between builds (client_source_keep.go), so +// a page change is a reload on every version again, and step 4 asserts the run +// log shows no serve restart. Every fallback that restarts the serve says +// "restarting mxbuild --serve" (serve_recovery.go): a failure naming it means the +// keep did not hold what the build needed. func TestRunLifecycle_DetachStatusExecWaitStop(t *testing.T) { version := os.Getenv("MXCLI_RUNLIFE_VERSION") if version == "" { @@ -72,7 +75,6 @@ func TestRunLifecycle_DetachStatusExecWaitStop(t *testing.T) { t.Skipf("mx create-project: %v\n%s", err, out) } mpr := filepath.Join(work, "App.mpr") - serveRebuildsBroken := projectAtLeast(t, mpr, 11, 14) const appPort, adminPort, servePort = 8287, 8297, 6743 for _, p := range []int{appPort, adminPort, servePort} { @@ -110,21 +112,22 @@ func TestRunLifecycle_DetachStatusExecWaitStop(t *testing.T) { return f } - // wantBuild2 checks a `run wait` that should report build #2 applied. On - // 11.14+ that build only applies because mxcli restarted mxbuild --serve after - // the incremental build failed, so the run log must show that recovery. - // orDirect drops that requirement, for a step whose claim is only that build - // #2 ran. + // wantBuild2 checks a `run wait` that should report build #2 applied and, + // unless orDirect, that mxcli did not restart mxbuild --serve to get there: on + // 11.14+ that restart is the fallback for client source the keep did not hold, + // a cold build per change instead of an incremental one. orDirect drops that + // requirement, for a step whose claim is only that build #2 ran. wantBuild2 := func(step, out string, code int, orDirect bool) { t.Helper() if code != 0 || !strings.HasPrefix(out, "applied: build #2") { t.Fatalf("%s: exit %d, %q\n%s", step, code, out, tailOf(docker.RunLogPath(mpr))) } - if serveRebuildsBroken && !orDirect { + if !orDirect { log, _ := os.ReadFile(docker.RunLogPath(mpr)) - if !strings.Contains(string(log), "cannot build this change incrementally") { - t.Fatalf("%s (Mendix 11.14+): build #2 applied without mxcli restarting mxbuild --serve; "+ - "if this mxbuild builds incrementally again, drop the 11.14+ branch\n%s", step, tailOf(docker.RunLogPath(mpr))) + if strings.Contains(string(log), "restarting mxbuild --serve") { + t.Fatalf("%s: build #2 applied only after mxcli restarted mxbuild --serve; the client source kept "+ + "between builds (client_source_keep.go, ako/mxcli#2150) should let it build incrementally\n%s", + step, tailOf(docker.RunLogPath(mpr))) } } } @@ -157,7 +160,7 @@ func TestRunLifecycle_DetachStatusExecWaitStop(t *testing.T) { } out, code = mxcli("run", "wait", "-p", mpr, "--timeout", "3m") wantBuild2("wait", out, code, false) - if !serveRebuildsBroken && !strings.HasPrefix(out, "applied: build #2 via reload") { + if !strings.HasPrefix(out, "applied: build #2 via reload") { t.Fatalf("wait: applied, but not via reload: %q", out) } // Waiting again with nothing changed reports the same build at once. @@ -207,11 +210,10 @@ func TestRunLifecycle_DetachStatusExecWaitStop(t *testing.T) { if err := bg.Wait(); err != nil { t.Fatalf("detach (boot race): %v\n%s", err, tailOf(docker.RunLogPath(mpr))) } - // What this step claims is that the change is not lost: build #2 applies. On - // 11.14+ this build was measured applying without a serve restart where step - // 4's needed one; the likely difference is how much of the change the boot - // build had already read, which the test does not control. So the recovery is - // not required in the log here; "nothing to apply" never passes. + // What this step claims is that the change is not lost: build #2 applies; + // "nothing to apply" never passes. A serve restart is not ruled out here: the + // change lands while the boot build runs, and how much of it that build had + // already read, and so what the keep holds, is timing the test does not control. out, code = mxcli("run", "wait", "-p", mpr, "--timeout", "2m") wantBuild2("a change made during the boot", out, code, true) if out, code := mxcli("run", "stop", "-p", mpr); code != 0 { diff --git a/cmd/mxcli/cpuprofile.go b/cmd/mxcli/cpuprofile.go new file mode 100644 index 000000000..83130f816 --- /dev/null +++ b/cmd/mxcli/cpuprofile.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "runtime/pprof" +) + +// cpuProfileEnv names a file to write a CPU profile of this command to, for +// `go tool pprof`. Unset, nothing is profiled. +const cpuProfileEnv = "MXCLI_CPUPROFILE" + +// startCPUProfile starts profiling when cpuProfileEnv is set and returns the +// function that stops it and closes the file. +func startCPUProfile() (stop func()) { + path := os.Getenv(cpuProfileEnv) + if path == "" { + return func() {} + } + f, err := os.Create(path) + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: %s: %v\n", cpuProfileEnv, err) + return func() {} + } + if err := pprof.StartCPUProfile(f); err != nil { + fmt.Fprintf(os.Stderr, "Warning: %s: %v\n", cpuProfileEnv, err) + f.Close() + return func() {} + } + return func() { + pprof.StopCPUProfile() + f.Close() + } +} diff --git a/cmd/mxcli/describe_members_1262_test.go b/cmd/mxcli/describe_members_1262_test.go new file mode 100644 index 000000000..873fecfb6 --- /dev/null +++ b/cmd/mxcli/describe_members_1262_test.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +// ako/mxcli#1262 (mendixlabs/mxcli#658, item 1a): `describe ` without a +// type resolved documents only. A module role, a user role or a navigation +// profile — each with a describe type of its own — answered "no describable +// document named …". +func TestDescribeAutoDetectsMembers_Issue1262(t *testing.T) { + mpr := filepath.Join("..", "..", "testdata", "testapp", "TestApp", "TestApp.mpr") + if _, err := os.Stat(mpr); err != nil { + t.Skipf("testapp submodule not initialised: %v", err) + } + for name, want := range map[string]string{ + "Administration.Administrator": "modulerole", + "Administrator": "userrole", + "Responsive": "navigation", + // Control: a document still resolves as before. + "Administration.Account": "entity", + } { + got, candidates, err := resolveDescribeType(mpr, name) + if err != nil || got != want { + t.Errorf("describe %s: resolved to %q (candidates %v, err %v); want %q", name, got, candidates, err, want) + } + } + if _, _, err := resolveDescribeType(mpr, "Administration.NoSuchRole"); err == nil { + t.Error("a name nothing has resolved") + } +} diff --git a/cmd/mxcli/json_output_purity_test.go b/cmd/mxcli/json_output_purity_test.go index fd1d1266a..1b4a2ef30 100644 --- a/cmd/mxcli/json_output_purity_test.go +++ b/cmd/mxcli/json_output_purity_test.go @@ -327,8 +327,9 @@ func TestSearchQuietNamesIsOnlyNamesOnAColdCatalog(t *testing.T) { t.Errorf("not a typename line: %q", l) } } - // The progress is moved, not dropped: a cold build can take minutes. - if !strings.Contains(r.stderr, "✓ Entities:") { - t.Errorf("build progress vanished instead of moving to stderr:\n%s", r.stderr) + // Nor does stderr carry a line per table (#1860): a slow build reports + // a heartbeat there instead (catalogHeartbeat). + if lines := perTableLines(r.stderr); len(lines) > 0 { + t.Errorf("a quiet build printed %d per-table lines on stderr, e.g. %q", len(lines), lines[0]) } } diff --git a/cmd/mxcli/main.go b/cmd/mxcli/main.go index f314b1586..b933fe3e1 100644 --- a/cmd/mxcli/main.go +++ b/cmd/mxcli/main.go @@ -45,7 +45,10 @@ func main() { // (ako/mxcli#633). startSession(os.Args[1:]) - if code := runCLI(); code != 0 { + stopProfile := startCPUProfile() + code := runCLI() + stopProfile() + if code != 0 { os.Exit(code) } } @@ -125,6 +128,30 @@ func discoverProject(dir string) (path, note string) { } } +// noProjectFlag is the opt-out from project auto-detection, registered on the +// commands that are meaningful without a project (check). The root +// PersistentPreRun honours it on any command that registers it. +const noProjectFlag = "no-project" + +// noProjectOptOut reports whether cmd registers --no-project and, if so, the +// line that tells a user who got an auto-detected project how to avoid it — +// the notice is where they meet the guess, so it is where the way out goes. +func noProjectOptOut(cmd *cobra.Command) (bool, string) { + if cmd.Flags().Lookup(noProjectFlag) == nil { + return false, "" + } + return true, fmt.Sprintf("(pass --%s to %s without a project)", noProjectFlag, cmd.Name()) +} + +// skipsProjectDiscovery reports whether cmd was given --no-project. +func skipsProjectDiscovery(cmd *cobra.Command) bool { + if ok, _ := noProjectOptOut(cmd); !ok { + return false + } + v, _ := cmd.Flags().GetBool(noProjectFlag) + return v +} + var rootCmd = &cobra.Command{ Use: "mxcli", Short: "Mendix CLI - Work with Mendix projects using MDL syntax", @@ -157,10 +184,19 @@ beta language; --mdl 0 starts them in the alpha language, and in the REPL an Version: version, PersistentPreRun: func(cmd *cobra.Command, args []string) { projectPath, _ := cmd.Flags().GetString("project") - if projectPath == "" { + // A command with a --no-project opt-out (check) skips auto-detection + // when it is given: run from a project directory, `check script.mdl` + // otherwise resolves every reference against the .mpr it finds, and + // the syntax-only check its help offered could not be had there at + // all (mendixlabs/mxcli#1817). Without the flag nothing changes. + _, optOut := noProjectOptOut(cmd) + if projectPath == "" && !skipsProjectDiscovery(cmd) { discovered, note := discoverProject(".") if discovered != "" { _ = cmd.Flags().Set("project", discovered) + if optOut != "" { + note += "\n" + optOut + } fmt.Fprintln(os.Stderr, note) } else if note != "" && !shouldSuppressWarning() { fmt.Fprintln(os.Stderr, note) @@ -177,6 +213,7 @@ beta language; --mdl 0 starts them in the alpha language, and in the REPL an warnIfToolingNewer(os.Stderr, projectPath) } globalJSONFlag, _ = cmd.Flags().GetBool("json") + globalQuietFlag, _ = cmd.Flags().GetBool("quiet") globalMCPURL, _ = cmd.Flags().GetString("mcp") globalMCPDial, _ = cmd.Flags().GetString("mcp-dial") globalMCPCheck, _ = cmd.Flags().GetBool("mcp-check") @@ -241,6 +278,13 @@ beta language; --mdl 0 starts them in the alpha language, and in the REPL an // globalJSONFlag is set by PersistentPreRun when --json is passed. var globalJSONFlag bool +// globalQuietFlag is --quiet / -q: no warning banner (shouldSuppressWarning +// reads argv for that, before cobra runs) and no connection or status +// messages from the executor. The banner check named -q for a long time while +// no command but `search` accepted it, so `mxcli -q -c …` failed outright +// (ako/mxcli#1860). +var globalQuietFlag bool + // globalMCPURL / globalMCPDial are set by PersistentPreRun from --mcp / --mcp-dial. // When globalMCPURL is non-empty, newLoggedExecutor builds an MCP backend that // routes model writes to a live Studio Pro while reads come from the local @@ -306,6 +350,9 @@ func newLoggedExecutorTo(mode string, out io.Writer) (*executor.Executor, *diagl if globalJSONFlag { exec.SetFormat(executor.FormatJSON) } + if globalQuietFlag { + exec.SetQuiet(true) + } return exec, logger } @@ -349,6 +396,7 @@ func init() { // Global flags rootCmd.PersistentFlags().StringP("project", "p", "", "Path to Mendix project (.mpr file)") rootCmd.PersistentFlags().Bool("json", false, "Output in JSON format") + rootCmd.PersistentFlags().BoolP("quiet", "q", false, "No warning banner, and no connection or status messages (catalog build progress included)") rootCmd.PersistentFlags().String("mcp", "", "Route model writes to a live Studio Pro MCP server (e.g. http://localhost:7782/mcp); reads come from -p") rootCmd.PersistentFlags().String("mcp-dial", "", "Override the TCP address dialed for --mcp (e.g. host.docker.internal:7782 from a devcontainer)") rootCmd.PersistentFlags().Bool("mcp-check", false, "After the command, run Studio Pro's project-wide error check (ped_check_errors, Studio Pro 11.15+) and print the report") @@ -364,6 +412,8 @@ func init() { checkCmd.Flags().BoolP("references", "r", false, "Validate references against the project (implied by -p; kept for compatibility)") checkCmd.Flags().String("format", "text", "Output format: text, json, sarif") checkCmd.Flags().Bool("post-migration", false, "Scan the project for legacy native widgets that survived a Mendix upgrade (requires -p)") + checkCmd.Flags().Bool(noProjectFlag, false, + "Check syntax and the no-model rules only: do not auto-detect the .mpr in the current directory (conflicts with -p, --references, --post-migration)") // Diff command flags diffCmd.Flags().StringP("format", "f", "unified", "Output format: unified, side, struct") @@ -386,7 +436,6 @@ func init() { // Search command flags searchCmd.Flags().StringP("format", "f", "table", "Output format: table, names (for JSON use --json; \"json\" is accepted as a deprecated alias)") - searchCmd.Flags().BoolP("quiet", "q", false, "Suppress connection and status messages (for piping)") searchCmd.Flags().Bool("fail-on-match", false, "Exit 1 when there is at least one hit, 0 when there is none (an error exits 2)") searchCmd.Flags().Bool("exclude-marketplace", false, "Leave out hits inside modules downloaded from the Marketplace") searchCmd.Flags().Bool("no-build", false, "Never build the source index; without one, exit 3 because flow, page and expression bodies were not searched") diff --git a/cmd/mxcli/syntax/features_page.go b/cmd/mxcli/syntax/features_page.go index 5b5556575..c0a6fe88b 100644 --- a/cmd/mxcli/syntax/features_page.go +++ b/cmd/mxcli/syntax/features_page.go @@ -864,7 +864,7 @@ LIST IMPACT OF htmlelement; "binding", "rebind", "building block override", "reusable component", }, Syntax: "-- Declare typed params, reference with $name in a datasource/action slot:\nCREATE FRAGMENT Name ($data: datasource, $onEdit: action) AS { … };\n-- Supply values at the use site:\nUSE FRAGMENT Name ($data: , $onEdit: ) [{ payload }]\n-- Building blocks: rebind the outermost datasource / first button:\nUSE BUILDING BLOCK Module.Block (datasource: , action: ) [AS prefix_]", - Example: "CREATE FRAGMENT DataPanel ($data: datasource, $onEdit: action) AS {\n CONTAINER panel (Class: 'card') {\n LISTVIEW lv (DataSource: $data) {\n SLOT content\n ACTIONBUTTON edit (Caption: 'Edit', Action: $onEdit, ButtonStyle: Primary)\n }\n }\n};\n\nCREATE PAGE Module.Orders (Title: 'Orders', Layout: Atlas_Core.Atlas_Default) {\n USE FRAGMENT DataPanel ($data: DATABASE Sales.Order, $onEdit: MICROFLOW Sales.Edit) {\n DYNAMICTEXT heading (Content: 'Orders', RenderMode: H4)\n }\n};\n\n-- Rebind a building block's datasource and primary button:\nUSE BUILDING BLOCK Atlas_Web_Content.List_Cards (datasource: DATABASE Sales.Order, action: CALL MICROFLOW Sales.Open) AS orders_\n\n-- Notes:\n-- * Param kinds: datasource | action. Every declared param must be supplied.\n-- * A microflow value parses as a datasource and is reinterpreted for an action param.\n-- * BB binding-point rule: datasource → first datasource widget; action → first button.", + Example: "CREATE FRAGMENT DataPanel ($data: datasource, $onEdit: action) AS {\n CONTAINER panel (Class: 'card') {\n LISTVIEW lv (DataSource: $data) {\n SLOT content\n ACTIONBUTTON edit (Caption: 'Edit', Action: $onEdit, ButtonStyle: Primary)\n }\n }\n};\n\nCREATE PAGE Module.Orders (Title: 'Orders', Layout: Atlas_Core.Atlas_Default) {\n USE FRAGMENT DataPanel ($data: DATABASE Sales.Order, $onEdit: MICROFLOW Sales.Edit) {\n DYNAMICTEXT heading (Content: 'Orders', RenderMode: H4)\n }\n};\n\n-- Rebind a building block's datasource and primary button:\nUSE BUILDING BLOCK Atlas_Web_Content.List_Cards (datasource: DATABASE Sales.Order, action: CALL MICROFLOW Sales.Open) AS orders_\n\n-- Notes:\n-- * Param kinds: datasource | action. Every declared param must be supplied.\n-- * A microflow value parses as a datasource and is reinterpreted for an action param.\n-- * BB binding-point rule: datasource → first datasource widget; action → first button (a pluggable button such as pdsbutton counts).", SeeAlso: []string{"fragment", "fragment.slot", "fragment.use"}, }) diff --git a/cmd/mxcli/tool_templates.go b/cmd/mxcli/tool_templates.go index cb63e1836..1a1221b65 100644 --- a/cmd/mxcli/tool_templates.go +++ b/cmd/mxcli/tool_templates.go @@ -149,7 +149,7 @@ The mxcli tool is in the PROJECT ROOT, not in system PATH. Always use: ### Validate MDL Scripts `+"```bash"+` -./mxcli check script.mdl # Syntax only +./mxcli check script.mdl --no-project # Syntax only (else the .mpr here is auto-detected) ./mxcli check script.mdl -p %s --references # With refs `+"```"+` diff --git a/docs-site/src/ide/vscode-syntax.md b/docs-site/src/ide/vscode-syntax.md index 19642c536..395ed3496 100644 --- a/docs-site/src/ide/vscode-syntax.md +++ b/docs-site/src/ide/vscode-syntax.md @@ -80,8 +80,8 @@ These diagnostics appear as warnings or errors in the Problems panel, identical The same checks available in the extension can be run from the command line: ```bash -# Syntax check only (no project needed) -mxcli check script.mdl +# Syntax check only (--no-project: do not auto-detect an .mpr in this directory) +mxcli check script.mdl --no-project # Syntax + reference validation mxcli check script.mdl -p app.mpr --references diff --git a/docs-site/src/language/script-files.md b/docs-site/src/language/script-files.md index 350ee5cd5..6de398250 100644 --- a/docs-site/src/language/script-files.md +++ b/docs-site/src/language/script-files.md @@ -98,8 +98,8 @@ EXECUTE SCRIPT './scripts/setup_domain_model.mdl'; You can validate a script without connecting to a project: ```bash -# Syntax only -mxcli check script.mdl +# Syntax only (--no-project: do not auto-detect an .mpr in this directory) +mxcli check script.mdl --no-project # Syntax + reference validation (requires project) mxcli check script.mdl -p app.mpr --references diff --git a/docs-site/src/migration/generation.md b/docs-site/src/migration/generation.md index 242eba7e1..62568e416 100644 --- a/docs-site/src/migration/generation.md +++ b/docs-site/src/migration/generation.md @@ -117,8 +117,8 @@ CREATE PAGE CRM.Customer_Overview ( Validate after each script to catch errors early: ```bash -# Syntax check (fast, no project needed) -mxcli check domain-model.mdl +# Syntax check (fast, no project used) +mxcli check domain-model.mdl --no-project # Reference validation (checks entity/microflow names exist) mxcli check pages.mdl -p app.mpr --references diff --git a/docs-site/src/migration/validation.md b/docs-site/src/migration/validation.md index 91e46a821..c475b5f1b 100644 --- a/docs-site/src/migration/validation.md +++ b/docs-site/src/migration/validation.md @@ -5,8 +5,8 @@ mxcli provides a multi-level validation pipeline. The agent uses these tools to ## Validation Steps ```bash -# 1. Syntax check (fast, no project needed) -mxcli check script.mdl +# 1. Syntax check (fast, no project used) +mxcli check script.mdl --no-project # 2. Reference validation (checks entity/microflow names exist) mxcli check script.mdl -p app.mpr --references diff --git a/docs-site/src/tools/catalog-tables.md b/docs-site/src/tools/catalog-tables.md index 4572bcb12..eb6cb462b 100644 --- a/docs-site/src/tools/catalog-tables.md +++ b/docs-site/src/tools/catalog-tables.md @@ -250,6 +250,37 @@ JOIN CATALOG.ACTIVITIES a ON a.Id = f.DestinationId WHERE f.FlowType = 'AnnotationFlow'; ``` +### CATALOG.WORKFLOW_ACTIVITIES + +One row per activity of a workflow, in walk order: the main flow, each +activity followed by the paths of its boundary events, then the event +sub-processes. Populated by every catalog build. A slot that names nothing is +NULL. + +| Column | Description | +|--------|-------------| +| `Id` | The activity's ID | +| `WorkflowId`, `WorkflowQualifiedName`, `ModuleName` | The workflow it belongs to (`CATALOG.WORKFLOWS.Id`) | +| `Sequence` | Walk order within the workflow | +| `Name`, `Caption` | As in Studio Pro | +| `ActivityType` | `UserTask`, `MultiUserTask`, `SystemTask`, `CallMicroflow`, `AIAgentTask`, `CallWorkflow`, `Decision`, `ParallelSplit`, `JumpTo`, `WaitForTimer`, `WaitForNotification`, `StartWorkflow`, `EndWorkflow`, and the event sub-process and path-end kinds | +| `Microflow` | The microflow a system task, call-microflow activity or AI agent task runs | +| `UserSourceMicroflow` | A user task's targeting microflow (users or groups) | +| `OnCreatedMicroflow` | A user task's on-created microflow | +| `CompletionMicroflow` | A multi-user task's completion microflow | +| `Page`, `UserTaskEntity` | A user task's page and task entity | +| `CalledWorkflow` | The workflow a call-workflow activity calls | + +`CATALOG.REFS` records that a workflow calls a microflow; this table says from +which kind of activity and which slot. + +```sql +-- Microflows that system tasks run, without the WF_SYS_ prefix +SELECT WorkflowQualifiedName, Name, Microflow +FROM CATALOG.WORKFLOW_ACTIVITIES +WHERE ActivityType = 'SystemTask' AND Microflow NOT LIKE '%.WF_SYS_%'; +``` + ### CATALOG.PAGES Information about pages and their properties. diff --git a/docs-site/src/tools/refresh-catalog.md b/docs-site/src/tools/refresh-catalog.md index 01b35c841..c651ad717 100644 --- a/docs-site/src/tools/refresh-catalog.md +++ b/docs-site/src/tools/refresh-catalog.md @@ -71,6 +71,12 @@ mxcli -p app.mpr -c "REFRESH CATALOG FULL FORCE" The catalog is cached in `.mxcli/catalog.db` next to the MPR file. On subsequent refreshes, mxcli checks whether the project has changed and only rebuilds if necessary (unless `FORCE` is specified). +### Progress output + +A build prints a line per table (`✓ Entities: 111`) only when its output is a terminal. Piped, logged or run by an agent, it prints nothing per table while it is quick, and the latest progress line every 15 seconds while it is slow (`… building catalog, source 250/300 documents (32s)`), so a build of several minutes still shows it is running. Lines that carry information, such as the incremental summary and documents that could not be described, print either way. + +`-q` / `--quiet` works on every command: it drops the warning banner and the connection and status messages, and sends what catalog progress remains to stderr. + ### Incremental rebuilds A rebuild after a save does not start from scratch. The cache records which document every row came from, and the next rebuild of the same level (fast, `FULL` or `FULL SOURCE`) starts from the cached catalog and produces again only the rows of: diff --git a/docs-site/src/tutorial/skills.md b/docs-site/src/tutorial/skills.md index cfbe7791b..e4ee243b3 100644 --- a/docs-site/src/tutorial/skills.md +++ b/docs-site/src/tutorial/skills.md @@ -118,8 +118,8 @@ A list of errors the AI should avoid. For example, `write-microflows.md` warns a Steps the AI should follow after writing MDL to confirm correctness: ```bash -# Syntax check (no project needed) -./mxcli check script.mdl +# Syntax check only (--no-project: do not auto-detect an .mpr in this directory) +./mxcli check script.mdl --no-project # Syntax + reference validation ./mxcli check script.mdl -p app.mpr --references diff --git a/docs-site/src/tutorial/validation.md b/docs-site/src/tutorial/validation.md index 60b357d16..ce78aec9c 100644 --- a/docs-site/src/tutorial/validation.md +++ b/docs-site/src/tutorial/validation.md @@ -7,9 +7,11 @@ You've created entities, microflows, and pages. Before opening your project in S The fastest check. It parses your MDL script and reports syntax errors without touching any `.mpr` file: ```bash -mxcli check script.mdl +mxcli check script.mdl --no-project ``` +Without `--no-project`, a plain `mxcli check script.mdl` run from a directory holding exactly one `.mpr` auto-detects it (it prints `Using project: ...`) and goes on to Level 2 against it. `--no-project` gives the syntax-only check wherever you run it. + This catches: - Typos in keywords (`CRETE` instead of `CREATE`) @@ -147,8 +149,8 @@ BEGIN END; EOF -# 2. Check syntax (fast, no project needed) -mxcli check changes.mdl +# 2. Check syntax (fast, no project used) +mxcli check changes.mdl --no-project # 3. Check references (needs project, still fast) mxcli check changes.mdl -p app.mpr --references diff --git a/docs-wiki/bug-patterns/flow-graph-geometry.md b/docs-wiki/bug-patterns/flow-graph-geometry.md index 3a1d12a62..fb97388c4 100644 --- a/docs-wiki/bug-patterns/flow-graph-geometry.md +++ b/docs-wiki/bug-patterns/flow-graph-geometry.md @@ -43,6 +43,16 @@ outside it — worse than a box that is too wide. Those runs fall back to the conservative measure deliberately. The containment check is the honest test: every child of a looped activity must lie inside the container's box. +**The measurer and the builder must agree, and a loop box says whether they +do.** Whatever follows a body — an IF's closing merge, the next row, a loop's +box — is placed from the body's *measured* width, while the box around a loop +is fitted to what was *built*. Wrapping a run in a loop and comparing the two +widths is an exact oracle that needs no expected coordinates, and it catches +the over-measures no overlap check can see. Every "already laid out" report +from `layout flows` is the builder agreeing with itself, so it proves nothing +about this class. Keep the main-line cursor apart from the drawn extent: an IF +without a merge resumes at a point that is not the edge of anything. + **Wiring: the invalid shapes come from branches that do not merge.** A decision whose non-terminal branch falls off the end of a loop body, a `break` nested inside an `if`, a split followed by a statement — each is a place where a diff --git a/docs/03-development/MODELSDK_ENGINE_ARCHITECTURE.md b/docs/03-development/MODELSDK_ENGINE_ARCHITECTURE.md index 81556a1e0..c961d5013 100644 --- a/docs/03-development/MODELSDK_ENGINE_ARCHITECTURE.md +++ b/docs/03-development/MODELSDK_ENGINE_ARCHITECTURE.md @@ -70,6 +70,35 @@ ALTER read-modify-write path round-trips (else ALTER silently drops data — see microflow `ConcurrencyErrorMessage`→`ConcurrenyErrorMessage` (Mendix typo), `StableId` is binary-not-string (handled via `FreshGUIDFields`). gen MicroflowParameter was also missing fields (completed by hand). +## A listed element may be shared: read it, or change it and write it + +`mprread`'s listings (`ListUnitsWithContainer` and the lookups beside it — what +the backend's `listUnits` calls) keep the decoded form of every unit they have +decoded twice, and hand the **same** element to every caller until the unit's +stored bytes change (`modelsdk/mprread/decode_cache.go`, ako/mxcli#2160; +the decision and its alternatives are +[ADR-0015](../13-decisions/0015-content-keyed-unit-caches.md)). A +script used to cost statements × project, because each statement listed whole +document types and each listing decoded every unit again. + +What that means for a converter or a write path: + +- **Reading is free and safe**, from any goroutine: every lazily decoded value + is loaded before an element is shared, so a read writes nothing. +- **A setter on a listed element is seen by no other caller.** The element + becomes dirty, and a dirty entry is decoded afresh for the next caller. Set + it, encode it, write it — as `UpdateOqlQueriesForMovedEntity` does. +- **Do not change a listed element by any route that does not mark it dirty** + (a field assignment, a slice edited in place). Nothing detects that, and the + next caller would read the change. +- **Never hold a listed element across your own write and compare it with a + fresh listing** expecting two objects: when nothing changed they are the + same one. + +The cache is keyed by a unit's stored bytes, so no write path invalidates it +and none can forget to. `MXCLI_DECODE_CACHE=off` turns it off for a process, to +rule it in or out when a result looks wrong. + ## Harvest rule (engalar/dev) engalar/dev has a complete, tested gen-native write suite, but built **AST-direct with a gen-typed diff --git a/docs/13-decisions/0015-content-keyed-unit-caches.md b/docs/13-decisions/0015-content-keyed-unit-caches.md new file mode 100644 index 000000000..549d59341 --- /dev/null +++ b/docs/13-decisions/0015-content-keyed-unit-caches.md @@ -0,0 +1,121 @@ +# ADR-0015: What mxcli keeps of a unit is keyed by the unit's content + +- **Status**: Proposed +- **Date**: 2026-11-12 +- **Related**: ako/mxcli#2160 (the defect), ako/mxcli#2159 (the change); [ADR-0008](0008-identity-and-idempotence.md) (content comparison for writes); [ADR-0014](0014-incremental-catalog-builds.md) (the same rule for catalog rows); ako/mxcli#2121 (a session sees another process's writes) + +## Context + +A script cost statements × project. The check before an `exec`, and the +`exec` itself, list whole document types for each statement: every microflow +to resolve one call, every domain model to resolve one entity. Each listing +read every unit file of the type and decoded every unit from its bytes. +Seeding 300 entities, 600 microflows and 600 pages into a 369-unit project +took 45 s, 42 s and 286 s per stage; 1,600 pages had not finished after 15 +minutes. A user works on a project of 8,000 microflows. + +One listing is cheap: 616 microflows decode in 0.11 s. The cost was the +count, about a thousand full microflow listings for 600 page statements, and +it was spread over more call sites than a profile shows at once. The fixes +before this one (#1789, #1791, #1850, #2094) each removed one call site and +left a budget test for one command. None of those tests covers many +statements in one script, and the next script shape finds the next site. + +So the remedy had to be below the call sites, where every listing passes, +and it had to stay correct while the same process writes the project between +listings. That is the hard part of caching here. The reader had a byte cache +already, and every write emptied all of it, because a cache keyed by a +unit's identity has to be told about every write by every path, and the +simplest way to be sure is to throw everything away. + +## Decision + +What mxcli keeps that is derived from a unit — its bytes in memory, its +decoded form — is valid for the unit's **content** and for nothing else: an +entry records the content it was made from and is served only while the +unit's stored content is still that, so no write path invalidates it and +none can forget to. Three rules follow from handing the same thing to more +than one caller. A decoded element is shared only once reading it writes +nothing, which means every lazily decoded value in it has been loaded. An +element a caller has changed is never served to the next caller. And when +another process is known to have written the project, cached bytes are +dropped outright, because that writer is not obliged to move the hash the +bytes are keyed by. Separately, a check may list each document type once for +its whole run, because a check never writes; execution may not, because a +statement must see what the one before it wrote. + +## Consequences + +- The seed above takes 20 s, 14 s and 44 s. On the PedApp fixture an added + page statement decodes 2 units where it decoded 240, and reads 1 unit file + where it read 120. The check lists the domain models, microflows and + nanoflows once per run, where it listed them four, three and two times per + page statement. Commands that run one statement are unchanged, by design: + a unit is only remembered the first time it is decoded and kept the + second, so a single listing pays nothing. +- **Two keys, on purpose.** The decode cache holds the bytes an element was + decoded from and compares them exactly. The byte cache is keyed by the + `ContentsHash` in the unit's row of the Unit table, which the writer + already updates with every unit it writes. The first needs nothing from + anyone. The second trusts that hash for this process's own writes — every + site that writes a unit file was read, and does move it — and trusts + nothing for another process's, which is why a foreign change empties it. + A future write path that rewrote a unit file without moving its hash would + serve stale bytes for the rest of a script run, and no test would notice. +- **A listed element may be one another caller holds.** Before this, every + listing returned objects of its own, and a caller could do as it liked + with them. Now a change made through a setter is safe, because it marks + the element dirty and the entry is decoded afresh. A change made by any + route that does not mark it dirty — a field assignment, a slice edited in + place, a write through `Raw()` — is seen by the next caller, and nothing + detects it. `docs/03-development/MODELSDK_ENGINE_ARCHITECTURE.md` carries + the rule for converter and write-path authors. +- **Each safeguard has a test that fails without it, and one of them needs + the race detector.** Without the loading of lazy values, reading a shared + element from several goroutines is 42 data races; `make test` does not run + with `-race`, so that test only bites where the detector is on. Without + the dirty check, a caller's change reaches the next caller. Without the + foreign-write rule, a rewrite by another process that leaves the hash + alone is not seen while a cache scope is held. +- **Within one check, the validators share what was listed.** None changes a + listed document in place today. One that did would be seen by every + validator after it in the run. +- **Memory is held longer.** Units of a type listed more than once stay + decoded for the life of the connection. A script run keeps the bytes of + every unit file it reads until it ends. A long-lived session does not keep + bytes between statements, so that it cannot go on serving what Studio Pro + has since rewritten. +- **`MXCLI_DECODE_CACHE=off`** turns the decode cache off for a process, to + rule it in or out when a result looks wrong. There is no switch for the + byte cache or for the check's listings. +- What this does not remove: the conversion of each listing into the + semantic model, the per-statement index builds during execution, and the + rewriting of one domain model once per statement. A script still grows + faster than its statement count. + +## Alternatives considered + +- **Fix the call sites one at a time**, as the earlier fixes did: give each + validator a by-name lookup or a narrower listing. Lowest risk per change, + and it leaves the cost in every site nobody has profiled. The profile of + one script showed eight sites; the fixes of the preceding weeks had each + found one. Still the right answer where a site lists a type to find one + document, and compatible with this decision. +- **Cache the semantic-model objects a listing returns** (`[]*Microflow`, + `[]*DomainModel`), which would also save the conversion. They are plain + structs that callers change freely — `create or modify` builds on what it + reads — and they carry no dirty flag, so a shared one cannot tell that it + was changed. Sound only inside a check, where nothing is written, which is + where it was done. +- **Key by identity and invalidate on write**, as the reader's byte cache + did. It needs every write path to say which unit it wrote; the paths that + could not say emptied everything, which is how the cache came to do + nothing during a script. It also cannot see another process's write. +- **Make lazy decoding safe to share instead of loading everything first**, + with a lock or an atomic per property. That puts a cost on every property + of every element, in the engine's hottest path, to save one walk of an + element that is about to be read many times. +- **Hold decoded listings for the length of one command only**, with no + validity rule at all. That is what a check gets from its memo, and it is + correct there because nothing is written. During execution the listing + changes under the statements that read it. diff --git a/docs/13-decisions/README.md b/docs/13-decisions/README.md index e8a252b53..7a466b83d 100644 --- a/docs/13-decisions/README.md +++ b/docs/13-decisions/README.md @@ -108,6 +108,7 @@ This preserves the audit trail. | [0012](0012-mdl-first-and-data-first-editing.md) | MDL-first and data-first editing share one syntax and one patch engine | Accepted | | [0013](0013-activity-settings-property-list.md) | Microflow activity settings: words for the statement, one property list for the dialog (refines R2/R3 of 0010 for activities) | Proposed | | [0014](0014-incremental-catalog-builds.md) | Catalog rows belong to a unit, and a refresh rebuilds only the changed units | Accepted | +| [0015](0015-content-keyed-unit-caches.md) | What mxcli keeps of a unit is keyed by the unit's content | Proposed | ## Candidates to back-fill diff --git a/mdl-examples/bug-tests/1773-layout-measurer-if-join-rejoin.mdl b/mdl-examples/bug-tests/1773-layout-measurer-if-join-rejoin.mdl new file mode 100644 index 000000000..35337a185 --- /dev/null +++ b/mdl-examples/bug-tests/1773-layout-measurer-if-join-rejoin.mdl @@ -0,0 +1,82 @@ +mdl 1; +-- ============================================================================ +-- ako/mxcli#1773: the layout measurer disagreed with the flow builder in three +-- shapes, so an object landed on or behind its neighbour, and `layout flows` +-- reported such a flow as already laid out because it uses the same builder. +-- +-- 1. "An IF is measured 45 px short. The measurer counts the split's centre as +-- its left edge. Take an if/else followed by an activity inside a THEN +-- branch: the enclosing merge ends up 5 px inside that activity." +-- A decision that opens a branch was a further 15px short: the builder +-- centres it where an activity's centre would be, and a split is narrower. +-- 2. "A `join` is measured as an activity plus an end event, though it draws +-- nothing. A handler row that ends in joins reserves roughly twice its +-- drawn width." +-- 3. "A fall-through error handler of two or more statements rejoins +-- backwards." The merge stood between the activity and the next one, left +-- of the row's last activity. It now goes one gap past the row, and the +-- main line waits for it. +-- +-- Geometry is invisible below Studio Pro: every flow here passes `mx check` +-- either way. Execute without @position and lint: MPR008 must report nothing. +-- +-- mxcli exec 1773-layout-measurer-if-join-rejoin.mdl -p app.mpr +-- mxcli lint -p app.mpr --rules MPR008,MPR011 +-- +-- Regression coverage: mdl/executor/layout_measurer_1773_test.go and +-- mdl/executor/layout_handler_lane_1768_test.go (now without skips). +-- ============================================================================ + +create or modify module Bug1773; + +create or modify persistent entity Bug1773.Item ( "Code": String(50) ); + +-- Shape 1: before the fix the closing merge of the outer IF stood inside the +-- 'last in then' activity. +create or modify microflow Bug1773.NestedDecisionInBranch ($Flag: Boolean, $Other: Boolean) +begin + if $Other then + if $Flag then + log info node 'Bug1773' 'yes'; + else + log info node 'Bug1773' 'no'; + end if; + log info node 'Bug1773' 'last in then'; + else + log info node 'Bug1773' 'else'; + end if; + log info node 'Bug1773' 'after'; +end; + +-- Shape 2: handler rows that end in a join, in a THEN branch whose closing +-- merge is placed from the branch's measured width. +create or modify microflow Bug1773.HandlerRowsEndingInJoins ($Flag: Boolean) +begin + if $Flag then + retrieve $A from Bug1773.Item on error without rollback begin + log error node 'Bug1773' 'a1'; + join recovered; + end error; + retrieve $B from Bug1773.Item on error without rollback begin + log error node 'Bug1773' 'b1'; + log error node 'Bug1773' 'b2'; + join recovered; + end error; + log info node 'Bug1773' 'between'; + else + log info node 'Bug1773' 'else'; + end if; + merge recovered; + log info node 'Bug1773' 'done'; +end; + +-- Shape 3: a handler of three statements that falls through to the main line. +create or modify microflow Bug1773.LongHandlerRejoins () +begin + retrieve $A from Bug1773.Item on error without rollback begin + log error node 'Bug1773' 'a1'; + log error node 'Bug1773' 'a2'; + log error node 'Bug1773' 'a3'; + end error; + log info node 'Bug1773' 'next'; +end; diff --git a/mdl-examples/bug-tests/1841-domain-model-svg-stored-anchors.mdl b/mdl-examples/bug-tests/1841-domain-model-svg-stored-anchors.mdl new file mode 100644 index 000000000..82b2f5143 --- /dev/null +++ b/mdl-examples/bug-tests/1841-domain-model-svg-stored-anchors.mdl @@ -0,0 +1,61 @@ +mdl 1; +-- ============================================================================ +-- ako/mxcli#1841 (part 1 of mendixlabs/mxcli#1692) — the domain model SVG +-- draws associations from their stored anchors +-- ============================================================================ +-- +-- Symptom, verbatim: "`mxcli describe --format svg domainmodel ` draws +-- every association centre to centre. The stored association anchors are +-- ignored (tested with extreme anchor values), so the SVG cannot be used to +-- verify "corner to corner" lines." +-- +-- An association's anchors are DomainModels$Association.ParentConnection (FROM +-- entity) and ChildConnection (TO entity), each the string "x;y" — a percentage +-- of the entity box (a blank 11.14.0 app's Studio Pro-authored +-- Administration.AccountPasswordData_Account stores "0;54" / "100;54"). The SVG +-- now attaches each end at that percentage of the drawn box; values outside +-- 0..100 are clamped onto the box. A self-association is routed round the +-- outside of its box from one anchor to the other. +-- +-- Run, then render: +-- mxcli exec 1841-domain-model-svg-stored-anchors.mdl -p app.mpr +-- mxcli describe --format svg -p app.mpr domainmodel Bug1841 +-- +-- Expected, with the entities drawn as below (Customer 90x44 at (100,100), +-- Order at (500,300), Category 90x44 at (100,450)): +-- Order_Customer corner to corner +-- Order_Category mxcli's default 0;50 / 100;50 +-- Category_Parent +-- Before the fix all three were centre-line lines, and Category_Parent was a +-- zero-length line at the box centre. + +create module Bug1841; + +@position(100, 100) +create persistent entity Bug1841.Customer ( + "Name": string(200) +); + +@position(500, 300) +create persistent entity Bug1841.Order ( + "Number": integer +); + +@position(100, 450) +create persistent entity Bug1841.Category ( + "Title": string(100) +); + +-- Corner to corner: Order's top-left to Customer's bottom-right. +@anchor(from: (0, 0), to: (100, 100)) +create association Bug1841.Order_Customer +from Bug1841.Order to Bug1841.Customer; + +-- No anchor given: mxcli stores its default 0;50 / 100;50, and the SVG draws that. +create association Bug1841.Order_Category +from Bug1841.Order to Bug1841.Category; + +-- Self-association: out of the right edge at 30%, back into the bottom at 50%. +@anchor(from: (100, 30), to: (50, 100)) +create association Bug1841.Category_Parent +from Bug1841.Category to Bug1841.Category; diff --git a/mdl-examples/bug-tests/2145-building-block-action-override-pluggable-button.mdl b/mdl-examples/bug-tests/2145-building-block-action-override-pluggable-button.mdl new file mode 100644 index 000000000..f044d3e11 --- /dev/null +++ b/mdl-examples/bug-tests/2145-building-block-action-override-pluggable-button.mdl @@ -0,0 +1,29 @@ +mdl 1; +-- Bug ako/mxcli#2145 (mendixlabs/mxcli#1821): the `action:` override of +-- `use building block` only recognised the core buttons (actionbutton, +-- linkbutton, button). A PlatformDesignSystem block is a pluggable PDS Button, +-- so +-- use building block PlatformDesignSystem.Button (action: call microflow PDSRepro.ACT_Noop) as b_ +-- failed with +-- Error: failed to build page: use building block PlatformDesignSystem.Button: +-- action override supplied, but the block has no button to rebind +-- +-- Fix: with no core button in the block, the override binds the first pluggable +-- widget whose definition has a CLICK action (Source OnClick — PDS Button's +-- onClickAction, Badge Button's onClickEvent, Image's onClick), skipping data +-- containers. A block with a core button binds it as before. +-- +-- PlatformDesignSystem is a marketplace module this repo cannot fetch without a +-- marketplace token, so the regression case uses the equivalent Atlas block: +-- Card_WithImage has no core button, only a pluggable Image whose definition +-- declares an onClick action. Verify (project with Atlas_Web_Content, e.g. +-- `mxcli new`): exec writes `image w_image1 (…, OnClick: call microflow +-- BugTest2145.ACT_Noop, …)`; mxcli docker check reports 0 errors. + +create module BugTest2145; + +create microflow BugTest2145.ACT_Noop () begin end; + +create page BugTest2145.Action_Rebind (Title: 'Rebind', Layout: Atlas_Core.Atlas_Default) { + use building block Atlas_Web_Content.Card_WithImage (action: call microflow BugTest2145.ACT_Noop) as w_ +}; diff --git a/mdl-examples/bug-tests/559-prefixed-building-block-false-ce0495.mdl b/mdl-examples/bug-tests/559-prefixed-building-block-false-ce0495.mdl new file mode 100644 index 000000000..f414b5a94 --- /dev/null +++ b/mdl-examples/bug-tests/559-prefixed-building-block-false-ce0495.mdl @@ -0,0 +1,26 @@ +mdl 1; +-- Bug ako/mxcli#559: placing one building block several times, each with its +-- own `as `, was refused by `check --references`: +-- duplicate widget name 'Atlas_Web_Content.Card' (used 4 times) +-- — Mendix requires unique widget names per page (CE0495) +-- check counted the `use building block` statement as a widget named after the +-- block, before expansion; exec expands and prefixes (a_container2, +-- b_container2, …) and mx check reports no CE0495. +-- +-- Fix: the page-context check counts the names of the EXPANDED tree +-- (expandUseRefsForCheck). The same block placed twice WITHOUT a prefix is +-- still reported, by its real widget names ('container2', 'text22'), which is +-- what exec refuses. +-- +-- Verify (needs a project with Atlas_Web_Content, e.g. `mxcli new`): +-- mxcli check 559-prefixed-building-block-false-ce0495.mdl -p app.mpr --references +-- passes; exec writes a_/b_/c_/d_ copies; mxcli docker check: 0 errors. + +create module BugTest559; + +create page BugTest559.Dashboard (Title: 'Dashboard', Layout: Atlas_Core.Atlas_Default) { + use building block Atlas_Web_Content.Card as a_ + use building block Atlas_Web_Content.Card as b_ + use building block Atlas_Web_Content.Card as c_ + use building block Atlas_Web_Content.Card as d_ +}; diff --git a/mdl-examples/bug-tests/560-building-block-prefix-intra-block-references.mdl b/mdl-examples/bug-tests/560-building-block-prefix-intra-block-references.mdl new file mode 100644 index 000000000..c8503f2fe --- /dev/null +++ b/mdl-examples/bug-tests/560-building-block-prefix-intra-block-references.mdl @@ -0,0 +1,25 @@ +mdl 1; +-- Bug ako/mxcli#560: `use building block … as ` renamed the block's +-- widgets but not the references between them. Atlas_Web_Content.Master_Detail +-- holds `gallery gallery2` and `dataview dataView7 (DataSource: selection +-- gallery2)`; copied `as md_`, the widgets became md_gallery2 / md_dataView7 +-- and exec failed with +-- Error: failed to build page: failed to build datasource: widget not found: gallery2 +-- +-- Fix: prefixWidgetNames rewrites, in the same pass, every reference to a name +-- the block declares — a `selection` datasource, `delete $w`, a flow argument +-- `$w`, a template parameter `$w.Attr`, an expression `$w/Attr` — and leaves +-- references to widgets outside the block alone. `use fragment … as` shares it. +-- +-- Verify (needs a project with Atlas_Web_Content, e.g. `mxcli new`): exec +-- writes `dataview md_dataView7 (DataSource: selection md_gallery2)`. mxcli +-- docker check then reports only the block template's unbound text boxes +-- (CE7005); with the selection left on `gallery2` exec refused the page. + +create module BugTest560; + +create persistent entity BugTest560.Person (Name: string(100)); + +create page BugTest560.MasterDetail (Title: 'Master detail', Layout: Atlas_Core.Atlas_Default) { + use building block Atlas_Web_Content.Master_Detail (datasource: database from BugTest560.Person) as md_ +}; diff --git a/mdl-examples/bug-tests/684-layout-flows-merge-overlaps.mdl b/mdl-examples/bug-tests/684-layout-flows-merge-overlaps.mdl new file mode 100644 index 000000000..1198f06d7 --- /dev/null +++ b/mdl-examples/bug-tests/684-layout-flows-merge-overlaps.mdl @@ -0,0 +1,92 @@ +mdl 1; +-- ============================================================================ +-- ako/mxcli#684: `mxcli layout flows` introduced MPR008 overlaps at merge nodes. +-- After the reporter's correction, two remain: +-- +-- '(merge)' (2355,80) and '(merge)' (2360,80) ACT_DealGame 5px +-- '(merge)' (2815,1730) and '(unnamed)' (2740,1730) ACT_Refresh 75px +-- +-- "both merge nodes, at the same y on the main path." An IF's closing merge is +-- placed from the measured width of its branch, and a decision inside the branch +-- was measured from its split's centre instead of its left edge (ako/mxcli#1773), +-- so the merge landed in the activity — or the merge — before it. +-- +-- The reporter's project is not available. These are the shapes that put +-- merges side by side on the main path: decisions closing at the same point, a +-- decision followed by an activity in a branch, a loop ending in a decision, and +-- nested loops. Execute without @position, or execute and run `layout flows`, +-- then lint: MPR008 and MPR011 must not increase. +-- +-- mxcli exec 684-layout-flows-merge-overlaps.mdl -p app.mpr +-- mxcli layout flows -p app.mpr --module Bug684 +-- mxcli lint -p app.mpr --rules MPR008,MPR011 +-- +-- Regression coverage: mdl/executor/layout_merge_overlap_684_test.go +-- ============================================================================ + +create or modify module Bug684; + +create or modify persistent entity Bug684.Cell ( "Value": Integer ); + +create or modify microflow Bug684.DecisionsCloseTogether ($Flag: Boolean, $Other: Boolean) +begin + if $Flag then + log info node 'Bug684' 'a'; + if $Other then + log info node 'Bug684' 'b'; + else + log info node 'Bug684' 'c'; + end if; + else + log info node 'Bug684' 'd'; + end if; + log info node 'Bug684' 'done'; +end; + +create or modify microflow Bug684.ThreeDecisionsCloseTogether ($Flag: Boolean, $Other: Boolean) +begin + if $Flag then + if $Other then + if $Flag then + log info node 'Bug684' 'b'; + end if; + end if; + end if; + log info node 'Bug684' 'done'; +end; + +create or modify microflow Bug684.DecisionThenActivityInBranch ($Flag: Boolean, $Other: Boolean) +begin + if $Flag then + if $Other then + log info node 'Bug684' 'b'; + else + log info node 'Bug684' 'c'; + end if; + log info node 'Bug684' 'x'; + else + log info node 'Bug684' 'd'; + end if; + log info node 'Bug684' 'done'; +end; + +create or modify microflow Bug684.NestedLoopsWithMerges ($Flag: Boolean, $Other: Boolean, $Cells: list of Bug684.Cell) +begin + if $Flag then + loop $Row in $Cells begin + if $Other then + log info node 'Bug684' 'a'; + end if; + loop $Col in $Cells begin + if $Other then + log info node 'Bug684' 'b'; + else + log info node 'Bug684' 'c'; + end if; + log info node 'Bug684' 'd'; + end loop; + log info node 'Bug684' 'e'; + end loop; + end if; + log info node 'Bug684' 'done'; +end; diff --git a/mdl-examples/doctype-tests/15c-fragment-bindings-examples.mdl b/mdl-examples/doctype-tests/15c-fragment-bindings-examples.mdl index 278bb9a48..cdda4fb42 100644 --- a/mdl-examples/doctype-tests/15c-fragment-bindings-examples.mdl +++ b/mdl-examples/doctype-tests/15c-fragment-bindings-examples.mdl @@ -83,7 +83,13 @@ describe page BindTest.P001_Bound_Panel; -- Binding-point rule (prototype): -- datasource → the first widget carrying a datasource -- action → the first button widget (Atlas placeholder buttons ship with --- no action, so the target is matched by widget type) +-- no action, so the target is matched by widget type). In a +-- block with none: the first pluggable button (its definition +-- names it a button and takes `OnClick:`, such as the PDS Button +-- the PlatformDesignSystem blocks are built from), else the first +-- other pluggable widget with a click action (an Image). A PDS +-- Link before a PDS Button is skipped; a gallery row click never +-- counts. -- -- Requires the Atlas_Web_Content.List_Cards building block, which ships with the -- Atlas UI in Mendix 11.x (it is absent from the 10.x Atlas), so this section is diff --git a/mdl/backend/connection.go b/mdl/backend/connection.go index 275dffe99..8d8f33c5a 100644 --- a/mdl/backend/connection.go +++ b/mdl/backend/connection.go @@ -29,6 +29,16 @@ type ConnectionBackend interface { GetMendixVersion() (string, error) } +// DiskChangeSyncer is implemented by a backend that reads a project on disk +// another process can write to (ako/mxcli#2121). SyncWithDisk reports whether +// something other than this connection changed the project since it last +// looked, and when it did, has already dropped what the connection kept, so +// the next read sees the disk. It must be cheap — a session calls it before +// every statement — and must not report the connection's own writes. +type DiskChangeSyncer interface { + SyncWithDisk() (changed bool, err error) +} + // ModuleBackend provides module-level operations. type ModuleBackend interface { // ListModules returns all modules in the project. diff --git a/mdl/backend/modelsdk/backend.go b/mdl/backend/modelsdk/backend.go index 4470d08c5..1d900e763 100644 --- a/mdl/backend/modelsdk/backend.go +++ b/mdl/backend/modelsdk/backend.go @@ -25,9 +25,9 @@ import ( "github.com/mendixlabs/mxcli/mdl/backend" "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/model" - "github.com/mendixlabs/mxcli/modelsdk/codec" genPr "github.com/mendixlabs/mxcli/modelsdk/gen/projects" mmpr "github.com/mendixlabs/mxcli/modelsdk/mpr" + "github.com/mendixlabs/mxcli/modelsdk/mprread" ) // Compile-time guarantee that the backend satisfies the whole interface (via the @@ -35,6 +35,7 @@ import ( var _ backend.FullBackend = (*Backend)(nil) var _ backend.WriteStatsReporter = (*Backend)(nil) var _ backend.SourceFileReporter = (*Backend)(nil) +var _ backend.DiskChangeSyncer = (*Backend)(nil) // Backend reads and writes a Mendix project through the modelsdk codec engine. // It embeds `unimplemented` (generated, see gen_unimplemented.go) so any @@ -47,6 +48,9 @@ type Backend struct { writer *mmpr.Writer path string + // domainModelListings counts ListDomainModels calls (DomainModelListings). + domainModelListings int64 + // fileWrites counts writes that do not go through unit storage — the // generated .java/.js source of a code action, whose body lives in // javasource/ or javascriptsource/ rather than in its unit. They are folded @@ -196,6 +200,46 @@ func (b *Backend) Disconnect() error { return flushErr } +// SyncWithDisk reports whether another process changed the project since this +// connection last looked, having first dropped every view of it the +// connection keeps, so the next read sees the disk (ako/mxcli#2121). A session +// calls it before each statement: one PRAGMA and one stat, not a re-read. +// +// When the .mpr was replaced by a new file (a git checkout renames one into +// place) the connection still holds the old one open, so the project is +// reopened. Not while a deferred run or a simulation holds unwritten units: +// reopening would write them into, or drop them from, a project they were not +// made against, so that is refused with an error instead. +func (b *Backend) SyncWithDisk() (changed bool, err error) { + if b.reader == nil { + return false, nil + } + switch b.reader.SyncWithDisk() { + case mmpr.DiskUnchanged: + return false, nil + case mmpr.DiskChanged: + return true, nil + } + if b.writer != nil && b.writer.HoldsWrites() { + return true, fmt.Errorf("%s was replaced on disk while unwritten changes were held; reconnect to read the new file", b.path) + } + readOnly := b.writer == nil + path := b.path + if err := b.reader.Close(); err != nil { + return true, fmt.Errorf("close the replaced project: %w", err) + } + b.reader, b.writer = nil, nil + if readOnly { + err = b.ConnectReadOnly(path) + } else { + err = b.Connect(path) + } + if err != nil { + return true, fmt.Errorf("reopen %s after it was replaced on disk: %w", path, err) + } + return true, nil +} + // DeferUnitWrites holds unit updates in memory until FlushDeferredWrites, so a // run of statements is written once and judged by its net result // (mmpr.Writer.DeferUnitWrites, ako/mxcli#872). Reads during the run see the @@ -206,6 +250,24 @@ func (b *Backend) DeferUnitWrites() { } } +// DeferUnitWritesJudged is DeferUnitWrites for a run whose statements report +// for themselves: each held write is counted in WriteStats as it is held +// (mmpr.Writer.DeferUnitWritesJudged, ako/mxcli#2160). +func (b *Backend) DeferUnitWritesJudged() { + if b.writer != nil { + b.writer.DeferUnitWritesJudged() + } +} + +// UnitReconciles is how many times this backend's writer has compared a write +// with the stored unit. For tests. +func (b *Backend) UnitReconciles() int { + if b.writer == nil { + return 0 + } + return b.writer.Reconciles() +} + // FlushDeferredWrites writes what a deferred run holds, each unit once and // reconciled against what was stored before the run. func (b *Backend) FlushDeferredWrites() error { @@ -271,6 +333,34 @@ func (b *Backend) UnitFileReads() int64 { return b.reader.UnitFileReads() } +// UnitJudgedDigests is how many canonical digests this backend's writer has +// computed to judge the held writes of a judged run. For tests. +func (b *Backend) UnitJudgedDigests() int { + if b.writer == nil { + return 0 + } + return b.writer.JudgedDigests() +} + +// UnitIndexBuilds is how many times this backend's reader has scanned the Unit +// table to rebuild its list of units. For tests. +func (b *Backend) UnitIndexBuilds() int64 { + if b.reader == nil { + return 0 + } + return b.reader.UnitIndexBuilds() +} + +// UnitStoreHits is how many unit reads this backend's reader served from the +// persisted unit store instead of a file. With UnitFileReads it is every read +// that was not served from memory. For tests. +func (b *Backend) UnitStoreHits() int64 { + if b.reader == nil { + return 0 + } + return b.reader.UnitStoreHits() +} + func (b *Backend) Version() types.MPRVersion { if b.reader == nil { return 0 @@ -345,18 +435,14 @@ func (b *Backend) enrichModule(m *model.Module) { if pm, ok := b.reader.PackagedModuleOf(string(m.ID)); ok { m.PackageFile = pm.File } - raw, err := b.reader.GetRawUnitBytes(string(m.ID)) - if err != nil || len(raw) == 0 { - return - } - el, err := codec.NewDecoder(codec.DefaultRegistry).Decode(raw) - if err != nil { - return - } - gm, ok := el.(*genPr.Module) - if !ok { + // Through the decode cache: a module is looked up for every statement of a + // script, and decoding its unit afresh each time was a full decode per + // module per listing (ako/mxcli#2160). + u, ok, err := mprread.UnitByIDAs[*genPr.Module](b.reader, m.ID) + if err != nil || !ok { return } + gm := u.Element m.FromAppStore = gm.FromAppStore() m.AppStoreVersion = gm.AppStoreVersion() m.AppStoreGuid = gm.AppStoreGuid() diff --git a/mdl/backend/modelsdk/domainmodel.go b/mdl/backend/modelsdk/domainmodel.go index 3d6a05b88..a469b100f 100644 --- a/mdl/backend/modelsdk/domainmodel.go +++ b/mdl/backend/modelsdk/domainmodel.go @@ -7,6 +7,7 @@ import ( "sort" "strconv" "strings" + "sync/atomic" "github.com/mendixlabs/mxcli/modelsdk/element" genDm "github.com/mendixlabs/mxcli/modelsdk/gen/domainmodels" @@ -27,8 +28,14 @@ import ( // faithful member counts); full attribute-type / association-detail fidelity // (DESCRIBE level) is a later phase. +// DomainModelListings is how many times this backend has listed and converted +// every domain model of the project. A statement that makes one per lookup +// costs the project once per lookup (ako/mxcli#2160). For tests. +func (b *Backend) DomainModelListings() int64 { return atomic.LoadInt64(&b.domainModelListings) } + // ListDomainModels reads every domain model through the codec engine. func (b *Backend) ListDomainModels() ([]*domainmodel.DomainModel, error) { + atomic.AddInt64(&b.domainModelListings, 1) units, err := listUnits[*genDm.DomainModel](b) if err != nil { return nil, err diff --git a/mdl/catalog/builder.go b/mdl/catalog/builder.go index 4e7b75b25..d22dc5f24 100644 --- a/mdl/catalog/builder.go +++ b/mdl/catalog/builder.go @@ -91,6 +91,7 @@ type Builder struct { reader CatalogReader snapshot *Snapshot progress ProgressFunc + routine ProgressFunc // routine counts; nil sends them to progress (SetRoutineProgress) hierarchy *hierarchy tx *builderTx // Transaction for batched inserts; records row owners fullMode bool // If true, do full parsing (activities/widgets) @@ -764,10 +765,30 @@ func (b *Builder) resetCollected() { // read as the table's size and is left out; say reports regardless. func (b *Builder) report(table string, count int) { if b.scope == nil { - b.say(table, count) + b.count(table, count) } } +// SetRoutineProgress sends the routine lines — a table's row count, the +// source pass's periodic "source n/m documents" — to fn instead of to the +// Build callback. Those are about fifty lines a build that say nothing but +// "still going", so a caller writing to a log or an agent's context can +// throttle them, while the lines that carry information (the incremental +// summary, documents that could not be described) still reach Build's +// callback (ako/mxcli#1860). +func (b *Builder) SetRoutineProgress(fn ProgressFunc) { + b.routine = fn +} + +// count reports a routine line in every build. +func (b *Builder) count(line string, n int) { + if b.routine != nil { + b.routine(line, n) + return + } + b.say(line, n) +} + // say reports a progress line in every build. func (b *Builder) say(line string, count int) { if b.progress != nil { diff --git a/mdl/catalog/builder_source.go b/mdl/catalog/builder_source.go index 57443d030..50961f4b5 100644 --- a/mdl/catalog/builder_source.go +++ b/mdl/catalog/builder_source.go @@ -260,7 +260,7 @@ func (b *Builder) buildSource() error { case <-stop: return case <-ticker.C: - b.say(fmt.Sprintf("source %d/%d documents", done.Load(), total), int(done.Load())) + b.count(fmt.Sprintf("source %d/%d documents", done.Load(), total), int(done.Load())) } } }() diff --git a/mdl/catalog/builder_widget_definitions.go b/mdl/catalog/builder_widget_definitions.go index d7ac669b1..b058c13d0 100644 --- a/mdl/catalog/builder_widget_definitions.go +++ b/mdl/catalog/builder_widget_definitions.go @@ -165,9 +165,7 @@ func (b *Builder) buildWidgetDefinitions() error { count++ } - if b.progress != nil { - b.progress("widget_definitions", count) - } + b.count("widget_definitions", count) return nil } diff --git a/mdl/catalog/builder_workflow_activities.go b/mdl/catalog/builder_workflow_activities.go new file mode 100644 index 000000000..e7b10d513 --- /dev/null +++ b/mdl/catalog/builder_workflow_activities.go @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "github.com/mendixlabs/mxcli/mdl/backend/wfnames" + "github.com/mendixlabs/mxcli/sdk/workflows" +) + +// workflowActivityRow is one row of workflow_activities (ako/mxcli#1295). +type workflowActivityRow struct { + ID, Name, Caption, ActivityType string + Microflow, UserSourceMicroflow, OnCreatedMicroflow, CompletionMicroflow string + Page, UserTaskEntity, CalledWorkflow string +} + +// workflowActivityRows lists a workflow's activities in walk order with the +// documents each one names. A lint rule that wants "every microflow a system +// task runs" or "every page a user task opens" reads it, where refs says only +// that the workflow calls the microflow, not from which kind of activity. +func workflowActivityRows(wf *workflows.Workflow) []workflowActivityRow { + var out []workflowActivityRow + wfnames.WalkActivities(wf, func(act workflows.WorkflowActivity) { + r := workflowActivityRow{ + ID: string(act.GetID()), + Name: act.GetName(), + Caption: act.GetCaption(), + ActivityType: act.ActivityType(), + } + switch a := act.(type) { + case *workflows.UserTask: + if a.IsMulti { + r.ActivityType = "MultiUserTask" + } + r.Page, r.UserTaskEntity, r.OnCreatedMicroflow = a.Page, a.UserTaskEntity, a.OnCreated + switch us := a.UserSource.(type) { + case *workflows.MicroflowBasedUserSource: + r.UserSourceMicroflow = us.Microflow + case *workflows.MicroflowGroupSource: + r.UserSourceMicroflow = us.Microflow + } + if a.CompletionCriteria != nil { + r.CompletionMicroflow = a.CompletionCriteria.Microflow + } + case *workflows.CallMicroflowTask: + if a.IsAgent { + r.ActivityType = "AIAgentTask" + } + r.Microflow = a.Microflow + case *workflows.SystemTask: + r.Microflow = a.Microflow + case *workflows.CallWorkflowActivity: + r.CalledWorkflow = a.Workflow + } + out = append(out, r) + }) + return out +} + +// buildWorkflowActivities writes workflow_activities. +func (b *Builder) buildWorkflowActivities(wfs []*workflows.Workflow) (int, error) { + stmt, err := b.tx.Prepare(` + INSERT INTO workflow_activities_data (Id, Sequence, WorkflowId, WorkflowQualifiedName, ModuleName, + Name, Caption, ActivityType, Microflow, UserSourceMicroflow, OnCreatedMicroflow, + CompletionMicroflow, Page, UserTaskEntity, CalledWorkflow, ProjectId, SnapshotId) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `) + if err != nil { + return 0, err + } + defer stmt.Close() + projectID, snapshotID := b.snapshotMeta() + + count := 0 + for _, wf := range wfs { + if !b.own(string(wf.ID)) { + continue + } + moduleName := b.hierarchy.getModuleName(b.hierarchy.findModuleID(wf.ContainerID)) + qualifiedName := moduleName + "." + wf.Name + for i, r := range workflowActivityRows(wf) { + if _, err := stmt.Exec(r.ID, i, string(wf.ID), qualifiedName, moduleName, + r.Name, r.Caption, r.ActivityType, + nullIfEmpty(r.Microflow), nullIfEmpty(r.UserSourceMicroflow), nullIfEmpty(r.OnCreatedMicroflow), + nullIfEmpty(r.CompletionMicroflow), nullIfEmpty(r.Page), nullIfEmpty(r.UserTaskEntity), + nullIfEmpty(r.CalledWorkflow), projectID, snapshotID); err != nil { + return count, err + } + count++ + } + } + b.disown() + return count, nil +} + +// nullIfEmpty is a reference column: the referenced document's qualified +// name, or NULL when the slot is empty — one rule since schema 47: absent is +// NULL (ako/mxcli#2093, docRefName). +func nullIfEmpty(s string) any { + if s == "" { + return nil + } + return s +} diff --git a/mdl/catalog/builder_workflow_activities_test.go b/mdl/catalog/builder_workflow_activities_test.go new file mode 100644 index 000000000..445e30e21 --- /dev/null +++ b/mdl/catalog/builder_workflow_activities_test.go @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "fmt" + "strings" + "testing" + + modelsdkbackend "github.com/mendixlabs/mxcli/mdl/backend/modelsdk" +) + +// ako/mxcli#1295 (mendixlabs/mxcli#1269): a lint rule could not ask which kind +// of workflow activity runs a microflow — "a system task's microflow is +// prefixed WF_SYS_", "a user task's targeting microflow WF_TGT_". refs records +// only that the WORKFLOW calls the MICROFLOW, and workflows_data only counts +// activities. + +func TestWorkflowActivityRows_KindAndSlots_Issue1295(t *testing.T) { + var got []string + for _, r := range workflowActivityRows(everySlotWorkflow()) { + got = append(got, fmt.Sprintf("%s mf=%s src=%s created=%s done=%s page=%s wf=%s", + r.ActivityType, r.Microflow, r.UserSourceMicroflow, r.OnCreatedMicroflow, + r.CompletionMicroflow, r.Page, r.CalledWorkflow)) + } + want := []string{ + // The main flow, each activity followed by its boundary-event paths. + "UserTask mf= src=M.TargetUsers created=M.OnCreated done= page=M.TaskPage wf=", + "CallMicroflow mf=M.InUserTaskBoundary src= created= done= page= wf=", + "MultiUserTask mf= src=M.TargetGroups created= done=M.Completion page= wf=", + "CallMicroflow mf=M.MainCall src= created= done= page= wf=", + "SystemTask mf=M.InCallBoundary src= created= done= page= wf=", + "CallWorkflow mf= src= created= done= page= wf=M.SubWorkflow", + "CallMicroflow mf=M.InCallWorkflowBoundary src= created= done= page= wf=", + "WaitForNotification mf= src= created= done= page= wf=", + "CallMicroflow mf=M.InWaitBoundary src= created= done= page= wf=", + // Then the event sub-process. + "InterruptingNotificationEventSubProcessStart mf= src= created= done= page= wf=", + "CallMicroflow mf=M.InEventSubProcess src= created= done= page= wf=", + "UserTask mf= src= created= done= page=M.SubProcessTaskPage wf=", + } + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Errorf("rows:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +} + +// The table is written by a fast build: model-quality rules run on one. +func TestTestApp_WorkflowActivitiesInFastBuild_Issue1295(t *testing.T) { + be := modelsdkbackend.New() + if err := be.Connect(copyTestApp(t)); err != nil { + t.Fatalf("connect: %v", err) + } + t.Cleanup(func() { be.Disconnect() }) + cat, err := New() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { cat.Close() }) + if err := NewBuilder(cat, be).Build(nil); err != nil { + t.Fatalf("catalog build: %v", err) + } + + // workflow.UserTaskEventHandle is a user task's on-created microflow + // (TestTestApp_WorkflowHandlersAreNotDead). + rows := queryRows(t, cat, `SELECT WorkflowQualifiedName, ActivityType FROM workflow_activities + WHERE OnCreatedMicroflow = 'workflow.UserTaskEventHandle'`) + if len(rows) == 0 { + t.Fatal("no activity has workflow.UserTaskEventHandle as its on-created microflow") + } + for _, r := range rows { + if typ := r[1].(string); typ != "UserTask" && typ != "MultiUserTask" { + t.Errorf("%v: on-created microflow on a %s", r[0], typ) + } + } + // An empty slot is NULL, not ''. + if n := queryRows(t, cat, `SELECT COUNT(*) FROM workflow_activities WHERE Microflow = '' OR Page = ''`); n[0][0].(int64) != 0 { + t.Errorf("%v rows store an empty slot as ''", n[0][0]) + } + // Every workflow's activity count matches its rows. + for _, r := range queryRows(t, cat, `SELECT w.QualifiedName, w.ActivityCount, + (SELECT COUNT(*) FROM workflow_activities a WHERE a.WorkflowId = w.Id) + FROM workflows w`) { + if r[1].(int64) != r[2].(int64) { + t.Errorf("%v: ActivityCount %v but %v activity rows", r[0], r[1], r[2]) + } + } +} diff --git a/mdl/catalog/builder_workflows.go b/mdl/catalog/builder_workflows.go index ec1cd00bf..e904a7bb3 100644 --- a/mdl/catalog/builder_workflows.go +++ b/mdl/catalog/builder_workflows.go @@ -72,6 +72,12 @@ func (b *Builder) buildWorkflows() error { b.disown() b.report("Workflows", count) + + activities, err := b.buildWorkflowActivities(wfs) + if err != nil { + return err + } + b.report("Workflow activities", activities) return nil } diff --git a/mdl/catalog/catalog.go b/mdl/catalog/catalog.go index 47ee873f4..9528df35c 100644 --- a/mdl/catalog/catalog.go +++ b/mdl/catalog/catalog.go @@ -157,6 +157,7 @@ func (c *Catalog) Tables() []string { "CATALOG.SNAPSHOTS", "CATALOG.OBJECTS", "CATALOG.WORKFLOWS", + "CATALOG.WORKFLOW_ACTIVITIES", "CATALOG.CONSUMED_ODATA_SERVICES", "CATALOG.PUBLISHED_ODATA_SERVICES", "CATALOG.BUSINESS_EVENT_SERVICES", diff --git a/mdl/catalog/tables.go b/mdl/catalog/tables.go index e01478b59..04bc71e79 100644 --- a/mdl/catalog/tables.go +++ b/mdl/catalog/tables.go @@ -7,6 +7,11 @@ package catalog // // History: // +// 48 (workflow activities): new workflow_activities table, one row per +// activity of a workflow — boundary-event paths and event sub-processes +// included — with its kind and the microflow, page or workflow each of +// its slots names (ako/mxcli#1295). Without the bump a cached catalog +// lacks the table and workflow_activities() fails with "no such table". // 47 (absent references are NULL): the agent-editor tables (agents, // agent_tools, agent_knowledge_bases, ai_models, knowledge_bases, // consumed_mcp_services, published_mcp_services) store an absent @@ -257,7 +262,7 @@ package catalog // SnapshotSource / SourceId / SourceBranch / SourceRevision columns // from every row (issue #576). // 1 — initial flat schema with denormalized snapshot columns on every row. -const CatalogSchemaVersion = "47" +const CatalogSchemaVersion = "48" // MetaSchemaVersion is the catalog_meta key that records the schema version // the cache was built against. @@ -1135,6 +1140,41 @@ func (c *Catalog) createTables() error { )`, viewWithFullSnapshot("workflows"), + // workflow_activities — one row per activity of a workflow, in the + // order wfnames.WalkActivities visits them: the main flow, the paths + // of its boundary events, then its event sub-processes + // (ako/mxcli#1295). ActivityType is the activity's kind as + // workflows_data counts it, with the two kinds that share a Go type + // told apart: MultiUserTask from UserTask, AIAgentTask from + // CallMicroflow. Each slot that names another document is a column of + // its own, so a rule can ask which microflow runs where: + // Microflow a system task's, call-microflow or AI agent task's + // UserSourceMicroflow a user task's targeting microflow (users or groups) + // OnCreatedMicroflow a user task's on-created microflow + // CompletionMicroflow a multi-user task's completion microflow + // Page a user task's page + // CalledWorkflow a call-workflow activity's workflow + `CREATE TABLE IF NOT EXISTS workflow_activities_data ( + Id TEXT PRIMARY KEY, + Sequence INTEGER DEFAULT 0, + WorkflowId TEXT, + WorkflowQualifiedName TEXT, + ModuleName TEXT, + Name TEXT, + Caption TEXT, + ActivityType TEXT, + Microflow TEXT, + UserSourceMicroflow TEXT, + OnCreatedMicroflow TEXT, + CompletionMicroflow TEXT, + Page TEXT, + UserTaskEntity TEXT, + CalledWorkflow TEXT, + ProjectId TEXT, + SnapshotId TEXT + )`, + viewWithFullSnapshot("workflow_activities"), + // business_event_services `CREATE TABLE IF NOT EXISTS business_event_services_data ( Id TEXT PRIMARY KEY, @@ -1868,6 +1908,7 @@ func (c *Catalog) createTables() error { `CREATE INDEX IF NOT EXISTS idx_odata_services_module ON odata_services_data(ModuleName)`, `CREATE INDEX IF NOT EXISTS idx_workflows_name ON workflows_data(QualifiedName)`, `CREATE INDEX IF NOT EXISTS idx_workflows_module ON workflows_data(ModuleName)`, + `CREATE INDEX IF NOT EXISTS idx_workflow_activities_workflow ON workflow_activities_data(WorkflowQualifiedName)`, `CREATE INDEX IF NOT EXISTS idx_be_services_name ON business_event_services_data(QualifiedName)`, `CREATE INDEX IF NOT EXISTS idx_be_services_module ON business_event_services_data(ModuleName)`, `CREATE INDEX IF NOT EXISTS idx_role_mappings_user_role ON role_mappings(UserRoleName)`, diff --git a/mdl/executor/catalog_progress.go b/mdl/executor/catalog_progress.go new file mode 100644 index 000000000..ff9892450 --- /dev/null +++ b/mdl/executor/catalog_progress.go @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "io" + "os" + "sync" + "time" + + "golang.org/x/term" +) + +// catalogHeartbeatEvery is how often a catalog build that is not writing to a +// terminal says it is still going. +const catalogHeartbeatEvery = 15 * time.Second + +// catalogHeartbeat throttles a catalog build's routine progress lines — one +// "✓ Table: N" per table, the source pass's "source n/m documents" every 2 s — +// for a reader who is not watching a terminal. +// +// Printed in full they were about fifty lines on every build, and a build runs +// on the first command after each model change, so an agent paid for them in +// context over and over (ako/mxcli#1860). Dropping them is not the answer +// either: a cold build of a large project takes minutes, and a log with +// nothing in it for that long reads as a hang. So a build that finishes +// quickly prints none of them, and a slow one prints the latest at most every +// catalogHeartbeatEvery. +type catalogHeartbeat struct { + w io.Writer + every time.Duration + now func() time.Time + + mu sync.Mutex + start time.Time + last time.Time +} + +func newCatalogHeartbeat(w io.Writer) *catalogHeartbeat { + h := &catalogHeartbeat{w: w, every: catalogHeartbeatEvery, now: time.Now} + h.start = h.now() + h.last = h.start + return h +} + +// line is a catalog.ProgressFunc. +func (h *catalogHeartbeat) line(text string, count int) { + h.mu.Lock() + defer h.mu.Unlock() + t := h.now() + if t.Sub(h.last) < h.every { + return + } + h.last = t + fmt.Fprintf(h.w, "… building catalog, %s (%s)\n", progressText(text, count), formatDuration(t.Sub(h.start))) +} + +// progressText is a routine progress line as the per-table form prints it: +// "Entities: 111", or the line itself when it already carries its numbers. +func progressText(text string, count int) string { + for _, r := range text { + if r >= '0' && r <= '9' { + return text + } + } + return fmt.Sprintf("%s: %d", text, count) +} + +// isTerminal reports whether w writes to a terminal. +func isTerminal(w io.Writer) bool { + f, ok := w.(*os.File) + return ok && term.IsTerminal(int(f.Fd())) +} diff --git a/mdl/executor/catalog_progress_test.go b/mdl/executor/catalog_progress_test.go new file mode 100644 index 000000000..a39c00afa --- /dev/null +++ b/mdl/executor/catalog_progress_test.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "bytes" + "strings" + "testing" + "time" +) + +// ako/mxcli#1860: off a terminal a catalog build prints none of its routine +// lines while it is quick, and the latest one at most every interval while it +// is slow, so a minutes-long build still shows it is alive. +func TestCatalogHeartbeatThrottlesRoutineLines_Issue1860(t *testing.T) { + var out bytes.Buffer + clock := time.Unix(0, 0) + h := newCatalogHeartbeat(&out) + h.now = func() time.Time { return clock } + h.start, h.last = clock, clock + + for _, table := range []string{"Modules", "Entities", "Attributes"} { + clock = clock.Add(time.Second) + h.line(table, 7) + } + if out.Len() != 0 { + t.Fatalf("a 3-second build printed:\n%s", out.String()) + } + + clock = clock.Add(13 * time.Second) // 16 s in + h.line("Microflows", 42) + clock = clock.Add(2 * time.Second) + h.line("source 10/300 documents", 10) + clock = clock.Add(14 * time.Second) // 32 s in + h.line("source 250/300 documents", 250) + + got := strings.Split(strings.TrimSpace(out.String()), "\n") + want := []string{ + "… building catalog, Microflows: 42 (16s)", + "… building catalog, source 250/300 documents (32s)", + } + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Errorf("heartbeat:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +} diff --git a/mdl/executor/check_unit_reads_pedapp_test.go b/mdl/executor/check_unit_reads_pedapp_test.go index 00b077a46..8311dccd2 100644 --- a/mdl/executor/check_unit_reads_pedapp_test.go +++ b/mdl/executor/check_unit_reads_pedapp_test.go @@ -31,6 +31,21 @@ func unitFileReads(t *testing.T, e *Executor) int64 { return b.UnitFileReads() } +// unitReadsNotFromMemory is every unit read the backend did not serve from +// its in-memory cache: the files it opened and the reads its persisted store +// answered. A run that ends inside a cache scope flushes what it read to that +// store, so afterwards an uncached read of the same unit is a store read, not +// a file read — and counting files alone would call a pass that reads each +// flow nine times a pass that reads almost nothing. +func unitReadsNotFromMemory(t *testing.T, e *Executor) int64 { + t.Helper() + b, ok := e.Backend().(interface{ UnitStoreHits() int64 }) + if !ok { + t.Fatalf("backend %T does not count its store reads", e.Backend()) + } + return unitFileReads(t, e) + b.UnitStoreHits() +} + // redeclareChain is the issue's generator: n microflows, each calling the one // before it. func redeclareChain(n int, verb string) string { @@ -75,9 +90,12 @@ func TestCheckPasses_ReadEachUnitFileOnce(t *testing.T) { // the unit index names flows without opening them (ako/mxcli#1791) it // reads only flows it looks up, but each of them several times — 103 // reads for these 12 flows, against 12 with the cache. - before := unitFileReads(t, exec) + // Counted as reads not served from memory: the setup above ran as a + // script, which holds the cache and leaves what it read in the persisted + // store (ako/mxcli#2160). + before := unitReadsNotFromMemory(t, exec) CheckFlowVerdicts(exec.newExecContext(context.Background()), prog) - if reads := unitFileReads(t, exec) - before; reads <= 2*n { + if reads := unitReadsNotFromMemory(t, exec) - before; reads <= 2*n { t.Fatalf("control: the uncached verdict pass read %d files for %d flows; the bound below would prove nothing", reads, n) } diff --git a/mdl/executor/cmd_catalog.go b/mdl/executor/cmd_catalog.go index 522e79a05..9d9ca8ffb 100644 --- a/mdl/executor/cmd_catalog.go +++ b/mdl/executor/cmd_catalog.go @@ -571,15 +571,22 @@ func buildCatalog(ctx *ExecContext, full, isSource, communities bool, resolution if hook := catalogBuilderHook; hook != nil { hook(builder) } - // Per-table lines are the only sign of life during a build that takes - // minutes on a large app (18 on a 140MB one), so quiet mode moves them off - // stdout rather than dropping them: `search -q --format names` promises one - // name per line, and forty "✓ Table: N" lines ahead of the names on a cold - // cache broke that promise. + // Quiet mode moves progress off stdout: `search -q --format names` + // promises one name per line, and forty "✓ Table: N" lines ahead of the + // names on a cold cache broke that promise. buildProgress := ctx.progress() if ctx.Quiet { buildProgress = ctx.diagnostics() } + // A line per table is for someone watching a terminal. Anywhere else — + // a pipe, a log, an agent's context — the build prints only a heartbeat, + // so the ~50 lines are not paid for after every model change, and a build + // that takes minutes (18 on a 140MB app) still shows it is alive + // (ako/mxcli#1860). Lines that carry information still print in full. + perTable := ctx.terminal && !ctx.Quiet + if !perTable { + builder.SetRoutineProgress(newCatalogHeartbeat(buildProgress).line) + } err = builder.Build(func(table string, count int) { fmt.Fprintf(buildProgress, "✓ %s: %d\n", table, count) }) @@ -638,7 +645,7 @@ func buildCatalog(ctx *ExecContext, full, isSource, communities bool, resolution // and the in-place write that followed raced it (ako/mxcli#951). if err := cat.SaveToFile(cachePath); err != nil { fmt.Fprintf(ctx.progress(), "Warning: failed to save catalog cache: %v\n", err) - } else { + } else if perTable { fmt.Fprintf(buildProgress, "✓ Catalog cached to %s\n", cachePath) } } diff --git a/mdl/executor/cmd_domainmodel_svg.go b/mdl/executor/cmd_domainmodel_svg.go index 5ccfab481..692bb0de8 100644 --- a/mdl/executor/cmd_domainmodel_svg.go +++ b/mdl/executor/cmd_domainmodel_svg.go @@ -117,6 +117,109 @@ func (b svgBox) edgePointFrom(sx, sy, tx, ty float64) (float64, float64) { return sx + dx*scale, sy + dy*scale } +// anchorPoint is where a stored association anchor sits on the drawn box. +// +// Mendix stores an association's anchors (ParentConnection on the FROM entity, +// ChildConnection on the TO entity) as "x;y", a PERCENTAGE of the entity box +// (see sdk/domainmodel/connection.go for the measurement). The box here is this +// renderer's estimate, not Studio Pro's, so the percentage is applied to the +// box as drawn — the line attaches to the same edge, the same fraction along. +// Mendix's loader does not range-check, so a value outside 0..100 is clamped +// onto the box: a connector attaches to its entity. nil means nothing (or +// nothing readable) was stored, and the caller keeps the centre-line drawing. +func (b svgBox) anchorPoint(p *model.Point) (x, y float64, ok bool) { + if p == nil { + return 0, 0, false + } + fx := math.Max(0, math.Min(100, float64(p.X))) / 100 + fy := math.Max(0, math.Min(100, float64(p.Y))) / 100 + return b.x + fx*b.w, b.y + fy*b.h, true +} + +// outward is the unit normal of the box edge nearest to (x, y), a point on it. +func (b svgBox) outward(x, y float64) (float64, float64) { + nx, ny, best := -1.0, 0.0, x-b.x + if d := b.x + b.w - x; d < best { + nx, ny, best = 1, 0, d + } + if d := y - b.y; d < best { + nx, ny, best = 0, -1, d + } + if d := b.y + b.h - y; d < best { + nx, ny = 0, 1 + } + return nx, ny +} + +// selfLoop routes a self-association from anchor (x1, y1) back to anchor +// (x2, y2) of the same box: a short stub straight out of each anchor's edge, +// joined the shorter way round a rectangle `d` outside the box, so the line +// never crosses the entity it belongs to. +func (b svgBox) selfLoop(x1, y1, x2, y2 float64) [][2]float64 { + const d = 20.0 + r := svgBox{x: b.x - d, y: b.y - d, w: b.w + 2*d, h: b.h + 2*d} + n1x, n1y := b.outward(x1, y1) + n2x, n2y := b.outward(x2, y2) + s1 := [2]float64{x1 + n1x*d, y1 + n1y*d} + s2 := [2]float64{x2 + n2x*d, y2 + n2y*d} + // Position along r's perimeter, clockwise from its top-left corner. + per := 2 * (r.w + r.h) + at := func(p [2]float64) float64 { + switch { + case p[1] <= r.y: + return p[0] - r.x + case p[0] >= r.x+r.w: + return r.w + p[1] - r.y + case p[1] >= r.y+r.h: + return r.w + r.h + r.x + r.w - p[0] + default: + return 2*r.w + r.h + r.y + r.h - p[1] + } + } + corners := []struct { + t float64 + p [2]float64 + }{ + {0, [2]float64{r.x, r.y}}, + {r.w, [2]float64{r.x + r.w, r.y}}, + {r.w + r.h, [2]float64{r.x + r.w, r.y + r.h}}, + {2*r.w + r.h, [2]float64{r.x, r.y + r.h}}, + } + t1, t2 := at(s1), at(s2) + cw := math.Mod(t2-t1+per, per) + clockwise := cw <= per-cw + walk := cw + if !clockwise { + walk = per - cw + } + // The corners passed on the way, each with its distance from s1 along the + // walk; at most four, so sorting them by insertion is enough. + type passed struct { + p [2]float64 + dist float64 + } + var via []passed + for _, c := range corners { + dc := math.Mod(c.t-t1+per, per) + if !clockwise { + dc = math.Mod(t1-c.t+per, per) + } + if dc <= 0 || dc >= walk { + continue + } + i := len(via) + via = append(via, passed{c.p, dc}) + for ; i > 0 && via[i-1].dist > via[i].dist; i-- { + via[i-1], via[i] = via[i], via[i-1] + } + } + pts := [][2]float64{{x1, y1}, s1} + for _, v := range via { + pts = append(pts, v.p) + } + return append(pts, s2, [2]float64{x2, y2}) +} + func svgNum(f float64) string { return fmt.Sprintf("%.0f", f) } func svgText(s string) string { @@ -239,11 +342,44 @@ func renderDomainModelSVG(moduleName string, entities []*domainmodel.Entity, ass } x1, y1 := from.edgePointFrom(fcx, fcy, tcx, tcy) x2, y2 := to.edgePointFrom(tcx, tcy, fcx, fcy) + // The stored anchors say where Studio Pro attaches each end; an end + // with none stored keeps the centre-line point. (ako/mxcli#1841) + ax1, ay1, okA1 := from.anchorPoint(a.ParentConnection) + ax2, ay2, okA2 := to.anchorPoint(a.ChildConnection) + if okA1 { + x1, y1 = ax1, ay1 + } + if okA2 { + x2, y2 = ax2, ay2 + } fromMult, toMult := "*", "1" if a.Type == domainmodel.AssociationTypeReferenceSet { toMult = "*" } fmt.Fprintf(&sb, `%s`, svgText(a.Name)) + if a.ParentID == a.ChildID && okA1 && okA2 { + // A self-association leaves its box and comes back in, round + // the outside; a straight line would cross the entity. + route := from.selfLoop(x1, y1, x2, y2) + sb.WriteString(``) + n := len(route) + writeEndLabel(&sb, x1, y1, route[1][0], route[1][1], fromMult) + writeEndLabel(&sb, x2, y2, route[n-2][0], route[n-2][1], toMult) + mid := route[n/2] + fmt.Fprintf(&sb, `%s`, + svgNum(mid[0]), svgNum(mid[1]-4+offset/16*12), svgText(a.Name)) + sb.WriteString(``) + continue + } fmt.Fprintf(&sb, ``, svgNum(x1), svgNum(y1), svgNum(x2), svgNum(y2)) writeEndLabel(&sb, x1, y1, x2, y2, fromMult) writeEndLabel(&sb, x2, y2, x1, y1, toMult) diff --git a/mdl/executor/cmd_domainmodel_svg_test.go b/mdl/executor/cmd_domainmodel_svg_test.go index 25495d7e0..249e60e99 100644 --- a/mdl/executor/cmd_domainmodel_svg_test.go +++ b/mdl/executor/cmd_domainmodel_svg_test.go @@ -182,3 +182,104 @@ func TestDomainModelSVG_SpreadsTheLayoutSoNamesFit(t *testing.T) { t.Errorf("want the left entity kept at 0 and the right one moved out from 150, got %v", m) } } + +// assocLine returns the endpoints of the one association line drawn for name. +func assocLine(t *testing.T, svg, name string) (x1, y1, x2, y2 int) { + t.Helper() + re := regexp.MustCompile(`` + regexp.QuoteMeta(name) + + `Customer_Parent 100 && p[0] < 190 && p[1] > 100 && p[1] < 144 { + t.Errorf("bend %v lies inside the entity box", p) + } + } +} diff --git a/mdl/executor/cmd_microflows_builder.go b/mdl/executor/cmd_microflows_builder.go index 56512ac31..8be99934b 100644 --- a/mdl/executor/cmd_microflows_builder.go +++ b/mdl/executor/cmd_microflows_builder.go @@ -185,6 +185,10 @@ type flowBuilder struct { // rejoins the normal path, keyed by the activity. describe folds that merge // away into the fall-through form and carries its position here (#750). rejoinMergeAt map[model.ID]*ast.Position + // rejoinPastRowX is the x of the merge an activity's falling-through + // handler row rejoins at when the row is too long to rejoin before the next + // statement (rejoinAfterRow). An @merge in rejoinMergeAt still wins. + rejoinPastRowX map[model.ID]int // labelReg holds the `merge