diff --git a/.claude/lint-rules/cross_module_data.star b/.claude/lint-rules/cross_module_data.star index df770ad54..d27b34dba 100644 --- a/.claude/lint-rules/cross_module_data.star +++ b/.claude/lint-rules/cross_module_data.star @@ -14,6 +14,12 @@ # (CE1843) — so there is no view entity or microflow to use instead # (ako/mxcli#2034). # +# Exempt too: an entity the page hands to a snippet or page of the entity's own +# module that takes it as a parameter. Conversational UI's chat snippet needs a +# data view on ConversationalUI.ChatContext, exactly as the module's own +# full-screen chat page has it; a view entity or microflow cannot stand in for +# the persistent object the snippet expects (ako/mxcli#2069). +# # Widget properties: # .id - Widget ID # .name - Widget name @@ -41,6 +47,32 @@ def is_platform_required(entity_ref): """The System workflow entities a task page or task inbox must use.""" return entity_ref.startswith("System.Workflow") +def takes_parameter(document, entity_ref): + """Whether a snippet or page has a parameter of type entity_ref.""" + for ref in refs_from(document): + if ref.ref_kind == "parameter" and ref.target_name == entity_ref: + return True + return False + +def fed_to_owning_module(container, entity_ref, entity_module, cache): + """Whether container calls a snippet, or links to a page, of entity_module + that takes entity_ref as a parameter — the module's own UI asking for its + own entity, which only a data view on that entity can supply.""" + key = container + "|" + entity_ref + if key in cache: + return cache[key] + found = False + for ref in refs_from(container): + if ref.target_type not in ("SNIPPET", "PAGE"): + continue + if get_module_from_qualified_name(ref.target_name) != entity_module: + continue + if takes_parameter(ref.target_name, entity_ref): + found = True + break + cache[key] = found + return found + def check(): """ Check that widgets don't reference entities from other modules. @@ -50,6 +82,7 @@ def check(): # Track violations per container to avoid duplicate messages seen = {} + fed = {} for widget in widgets(): # Skip widgets without entity references @@ -65,6 +98,8 @@ def check(): # Check if entity is from a different module if entity_module != widget.module_name: + if fed_to_owning_module(widget.container_qualified_name, widget.entity_ref, entity_module, fed): + continue # Create a unique key for this violation key = "{}:{}".format(widget.container_qualified_name, widget.entity_ref) if key in seen: diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-09-docker-build-normalises-widgets-builds-stale-project.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-09-docker-build-normalises-widgets-builds-stale-project.json new file mode 100644 index 000000000..1ca68409f --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-09-docker-build-normalises-widgets-builds-stale-project.json @@ -0,0 +1 @@ +{"area": "cmd/mxcli/docker", "date": "2026-11-09", "symptom": "ako/mxcli#1794: for a project with stale stored widget definitions, docker check failed (CE0463) while docker build / run / reload succeeded, building a widget-normalised copy; run --local and Studio Pro failed on the same project.", "cause": "buildOnCopy ran mx update-widgets on its temporary copy before the pre-build mx check and MxBuild — the pre-#646 docker check behaviour, never revisited for the build.", "file": "cmd/mxcli/docker/build.go (buildOnCopy checks the stored copy, recheckCE0463 labels, UpdateWidgetsFirst opt-in), cmd/mxcli/docker.go (--update-widgets on build/run/reload)", "insight": "When one gate's default changes (#646: check what is stored), every other command that runs the same tool on a copy needs the same decision, or two gates disagree about one project and the green one is believed.", "refs": ["ako/mxcli#1794", "ako/mxcli#646", "ako/mxcli#568", "ako/mxcli#1792"], "test": "cmd/mxcli/docker/build_stored_test.go, cmd/mxcli/docker/build_readonly_test.go"} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-09-opencode-instructions-inline-every-skill.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-09-opencode-instructions-inline-every-skill.json new file mode 100644 index 000000000..5f5ca4b31 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-09-opencode-instructions-inline-every-skill.json @@ -0,0 +1 @@ +{"area": "cmd/mxcli", "date": "2026-11-09", "symptom": "ako/mxcli#1822 (mendixlabs/mxcli#1683): an OpenCode session in an `init --tool opencode` project started with 150 instruction files, 2.7 MB (~73% of a 1M window); .opencode/skills/ copies were described by their title ('Agents') instead of the skill's 'Use when…' description.", "cause": "generateOpenCodeConfig listed .opencode/skills/**/SKILL.md and .ai-context/skills/*/SKILL.md as `instructions`, which OpenCode inlines; wrapSkillContent prepended a second frontmatter described by the first heading in front of the skill's own, and OpenCode reads the first.", "file": "cmd/mxcli/tool_templates.go (generateOpenCodeConfig), cmd/mxcli/init_claudemd.go (wrapSkillContent, splitSkillFrontmatter)", "insight": "A tool adapter written before the Agent Skills migration (#906/#959) kept the force-feed pattern that migration removed; when skills change shape, re-check every per-tool wrapper, which each have their own copy of the old assumption (Vibe's wrapper still reads the heading).", "refs": ["ako/mxcli#1822", "mendixlabs/mxcli#1683", "ako/mxcli#906"], "test": "cmd/mxcli/init_opencode_context_test.go"} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-09-empty-placeholder-hidden-slot-ce0463.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-09-empty-placeholder-hidden-slot-ce0463.json new file mode 100644 index 000000000..c5125cc02 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-09-empty-placeholder-hidden-slot-ce0463.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-09", "symptom": "ako/mxcli#1862 (mendixlabs/mxcli#1704): no documented MDL form for Data grid 2's emptyPlaceholder. On main `emptyplaceholder { … }` parsed, checked and executed, but mx check (11.14) then reported CE0463 on the grid.", "cause": "The slot is hidden while showEmptyPlaceholder is 'none' (the default); widgets in a hidden slot are a definition mismatch. Studio Pro's own grid with a placeholder (TestApp WorkflowCommons.ConflictResolution) stores showEmptyPlaceholder 'custom'. MDL-WIDGET10 never saw it because widgetValueMap only counted `key: value` properties as set, not slots filled by child blocks.", "file": "mdl/executor/validate_widgets.go (widgetValueMap marks filled slots), mdl/executor/validate_widget_hidden.go (filled hidden slot is an error), cmd/mxcli/syntax/features_page.go (documented form)", "insight": "'Accepted and round-trips' is not 'works': run mx check on a control page with and without the feature — a plain grid was 0 errors, the same grid plus a placeholder CE0463. Then diff against a Studio Pro-authored instance of the same feature in the fixture (a describe sweep for the keyword finds one) — the difference was a sibling property, not the slot's own BSON.", "refs": ["ako/mxcli#1862", "mendixlabs/mxcli#1704", "ako/mxcli#956"], "ce": ["CE0463"], "test": "mdl/executor/validate_slot_hidden_1862_test.go"} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-09-move-leaves-xpath-path-steps-and-roles.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-09-move-leaves-xpath-path-steps-and-roles.json new file mode 100644 index 000000000..1cf582a51 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-09-move-leaves-xpath-path-steps-and-roles.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-09", "symptom": "ako/mxcli#1836 (mendixlabs/mxcli#1687): after moving entities and flows to another module, mx check (11.12.6) reported CE1613 'association Src.Order_Customer / entity Src.Customer no longer exists' at the moved entity's access rule; the rule kept Src.User and a moved nanoflow kept its Src grants while the microflow's were remapped. Enumeration/constant refs in expressions were already fixed (#1671, #1674).", "cause": "UpdateQualifiedNameInAllUnits matched whole strings, prefixes and RewriteExpressionReferences tokens, which deliberately skips path steps (an enumeration rename must not touch a same-named association); moveNanoflow had no remapDocumentAccessRoles call; nothing remapped entity access-rule roles.", "file": "mdl/types/expression_rename.go (RewritePathSteps), mdl/backend/modelsdk/infrastructure_write.go (path steps only when newName is an entity/association), mdl/executor/cmd_move.go (moveNanoflow remap, remapEntityAccessRoles)", "insight": "Whether a path step may be rewritten depends on the renamed element's KIND, which a name-based sweep cannot see — but after the move the element exists under its new name, so asking the domain models whether newName is an entity or association decides it without a new backend method.", "refs": ["ako/mxcli#1836", "mendixlabs/mxcli#1687", "ako/mxcli#1671", "ako/mxcli#605"], "test": "mdl/executor/move_module_split_test.go, mdl/types/expression_rename_test.go"} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-09-quoted-entity-type-argument-not-reproduced.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-09-quoted-entity-type-argument-not-reproduced.json new file mode 100644 index 000000000..1d11e6912 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-09-quoted-entity-type-argument-not-reproduced.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-09", "symptom": "ako/mxcli#2057 (mendixlabs/mxcli#1785, mxcli 0.26.0): `CALL JAVASCRIPT ACTION \"NanoflowCommons\".\"RefreshEntity\" (EntityToRefresh = \"CustomModule\".\"NPActivity\")` reported stored with its MDL quotes, CE1613 at mxbuild.", "cause": "Not reproduced on main: the stored EntityTypeCodeActionParameterValue.Entity is the bare qualified name for quoted and unquoted spellings alike, and mx check (11.14, TestApp with NanoflowCommons) reports 0 errors. The entity-type branch (#1137) and its whitespace normalisation (#1171) are the fixes in that path; history here is squashed at v0.26.0, so the exact release cannot be pinned.", "file": "mdl/executor/cmd_microflows_builder_calls.go (entityTypeArgument)", "insight": "For this class, verify with the BSON value (`mxcli bson dump --type nanoflow --object X | grep -A1 Entity`) and mx check, never describe — describe prints quoted and broken values identically. A report against an old version that main does not reproduce still earns a verbatim regression test, so a regression is caught by name.", "refs": ["ako/mxcli#2057", "mendixlabs/mxcli#1785", "mendixlabs/mxcli#1137", "mendixlabs/mxcli#1171"], "test": "mdl/executor/cmd_microflows_builder_entity_type_arg_test.go (TestBuildJavaScriptAction_QuotedEntityTypeArgument)"} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-09-snippet-param-from-outer-context-and-specialised-hop.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-09-snippet-param-from-outer-context-and-specialised-hop.json new file mode 100644 index 000000000..a62a05916 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-09-snippet-param-from-outer-context-and-specialised-hop.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-09", "symptom": "ako/mxcli#2055: describe → exec of TestApp's WorkflowCommons.ManageTaskAssignments refused 'snippet …Snip_TaskAssignment_UserTask_Assignees requires parameter $Account'; with that refusal lifted, the rebuilt page failed mx check with CE1613 'System.User.FullName no longer exists' at two labels.", "cause": "(1) buildSnippetCallParams accepted an omitted snippet parameter only from the nearest data container (pb.entityContext); Mendix takes it from any enclosing one — here a System.User data view around a Data grid 2 of System.WorkflowUserTask, stored with no mapping, mx check 0 errors. (2) associationSourcePath printed a single hop as the bare association, dropping a stored DestinationEntity that specialises the association's end.", "file": "mdl/executor/cmd_pages_builder_v3_widgets.go (outerEntityContexts, contextSupplies), mdl/executor/widget_engine.go (Build pushes the outer context), mdl/executor/cmd_pages_describe_datasource.go (withSpecialisedDestination)", "insight": "Lifting a refusal can turn a loud failure into a silent broken write: after making the builder accept the call, run mx check on the result before calling it fixed — the refusal had been hiding a describe gap (#721 L5's specialised-destination class, single-hop case) that produced CE1613.", "refs": ["ako/mxcli#2055", "ako/mxcli#1897", "ako/mxcli#868", "mendixlabs/mxcli#1467"], "test": "mdl/executor/snippet_call_outer_context_test.go"} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-10-check-misses-doc-renamed-earlier-in-script.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-10-check-misses-doc-renamed-earlier-in-script.json new file mode 100644 index 000000000..ce5b85945 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-10-check-misses-doc-renamed-earlier-in-script.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-10", "symptom": "ako/mxcli#2070 (mendixlabs/mxcli#1790): `rename page M.Old_Page to New_Page;` then `show page M.New_Page` in a microflow of the same script: check --references reported 'page not found: M.New_Page'; exec ran it and mx check reported 0 errors.", "cause": "collectSingle recorded a RENAME (and a cross-module MOVE) only in scriptContext.indirectDocs, which only the alter-target check (#1821) reads. Every reference check — show page, call microflow, snippet calls, navigation — consults the kind's own script-defined set (sc.pages, sc.microflows, …), so the new name was never known to them.", "file": "mdl/executor/validate.go (markIndirectDoc)", "insight": "A second registry for 'documents the script provides' drifts from the first one the moment a new reader appears. Write the indirect name into the same per-kind set the CREATEs fill, so every existing and future reference check inherits it. Verified on TestApp: check refused before, passes after; exec + mx check 0 errors.", "refs": ["ako/mxcli#2070", "mendixlabs/mxcli#1790", "ako/mxcli#1821", "mendixlabs/mxcli#1212"], "test": "mdl/executor/validate_renamed_doc_refs_1790_test.go"} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-10-consumed-mcp-service-no-endpoint.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-10-consumed-mcp-service-no-endpoint.json new file mode 100644 index 000000000..d948a149e --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-10-consumed-mcp-service-no-endpoint.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-10", "refs": ["ako/mxcli#2060", "ako/mxcli#2067"], "ce": [], "symptom": "A consumed MCP service written by mxcli connects nowhere: Agent Editor skips it at startup (\"No value could be found for the local configuration for the constant\") and an agent's `mcp service` block exposes no tools. mx check and the build are green; MDL had no way to write the endpoint constant or the credentials microflow.", "file": "sdk/agenteditor/types.go and mdl/types/agenteditor.go (ConsumedMCPService.Endpoint, AuthenticationMicroflow), agenteditor_read.go / mprread reader_documents.go, agenteditor_write.go encodeConsumedMCPServiceContents (endpoint first, credentials last), agenteditor_carry.go consumedMCPServiceContentsSpec, visitor_agenteditor.go + visitor_strict_properties.go, cmd_agenteditor_write.go / cmd_agenteditor_alter.go (applyConsumedMCPServiceChange takes ctx), cmd_agenteditor_mcpservices.go (describe, consumedMCPEndpointRef, consumedMCPCredentialsProblem), validate.go validateConsumedMCPRefs (check --references), validate_consumed_mcp_endpoint.go (MDL-AGENT03). Tests: agenteditor_consumedmcp_test.go (fixtures testdata/agenteditor/FactoryManagement.{GraphStudio_MCP,JiraMCP}.11_16.json), cmd_agenteditor_consumedmcp_endpoint_test.go, visitor_agent_settings_test.go", "cause": "ConsumedMCPService modelled only protocolVersion, version, connectionTimeoutSeconds and documentation. AgentEditorCommons' IM_ConsumedMCPService (1.1.0, 2.2.1 and 2.6.0 alike) maps endpoint/qualifiedName to MCPEndpointConstant and authenticationMicroflow/qualifiedName to GetCredentialsMicroflow; ConsumedMCPService_CreateUpdateFromModel imports the service only when GetConstantValue of that constant is non-empty, and MCPClient's ConsumedMCPService_CreateMCPClient calls the credentials microflow with no arguments and maps its result to System.HttpHeader objects.", "insight": "**The contract of a referenced microflow is in the consumer's Java, not in the document.** The stored document only names GraphAuth; reading ConsumedMCPService_CreateMCPClient.java (Core.microflowCall(name).execute(ctx) with no params, result mapped to HttpHeader) fixed what check --references must refuse, and the stored GraphAuth (no parameters, returns List of System.HttpHeader via MCPClient.HttpHeader_CreateAndAddToList) confirmed it. Studio Pro saved the six FactoryManagement services in two key orders (endpoint first in five, after protocolVersion in one), each reference documentId first; both fixtures rebuild byte for byte through the carry, and a new document takes the majority order. documentId is the referenced unit's id (the constant's and the microflow's mxunit file names). Controls: dropping the encoder field, leaving authenticationMicroflow unmodelled, ignoring the statement's Endpoint, not keeping the stored references on create or modify, omitting them in describe, accepting a non-String constant, silencing validateConsumedMCPRefs or MDL-AGENT03, and alter not assigning the endpoint each fail their test."} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-10-create-agent-no-selected-model.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-10-create-agent-no-selected-model.json new file mode 100644 index 000000000..ba810b28e --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-10-create-agent-no-selected-model.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-10", "refs": ["ako/mxcli#2031", "ako/mxcli#2067"], "ce": [], "symptom": "An agent written with `create agent` (Agent Editor 2.2+) is refused at startup by ASU_AgentEditor: \"Model with name is referenced but could not be found for the Agent\", and the refusal stops the import of every agent document. mx check and the build are green; MDL had no way to write the agent's selectedModel.", "file": "sdk/agenteditor/types.go and mdl/types/agenteditor.go (SelectedModel), agenteditor_read.go / mprread reader_documents.go, agenteditor_write.go (encoder, after model), agenteditor_carry.go (selectedModel modelled), visitor_agenteditor.go + visitor_strict_properties.go (SelectedModel, SelectedModelName), cmd_agenteditor_write.go + cmd_agenteditor_carry.go selectedModelFrom, cmd_agenteditor_alter.go (keys applied in sorted order), cmd_agenteditor_agents.go describe, validate_agent_selected_model.go (MDL-AGENT02). Tests: agenteditor_carry_test.go TestEncodeAgentContents_SelectedModel, cmd_agenteditor_selectedmodel_test.go, validate_agent_selected_model_test.go, visitor_agent_settings_test.go", "cause": "selectedModel {id, name} was carried on rewrite (#2037) but never modelled, so a new agent had none. AgentEditorCommons 2.2.1 and 2.6.0 AgentFromModel_Validate require SelectedModelID, and AgentFromModel_CreateUpdateAgent retrieves the DeployedModel where Model = SelectedModelID; IM_AgentFromModel maps only selectedModel/id (name is Studio Pro's display text). AgentEditorCommons 1.1.0 does not read it.", "insight": "**Read the importer, not the document, to learn what is required.** The stored documents showed selectedModel on one agent in five apps, which looked optional; the import mapping and the validation microflow (describe microflow AgentEditorCommons.AgentFromModel_Validate) showed it is mandatory from 2.2 and that only the id matters, which fixed both the syntax (id key, optional display name defaulting to the id) and the check rule's scope (every plain create agent, not only MxCloudGenAI). A key that is set by two alter assignments (id and name) must be applied in a fixed order: the Sets map iterates randomly, so the test sets both in one statement and the control (reverse order) fails. Controls: dropping the encoder field, leaving selectedModel unmodelled in the carry, ignoring the statement's SelectedModel, omitting it in describe and silencing MDL-AGENT02 each fail their test; a build without the encoder field changes GraphExplorer's unit on an ICXGraphPilot describe→exec round trip, where the fixed build changes none."} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-10-move-same-name-other-doctype.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-10-move-same-name-other-doctype.json new file mode 100644 index 000000000..12592f9cf --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-10-move-same-name-other-doctype.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-10", "symptom": "ako/mxcli#2072 (mendixlabs/mxcli#1792): with a microflow M.Nightly and a scheduled event M.Nightly, `move scheduled event M.Nightly to folder 'Jobs'` was refused: 'M.Nightly is a microflow, not a scheduled event'.", "cause": "The generic MOVE resolved the name through FindDocumentUnit, which returns the first document of that name in the module whatever its type, then refused it as a mistyped doctype. The comment beside it assumed document names are unique per module; they are unique per document type.", "file": "mdl/executor/cmd_move_document.go (findDocumentOfKind), mdl/backend/modelsdk/placement.go (FindDocumentUnits)", "insight": "A type-agnostic lookup by name is only safe as an authority when the caller does not know the kind. When the statement names one, ask for all matches and pick by kind; keep the first-match fallback so a true typo still gets the 'is a X, not a Y' message. Reproduced on TestApp with the unfixed binary before trusting the mock test.", "refs": ["ako/mxcli#2072", "mendixlabs/mxcli#1792", "ako/mxcli#932"], "test": "mdl/executor/cmd_move_same_name_1792_test.go"} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-09-conv013-flags-only-valid-rollback.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-09-conv013-flags-only-valid-rollback.json new file mode 100644 index 000000000..595f3b86b --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-09-conv013-flags-only-valid-rollback.json @@ -0,0 +1 @@ +{"area": "mdl/linter", "date": "2026-11-09", "symptom": "ako/mxcli#1838 (mendixlabs/mxcli#1689): CONV013 'uses Rollback error handling instead of Custom' fired on a call to CommunityCommons.ThrowException (a deliberate rollback) and on Java action calls inside a loop, where custom handling is CE0644 — one rule or another always fired.", "cause": "findUnhandledCalls descended into loop bodies and treated every Java action call alike.", "file": "mdl/linter/rules/conv_error_handling.go (no descent into LoopedActivity; isThrowingJavaAction)", "insight": "A lint rule and a check rule that demand opposite things for the same element (CONV013 vs MDL006/CE0644) are a defect in the lint rule: check encodes what Studio Pro accepts. Grep the CE number across mdl/executor before deciding which side is right.", "refs": ["ako/mxcli#1838", "mendixlabs/mxcli#1689", "mendixlabs/mxcli#1202"], "test": "mdl/linter/rules/conv_error_handling_test.go"} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-09-domain-model-annotations-not-in-catalog.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-09-domain-model-annotations-not-in-catalog.json new file mode 100644 index 000000000..c8416898f --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-09-domain-model-annotations-not-in-catalog.json @@ -0,0 +1 @@ +{"area": "mdl/catalog", "date": "2026-11-09", "symptom": "A Starlark rule could not read a domain model annotation: not in strings(), modules(), entities() or documentable_elements(), and annotations() was undefined. A 'no TODO notes' rule covered microflows (annotations are activities) and passed every domain model.", "cause": "No catalog table held DomainModel.Annotations, though the decoded domain model already carries them (list annotations reads them).", "file": "mdl/catalog/builder_annotations.go; mdl/linter/context_catalog_tables.go", "insight": "A domain model annotation's Caption is a plain string, not a Texts$Text, so the translatable-strings walk never reaches it and adding it to strings() would have been the wrong home. Own the rows by the domain model unit (b.own(dm.ID)), like the entities, so TestCatalog_RowsBelongToADocument and the incremental tests need no allowlist. CATALOG.ANNOTATIONS hit the keyword trap (LIST ANNOTATIONS) and TestEveryCatalogTableParses caught it before any manual run. A microflow annotation is activity_type == 'Annotation', not action_type: check a field name against starlark.go before documenting it. Cross-check counts per module with LIST ANNOTATIONS.", "issue": "ako/mxcli#1300"} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-09-enum-value-id-for-translations.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-09-enum-value-id-for-translations.json new file mode 100644 index 000000000..53f3c50b9 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-09-enum-value-id-for-translations.json @@ -0,0 +1 @@ +{"area": "mdl/linter", "date": "2026-11-09", "symptom": "A Starlark rule could not tie a caption in another language to its enumeration value: enumerations().values had name, qualified_name, caption (default language) and ordinal, and strings() rows for Enumerations$EnumerationValue.Caption carried an element_id that matched nothing a rule could see. `v.id` stopped the rule with 'enumeration_value struct has no .id attribute'.", "cause": "The value's Id was in enumeration_values_data all along and equals the translations walk's ElementID (the owner's $ID); the linter query did not select it.", "file": "mdl/linter/context.go; mdl/linter/starlark.go", "insight": "Half of #1301 had already shipped with #1453, so the issue looked done; read the issue's Expected section line by line before closing as stale. Measure the join in SQL before writing code: on CitrusGrove `strings s left join enumeration_values v on v.Id = s.ElementId` matched 144 of 144 caption rows, which settled that exposing the id was the whole fix. A user module is needed to see it end to end: enumerations() leaves out Marketplace modules, and CitrusGrove's own modules have none, so the first run reported values=0.", "issue": "ako/mxcli#1301"} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-09-folders-not-in-catalog.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-09-folders-not-in-catalog.json new file mode 100644 index 000000000..8cb70fcd9 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-09-folders-not-in-catalog.json @@ -0,0 +1 @@ +{"area": "mdl/catalog", "date": "2026-11-09", "symptom": "A Starlark rule could not see an empty folder: the catalog had no folders table, so a folder existed for a rule only as the folder of a document in it, and 'every module has the folders Objects and UI' reported an empty folder as missing. Separately, `select ... from CATALOG.AGENTS` printed nothing and exited 0.", "cause": "No folders pass, although buildHierarchy already lists every folder for the documents' folder paths. CATALOG.AGENTS: AGENTS is a lexer keyword missing from catalogTableName, so the SELECT parsed to zero statements, with no error.", "file": "mdl/catalog/builder_folders.go; mdl/grammar/domains/MDLCatalog.g4; mdl/visitor/visitor_catalog_tables_parse_test.go", "insight": "Three things that would have saved time. (1) A new CATALOG. whose name is a lexer keyword (FOLDERS from LIST FOLDERS) parses to NOTHING, not to an error, and exits 0; this was the fourth such table after COMMUNITIES, QUEUES and AGENTS. TestEveryCatalogTableParses now parses every name Catalog.Tables() lists, and found AGENTS on its first run. If `select` prints nothing, try the same table in a subquery: that goes to SQL unparsed and works. (2) LIST FOLDERS already lists empty folders and is the cross-check: diff its per-folder counts against the table. On CitrusGrove all 179 folders matched; the one spelling difference was DeepLink's folder NAMED 'Pages/Snippets', which LIST FOLDERS escapes and the Folder column does not, and it broke a depth computed by counting slashes. Walk the parents. (3) A pass must not list through b.reader (TestBuilderListsDocumentsThroughTheScope); build from b.hierarchy, which already holds every folder.", "issue": "ako/mxcli#1436"} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-09-jar-files-not-visible-to-lint.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-09-jar-files-not-visible-to-lint.json new file mode 100644 index 000000000..5144571e5 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-09-jar-files-not-visible-to-lint.json @@ -0,0 +1 @@ +{"area": "mdl/linter", "date": "2026-11-09", "symptom": "A Starlark rule could not see the .jar files in userlib/ and vendorlib/: jar_dependencies() lists only the Maven dependencies module settings declare, and jar_files() was undefined, so one library on the classpath in two versions went unseen.", "cause": "The jars are files beside the .mpr, not model data, so nothing the catalog builds held them and no builtin read the file system.", "file": "mdl/linter/jar_files.go", "insight": "Read them at call time from the project folder rather than adding a catalog table: they change without the .mpr changing, so a cached catalog would answer stale. The project folder reaches the linter through the backend's Path() (an optional interface on the reader), so no lint entry point needed a new argument. Split name and version at the LAST '-digit', not the first: log4j-1.2-api-2.17.1 and guava-32.0.1-jre both need it. Measured on CitrusGrove: 65 userlib + 14 vendorlib jars, matching ls, and the duplicate-version rule found real conflicts the Marketplace modules ship (guava android/jre across the two folders, seven netty libraries 4.1.77 vs 4.1.94, slf4j-api 1.7.36 vs 2.0.0). Some .RequiredLib markers name a jar that is not there; they are ignored, not reported as rows.", "issue": "ako/mxcli#1304"} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-10-agent-and-code-action-param-refs-missing.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-10-agent-and-code-action-param-refs-missing.json new file mode 100644 index 000000000..0b076d89d --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-10-agent-and-code-action-param-refs-missing.json @@ -0,0 +1 @@ +{"area": "mdl/catalog", "date": "2026-11-10", "symptom": "Agent-editor documents produced no refs and a microflow passed as a microflow-typed Java action parameter was no reference: an agent's tool microflow, a chat context's ActionMicroflow and MCPServer.AddPrompt's Microflow showed '(no callers found)' and sat in GRAPH_DEAD_ASSETS, though deleting one breaks the app at runtime and neither mx check nor a build notices.", "cause": "The agent pass read only the CustomBlobDocument wrapper (name, documentation), not the JSON Contents with tools, model, knowledge bases and key; and microflowActionRef emitted only the Java action itself, skipping MicroflowParameterValue and CustomBlobDocumentCodeActionParameterValue in the parameter mappings.", "file": "mdl/catalog/builder_agent_refs.go; mdl/catalog/builder_agenteditor.go; modelsdk/mprread/reader_documents.go (DecodeAgentEditorUnit)", "insight": "The decoders already existed for describe agent in modelsdk/mprread; exporting one entry point beat re-parsing the JSON in the catalog. Resolve tool targets in buildReferences, not in the agent pass: the agent phase runs before microflows are catalogued. Measure with GRAPH_DEAD_ASSETS, main vs fix, on the Agent Builder starter app: 238 -> 208, 30 assets no longer dead and none newly dead; spot-check that each is real (MCPServer.AddPrompt(Microflow = ...) in MCPServer_Initialize). Build the 'main' binary with make build, not go build: go build fails on the generated embed dirs, and an empty baseline made every dead asset look new. Not covered: a nanoflow passed to a JavaScript action (Nanoflows$NanoflowParameterValue), which the SDK does not decode.", "issue": "ako/mxcli#2074"} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-10-agent-details-not-in-catalog.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-10-agent-details-not-in-catalog.json new file mode 100644 index 000000000..ebb3259e1 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-10-agent-details-not-in-catalog.json @@ -0,0 +1 @@ +{"area": "mdl/catalog", "date": "2026-11-10", "symptom": "CATALOG.AGENTS, AI_MODELS, KNOWLEDGE_BASES and CONSUMED_MCP_SERVICES held only Id, Name, QualifiedName, ModuleName, Folder and Description, and published MCP services were not catalogued at all: which agents run a tool without approval, which use a model, or have streaming on took one describe agent per agent; a published MCP service's tool and authentication microflows sat in GRAPH_DEAD_ASSETS.", "cause": "The agent pass inserted the CustomBlobDocument wrapper's fields only; the settings live in the JSON Contents, and DecodeAgentEditorUnit had no case for agenteditor.publishedMCPService (its decoder lived only in the backend, on the sdk/agenteditor types).", "file": "mdl/catalog/builder_agenteditor.go; mdl/catalog/tables.go; modelsdk/mprread/reader_documents.go", "insight": "Test against the real 11.16 Agent Editor fixtures in mdl/backend/modelsdk/testdata/agenteditor, not hand-written JSON: one stores the streaming switches and one does not, which is the NULL-vs-0 control for free. A boolean the document may not store must be a nullable column, or 'streaming off' and 'older Agent Editor' read the same. A published service's microflows are `publish` edges, matching published REST/OData, not `call`. Cross-check an empty column with describe before suspecting the decoder: an AI model with no type or display name and an agent with no model are real in the starter app. Measured on the Agent Builder starter app: GRAPH_DEAD_ASSETS 212 -> 210 (mcptool1, auth_mf), none newly dead.", "issue": "ako/mxcli#2075"} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-10-agent-prompts-not-in-strings.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-10-agent-prompts-not-in-strings.json new file mode 100644 index 000000000..09cbfd1eb --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-10-agent-prompts-not-in-strings.json @@ -0,0 +1 @@ +{"area": "mdl/catalog", "date": "2026-11-10", "symptom": "search could not find text in an agent's system or user prompt or in its tool and knowledge base descriptions: CATALOG.STRINGS held only the agent's documentation.", "cause": "The prompts live in the CustomBlobDocument's JSON Contents, not in a Texts$Text, so the type-agnostic translatable walk never sees them, and no typed extraction read them.", "file": "mdl/catalog/builder_agent_strings.go; mdl/catalog/builder_strings.go", "insight": "Do not decode the agent again in the strings pass: the agent pass already holds every in-scope decoded document with its owning unit (b.agentDocRefs, from #2074), and it runs before strings, so b.as(d.owner) attributes each row to the agent for incremental builds. Control on the Agent Builder starter app with the wiring stubbed: strings 7529 -> 7509 and search 'ticket management system' loses its only string match.", "issue": "ako/mxcli#2075"} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-10-arch001-entity-fed-to-own-module-snippet.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-10-arch001-entity-fed-to-own-module-snippet.json new file mode 100644 index 000000000..1c87f4fa6 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-10-arch001-entity-fed-to-own-module-snippet.json @@ -0,0 +1 @@ +{"area": "mdl/other", "date": "2026-11-10", "symptom": "ako/mxcli#2069: ARCH001 flagged the data view on ConversationalUI.ChatContext that hosts Conversational UI's chat snippet on every app page with a chat; the snippet takes that entity as its parameter, so no view entity or microflow can stand in.", "cause": "cross_module_data.star treated every cross-module entity on a widget as avoidable. #2034 exempted the System workflow entities by name; this is the general shape of the same false positive: the module's own UI asks for its own entity.", "file": ".claude/lint-rules/cross_module_data.star (fed_to_owning_module)", "insight": "Exempt by structure, not by module name: the page calls a snippet or links to a page of the entity's module whose parameter (refs kind 'parameter') is that entity. The refs table already carries both halves, so no new builtin was needed. Verified on a real catalog (TestApp + a ChatMod module) with the old rule as control: dvChat flagged before, not after; the bare data view stays flagged.", "refs": ["ako/mxcli#2069", "ako/mxcli#2034"], "test": "mdl/linter/starlark_arch001_test.go (TestARCH001ExemptsAnEntityFedToItsOwnModulesSnippet)"} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-10-widget-caption-not-tied-to-widget.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-10-widget-caption-not-tied-to-widget.json new file mode 100644 index 000000000..801d4af1b --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-10-widget-caption-not-tied-to-widget.json @@ -0,0 +1 @@ +{"area": "mdl/catalog", "date": "2026-11-10", "symptom": "A Starlark rule could not read a button's caption or tooltip: widgets() had neither, and the strings() caption row's element_id was the button's Forms$ClientTemplate, the confirmation rows' a Forms$ConfirmationInfo, so no text could be tied to a widget id.", "cause": "The widget walk never read the widget's own texts, and the translations walk recorded only a text's direct owner.", "file": "mdl/catalog/builder_pages.go (widgetTexts); mdl/translations/sites.go (WidgetID); mdl/catalog/builder_strings.go", "insight": "The storage key is CaptionTemplate, not Caption, for a button: the first version read Caption, passed a unit test built on the assumed key, and gave 0 of 73 captions on CitrusGrove. Before writing a raw-BSON reader, tally the real (parent $Type, key) pairs from `mxcli bson dump --format ndsl` over every page, snippet and layout -- it showed CaptionTemplate (buttons), Caption (labels), Content (dynamic text) and LabelTemplate (inputs) in one pass. Second trap: rebuilding mxcli without a schema bump reused the cached catalog, so the fixed build still showed 0 until `refresh catalog full force`. Coverage measured after: 66/73 buttons have a caption (the 7 are icon-only, empty in storage), 739/810 page texts carry a widget_id, the 71 others are exactly the page titles, and the 7 that name no widgets() row are tab page captions.", "issue": "ako/mxcli#1303"} diff --git a/.claude/skills/fix-issue/findings/modelsdk/2026-11-09-page-round-trip-rewrites-unexpressible-values.json b/.claude/skills/fix-issue/findings/modelsdk/2026-11-09-page-round-trip-rewrites-unexpressible-values.json new file mode 100644 index 000000000..bb8150f1e --- /dev/null +++ b/.claude/skills/fix-issue/findings/modelsdk/2026-11-09-page-round-trip-rewrites-unexpressible-values.json @@ -0,0 +1 @@ +{"area": "modelsdk/canon", "date": "2026-11-09", "symptom": "ako/mxcli#1897: describe page X > p.mdl; exec p.mdl reported 'Replaced page' with nothing changed. On TestApp 15 of 68 pages did (Administration.Account_Overview, most WorkflowCommons pages); a second exec reported Unchanged.", "cause": "Three values MDL cannot state were regenerated on rebuild: Forms$TabControl.DefaultPagePointer (stored null → first tab; a stored non-first default reset too), Expression strings with a trailing newline (MDL writes expressions bare), and an empty Forms$ClientTemplate ScreenReaderLabel (written null).", "file": "modelsdk/canon/tabdefault.go (CarryDefaultTabPages), modelsdk/canon/unexpressed.go (CarryUnexpressed), modelsdk/canon/identity.go (Reconcile runs both after TransplantIDs)", "insight": "Diff the unit bytes before and after the FIRST round trip with a value-only BSON walk (key order differs on any unit an older writer sorted, and NDSL dumps hide both); the second exec is always Unchanged, so testing a re-run proves nothing. Sweep every page of a project with describe→exec and grep for Replaced — one named page in a report is usually a class.", "refs": ["ako/mxcli#1897", "ako/mxcli#949"], "test": "modelsdk/canon/tabdefault_test.go, modelsdk/canon/unexpressed_test.go"} diff --git a/.claude/skills/mendix/agents/SKILL.md b/.claude/skills/mendix/agents/SKILL.md index 1e8c05682..d02ce4bb9 100644 --- a/.claude/skills/mendix/agents/SKILL.md +++ b/.claude/skills/mendix/agents/SKILL.md @@ -11,12 +11,12 @@ description: "Author Mendix AI agent documents in MDL — Model, Knowledge Base, | lines | section | |---|---| | 24-50 | Overview | -| 51-166 | Syntax | -| 167-193 | Altering existing documents | -| 194-222 | Gotchas | -| 223-244 | Common Patterns | -| 245-309 | Calling Agents from Microflows | -| 310-312 | AI agent documents | +| 51-192 | Syntax | +| 193-219 | Altering existing documents | +| 220-268 | Gotchas | +| 269-291 | Common Patterns | +| 292-356 | Calling Agents from Microflows | +| 357-359 | AI agent documents | # Agents @@ -92,14 +92,39 @@ create knowledge base Module.ProductDocs ( ```sql mdl 1; +create constant Module.WebSearchUrl (Type: string, DefaultValue: 'https://example.com/mcp'); create consumed mcp service Module.WebSearch ( ProtocolVersion: v2025_03_26, + Endpoint: @Module.WebSearchUrl, -- String constant: the server URL + AuthenticationMicroflow: Module.GetMcpHeaders, -- optional: returns the request headers version: '1.0', ConnectionTimeoutSeconds: 30, documentation: 'Web search MCP server' ); ``` +**`Endpoint` is what makes it connect.** It names a String constant holding the +server URL. At startup Agent Editor reads the constant's value and, when there +is none, skips the service ("No value could be found for the local +configuration for the constant"), so an agent's `mcp service` block gets no +tools. `check` warns on a `create` without it (MDL-AGENT03), and +`check --references` refuses a constant that is not a String. + +`AuthenticationMicroflow` supplies credentials: MCP Client calls it with **no +parameters** and sends the `List of System.HttpHeader` it returns with every +request (`check --references` refuses any other signature): + +```sql +mdl 1; +create microflow Module.GetMcpHeaders () +returns list of System.HttpHeader as $Headers +begin + $Headers = create list of System.HttpHeader; + call microflow MCPClient.HttpHeader_CreateAndAddToList(HttpHeaderList = $Headers, Key = 'Authorization', Value = 'Bearer ' + @Module.WebSearchToken); + return $Headers; +end; +``` + ### Published MCP Service The other direction: the app publishes its microflows as an MCP server @@ -123,6 +148,7 @@ mdl 1; create agent Module.MyAgent ( UsageType: task, -- Task | Conversational model: Module.MyModel, -- must exist + SelectedModel: 'eu.anthropic.claude-sonnet-5', description: 'Agent description', MaxTokens: 4096, Temperature: 0.7, -- float @@ -193,6 +219,26 @@ Property names match `CREATE` (`SystemPrompt`, `UserPrompt`, `Temperature`, `Max ## Gotchas +### An agent needs `SelectedModel`, or no agent imports +Agent Editor 2.2+ imports an agent only with a selected model: the id of one of +the models the Model document's resource offers. Without it, `ASU_AgentEditor` +logs "Model with name is referenced but could not be found for the Agent" at +startup and **stops importing every agent document**, not just that one. `mx +check` and the build stay green. `mxcli check` warns on a `create agent` +without it (MDL-AGENT02). + +```sql +mdl 1; +alter agent Module.MyAgent set SelectedModel = 'eu.anthropic.claude-sonnet-5'; +``` + +The runtime reads only the id and matches it against +`GenAICommons.DeployedModel.Model`. `SelectedModelName` is the display name +Studio Pro stores; it defaults to the id. To list the ids a resource offers, +with the app running (reported in ako/mxcli#2031, not measured here): +`mxcli oql "select Model from MxGenAIConnector.MxCloudDeployedModel"`. Or read +an agent Studio Pro saved: `describe agent` prints its `SelectedModel`. + ### Dollar-quoting for multi-line prompts Single-quoted strings cannot span lines. Use `$$...$$` for any SystemPrompt or UserPrompt that contains newlines. DESCRIBE always emits `$$...$$` when the value contains newlines, so DESCRIBE output re-parses cleanly. @@ -229,6 +275,7 @@ create ai model Module.M (Provider: MxCloudGenAI, key: @Module.K); create agent Module.A ( UsageType: task, model: Module.M, + SelectedModel: 'eu.anthropic.claude-sonnet-5', SystemPrompt: 'You are a helpful assistant.', UserPrompt: 'Ask me anything.' ); diff --git a/.claude/skills/mendix/project-brain/SKILL.md b/.claude/skills/mendix/project-brain/SKILL.md index 17f7cda5c..6060297a7 100644 --- a/.claude/skills/mendix/project-brain/SKILL.md +++ b/.claude/skills/mendix/project-brain/SKILL.md @@ -19,11 +19,11 @@ description: "Project-specific knowledge mxcli cannot compute — the requiremen | 255-274 | Write the anchor, not the name | | 275-296 | Checking it | | 297-312 | Size | -| 313-329 | Commands | -| 330-344 | Renaming | -| 345-354 | What not to record | -| 355-380 | Handing a slice to another agent | -| 381-383 | Project brain (`mxcli brain init/capture/staged/promote/drop/check/show`) | +| 313-330 | Commands | +| 331-345 | Renaming | +| 346-355 | What not to record | +| 356-381 | Handing a slice to another agent | +| 382-384 | Project brain (`mxcli brain init/capture/staged/promote/drop/check/show`) | # Project brain @@ -317,7 +317,8 @@ cap: the cap is what stops the store becoming a file nobody reads. | `mxcli brain init -p app.mpr` | Creates `docs/brain/`. Refuses a `docs/brain/` it did not write | | `mxcli brain capture "" [-a @Anchor]…` | Queues an entry. Never commits | | `mxcli brain staged [--since ] [--slice ] [--fail-if-empty]` | Lists the queue with the shard each entry would land in. `--since` is the slice boundary — see below | -| `mxcli brain promote [--to ]` | Writes it into its shard. The human step | +| `mxcli brain capture --from-json ` | Queues a JSON array of entries as one act: every item validated first, duplicates skipped. Use it for a list, not a loop of captures | +| `mxcli brain promote … [--to ]` | Writes them into their shards, all or none. The human step. Re-run after an interruption completes it | | `mxcli brain drop ` | Removes it from the queue or from its shard | | `mxcli brain capture "" --slice [-a @Anchor]…` | Queues a **requirement** of that slice | | `mxcli brain capture "" --open [-a @Anchor]…` | Queues an **open question**; its anchors are not checked | diff --git a/.claude/skills/mendix/write-lint-rules/SKILL.md b/.claude/skills/mendix/write-lint-rules/SKILL.md index a0fe4d77f..895a4a72d 100644 --- a/.claude/skills/mendix/write-lint-rules/SKILL.md +++ b/.claude/skills/mendix/write-lint-rules/SKILL.md @@ -11,11 +11,11 @@ description: "Write custom Starlark lint rules in .claude/lint-rules/ that run b | lines | section | |---|---| | 25-67 | Rule File Structure | -| 68-163 | Available Query Functions | -| 164-548 | Object Properties | -| 549-562 | Helper Functions | -| 563-649 | Common Patterns | -| 650-658 | Validation | +| 68-166 | Available Query Functions | +| 167-552 | Object Properties | +| 553-566 | Helper Functions | +| 567-653 | Common Patterns | +| 654-662 | Validation | # Writing Custom Starlark Lint Rules @@ -100,16 +100,19 @@ Without this, a rule that reads a full-only table under a fast build gets | `languages()` | list of language | The languages **enabled** in the project settings, in settings order (requires MPR reader; `[]` without one). Use it to scope per-language checks: `strings()` has a row for every stored translation, including languages the project never enabled | | `xpath_expressions()` | list of xpath_expression | All XPath constraint expressions in the catalog (access rules, retrieve actions, widgets) (full catalog — auto-detected) | | `modules()` | list of module | The user's modules (not System, not Marketplace), with their domain model's documentation | +| `folders()` | list of folder | Every folder of the user's modules, **empty folders included**, with its path, parent and the documents and folders directly in it. A document's `folder` only names folders that hold a document, so check a module's folder layout with this | +| `annotations()` | list of annotation | Every **domain model** annotation in the user's modules, with its full caption (newlines kept) and its place on the canvas. A microflow's annotations are activities: read them with `activities_for()` (`activity_type == "Annotation"`) | | `associations()` | list of association | All non-system associations, same-module and cross-module, with the delete behaviour of both ends | | `entity_event_handlers()` | list of entity_event_handler | Every before/after event handler on a non-system entity: which moment, which event, which microflow | | `navigation_menu_items()` | list of navigation_menu_item | Every navigation menu item of every profile, at every depth. Navigation belongs to the project, so no module filter applies | | `jar_dependencies()` | list of jar_dependency | Maven dependencies declared by non-system, non-marketplace modules | +| `jar_files()` | list of jar_file | The `.jar` files on disk in `userlib/` and `vendorlib/` (top level, as the runtime reads them), with the library name and version split from the file name and the modules whose `.RequiredLib` marker names each. Read from disk at each call, not from the catalog; `[]` without a project path. Use it to find one library on the classpath in two versions | | `strings(language = None)` | list of catalog_string | User-facing and documentary text, one row per text and language; pass `language` (`"nl_NL"`) to narrow. An untranslated language has **no row**. Needs a FULL catalog, which `mxcli lint` builds automatically for a rule that calls it | | `layouts()` | list of layout | All non-system layouts | | `published_rest_operations()` | list of published_rest_operation | Operations of published REST services, with the microflow behind each | -The fields of `module`, `association`, `entity_event_handler`, -`navigation_menu_item`, `jar_dependency`, `catalog_string`, `layout` and +The fields of `module`, `folder`, `annotation`, `association`, `entity_event_handler`, +`navigation_menu_item`, `jar_dependency`, `jar_file`, `catalog_string`, `layout` and `published_rest_operation` are in [catalog-tables.md](catalog-tables.md). ### Graph-analysis functions (architecture rules) @@ -268,6 +271,7 @@ def check(): | `qualified_name` | string | `"Sales.ApiTimeout.Slow"` — the name `refs_from()` and an expression's `getCaption(Sales.ApiTimeout.Slow)` use | | `caption` | string | Caption in the project's default language: `"400"` | | `ordinal` | int | Position in the enumeration, from 0 | +| `id` | string | The value's UUID. A caption in another language is the `strings(language = "nl_NL")` row with `context == "Enumerations$EnumerationValue.Caption"` and `element_id == v.id` | ### constant | Property | Type | Example | diff --git a/.claude/skills/mendix/write-lint-rules/catalog-tables.md b/.claude/skills/mendix/write-lint-rules/catalog-tables.md index 01f8106ab..4d4262381 100644 --- a/.claude/skills/mendix/write-lint-rules/catalog-tables.md +++ b/.claude/skills/mendix/write-lint-rules/catalog-tables.md @@ -1,7 +1,7 @@ # Catalog-table builtins: object properties -The structs returned by `modules()`, `associations()`, `entity_event_handlers()`, -`navigation_menu_items()`, `jar_dependencies()`, `strings()`, `layouts()`, +The structs returned by `modules()`, `folders()`, `annotations()`, `associations()`, `entity_event_handlers()`, +`navigation_menu_items()`, `jar_dependencies()`, `jar_files()`, `strings()`, `layouts()`, `published_rest_operations()` and `widgets()`. The functions themselves are listed in [SKILL.md](SKILL.md) under "Available Query Functions". Every builtin leaves out System and Marketplace modules, except `navigation_menu_items()`, whose rows @@ -16,6 +16,38 @@ Returned by `modules()`. | `name` | string | `"Sales"` | | `domain_model_documentation` | string | The module's domain model's documentation (Studio Pro's Documentation pane with nothing selected on the canvas). A Mendix module has no documentation of its own; this is the module-level text an author can write. `""` when none | +### folder +Returned by `folders()`. One per folder, empty folders included (ako/mxcli#1436). + +| Property | Type | Example | +|----------|------|---------| +| `id` | string | Folder UUID | +| `name` | string | `"Order"` | +| `module_name` | string | `"Sales"` | +| `path` | string | `"Objects/Order"` — spelled as a document's `folder`, so `f.path == d.folder` matches the documents in it. A name that itself holds `/` is not escaped | +| `parent_path` | string | `"Objects"`; `""` for a folder directly in the module | +| `depth` | int | `1` for a folder directly in the module; counted from the parents, not from the slashes in `path` | +| `document_count` | int | Documents directly in this folder, not in its subfolders | +| `folder_count` | int | Folders directly in this folder | + +An empty folder has `document_count == 0 and folder_count == 0`. A folder +with `document_count == 0` alone may still hold subfolders. + +### annotation +Returned by `annotations()`. One per domain model annotation (ako/mxcli#1300). + +| Property | Type | Example | +|----------|------|---------| +| `id` | string | Annotation UUID | +| `module_name` | string | `"Sales"` — the module whose domain model holds it | +| `caption` | string | `"TODO split sales\nper region"` — the full text, newlines kept | +| `x` | int | `60` — left edge on the domain model canvas | +| `y` | int | `40` — top edge on the canvas | +| `width` | int | `400` | + +A "no TODO notes" rule needs both this and the microflow annotations, which +are activities: `activities_for(flow)` with `activity_type == "Annotation"`. + ### association Returned by `associations()`. @@ -73,6 +105,34 @@ Returned by `jar_dependencies()`. | `coordinate` | string | `"org.apache.commons:commons-lang3"` — group and artifact, without the version, so it identifies the library across versions | | `is_included` | bool | Studio Pro's "Included" setting; False when the dependency is declared but not included | +### jar_file +Returned by `jar_files()` (ako/mxcli#1304). Read from the project folder at +each call: the jars are not in the `.mpr`, so the catalog does not hold them. + +| Property | Type | Example | +|----------|------|---------| +| `folder` | string | `"userlib"` or `"vendorlib"` | +| `file_name` | string | `"commons-lang3-3.12.0.jar"` | +| `name` | string | `"commons-lang3"` — the file name up to the last `-` followed by a digit, so `log4j-1.2-api-2.17.1.jar` is `log4j-1.2-api` | +| `version` | string | `"3.12.0"`; `""` when the file name carries no version | +| `required_by` | list of string | `["MyFirstModule"]` — modules with a `..RequiredLib` marker beside the jar, as a Marketplace module ships one | + +The same `name` in two versions is one library on the classpath twice: + +```python +def check(): + seen = {} + for j in jar_files(): + seen.setdefault(j.name, []).append(j) + out = [] + for name, jars in seen.items(): + versions = sorted({j.version for j in jars}) + if len(versions) > 1: + out.append(violation(message = "%s on the classpath in versions %s" % (name, ", ".join(versions)), + location = location(module = "", document_type = "project", document_name = ""))) + return out +``` + ### catalog_string Returned by `strings(language = None)`. @@ -84,6 +144,7 @@ Returned by `strings(language = None)`. | `context` | string | What the text is. Translatable text names its stored type and property: `"Forms$Page.Title"`, `"Forms$TabPage.Caption"`, `"Enumerations$EnumerationValue.Caption"`; the rest a lower-case label: `"documentation"` (of an entity, attribute, constant, page, … — `object_type` says which; a module's domain model documentation is `"domain_model_documentation"`), `"page_url"`, `"log_node"`, `"log_message"`, … | | `language` | string | `"en_US"`; `""` for text that is not translatable (documentation, URLs) | | `element_id` | string | UUID of the element carrying the text | +| `widget_id` | string | UUID of the widget around the text, matching `widgets().id`; `""` outside a widget (a page's own title). A tab page's caption names the tab page, which `widgets()` does not list. A button's caption row has `element_id` of its `Forms$ClientTemplate` and its confirmation rows that of a `Forms$ConfirmationInfo`, so collect a widget's texts in every language by `widget_id`, not `element_id` | | `module_name` | string | `"Sales"` | ### layout @@ -141,6 +202,8 @@ Returned by `widgets()` (full catalog — auto-detected). | `dynamic_classes` | string | The `Dynamic classes` expression (Appearance), else `""` | | `action_type` | string | Stored type of the widget's primary action — the button's action, else a container's on-click action, else a list view's or image's click action: `"Forms$DeleteClientAction"`, `"Forms$MicroflowAction"`, `"Forms$CallNanoflowClientAction"`, `"Forms$FormAction"` (show page), `"Forms$SaveChangesClientAction"`, `"Forms$CancelChangesClientAction"`, `"Forms$ClosePageClientAction"`, `"Forms$CreateObjectClientAction"`, `"Forms$OpenLinkClientAction"`, `"Forms$NoAction"`; `""` for a widget with no action property. Pluggable widgets' actions (inside their property bag) are not read | | `has_confirmation` | bool | The primary action asks for confirmation. Only a microflow, nanoflow or workflow call can; a delete action (`"Forms$DeleteClientAction"`) has no confirmation setting at all, so "a delete must confirm" is enforced as "no button uses the delete action directly — call a microflow or nanoflow with a confirmation" | +| `caption` | string | `"Delete sale"` — the widget's own caption in the project's default language (buttons and labels; an input's label and a dynamic text's content are not its caption); `""` when it has none. Other languages: `strings(language = …)` rows with `widget_id == w.id` | +| `tooltip` | string | The tooltip in the project's default language; `""` when none is set | ```python # Inline styles, classes outside an allow-list, and direct delete buttons. diff --git a/.claude/skills/mendix/write-lint-rules/security-and-settings.md b/.claude/skills/mendix/write-lint-rules/security-and-settings.md index 466e869ee..aa70212b7 100644 --- a/.claude/skills/mendix/write-lint-rules/security-and-settings.md +++ b/.claude/skills/mendix/write-lint-rules/security-and-settings.md @@ -33,13 +33,13 @@ MPR reader, so they need no particular catalog depth. ### reference | Property | Type | Example | |----------|------|---------| -| `source_type` | string | The document the edge comes FROM, upper-case: `"MICROFLOW"`, `"NANOFLOW"`, `"RULE"`, `"PAGE"`, `"SNIPPET"`, `"LAYOUT"`, `"ENTITY"`, `"ASSOCIATION"`, `"WORKFLOW"`, `"NAVIGATION"`, `"SCHEDULED_EVENT"`, `"PUBLISHED_REST_OPERATION"`, `"PUBLISHED_REST_SERVICE"`, `"PUBLISHED_ODATA_SERVICE"`, `"CONSUMED_ODATA_SERVICE"`, `"BUSINESS_EVENT_SERVICE"`, `"PROJECT_SETTINGS"`, `"IMPORT_MAPPING"`, `"EXPORT_MAPPING"`, `"MESSAGE_DEFINITION_COLLECTION"`, `"MESSAGE_DEFINITION"` | +| `source_type` | string | The document the edge comes FROM, upper-case: `"MICROFLOW"`, `"NANOFLOW"`, `"RULE"`, `"PAGE"`, `"SNIPPET"`, `"LAYOUT"`, `"ENTITY"`, `"ASSOCIATION"`, `"WORKFLOW"`, `"NAVIGATION"`, `"SCHEDULED_EVENT"`, `"PUBLISHED_REST_OPERATION"`, `"PUBLISHED_REST_SERVICE"`, `"PUBLISHED_ODATA_SERVICE"`, `"CONSUMED_ODATA_SERVICE"`, `"BUSINESS_EVENT_SERVICE"`, `"PROJECT_SETTINGS"`, `"IMPORT_MAPPING"`, `"EXPORT_MAPPING"`, `"MESSAGE_DEFINITION_COLLECTION"`, `"MESSAGE_DEFINITION"`, `"AGENT"`, `"AI_MODEL"`, `"KNOWLEDGE_BASE"`, `"PUBLISHED_MCP_SERVICE"` | | `source_id` | string | Source UUID | | `source_name` | string | `"Sales.ACT_Customer_Create"` | -| `target_type` | string | What it points AT, upper-case: `"ENTITY"`, `"ASSOCIATION"`, `"MICROFLOW"`, `"NANOFLOW"`, `"RULE"`, `"PAGE"`, `"SNIPPET"`, `"LAYOUT"`, `"WORKFLOW"`, `"WIDGET"`, `"JAVA_ACTION"`, `"JAVASCRIPT_ACTION"`, `"REST_OPERATION"`, `"REGULAR_EXPRESSION"`, `"ATTRIBUTE"`, `"ENUMERATION"`, `"ENUMERATION_VALUE"`, `"IMPORT_MAPPING"`, `"EXPORT_MAPPING"`, `"JSON_STRUCTURE"`. `WIDGET`, `ATTRIBUTE`, `ENUMERATION`, `ENUMERATION_VALUE` and `JSON_STRUCTURE` are only ever targets; `SCHEDULED_EVENT`, `PROJECT_SETTINGS` and the service types only ever sources | +| `target_type` | string | What it points AT, upper-case: `"ENTITY"`, `"ASSOCIATION"`, `"MICROFLOW"`, `"NANOFLOW"`, `"RULE"`, `"PAGE"`, `"SNIPPET"`, `"LAYOUT"`, `"WORKFLOW"`, `"WIDGET"`, `"JAVA_ACTION"`, `"JAVASCRIPT_ACTION"`, `"REST_OPERATION"`, `"REGULAR_EXPRESSION"`, `"ATTRIBUTE"`, `"ENUMERATION"`, `"ENUMERATION_VALUE"`, `"IMPORT_MAPPING"`, `"EXPORT_MAPPING"`, `"JSON_STRUCTURE"`, `"AGENT"`, `"AI_MODEL"`, `"KNOWLEDGE_BASE"`, `"CONSUMED_MCP_SERVICE"`, `"CONSTANT"`. `WIDGET`, `ATTRIBUTE`, `ENUMERATION`, `ENUMERATION_VALUE` and `JSON_STRUCTURE` are only ever targets; `SCHEDULED_EVENT`, `PROJECT_SETTINGS` and the service types only ever sources | | `target_id` | string | Target UUID | | `target_name` | string | `"Sales.Customer"`; three-part for an attribute or an enumeration value: `"Sales.Order.Total"`, `"Sales.OrderStatus.Open"` | -| `ref_kind` | string | How it references: `"call"`, `"create"`, `"retrieve"`, `"change"`, `"delete"`, `"commit"` (a commit action, or a create/change that commits — beside its `"create"`/`"change"` edge; a commit of a variable whose entity the flow cannot tell has no edge), `"show_page"`, `"snippet_call"` (a page, snippet or layout places a snippet), `"datasource"`, `"action"`, `"layout"`, `"parameter"` (a page, snippet or flow parameter's entity type), `"return"`, `"generalize"`, `"associate"`, `"home_page"`, `"login_page"`, `"menu_item"`, `"calculate"`, `"schedule"`, `"validate"`, `"settings"`, `"widget"`, `"sync"`, `"publish"` (a published REST operation, OData resource or action, or a service's authentication microflow), `"event"` (an entity event or a business event handler), `"member"` (binds/reads/writes an attribute, navigates an association, or reaches the entity an association step arrives at), `"xpath"` (an XPath constraint names it, or reaches the entity through an association), `"type"` (typed as an enumeration; or an entity named as a type: an action's result variable, a list a flow creates, an entity-type argument to a Java or JavaScript action), `"value"` (an expression names an enumeration value; an ENUMERATION target when the value named does not exist), `"mapping"` (an import/export mapping maps the entity, or a message definition exposes it), `"split"` (a decision branches on an object being the entity), `"mapping_call"` (a flow applies an import or export mapping: an import/export action, or a REST or web service call's request or response mapping), `"structure"` (a mapping is built on a JSON structure) — lower-case, unlike the types above. Attribute names used only through a variable in a free-text expression (`$Order/Total`) have no edge | +| `ref_kind` | string | How it references: `"call"`, `"create"`, `"retrieve"`, `"change"`, `"delete"`, `"commit"` (a commit action, or a create/change that commits — beside its `"create"`/`"change"` edge; a commit of a variable whose entity the flow cannot tell has no edge), `"show_page"`, `"snippet_call"` (a page, snippet or layout places a snippet), `"datasource"`, `"action"`, `"layout"`, `"parameter"` (a page, snippet or flow parameter's entity type), `"return"`, `"generalize"`, `"associate"`, `"home_page"`, `"login_page"`, `"menu_item"`, `"calculate"`, `"schedule"`, `"validate"`, `"settings"`, `"widget"`, `"sync"`, `"publish"` (a published REST operation, OData resource or action, a published MCP service's tool microflow, or a service's authentication microflow), `"event"` (an entity event or a business event handler), `"member"` (binds/reads/writes an attribute, navigates an association, or reaches the entity an association step arrives at), `"xpath"` (an XPath constraint names it, or reaches the entity through an association), `"type"` (typed as an enumeration; or an entity named as a type: an action's result variable, a list a flow creates, an entity-type argument to a Java or JavaScript action), `"value"` (an expression names an enumeration value; an ENUMERATION target when the value named does not exist), `"mapping"` (an import/export mapping maps the entity, or a message definition exposes it), `"split"` (a decision branches on an object being the entity), `"mapping_call"` (a flow applies an import or export mapping: an import/export action, or a REST or web service call's request or response mapping), `"structure"` (a mapping is built on a JSON structure), `"tool"` (an agent draws on a knowledge base or consumed MCP service; an agent's tool microflow, nanoflow or agent is a `"call"`, as is a microflow or agent document passed as a Java action parameter), `"model"` (an agent runs on an AI model), `"key"` (an AI model or knowledge base reads its key from a constant) — lower-case, unlike the types above. Attribute names used only through a variable in a free-text expression (`$Order/Total`) have no edge | | `module_name` | string | Source module | ### project_security diff --git a/CHANGELOG.md b/CHANGELOG.md index aa0b67480..ff01f0a39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,35 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +### Added + +- **Agent prompts in full-text search** (ako/mxcli#2075) — `CATALOG.STRINGS` indexed only an agent's documentation, so `search` could not find its system or user prompt, or the description the model reads for each tool and knowledge base: the most important text in an agentic app. Each is now a row with `ObjectType` `AGENT` and a `StringContext` naming it (`system_prompt`, `user_prompt`, `agent_description`, `tool_description`, `tool_display_title`, `tool_display_description`, `knowledge_base_description`). A published MCP service's tool descriptions and titles are rows of the service too (`mcp_tool_description`, `mcp_tool_title`). Catalog schema 43. +- **Agent details in the catalog: `AGENT_TOOLS`, `AGENT_KNOWLEDGE_BASES`, `AGENT_MCP_SERVICES`, `PUBLISHED_MCP_SERVICES`, and each agent's settings** (ako/mxcli#2075) — `CATALOG.AGENTS` and its siblings held a document's name only, so which agents can run a tool without the user's approval, which use a model, or which have streaming on took one `describe agent` per agent. `AGENTS` gains the usage type, model, selected model, tool choice, generation settings, streaming switches (`NULL` when not stored), context entity, variables and tool counts; `AI_MODELS` and `KNOWLEDGE_BASES` the provider and key constant; `CONSUMED_MCP_SERVICES` the protocol version, version and timeout. An agent's tools and knowledge base tools are rows of their own, and a published MCP service is a catalog object whose tool and authentication microflows are `publish` edges, so they are no longer listed in `GRAPH_DEAD_ASSETS`. Catalog schema 42. +- **A widget's caption and tooltip in lint rules, and its texts tied to it in every language** (ako/mxcli#1303, mendixlabs/mxcli#1312) — with two buttons on a page a rule could not tell which caption belonged to the one without a confirmation: `widgets()` had no caption, and the `strings()` caption row named the button's inner `Forms$ClientTemplate`, not the button. `widgets()` gains `caption` and `tooltip` in the default language, and `strings()` gains `widget_id`, so a widget's caption, tooltip and confirmation texts can be collected per language. Full catalog; the schema moves to version 40, so a cached catalog is rebuilt once. +- **`brain capture --from-json` and `brain promote ...`: many entries as one act** (ako/mxcli#2054) — accepting fifty requirements meant fifty captures and fifty promotes, and a bad anchor or a cap refusal at number 23 left half of them done. A batch capture validates every item before queueing any (a bad one is named by its index) and skips duplicates of committed, queued or earlier items with the existing id, so the same list twice is a no-op. A batch promote checks each shard's cap with all of its additions, writes the shards and then the queue once, and a re-run after an interruption completes it rather than refusing with "already carries entry". Shard files are now written atomically. +- **`jar_files()`: the `.jar` files in `userlib/` and `vendorlib/` in lint rules** (ako/mxcli#1304, mendixlabs/mxcli#1313) — `jar_dependencies()` lists the Maven dependencies module settings declare, not what is on disk, so a hand-added jar and the managed one in two versions went unseen. Each row has the folder, the file name, the library name and version split from it, and the modules whose `.RequiredLib` marker names the jar. Read from disk at each call; no catalog change. +- **`annotations()` and `CATALOG.ANNOTATIONS`: domain model annotations in lint rules and the catalog** (ako/mxcli#1300, mendixlabs/mxcli#1309) — a microflow's annotations reached a rule through `activities_for()`, a domain model's through nothing, so a "no TODO notes" rule passed a domain model full of them. Each row has the module, the full caption with its newlines, and the place on the canvas. The schema moves to version 39, so a cached catalog is rebuilt once. +- **An enumeration value's `id` in Starlark lint rules** (ako/mxcli#1301, mendixlabs/mxcli#1310) — `enumerations()` gave each value's caption in the default language only, and a rule could not tell which value a caption in another language belongs to. Each value now carries `id`, which is the `element_id` of its caption rows in `strings()`, so `getCaption(Module.Enum.Value)` can be resolved per language. +- **`folders()` and `CATALOG.FOLDERS`: every folder, empty ones included** (ako/mxcli#1436, mendixlabs/mxcli#1538) — a rule could only derive a folder from the `folder` of a document in it, so "every module has the folders Objects and UI" reported an empty folder as missing. Each row has the module, the path (spelled as a document's `folder`), the parent path, the depth, and the documents and folders directly in it. `CATALOG.AGENTS` also parses now: like `FOLDERS`, its name is a keyword, and the `select` returned nothing without an error; a test now parses every name `show catalog tables` lists. The schema moves to version 38, so a cached catalog is rebuilt once. + +### Changed + +- **`docker build`, `docker run` and `docker reload` check and build the project as stored** (ako/mxcli#1794) — they ran `mx update-widgets` on their temporary copy first, so a project whose stored widget definitions are stale built, while `docker check`, `run --local` and Studio Pro all reported the CE0463s: the deployment was not built from what is committed. A stale widget now fails the build, each CE0463 labelled as one `mxcli fix widgets` clears or one that persists, as `docker check` labels them. `--update-widgets` builds a widget-normalised copy as before, leaving the project as it is; `--no-update-widgets` skips the labelling re-check. + +### Fixed + +- **A consumed MCP service written by mxcli connects to its server** (ako/mxcli#2060) — it had no `endpoint` and no credentials microflow, so Agent Editor skipped it at startup and the agents using it got no tools, while `mx check` stayed green. `create`/`alter consumed mcp service` take `Endpoint: @Module.UrlConstant` (a String constant) and `AuthenticationMicroflow: Module.GetHeaders`, `describe` prints them, `create or modify` keeps stored ones, `check` warns on a `create` without an endpoint (MDL-AGENT03), and `check --references` refuses a non-String constant or a credentials microflow MCP Client cannot call (it must take no parameters and return a list of `System.HttpHeader`). The agents skill and syntax help show it (part of ako/mxcli#2067). +- **`create agent` writes the selected model, so Agent Editor imports the agent** (ako/mxcli#2031) — Agent Editor 2.2+ refused every mxcli-created agent at startup ("Model with name is referenced but could not be found"), and one refused agent stopped the import of all of them, while `mx check` and the build stayed green. `create`/`alter agent` take `SelectedModel: ''` (and an optional `SelectedModelName`, defaulting to the id), `describe` prints it, and `check` warns on a `create agent` without one (MDL-AGENT02). The skill, syntax help and examples now show it (part of ako/mxcli#2067). +- **`move scheduled event M.Nightly` moves the scheduled event when a microflow shares its name** (ako/mxcli#2072, mendixlabs/mxcli#1792) — the move looked the name up regardless of type, met the microflow first and refused with "M.Nightly is a microflow, not a scheduled event". It now picks the document of the type the statement names; a name no document of that type has is still refused, naming what it is. +- **Lint ARCH001 no longer flags a data view on another module's entity that feeds that module's own snippet or page** (ako/mxcli#2069) — a page hosting Conversational UI's chat needs a data view on `ConversationalUI.ChatContext`, because the chat snippet takes it as its parameter, and every such page was told to use a view entity or microflow instead. A cross-module entity is now exempt when the page calls a snippet, or links to a page, of that entity's module that takes the entity as a parameter. Any other cross-module data view is still reported. +- **`check` resolves a document by the name an earlier `rename` or `move` in the same script gives it** (ako/mxcli#2070, mendixlabs/mxcli#1790) — `rename page M.Old_Page to New_Page;` followed by `show page M.New_Page` was refused with "page not found: M.New_Page", although exec ran it; the script had to be split in two. A rename or a move to another module now counts as providing that document to every reference check, as a create does. +- **A Data grid 2 or Gallery empty placeholder is documented, and refused where it cannot build** (ako/mxcli#1862, mendixlabs/mxcli#1704) — `emptyplaceholder { … }` was accepted by `check` and `exec`, and the page then failed `mx check` with CE0463 on the grid (11.14): the slot is hidden while `showEmptyPlaceholder` is `none`, its default. MDL-WIDGET10 now counts a filled slot as set, so `check` refuses widgets in a hidden slot and names the property; `mxcli syntax page.widgets` documents the working form (`showEmptyPlaceholder: 'custom'` with an `emptyplaceholder` block) and its `alter page` equivalent. It creates, alters and round-trips through `describe` unchanged. +- **A page whose snippet call takes a parameter from an enclosing data container round-trips** (ako/mxcli#2055) — `describe` → `exec` of TestApp's `WorkflowCommons.ManageTaskAssignments` was refused ("snippet … requires parameter $Account"): Studio Pro stores the call with no parameter mappings and `mx check` accepts it, the parameter coming from a data view two containers out, while the builder looked only at the nearest container. It now accepts an omitted parameter that any enclosing data container supplies (the entity, a specialisation or a generalisation of it). The same page then built with CE1613 — `describe` printed a single association hop without its stored specialised destination (`TaskAssignmentHelper_User` arriving at `Administration.Account`, not `System.User`), so labels bound to `FullName` lost their attribute; such a hop now prints `Assoc/Destination`. The page now reports `Unchanged page` and `mx check` 0 errors (11.14). +- **MOVE between modules rewrites access-rule XPaths and maps entity and nanoflow roles** (ako/mxcli#1836, mendixlabs/mxcli#1687) — splitting a module left an entity access rule's XPath naming the old module (`[Src.Order_Customer/Src.Customer/…]`: on 11.12.6, `mx check` reports CE1613 for the association and the entity), its access rules on the old module's roles, and a moved nanoflow's grants unmapped where a microflow's are mapped. The reference sweep now rewrites path steps — in XPaths and in expressions such as `$O/Src.Assoc/Name` — when the moved element is an entity or an association; an entity's access rules and a nanoflow's grants move to the target's same-named roles, and a role the target lacks is kept with a warning. +- **A describe → create or replace round trip of an unchanged page no longer rewrites it** (ako/mxcli#1897) — on TestApp 15 of 68 pages reported `Replaced page` with nothing changed, for three things MDL cannot state: a tab container's default tab (stored null, written as the first tab — a stored second-tab default was also reset on every pass), an expression's trailing newline, and an empty screen-reader label Studio Pro stores as a client template. The write path now carries each from the stored element of the same `$ID`; all 15 report `Unchanged page`, and a real change still lands. +- **Lint CONV013 no longer flags error handling that is the only valid choice** (ako/mxcli#1838, mendixlabs/mxcli#1689) — it reported Rollback on a call inside a loop, where Mendix allows only Rollback (CE0644, and `check` warns MDL006 on anything else), and on a call to a Java action that exists to throw, such as `CommunityCommons.ThrowException`, which is how a flow rolls itself back. Neither could be fixed, so every app needed a baseline entry. Both are no longer reported; other external calls are reported as before. +- **`init --tool opencode` no longer puts every skill into every prompt** (ako/mxcli#1822, mendixlabs/mxcli#1683) — the generated `opencode.json` listed both skill trees as `instructions`, which OpenCode inlines whole: 150 files, 2.7 MB, about 73% of a 1M context window before the user typed anything. Its instructions are `AGENTS.md` alone; OpenCode discovers `.opencode/skills/` itself and loads a skill when its description matches. Those copies now keep the skill's own "Use when…" description, in one frontmatter block, instead of a second block described by the skill's title. + ## [0.31.0] - 2026-11-09 Headline: **Publish your app's microflows as an MCP server, and keep secrets off the command line.** `create published mcp service` writes the MCP Server document Agent Editor starts at runtime, and agents gain tool approval, display text and streaming. The secret `run` flags and `--constant` take `@env:VAR` and `@file:path`, and the `brain` can be started before the project and edited in place. Alongside them come a long run of fixes that make `check --references` catch more of what the build would refuse, and `mxcli test --local` runs that start clean every time. @@ -39,6 +68,13 @@ Headline: **Publish your app's microflows as an MCP server, and keep secrets off ### Fixed +- **Agents and the microflows a Java action runs are references now** (ako/mxcli#2074) — an agent-editor document produced no `refs` row, and a microflow passed as a microflow-typed Java action parameter was no reference, so an agent's tool microflow, a chat context's `ActionMicroflow` and the `Microflow` of `MCPServer.AddPrompt` showed "(no callers found)" and were listed in `GRAPH_DEAD_ASSETS`, though deleting one breaks the app at runtime. An agent now has a `call` edge to each tool microflow, nanoflow or agent, a `tool` edge to its knowledge bases and MCP services, a `model` edge to its model and a `type` edge to its context entity; a model or knowledge base has a `key` edge to its key constant; and a microflow or agent document passed as a Java action parameter is a `call`. `show callers` counts `tool` and `model`. On the Agent Builder starter app, 30 of 238 dead assets turn out to be in use. +- **A Data grid 2 or Gallery empty placeholder is documented, and refused where it cannot build** (ako/mxcli#1862, mendixlabs/mxcli#1704) — `emptyplaceholder { … }` was accepted by `check` and `exec`, and the page then failed `mx check` with CE0463 on the grid (11.14): the slot is hidden while `showEmptyPlaceholder` is `none`, its default. MDL-WIDGET10 now counts a filled slot as set, so `check` refuses widgets in a hidden slot and names the property; `mxcli syntax page.widgets` documents the working form (`showEmptyPlaceholder: 'custom'` with an `emptyplaceholder` block) and its `alter page` equivalent. It creates, alters and round-trips through `describe` unchanged. +- **A page whose snippet call takes a parameter from an enclosing data container round-trips** (ako/mxcli#2055) — `describe` → `exec` of TestApp's `WorkflowCommons.ManageTaskAssignments` was refused ("snippet … requires parameter $Account"): Studio Pro stores the call with no parameter mappings and `mx check` accepts it, the parameter coming from a data view two containers out, while the builder looked only at the nearest container. It now accepts an omitted parameter that any enclosing data container supplies (the entity, a specialisation or a generalisation of it). The same page then built with CE1613 — `describe` printed a single association hop without its stored specialised destination (`TaskAssignmentHelper_User` arriving at `Administration.Account`, not `System.User`), so labels bound to `FullName` lost their attribute; such a hop now prints `Assoc/Destination`. The page now reports `Unchanged page` and `mx check` 0 errors (11.14). +- **MOVE between modules rewrites access-rule XPaths and maps entity and nanoflow roles** (ako/mxcli#1836, mendixlabs/mxcli#1687) — splitting a module left an entity access rule's XPath naming the old module (`[Src.Order_Customer/Src.Customer/…]`: on 11.12.6, `mx check` reports CE1613 for the association and the entity), its access rules on the old module's roles, and a moved nanoflow's grants unmapped where a microflow's are mapped. The reference sweep now rewrites path steps — in XPaths and in expressions such as `$O/Src.Assoc/Name` — when the moved element is an entity or an association; an entity's access rules and a nanoflow's grants move to the target's same-named roles, and a role the target lacks is kept with a warning. +- **A describe → create or replace round trip of an unchanged page no longer rewrites it** (ako/mxcli#1897) — on TestApp 15 of 68 pages reported `Replaced page` with nothing changed, for three things MDL cannot state: a tab container's default tab (stored null, written as the first tab — a stored second-tab default was also reset on every pass), an expression's trailing newline, and an empty screen-reader label Studio Pro stores as a client template. The write path now carries each from the stored element of the same `$ID`; all 15 report `Unchanged page`, and a real change still lands. +- **Lint CONV013 no longer flags error handling that is the only valid choice** (ako/mxcli#1838, mendixlabs/mxcli#1689) — it reported Rollback on a call inside a loop, where Mendix allows only Rollback (CE0644, and `check` warns MDL006 on anything else), and on a call to a Java action that exists to throw, such as `CommunityCommons.ThrowException`, which is how a flow rolls itself back. Neither could be fixed, so every app needed a baseline entry. Both are no longer reported; other external calls are reported as before. +- **`init --tool opencode` no longer puts every skill into every prompt** (ako/mxcli#1822, mendixlabs/mxcli#1683) — the generated `opencode.json` listed both skill trees as `instructions`, which OpenCode inlines whole: 150 files, 2.7 MB, about 73% of a 1M context window before the user typed anything. Its instructions are `AGENTS.md` alone; OpenCode discovers `.opencode/skills/` itself and loads a skill when its description matches. Those copies now keep the skill's own "Use when…" description, in one frontmatter block, instead of a second block described by the skill's title. - **Building a flow no longer reads every microflow of the project once per called flow** (ako/mxcli#2043) — resolving a `call microflow` target looked its name up by reading and decoding every unit of the type, so one large flow in a large project took minutes, and outlasted the per-statement limit. A lookup now records where each name is and later reads that one unit, re-checking its name, type and module so a unit renamed or moved outside mxcli is not returned; a miss scans as before. On TestApp a flow calling 44 microflows builds in 0.29 s instead of 0.6 s; the saving grows with the number of flows in the project. - **A statement that times out or panics no longer crashes `exec` past its rollback** (ako/mxcli#2043, mendixlabs/mxcli#1780) — a statement that outlasted the per-statement limit kept running after `Execute` returned the timeout; `exec` then closed the project to roll back, the statement read through the closed reader, and the nil-pointer panic in its goroutine ended the process before the rollback and before the timeout was printed. The drop and every earlier statement stayed written. `Close` now waits (up to 2 minutes) for such a statement to stop, and a panic in any statement is that statement's error, so the script is rolled back as for any other failure. - **References to entities named as a type or reached through an association** (ako/mxcli#1829) — a flow whose action result, created list or Java/JavaScript entity-type argument names an entity, a snippet's parameters, and the entity an association path arrives at made no `refs` row, so `show references`, `impact` and the unused-entity check missed them (five entities in use on one project read as unreferenced), and an incremental build left a flow stale when an enumeration value it named was added. They are now `type`, `parameter`, `member` and `xpath` edges to the entity, and `value` to the enumeration when the value does not exist. Annotation and text prose and module-role names no longer produce member edges. Full catalog; the schema moves to version 37, so a cached catalog is rebuilt once. diff --git a/cmd/mxcli/brain/batch.go b/cmd/mxcli/brain/batch.go new file mode 100644 index 000000000..2b0683977 --- /dev/null +++ b/cmd/mxcli/brain/batch.go @@ -0,0 +1,386 @@ +// SPDX-License-Identifier: Apache-2.0 + +// batch.go - capturing and promoting many entries as one act (ako/mxcli#2054). +// +// The point is the transaction boundary, not speed. Fifty single captures or +// promotes are fifty places to stop half way — a cap refusal, a bad anchor or a +// killed process at entry 23 leaves 22 done, 28 not, and nobody sure which. So +// a batch validates every item before it writes anything, and a write that +// cannot be atomic (a promote touches several shards and the queue) is ordered +// so that a crash leaves a state a plain re-run completes. +package brain + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "time" +) + +// BatchItem is one entry of `brain capture --from-json`. The fields are the +// input half of what `staged --json` and `list --json` print, so a caller can +// feed a list straight back in. +type BatchItem struct { + // Text is the single-argument form of capture: first line title, the rest + // body. Give it or Title (with an optional Body), not both. + Text string `json:"text,omitempty"` + Title string `json:"title,omitempty"` + Body string `json:"body,omitempty"` + Anchors []string `json:"anchors,omitempty"` + Slice string `json:"slice,omitempty"` + Open bool `json:"open,omitempty"` + Source string `json:"source,omitempty"` +} + +// ParseBatch reads a JSON array of items. Unknown fields are refused: a +// misspelt "anchor" would otherwise capture an unanchored entry in silence, +// which is the kind of half-success a batch exists to rule out. +func ParseBatch(r io.Reader) ([]BatchItem, error) { + raw, err := io.ReadAll(r) + if err != nil { + return nil, err + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + var items []BatchItem + if err := dec.Decode(&items); err != nil { + return nil, fmt.Errorf("--from-json expects a JSON array of entries: %w", err) + } + if dec.More() { + return nil, fmt.Errorf("--from-json expects one JSON array; found more after it") + } + return items, nil +} + +// BuildBatch turns items into entries, validating every one before the caller +// writes anything. The first invalid item fails the whole batch, named by its +// index in the array, so nothing is queued from a list that is partly wrong. +// +// normalizeSource resolves an item's source the way --source does on the +// command line; nil leaves sources as given. +func BuildBatch(items []BatchItem, now time.Time, normalizeSource func(string) (string, error)) ([]Entry, error) { + out := make([]Entry, 0, len(items)) + for i, it := range items { + e, err := it.entry(now) + if err == nil && it.Source != "" && normalizeSource != nil { + e.Source, err = normalizeSource(it.Source) + } else if err == nil { + e.Source = it.Source + } + if err != nil { + return nil, fmt.Errorf("items[%d]: %w; nothing was queued", i, err) + } + out = append(out, e) + } + return out, nil +} + +func (it BatchItem) entry(now time.Time) (Entry, error) { + text := it.Text + switch { + case it.Text != "" && (it.Title != "" || it.Body != ""): + return Entry{}, fmt.Errorf("give text, or title and body, not both") + case it.Text == "" && it.Title == "" && it.Body != "": + return Entry{}, fmt.Errorf("a body needs a title") + case it.Text == "": + text = it.Title + if it.Body != "" { + text += "\n" + it.Body + } + } + switch { + case it.Open: + return NewQuestion(text, it.Anchors, it.Slice, now) + case it.Slice != "": + return NewRequirement(text, it.Anchors, it.Slice, now) + default: + return NewEntry(text, it.Anchors, now) + } +} + +// Capture statuses, shared with the single form's --json report. +const ( + StatusQueued = "queued" + StatusAlreadyQueued = "already_queued" + StatusAlreadyCommitted = "already_committed" +) + +// CaptureResult is one item's outcome. A duplicate carries the id of the entry +// it duplicates, so a caller can promote by it without a second lookup. +type CaptureResult struct { + Entry Entry + Added bool + Status string +} + +// CaptureBatch queues entries as one act. +// +// Duplicates are not errors: an entry matching something committed, something +// queued, or an earlier item of this batch is reported with the twin's id and +// skipped, so accepting the same list twice is a no-op rather than a failure. +// Ids are assigned against the stored ones AND those given earlier in the +// batch, or two items colliding only with each other would share a handle. +// +// The queue lock is held from the duplicate check to the single append, so a +// promote cannot rebuild the file between them. +func CaptureBatch(s *Store, q *Queue, entries []Entry) ([]CaptureResult, error) { + release, err := acquireQueueLock(q.Path) + if err != nil { + return nil, err + } + defer release() + + staged, err := q.Load() + if err != nil { + return nil, err + } + taken, err := TakenIDs(s, q) + if err != nil { + return nil, err + } + committed := map[string]Entry{} + if s.Exists() { + shards, err := s.ListShards() + if err != nil { + return nil, err + } + for _, sh := range shards { + es, _, err := s.LoadShard(sh) + if err != nil { + return nil, err + } + for _, e := range es { + if _, ok := committed[e.ContentKey()]; !ok { + committed[e.ContentKey()] = e + } + } + } + } + queued := map[string]Entry{} + for _, e := range staged { + if _, ok := queued[e.ContentKey()]; !ok { + queued[e.ContentKey()] = e + } + } + + results := make([]CaptureResult, 0, len(entries)) + var fresh []Entry + for _, e := range entries { + key := e.ContentKey() + if dup, ok := committed[key]; ok { + results = append(results, CaptureResult{Entry: dup, Status: StatusAlreadyCommitted}) + continue + } + // An earlier item of this batch is in `queued` by now, so an in-batch + // twin reads as already queued — which, once the append lands, it is. + if dup, ok := queued[key]; ok { + results = append(results, CaptureResult{Entry: dup, Status: StatusAlreadyQueued}) + continue + } + e = AssignFreeID(e, taken) + taken[e.ID] = true + queued[key] = e + fresh = append(fresh, e) + results = append(results, CaptureResult{Entry: e, Added: true, Status: StatusQueued}) + } + if len(fresh) == 0 { + return results, nil + } + if err := q.appendLocked(fresh); err != nil { + return nil, err + } + return results, nil +} + +// appendLocked writes entries to the end of the queue in one write. The +// caller holds the queue lock. +func (q *Queue) appendLocked(entries []Entry) error { + var buf bytes.Buffer + for _, e := range entries { + b, err := json.Marshal(e) + if err != nil { + return err + } + buf.Write(b) + buf.WriteByte('\n') + } + if err := os.MkdirAll(filepath.Dir(q.Path), 0o755); err != nil { + return err + } + f, err := os.OpenFile(q.Path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return err + } + if _, err := f.Write(buf.Bytes()); err != nil { + f.Close() + return err + } + return f.Close() +} + +// Promote statuses. +const ( + StatusPromoted = "promoted" + // StatusAlreadyPromoted is an id whose entry is already in its shard with + // the same content: what a promote interrupted after its shard write and + // before its queue rewrite leaves behind, or a batch run twice. + StatusAlreadyPromoted = "already_promoted" +) + +// PromoteResult is one id's outcome. +type PromoteResult struct { + Entry Entry + Shard string + Status string +} + +// afterShardWrites runs between the shard writes and the queue rewrite. It is +// a test seam for the one interruption a multi-file write cannot rule out. +var afterShardWrites func() error + +// PromoteBatch commits staged entries as one act. to, when set, is the shard +// for every id; otherwise each entry goes where its anchors route it. +// +// Everything is validated before anything is written: every id must resolve, +// and each target shard's size is computed with ALL of this batch's additions +// to it — checked entry by entry, each passes and the shard still overruns. +// +// A multi-file write cannot be atomic, so it is ordered instead: every shard +// first (each atomically), then one queue rewrite. A crash between the two +// leaves entries committed and still queued — visible, and completed by +// re-running the same promote, which counts an id already in its shard with +// the same content as promoted. The reverse order could lose an entry from +// both places; this one never can. +func PromoteBatch(s *Store, q *Queue, ids []string, to string) ([]PromoteResult, error) { + if !s.Exists() { + return nil, fmt.Errorf("no store yet — run 'mxcli brain init' first") + } + release, err := acquireQueueLock(q.Path) + if err != nil { + return nil, err + } + defer release() + + staged, err := q.Load() + if err != nil { + return nil, err + } + byID := make(map[string]Entry, len(staged)) + for _, e := range staged { + byID[e.ID] = e + } + + type shardPlan struct { + entries []Entry + changed bool + } + plans := map[string]*shardPlan{} + var order []string + load := func(shard string) (*shardPlan, error) { + if p, ok := plans[shard]; ok { + return p, nil + } + es, _, err := s.LoadShard(shard) + if err != nil { + return nil, err + } + p := &shardPlan{entries: es} + plans[shard] = p + order = append(order, shard) + return p, nil + } + + seen := map[string]bool{} + results := make([]PromoteResult, 0, len(ids)) + drop := map[string]bool{} + for _, id := range ids { + if seen[id] { + continue + } + seen[id] = true + e, queuedHere := byID[id] + if !queuedHere { + // Not queued, but perhaps already promoted: a batch re-run after it + // completed must read as done, not fail on its first id. + if found, shard, err := s.Find(id); err != nil { + return nil, err + } else if found.ID != "" { + results = append(results, PromoteResult{Entry: found, Shard: shard, Status: StatusAlreadyPromoted}) + continue + } + return nil, fmt.Errorf("no staged entry with id %s; nothing was promoted", id) + } + shard := e.Shard() + if to != "" { + shard = to + } + p, err := load(shard) + if err != nil { + return nil, err + } + status := StatusPromoted + for _, existing := range p.entries { + if existing.ID == e.ID { + if existing.ContentKey() != e.ContentKey() { + return nil, fmt.Errorf("%s already carries entry %s (%q); nothing was promoted", shard, e.ID, existing.Title) + } + status = StatusAlreadyPromoted + break + } + if existing.ContentKey() == e.ContentKey() { + return nil, fmt.Errorf("%s already says this, as entry %s (%q); nothing was promoted", shard, existing.ID, existing.Title) + } + } + if status == StatusPromoted { + p.entries = append(p.entries, e) + p.changed = true + } + drop[e.ID] = true + results = append(results, PromoteResult{Entry: e, Shard: shard, Status: status}) + } + + // The cap, per shard, with the whole batch in it. Counted with frontmatter, + // as the single promote counts it and `brain show` measures it. + rendered := map[string]string{} + for _, shard := range order { + p := plans[shard] + if !p.changed { + continue + } + r := RenderShardWithFrontmatter(shard, p.entries, s.Frontmatter(shard)) + if lines, limit := CountLines(r), CapFor(shard); lines > limit { + return nil, &ErrCapExceeded{Shard: shard, Would: lines, Cap: limit} + } + rendered[shard] = r + } + + for _, shard := range order { + r, ok := rendered[shard] + if !ok { + continue + } + if err := writeFileAtomic(s.ShardPath(shard), []byte(r)); err != nil { + return nil, err + } + } + if afterShardWrites != nil { + if err := afterShardWrites(); err != nil { + return nil, err + } + } + if len(drop) > 0 { + kept := make([]Entry, 0, len(staged)) + for _, e := range staged { + if !drop[e.ID] { + kept = append(kept, e) + } + } + if err := q.write(kept); err != nil { + return nil, err + } + } + return results, nil +} diff --git a/cmd/mxcli/brain/batch_test.go b/cmd/mxcli/brain/batch_test.go new file mode 100644 index 000000000..9b86ceddf --- /dev/null +++ b/cmd/mxcli/brain/batch_test.go @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: Apache-2.0 + +package brain + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +// ako/mxcli#2054: capture and promote as one act each, so accepting many +// entries cannot stop half way with nobody sure which half. + +func batchStore(t *testing.T) (*Store, *Queue) { + t.Helper() + dir := t.TempDir() + s := NewStore(dir) + if _, err := s.Init(); err != nil { + t.Fatal(err) + } + return s, NewQueue(dir) +} + +func items(texts ...string) []BatchItem { + out := make([]BatchItem, 0, len(texts)) + for _, t := range texts { + out = append(out, BatchItem{Text: t, Anchors: []string{"@Sales.Order"}}) + } + return out +} + +func queueLen(t *testing.T, q *Queue) int { + t.Helper() + es, err := q.Load() + if err != nil { + t.Fatal(err) + } + return len(es) +} + +func TestBuildBatchRefusesTheWholeBatchNamingTheBadItem(t *testing.T) { + bad := items("One", "Two", "Three") + bad[1].Anchors = []string{"not an anchor"} + if _, err := BuildBatch(bad, day, nil); err == nil || !strings.Contains(err.Error(), "items[1]") { + t.Fatalf("err = %v, want a refusal naming items[1]", err) + } + + // Control: the same batch without the bad item builds, and queues N-1. + s, q := batchStore(t) + good := append(bad[:1:1], bad[2]) + es, err := BuildBatch(good, day, nil) + if err != nil { + t.Fatalf("control batch refused: %v", err) + } + if _, err := CaptureBatch(s, q, es); err != nil { + t.Fatal(err) + } + if n := queueLen(t, q); n != 2 { + t.Errorf("queued %d, want 2", n) + } +} + +func TestBuildBatchRefusesAmbiguousOrEmptyItems(t *testing.T) { + for name, it := range map[string]BatchItem{ + "text and title": {Text: "A", Title: "B"}, + "body only": {Body: "B"}, + "nothing": {}, + "bare module requirement": {Title: "Needs a module element", Slice: "01-x", + Anchors: []string{"@Sales"}}, + } { + if _, err := BuildBatch([]BatchItem{it}, day, nil); err == nil { + t.Errorf("%s: accepted, want a refusal", name) + } + } + // title + body is the two-field form of text. + es, err := BuildBatch([]BatchItem{{Title: "Head", Body: "Prose"}}, day, nil) + if err != nil || es[0].Title != "Head" || es[0].Body != "Prose" { + t.Fatalf("title+body = %+v, %v", es, err) + } +} + +func TestParseBatchRefusesUnknownFields(t *testing.T) { + // "anchor" for "anchors" would otherwise queue an unanchored entry. + if _, err := ParseBatch(strings.NewReader(`[{"text":"A","anchor":["@Sales.Order"]}]`)); err == nil { + t.Fatal("a misspelt field was accepted") + } + got, err := ParseBatch(strings.NewReader(`[{"text":"A","anchors":["@Sales.Order"]}]`)) + if err != nil || len(got) != 1 || got[0].Anchors[0] != "@Sales.Order" { + t.Fatalf("control: %+v, %v", got, err) + } +} + +func TestCaptureBatchSkipsDuplicatesAndIsIdempotent(t *testing.T) { + s, q := batchStore(t) + committed := mustEntry(t, "Already decided", "@Sales.Order") + if err := s.Promote(committed, committed.Shard()); err != nil { + t.Fatal(err) + } + + es, err := BuildBatch(items("Fresh one", "Already decided", "Fresh two", "Fresh one"), day, nil) + if err != nil { + t.Fatal(err) + } + res, err := CaptureBatch(s, q, es) + if err != nil { + t.Fatal(err) + } + want := []string{StatusQueued, StatusAlreadyCommitted, StatusQueued, StatusAlreadyQueued} + for i, r := range res { + if r.Status != want[i] || r.Added != (want[i] == StatusQueued) { + t.Errorf("item %d: status %s added %v, want %s", i, r.Status, r.Added, want[i]) + } + } + if res[1].Entry.ID != committed.ID { + t.Errorf("committed duplicate reported as %s, want the committed id %s", res[1].Entry.ID, committed.ID) + } + if res[3].Entry.ID != res[0].Entry.ID { + t.Errorf("in-batch duplicate reported as %s, want its twin's id %s", res[3].Entry.ID, res[0].Entry.ID) + } + if n := queueLen(t, q); n != 2 { + t.Fatalf("queued %d, want 2", n) + } + + // The same batch again changes nothing. + before, _ := os.ReadFile(q.Path) + again, err := CaptureBatch(s, q, es) + if err != nil { + t.Fatal(err) + } + for i, r := range again { + if r.Added { + t.Errorf("re-run item %d added again", i) + } + } + after, _ := os.ReadFile(q.Path) + if string(before) != string(after) { + t.Errorf("re-running the batch changed the queue:\n%s\n---\n%s", before, after) + } +} + +// Two items whose ids collide only with each other must get different handles: +// the second has to see the id the first was given in this batch. +func TestCaptureBatchAssignsIDsAgainstEarlierItems(t *testing.T) { + s, q := batchStore(t) + a, b := mustEntry(t, "Alpha", "@Sales.Order"), mustEntry(t, "Beta", "@Sales.Order") + b.ID = a.ID // a collision nothing stored shares + res, err := CaptureBatch(s, q, []Entry{a, b}) + if err != nil { + t.Fatal(err) + } + if !res[0].Added || !res[1].Added { + t.Fatalf("both are different facts and should be queued: %+v", res) + } + if res[0].Entry.ID == res[1].Entry.ID { + t.Fatalf("both items got id %s", res[0].Entry.ID) + } +} + +func stage(t *testing.T, s *Store, q *Queue, texts ...string) []string { + t.Helper() + es, err := BuildBatch(items(texts...), day, nil) + if err != nil { + t.Fatal(err) + } + res, err := CaptureBatch(s, q, es) + if err != nil { + t.Fatal(err) + } + ids := make([]string, 0, len(res)) + for _, r := range res { + ids = append(ids, r.Entry.ID) + } + return ids +} + +// The cap is checked with every addition of the batch in the shard. Checked one +// by one, each addition fits and the shard still overruns. +func TestPromoteBatchChecksTheCapWithTheWholeBatch(t *testing.T) { + s, q := batchStore(t) + const shard = "Sales" + // Find how many fillers leave room for exactly one more entry. + var fill []Entry + for i := 0; ; i++ { + fill = append(fill, mustEntry(t, fmt.Sprintf("Filler %03d", i), "@Sales.Order")) + next := append(append([]Entry{}, fill...), mustEntry(t, "Probe one", "@Sales.Order"), mustEntry(t, "Probe two", "@Sales.Order")) + oneMore := next[:len(next)-1] + if CountLines(RenderShard(shard, oneMore)) <= CapFor(shard) && + CountLines(RenderShard(shard, next)) > CapFor(shard) { + break + } + if i > 1000 { + t.Fatal("never found the boundary") + } + } + if err := s.SaveShard(shard, fill); err != nil { + t.Fatal(err) + } + ids := stage(t, s, q, "Probe one", "Probe two") + shardBefore, _ := os.ReadFile(s.ShardPath(shard)) + queueBefore, _ := os.ReadFile(q.Path) + + _, err := PromoteBatch(s, q, ids, "") + var capErr *ErrCapExceeded + if !errors.As(err, &capErr) || capErr.Shard != shard { + t.Fatalf("err = %v, want the cap refused for %s", err, shard) + } + shardAfter, _ := os.ReadFile(s.ShardPath(shard)) + queueAfter, _ := os.ReadFile(q.Path) + if string(shardBefore) != string(shardAfter) || string(queueBefore) != string(queueAfter) { + t.Fatal("a refused batch changed the shard or the queue") + } + + // Control: either one alone fits. + res, err := PromoteBatch(s, q, ids[:1], "") + if err != nil || len(res) != 1 || res[0].Status != StatusPromoted { + t.Fatalf("a batch that fits was refused: %+v, %v", res, err) + } +} + +func TestPromoteBatchUnknownIDPromotesNothing(t *testing.T) { + s, q := batchStore(t) + ids := stage(t, s, q, "One", "Two") + if _, err := PromoteBatch(s, q, append(ids, "ffffff"), ""); err == nil { + t.Fatal("an unknown id was accepted") + } + if n := queueLen(t, q); n != 2 { + t.Errorf("queue has %d, want both still staged", n) + } + if es, _, _ := s.LoadShard("Sales"); len(es) != 0 { + t.Errorf("shard has %d entries, want none", len(es)) + } +} + +// The seam a multi-file write cannot close: shards written, queue not. A re-run +// of the same promote completes it, each entry committed exactly once. +func TestPromoteBatchRetryAfterInterruptedQueueRewrite(t *testing.T) { + s, q := batchStore(t) + ids := stage(t, s, q, "One", "Two", "Three") + + afterShardWrites = func() error { return errors.New("killed") } + _, err := PromoteBatch(s, q, ids, "") + afterShardWrites = nil + if err == nil { + t.Fatal("the seam did not fire") + } + if es, _, _ := s.LoadShard("Sales"); len(es) != 3 { + t.Fatalf("after the interruption the shard has %d entries, want 3", len(es)) + } + if n := queueLen(t, q); n != 3 { + t.Fatalf("after the interruption the queue has %d, want 3 (committed and still queued)", n) + } + + res, err := PromoteBatch(s, q, ids, "") + if err != nil { + t.Fatalf("re-run refused: %v", err) + } + for _, r := range res { + if r.Status != StatusAlreadyPromoted { + t.Errorf("%s: status %s, want %s", r.Entry.ID, r.Status, StatusAlreadyPromoted) + } + } + es, _, _ := s.LoadShard("Sales") + if len(es) != 3 { + t.Errorf("shard has %d entries after the retry, want each exactly once (3)", len(es)) + } + if n := queueLen(t, q); n != 0 { + t.Errorf("queue has %d after the retry, want 0", n) + } + + // And once more, with the queue already empty: still done, not an error. + if _, err := PromoteBatch(s, q, ids, ""); err != nil { + t.Errorf("a completed batch re-run failed: %v", err) + } +} + +func TestPromoteBatchToAppliesToEveryID(t *testing.T) { + s, q := batchStore(t) + ids := stage(t, s, q, "One", "Two") + res, err := PromoteBatch(s, q, ids, ProjectShard) + if err != nil { + t.Fatal(err) + } + for _, r := range res { + if r.Shard != ProjectShard { + t.Errorf("%s went to %s, want %s", r.Entry.ID, r.Shard, ProjectShard) + } + } +} + +// Shards are written through writeFileAtomic, which leaves no temporary file +// behind and replaces rather than truncates. +func TestShardWritesLeaveNoTemporaries(t *testing.T) { + s, q := batchStore(t) + ids := stage(t, s, q, "One", "Two") + if _, err := PromoteBatch(s, q, ids, ""); err != nil { + t.Fatal(err) + } + matches, _ := filepath.Glob(filepath.Join(filepath.Dir(s.ShardPath("Sales")), ".*tmp-*")) + if len(matches) != 0 { + t.Errorf("temporary files left behind: %v", matches) + } +} diff --git a/cmd/mxcli/brain/store.go b/cmd/mxcli/brain/store.go index 468038556..71bd0107d 100644 --- a/cmd/mxcli/brain/store.go +++ b/cmd/mxcli/brain/store.go @@ -127,10 +127,9 @@ func (s *Store) saveShardWith(shard string, entries []Entry, frontmatter string) } return err } - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - return err - } - return os.WriteFile(path, []byte(RenderShardWithFrontmatter(shard, entries, frontmatter)), 0644) + // Atomically: a crash mid-write must not leave a half-written shard, which + // would read back as a shorter one and silently lose the tail (#2054). + return writeFileAtomic(path, []byte(RenderShardWithFrontmatter(shard, entries, frontmatter))) } // Frontmatter returns the YAML block at the top of a shard, or "" if the shard diff --git a/cmd/mxcli/cmd_brain.go b/cmd/mxcli/cmd_brain.go index c87373443..c94ba3044 100644 --- a/cmd/mxcli/cmd_brain.go +++ b/cmd/mxcli/cmd_brain.go @@ -6,6 +6,7 @@ package main import ( "encoding/json" "fmt" + "io" "os" osexec "os/exec" "path/filepath" @@ -94,11 +95,34 @@ var brainInitCmd = &cobra.Command{ } var brainCaptureCmd = &cobra.Command{ - Use: "capture ", + Use: "capture | --from-json ", Short: "Queue something worth remembering (does not commit it)", - Args: cobra.ExactArgs(1), + Long: `Queue an entry for a person to review and promote. + +--from-json queues a JSON array of entries in one act, each +{"text" | "title"+"body", "anchors": [...], "slice"?, "open"?, "source"?}. +Every item is validated before anything is queued, so one bad item fails the +batch with its index named. A duplicate of something committed, queued or +earlier in the batch is skipped and reported with the existing id, so the same +list captured twice is a no-op.`, + Args: cobra.MaximumNArgs(1), Run: func(cmd *cobra.Command, args []string) { dir := brainProjectDir(cmd) + if from, _ := cmd.Flags().GetString("from-json"); from != "" { + if len(args) > 0 { + brainFatal(fmt.Errorf("give the entry as text or --from-json, not both")) + } + for _, f := range []string{"anchor", "slice", "open", "source"} { + if cmd.Flags().Changed(f) { + brainFatal(fmt.Errorf("--%s does not combine with --from-json; set it per item in the JSON", f)) + } + } + runBrainCaptureBatch(dir, from) + return + } + if len(args) != 1 { + brainFatal(fmt.Errorf("capture needs the entry's text, or --from-json ")) + } anchors, _ := cmd.Flags().GetStringSlice("anchor") slice, _ := cmd.Flags().GetString("slice") // --slice is the only signal needed: a requirement is a requirement @@ -252,40 +276,115 @@ append-only, so its own order is the honest timeline.`, } var brainPromoteCmd = &cobra.Command{ - Use: "promote ", - Short: "Write a staged entry into its shard (the human step)", - Args: cobra.ExactArgs(1), + Use: "promote ...", + Short: "Write staged entries into their shards (the human step)", + Long: `Commit staged entries into the shards their anchors route them to. + +Several ids are promoted as one act: every id is resolved and every target +shard's cap is checked with all of the batch's additions before anything is +written, so a batch either lands whole or not at all. --to applies to every id. + +If a promote is interrupted after the shards are written and before the queue +is, the entries are both committed and still queued; re-running the same +command completes it. An id already in its shard with the same content counts +as promoted.`, + Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { dir := brainProjectDir(cmd) store, queue := brain.NewStore(dir), brain.NewQueue(dir) - if !store.Exists() { - brainFatal(fmt.Errorf("no store yet — run 'mxcli brain init' first")) - } - e, ok, err := queue.Get(args[0]) + to, _ := cmd.Flags().GetString("to") + results, err := brain.PromoteBatch(store, queue, args, to) if err != nil { brainFatal(err) } - if !ok { - brainFatal(fmt.Errorf("no staged entry with id %s", args[0])) - } - shard := e.Shard() - if to, _ := cmd.Flags().GetString("to"); to != "" { - shard = to - } - if err := store.Promote(e, shard); err != nil { - brainFatal(err) - } - if _, err := queue.Drop(e.ID); err != nil { - brainFatal(err) + reports := make([]promoteReport, 0, len(results)) + promoted := 0 + for _, r := range results { + if r.Status == brain.StatusPromoted { + promoted++ + } + reports = append(reports, promoteReport{ID: r.Entry.ID, Kind: string(r.Entry.EntryKind()), + Shard: r.Shard, Path: store.ShardPath(r.Shard), Status: r.Status}) } if globalJSONFlag { - brainJSON(promoteReport{ID: e.ID, Kind: string(e.EntryKind()), Shard: shard, Path: store.ShardPath(shard)}) + // One id keeps the single object #2032 introduced, so existing + // callers are unaffected; several get one result per id. + if len(args) == 1 && len(reports) == 1 { + brainJSON(reports[0]) + return + } + brainJSON(map[string]any{"results": reports, "promoted": promoted, + "already_promoted": len(reports) - promoted, "total": len(reports)}) return } - fmt.Printf("Promoted %s into %s\n", e.ID, store.ShardPath(shard)) + for _, r := range reports { + if r.Status == brain.StatusPromoted { + fmt.Printf("Promoted %s into %s\n", r.ID, r.Path) + } else { + fmt.Printf("Already promoted %s in %s\n", r.ID, r.Path) + } + } + if len(reports) > 1 { + fmt.Printf("\n%d promoted, %d already promoted.\n", promoted, len(reports)-promoted) + } }, } +// runBrainCaptureBatch is `capture --from-json`: read, validate every item, +// then queue the survivors in one write. +func runBrainCaptureBatch(dir, from string) { + var ( + r io.Reader = os.Stdin + err error + ) + if from != "-" { + f, ferr := os.Open(from) + if ferr != nil { + brainFatal(ferr) + } + defer f.Close() + r = f + } + items, err := brain.ParseBatch(r) + if err != nil { + brainFatal(err) + } + entries, err := brain.BuildBatch(items, time.Now(), func(raw string) (string, error) { + return brainSource(dir, raw) + }) + if err != nil { + brainFatal(err) + } + results, err := brain.CaptureBatch(brain.NewStore(dir), brain.NewQueue(dir), entries) + if err != nil { + brainFatal(err) + } + added := 0 + reports := make([]captureReport, 0, len(results)) + for _, res := range results { + if res.Added { + added++ + } + reports = append(reports, newCaptureReport(res.Entry, res.Added, res.Status)) + } + if globalJSONFlag { + brainJSON(map[string]any{"results": reports, "added": added, + "skipped": len(reports) - added, "total": len(reports)}) + return + } + for _, rep := range reports { + switch rep.Status { + case brain.StatusQueued: + fmt.Printf("Queued %s -> would promote into %s\n", rep.ID, shardLabel(rep.Shard)) + case brain.StatusAlreadyCommitted: + fmt.Printf("Already committed as %s — not added again.\n", rep.ID) + default: + fmt.Printf("Already queued as %s — not added again.\n", rep.ID) + } + } + fmt.Printf("\n%d queued, %d already present.\n", added, len(reports)-added) +} + var brainDropCmd = &cobra.Command{ Use: "drop ", Short: "Remove an entry from the queue or from its shard", @@ -1181,6 +1280,8 @@ func init() { brainListCmd.Flags().String("slice", "", "Only this slice's entries") brainListCmd.Flags().String("kind", "", "Only decisions or only requirements (decision|requirement)") brainListCmd.Flags().Bool("open", false, "Only open questions; --open=false for only settled entries") + brainCaptureCmd.Flags().String("from-json", "", + "Queue a JSON array of entries from a file, or - for stdin, as one act") brainPromoteCmd.Flags().String("to", "", "Override the derived shard (use 'project' for a cross-cutting fact)") brainStagedCmd.Flags().String("since", "", @@ -1247,6 +1348,9 @@ type promoteReport struct { Kind string `json:"kind"` Shard string `json:"shard"` Path string `json:"path"` + // Status is promoted, or already_promoted for an id already committed + // with the same content (a re-run after an interrupted promote). + Status string `json:"status"` } // stagedEntry is one queued entry as a machine sees it. The shard is included diff --git a/cmd/mxcli/cmd_brain_batch_test.go b/cmd/mxcli/cmd_brain_batch_test.go new file mode 100644 index 000000000..650400c71 --- /dev/null +++ b/cmd/mxcli/cmd_brain_batch_test.go @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +// ako/mxcli#2054: `capture --from-json` and `promote ...` end to end, in +// the JSON shapes a caller reads. + +type captureBatchJSON struct { + Results []captureJSON `json:"results"` + Added int `json:"added"` + Skipped int `json:"skipped"` + Total int `json:"total"` +} + +type promoteBatchJSON struct { + Results []promoteJSON `json:"results"` + Promoted int `json:"promoted"` + Total int `json:"total"` +} + +func TestBrainBatchCaptureAndPromote(t *testing.T) { + mpr := brainJSONProject(t) + in := filepath.Join(t.TempDir(), "entries.json") + if err := os.WriteFile(in, []byte(`[ + {"text": "Orders are committed by Finance", "anchors": ["@Sales.Order"]}, + {"title": "Invoices are immutable", "body": "Corrections are credit notes.", "anchors": ["@Sales.Order"]}, + {"text": "Orders are committed by Finance", "anchors": ["@Sales.Order"]} + ]`), 0o644); err != nil { + t.Fatal(err) + } + + args := []string{"brain", "capture", "--from-json", in, "-p", mpr, "--json"} + c := decodeJSON[captureBatchJSON](t, args, runBrainForTest(t, args)) + if c.Total != 3 || c.Added != 2 || c.Skipped != 1 || len(c.Results) != 3 { + t.Fatalf("capture --from-json = %+v, want 2 added and the in-batch duplicate skipped", c) + } + if c.Results[2].Added || c.Results[2].ID != c.Results[0].ID { + t.Errorf("in-batch duplicate = %+v, want added false with id %s", c.Results[2], c.Results[0].ID) + } + + // The same file again is a no-op. + again := decodeJSON[captureBatchJSON](t, args, runBrainForTest(t, args)) + if again.Added != 0 { + t.Errorf("re-capture added %d, want 0", again.Added) + } + + ids := []string{c.Results[0].ID, c.Results[1].ID} + pargs := append([]string{"brain", "promote"}, append(ids, "-p", mpr, "--json")...) + p := decodeJSON[promoteBatchJSON](t, pargs, runBrainForTest(t, pargs)) + if p.Promoted != 2 || p.Total != 2 || p.Results[0].Shard != "Sales" { + t.Fatalf("promote --json = %+v, want both promoted into Sales", p) + } + + // Run again: done already, not an error. + p = decodeJSON[promoteBatchJSON](t, pargs, runBrainForTest(t, pargs)) + if p.Promoted != 0 || p.Total != 2 || p.Results[0].Status != "already_promoted" { + t.Errorf("re-promote --json = %+v, want both already_promoted", p) + } +} diff --git a/cmd/mxcli/cmd_brain_capture_promote_json_test.go b/cmd/mxcli/cmd_brain_capture_promote_json_test.go index a97f31fdf..a8b18b443 100644 --- a/cmd/mxcli/cmd_brain_capture_promote_json_test.go +++ b/cmd/mxcli/cmd_brain_capture_promote_json_test.go @@ -28,10 +28,11 @@ type captureJSON struct { } type promoteJSON struct { - ID string `json:"id"` - Kind string `json:"kind"` - Shard string `json:"shard"` - Path string `json:"path"` + ID string `json:"id"` + Kind string `json:"kind"` + Shard string `json:"shard"` + Path string `json:"path"` + Status string `json:"status"` } func brainJSONProject(t *testing.T) string { diff --git a/cmd/mxcli/cmd_describe.go b/cmd/mxcli/cmd_describe.go index 32ee7b191..04391eb1d 100644 --- a/cmd/mxcli/cmd_describe.go +++ b/cmd/mxcli/cmd_describe.go @@ -405,6 +405,7 @@ var objectTypeToDescribe = map[string]string{ "AI_MODEL": "aimodel", "KNOWLEDGE_BASE": "knowledgebase", "CONSUMED_MCP_SERVICE": "consumedmcpservice", + "PUBLISHED_MCP_SERVICE": "publishedmcpservice", "JAVASCRIPT_ACTION": "javascriptaction", } diff --git a/cmd/mxcli/docker.go b/cmd/mxcli/docker.go index 5917e06ba..c6a065436 100644 --- a/cmd/mxcli/docker.go +++ b/cmd/mxcli/docker.go @@ -86,17 +86,19 @@ Examples: wait, _ := cmd.Flags().GetBool("wait") waitTimeout, _ := cmd.Flags().GetInt("wait-timeout") portOffset, _ := cmd.Flags().GetInt("port-offset") + updateWidgets, _ := cmd.Flags().GetBool("update-widgets") opts := docker.RunOptions{ - ProjectPath: projectPath, - MxBuildPath: mxbuildPath, - SkipCheck: skipCheck, - Fresh: fresh, - Wait: wait, - WaitTimeout: time.Duration(waitTimeout) * time.Second, - PortOffset: portOffset, - Stdout: os.Stdout, - Stderr: os.Stderr, + ProjectPath: projectPath, + MxBuildPath: mxbuildPath, + SkipCheck: skipCheck, + UpdateWidgetsFirst: updateWidgets, + Fresh: fresh, + Wait: wait, + WaitTimeout: time.Duration(waitTimeout) * time.Second, + PortOffset: portOffset, + Stdout: os.Stdout, + Stderr: os.Stderr, } if err := docker.Run(opts); err != nil { @@ -115,16 +117,20 @@ This command: 1. Detects the Mendix project version (requires >= 11.6.1) 2. Locates MxBuild and a JDK matching the project's JavaVersion (auto-downloads MxBuild from CDN if not found) -3. Copies the project to a temporary directory and runs 'mx update-widgets', - 'mx check' and MxBuild (--target=portable-app-package) on that copy +3. Copies the project to a temporary directory and runs 'mx check' and + MxBuild (--target=portable-app-package) on that copy 4. Applies version-aware patches to fix known PAD issues The project itself is not modified; only the output directory is written. Each of the mx tools writes into the project it is given (update-widgets rewrites the model, mx check and MxBuild write theme-cache/, deployment/ -and javasource/ proxies), so they never run on the original. Widget -definitions are normalised on the copy only; 'mxcli fix widgets' applies -that to the project. +and javasource/ proxies), so they never run on the original. + +The project is checked and built as stored, as 'docker check' checks it: a +project whose stored widget definitions are stale (CE0463) fails the build, +with each CE0463 labelled as one 'mxcli fix widgets' clears or one that +persists. --update-widgets builds a widget-normalised copy instead (the +behaviour before ako/mxcli#1794), leaving the project as it is. MxBuild is cached at ~/.mxcli/mxbuild/{version}/ and reused across builds. You can also pre-download with: mxcli setup mxbuild -p app.mpr @@ -152,15 +158,17 @@ Examples: dryRun, _ := cmd.Flags().GetBool("dry-run") skipCheck, _ := cmd.Flags().GetBool("skip-check") noUpdateWidgets, _ := cmd.Flags().GetBool("no-update-widgets") + updateWidgets, _ := cmd.Flags().GetBool("update-widgets") opts := docker.BuildOptions{ - ProjectPath: projectPath, - MxBuildPath: mxbuildPath, - OutputDir: outputDir, - DryRun: dryRun, - SkipCheck: skipCheck, - SkipUpdateWidgets: noUpdateWidgets, - Stdout: os.Stdout, + ProjectPath: projectPath, + MxBuildPath: mxbuildPath, + OutputDir: outputDir, + DryRun: dryRun, + SkipCheck: skipCheck, + SkipUpdateWidgets: noUpdateWidgets, + UpdateWidgetsFirst: updateWidgets, + Stdout: os.Stdout, } if err := docker.Build(opts); err != nil { @@ -485,6 +493,7 @@ Examples: mxbuildPath, _ := cmd.Flags().GetString("mxbuild-path") skipCheck, _ := cmd.Flags().GetBool("skip-check") modelOnly, _ := cmd.Flags().GetBool("model-only") + updateWidgets, _ := cmd.Flags().GetBool("update-widgets") cssOnly, _ := cmd.Flags().GetBool("css") host, _ := cmd.Flags().GetString("host") port, _ := cmd.Flags().GetInt("port") @@ -492,17 +501,18 @@ Examples: direct, _ := cmd.Flags().GetBool("direct") opts := docker.ReloadOptions{ - ProjectPath: projectPath, - MxBuildPath: mxbuildPath, - SkipCheck: skipCheck, - SkipBuild: modelOnly, - CSSOnly: cssOnly, - Host: host, - Port: port, - Token: token, - Direct: direct, - Stdout: os.Stdout, - Stderr: os.Stderr, + ProjectPath: projectPath, + MxBuildPath: mxbuildPath, + SkipCheck: skipCheck, + UpdateWidgetsFirst: updateWidgets, + SkipBuild: modelOnly, + CSSOnly: cssOnly, + Host: host, + Port: port, + Token: token, + Direct: direct, + Stdout: os.Stdout, + Stderr: os.Stderr, } if err := docker.Reload(opts); err != nil { @@ -547,6 +557,7 @@ func init() { // Run command flags dockerRunCmd.Flags().String("mxbuild-path", "", "Path to MxBuild executable") dockerRunCmd.Flags().Bool("skip-check", false, "Skip 'mx check' pre-build validation") + dockerRunCmd.Flags().Bool("update-widgets", false, "Build a widget-normalised copy (see 'docker build --update-widgets')") dockerRunCmd.Flags().Bool("fresh", false, "Remove volumes and rebuild from scratch") dockerRunCmd.Flags().BoolP("wait", "w", false, "Wait for runtime to start before returning") dockerRunCmd.Flags().Int("wait-timeout", 300, "Timeout in seconds for --wait (default: 300)") @@ -557,7 +568,9 @@ func init() { dockerBuildCmd.Flags().StringP("output", "o", "", "Output directory for PAD package") dockerBuildCmd.Flags().Bool("dry-run", false, "Detect tools and show patch plan without building") dockerBuildCmd.Flags().Bool("skip-check", false, "Skip 'mx check' pre-build validation") - dockerBuildCmd.Flags().Bool("no-update-widgets", false, "Skip 'mx update-widgets' (run on the temporary copy) before check and build") + dockerBuildCmd.Flags().Bool("no-update-widgets", false, "Check the project as stored without the update-widgets re-check that labels its CE0463 errors") + dockerBuildCmd.Flags().Bool("update-widgets", false, "Run 'mx update-widgets' on the temporary copy first, then check and build that normalised copy (builds a project whose stored widgets are stale)") + dockerBuildCmd.MarkFlagsMutuallyExclusive("no-update-widgets", "update-widgets") // Check command flags dockerCheckCmd.Flags().String("mxbuild-path", "", "Path to MxBuild/Mendix installation (used to find mx)") @@ -589,6 +602,7 @@ func init() { // Reload command flags dockerReloadCmd.Flags().String("mxbuild-path", "", "Path to MxBuild executable") dockerReloadCmd.Flags().Bool("skip-check", false, "Skip 'mx check' pre-build validation") + dockerReloadCmd.Flags().Bool("update-widgets", false, "Build a widget-normalised copy (see 'docker build --update-widgets')") dockerReloadCmd.Flags().Bool("model-only", false, "Skip build, just call reload_model") dockerReloadCmd.Flags().Bool("css", false, "CSS hot reload only (update_styling, no build)") dockerReloadCmd.Flags().String("host", "", "M2EE admin API host") diff --git a/cmd/mxcli/docker/build.go b/cmd/mxcli/docker/build.go index da8762c77..dc45e232e 100644 --- a/cmd/mxcli/docker/build.go +++ b/cmd/mxcli/docker/build.go @@ -4,6 +4,7 @@ package docker import ( "archive/zip" + "bytes" "fmt" "io" "os" @@ -34,9 +35,16 @@ type BuildOptions struct { // SkipCheck skips the 'mx check' pre-build validation. SkipCheck bool - // SkipUpdateWidgets skips the 'mx update-widgets' step before checking. + // SkipUpdateWidgets skips the update-widgets re-check that labels the + // stored project's CE0463 errors (--no-update-widgets). SkipUpdateWidgets bool + // UpdateWidgetsFirst runs 'mx update-widgets' on the temporary copy before + // the check and builds that widget-normalised copy (--update-widgets): the + // behaviour before ako/mxcli#1794. By default the project is checked and + // built as stored. + UpdateWidgetsFirst bool + // Stdout for output messages. Stdout io.Writer } @@ -44,13 +52,14 @@ type BuildOptions struct { // buildSteps is what buildOnCopy needs once the tools are resolved. An empty // MxPath skips update-widgets and the check (--skip-check, or no mx found). type buildSteps struct { - ProjectPath string // absolute - MxPath string - MxBuildPath string - JavaHome string - OutputDir string // absolute - SkipUpdateWidgets bool - DryRun bool + ProjectPath string // absolute + MxPath string + MxBuildPath string + JavaHome string + OutputDir string // absolute + SkipUpdateWidgets bool + UpdateWidgetsFirst bool + DryRun bool } // mxbuildCmd runs MxBuild. A package variable so tests can substitute a stub @@ -98,24 +107,47 @@ func buildOnCopy(s buildSteps, w, stderr io.Writer) error { fmt.Fprintln(w, " they are given; the project on disk is not changed, only the output directory is written).") if s.MxPath != "" { - if !s.SkipUpdateWidgets { - // Normalise pluggable widget definitions so neither the check nor - // MxBuild reports CE0463 for definitions that only need a resync. - fmt.Fprintln(w, "Normalising widget definitions on the temporary copy (the project keeps its own;") - fmt.Fprintln(w, " `mxcli fix widgets` applies the normalisation to it)...") + // The project is checked — and built — as stored, as `docker check` + // checks it (#646): a build that normalised widget definitions on its + // copy succeeded on a project whose stored widgets `docker check`, + // `run --local` and Studio Pro all report as stale, so "it built" did + // not mean "what is committed builds" (ako/mxcli#1794). + // --update-widgets is the old behaviour. + normalised := false + if s.UpdateWidgetsFirst { + fmt.Fprintln(w, "Normalising widget definitions on the temporary copy (--update-widgets; the project") + fmt.Fprintln(w, " keeps its own, `mxcli fix widgets` applies the normalisation to it)...") if err := updateWidgetsCmd(s.MxPath, workMpr, out, errOut); err != nil { out.Flush() fmt.Fprintf(w, "Warning: update-widgets failed (continuing): %v\n", err) + } else { + normalised = true } } fmt.Fprintln(w, "Checking project for errors...") - err := mxCheckCmd(s.MxPath, workMpr, nil, out, errOut) + var checkOut, checkErrOut bytes.Buffer + err := mxCheckCmd(s.MxPath, workMpr, nil, io.MultiWriter(out, &checkOut), io.MultiWriter(errOut, &checkErrOut)) out.Flush() errOut.Flush() if err != nil { - return fmt.Errorf("project has errors (fix them or use --skip-check to bypass): %w", err) + hint := "fix them, or use --skip-check to bypass" + if !normalised { + if stored := ce0463Lines(checkOut.String() + "\n" + checkErrOut.String()); len(stored) > 0 { + if !s.SkipUpdateWidgets { + recheckCE0463(w, out, s.MxPath, workMpr, s.ProjectPath, stored, checkOut.String()) + } + hint = "`mxcli fix widgets` resyncs the stale widget definitions in the project; " + + "--update-widgets builds a widget-normalised copy instead, leaving the project as it is; " + + "--skip-check bypasses the check" + } + } + return fmt.Errorf("project has errors (%s): %w", hint, err) } fmt.Fprintln(w, " Project check passed.") + if normalised { + fmt.Fprintln(w, "Note: this builds a widget-normalised copy (--update-widgets); the project as stored") + fmt.Fprintln(w, " may still have stale widget definitions — `mxcli fix widgets` applies the normalisation to it.") + } } if s.DryRun { return nil @@ -214,13 +246,14 @@ func Build(opts BuildOptions) error { } } if err := buildOnCopy(buildSteps{ - ProjectPath: projectPath, - MxPath: mxPath, - MxBuildPath: mxbuildPath, - JavaHome: javaHome, - OutputDir: outputDir, - SkipUpdateWidgets: opts.SkipUpdateWidgets, - DryRun: opts.DryRun, + ProjectPath: projectPath, + MxPath: mxPath, + MxBuildPath: mxbuildPath, + JavaHome: javaHome, + OutputDir: outputDir, + SkipUpdateWidgets: opts.SkipUpdateWidgets, + UpdateWidgetsFirst: opts.UpdateWidgetsFirst, + DryRun: opts.DryRun, }, w, os.Stderr); err != nil { return err } @@ -315,6 +348,10 @@ type RunOptions struct { // SkipCheck skips the 'mx check' pre-build validation. SkipCheck bool + // UpdateWidgetsFirst builds a widget-normalised copy (--update-widgets); + // see BuildOptions.UpdateWidgetsFirst. + UpdateWidgetsFirst bool + // Fresh removes volumes before starting. Fresh bool @@ -417,10 +454,11 @@ func Run(opts RunOptions) error { // Step 5: Build fmt.Fprintln(w, "") buildOpts := BuildOptions{ - ProjectPath: opts.ProjectPath, - MxBuildPath: opts.MxBuildPath, - SkipCheck: opts.SkipCheck, - Stdout: w, + ProjectPath: opts.ProjectPath, + MxBuildPath: opts.MxBuildPath, + SkipCheck: opts.SkipCheck, + UpdateWidgetsFirst: opts.UpdateWidgetsFirst, + Stdout: w, } if err := Build(buildOpts); err != nil { return fmt.Errorf("docker build: %w", err) diff --git a/cmd/mxcli/docker/build_readonly_test.go b/cmd/mxcli/docker/build_readonly_test.go index a22e0f505..a0d18793b 100644 --- a/cmd/mxcli/docker/build_readonly_test.go +++ b/cmd/mxcli/docker/build_readonly_test.go @@ -62,20 +62,23 @@ func stubMxBuild(t *testing.T) (sawModel *string, sawPath *string) { func TestBuildOnCopy_DoesNotModifyProject(t *testing.T) { for _, tc := range []struct { - name string - fixture func(*testing.T) string - skipCheck bool - skipUpdate bool - dryRun bool + name string + fixture func(*testing.T) string + skipCheck bool + skipUpdate bool + dryRun bool + updateFirst bool }{ - {"v1", v1Fixture, false, false, false}, - {"v2", v2Fixture, false, false, false}, - {"v1 --no-update-widgets", v1Fixture, false, true, false}, - {"v2 --no-update-widgets", v2Fixture, false, true, false}, - {"v1 --skip-check", v1Fixture, true, false, false}, - {"v2 --skip-check", v2Fixture, true, false, false}, - {"v1 --dry-run", v1Fixture, false, false, true}, - {"v2 --dry-run", v2Fixture, false, false, true}, + {"v1", v1Fixture, false, false, false, false}, + {"v2", v2Fixture, false, false, false, false}, + {"v1 --no-update-widgets", v1Fixture, false, true, false, false}, + {"v2 --no-update-widgets", v2Fixture, false, true, false, false}, + {"v1 --update-widgets", v1Fixture, false, false, false, true}, + {"v2 --update-widgets", v2Fixture, false, false, false, true}, + {"v1 --skip-check", v1Fixture, true, false, false, false}, + {"v2 --skip-check", v2Fixture, true, false, false, false}, + {"v1 --dry-run", v1Fixture, false, false, true, false}, + {"v2 --dry-run", v2Fixture, false, false, true, false}, } { t.Run(tc.name, func(t *testing.T) { mprPath := tc.fixture(t) @@ -102,13 +105,14 @@ func TestBuildOnCopy_DoesNotModifyProject(t *testing.T) { } var out bytes.Buffer if err := buildOnCopy(buildSteps{ - ProjectPath: mprPath, - MxPath: mxPath, - MxBuildPath: "mxbuild", - JavaHome: "/jdk", - OutputDir: outputDir, - SkipUpdateWidgets: tc.skipUpdate, - DryRun: tc.dryRun, + ProjectPath: mprPath, + MxPath: mxPath, + MxBuildPath: "mxbuild", + JavaHome: "/jdk", + OutputDir: outputDir, + SkipUpdateWidgets: tc.skipUpdate, + UpdateWidgetsFirst: tc.updateFirst, + DryRun: tc.dryRun, }, &out, io.Discard); err != nil { t.Fatalf("buildOnCopy: %v\n%s", err, out.String()) } @@ -144,9 +148,10 @@ func TestBuildOnCopy_DoesNotModifyProject(t *testing.T) { if _, err := os.Stat(filepath.Join(outputDir, "Dockerfile")); err != nil { t.Errorf("the PAD was not written to the output directory: %v", err) } - // MxBuild builds the widget-normalised model, which is what - // update-widgets is there for — on the copy, not the project. - wantNormalised := !tc.skipCheck && !tc.skipUpdate + // MxBuild builds the project as stored; only --update-widgets + // builds a widget-normalised model — on the copy, not the + // project (ako/mxcli#1794). + wantNormalised := !tc.skipCheck && tc.updateFirst if got := *sawModel == "rewritten by update-widgets"; got != wantNormalised { t.Errorf("MxBuild saw the normalised model = %v, want %v", got, wantNormalised) } diff --git a/cmd/mxcli/docker/build_stored_test.go b/cmd/mxcli/docker/build_stored_test.go new file mode 100644 index 000000000..3067e87a0 --- /dev/null +++ b/cmd/mxcli/docker/build_stored_test.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 + +// ako/mxcli#1794: `docker build` (and `docker run` / `docker reload`, which +// build through it) ran `mx update-widgets` on its temporary copy before its +// pre-build check and MxBuild. A project whose stored widgets were stale +// therefore built — from a widget-normalised copy — while `docker check`, +// `run --local` and Studio Pro all reported the CE0463s: the deployment was not +// built from what is committed. The build now checks the project as stored, +// as `docker check` does since #646, and labels each CE0463; the old behaviour +// is --update-widgets. +package docker + +import ( + "bytes" + "io" + "path/filepath" + "strings" + "testing" +) + +func TestBuildOnCopy_StaleStoredWidgetsFailTheBuild(t *testing.T) { + mprPath := v1Fixture(t) + calls := stubStaleWidgets(t, []string{"Page 'Home' / widget 'grid1'"}, nil, nil) + _, sawPath := stubMxBuild(t) + t.Setenv("TMPDIR", t.TempDir()) + var out bytes.Buffer + err := buildOnCopy(buildSteps{ProjectPath: mprPath, MxPath: "mx", OutputDir: filepath.Join(t.TempDir(), "out")}, &out, io.Discard) + if err == nil || !strings.Contains(err.Error(), "project has errors") { + t.Fatalf("err = %v, want the stored project's check to fail the build\n%s", err, out.String()) + } + if *sawPath != "" { + t.Error("MxBuild ran on a project whose stored widgets are stale") + } + // The check ran on the copy as stored; update-widgets only for the + // re-check that labels the CE0463. + if got := strings.Join(*calls, ","); !strings.HasPrefix(got, "check,") { + t.Errorf("first call = %q, want the stored project checked first", got) + } + s := out.String() + err.Error() + for _, want := range []string{"mxcli fix widgets", "--update-widgets"} { + if !strings.Contains(s, want) { + t.Errorf("the failure does not name %q:\n%s", want, s) + } + } +} + +// --update-widgets is the old behaviour: normalise the copy, check it and +// build it, saying so. +func TestBuildOnCopy_UpdateWidgetsBuildsTheNormalisedCopy(t *testing.T) { + mprPath := v1Fixture(t) + calls := stubStaleWidgets(t, []string{"Page 'Home' / widget 'grid1'"}, nil, nil) + _, sawPath := stubMxBuild(t) + t.Setenv("TMPDIR", t.TempDir()) + var out bytes.Buffer + if err := buildOnCopy(buildSteps{ProjectPath: mprPath, MxPath: "mx", OutputDir: filepath.Join(t.TempDir(), "out"), UpdateWidgetsFirst: true}, &out, io.Discard); err != nil { + t.Fatalf("buildOnCopy --update-widgets: %v\n%s", err, out.String()) + } + if *sawPath == "" { + t.Error("MxBuild did not run") + } + if got := strings.Join(*calls, ","); got != "update-widgets,check" { + t.Errorf("calls = %q, want update-widgets then check", got) + } + if !strings.Contains(out.String(), "mxcli fix widgets") { + t.Errorf("the build does not say the project keeps its stale widgets:\n%s", out.String()) + } +} + +// Control: a clean project builds, with no update-widgets anywhere. +func TestBuildOnCopy_CleanProjectBuildsAsStored(t *testing.T) { + mprPath := v1Fixture(t) + calls := stubStaleWidgets(t, nil, nil, nil) + _, sawPath := stubMxBuild(t) + t.Setenv("TMPDIR", t.TempDir()) + var out bytes.Buffer + if err := buildOnCopy(buildSteps{ProjectPath: mprPath, MxPath: "mx", OutputDir: filepath.Join(t.TempDir(), "out")}, &out, io.Discard); err != nil { + t.Fatalf("buildOnCopy: %v\n%s", err, out.String()) + } + if *sawPath == "" { + t.Error("MxBuild did not run") + } + if got := strings.Join(*calls, ","); got != "check" { + t.Errorf("calls = %q, want only the check", got) + } +} diff --git a/cmd/mxcli/docker/reload.go b/cmd/mxcli/docker/reload.go index aa34c868d..4082a7b49 100644 --- a/cmd/mxcli/docker/reload.go +++ b/cmd/mxcli/docker/reload.go @@ -21,6 +21,10 @@ type ReloadOptions struct { // SkipCheck skips mx check before build. SkipCheck bool + // UpdateWidgetsFirst builds a widget-normalised copy (--update-widgets); + // see BuildOptions.UpdateWidgetsFirst. + UpdateWidgetsFirst bool + // SkipBuild skips mxbuild, just calls reload_model. SkipBuild bool @@ -83,10 +87,11 @@ func Reload(opts ReloadOptions) error { // Build step (unless --model-only) if !opts.SkipBuild { buildOpts := BuildOptions{ - ProjectPath: opts.ProjectPath, - MxBuildPath: opts.MxBuildPath, - SkipCheck: opts.SkipCheck, - Stdout: w, + ProjectPath: opts.ProjectPath, + MxBuildPath: opts.MxBuildPath, + SkipCheck: opts.SkipCheck, + UpdateWidgetsFirst: opts.UpdateWidgetsFirst, + Stdout: w, } if err := Build(buildOpts); err != nil { return fmt.Errorf("build failed: %w", err) diff --git a/cmd/mxcli/init_claudemd.go b/cmd/mxcli/init_claudemd.go index e638d6968..7a536d07d 100644 --- a/cmd/mxcli/init_claudemd.go +++ b/cmd/mxcli/init_claudemd.go @@ -5,6 +5,7 @@ package main import ( "fmt" + "gopkg.in/yaml.v3" "path/filepath" "strings" ) @@ -18,14 +19,41 @@ func yamlSingleQuote(s string) string { return "'" + s + "'" } -// wrapSkillContent prepends OpenCode-compatible YAML frontmatter to a skill file. -// OpenCode requires each skill to live in its own subdirectory as SKILL.md and -// the file must start with YAML frontmatter containing name, description, and -// compatibility fields. +// wrapSkillContent gives a skill the frontmatter OpenCode reads: its name, its +// description, and the opencode marker. A skill that already has frontmatter +// (the Agent Skills layout every bundled skill uses) keeps its own description +// and is given ONE block: prepending a second one, described by the skill's +// first heading, made OpenCode read "Agents" where the skill says "… Use when +// adding GenAI features", and the description is what decides whether a skill +// is invoked at all (ako/mxcli#1822). func wrapSkillContent(skillName string, content []byte) []byte { - description := extractSkillDescription(content) + description, body := splitSkillFrontmatter(content) + if description == "" { + description = extractSkillDescription(body) + } frontmatter := fmt.Sprintf("---\nname: %s\ndescription: %s\ncompatibility: opencode\n---\n\n", yamlSingleQuote(skillName), yamlSingleQuote(description)) - return append([]byte(frontmatter), content...) + return append([]byte(frontmatter), body...) +} + +// splitSkillFrontmatter returns the description a skill's leading YAML +// frontmatter declares ("" when it has none, or none readable) and the content +// after that frontmatter. Content without frontmatter is returned whole. +func splitSkillFrontmatter(content []byte) (description string, body []byte) { + text := string(content) + if !strings.HasPrefix(text, "---\n") { + return "", content + } + head, rest, ok := strings.Cut(text[len("---\n"):], "\n---\n") + if !ok { + return "", content + } + var fm struct { + Description string `yaml:"description"` + } + if err := yaml.Unmarshal([]byte(head), &fm); err != nil { + return "", content + } + return strings.TrimSpace(fm.Description), []byte(strings.TrimLeft(rest, "\n")) } // extractSkillDescription returns a one-line description for the skill by diff --git a/cmd/mxcli/init_opencode_context_test.go b/cmd/mxcli/init_opencode_context_test.go new file mode 100644 index 000000000..6d32b0b21 --- /dev/null +++ b/cmd/mxcli/init_opencode_context_test.go @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// ako/mxcli#1822 (mendixlabs/mxcli#1683): `init --tool opencode` wrote an +// opencode.json whose `instructions` globbed both skill trees. OpenCode +// inlines every instruction file's body into the system prompt, so a fresh +// session started with 150 files, 2.7 MB — about 73% of a 1M window — before +// the user typed anything. OpenCode discovers .opencode/skills/ by itself and +// loads a skill's body on demand; the instructions are AGENTS.md alone. +func TestOpenCodeConfig_InstructionsAreAgentsMdOnly(t *testing.T) { + var cfg struct { + Instructions []string `json:"instructions"` + } + if err := json.Unmarshal([]byte(generateOpenCodeConfig("App", "App.mpr")), &cfg); err != nil { + t.Fatalf("opencode.json is not JSON: %v", err) + } + if len(cfg.Instructions) != 1 || cfg.Instructions[0] != "AGENTS.md" { + t.Errorf("instructions = %v, want only AGENTS.md: a skill glob inlines every skill body into every prompt", cfg.Instructions) + } +} + +// The same report's second symptom: the .opencode/skills/ copies were given a +// frontmatter of their own, described by the skill's first heading +// ("Agents"), in front of the skill's own frontmatter. OpenCode reads the +// first, so the "Use when…" description that decides whether a skill is +// invoked was replaced by a title. A skill that has frontmatter keeps its +// description, in one frontmatter block. +func TestWrapSkillContent_KeepsTheSkillsOwnDescription(t *testing.T) { + src := "---\nname: agents\ndescription: \"Author agents in MDL. Use when adding GenAI features.\"\n---\n# Agents\n\nbody\n" + out := string(wrapSkillContent("agents", []byte(src))) + if n := strings.Count(out, "\n---\n") + boolInt(strings.HasPrefix(out, "---\n")); n != 2 { + t.Errorf("want exactly one frontmatter block (2 delimiters), got %d:\n%s", n, out) + } + if !strings.Contains(out, "Use when adding GenAI features.") { + t.Errorf("the skill's own description was lost:\n%s", out) + } + if !strings.Contains(out, "compatibility: opencode") || !strings.Contains(out, "# Agents\n\nbody\n") { + t.Errorf("want the opencode marker and the body:\n%s", out) + } +} + +func boolInt(b bool) int { + if b { + return 1 + } + return 0 +} + +// End to end on the shipped skills: every .opencode/skills/ copy carries the +// description of the skill it copies. +func TestInitOpenCode_SkillsKeepTheirDescriptions(t *testing.T) { + dir := t.TempDir() + runInit(t, []string{"opencode"}, dir) + body, err := os.ReadFile(filepath.Join(dir, ".opencode", "skills", "agents", "SKILL.md")) + if err != nil { + t.Fatal(err) + } + head, _, _ := strings.Cut(strings.TrimPrefix(string(body), "---\n"), "\n---\n") + if !strings.Contains(head, "Use when") { + t.Errorf(".opencode/skills/agents/SKILL.md frontmatter lacks the skill's \"Use when\" description:\n%s", head) + } +} diff --git a/cmd/mxcli/lint_rules_shipped.txt b/cmd/mxcli/lint_rules_shipped.txt index d95855167..11e274345 100644 --- a/cmd/mxcli/lint_rules_shipped.txt +++ b/cmd/mxcli/lint_rules_shipped.txt @@ -79,3 +79,4 @@ df118032d8e9554b860ed09401a06397efaff09738dfe403c77a2ee9f9cbe941 sec_strict_mod 72060544ea8fd6dd0d2d73b0cce09f3cb3802677acfa4f7cce02e5204bb7153f sec_unconstrained_pii_read.star d926cf53fd54bc73473b567849bf866f045b5b2bed60eeacc66e982d5e21225f sec_unconstrained_pii_read.star 9bdd2ef0fd62f6c1879a81850a2cf136acb401ed1a3a741726521c410af3bbbc cross_module_data.star +cbf54d4123e26875a48d54998d2f15b678e12caef832705c10b0a33c5d88e021 cross_module_data.star diff --git a/cmd/mxcli/syntax/features_integration.go b/cmd/mxcli/syntax/features_integration.go index 87a0ab347..cc2b43005 100644 --- a/cmd/mxcli/syntax/features_integration.go +++ b/cmd/mxcli/syntax/features_integration.go @@ -983,9 +983,9 @@ DESCRIBE DATABASE CONNECTION Ops.Erp;`, Path: "agents.mcp-service", Features: []string{"agent_documents/agent_consumed_mcp_service"}, Summary: "CREATE/DROP CONSUMED MCP SERVICE documents for AI agents", - Keywords: []string{"consumed mcp service", "mcp", "mcp service", "protocol version"}, - Syntax: "CREATE [OR MODIFY] CONSUMED MCP SERVICE Module.Name [FOLDER 'path'] (\n ProtocolVersion: v2025_03_26,\n Version: '1.0',\n ConnectionTimeoutSeconds: 30,\n Documentation: 'description'\n);\nDESCRIBE CONSUMED MCP SERVICE Module.Name;\nLIST CONSUMED MCP SERVICES [IN Module];\nDROP CONSUMED MCP SERVICE Module.Name;", - Example: "create consumed mcp service MyModule.WebSearch (\n ProtocolVersion: v2025_03_26,\n Version: '1.0',\n ConnectionTimeoutSeconds: 30\n);", + Keywords: []string{"consumed mcp service", "mcp", "mcp service", "protocol version", "endpoint", "authenticationmicroflow"}, + Syntax: "CREATE [OR MODIFY] CONSUMED MCP SERVICE Module.Name [FOLDER 'path'] (\n ProtocolVersion: v2025_03_26,\n Endpoint: @Module.UrlConstant, -- String constant; without it the service is skipped (MDL-AGENT03)\n [AuthenticationMicroflow: Module.GetHeaders,] -- no parameters, returns List of System.HttpHeader\n Version: '1.0',\n ConnectionTimeoutSeconds: 30,\n Documentation: 'description'\n);\nDESCRIBE CONSUMED MCP SERVICE Module.Name;\nLIST CONSUMED MCP SERVICES [IN Module];\nDROP CONSUMED MCP SERVICE Module.Name;", + Example: "create consumed mcp service MyModule.WebSearch (\n ProtocolVersion: v2025_03_26,\n Endpoint: @MyModule.WebSearchUrl,\n Version: '1.0',\n ConnectionTimeoutSeconds: 30\n);", SeeAlso: []string{"agents"}, }) @@ -1009,10 +1009,12 @@ DESCRIBE DATABASE CONNECTION Ops.Erp;`, "create agent", "drop agent", "usagetype", "systemprompt", "userprompt", "variables", "toolchoice", "temperature", "topp", "maxtokens", "useraccessapproval", "approval", "streaming", "displaytitle", "minsimilarity", + "selectedmodel", "selected model", }, Syntax: `CREATE [OR MODIFY] AGENT Module.Name [FOLDER 'path'] ( UsageType: Task|Chat, Model: Module.MyModel, + SelectedModel: 'model id', [SelectedModelName: 'display name',] [Description: 'text',] [MaxTokens: N,] [Temperature: 0.7,] @@ -1032,11 +1034,13 @@ DESCRIBE DATABASE CONNECTION Ops.Erp;`, [DisplayTitle: 'shown to the user', DisplayDescription: '…'] )] }; -A TOOL's Document is the microflow it runs. On OR MODIFY, the tools, -knowledge bases and entity keep their stored identity, and settings MDL has -no spelling for (the selected model) keep their stored values, and so does -a setting the statement leaves out.`, - Example: "create agent MyModule.Assistant (\n UsageType: Chat,\n Model: MyModule.GPT4,\n SystemPrompt: $$You are a helpful assistant.$$,\n UserPrompt: 'Ask me anything.'\n)\n{\n MCP SERVICE MyModule.WebSearch ( Enabled: true )\n};", +A TOOL's Document is the microflow it runs. SelectedModel is the id of the +model the Model document's resource offers (e.g. 'eu.anthropic.claude-sonnet-5'); +Agent Editor 2.2+ refuses to import an agent without one, and that stops the +import of every agent (check: MDL-AGENT02). SelectedModelName defaults to the +id. On OR MODIFY, the tools, knowledge bases and entity keep their stored +identity, and a setting the statement leaves out keeps its stored value.`, + Example: "create agent MyModule.Assistant (\n UsageType: Chat,\n Model: MyModule.GPT4,\n SelectedModel: 'eu.anthropic.claude-sonnet-5',\n SystemPrompt: $$You are a helpful assistant.$$,\n UserPrompt: 'Ask me anything.'\n)\n{\n MCP SERVICE MyModule.WebSearch ( Enabled: true )\n};", SeeAlso: []string{"agents", "agents.model", "agents.knowledge-base", "agents.mcp-service"}, }) } diff --git a/cmd/mxcli/syntax/features_page.go b/cmd/mxcli/syntax/features_page.go index 3fd99c7c9..f6b3cd71c 100644 --- a/cmd/mxcli/syntax/features_page.go +++ b/cmd/mxcli/syntax/features_page.go @@ -123,7 +123,7 @@ LIST IMPACT OF htmlelement; "gallery", "listview", "textbox", "textarea", "datepicker", "combobox", "checkbox", "radiobuttons", "filemanager", "file upload", "actionbutton", "linkbutton", "dynamictext", "title", "snippetcall", "navigationlist", "item", - "column", "row", "footer", "header", "controlbar", + "column", "row", "footer", "header", "controlbar", "emptyplaceholder", "empty placeholder", "no items", "template", "specialization", "list view template", "tabcontainer", "tabpage", "tabs", "groupbox", "collapsible", "image", "filter", "textfilter", "numberfilter", "datefilter", @@ -151,7 +151,16 @@ LIST IMPACT OF htmlelement; "GALLERY g (...) { FILTER f { TEXTFILTER tf (Attribute: A) } }\n" + "-- A FILTER block written on a DATAGRID is not a column filter and not a\n" + "-- container the grid declares — it used to be dropped on write with no\n" + - "-- diagnostic, and is now refused (MDL-WIDGET30).\n\n" + + "-- diagnostic, and is now refused (MDL-WIDGET30).\n" + + "-- What an EMPTY grid or gallery shows is the EMPTYPLACEHOLDER slot. It is\n" + + "-- hidden unless showEmptyPlaceholder is 'custom', and widgets in a hidden\n" + + "-- slot are CE0463 at mx check, so name both (MDL-WIDGET10 otherwise):\n" + + "DATAGRID dg (DataSource: DATABASE Module.Entity, showEmptyPlaceholder: 'custom') {\n" + + " COLUMN (Attribute: A)\n" + + " EMPTYPLACEHOLDER { DYNAMICTEXT tEmpty (Content: 'No items found.') }\n" + + "}\n" + + "-- On an existing grid: ALTER PAGE M.P { SET (showEmptyPlaceholder: 'custom') ON dg;\n" + + "-- INSERT INTO dg.emptyplaceholder { DYNAMICTEXT tEmpty (Content: '…') }; };\n\n" + "-- Inputs\nTEXTBOX name (Label: 'L', Attribute: Attr)\nTEXTAREA | DATEPICKER | COMBOBOX | CHECKBOX | RADIOBUTTONS\n" + "-- Read-only style of TEXTBOX, TEXTAREA, DATEPICKER, CHECKBOX, RADIOBUTTONS (and DATAVIEW):\n" + "-- Inherit (default) | Control | Text — how the widget renders when it is read-only.\n" + diff --git a/cmd/mxcli/tool_templates.go b/cmd/mxcli/tool_templates.go index 1c67520b8..cb63e1836 100644 --- a/cmd/mxcli/tool_templates.go +++ b/cmd/mxcli/tool_templates.go @@ -571,12 +571,15 @@ func generateCopilotInstructions(projectName, mprPath string) string { } func generateOpenCodeConfig(projectName, mprPath string) string { + // AGENTS.md only. `instructions` inlines each matched file's whole body + // into the system prompt; the two skill globs this listed put 150 files, + // 2.7 MB — ~73% of a 1M window — in front of every session (ako/mxcli#1822). + // OpenCode discovers .opencode/skills/ itself and loads a skill's body when + // its description matches, which is what the skills are written for. return `{ "$schema": "https://opencode.ai/config.json", "instructions": [ - "AGENTS.md", - ".opencode/skills/**/SKILL.md", - ".ai-context/skills/*/SKILL.md" + "AGENTS.md" ] } ` diff --git a/docs-site/src/appendixes/error-messages.md b/docs-site/src/appendixes/error-messages.md index 09fbc795e..647d0ae2b 100644 --- a/docs-site/src/appendixes/error-messages.md +++ b/docs-site/src/appendixes/error-messages.md @@ -281,6 +281,40 @@ two tools with one name, is refused too. As a warning: `AuthenticationMicroflow` **Solution:** Fix the property the message names; `create published mcp service` documents each. +### MDL-AGENT02: An agent created without a selected model + +``` +agent Shop.SupportAgent has no SelectedModel: Agent Editor 2.2+ refuses to import it at +startup ("Model with name is referenced but could not be found"), and that stops the +import of every agent [MDL-AGENT02] +``` + +**Cause:** Agent Editor 2.2+ (AgentEditorCommons `AgentFromModel_Validate`) requires the +agent's `selectedModel` id and retrieves the deployed model by it. mxbuild does not read agent +documents, so the build is green; `ASU_AgentEditor` refuses the agent at startup and stops +importing the rest. + +**Solution:** Add `SelectedModel: ''` (e.g. `'eu.anthropic.claude-sonnet-5'`), the id +of a model the Model document's resource offers. `create or modify agent` is not flagged: left +out, it keeps the stored value. + +### MDL-AGENT03: A consumed MCP service without an endpoint + +``` +consumed mcp service Shop.CarrierTracking has no Endpoint: Agent Editor skips it at startup +("No value could be found for the local configuration for the constant"), so it connects +nowhere [MDL-AGENT03] +``` + +**Cause:** Agent Editor imports a consumed MCP service only when its `endpoint` constant has a +value: `ConsumedMCPService_CreateUpdateFromModel` reads it with `GetConstantValue` and skips the +service otherwise. mxbuild does not read agent documents, so the build is green; the agents that +use the service get none of its tools. + +**Solution:** Add `Endpoint: @Module.UrlConstant`, a String constant holding the server URL, and +set its value per environment. `create or modify` is not flagged: left out, it keeps the stored +endpoint. + ### MDL-LISTOP02: A list operation nested inside another one ``` diff --git a/docs-site/src/internals/catalog-schema.md b/docs-site/src/internals/catalog-schema.md index 312b8b854..ee58d46b3 100644 --- a/docs-site/src/internals/catalog-schema.md +++ b/docs-site/src/internals/catalog-schema.md @@ -249,6 +249,8 @@ CREATE TABLE WIDGETS ( DynamicClasses TEXT, ActionType TEXT, -- $Type of Action / OnClickAction / ClickAction HasConfirmation INTEGER, -- 1 when that action has a ConfirmationInfo + Caption TEXT, -- own Caption, project's default language + Tooltip TEXT, -- own Tooltip, project's default language ProjectId TEXT, SnapshotId TEXT ); @@ -293,7 +295,7 @@ rather than trusting a list here: | RefKind | Edge | |---------|------| -| `call` | flow calls a microflow / nanoflow / rule / Java action / REST operation; a workflow calls a handler microflow; a consumed OData service calls its configuration / headers / error-handling microflow | +| `call` | flow calls a microflow / nanoflow / rule / Java action / REST operation; a workflow calls a handler microflow; a consumed OData service calls its configuration / headers / error-handling microflow; an agent calls its tool microflow, nanoflow or agent; a flow passes a microflow or agent document as a Java action parameter, which the action runs | | `create` / `change` / `delete` / `retrieve` | flow acts on an entity object | | `commit` | flow commits an entity object: a commit action, or a create / change that commits (`Yes` or `YesWithoutEvents`), beside its `create` / `change` edge | | `return` | flow returns an entity type | @@ -308,12 +310,15 @@ rather than trusting a list here: | `home_page` / `login_page` / `menu_item` | navigation profile references a page, or a microflow (home microflow, menu item that calls one) | | `calculate` | calculated attribute uses a microflow | | `schedule` | scheduled event runs a microflow | -| `publish` | published service runs a microflow: a REST operation, an OData resource's read / insert / update / delete microflow, an OData action, a REST or OData service's custom authentication microflow | +| `publish` | published service runs a microflow: a REST operation, an OData resource's read / insert / update / delete microflow, an OData action, a REST or OData service's custom authentication microflow; a published MCP service's tool microflow and authentication microflow | | `event` | event handler runs a microflow: an entity event, or a business event a service subscribes to | | `settings` | a project setting (after-startup / before-shutdown / health check) names a microflow | | `sync` | offline navigation profile synchronizes an entity | | `mapping_call` | flow applies an import or export mapping: an import / export action, or a REST or web service call's request or response mapping | | `structure` | import or export mapping is built on a JSON structure | +| `tool` | agent draws on a knowledge base or consumed MCP service (a tool microflow, nanoflow or agent is a `call`) | +| `model` | agent runs on an AI model | +| `key` | AI model or knowledge base reads its key from a constant | | `validate` | attribute validation rule uses a regular expression | | `member` | document binds, reads or writes an attribute, navigates an association, or reaches the entity an association step arrives at | | `xpath` | XPath constraint names an attribute, association, enumeration value, or an entity reached through an association | @@ -385,11 +390,17 @@ CREATE VIRTUAL TABLE STRINGS USING fts5( -- ., e.g. Forms$ActionButton.Caption. -- Non-translatable strings keep a plain label: page_url, -- log_node, log_message, documentation, rest_path, - -- task_name, domain_model_documentation, ... + -- task_name, domain_model_documentation, an agent's + -- system_prompt / user_prompt / tool_description, ... Language, -- Language code, empty for non-translatable strings ElementId, -- The owning element's $ID — what distinguishes an -- enumeration's twelve values from each other - ModuleName + ModuleName, + WidgetId UNINDEXED -- The widget around the text: a button's caption is + -- owned by its Forms$ClientTemplate and its confirmation + -- by a Forms$ConfirmationInfo, so ElementId is never the + -- button. '' outside a widget, and for non-translatable rows. + -- A tab page's caption names the tab page, not in widgets ); ``` diff --git a/docs-site/src/reference/agent/create-agent.md b/docs-site/src/reference/agent/create-agent.md index a6f8cde7f..c9efa3b2c 100644 --- a/docs-site/src/reference/agent/create-agent.md +++ b/docs-site/src/reference/agent/create-agent.md @@ -6,6 +6,7 @@ CREATE [ OR MODIFY ] AGENT module.Name ( UsageType: { Task | Conversational }, model: module.ModelName, + SelectedModel: 'model id', SystemPrompt: 'prompt' | $$ multi-line prompt $$, UserPrompt: 'prompt' | $$ multi-line prompt $$ [, description: 'description' ] @@ -82,6 +83,12 @@ If `OR MODIFY` is specified and the agent already exists, its properties and bod `model: module.ModelName` : Qualified reference to the model document the agent uses. +`SelectedModel` +: The id of the model the agent uses, one the Model document's resource offers (e.g. `'eu.anthropic.claude-sonnet-5'`). Required in practice from Agent Editor 2.2: its import refuses an agent without one ("Model with name is referenced but could not be found for the Agent"), and that refusal stops the import of every agent document at startup. `mxcli check` warns on a `create agent` without it (MDL-AGENT02). Only the id is read at runtime; it is matched against `GenAICommons.DeployedModel.Model`. To list the ids, with the app running: `mxcli oql "select Model from MxGenAIConnector.MxCloudDeployedModel"` (reported to work in ako/mxcli#2031). `''` clears it. + +`SelectedModelName` +: Optional. The display name Studio Pro stores next to the id (e.g. `'Anthropic Claude Sonnet 5'`); nothing at runtime reads it. Defaults to the id, or keeps the stored name while the id is unchanged. + `SystemPrompt` : Instructions for the AI model's behavior and persona. @@ -132,6 +139,7 @@ On `OR MODIFY`, a setting the statement leaves out keeps its stored value. CREATE AGENT MyModule."Summarizer" ( UsageType: Task, model: MyModule.GPT4Model, + SelectedModel: 'eu.anthropic.claude-sonnet-5', SystemPrompt: 'Summarize the given text in 3 sentences.', UserPrompt: 'Enter text to summarize.' ); @@ -144,6 +152,7 @@ CREATE AGENT MyModule."Summarizer" ( CREATE AGENT MyModule."Translator" ( UsageType: Task, model: MyModule.GPT4Model, + SelectedModel: 'eu.anthropic.claude-sonnet-5', variables: ("Language": EntityAttribute), SystemPrompt: 'Translate the text into {{Language}}.', UserPrompt: 'Hello world' @@ -157,6 +166,7 @@ CREATE AGENT MyModule."Translator" ( CREATE AGENT MyModule."Support" ( UsageType: Chat, model: MyModule.GPT4Model, + SelectedModel: 'eu.anthropic.claude-sonnet-5', EnableBackendStreaming: true, SystemPrompt: 'Help the customer with their orders.' ) @@ -177,6 +187,7 @@ CREATE AGENT MyModule."Support" ( CREATE AGENT MyModule."CodeReviewer" ( UsageType: Task, model: MyModule.GPT4Model, + SelectedModel: 'eu.anthropic.claude-sonnet-5', SystemPrompt: $$You are a code review assistant. Review the provided code for: @@ -199,6 +210,7 @@ CREATE AGENT MyModule."ResearchAssistant" ( UsageType: Conversational, description: 'Research assistant with web search and documentation', model: MyModule.GPT4Model, + SelectedModel: 'eu.anthropic.claude-sonnet-5', MaxTokens: 16384, ToolChoice: Auto, SystemPrompt: 'You are a research assistant. Use available tools.', diff --git a/docs-site/src/reference/agent/create-consumed-mcp-service.md b/docs-site/src/reference/agent/create-consumed-mcp-service.md index 7cf78bb1c..5bab17187 100644 --- a/docs-site/src/reference/agent/create-consumed-mcp-service.md +++ b/docs-site/src/reference/agent/create-consumed-mcp-service.md @@ -5,6 +5,8 @@ ```sql CREATE [ OR MODIFY ] CONSUMED MCP SERVICE module.Name ( ProtocolVersion: v2025_03_26, + Endpoint: @module.UrlConstant, + [ AuthenticationMicroflow: module.GetHeaders, ] version: '1.0' [, ConnectionTimeoutSeconds: 30 ] [, documentation: 'description' ] @@ -31,6 +33,12 @@ If `OR MODIFY` is specified and the service already exists, its properties are u `ProtocolVersion: v2025_03_26` : The MCP protocol version. Use the token form (e.g. `v2025_03_26`), not a quoted string. +`Endpoint: @module.UrlConstant` +: The String constant holding the MCP server's URL (e.g. `https://example.com/mcp`). Required in practice: at startup Agent Editor reads the constant's value and skips a service whose constant has none ("No value could be found for the local configuration for the constant"), so the agents that use it get no tools. `mxcli check` warns on a `CREATE` without it (MDL-AGENT03); `check --references` and `exec` refuse a constant that does not exist or is not a String. Set the value per environment. On `OR MODIFY`, left out, the stored endpoint stays. + +`AuthenticationMicroflow: module.GetHeaders` +: Optional. The microflow that supplies credentials. MCP Client calls it with no arguments and sends the `List of System.HttpHeader` it returns with each request (build the list with `MCPClient.HttpHeader_CreateAndAddToList`). `check --references` refuses one with parameters or another return type. On `OR MODIFY`, left out, the stored microflow stays; `ALTER … SET AuthenticationMicroflow = ''` removes it. + `version: '1.0'` : The service version as a quoted string. @@ -45,8 +53,12 @@ If `OR MODIFY` is specified and the service already exists, its properties are u ### Register an MCP service ```sql +mdl 1; +CREATE CONSTANT MyModule.WebSearchUrl (Type: String, DefaultValue: 'https://example.com/mcp'); + CREATE CONSUMED MCP SERVICE MyModule."WebSearch" ( ProtocolVersion: v2025_03_26, + Endpoint: @MyModule.WebSearchUrl, version: '1.0', ConnectionTimeoutSeconds: 30, documentation: 'Web search MCP server for research tasks' diff --git a/docs-site/src/tools/builtin-rules.md b/docs-site/src/tools/builtin-rules.md index e38cc3e86..4093122c5 100644 --- a/docs-site/src/tools/builtin-rules.md +++ b/docs-site/src/tools/builtin-rules.md @@ -40,7 +40,7 @@ The **lint rules** below run with `mxcli lint`. There is also a separate group o |------|-------------| | **CONV011** | No commit in loop -- Detects COMMIT statements, and CREATE / CHANGE with a COMMIT clause, inside LOOP blocks (performance anti-pattern) | | **CONV012** | Exclusive split captions -- Checks that decision branches have meaningful captions | -| **CONV013** | Error handling on external calls -- Ensures external service calls have error handling | +| **CONV013** | Error handling on external calls -- Ensures external service calls have error handling. Not reported: a call inside a loop, where Mendix allows only Rollback (CE0644), and a call to a Java action that exists to throw (`Throw…`, e.g. `CommunityCommons.ThrowException`), whose Rollback is the point | | **CONV014** | No continue error handling -- Warns against using CONTINUE error handling without logging | | **CONV020** | Created object never persisted -- A microflow creates an object of a persistent entity without commit, and never commits, returns or passes it on, so the object is discarded when the microflow ends ([details](#conv020-details)) | | **CONV021** | Create/change then commit -- A Create or Change object activity without commit, directly followed by a Commit of the same object; one create/change with Commit set does both ([details](#conv021-details)) | diff --git a/docs-site/src/tools/catalog-tables.md b/docs-site/src/tools/catalog-tables.md index 38dc9f766..399fd2c52 100644 --- a/docs-site/src/tools/catalog-tables.md +++ b/docs-site/src/tools/catalog-tables.md @@ -68,6 +68,47 @@ SELECT Name FROM CATALOG.MODULES WHERE Source = '' AND COALESCE(DomainModelDocumentation, '') = ''; ``` +### CATALOG.FOLDERS + +One row per folder, in every module, empty folders included. A document's +`Folder` column names only the folders that hold a document, so an empty folder +appears nowhere else. + +| Column | Description | +|--------|-------------| +| `Id` | Folder UUID | +| `Name` | Folder name | +| `ModuleName` | Module the folder is in | +| `Path` | `Objects/Order` — spelled as a document's `Folder` column, so the two join. A folder name that itself holds `/` is not escaped (`LIST FOLDERS` prints it as `\/`) | +| `ParentPath` | Path of the enclosing folder; `''` directly in the module | +| `Depth` | `1` directly in the module | +| `DocumentCount` | Documents directly in the folder, not in its subfolders | +| `FolderCount` | Folders directly in the folder | + +```sql +-- Empty folders in your own modules +SELECT f.ModuleName, f.Path FROM CATALOG.FOLDERS f +JOIN CATALOG.MODULES m ON m.Name = f.ModuleName +WHERE m.Source = '' AND f.DocumentCount = 0 AND f.FolderCount = 0; +``` + +### CATALOG.ANNOTATIONS + +One row per **domain model** annotation, in every module. A microflow's +annotations are rows of `CATALOG.ACTIVITIES` (`ActivityType = 'Annotation'`). + +| Column | Description | +|--------|-------------| +| `Id` | Annotation UUID | +| `DomainModelId` | The domain model holding it | +| `ModuleName` | Module of that domain model | +| `Caption` | The full text, newlines kept | +| `X`, `Y`, `Width` | Place and width on the domain model canvas | + +```sql +SELECT ModuleName, Caption FROM CATALOG.ANNOTATIONS WHERE Caption LIKE '%TODO%'; +``` + ### CATALOG.ASSOCIATIONS Information about entity associations, same-module and cross-module. @@ -223,6 +264,8 @@ One row per widget **instance** on a page or snippet. Full build only | `Class`, `Style`, `DynamicClasses` | The widget's Appearance | | `ActionType` | Stored type of the primary action — `Action` (buttons), else `OnClickAction` (containers), else `ClickAction` (list views, images): `Forms$DeleteClientAction`, `Forms$MicroflowAction`, `Forms$CallNanoflowClientAction`, `Forms$FormAction` (show page), `Forms$NoAction`, …; empty for a widget without one. A pluggable widget's actions are not read | | `HasConfirmation` | 1 when the primary action asks for confirmation. Only microflow, nanoflow and workflow calls have that setting; a delete action never does | +| `Caption` | The widget's own caption in the project's default language (buttons and labels; an input's label and a dynamic text's content are not its caption); `''` when it has none | +| `Tooltip` | The widget's tooltip in the project's default language; `''` when none is set | ```sql -- Widgets nested more than five deep, deepest first @@ -430,6 +473,69 @@ The edge carries neither the moment nor the event; `refs` has no column for them, and a kind per combination would put eight kinds into every consumer's list to say one thing. Which moment and which event is this table's question. +## Agent Tables + +The Agent Editor's documents, with the settings their JSON contents hold +([ako/mxcli#2075](https://github.com/ako/mxcli/issues/2075)). A boolean that a +document does not store, as an older Agent Editor does not, is `NULL` rather +than `0`, so a query can tell "off" from "never set". + +### CATALOG.AGENTS + +| Column | Description | +|--------|-------------| +| `UsageType` | `Chat` or `Task` | +| `Model` | The AI model document, qualified name | +| `SelectedModel`, `SelectedModelName` | The model picked from the provider's list, by id and display name (`NULL` if none) | +| `ToolChoice`, `MaxTokens`, `Temperature`, `TopP` | Generation settings, `NULL` or empty when not set | +| `EnableBackendStreaming`, `EnableUIStreaming` | `1`, `0`, or `NULL` when the document does not store the switch | +| `Entity` | The context entity, qualified name | +| `Variables` | The prompt variable keys, comma-separated | +| `ToolCount`, `KnowledgeBaseCount`, `MCPServiceCount` | Entries in the agent's tool lists. `ToolCount` excludes MCP services | + +### CATALOG.AGENT_TOOLS + +One row per entry of an agent's tools list: a microflow, nanoflow or agent it +calls, or a consumed MCP service (`ToolType = 'MCP'`). + +| Column | Description | +|--------|-------------| +| `AgentId`, `Agent` | The agent's id and qualified name | +| `Name`, `Description` | What the model sees | +| `ToolType` | `Microflow`, `MCP`, or empty | +| `Document` | The microflow, nanoflow, agent or MCP service, qualified name | +| `Enabled` | `1` or `0` | +| `UserAccessApproval` | `HiddenForUser`, `VisibleForUser`, `UserConfirmationRequired`, or empty when not stored | +| `DisplayTitle`, `DisplayDescription` | What the user sees of the call, `NULL` when not stored | + +`CATALOG.AGENT_MCP_SERVICES` is the same rows filtered to `ToolType = 'MCP'`, +with `Document` named `Service`. + +```sql +-- Tools an agent can run without the user's approval +SELECT Agent, Name, Document FROM CATALOG.AGENT_TOOLS +WHERE Enabled = 1 AND UserAccessApproval <> 'UserConfirmationRequired'; +``` + +### CATALOG.AGENT_KNOWLEDGE_BASES + +One row per entry of an agent's knowledge base tools list: `Agent`, `Name`, +`Description`, `KnowledgeBase` (qualified name), `Collection`, `MaxResults`, +`MinSimilarity`, `IsVisible` and `Enabled`. + +### Models, knowledge bases and MCP services + +| Table | Columns beyond the document's | +|-------|-------------------------------| +| `CATALOG.AI_MODELS` | `Provider`, `ModelType`, `DisplayName`, `KeyConstant` | +| `CATALOG.KNOWLEDGE_BASES` | `Provider`, `KeyConstant` | +| `CATALOG.CONSUMED_MCP_SERVICES` | `ProtocolVersion`, `Version`, `ConnectionTimeoutSeconds` | +| `CATALOG.PUBLISHED_MCP_SERVICES` | `ServerName`, `EndpointPath`, `Version`, `ProtocolVersion`, `AuthenticationEnabled`, `AuthenticationMicroflow`, `ToolCount` | + +What an agent references (its tools, model, knowledge bases and context entity) +and what a published MCP service runs are also edges in `CATALOG.REFS`, so +`show callers` and `show impact` see them. + ## Graph-Analysis Tables The dependency graph (`CATALOG.REFS`, full refresh) is analysed by a family of diff --git a/docs-site/src/tools/docker-build.md b/docs-site/src/tools/docker-build.md index 3c472cbd9..915b6eae2 100644 --- a/docs-site/src/tools/docker-build.md +++ b/docs-site/src/tools/docker-build.md @@ -17,22 +17,37 @@ mxcli docker build -p app.mpr ## The Project Is Not Modified -`mx update-widgets`, `mx check` and MxBuild all write into the project they are -given: update-widgets rewrites the model (an MPRv1 `.mpr` in place, an MPRv2 -project into MPRv1), mx check compiles the theme into `theme-cache/` and -`deployment/sass/`, and MxBuild regenerates the `javasource/` proxies, the -`.launch`, `.classpath` and `.project` files and all of `deployment/`. +`mx check` and MxBuild write into the project they are given: mx check +compiles the theme into `theme-cache/` and `deployment/sass/`, and MxBuild +regenerates the `javasource/` proxies, the `.launch`, `.classpath` and +`.project` files and all of `deployment/`. So `docker build` copies the project to a temporary directory (in `$TMPDIR`) -and runs all three there. Widget definitions are normalised on that copy, so the -build uses the normalised model while the project keeps its own; run -`mxcli fix widgets` to apply the normalisation to the project. (So the build's -pre-build check does not report a stale widget definition; `mxcli docker check`, -which checks the project as stored, does.) The only thing +and runs them there. The only thing written is the output directory (`.docker/build/` by default, or `-o`). Build output, caches and VCS folders (`deployment/`, `.docker/`, `.git/`, `node_modules/`, ...) are not copied. +## The Project Is Built as Stored + +The pre-build check runs on the project as stored, exactly as +[`mxcli docker check`](docker-check.md) checks it, and MxBuild builds that same +model: a deployment is built from what is committed. A project whose stored +widget definitions are stale (CE0463) therefore fails the build, with each +CE0463 labelled as one `mxcli fix widgets` clears or one that persists — the +same verdict `docker check`, `mxcli run --local` and Studio Pro give. + +| Flag | Checks and builds | Use | +|------|-------------------|-----| +| *(default)* | the project as stored | a build that matches what is committed | +| `--update-widgets` | a copy normalised by `mx update-widgets` first | build anyway; the project keeps its stale widgets, `mxcli fix widgets` fixes them | +| `--no-update-widgets` | the project as stored, without the re-check that labels CE0463s | scripts that passed it before | +| `--skip-check` | no check at all | — | + +`--update-widgets` is how `docker build` (and `docker run` / `docker reload`, +which build through it) behaved before ako/mxcli#1794; `docker run` and +`docker reload` take the same flag. + ## PAD Patching PAD (Platform-Agnostic Deployment) patching modifies the build output to be compatible with container-based deployment platforms. This is essential for deploying Mendix applications to Kubernetes, Cloud Foundry, or other container orchestrators. diff --git a/docs-site/src/tools/docker-check.md b/docs-site/src/tools/docker-check.md index 63882a915..19e26a13d 100644 --- a/docs-site/src/tools/docker-check.md +++ b/docs-site/src/tools/docker-check.md @@ -130,7 +130,7 @@ Error: project check failed: exit status 1 `mxcli docker check` is faster than a full build (`mxcli docker build`) because it only validates the project structure without compiling or packaging. This makes it useful for: - **Quick validation** after making MDL changes -- **Pre-build gate** -- `mxcli docker build` runs `mx check` automatically before building (use `--skip-check` to bypass). That check runs on the build's widget-normalised copy, as `docker check --update-widgets` does, so it does not report stale widget definitions; `docker check` does +- **Pre-build gate** -- `mxcli docker build` runs the same check automatically before building, on the project as stored (use `--skip-check` to bypass; `--update-widgets` checks and builds a widget-normalised copy instead) - **CI pipelines** -- fail fast on invalid projects before spending time on a full build - **Iterative development** -- check after each MDL script execution to catch errors early diff --git a/docs-site/src/tools/project-brain.md b/docs-site/src/tools/project-brain.md index 279ead1c6..1aeebaf7d 100644 --- a/docs-site/src/tools/project-brain.md +++ b/docs-site/src/tools/project-brain.md @@ -449,7 +449,8 @@ progress from 65/65 to 63/65 and the number was the only symptom. | `brain init` | Creates `docs/brain/`. Refuses a `docs/brain/` it did not write | | `brain capture "" [-a @Anchor]…` | Queues an entry. Never commits | | `brain staged [--since ] [--slice ] [--fail-if-empty]` | Lists the queue with the shard each entry would land in | -| `brain promote [--to ]` | Writes it into its shard | +| `brain capture --from-json ` | Queues a JSON array of entries as one act: all validated first, duplicates skipped | +| `brain promote … [--to ]` | Writes them into their shards, all or none | | `brain drop ` | Removes it from the queue or from its shard | | `brain capture "" --source [#]` | Records which document, and where in it, the entry came from | | `brain edit [""] [-a @Anchor]… [--source ]` | Rewrites it in place, staged or committed; the id does not change | @@ -471,10 +472,38 @@ act on the answers rather than read them. `capture --json` returns the entry's `id`, `kind`, `slice`, `shard`, `added` and a `status` of `queued`, `already_queued` or `already_committed` (a duplicate is reported with its twin's id, and exits 0 as before); `promote --json` returns the `id`, `kind`, -`shard` and the shard file's `path`. `brain staged --since ` is the +`shard`, the shard file's `path` and a `status` of `promoted` or +`already_promoted`. `brain staged --since ` is the slice boundary: note `last_id` before handing a slice off, pass it back afterwards, and `--fail-if-empty` exits 1 on a slice that recorded nothing. +### Many entries at once + +A tool that turns a specification into fifty requirements, and a person who +accepts them all, should not mean fifty captures and fifty promotes — each is a +place to stop half way. Both commands take a batch, and a batch is one act: + +```bash +mxcli brain capture --from-json requirements.json -p app.mpr --json +mxcli brain promote 1a2b3c 4d5e6f 7a8b9c -p app.mpr +``` + +`--from-json` reads a JSON array (`-` for stdin) of +`{"text" | "title"+"body", "anchors": [...], "slice"?, "open"?, "source"?}`, +the input fields `staged --json` prints. Every item is checked before anything +is queued, so one bad item fails the batch and names its index (`items[3]`). +A duplicate — of a committed entry, a queued one, or an earlier item — is +skipped and reported with the existing id, so the same list twice is a no-op. + +`promote` with several ids resolves every id and checks each shard's cap with +**all** of the batch's additions to it before writing anything; additions that +each fit can still overrun the shard together, and then nothing is written. +`--to` applies to every id. The shards are written first, then the queue once: +interrupted between the two, the entries are committed and still queued, and +running the same `promote` again completes it. With `--json`, both return +`results` (one per item, in input order) plus totals; `promote` with a single +id keeps its single-object shape. + `--since` rather than `--slice`, because `capture --slice` is what makes an entry a *requirement* — a decision found while building a slice carries no slice at all, and a slice's findings are mostly decisions. The queue is append-only, so diff --git a/docs-site/src/tools/search.md b/docs-site/src/tools/search.md index 490c5a262..30e1bf81f 100644 --- a/docs-site/src/tools/search.md +++ b/docs-site/src/tools/search.md @@ -83,6 +83,7 @@ mxcli search -p app.mpr "error" -q --format names | head -1 | awk '{print $2}' | - Log messages and log node names of microflows - Page URLs and the paths, resources and operation summaries of published REST services - Workflow names, descriptions, task names and activity captions +- Agents' system and user prompts, descriptions, and the descriptions of their tools and knowledge bases, with the title a tool call shows the user (`ObjectType` `AGENT`; `StringContext` `system_prompt`, `user_prompt`, `agent_description`, `tool_description`, `tool_display_title`, `tool_display_description`, `knowledge_base_description`), and a published MCP service's tool descriptions and titles (`PUBLISHED_MCP_SERVICE`; `mcp_tool_description`, `mcp_tool_title`) - Element names: the qualified name is a searchable column too, so `SEARCH 'Customer'` also returns the rows of elements named Customer that have some indexed text - **Documentation of project elements.** A hit's `ObjectType` names the element and its `StringContext` is `documentation`: @@ -101,7 +102,7 @@ mxcli search -p app.mpr "error" -q --format names | head -1 | awk '{print $2}' | | Consumed and published REST service, business event service | `CONSUMED_REST_SERVICE`, `PUBLISHED_REST_SERVICE`, `BUSINESS_EVENT_SERVICE` | | Import mapping, export mapping, JSON structure, data transformer | `IMPORT_MAPPING`, `EXPORT_MAPPING`, `JSON_STRUCTURE`, `DATA_TRANSFORMER` | | Scheduled event, task queue, regular expression, image and icon collection, menu | `SCHEDULED_EVENT`, `TASK_QUEUE`, `REGULAR_EXPRESSION`, `IMAGE_COLLECTION`, `ICON_COLLECTION`, `MENU` | - | Agent, AI model, knowledge base, consumed MCP service | `AGENT`, `AI_MODEL`, `KNOWLEDGE_BASE`, `CONSUMED_MCP_SERVICE` | + | Agent, AI model, knowledge base, consumed and published MCP service | `AGENT`, `AI_MODEL`, `KNOWLEDGE_BASE`, `CONSUMED_MCP_SERVICE`, `PUBLISHED_MCP_SERVICE` | | A module's domain model documentation | `MODULE`, with `StringContext` `domain_model_documentation` | This is the complete set: every document type whose catalog table carries a documentation text. Not covered: the documentation of an enumeration value (Mendix has no such field) and the summary text of a remote service's contract entities (written by the remote service, not by the project). diff --git a/docs-site/src/tutorial/opencode.md b/docs-site/src/tutorial/opencode.md index 4d1b882c0..6f49d6f80 100644 --- a/docs-site/src/tutorial/opencode.md +++ b/docs-site/src/tutorial/opencode.md @@ -46,27 +46,27 @@ my-mendix-project/ ### opencode.json -The `opencode.json` file is OpenCode's primary configuration. It points to `AGENTS.md` for instructions and to both the OpenCode-format skills in `.opencode/skills/` and the universal skill files in `.ai-context/skills/`: +The `opencode.json` file is OpenCode's primary configuration. Its instructions are `AGENTS.md` alone: ```json { "$schema": "https://opencode.ai/config.json", "instructions": [ - "AGENTS.md", - ".opencode/skills/**/SKILL.md", - ".ai-context/skills/*/SKILL.md" + "AGENTS.md" ] } ``` +The skills are not listed there. OpenCode inlines every instruction file's whole body into the system prompt, so listing the skill trees put about 2.7 MB of skill text in front of every session. OpenCode discovers the skills in `.opencode/skills/` itself and loads a skill's body only when its description matches the task. + ### Skills -Skills live in `.opencode/skills//SKILL.md` and use YAML frontmatter that OpenCode understands: +Skills live in `.opencode/skills//SKILL.md` and use YAML frontmatter that OpenCode understands. The description is the skill's own, with its "Use when…" clause, since that is what OpenCode matches a task against: ```markdown --- name: write-microflows -description: MDL syntax and patterns for creating microflows +description: 'Microflow syntax reference in MDL — … Use before writing any CREATE MICROFLOW …' compatibility: opencode --- diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index 27d7bf428..222aa7621 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -1101,6 +1101,7 @@ the `AgentEditorCommons` marketplace module and Mendix 11.9+. create agent Module.MyAgent ( UsageType: task, model: Module.MyModel, + SelectedModel: 'eu.anthropic.claude-sonnet-5', MaxTokens: 4096, Temperature: 0.7, TopP: 0.9, diff --git a/mdl-examples/bug-tests/1300-domain-model-annotations.mdl b/mdl-examples/bug-tests/1300-domain-model-annotations.mdl new file mode 100644 index 000000000..ba27ed30d --- /dev/null +++ b/mdl-examples/bug-tests/1300-domain-model-annotations.mdl @@ -0,0 +1,18 @@ +mdl 1; +-- ako/mxcli#1300: a domain model annotation was in no catalog table, so a +-- Starlark rule looking for "TODO" notes saw microflow annotations only. +-- +-- Run on a fresh app, then: +-- mxcli -p App.mpr -c "refresh catalog force" +-- mxcli -p App.mpr -c "select ModuleName, Caption, X, Y, Width from CATALOG.ANNOTATIONS where ModuleName = 'Bug1300'" +-- Expected: Bug1300 | TODO split sales per region | 60 | 40 | 400 +-- and annotations() in a lint rule returns the same row, the caption with its +-- newline. Before the fix the table did not exist and annotations() was +-- undefined. + +create module Bug1300; + +create persistent entity Bug1300.Sale (Amount: Integer); + +create annotation in Bug1300 (Caption: 'TODO split sales +per region', Position: (60, 40), Width: 400); diff --git a/mdl-examples/bug-tests/1301-enum-value-id.mdl b/mdl-examples/bug-tests/1301-enum-value-id.mdl new file mode 100644 index 000000000..a8760c6d7 --- /dev/null +++ b/mdl-examples/bug-tests/1301-enum-value-id.mdl @@ -0,0 +1,24 @@ +mdl 1; +-- ako/mxcli#1301: a Starlark rule could not tie an enumeration caption in +-- another language to its value: enumerations().values had no id, and the +-- strings() caption rows name the value only by element_id. +-- +-- Run on a fresh app, then lint with a rule that does +-- for e in enumerations(): +-- for v in e.values: ... v.id ... +-- and matches it to strings(language = "nl_NL") rows whose +-- context == "Enumerations$EnumerationValue.Caption" and element_id == v.id. +-- Expected: Bug1301.ApiTimeout.Quick -> 'dertig', .Slow -> 'vierhonderd'. +-- Before the fix the rule stopped: "enumeration_value" struct has no .id. + +create module Bug1301; + +create enumeration Bug1301.ApiTimeout ( + Quick '30', + Slow '400' +); + +create or modify translations in Bug1301 for nl_NL ( + '30' as 'dertig', + '400' as 'vierhonderd' +); diff --git a/mdl-examples/bug-tests/1303-widget-caption-tooltip.mdl b/mdl-examples/bug-tests/1303-widget-caption-tooltip.mdl new file mode 100644 index 000000000..8ff581bba --- /dev/null +++ b/mdl-examples/bug-tests/1303-widget-caption-tooltip.mdl @@ -0,0 +1,38 @@ +mdl 1; +-- ako/mxcli#1303: a Starlark rule could not read a button's caption or +-- tooltip, and a strings() caption row named the button's inner +-- Forms$ClientTemplate, so with two buttons a rule could not tell which caption +-- belonged to the one without a confirmation. +-- +-- Run on a fresh app, then: +-- mxcli -p App.mpr -c "refresh catalog full force" +-- mxcli -p App.mpr -c "select Name, Caption, HasConfirmation from CATALOG.WIDGETS where ContainerQualifiedName = 'Bug1303.Sale_Edit' and WidgetType = 'Forms\$ActionButton' order by Name" +-- Expected: +-- btnDelete Delete sale 0 +-- btnRemove Remove 1 +-- and in a lint rule, strings(language = "nl_NL") rows whose widget_id is +-- btnRemove's id: 'Weghalen' and 'Deze verkoop verwijderen?'. +-- Before the fix the Caption column did not exist and no strings() row named a +-- widget id. + +create module Bug1303; + +create persistent entity Bug1303.Sale (Amount: Integer); + +create microflow Bug1303.Act_Sale_Delete ($Sale: Bug1303.Sale) +begin + delete $Sale; +end; + +create page Bug1303.Sale_Edit (params: ($Sale: Bug1303.Sale), title: 'Edit sale', layout: Atlas_Core.Atlas_Default) { + dataview dvSale (datasource: $Sale) { + actionbutton btnDelete (caption: 'Delete sale', action: call microflow Bug1303.Act_Sale_Delete(Sale = $currentObject)) + actionbutton btnRemove (caption: 'Remove', action: call microflow Bug1303.Act_Sale_Delete(Sale = $currentObject) with (Confirmation: 'Delete this sale?')) + } +}; + +create or modify translations in Bug1303 for nl_NL ( + 'Delete sale' as 'Verkoop verwijderen', + 'Remove' as 'Weghalen', + 'Delete this sale?' as 'Deze verkoop verwijderen?' +); diff --git a/mdl-examples/bug-tests/1304-jar-files.mdl b/mdl-examples/bug-tests/1304-jar-files.mdl new file mode 100644 index 000000000..2bdb1756d --- /dev/null +++ b/mdl-examples/bug-tests/1304-jar-files.mdl @@ -0,0 +1,23 @@ +mdl 1; +-- ako/mxcli#1304: a Starlark rule could not see the .jar files in userlib/ +-- and vendorlib/; jar_dependencies() lists only what module settings declare. +-- +-- Run on a fresh app, then put two versions of one library on disk: +-- mkdir -p App/vendorlib +-- touch App/userlib/commons-lang3-3.12.0.jar App/userlib/commons-lang3-3.12.0.jar.Bug1304.RequiredLib +-- touch App/vendorlib/commons-lang3-3.14.0.jar +-- and lint with a rule that iterates jar_files(). +-- Expected rows: +-- userlib commons-lang3-3.12.0.jar commons-lang3 3.12.0 required_by ["Bug1304"] +-- vendorlib commons-lang3-3.14.0.jar commons-lang3 3.14.0 required_by [] +-- Before the fix the rule was skipped: undefined: jar_files. + +create module Bug1304; + +alter module Bug1304 + add jar dependency ( + group = 'org.apache.commons', + artifact = 'commons-lang3', + version = '3.14.0', + included = true, + ); diff --git a/mdl-examples/bug-tests/1436-folders-in-catalog.mdl b/mdl-examples/bug-tests/1436-folders-in-catalog.mdl new file mode 100644 index 000000000..93132a8f2 --- /dev/null +++ b/mdl-examples/bug-tests/1436-folders-in-catalog.mdl @@ -0,0 +1,25 @@ +mdl 1; +-- ako/mxcli#1436: the catalog held no folders, so a Starlark rule saw a folder +-- only through the `folder` of a document in it, and an empty folder did not +-- exist for it. +-- +-- Run on a fresh app, then: +-- mxcli -p App.mpr -c "refresh catalog force" +-- mxcli -p App.mpr -c "select Path, ParentPath, Depth, DocumentCount, FolderCount from CATALOG.FOLDERS where ModuleName = 'Bug1436' order by Path" +-- Expected: +-- Objects '' 1 1 1 +-- Objects/Empty Objects 2 0 0 +-- Before the fix CATALOG.FOLDERS did not exist (and, its name being a keyword, +-- the select printed nothing at all), and LIST FOLDERS was the only place the +-- empty folder showed. + +create module Bug1436; + +-- Created in Objects/Empty, then moved up: Objects/Empty stays, empty. +create microflow Bug1436.ACT_Probe () +folder 'Objects/Empty' +begin + return; +end; + +move microflow Bug1436.ACT_Probe to folder 'Objects'; diff --git a/mdl-examples/bug-tests/2074-agent-and-java-param-refs.mdl b/mdl-examples/bug-tests/2074-agent-and-java-param-refs.mdl new file mode 100644 index 000000000..b711eea56 --- /dev/null +++ b/mdl-examples/bug-tests/2074-agent-and-java-param-refs.mdl @@ -0,0 +1,44 @@ +mdl 1; +-- ako/mxcli#2074: an agent-editor document produced no refs, and a microflow +-- passed as a microflow-typed Java action parameter was no reference, so both +-- showed "(no callers found)" and sat in GRAPH_DEAD_ASSETS. +-- +-- Needs an app with Agent Editor and AgentCommons (the Agent Builder starter +-- app has both). Run, then: +-- mxcli -p App.mpr -c "refresh catalog full force" +-- mxcli -p App.mpr -c "show callers of Bug2074.Tool_GetOrder" +-- expected: Bug2074.SupportAgent +-- mxcli -p App.mpr -c "show callers of Bug2074.Chat_Action" +-- expected: Bug2074.ChatContext_Create +-- Before the fix both said "(no callers found)". + +create module Bug2074; + +create microflow Bug2074.Tool_GetOrder ($OrderId: String) returns String as $Result +begin + declare $Result String = 'order ' + $OrderId; + return $Result; +end; + +create agent Bug2074.SupportAgent ( + UsageType: Chat, + SystemPrompt: 'You answer questions about orders.' +) { + tool GetOrder ( + ToolType: Microflow, + Document: Bug2074.Tool_GetOrder, + Description: 'Look an order up by its id' + ) +}; + +create microflow Bug2074.Chat_Action ($ChatContext: ConversationalUI.ChatContext) returns Boolean as $Ok +begin + declare $Ok Boolean = true; + return $Ok; +end; + +create microflow Bug2074.ChatContext_Create ($Agent: AgentCommons.Agent) returns ConversationalUI.ChatContext as $ChatContext +begin + $ChatContext = call java action AgentCommons.ChatContext_Create_ForAgent(OverwritingDeployedModel = empty, ActionMicroflow = 'Bug2074.Chat_Action', Agent = $Agent, ContextObject = empty); + return $ChatContext; +end; diff --git a/mdl-examples/bug-tests/2075-agent-details.mdl b/mdl-examples/bug-tests/2075-agent-details.mdl new file mode 100644 index 000000000..9fe52e2b8 --- /dev/null +++ b/mdl-examples/bug-tests/2075-agent-details.mdl @@ -0,0 +1,49 @@ +mdl 1; +-- ako/mxcli#2075: the catalog listed agent-editor documents by name only, so +-- "which agents can run a tool without the user's approval" needed one +-- describe agent per agent and text parsing. +-- +-- Needs an app with Agent Editor 2.6+ (the Agent Builder starter app). Run, then: +-- mxcli -p App.mpr -c "refresh catalog full force" +-- mxcli -p App.mpr -c "select Agent, Name, Document, UserAccessApproval from CATALOG.AGENT_TOOLS where Agent like 'Bug2075.%'" +-- expected: Bug2075.RefundAgent | IssueRefund | Bug2075.Tool_IssueRefund | VisibleForUser +-- mxcli -p App.mpr -c "select UsageType, ToolChoice, MaxTokens, EnableBackendStreaming, ToolCount from CATALOG.AGENTS where QualifiedName = 'Bug2075.RefundAgent'" +-- expected: Task | Auto | 1024 | 1 | 1 +-- mxcli -p App.mpr -c "select EndpointPath, ToolCount from CATALOG.PUBLISHED_MCP_SERVICES where QualifiedName = 'Bug2075.RefundMCP'" +-- expected: refunds | 1 +-- mxcli -p App.mpr -c "show callers of Bug2075.Tool_IssueRefund" +-- expected: Bug2075.RefundAgent and Bug2075.RefundMCP +-- Before the fix the columns and tables did not exist, and the published MCP +-- service was no caller. + +create module Bug2075; + +create microflow Bug2075.Tool_IssueRefund ($OrderId: String) returns String as $Result +begin + declare $Result String = 'refunded ' + $OrderId; + return $Result; +end; + +create agent Bug2075.RefundAgent ( + UsageType: Task, + ToolChoice: Auto, + MaxTokens: 1024, + EnableBackendStreaming: true, EnableUIStreaming: false, + SystemPrompt: 'You issue refunds for orders.', + UserPrompt: 'Refund order {{OrderId}}' +) { + tool IssueRefund ( + Document: Bug2075.Tool_IssueRefund, + Description: 'Refund an order by its id', + Enabled: true, + UserAccessApproval: VisibleForUser + ) +}; + +create published mcp service Bug2075.RefundMCP ( + EndpointPath: 'refunds', + Version: '1.0.0', + ProtocolVersion: v2025_03_26 +) { + tool issue_refund ( Microflow: Bug2075.Tool_IssueRefund, Description: 'Refund an order.' ) +}; diff --git a/mdl-examples/bug-tests/2075-agent-prompts-search.mdl b/mdl-examples/bug-tests/2075-agent-prompts-search.mdl new file mode 100644 index 000000000..d1120a42b --- /dev/null +++ b/mdl-examples/bug-tests/2075-agent-prompts-search.mdl @@ -0,0 +1,31 @@ +mdl 1; +-- ako/mxcli#2075: CATALOG.STRINGS indexed only an agent's documentation, so +-- search could not find its prompts or tool descriptions. +-- +-- Needs an app with Agent Editor (the Agent Builder starter app). Run, then: +-- mxcli -p App.mpr -c "refresh catalog full force" +-- mxcli -p App.mpr -c "search 'quarantined warehouse'" +-- expected: Bug2075P.StockAgent, StringContext system_prompt +-- mxcli -p App.mpr -c "search 'reorder threshold'" +-- expected: Bug2075P.StockAgent, StringContext tool_description +-- Before the fix both found no string match. + +create module Bug2075P; + +create microflow Bug2075P.Tool_CheckStock ($Sku: String) returns String as $Result +begin + declare $Result String = 'in stock: ' + $Sku; + return $Result; +end; + +create agent Bug2075P.StockAgent ( + UsageType: Task, + SystemPrompt: 'You answer stock questions. Never ship from a quarantined warehouse.', + UserPrompt: 'Check stock for {{Sku}}' +) { + tool CheckStock ( + Document: Bug2075P.Tool_CheckStock, + Description: 'Compare stock with the reorder threshold', + Enabled: true + ) +}; diff --git a/mdl-examples/doctype-tests/28-agent-editor-examples.mdl b/mdl-examples/doctype-tests/28-agent-editor-examples.mdl index d3b2cfecc..5b1a85656 100644 --- a/mdl-examples/doctype-tests/28-agent-editor-examples.mdl +++ b/mdl-examples/doctype-tests/28-agent-editor-examples.mdl @@ -38,6 +38,10 @@ create constant AgentTest."ModelKey" create constant AgentTest."KBKey" ( Type: string, DefaultValue: '' ); +-- The consumed MCP service's server URL (#2060); set it per environment. +create constant AgentTest."WebSearchUrl" + ( Type: string, DefaultValue: 'https://example.com/mcp' ); + -- ============================================================================ -- LEVEL 1: Model (prerequisite for Agents) -- ============================================================================ @@ -62,6 +66,7 @@ create knowledge base AgentTest."ProductDocs" ( create consumed mcp service AgentTest."WebSearch" ( ProtocolVersion: v2025_03_26, + Endpoint: @AgentTest.WebSearchUrl, version: '1.0', ConnectionTimeoutSeconds: 30, documentation: 'Web search MCP server for research tasks' @@ -74,6 +79,7 @@ create consumed mcp service AgentTest."WebSearch" ( create agent AgentTest."Summarizer" ( UsageType: task, model: AgentTest.TestModel, + SelectedModel: 'eu.anthropic.claude-sonnet-5', SystemPrompt: 'Summarize the given text in 3 sentences.', UserPrompt: 'Enter text to summarize.' ); @@ -85,6 +91,7 @@ create agent AgentTest."Summarizer" ( create agent AgentTest."Translator" ( UsageType: task, model: AgentTest.TestModel, + SelectedModel: 'eu.anthropic.claude-sonnet-5', variables: ("Language": EntityAttribute), SystemPrompt: 'Translate the text into {{Language}}.', UserPrompt: 'Hello world' @@ -97,6 +104,7 @@ create agent AgentTest."Translator" ( create agent AgentTest."CodeReviewer" ( UsageType: task, model: AgentTest.TestModel, + SelectedModel: 'eu.anthropic.claude-sonnet-5', SystemPrompt: $$You are a code review assistant. Review the provided code for: @@ -118,6 +126,7 @@ create agent AgentTest."ResearchAssistant" ( UsageType: Conversational, description: 'Research assistant with web search and documentation', model: AgentTest.TestModel, + SelectedModel: 'eu.anthropic.claude-sonnet-5', MaxTokens: 16384, ToolChoice: Auto, SystemPrompt: 'You are a research assistant. Use available tools.', diff --git a/mdl-examples/use-cases/ai-agent-platform-demo.mdl b/mdl-examples/use-cases/ai-agent-platform-demo.mdl index e0bf812da..62a5a2c20 100644 --- a/mdl-examples/use-cases/ai-agent-platform-demo.mdl +++ b/mdl-examples/use-cases/ai-agent-platform-demo.mdl @@ -332,6 +332,9 @@ create constant AICustSupport."LLMKey" create constant AICustSupport."KBKey" ( Type: string, DefaultValue: '' ); +create constant AICustSupport."WebSearchUrl" + ( Type: string, DefaultValue: 'https://example.com/mcp' ); + -- Model document (visible in Studio Pro Agent Editor — developer UX) create ai model AICustSupport."GPT4TurboModel" ( Provider: MxCloudGenAI, @@ -349,6 +352,7 @@ create knowledge base AICustSupport."SupportKnowledgeBase" ( -- Consumed MCP service: real-time web search tool (MCP 2025-03-26 / A2A) create consumed mcp service AICustSupport."WebSearchMCP" ( ProtocolVersion: v2025_03_26, + Endpoint: @AICustSupport.WebSearchUrl, version: '1.0', ConnectionTimeoutSeconds: 30, documentation: 'Real-time web search for current product and support information' @@ -361,6 +365,7 @@ create agent AICustSupport."SupportAgent" ( UsageType: Conversational, description: 'AI first-line support agent with RAG and web-search capability', model: AICustSupport.GPT4TurboModel, + SelectedModel: 'eu.anthropic.claude-sonnet-5', MaxTokens: 8192, ToolChoice: Auto, variables: ("CustomerTier": EntityAttribute, "CasePriority": EntityAttribute), @@ -396,6 +401,7 @@ Guidelines: create agent AICustSupport."ResponseGuardrail" ( UsageType: task, model: AICustSupport.GPT4TurboModel, + SelectedModel: 'eu.anthropic.claude-sonnet-5', SystemPrompt: $$You are a content safety and policy compliance checker. Evaluate the proposed AI response for: diff --git a/mdl/ast/ast_agenteditor.go b/mdl/ast/ast_agenteditor.go index f827a68a8..67413ebb3 100644 --- a/mdl/ast/ast_agenteditor.go +++ b/mdl/ast/ast_agenteditor.go @@ -69,6 +69,11 @@ type CreateConsumedMCPServiceStmt struct { ConnectionTimeoutSeconds int InnerDocumentation string // Contents.documentation field CreateOrModify bool // true for CREATE OR MODIFY / CREATE OR REPLACE + // Endpoint is the String constant holding the server URL (#2060); + // AuthenticationMicroflow returns the headers sent with each request. Left + // out, a create or modify keeps the stored reference. + Endpoint *QualifiedName + AuthenticationMicroflow *QualifiedName } func (s *CreateConsumedMCPServiceStmt) isStatement() {} @@ -211,14 +216,19 @@ type CreateAgentStmt struct { // stored value. EnableBackendStreaming *bool EnableUIStreaming *bool - Temperature *float64 - TopP *float64 - SystemPrompt string - UserPrompt string - Variables []AgentVarDef - Tools []AgentToolDef - KBTools []AgentKBToolDef - CreateOrModify bool // true for CREATE OR MODIFY / CREATE OR REPLACE + // The model on a multi-model resource (#2031): its id, and the display + // name Studio Pro stores. Left out, a create or modify keeps the stored + // one; '' clears it. + SelectedModel *string + SelectedModelName *string + Temperature *float64 + TopP *float64 + SystemPrompt string + UserPrompt string + Variables []AgentVarDef + Tools []AgentToolDef + KBTools []AgentKBToolDef + CreateOrModify bool // true for CREATE OR MODIFY / CREATE OR REPLACE } func (s *CreateAgentStmt) isStatement() {} diff --git a/mdl/backend/mcp/unsupported_gen.go b/mdl/backend/mcp/unsupported_gen.go index be675b6ae..d89dbc13a 100644 --- a/mdl/backend/mcp/unsupported_gen.go +++ b/mdl/backend/mcp/unsupported_gen.go @@ -534,6 +534,11 @@ func (unsupportedBackend) FindDocumentUnit(_ string, _ string) (r0 *types.Docume return } +func (unsupportedBackend) FindDocumentUnits(_ string, _ string) (r0 []*types.DocumentUnit, err1 error) { + err1 = errUnsupported("FindDocumentUnits") + return +} + func (unsupportedBackend) FindViewEntitySourceDocumentID(_ string, _ string) (r0 model.ID, err1 error) { err1 = errUnsupported("FindViewEntitySourceDocumentID") return diff --git a/mdl/backend/mfmutator/splice_handler_test.go b/mdl/backend/mfmutator/splice_handler_test.go index 4a6ff434a..7e4eb0099 100644 --- a/mdl/backend/mfmutator/splice_handler_test.go +++ b/mdl/backend/mfmutator/splice_handler_test.go @@ -45,18 +45,18 @@ func handlerFlow() ([]bson.D, []bson.D) { a := withEH(obj("a", "Microflows$ActionActivity", 250, 200), microflows.ErrorHandlingTypeCustomWithoutRollback) dSet(a, "Size", "160;60") // resized in Studio Pro: kept return []bson.D{ - obj("start", "Microflows$StartEvent", 100, 200), - a, - obj("h", "Microflows$ActionActivity", 250, 330), - obj("m", "Microflows$ExclusiveMerge", 400, 200), - obj("end", "Microflows$EndEvent", 550, 200), - }, []bson.D{ - flow("f1", "start", "a", 1, 3, false), - flow("f2", "a", "m", 1, 3, false), - flow("err", "a", "h", 2, 0, true), - flow("fh", "h", "m", 1, 2, false), - flow("f3", "m", "end", 1, 3, false), - } + obj("start", "Microflows$StartEvent", 100, 200), + a, + obj("h", "Microflows$ActionActivity", 250, 330), + obj("m", "Microflows$ExclusiveMerge", 400, 200), + obj("end", "Microflows$EndEvent", 550, 200), + }, []bson.D{ + flow("f1", "start", "a", 1, 3, false), + flow("f2", "a", "m", 1, 3, false), + flow("err", "a", "h", 2, 0, true), + flow("fh", "h", "m", 1, 2, false), + flow("f3", "m", "end", 1, 3, false), + } } func TestSplice_ReplaceKeepsTheErrorHandler(t *testing.T) { diff --git a/mdl/backend/mock/backend.go b/mdl/backend/mock/backend.go index 63d2518db..21d87e6c9 100644 --- a/mdl/backend/mock/backend.go +++ b/mdl/backend/mock/backend.go @@ -57,6 +57,7 @@ type MockBackend struct { // DocumentPlacementBackend MoveDocumentFunc func(unitID, containerID model.ID) error FindDocumentUnitFunc func(moduleName, name string) (*types.DocumentUnit, error) + FindDocumentUnitsFunc func(moduleName, name string) ([]*types.DocumentUnit, error) ListDocumentUnitsFunc func() ([]*types.DocumentUnit, error) // DomainModelBackend diff --git a/mdl/backend/mock/mock_placement.go b/mdl/backend/mock/mock_placement.go index cd1d520f0..5d8594cd1 100644 --- a/mdl/backend/mock/mock_placement.go +++ b/mdl/backend/mock/mock_placement.go @@ -30,6 +30,19 @@ func (m *MockBackend) FindDocumentUnit(moduleName, name string) (*types.Document return nil, nil } +// FindDocumentUnits defaults to what FindDocumentUnit returns, so a test that +// configured only the single lookup sees the same document through both. +func (m *MockBackend) FindDocumentUnits(moduleName, name string) ([]*types.DocumentUnit, error) { + if m.FindDocumentUnitsFunc != nil { + return m.FindDocumentUnitsFunc(moduleName, name) + } + doc, err := m.FindDocumentUnit(moduleName, name) + if err != nil || doc == nil { + return nil, err + } + return []*types.DocumentUnit{doc}, nil +} + // ListDocumentUnits returns nothing by default. An empty project is the honest // default for a mock: it makes a folder listing show only what the test set up // through the typed list functions, rather than inventing documents. diff --git a/mdl/backend/modelsdk/agenteditor_carry.go b/mdl/backend/modelsdk/agenteditor_carry.go index 6070dea67..11b6387ec 100644 --- a/mdl/backend/modelsdk/agenteditor_carry.go +++ b/mdl/backend/modelsdk/agenteditor_carry.go @@ -46,8 +46,9 @@ var docRefSpec = &contentsSpec{modelled: contentKeys("documentId", "qualifiedNam var agentContentsSpec = &contentsSpec{ modelled: contentKeys("description", "systemPrompt", "userPrompt", "usageType", "variables", "tools", "knowledgebaseTools", "model", "entity", "maxTokens", "toolChoice", "temperature", "topP", - "enableBackendStreaming", "enableUIStreaming"), - objects: map[string]*contentsSpec{"model": docRefSpec, "entity": docRefSpec}, + "enableBackendStreaming", "enableUIStreaming", "selectedModel"), + objects: map[string]*contentsSpec{"model": docRefSpec, "entity": docRefSpec, + "selectedModel": {modelled: contentKeys("id", "name")}}, lists: map[string]*contentsSpec{ "tools": { modelled: contentKeys("id", "name", "description", "enabled", "toolType", "document", @@ -77,7 +78,9 @@ var knowledgeBaseContentsSpec = &contentsSpec{ } var consumedMCPServiceContentsSpec = &contentsSpec{ - modelled: contentKeys("protocolVersion", "documentation", "version", "connectionTimeoutSeconds"), + modelled: contentKeys("protocolVersion", "documentation", "version", "connectionTimeoutSeconds", + "endpoint", "authenticationMicroflow"), + objects: map[string]*contentsSpec{"endpoint": docRefSpec, "authenticationMicroflow": docRefSpec}, } var publishedMCPServiceContentsSpec = &contentsSpec{ diff --git a/mdl/backend/modelsdk/agenteditor_carry_test.go b/mdl/backend/modelsdk/agenteditor_carry_test.go index fd8135981..185c15e91 100644 --- a/mdl/backend/modelsdk/agenteditor_carry_test.go +++ b/mdl/backend/modelsdk/agenteditor_carry_test.go @@ -52,10 +52,16 @@ func typedAgent(t *testing.T, contents string) *agenteditor.Agent { // An `alter agent … set Temperature` rewrote Contents from the typed fields, // and the streaming switches, the selected model and each tool's approval and -// display title were gone. mxcli now models all of them but selectedModel; a -// rewrite keeps that one, and any key a later Agent Editor adds. +// display title were gone. mxcli models all of them now; a rewrite still keeps +// any key it does not model ("futureKey" stands in for what a later Agent +// Editor adds), at its stored position. func TestCarryStoredContents_AgentKeepsWhatItDoesNotModel(t *testing.T) { - stored := readContentsFixture(t, graphExplorerFixture) + stored := strings.Replace(readContentsFixture(t, graphExplorerFixture), + `"enableBackendStreaming": true,`, `"enableBackendStreaming": true, + "futureKey": "kept",`, 1) + if !strings.Contains(stored, "futureKey") { + t.Fatal("fixture changed; futureKey not injected") + } a := typedAgent(t, stored) temp := 0.3 a.Temperature = &temp @@ -64,9 +70,9 @@ func TestCarryStoredContents_AgentKeepsWhatItDoesNotModel(t *testing.T) { if err != nil { t.Fatal(err) } - // Control: the rebuild alone loses them, so the carry below is what keeps them. - if m := decodeMap(t, rebuilt); m["selectedModel"] != nil { - t.Fatalf("the encoder now writes the unmodelled keys itself; this test no longer covers the carry: %s", rebuilt) + // Control: the rebuild alone loses it, so the carry below is what keeps it. + if m := decodeMap(t, rebuilt); m["futureKey"] != nil { + t.Fatalf("the encoder writes futureKey itself; this test no longer covers the carry: %s", rebuilt) } got, err := carryStoredContents(stored, rebuilt, agentContentsSpec) @@ -84,6 +90,44 @@ func TestCarryStoredContents_AgentKeepsWhatItDoesNotModel(t *testing.T) { } } +// ako/mxcli#2031: Agent Editor 2.2+ refuses to import an agent without +// selectedModel ("Model with name is referenced but could not be found"), and +// mxcli could not write one. It is modelled now: the encoder writes it where +// Studio Pro does (after model, before entity), and clearing it removes it. +func TestEncodeAgentContents_SelectedModel(t *testing.T) { + stored := readContentsFixture(t, graphExplorerFixture) + a := typedAgent(t, stored) + if a.SelectedModel == nil || a.SelectedModel.ID != "eu.anthropic.claude-sonnet-5" || a.SelectedModel.Name != "Anthropic Claude Sonnet 5" { + t.Fatalf("selectedModel not read: %+v", a.SelectedModel) + } + rebuilt, err := encodeAgentContents(a) + if err != nil { + t.Fatal(err) + } + sm, _ := decodeMap(t, rebuilt)["selectedModel"].(map[string]any) + if sm["id"] != "eu.anthropic.claude-sonnet-5" || sm["name"] != "Anthropic Claude Sonnet 5" { + t.Errorf("selectedModel not written: %s", rebuilt) + } + keys := decodeObjectKeys(t, rebuilt) + pos := map[string]int{} + for i, k := range keys { + pos[k] = i + } + if !(pos["model"] < pos["selectedModel"] && pos["selectedModel"] < pos["entity"]) { + t.Errorf("selectedModel not between model and entity: %v", keys) + } + + a.SelectedModel = nil + rebuilt, _ = encodeAgentContents(a) + got, err := carryStoredContents(stored, rebuilt, agentContentsSpec) + if err != nil { + t.Fatal(err) + } + if _, ok := decodeMap(t, got)["selectedModel"]; ok { + t.Errorf("a cleared selectedModel came back from the stored contents: %s", got) + } +} + // A key on a knowledge base tool that mxcli does not model is kept, paired // with its stored entry by id. ("futureKey" stands in for what a later Agent // Editor adds: every key the fixture holds is modelled now.) diff --git a/mdl/backend/modelsdk/agenteditor_consumedmcp_test.go b/mdl/backend/modelsdk/agenteditor_consumedmcp_test.go new file mode 100644 index 000000000..a1395ada2 --- /dev/null +++ b/mdl/backend/modelsdk/agenteditor_consumedmcp_test.go @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/mendixlabs/mxcli/sdk/agenteditor" +) + +// Consumed MCP services saved by Studio Pro 11.16 (mx-test-projects +// FactoryManagement): GraphStudio_MCP has an endpoint constant and a +// credentials microflow, JiraMCP only an endpoint, stored first. +const ( + graphStudioMCPFixture = "testdata/agenteditor/FactoryManagement.GraphStudio_MCP.11_16.json" + jiraMCPFixture = "testdata/agenteditor/FactoryManagement.JiraMCP.11_16.json" +) + +func typedConsumedMCP(t *testing.T, contents string) *agenteditor.ConsumedMCPService { + t.Helper() + var p struct { + ProtocolVersion string `json:"protocolVersion"` + Documentation string `json:"documentation"` + Version string `json:"version"` + ConnectionTimeoutSeconds int `json:"connectionTimeoutSeconds"` + Endpoint *agenteditor.DocRef `json:"endpoint"` + AuthenticationMicroflow *agenteditor.DocRef `json:"authenticationMicroflow"` + } + if err := json.Unmarshal([]byte(contents), &p); err != nil { + t.Fatal(err) + } + return &agenteditor.ConsumedMCPService{ProtocolVersion: p.ProtocolVersion, InnerDocumentation: p.Documentation, + Version: p.Version, ConnectionTimeoutSeconds: p.ConnectionTimeoutSeconds, + Endpoint: p.Endpoint, AuthenticationMicroflow: p.AuthenticationMicroflow} +} + +// ako/mxcli#2060: mxcli wrote no endpoint and no credentials microflow, so an +// mxcli-written service connected nowhere (AgentEditorCommons imports it only +// when the endpoint constant has a value). Both are modelled now; an unchanged +// service rebuilds byte for byte in either stored key order. +func TestEncodeConsumedMCPServiceContents_EndpointAndCredentials(t *testing.T) { + for _, path := range []string{graphStudioMCPFixture, jiraMCPFixture} { + stored := readContentsFixture(t, path) + c := typedConsumedMCP(t, stored) + rebuilt, err := encodeConsumedMCPServiceContents(c) + if err != nil { + t.Fatal(err) + } + if m := decodeMap(t, rebuilt); m["endpoint"] == nil { + t.Errorf("%s: the encoder does not write endpoint: %s", path, rebuilt) + } + got, err := carryStoredContents(stored, rebuilt, consumedMCPServiceContentsSpec) + if err != nil { + t.Fatal(err) + } + if got != stored { + t.Errorf("%s: an unchanged service is rewritten:\n got %s\nwant %s", path, got, stored) + } + } + + // A new service, with no stored document: endpoint first, the credentials + // microflow last, each reference documentId first, as Agent Editor saves them. + c := &agenteditor.ConsumedMCPService{ProtocolVersion: "v2025_03_26", Version: "1.0.0", + Endpoint: &agenteditor.DocRef{DocumentID: "c1", QualifiedName: "M.Url"}, + AuthenticationMicroflow: &agenteditor.DocRef{DocumentID: "m1", QualifiedName: "M.GetCredentials"}} + fresh, err := encodeConsumedMCPServiceContents(c) + if err != nil { + t.Fatal(err) + } + keys := decodeObjectKeys(t, fresh) + if keys[0] != "endpoint" || keys[len(keys)-1] != "authenticationMicroflow" { + t.Errorf("new service key order %v", keys) + } + ep, _ := decodeObject(json.RawMessage(rawKey(t, fresh, "endpoint"))) + if !reflect.DeepEqual(ep.keys, []string{"documentId", "qualifiedName"}) { + t.Errorf("endpoint key order %v", ep.keys) + } + + // Clearing the credentials microflow removes it from a stored service. + stored := readContentsFixture(t, graphStudioMCPFixture) + c = typedConsumedMCP(t, stored) + c.AuthenticationMicroflow = nil + rebuilt, _ := encodeConsumedMCPServiceContents(c) + got, err := carryStoredContents(stored, rebuilt, consumedMCPServiceContentsSpec) + if err != nil { + t.Fatal(err) + } + if _, ok := decodeMap(t, got)["authenticationMicroflow"]; ok { + t.Errorf("a cleared credentials microflow came back from the stored contents: %s", got) + } +} + +func rawKey(t *testing.T, s, key string) string { + t.Helper() + var m map[string]json.RawMessage + if err := json.Unmarshal([]byte(s), &m); err != nil { + t.Fatal(err) + } + return string(m[key]) +} diff --git a/mdl/backend/modelsdk/agenteditor_read.go b/mdl/backend/modelsdk/agenteditor_read.go index 38244601c..6c90976af 100644 --- a/mdl/backend/modelsdk/agenteditor_read.go +++ b/mdl/backend/modelsdk/agenteditor_read.go @@ -179,14 +179,18 @@ func (b *Backend) ListAgentEditorConsumedMCPServices() ([]*agenteditor.ConsumedM c.ExportLevel = u.wrap.ExportLevel if u.wrap.Contents != "" { var p struct { - ProtocolVersion string `json:"protocolVersion"` - Documentation string `json:"documentation"` - Version string `json:"version"` - ConnectionTimeoutSeconds int `json:"connectionTimeoutSeconds"` + ProtocolVersion string `json:"protocolVersion"` + Documentation string `json:"documentation"` + Version string `json:"version"` + ConnectionTimeoutSeconds int `json:"connectionTimeoutSeconds"` + Endpoint *agenteditor.DocRef `json:"endpoint"` + AuthenticationMicroflow *agenteditor.DocRef `json:"authenticationMicroflow"` } if err := json.Unmarshal([]byte(u.wrap.Contents), &p); err != nil { return nil, fmt.Errorf("unmarshal ConsumedMCPService Contents: %w", err) } + c.Endpoint = p.Endpoint + c.AuthenticationMicroflow = p.AuthenticationMicroflow c.ProtocolVersion = p.ProtocolVersion c.InnerDocumentation = p.Documentation c.Version = p.Version @@ -268,21 +272,22 @@ func (b *Backend) ListAgentEditorAgents() ([]*agenteditor.Agent, error) { a.ExportLevel = u.wrap.ExportLevel if u.wrap.Contents != "" { var p struct { - Description string `json:"description"` - SystemPrompt string `json:"systemPrompt"` - UserPrompt string `json:"userPrompt"` - UsageType string `json:"usageType"` - Variables []agenteditor.AgentVar `json:"variables"` - Tools []agenteditor.AgentTool `json:"tools"` - KnowledgebaseTools []agenteditor.AgentKBTool `json:"knowledgebaseTools"` - Model *agenteditor.DocRef `json:"model"` - Entity *agenteditor.DocRef `json:"entity"` - MaxTokens *int `json:"maxTokens"` - ToolChoice string `json:"toolChoice"` - Temperature *float64 `json:"temperature"` - TopP *float64 `json:"topP"` - EnableBackend *bool `json:"enableBackendStreaming"` - EnableUI *bool `json:"enableUIStreaming"` + Description string `json:"description"` + SystemPrompt string `json:"systemPrompt"` + UserPrompt string `json:"userPrompt"` + UsageType string `json:"usageType"` + Variables []agenteditor.AgentVar `json:"variables"` + Tools []agenteditor.AgentTool `json:"tools"` + KnowledgebaseTools []agenteditor.AgentKBTool `json:"knowledgebaseTools"` + Model *agenteditor.DocRef `json:"model"` + Entity *agenteditor.DocRef `json:"entity"` + MaxTokens *int `json:"maxTokens"` + ToolChoice string `json:"toolChoice"` + Temperature *float64 `json:"temperature"` + TopP *float64 `json:"topP"` + EnableBackend *bool `json:"enableBackendStreaming"` + EnableUI *bool `json:"enableUIStreaming"` + SelectedModel *agenteditor.SelectedModel `json:"selectedModel"` } if err := json.Unmarshal([]byte(u.wrap.Contents), &p); err != nil { return nil, fmt.Errorf("unmarshal Agent Contents: %w", err) @@ -302,6 +307,7 @@ func (b *Backend) ListAgentEditorAgents() ([]*agenteditor.Agent, error) { a.TopP = p.TopP a.EnableBackendStreaming = p.EnableBackend a.EnableUIStreaming = p.EnableUI + a.SelectedModel = p.SelectedModel } out = append(out, a) } diff --git a/mdl/backend/modelsdk/agenteditor_write.go b/mdl/backend/modelsdk/agenteditor_write.go index a3e2e4f2f..00690b90d 100644 --- a/mdl/backend/modelsdk/agenteditor_write.go +++ b/mdl/backend/modelsdk/agenteditor_write.go @@ -382,14 +382,21 @@ func encodeKnowledgeBaseContents(k *agenteditor.KnowledgeBase) (string, error) { }) } +// encodeConsumedMCPServiceContents writes endpoint first and the credentials +// microflow last, as most Studio Pro 11.16 documents hold them (FactoryManagement); +// a stored document keeps its own order through the carry. func encodeConsumedMCPServiceContents(c *agenteditor.ConsumedMCPService) (string, error) { type contentsShape struct { - ProtocolVersion string `json:"protocolVersion"` - Documentation string `json:"documentation"` - Version string `json:"version"` - ConnectionTimeoutSeconds int `json:"connectionTimeoutSeconds"` + Endpoint *agenteditor.DocRef `json:"endpoint,omitempty"` + ProtocolVersion string `json:"protocolVersion"` + Documentation string `json:"documentation"` + Version string `json:"version"` + ConnectionTimeoutSeconds int `json:"connectionTimeoutSeconds"` + AuthenticationMicroflow *agenteditor.DocRef `json:"authenticationMicroflow,omitempty"` } return marshalJSON(contentsShape{ + Endpoint: c.Endpoint, + AuthenticationMicroflow: c.AuthenticationMicroflow, ProtocolVersion: c.ProtocolVersion, Documentation: c.InnerDocumentation, Version: c.Version, @@ -486,13 +493,15 @@ func encodeAgentContents(a *agenteditor.Agent) (string, error) { Tools []toolEntry `json:"tools"` KnowledgebaseTools []kbToolEntry `json:"knowledgebaseTools"` Model *agenteditor.DocRef `json:"model,omitempty"` - Entity *agenteditor.DocRef `json:"entity,omitempty"` - MaxTokens *int `json:"maxTokens,omitempty"` - ToolChoice string `json:"toolChoice,omitempty"` - Temperature *float64 `json:"temperature,omitempty"` - TopP *float64 `json:"topP,omitempty"` - EnableBackend *bool `json:"enableBackendStreaming,omitempty"` - EnableUI *bool `json:"enableUIStreaming,omitempty"` + // After model, before entity: where Studio Pro puts it. + SelectedModel *agenteditor.SelectedModel `json:"selectedModel,omitempty"` + Entity *agenteditor.DocRef `json:"entity,omitempty"` + MaxTokens *int `json:"maxTokens,omitempty"` + ToolChoice string `json:"toolChoice,omitempty"` + Temperature *float64 `json:"temperature,omitempty"` + TopP *float64 `json:"topP,omitempty"` + EnableBackend *bool `json:"enableBackendStreaming,omitempty"` + EnableUI *bool `json:"enableUIStreaming,omitempty"` } tools := make([]toolEntry, 0, len(a.Tools)) @@ -518,6 +527,7 @@ func encodeAgentContents(a *agenteditor.Agent) (string, error) { Tools: tools, KnowledgebaseTools: kbTools, Model: a.Model, + SelectedModel: a.SelectedModel, Entity: a.Entity, MaxTokens: a.MaxTokens, ToolChoice: a.ToolChoice, diff --git a/mdl/backend/modelsdk/infrastructure_write.go b/mdl/backend/modelsdk/infrastructure_write.go index de77624d4..d20c2c270 100644 --- a/mdl/backend/modelsdk/infrastructure_write.go +++ b/mdl/backend/modelsdk/infrastructure_write.go @@ -26,6 +26,23 @@ func (b *Backend) UpdateQualifiedNameInAllUnits(oldName, newName string) (int, e if err != nil { return 0, fmt.Errorf("UpdateQualifiedNameInAllUnits: list units: %w", err) } + // A path step — `[Src.Assoc/Src.Entity/Attr = 1]` in an access rule, + // `$O/Src.Assoc/Name` in an expression — names an entity or an association, + // and only those: an enumeration renamed must not rewrite an association + // that shares its name. By the time a move or rename sweeps, the element + // already carries newName, so that is what decides (ako/mxcli#1836). + paths := b.isDomainElementName(newName) + rewrite := func(s string) (any, bool) { + if v, ok := replaceQualifiedNameRef(s, oldName, newName); ok { + return v, true + } + if paths { + if v, n := types.RewritePathSteps(s, oldName, newName); n > 0 { + return v, true + } + } + return s, false + } updated := 0 for _, id := range ids { raw, err := b.reader.GetRawUnitBytes(id) @@ -36,7 +53,7 @@ func (b *Backend) UpdateQualifiedNameInAllUnits(oldName, newName string) (int, e if err := bson.Unmarshal(raw, &doc); err != nil { continue } - if replaced, ok := replaceQNInValue(doc, oldName, newName); ok { + if replaced, ok := replaceQNInValueWith(doc, rewrite); ok { contents, err := bson.Marshal(replaced) if err != nil { return updated, fmt.Errorf("UpdateQualifiedNameInAllUnits: marshal %s: %w", id, err) @@ -253,16 +270,59 @@ func replaceQNInValueCounted(v any, oldName, newName string, count *int) any { return v } -// replaceQNInValue recursively rewrites qualified-name references in a decoded -// BSON value, returning the (possibly mutated) value and whether anything changed. -func replaceQNInValue(v any, oldName, newName string) (any, bool) { +// isDomainElementName reports whether qn names an entity, an association or a +// cross-module association in the project. +func (b *Backend) isDomainElementName(qn string) bool { + mod, name, ok := strings.Cut(qn, ".") + if !ok || name == "" || strings.Contains(name, ".") { + return false + } + modules, err := b.ListModules() + if err != nil { + return false + } + var modID model.ID + for _, m := range modules { + if m.Name == mod { + modID = m.ID + } + } + if modID == "" { + return false + } + dm, err := b.GetDomainModel(modID) + if err != nil || dm == nil { + return false + } + for _, e := range dm.Entities { + if e.Name == name { + return true + } + } + for _, a := range dm.Associations { + if a.Name == name { + return true + } + } + for _, a := range dm.CrossAssociations { + if a.Name == name { + return true + } + } + return false +} + +// replaceQNInValueWith recursively applies rewrite to every string in a decoded +// BSON value, returning the (possibly mutated) value and whether anything +// changed. +func replaceQNInValueWith(v any, rewrite func(string) (any, bool)) (any, bool) { switch val := v.(type) { case string: - return replaceQualifiedNameRef(val, oldName, newName) - case bson.D: // also matches bson.D (alias) + return rewrite(val) + case bson.D: changed := false for i, elem := range val { - if nv, ok := replaceQNInValue(elem.Value, oldName, newName); ok { + if nv, ok := replaceQNInValueWith(elem.Value, rewrite); ok { val[i].Value = nv changed = true } @@ -271,7 +331,7 @@ func replaceQNInValue(v any, oldName, newName string) (any, bool) { case bson.A: changed := false for i, elem := range val { - if nv, ok := replaceQNInValue(elem, oldName, newName); ok { + if nv, ok := replaceQNInValueWith(elem, rewrite); ok { val[i] = nv changed = true } @@ -280,7 +340,7 @@ func replaceQNInValue(v any, oldName, newName string) (any, bool) { case []any: changed := false for i, elem := range val { - if nv, ok := replaceQNInValue(elem, oldName, newName); ok { + if nv, ok := replaceQNInValueWith(elem, rewrite); ok { val[i] = nv changed = true } diff --git a/mdl/backend/modelsdk/placement.go b/mdl/backend/modelsdk/placement.go index f8b080183..f40901660 100644 --- a/mdl/backend/modelsdk/placement.go +++ b/mdl/backend/modelsdk/placement.go @@ -31,6 +31,20 @@ func (b *Backend) MoveDocument(unitID, containerID model.ID) error { // table — restricting to the document containment is what keeps a folder named // like a document from being returned as one. func (b *Backend) FindDocumentUnit(moduleName, name string) (*types.DocumentUnit, error) { + docs, err := b.findDocumentUnits(moduleName, name, true) + if err != nil || len(docs) == 0 { + return nil, err + } + return docs[0], nil +} + +// FindDocumentUnits returns every document named name in moduleName, of any +// type. See backend.DocumentPlacementBackend. +func (b *Backend) FindDocumentUnits(moduleName, name string) ([]*types.DocumentUnit, error) { + return b.findDocumentUnits(moduleName, name, false) +} + +func (b *Backend) findDocumentUnits(moduleName, name string, firstOnly bool) ([]*types.DocumentUnit, error) { if b.reader == nil { return nil, fmt.Errorf("FindDocumentUnit: not connected") } @@ -50,13 +64,13 @@ func (b *Backend) FindDocumentUnit(moduleName, name string) (*types.DocumentUnit } containers := b.containerSetForModule(moduleID) - var found *types.DocumentUnit + var found []*types.DocumentUnit err = b.eachDocumentUnit(func(doc *types.DocumentUnit) bool { if doc.Name != name || !containers[string(doc.ContainerID)] { return true } - found = doc - return false + found = append(found, doc) + return !firstOnly }) if err != nil { return nil, fmt.Errorf("FindDocumentUnit: %w", err) diff --git a/mdl/backend/modelsdk/testdata/agenteditor/FactoryManagement.GraphStudio_MCP.11_16.json b/mdl/backend/modelsdk/testdata/agenteditor/FactoryManagement.GraphStudio_MCP.11_16.json new file mode 100644 index 000000000..dbe874eae --- /dev/null +++ b/mdl/backend/modelsdk/testdata/agenteditor/FactoryManagement.GraphStudio_MCP.11_16.json @@ -0,0 +1 @@ +{"protocolVersion":"v2025_03_26","endpoint":{"documentId":"7caf58c6-e15e-4105-93a7-5ba162cde0c7","qualifiedName":"AgenticAssistants.GraphStudioMCP_Endpoint"},"authenticationMicroflow":{"documentId":"84e3058c-0b5f-450d-9e9a-1ac946535967","qualifiedName":"GraphStudioIntegration.GraphAuth"}} diff --git a/mdl/backend/modelsdk/testdata/agenteditor/FactoryManagement.JiraMCP.11_16.json b/mdl/backend/modelsdk/testdata/agenteditor/FactoryManagement.JiraMCP.11_16.json new file mode 100644 index 000000000..74cfbe8bb --- /dev/null +++ b/mdl/backend/modelsdk/testdata/agenteditor/FactoryManagement.JiraMCP.11_16.json @@ -0,0 +1 @@ +{"endpoint":{"documentId":"4b2a5029-49cb-4f7d-ae9e-42b0e8f639b0","qualifiedName":"AgentLibrary.JiraMCP"},"protocolVersion":"v2025_03_26"} diff --git a/mdl/backend/modelsdk/unimplemented_gen.go b/mdl/backend/modelsdk/unimplemented_gen.go index afd4a0b5f..3bf8f88c1 100644 --- a/mdl/backend/modelsdk/unimplemented_gen.go +++ b/mdl/backend/modelsdk/unimplemented_gen.go @@ -442,6 +442,11 @@ func (unimplemented) FindDocumentUnit(_ string, _ string) (*types.DocumentUnit, return r0, errUnimplemented("FindDocumentUnit") } +func (unimplemented) FindDocumentUnits(_ string, _ string) ([]*types.DocumentUnit, error) { + var r0 []*types.DocumentUnit + return r0, errUnimplemented("FindDocumentUnits") +} + func (unimplemented) FindViewEntitySourceDocumentID(_ string, _ string) (model.ID, error) { var r0 model.ID return r0, errUnimplemented("FindViewEntitySourceDocumentID") diff --git a/mdl/backend/placement.go b/mdl/backend/placement.go index 1f80dc6ab..790ce4466 100644 --- a/mdl/backend/placement.go +++ b/mdl/backend/placement.go @@ -40,6 +40,13 @@ type DocumentPlacementBackend interface { // there is no such document. FindDocumentUnit(moduleName, name string) (*types.DocumentUnit, error) + // FindDocumentUnits returns every top-level document named name inside + // moduleName. A name is unique per document type, not per module: a + // microflow and a scheduled event may share one (mendixlabs/mxcli#1792), + // so a caller that knows the kind it wants picks it from these rather + // than take whichever FindDocumentUnit meets first. + FindDocumentUnits(moduleName, name string) ([]*types.DocumentUnit, error) + // ListDocumentUnits returns every top-level document in the project with // its container, name and type. // diff --git a/mdl/catalog/builder.go b/mdl/catalog/builder.go index ef0418360..4e7b75b25 100644 --- a/mdl/catalog/builder.go +++ b/mdl/catalog/builder.go @@ -138,6 +138,10 @@ type Builder struct { // and emitted by buildReferences (a later pass). Carried on the Builder // rather than re-queried because CatalogTx has no Query. scheduledEventRefs []scheduledEventRef + // agentDocRefs are the decoded agent-editor documents, whose edges + // buildReferences writes once every table their names resolve against is + // filled (ako/mxcli#2074). + agentDocRefs []agentDocRefs // Entity → regular-expression edges from attribute validation rules, // collected while cataloguing regexes and emitted by buildReferences. @@ -219,6 +223,9 @@ type hierarchy struct { moduleNames map[model.ID]string containerParent map[model.ID]model.ID folderNames map[model.ID]string + // unitTypes is each unit's $Type, so the folders pass can count the + // documents a folder holds directly (ako/mxcli#1436). + unitTypes map[model.ID]string } func (b *Builder) buildHierarchy() error { @@ -227,6 +234,7 @@ func (b *Builder) buildHierarchy() error { moduleNames: make(map[model.ID]string), containerParent: make(map[model.ID]model.ID), folderNames: make(map[model.ID]string), + unitTypes: make(map[model.ID]string), } // Load modules @@ -243,6 +251,7 @@ func (b *Builder) buildHierarchy() error { units, _ := b.reader.ListUnits() for _, u := range units { h.containerParent[u.ID] = u.ContainerID + h.unitTypes[u.ID] = u.Type } // Load folders @@ -437,7 +446,9 @@ type buildPhase struct { func (b *Builder) phases() []buildPhase { return []buildPhase{ {"modules", b.buildModules}, + {"folders", b.buildFolders}, {"entities", b.buildEntities}, + {"domain model annotations", b.buildDomainModelAnnotations}, {"entity event handlers", b.buildEntityEventHandlers}, {"associations", b.buildAssociations}, {"enumerations", b.buildEnumerations}, @@ -740,6 +751,7 @@ func (b *Builder) resetCollected() { b.databaseConnectionCache = nil b.moduleSettingsCache = nil b.scheduledEventRefs = nil + b.agentDocRefs = nil b.regexRuleRefs = nil b.publishedRestRefs = nil b.eventHandlerRefs = nil diff --git a/mdl/catalog/builder_agent_details_test.go b/mdl/catalog/builder_agent_details_test.go new file mode 100644 index 000000000..98bdacd21 --- /dev/null +++ b/mdl/catalog/builder_agent_details_test.go @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "database/sql" + "os" + "strings" + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/modelsdk/mprread" +) + +// ako/mxcli#2075: the catalog listed agent-editor documents by name only, so +// "which agents can run a refund tool without the user's approval" took one +// `describe agent` per agent and text parsing. The Contents of real Agent +// Editor documents (the 11.16 fixtures the writer's carry tests use) are the +// input here. + +const agentDetailsModuleID = model.ID("mod-agents") + +func agentEditorFixtureContents(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile("../backend/modelsdk/testdata/agenteditor/" + name) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func agentEditorRawUnit(t *testing.T, id, name, docType, contents string) *types.RawUnit { + t.Helper() + b, err := bson.Marshal(bson.D{ + {Key: "$Type", Value: "CustomBlobDocuments$CustomBlobDocument"}, + {Key: "Name", Value: name}, + {Key: "Documentation", Value: ""}, + {Key: "CustomDocumentType", Value: docType}, + {Key: "Contents", Value: contents}, + }) + if err != nil { + t.Fatal(err) + } + return &types.RawUnit{ID: model.ID(id), ContainerID: agentDetailsModuleID, + Type: "CustomBlobDocuments$CustomBlobDocument", Contents: b} +} + +func agentDetailsFixture(t *testing.T) *Catalog { + t.Helper() + cat, err := New() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { cat.Close() }) + tx, err := cat.db.Begin() + if err != nil { + t.Fatal(err) + } + units := []*types.RawUnit{ + // Agent Editor 2.2+: streaming switches, selected model, a microflow tool. + agentEditorRawUnit(t, "ag-1", "GraphExplorer", types.CustomTypeAgent, + agentEditorFixtureContents(t, "ICXGraphPilot.GraphExplorer.11_16.json")), + // No streaming keys stored, one knowledge base tool. + agentEditorRawUnit(t, "ag-2", "WorkInstructionAgent", types.CustomTypeAgent, + agentEditorFixtureContents(t, "FactoryManagement.WorkInstructionAgent.11_16.json")), + // An MCP tool and an approval-gated microflow tool. + agentEditorRawUnit(t, "ag-3", "RefundAgent", types.CustomTypeAgent, `{"usageType":"Task", + "tools":[ + {"id":"t1","name":"IssueRefund","toolType":"Microflow","enabled":true, + "userAccessApproval":"UserConfirmationRequired", + "document":{"qualifiedName":"Agents.Tool_IssueRefund"}}, + {"id":"t2","name":"Jira","toolType":"MCP","enabled":false, + "document":{"qualifiedName":"Agents.JiraMCP"}}], + "toolChoice":"Auto","maxTokens":2048,"temperature":0.2}`), + agentEditorRawUnit(t, "kb-1", "KB_WorkInstructions", types.CustomTypeKnowledgeBase, + agentEditorFixtureContents(t, "FactoryManagement.KB_WorkInstructions.11_16.json")), + agentEditorRawUnit(t, "m-1", "Gpt", types.CustomTypeModel, `{"type":"Text","name":"gpt","displayName":"GPT", + "provider":"MxCloudGenAI","providerFields":{"key":{"qualifiedName":"Agents.GptKey"}}}`), + agentEditorRawUnit(t, "mcp-1", "JiraMCP", types.CustomTypeConsumedMCPService, + `{"protocolVersion":"v2025_03_26","version":"1.0","connectionTimeoutSeconds":30}`), + agentEditorRawUnit(t, "pub-1", "Published_MCP_service", types.CustomTypePublishedMCPService, + agentEditorFixtureContents(t, "AgentBuilderStarterApp.Published_MCP_service.11_16.json")), + // Contents that do not decode: the row stays, without details. + agentEditorRawUnit(t, "ag-bad", "Broken", types.CustomTypeAgent, `{not json`), + } + b := &Builder{ + catalog: cat, + reader: &mock.MockBackend{ + ListRawUnitsByTypeFunc: func(prefix string) ([]*types.RawUnit, error) { + return units, nil + }, + }, + snapshot: &Snapshot{ID: "snap-1"}, + hierarchy: &hierarchy{ + moduleIDs: map[model.ID]bool{agentDetailsModuleID: true}, + moduleNames: map[model.ID]string{agentDetailsModuleID: "Agents"}, + containerParent: map[model.ID]model.ID{}, + folderNames: map[model.ID]string{}, + }, + tx: wrapTx(tx), + fullMode: true, + } + if err := b.buildAgentEditorDocs(); err != nil { + t.Fatalf("buildAgentEditorDocs: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + return cat +} + +// agentQueryRows renders every row of q as "a|b|c", NULL as "∅". +func agentQueryRows(t *testing.T, cat *Catalog, q string) []string { + t.Helper() + rows, err := cat.db.Query(q) + if err != nil { + t.Fatalf("%s: %v", q, err) + } + defer rows.Close() + cols, _ := rows.Columns() + var out []string + for rows.Next() { + vals := make([]sql.NullString, len(cols)) + ptrs := make([]any, len(cols)) + for i := range vals { + ptrs[i] = &vals[i] + } + if err := rows.Scan(ptrs...); err != nil { + t.Fatal(err) + } + var parts []string + for _, v := range vals { + if !v.Valid { + parts = append(parts, "∅") + } else { + parts = append(parts, v.String) + } + } + out = append(out, strings.Join(parts, "|")) + } + return out +} + +func assertAgentRows(t *testing.T, cat *Catalog, q string, want ...string) { + t.Helper() + got := agentQueryRows(t, cat, q) + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Errorf("%s\ngot:\n %s\nwant:\n %s", q, strings.Join(got, "\n "), strings.Join(want, "\n ")) + } +} + +func TestAgentDetails_Issue2075(t *testing.T) { + cat := agentDetailsFixture(t) + + assertAgentRows(t, cat, `SELECT Name, UsageType, Model, SelectedModel, ToolChoice, MaxTokens, Temperature, + EnableBackendStreaming, EnableUIStreaming, Entity, Variables, ToolCount, KnowledgeBaseCount, MCPServiceCount + FROM agents ORDER BY Name`, + // The row survives undecodable Contents, with no details. + "Broken|∅|∅|∅|∅|∅|∅|∅|∅|∅|∅|0|0|0", + "GraphExplorer|Chat|GraphPilot_Explorer.LLM_GenAIResource|eu.anthropic.claude-sonnet-5||∅|∅|1|0|GraphPilot_Explorer.OnboardingHelper|EnvironmentUrl,GraphmartUri,GeneralInfo,OntologyOverview,Questions|1|0|0", + "RefundAgent|Task||∅|Auto|2048|0.2|∅|∅|||1|0|1", + // Control: an agent that stores no streaming switch reads NULL, not "off". + "WorkInstructionAgent|Task|AgenticAssistants.ClaudeSonnet46|∅||∅|∅|∅|∅|AgenticAssistants.GenericAgentInterface|Question|0|1|0", + ) + + // The issue's query: which tools can run without the user's approval. + assertAgentRows(t, cat, `SELECT Agent, Name, ToolType, Document, Enabled, UserAccessApproval, IFNULL(DisplayTitle, '∅') + FROM agent_tools ORDER BY Agent, Name`, + "Agents.GraphExplorer|Tool_ExceuteSparql|Microflow|GraphPilot_Explorer.Tool_ExecuteSparql|1|VisibleForUser|Tool: Execute Sparql", + "Agents.RefundAgent|IssueRefund|Microflow|Agents.Tool_IssueRefund|1|UserConfirmationRequired|∅", + "Agents.RefundAgent|Jira|MCP|Agents.JiraMCP|0||∅", + ) + assertAgentRows(t, cat, `SELECT Agent, Service, Enabled FROM agent_mcp_services`, + "Agents.RefundAgent|Agents.JiraMCP|0") + assertAgentRows(t, cat, `SELECT Agent, Name, KnowledgeBase, Collection, MaxResults, MinSimilarity, IsVisible, Enabled + FROM agent_knowledge_bases`, + "Agents.WorkInstructionAgent|WorkInstructionKnowledgeBase|AgenticAssistants.KB_WorkInstructions|PortalManagedContent|5|0.2|∅|1") + + assertAgentRows(t, cat, `SELECT Name, Provider, ModelType, DisplayName, KeyConstant FROM ai_models`, + "Gpt|MxCloudGenAI|Text|GPT|Agents.GptKey") + assertAgentRows(t, cat, `SELECT Name, Provider, KeyConstant FROM knowledge_bases`, + "KB_WorkInstructions|MxCloudGenAI|AgenticAssistants.KB_WorkInstructionsKey") + assertAgentRows(t, cat, `SELECT Name, ProtocolVersion, Version, ConnectionTimeoutSeconds FROM consumed_mcp_services`, + "JiraMCP|v2025_03_26|1.0|30") + assertAgentRows(t, cat, `SELECT Name, ServerName, EndpointPath, Version, ProtocolVersion, AuthenticationEnabled, AuthenticationMicroflow, ToolCount + FROM published_mcp_services`, + "Published_MCP_service|Published_MCP_service|published_mcp_service|1.0.0|v2025_03_26|0||1") + assertAgentRows(t, cat, `SELECT ObjectType, QualifiedName FROM objects WHERE ObjectType = 'PUBLISHED_MCP_SERVICE'`, + "PUBLISHED_MCP_SERVICE|Agents.Published_MCP_service") +} + +// A published MCP service runs each tool's microflow and its authentication +// microflow, an entry point as a published REST operation is (`publish`): +// without the edge GRAPH_DEAD_ASSETS lists them. +func TestPublishedMCPServiceRefs_Issue2075(t *testing.T) { + u := agentEditorRawUnit(t, "pub-1", "Published_MCP_service", types.CustomTypePublishedMCPService, + `{"authenticationEnabled":true,"authenticationMicroflow":{"qualifiedName":"Agents.MCP_Auth"}, + "tools":[{"uuid":"u1","name":"tool1","sourceMicroflow":{"qualifiedName":"AgentTest.mcptool1"}}, + {"uuid":"u2","name":"unset"}]}`) + doc, err := mprread.DecodeAgentEditorUnit("pub-1", string(agentDetailsModuleID), u.Contents) + if err != nil { + t.Fatal(err) + } + if got := agentEditorSourceType(doc); got != RefObjectPublishedMCPService { + t.Errorf("source type = %q", got) + } + want := "publish MICROFLOW AgentTest.mcptool1|publish MICROFLOW Agents.MCP_Auth" + if got := strings.Join(edgeStrings(agentEditorRefs(doc, resolveFixture)), "|"); got != want { + t.Errorf("published MCP service edges = %q, want %q", got, want) + } +} diff --git a/mdl/catalog/builder_agent_refs.go b/mdl/catalog/builder_agent_refs.go new file mode 100644 index 000000000..9617f4214 --- /dev/null +++ b/mdl/catalog/builder_agent_refs.go @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// Edges from agent-editor documents and from code action parameters +// (ako/mxcli#2074). Neither produced a refs row, so an agent's tool microflow, +// a chat context's action microflow and an MCP server's authentication +// microflow read as dead code: `show callers` said "(no callers found)" and +// GRAPH_DEAD_ASSETS listed them, though deleting one breaks the app at +// runtime, where neither mx check nor a build would notice. + +// docRef is one edge out of a document: what it names and how. +type docRef struct { + targetType string + target string + kind string +} + +// agentDocRefs is a decoded agent-editor document waiting for buildReferences, +// which resolves its names once every document table is filled. +type agentDocRefs struct { + owner string + sourceType string + source string + moduleName string + doc any +} + +// agentEditorSourceType is the refs.SourceType of a decoded agent-editor +// document, the name the objects view gives it. +func agentEditorSourceType(doc any) string { + switch doc.(type) { + case *types.Agent: + return RefObjectAgent + case *types.Model: + return RefObjectAIModel + case *types.KnowledgeBase: + return RefObjectKnowledgeBase + case *types.ConsumedMCPService: + return RefObjectConsumedMCPService + case *types.PublishedMCPService: + return RefObjectPublishedMCPService + } + return "" +} + +// agentEditorRefs lists the edges out of one decoded agent-editor document. +// resolve gives the object type the catalog holds a qualified name under, "" +// when it holds none. +// +// - an agent calls its tools: a microflow, a nanoflow or another agent is a +// `call`, so `show callers` and the dead-asset view see it; a consumed MCP +// service or a knowledge base it draws on is a `tool`; +// - an agent runs on its `model`, and is typed on its context entity (`type`); +// - a model or knowledge base reads its key from a constant (`key`); +// - a published MCP service runs each tool's microflow and its +// authentication microflow (`publish`, as a published REST or OData +// service does; ako/mxcli#2075). +func agentEditorRefs(doc any, resolve func(string) string) []docRef { + var out []docRef + add := func(qn, targetType, kind string) { + if qn != "" && targetType != "" { + out = append(out, docRef{targetType: targetType, target: qn, kind: kind}) + } + } + switch d := doc.(type) { + case *types.Agent: + for _, tool := range d.Tools { + if tool.Document == nil || tool.Document.QualifiedName == "" { + continue + } + qn := tool.Document.QualifiedName + targetType := resolve(qn) + if targetType == "" { + // The document is not in the catalog (deleted, or in a module + // the build skipped): type it from the tool, as a call to a + // missing microflow is still recorded. + switch tool.ToolType { + case "MCP": + targetType = RefObjectConsumedMCPService + default: + targetType = RefObjectMicroflow + } + } + kind := RefKindCall + if targetType == RefObjectConsumedMCPService || targetType == RefObjectKnowledgeBase { + kind = RefKindTool + } + add(qn, targetType, kind) + } + for _, kb := range d.KBTools { + if kb.Document != nil { + add(kb.Document.QualifiedName, RefObjectKnowledgeBase, RefKindTool) + } + } + if d.Model != nil { + add(d.Model.QualifiedName, RefObjectAIModel, RefKindModel) + } + if d.Entity != nil { + add(d.Entity.QualifiedName, RefObjectEntity, RefKindType) + } + case *types.Model: + if d.Key != nil { + add(d.Key.QualifiedName, RefObjectConstant, RefKindKey) + } + case *types.KnowledgeBase: + if d.Key != nil { + add(d.Key.QualifiedName, RefObjectConstant, RefKindKey) + } + case *types.PublishedMCPService: + for _, tool := range d.Tools { + if tool.SourceMicroflow != nil { + add(tool.SourceMicroflow.QualifiedName, RefObjectMicroflow, RefKindPublish) + } + } + if d.AuthenticationMicroflow != nil { + add(d.AuthenticationMicroflow.QualifiedName, RefObjectMicroflow, RefKindPublish) + } + } + return out +} + +// codeActionParameterRefs lists the documents a Java or JavaScript action call +// passes by reference: a microflow for a parameter of type Microflow, and a +// custom document (an agent, model, knowledge base or MCP service) for one of +// a custom document type. Each is a `call`: the action runs it, by name, at +// runtime. An expression argument is not a reference, even when its text is +// a microflow's name. +func codeActionParameterRefs(action microflows.MicroflowAction, resolve func(string) string) []docRef { + var values []microflows.CodeActionParameterValue + switch a := action.(type) { + case *microflows.JavaActionCallAction: + for _, m := range a.ParameterMappings { + if m != nil { + values = append(values, m.Value) + } + } + case *microflows.JavaScriptActionCallAction: + for _, m := range a.ParameterMappings { + if m != nil { + values = append(values, m.Value) + } + } + } + var out []docRef + for _, v := range values { + switch p := v.(type) { + case *microflows.MicroflowParameterValue: + if p.Microflow != "" { + out = append(out, docRef{RefObjectMicroflow, p.Microflow, RefKindCall}) + } + case *microflows.CustomBlobDocumentCodeActionParameterValue: + if p.CustomDocument == "" { + continue + } + targetType := resolve(p.CustomDocument) + if targetType == "" { + targetType = RefObjectAgent + } + out = append(out, docRef{targetType, p.CustomDocument, RefKindCall}) + } + } + return out +} + +// agentEditorResolver maps the qualified name of every document an agent or +// a code action can name to its object type, from the tables this build has +// filled. +func (b *Builder) agentEditorResolver() func(string) string { + known := map[string]string{} + for table, objType := range map[string]string{ + "microflows_data": "", // MicroflowType says which + "agents_data": RefObjectAgent, + "ai_models_data": RefObjectAIModel, + "knowledge_bases_data": RefObjectKnowledgeBase, + "consumed_mcp_services_data": RefObjectConsumedMCPService, + "constants_data": RefObjectConstant, + "entities_data": RefObjectEntity, + } { + query := "SELECT QualifiedName, '" + objType + "' FROM " + table + if table == "microflows_data" { + query = "SELECT QualifiedName, MicroflowType FROM microflows_data" + } + rows, err := b.tx.Query(query) + if err != nil { + continue + } + for rows.Next() { + var qn, t string + if rows.Scan(&qn, &t) == nil { + known[qn] = t + } + } + rows.Close() + } + return func(qn string) string { return known[qn] } +} + +// extractAgentEditorRefs writes the edges of every agent-editor document the +// agent pass decoded, each owned by that document's unit. +func (b *Builder) extractAgentEditorRefs(stmt *Stmt, resolve func(string) string, projectID, snapshotID string) int { + defer b.disown() + count := 0 + for _, d := range b.agentDocRefs { + b.as(d.owner) + for _, r := range agentEditorRefs(d.doc, resolve) { + if _, err := stmt.Exec( + d.sourceType, "", d.source, + r.targetType, "", r.target, + r.kind, d.moduleName, projectID, snapshotID, + ); err == nil { + count++ + } + } + } + return count +} diff --git a/mdl/catalog/builder_agent_refs_test.go b/mdl/catalog/builder_agent_refs_test.go new file mode 100644 index 000000000..b2acd3538 --- /dev/null +++ b/mdl/catalog/builder_agent_refs_test.go @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "sort" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// ako/mxcli#2074: an agent-editor document produced no refs, so an agent's +// tool microflow showed "(no callers found)" and GRAPH_DEAD_ASSETS listed it +// as unused although the agent calls it in production. And a microflow passed +// as a microflow-typed Java action parameter (ActionMicroflow of +// ChatContext_Create_ForAgent, AuthenticationMicroflow of CreateMCPServer) was +// no reference either. + +// resolveFixture is what the catalog knows of each name: the object type the +// tables hold it under. +func resolveFixture(qn string) string { + return map[string]string{ + "Desk.Tool_GetOrder": RefObjectMicroflow, + "Desk.NF_Local": RefObjectNanoflow, + "Desk.Triage": RefObjectAgent, + "Desk.Jira": RefObjectConsumedMCPService, + "Desk.Docs": RefObjectKnowledgeBase, + "Desk.Gpt": RefObjectAIModel, + "Desk.GenAIKey": RefObjectConstant, + "Desk.Ticket": RefObjectEntity, + "Desk.Chat_Action": RefObjectMicroflow, + "Desk.SupportAgent": RefObjectAgent, + "Desk.MCP_Authenticate": RefObjectMicroflow, + }[qn] +} + +func edgeStrings(rs []docRef) []string { + var out []string + for _, r := range rs { + out = append(out, r.kind+" "+r.targetType+" "+r.target) + } + sort.Strings(out) + return out +} + +func TestAgentEditorRefs_Issue2074(t *testing.T) { + agent := &types.Agent{ + Tools: []types.AgentTool{ + {Name: "GetOrder", ToolType: "Microflow", Document: &types.DocRef{QualifiedName: "Desk.Tool_GetOrder"}}, + {Name: "Local", Document: &types.DocRef{QualifiedName: "Desk.NF_Local"}}, + {Name: "Triage", Document: &types.DocRef{QualifiedName: "Desk.Triage"}}, + {Name: "Jira", ToolType: "MCP", Document: &types.DocRef{QualifiedName: "Desk.Jira"}}, + {Name: "Unset", ToolType: "Microflow"}, // a tool with no document yet: no edge + // a tool whose document the catalog does not hold: the edge is still + // recorded, typed from its ToolType, as a call to a missing microflow is + {Name: "Gone", ToolType: "Microflow", Document: &types.DocRef{QualifiedName: "Desk.Tool_Deleted"}}, + }, + KBTools: []types.AgentKBTool{{Name: "Docs", Document: &types.DocRef{QualifiedName: "Desk.Docs"}}}, + Model: &types.DocRef{QualifiedName: "Desk.Gpt"}, + Entity: &types.DocRef{QualifiedName: "Desk.Ticket"}, + } + want := []string{ + "call AGENT Desk.Triage", + "call MICROFLOW Desk.Tool_Deleted", + "call MICROFLOW Desk.Tool_GetOrder", + "call NANOFLOW Desk.NF_Local", + "model AI_MODEL Desk.Gpt", + "tool CONSUMED_MCP_SERVICE Desk.Jira", + "tool KNOWLEDGE_BASE Desk.Docs", + "type ENTITY Desk.Ticket", + } + if got := edgeStrings(agentEditorRefs(agent, resolveFixture)); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Errorf("agent edges:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } + + key := &types.ConstantRef{QualifiedName: "Desk.GenAIKey"} + for _, doc := range []any{&types.Model{Key: key}, &types.KnowledgeBase{Key: key}} { + if got := edgeStrings(agentEditorRefs(doc, resolveFixture)); strings.Join(got, "|") != "key CONSTANT Desk.GenAIKey" { + t.Errorf("%T edges = %v, want the key constant", doc, got) + } + } + // Control: a model with no key set has no edge. + if got := agentEditorRefs(&types.Model{}, resolveFixture); len(got) != 0 { + t.Errorf("model without a key: %v", got) + } +} + +func TestCodeActionParameterRefs_Issue2074(t *testing.T) { + action := µflows.JavaActionCallAction{ + JavaAction: "AgentCommons.ChatContext_Create_ForAgent", + ParameterMappings: []*microflows.JavaActionParameterMapping{ + {Parameter: "ActionMicroflow", Value: µflows.MicroflowParameterValue{Microflow: "Desk.Chat_Action"}}, + {Parameter: "Agent", Value: µflows.CustomBlobDocumentCodeActionParameterValue{CustomDocument: "Desk.SupportAgent"}}, + {Parameter: "Unset", Value: µflows.MicroflowParameterValue{}}, // Studio Pro's placeholder: no edge + {Parameter: "Text", Value: µflows.BasicCodeActionParameterValue{Argument: "'Desk.Chat_Action'"}}, // control: a string, not a reference + }, + } + want := "call AGENT Desk.SupportAgent|call MICROFLOW Desk.Chat_Action" + if got := strings.Join(edgeStrings(codeActionParameterRefs(action, resolveFixture)), "|"); got != want { + t.Errorf("java action parameter edges = %q, want %q", got, want) + } + // The same holds for the AuthenticationMicroflow of CreateMCPServer. + mcp := µflows.JavaActionCallAction{ParameterMappings: []*microflows.JavaActionParameterMapping{ + {Parameter: "AuthenticationMicroflow", Value: µflows.MicroflowParameterValue{Microflow: "Desk.MCP_Authenticate"}}, + }} + if got := strings.Join(edgeStrings(codeActionParameterRefs(mcp, resolveFixture)), "|"); got != "call MICROFLOW Desk.MCP_Authenticate" { + t.Errorf("CreateMCPServer edges = %q", got) + } +} diff --git a/mdl/catalog/builder_agent_strings.go b/mdl/catalog/builder_agent_strings.go new file mode 100644 index 000000000..277049eb5 --- /dev/null +++ b/mdl/catalog/builder_agent_strings.go @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import "github.com/mendixlabs/mxcli/mdl/types" + +// agentStrings is the text of an agent the strings index holds +// (ako/mxcli#2075): its description, system and user prompts, and the +// descriptions of its tools and knowledge base tools, with the title a tool +// call shows the user. Before, only the agent's documentation was indexed, +// so the most important text in an agentic app was not searchable. +// +// None of it is a Texts$Text: it lives in the document's JSON Contents, so +// the translatable walk cannot see it and it has no language. A tool row's +// ElementId is the tool's id, which agent_tools rows carry too; the agent's +// own id when the tool has none. +// +// A published MCP service's tools are described to the client's model the +// same way: each tool's description and title is a row of the service. +func agentStrings(d agentDocRefs) []stringRow { + var out []stringRow + add := func(value, context, elementID string) { + if value == "" { + return + } + out = append(out, stringRow{ + QualifiedName: d.source, ObjectType: d.sourceType, + StringValue: value, StringContext: context, + ElementID: elementID, ModuleName: d.moduleName, + }) + } + if svc, ok := d.doc.(*types.PublishedMCPService); ok { + for _, t := range svc.Tools { + id := t.UUID + if id == "" { + id = string(svc.ID) + } + add(t.Description, "mcp_tool_description", id) + add(t.Title, "mcp_tool_title", id) + } + return out + } + agent, ok := d.doc.(*types.Agent) + if !ok { + return nil + } + agentID := string(agent.ID) + elementOf := func(id string) string { + if id != "" { + return id + } + return agentID + } + add(agent.Description, "agent_description", agentID) + add(agent.SystemPrompt, "system_prompt", agentID) + add(agent.UserPrompt, "user_prompt", agentID) + for _, t := range agent.Tools { + id := elementOf(t.ID) + add(t.Description, "tool_description", id) + if t.DisplayTitle != nil { + add(*t.DisplayTitle, "tool_display_title", id) + } + if t.DisplayDescription != nil { + add(*t.DisplayDescription, "tool_display_description", id) + } + } + for _, k := range agent.KBTools { + add(k.Description, "knowledge_base_description", elementOf(k.ID)) + } + return out +} diff --git a/mdl/catalog/builder_agent_strings_test.go b/mdl/catalog/builder_agent_strings_test.go new file mode 100644 index 000000000..1a0a7e01b --- /dev/null +++ b/mdl/catalog/builder_agent_strings_test.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/model" +) + +// ako/mxcli#2075: CATALOG.STRINGS indexed only an agent's documentation. Its +// system and user prompts and the descriptions of its tools and knowledge +// bases, the most important text in an agentic app, were not searchable. +func TestAgentStrings_Issue2075(t *testing.T) { + title := "Issuing a refund" + empty := "" + agent := &types.Agent{ + BaseElement: model.BaseElement{ID: "ag-1"}, + Description: "Handles refund requests", + SystemPrompt: "You issue refunds. Never refund more than the order total.", + UserPrompt: "Refund order {{OrderId}}", + Tools: []types.AgentTool{ + {ID: "t1", Name: "IssueRefund", Description: "Refund an order by its id", + DisplayTitle: &title, DisplayDescription: &empty}, + {Name: "NoID", Description: "A tool without an id"}, + {ID: "t3", Name: "Silent"}, // no description: no row + }, + KBTools: []types.AgentKBTool{ + {ID: "k1", Name: "Policies", Description: "The refund policy"}, + }, + } + d := agentDocRefs{owner: "ag-1", sourceType: RefObjectAgent, source: "Shop.RefundAgent", moduleName: "Shop", doc: agent} + + var got []string + for _, r := range agentStrings(d) { + got = append(got, strings.Join([]string{r.QualifiedName, r.ObjectType, r.StringContext, r.ElementID, r.ModuleName, r.StringValue}, "|")) + } + want := []string{ + "Shop.RefundAgent|AGENT|agent_description|ag-1|Shop|Handles refund requests", + "Shop.RefundAgent|AGENT|system_prompt|ag-1|Shop|You issue refunds. Never refund more than the order total.", + "Shop.RefundAgent|AGENT|user_prompt|ag-1|Shop|Refund order {{OrderId}}", + "Shop.RefundAgent|AGENT|tool_description|t1|Shop|Refund an order by its id", + "Shop.RefundAgent|AGENT|tool_display_title|t1|Shop|Issuing a refund", + "Shop.RefundAgent|AGENT|tool_description|ag-1|Shop|A tool without an id", + "Shop.RefundAgent|AGENT|knowledge_base_description|k1|Shop|The refund policy", + } + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Errorf("agent strings:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } + + // A published MCP service's tool descriptions are what the client's model reads. + svc := &types.PublishedMCPService{ + BaseElement: model.BaseElement{ID: "pub-1"}, + Tools: []types.PublishedMCPTool{ + {UUID: "u1", Name: "issue_refund", Description: "Refund an order.", Title: "Issue refund"}, + {Name: "bare"}, // no text: no row + }, + } + got = nil + for _, r := range agentStrings(agentDocRefs{sourceType: RefObjectPublishedMCPService, source: "Shop.RefundMCP", moduleName: "Shop", doc: svc}) { + got = append(got, strings.Join([]string{r.QualifiedName, r.ObjectType, r.StringContext, r.ElementID, r.StringValue}, "|")) + } + if want := "Shop.RefundMCP|PUBLISHED_MCP_SERVICE|mcp_tool_description|u1|Refund an order.\nShop.RefundMCP|PUBLISHED_MCP_SERVICE|mcp_tool_title|u1|Issue refund"; strings.Join(got, "\n") != want { + t.Errorf("published MCP service strings:\n%s", strings.Join(got, "\n")) + } + + // Control: a document that is neither has no rows. + if rows := agentStrings(agentDocRefs{doc: &types.Model{}}); len(rows) != 0 { + t.Errorf("model rows: %v", rows) + } +} diff --git a/mdl/catalog/builder_agenteditor.go b/mdl/catalog/builder_agenteditor.go index f7a06e6a5..ec293abaf 100644 --- a/mdl/catalog/builder_agenteditor.go +++ b/mdl/catalog/builder_agenteditor.go @@ -3,24 +3,138 @@ package catalog import ( + "fmt" + "strings" + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/modelsdk/mprread" ) // agentEditorTable maps a CustomBlobDocument's CustomDocumentType to its catalog -// table. The four agent-editor document types share one BSON wrapper +// table. The agent-editor document types share one BSON wrapper // (CustomBlobDocuments$CustomBlobDocument) and are distinguished by this field. var agentEditorTable = map[string]string{ - "agenteditor.agent": "agents", - "agenteditor.model": "ai_models", - "agenteditor.knowledgebase": "knowledge_bases", - "agenteditor.consumedMCPService": "consumed_mcp_services", + types.CustomTypeAgent: "agents", + types.CustomTypeModel: "ai_models", + types.CustomTypeKnowledgeBase: "knowledge_bases", + types.CustomTypeConsumedMCPService: "consumed_mcp_services", + types.CustomTypePublishedMCPService: "published_mcp_services", +} + +// agentEditorColumns is one document's row: the columns its type adds after +// the wrapper's, and their values. +type agentEditorColumns struct { + names []string + values []any +} + +func (c *agentEditorColumns) add(name string, value any) { + c.names = append(c.names, name) + c.values = append(c.values, value) +} + +// nullBool is a JSON boolean as a column: NULL when the document does not +// store it, which an older Agent Editor's does not, so a query can tell "off" +// from "never set". +func nullBool(b *bool) any { + if b == nil { + return nil + } + return boolToInt(*b) +} + +func docRefName(r *types.DocRef) string { + if r == nil { + return "" + } + return r.QualifiedName +} + +func constantRefName(r *types.ConstantRef) string { + if r == nil { + return "" + } + return r.QualifiedName } -// buildAgentEditorDocs catalogs the agent-editor documents (agent / model / -// knowledge base / consumed MCP service). The document name is a top-level field -// of the BSON wrapper (the type-specific config lives in the inner JSON blob, -// which the catalog does not need), so this reads through the raw-unit surface -// and decodes only the wrapper fields — no CatalogReader/backend change. +// agentEditorDetail is the type-specific part of a decoded document's row +// (ako/mxcli#2075). doc is nil when the Contents did not decode: the row then +// carries the wrapper's columns only. +func agentEditorDetail(doc any) agentEditorColumns { + var c agentEditorColumns + switch d := doc.(type) { + case *types.Agent: + tools, mcp := 0, 0 + for _, t := range d.Tools { + if t.ToolType == "MCP" { + mcp++ + } else { + tools++ + } + } + var vars []string + for _, v := range d.Variables { + vars = append(vars, v.Key) + } + var selected, selectedName any + if d.SelectedModel != nil { + selected, selectedName = d.SelectedModel.ID, d.SelectedModel.Name + } + var maxTokens, temperature, topP any + if d.MaxTokens != nil { + maxTokens = *d.MaxTokens + } + if d.Temperature != nil { + temperature = *d.Temperature + } + if d.TopP != nil { + topP = *d.TopP + } + c.add("UsageType", d.UsageType) + c.add("Model", docRefName(d.Model)) + c.add("SelectedModel", selected) + c.add("SelectedModelName", selectedName) + c.add("ToolChoice", d.ToolChoice) + c.add("MaxTokens", maxTokens) + c.add("Temperature", temperature) + c.add("TopP", topP) + c.add("EnableBackendStreaming", nullBool(d.EnableBackendStreaming)) + c.add("EnableUIStreaming", nullBool(d.EnableUIStreaming)) + c.add("Entity", docRefName(d.Entity)) + c.add("Variables", strings.Join(vars, ",")) + c.add("ToolCount", tools) + c.add("KnowledgeBaseCount", len(d.KBTools)) + c.add("MCPServiceCount", mcp) + case *types.Model: + c.add("Provider", d.Provider) + c.add("ModelType", d.Type) + c.add("DisplayName", d.DisplayName) + c.add("KeyConstant", constantRefName(d.Key)) + case *types.KnowledgeBase: + c.add("Provider", d.Provider) + c.add("KeyConstant", constantRefName(d.Key)) + case *types.ConsumedMCPService: + c.add("ProtocolVersion", d.ProtocolVersion) + c.add("Version", d.Version) + c.add("ConnectionTimeoutSeconds", d.ConnectionTimeoutSeconds) + case *types.PublishedMCPService: + c.add("ServerName", d.ServerName) + c.add("EndpointPath", d.EndpointPath) + c.add("Version", d.Version) + c.add("ProtocolVersion", d.ProtocolVersion) + c.add("AuthenticationEnabled", boolToInt(d.AuthenticationEnabled)) + c.add("AuthenticationMicroflow", docRefName(d.AuthenticationMicroflow)) + c.add("ToolCount", len(d.Tools)) + } + return c +} + +// buildAgentEditorDocs catalogs the agent-editor documents (agent, model, +// knowledge base, consumed and published MCP service) with the settings their +// JSON Contents holds, an agent's tools and knowledge base tools as rows of +// their own, and queues each document's references for buildReferences. func (b *Builder) buildAgentEditorDocs() error { units, err := b.lister().ListRawUnitsByType("CustomBlobDocuments$CustomBlobDocument") if err != nil { @@ -33,17 +147,15 @@ func (b *Builder) buildAgentEditorDocs() error { s.Close() } }() - stmtFor := func(table string) (*Stmt, error) { - if s, ok := stmts[table]; ok { + prepare := func(sql string) (*Stmt, error) { + if s, ok := stmts[sql]; ok { return s, nil } - s, err := b.tx.Prepare("INSERT INTO " + table + "_data " + - "(Id, Name, QualifiedName, ModuleName, Folder, Description, ProjectId, SnapshotId) " + - "VALUES (?, ?, ?, ?, ?, ?, ?, ?)") + s, err := b.tx.Prepare(sql) if err != nil { return nil, err } - stmts[table] = s + stmts[sql] = s return s, nil } @@ -54,37 +166,122 @@ func (b *Builder) buildAgentEditorDocs() error { if !b.own(string(u.ID)) { continue } - var doc struct { + var wrap struct { Name string `bson:"Name"` Documentation string `bson:"Documentation"` CustomDocumentType string `bson:"CustomDocumentType"` } - if err := bson.Unmarshal(u.Contents, &doc); err != nil || doc.Name == "" { + if err := bson.Unmarshal(u.Contents, &wrap); err != nil || wrap.Name == "" { continue } - table, ok := agentEditorTable[doc.CustomDocumentType] + table, ok := agentEditorTable[wrap.CustomDocumentType] if !ok { continue // not an agent-editor document type we catalog } - stmt, err := stmtFor(table) - if err != nil { - return err - } moduleID := b.hierarchy.findModuleID(u.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) - qualifiedName := moduleName + "." + doc.Name + qualifiedName := moduleName + "." + wrap.Name folderPath := b.hierarchy.buildFolderPath(u.ContainerID) - if _, err := stmt.Exec( - string(u.ID), doc.Name, qualifiedName, moduleName, folderPath, - doc.Documentation, projectID, snapshotID, - ); err != nil { + // A payload that does not decode still gets its row, with the + // wrapper's columns only, and no edges. + decoded, err := mprread.DecodeAgentEditorUnit(string(u.ID), string(u.ContainerID), u.Contents) + if err != nil { + decoded = nil + } + detail := agentEditorDetail(decoded) + names := append([]string{"Id", "Name", "QualifiedName", "ModuleName", "Folder", "Description"}, detail.names...) + names = append(names, "ProjectId", "SnapshotId") + values := append([]any{string(u.ID), wrap.Name, qualifiedName, moduleName, folderPath, wrap.Documentation}, detail.values...) + values = append(values, projectID, snapshotID) + stmt, err := prepare(fmt.Sprintf("INSERT INTO %s_data (%s) VALUES (?%s)", + table, strings.Join(names, ", "), strings.Repeat(", ?", len(names)-1))) + if err != nil { + return err + } + if _, err := stmt.Exec(values...); err != nil { return err } count++ + + if agent, ok := decoded.(*types.Agent); ok { + if err := b.insertAgentTools(prepare, agent, qualifiedName, moduleName, projectID, snapshotID); err != nil { + return err + } + } + + // What the document references; resolved and written by + // buildReferences (ako/mxcli#2074). + if decoded != nil { + b.agentDocRefs = append(b.agentDocRefs, agentDocRefs{ + owner: string(u.ID), sourceType: agentEditorSourceType(decoded), + source: qualifiedName, moduleName: moduleName, doc: decoded, + }) + } } b.disown() b.report("Agent-editor documents", count) return nil } + +// insertAgentTools writes one agent_tools row per entry of the agent's tools +// list and one agent_knowledge_bases row per knowledge base tool, owned by the +// agent's unit (ako/mxcli#2075). +func (b *Builder) insertAgentTools(prepare func(string) (*Stmt, error), a *types.Agent, agent, moduleName, projectID, snapshotID string) error { + toolStmt, err := prepare(`INSERT INTO agent_tools_data (Id, AgentId, Agent, ModuleName, + Name, ToolType, Document, Enabled, UserAccessApproval, Description, + DisplayTitle, DisplayDescription, ProjectId, SnapshotId) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) + if err != nil { + return err + } + optional := func(s *string) any { + if s == nil { + return nil + } + return *s + } + for i, t := range a.Tools { + if _, err := toolStmt.Exec( + rowID(string(a.ID), t.ID, i), string(a.ID), agent, moduleName, + t.Name, t.ToolType, docRefName(t.Document), boolToInt(t.Enabled), + t.UserAccessApproval, t.Description, + optional(t.DisplayTitle), optional(t.DisplayDescription), + projectID, snapshotID, + ); err != nil { + return err + } + } + kbStmt, err := prepare(`INSERT INTO agent_knowledge_bases_data (Id, AgentId, Agent, ModuleName, + Name, KnowledgeBase, Collection, MaxResults, MinSimilarity, IsVisible, + Enabled, Description, ProjectId, SnapshotId) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) + if err != nil { + return err + } + for i, k := range a.KBTools { + var minSimilarity any + if k.MinSimilarity != nil { + minSimilarity = *k.MinSimilarity + } + if _, err := kbStmt.Exec( + rowID(string(a.ID), k.ID, i), string(a.ID), agent, moduleName, + k.Name, docRefName(k.Document), k.CollectionIdentifier, k.MaxResults, + minSimilarity, nullBool(k.IsVisible), boolToInt(k.Enabled), k.Description, + projectID, snapshotID, + ); err != nil { + return err + } + } + return nil +} + +// rowID is a list entry's own id, or its agent's id and position when the +// entry has none. +func rowID(agentID, id string, i int) string { + if id != "" { + return id + } + return fmt.Sprintf("%s#%d", agentID, i) +} diff --git a/mdl/catalog/builder_annotations.go b/mdl/catalog/builder_annotations.go new file mode 100644 index 000000000..2869d0b20 --- /dev/null +++ b/mdl/catalog/builder_annotations.go @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +// buildDomainModelAnnotations writes one row per domain model annotation +// (ako/mxcli#1300). A microflow's annotations are activities and reach a rule +// through activities_for(); a domain model's were in no table, so a rule +// checking annotation text skipped every domain model and reported it clean. +// +// Each row belongs to its domain model's unit, as the entity rows do, so an +// incremental build rewrites a module's annotations exactly when it rewrites +// that module's entities. +func (b *Builder) buildDomainModelAnnotations() error { + domainModels, err := b.cachedDomainModels() + if err != nil { + return err + } + + stmt, err := b.tx.Prepare(` + INSERT INTO annotations_data (Id, DomainModelId, ModuleName, Caption, X, Y, Width, + ProjectId, SnapshotId) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `) + if err != nil { + return err + } + defer stmt.Close() + + projectID, snapshotID := b.snapshotMeta() + count := 0 + for _, dm := range domainModels { + if !b.own(string(dm.ID)) { + continue + } + moduleName := b.hierarchy.getModuleName(b.hierarchy.findModuleID(dm.ContainerID)) + for _, a := range dm.Annotations { + if _, err := stmt.Exec( + string(a.ID), string(dm.ID), moduleName, a.Caption, + a.Location.X, a.Location.Y, a.Width, + projectID, snapshotID, + ); err != nil { + return err + } + count++ + } + } + + b.report("Domain model annotations", count) + return nil +} diff --git a/mdl/catalog/builder_annotations_test.go b/mdl/catalog/builder_annotations_test.go new file mode 100644 index 000000000..2985ed78f --- /dev/null +++ b/mdl/catalog/builder_annotations_test.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "fmt" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// ako/mxcli#1300: a domain model annotation was in no catalog table, so a +// Starlark rule looking for "TODO" notes covered every microflow (whose +// annotations are activities) and silently skipped every domain model. Each +// row keeps the full caption, newlines included, and the annotation's place +// on the canvas. Module Empty has a domain model with no annotations: the +// control that a module contributes rows only for annotations it has. +func TestBuildDomainModelAnnotations_Issue1300(t *testing.T) { + cat, err := New() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { cat.Close() }) + + const sales, empty = model.ID("mod-sales"), model.ID("mod-empty") + note := func(id, caption string, x, y, width int) *domainmodel.Annotation { + a := &domainmodel.Annotation{Caption: caption, Location: model.Point{X: x, Y: y}, Width: width} + a.ID = model.ID(id) + return a + } + sdm := &domainmodel.DomainModel{ + ContainerID: sales, + Annotations: []*domainmodel.Annotation{ + note("an-todo", "TODO split sales\nper region", 60, 40, 400), + note("an-readme", "This domain model defines the data.", 60, 240, 440), + }, + } + sdm.ID = "dm-sales" + edm := &domainmodel.DomainModel{ContainerID: empty} + edm.ID = "dm-empty" + + tx, err := cat.CatalogDB().Begin() + if err != nil { + t.Fatal(err) + } + b := &Builder{ + catalog: cat, + snapshot: &Snapshot{ID: "snap"}, + hierarchy: &hierarchy{ + moduleIDs: map[model.ID]bool{sales: true, empty: true}, + moduleNames: map[model.ID]string{sales: "Sales", empty: "Empty"}, + containerParent: map[model.ID]model.ID{}, + folderNames: map[model.ID]string{}, + }, + domainModelCache: []*domainmodel.DomainModel{sdm, edm}, + tx: wrapTx(tx), + } + if err := b.buildDomainModelAnnotations(); err != nil { + t.Fatalf("buildDomainModelAnnotations: %v", err) + } + if err := b.tx.Commit(); err != nil { + t.Fatal(err) + } + + res, err := cat.Query(`SELECT Id, ModuleName, Caption, X, Y, Width FROM annotations ORDER BY Id`) + if err != nil { + t.Fatalf("query: %v -- the annotations view must exist", err) + } + var got []string + for _, r := range res.Rows { + got = append(got, strings.TrimSuffix(fmt.Sprintln(r...), "\n")) + } + want := []string{ + "an-readme Sales This domain model defines the data. 60 240 440", + "an-todo Sales TODO split sales\nper region 60 40 400", + } + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("annotations:\n%q\nwant:\n%q", got, want) + } +} diff --git a/mdl/catalog/builder_folders.go b/mdl/catalog/builder_folders.go new file mode 100644 index 000000000..59284c54c --- /dev/null +++ b/mdl/catalog/builder_folders.go @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "sort" + + "github.com/mendixlabs/mxcli/model" +) + +// buildFolders writes one row per folder, empty folders included +// (ako/mxcli#1436). Every other table names a folder only through the Folder +// column of a document in it, so a folder holding nothing appeared nowhere and +// a rule checking a module's folder layout reported it missing. +// +// The rows are project-wide, like the modules pass: a change to a folder is +// one no per-unit pass reads, which already makes the next build a full one. +// The folders come from the hierarchy, which listed them once for every +// document's folder path; listing them again would read the whole project on +// each incremental refresh. +func (b *Builder) buildFolders() error { + h := b.hierarchy + folders := make([]model.ID, 0, len(h.folderNames)) + for id := range h.folderNames { + folders = append(folders, id) + } + + // Direct children of each folder: the units it holds and the folders + // nested in it. A folder is listed as a unit too, so folderNames is + // checked first. + docs := map[model.ID]int{} + subs := map[model.ID]int{} + for child, parent := range h.containerParent { + if _, isFolder := h.folderNames[child]; isFolder { + subs[parent]++ + } else if t := h.unitTypes[child]; t != "" && t != "Projects$Folder" { + docs[parent]++ + } + } + + stmt, err := b.tx.Prepare(` + INSERT INTO folders_data (Id, Name, ModuleName, Path, ParentPath, Depth, + DocumentCount, FolderCount, ProjectId, SnapshotId) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `) + if err != nil { + return err + } + defer stmt.Close() + + // Sorted, so two builds of one project write the same bytes. + sort.Slice(folders, func(i, j int) bool { return folders[i] < folders[j] }) + projectID, snapshotID := b.snapshotMeta() + for _, id := range folders { + name, container := h.folderNames[id], h.containerParent[id] + moduleName := h.getModuleName(h.findModuleID(container)) + parentPath := h.buildFolderPath(container) + path := name + if parentPath != "" { + path = parentPath + "/" + name + } + if _, err := stmt.Exec( + string(id), name, moduleName, path, parentPath, + h.folderDepth(id), + docs[id], subs[id], + projectID, snapshotID, + ); err != nil { + return err + } + } + + b.report("Folders", len(folders)) + return nil +} + +// folderDepth counts the folders from id up to its module, id included. It +// walks the parents rather than counting "/" in the path: a folder name may +// itself hold a slash (DeepLink ships "Pages/Snippets"), which the path, like +// every document's Folder column, does not escape. +func (h *hierarchy) folderDepth(id model.ID) int { + depth := 0 + current := id + for range 100 { + if _, ok := h.folderNames[current]; !ok { + break + } + depth++ + parent, ok := h.containerParent[current] + if !ok || parent == current { + break + } + current = parent + } + return depth +} diff --git a/mdl/catalog/builder_folders_test.go b/mdl/catalog/builder_folders_test.go new file mode 100644 index 000000000..0357073ee --- /dev/null +++ b/mdl/catalog/builder_folders_test.go @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "fmt" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/model" +) + +// ako/mxcli#1436: the catalog held no folders, so a rule could only derive a +// folder from the `folder` of the documents in it, and an empty folder did not +// exist. "Every module has the folders Objects and UI" then reported a folder +// that is there, empty, as missing. +// +// Module Sales holds: +// +// Objects/ one document, one subfolder +// Objects/Order/ empty +// UI/ empty +// UI/Pages/Snippets one folder whose name holds a slash, as DeepLink's does: +// depth 2, though its path has two slashes +// (module root) one document, the control: it is in no folder +func TestBuildFolders_Issue1436(t *testing.T) { + cat, err := New() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { cat.Close() }) + + const sales = model.ID("mod-sales") + folders := []*types.FolderInfo{ + {ID: "f-objects", ContainerID: sales, Name: "Objects"}, + {ID: "f-order", ContainerID: "f-objects", Name: "Order"}, + {ID: "f-ui", ContainerID: sales, Name: "UI"}, + {ID: "f-slash", ContainerID: "f-ui", Name: "Pages/Snippets"}, + } + h := &hierarchy{ + moduleIDs: map[model.ID]bool{sales: true}, + moduleNames: map[model.ID]string{sales: "Sales"}, + containerParent: map[model.ID]model.ID{}, + folderNames: map[model.ID]string{}, + unitTypes: map[model.ID]string{}, + } + for _, f := range folders { + h.folderNames[f.ID] = f.Name + h.containerParent[f.ID] = f.ContainerID + } + for id, parent := range map[model.ID]model.ID{"u-flow": "f-objects", "u-dm": sales} { + h.containerParent[id] = parent + h.unitTypes[id] = "Microflows$Microflow" + } + + tx, err := cat.CatalogDB().Begin() + if err != nil { + t.Fatal(err) + } + b := &Builder{ + catalog: cat, + snapshot: &Snapshot{ID: "snap"}, + hierarchy: h, + tx: wrapTx(tx), + } + if err := b.buildFolders(); err != nil { + t.Fatalf("buildFolders: %v", err) + } + if err := b.tx.Commit(); err != nil { + t.Fatal(err) + } + + res, err := cat.Query(`SELECT Id, ModuleName, Name, Path, ParentPath, Depth, DocumentCount, FolderCount + FROM folders ORDER BY Path`) + if err != nil { + t.Fatalf("query: %v -- the folders view must exist", err) + } + var got []string + for _, r := range res.Rows { + got = append(got, strings.TrimSuffix(fmt.Sprintln(r...), "\n")) + } + want := []string{ + "f-objects Sales Objects Objects 1 1 1", + "f-order Sales Order Objects/Order Objects 2 0 0", + "f-ui Sales UI UI 1 0 1", + "f-slash Sales Pages/Snippets UI/Pages/Snippets UI 2 0 0", + } + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Errorf("folders:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +} diff --git a/mdl/catalog/builder_pages.go b/mdl/catalog/builder_pages.go index afe5f3f15..54e7a888d 100644 --- a/mdl/catalog/builder_pages.go +++ b/mdl/catalog/builder_pages.go @@ -37,9 +37,9 @@ func (b *Builder) buildPages() error { widgetStmt, err = b.tx.Prepare(` INSERT INTO widgets_data (Id, Name, WidgetType, ContainerId, ContainerQualifiedName, ContainerType, ModuleName, Folder, EntityRef, AttributeRef, MicroflowRef, NanoflowRef, PageRef, SnippetRef, Description, - ParentWidgetId, Depth, Class, Style, DynamicClasses, ActionType, HasConfirmation, + ParentWidgetId, Depth, Class, Style, DynamicClasses, ActionType, HasConfirmation, Caption, Tooltip, ProjectId, SnapshotId) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `) if err != nil { return err @@ -128,6 +128,7 @@ func (b *Builder) buildPages() error { "", w.ParentID, w.Depth, w.Class, w.Style, w.DynamicClasses, w.ActionType, w.HasConfirmation, + pickTranslation(w.CaptionTexts, b.defaultLanguage()), pickTranslation(w.TooltipTexts, b.defaultLanguage()), projectID, snapshotID, ); err != nil { return fmt.Errorf("insert widget %s for page %s: %w", w.Name, qualifiedName, err) @@ -172,9 +173,9 @@ func (b *Builder) buildSnippets() error { widgetStmt, err = b.tx.Prepare(` INSERT INTO widgets_data (Id, Name, WidgetType, ContainerId, ContainerQualifiedName, ContainerType, ModuleName, Folder, EntityRef, AttributeRef, MicroflowRef, NanoflowRef, PageRef, SnippetRef, Description, - ParentWidgetId, Depth, Class, Style, DynamicClasses, ActionType, HasConfirmation, + ParentWidgetId, Depth, Class, Style, DynamicClasses, ActionType, HasConfirmation, Caption, Tooltip, ProjectId, SnapshotId) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `) if err != nil { return err @@ -227,6 +228,7 @@ func (b *Builder) buildSnippets() error { w.EntityRef, w.AttributeRef, w.MicroflowRef, w.NanoflowRef, w.PageRef, w.SnippetRef, "", w.ParentID, w.Depth, w.Class, w.Style, w.DynamicClasses, w.ActionType, w.HasConfirmation, + pickTranslation(w.CaptionTexts, b.defaultLanguage()), pickTranslation(w.TooltipTexts, b.defaultLanguage()), projectID, snapshotID, ); err != nil { return fmt.Errorf("insert widget %s for snippet %s: %w", w.Name, qualifiedName, err) @@ -356,6 +358,57 @@ type rawWidgetInfo struct { // has no confirmation setting in the model at all. ActionType string HasConfirmation bool + + // CaptionTexts and TooltipTexts are the widget's own Caption and Tooltip, + // language code → text (ako/mxcli#1303). The row stores the project's + // default language, picked at insert time as pages.Title is. + CaptionTexts map[string]string + TooltipTexts map[string]string +} + +// widgetTexts reads a widget's own caption and tooltip. Measured on +// CitrusGrove (Mendix 10.24), a button stores its caption as CaptionTemplate, +// a Forms$ClientTemplate holding the Texts$Text under Template; a label stores +// Caption as a Texts$Text directly. A tooltip is a Texts$Text. A text with no +// translations, as an unset tooltip is stored, reads as nil. An input's +// LabelTemplate and a dynamic text's Content are not its caption; they reach a +// rule through strings().widget_id. +func widgetTexts(w map[string]any) (caption, tooltip map[string]string) { + text := func(v any) map[string]string { + m, ok := v.(map[string]any) + if !ok { + return nil + } + if extractString(m["$Type"]) == "Forms$ClientTemplate" { + if m, ok = m["Template"].(map[string]any); !ok { + return nil + } + } + if extractString(m["$Type"]) != "Texts$Text" { + return nil + } + var out map[string]string + for _, it := range getBsonArrayElements(m["Items"]) { + tr, ok := it.(map[string]any) + if !ok { + continue + } + lang, txt := extractString(tr["LanguageCode"]), extractString(tr["Text"]) + if lang == "" || txt == "" { + continue + } + if out == nil { + out = map[string]string{} + } + out[lang] = txt + } + return out + } + caption = text(w["CaptionTemplate"]) + if caption == nil { + caption = text(w["Caption"]) + } + return caption, text(w["Tooltip"]) } // widgetPrimaryAction returns the $Type of a widget's primary action and whether @@ -630,6 +683,7 @@ func walkWidget(w map[string]any, parentID string, depth int) []rawWidgetInfo { widget.AllRefs = scanWidgetAllRefs(w) widget.Class, widget.Style, widget.DynamicClasses = widgetAppearance(w) widget.ActionType, widget.HasConfirmation = widgetPrimaryAction(w) + widget.CaptionTexts, widget.TooltipTexts = widgetTexts(w) widget.ParentID, widget.Depth = parentID, depth // Index user-authored containers, but skip the synthetic @@ -1007,9 +1061,9 @@ func (b *Builder) buildLayouts() error { widgetStmt, err = b.tx.Prepare(` INSERT INTO widgets_data (Id, Name, WidgetType, ContainerId, ContainerQualifiedName, ContainerType, ModuleName, Folder, EntityRef, AttributeRef, MicroflowRef, NanoflowRef, PageRef, SnippetRef, Description, - ParentWidgetId, Depth, Class, Style, DynamicClasses, ActionType, HasConfirmation, + ParentWidgetId, Depth, Class, Style, DynamicClasses, ActionType, HasConfirmation, Caption, Tooltip, ProjectId, SnapshotId) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `) if err != nil { return err @@ -1065,6 +1119,7 @@ func (b *Builder) buildLayouts() error { w.EntityRef, w.AttributeRef, w.MicroflowRef, w.NanoflowRef, w.PageRef, w.SnippetRef, "", w.ParentID, w.Depth, w.Class, w.Style, w.DynamicClasses, w.ActionType, w.HasConfirmation, + pickTranslation(w.CaptionTexts, b.defaultLanguage()), pickTranslation(w.TooltipTexts, b.defaultLanguage()), projectID, snapshotID, ); err != nil { return fmt.Errorf("insert widget %s for layout %s: %w", w.Name, qualifiedName, err) diff --git a/mdl/catalog/builder_references.go b/mdl/catalog/builder_references.go index 234e5fba4..d057b837c 100644 --- a/mdl/catalog/builder_references.go +++ b/mdl/catalog/builder_references.go @@ -47,6 +47,9 @@ const ( RefKindSplit = "split" // A decision branches on an object being a specialization RefKindMappingCall = "mapping_call" // A flow applies an import or export mapping (import/export action, REST or web service call) RefKindStructure = "structure" // An import or export mapping is built on a JSON structure + RefKindTool = "tool" // An agent draws on a knowledge base or consumed MCP service (a tool microflow, nanoflow or agent is a call) + RefKindModel = "model" // An agent runs on an AI model + RefKindKey = "key" // An AI model or knowledge base reads its key from a constant ) // RefKinds is every value that reaches refs.RefKind. It is the vocabulary a @@ -64,7 +67,8 @@ var RefKinds = []string{ RefKindWidget, RefKindSettings, RefKindSync, RefKindPublish, RefKindEvent, RefKindMember, RefKindXPath, RefKindType, RefKindValue, RefKindMapping, RefKindSplit, RefKindSnippetCall, - RefKindMappingCall, RefKindStructure, + RefKindMappingCall, RefKindStructure, RefKindTool, RefKindModel, + RefKindKey, } // Object types recorded in refs.SourceType and refs.TargetType — the catalog's @@ -108,6 +112,14 @@ const ( // Message definitions: the pre-11.15 collection and the 11.15+ document. RefObjectMessageDefinitionCollection = "MESSAGE_DEFINITION_COLLECTION" RefObjectMessageDefinition = "MESSAGE_DEFINITION" + // Agent-editor documents and the constant a model's key is read from + // (ako/mxcli#2074), named as the objects view names them. + RefObjectAgent = "AGENT" + RefObjectAIModel = "AI_MODEL" + RefObjectKnowledgeBase = "KNOWLEDGE_BASE" + RefObjectConsumedMCPService = "CONSUMED_MCP_SERVICE" + RefObjectPublishedMCPService = "PUBLISHED_MCP_SERVICE" + RefObjectConstant = "CONSTANT" ) // RefSourceObjectTypes is every value that reaches refs.SourceType, and @@ -139,6 +151,10 @@ var ( RefObjectExportMapping, RefObjectMessageDefinitionCollection, RefObjectMessageDefinition, + RefObjectAgent, + RefObjectAIModel, + RefObjectKnowledgeBase, + RefObjectPublishedMCPService, } RefTargetObjectTypes = []string{ @@ -162,6 +178,11 @@ var ( RefObjectImportMapping, RefObjectExportMapping, RefObjectJsonStructure, + RefObjectAgent, + RefObjectAIModel, + RefObjectKnowledgeBase, + RefObjectConsumedMCPService, + RefObjectConstant, } ) @@ -571,6 +592,7 @@ func (b *Builder) buildReferences() error { snapshotID := b.snapshot.ID refCount := 0 assocs := b.associationEnds() + resolveDoc := b.agentEditorResolver() // emitActionRefs walks the action activities of a microflow or nanoflow and // records the document reference each action makes. Nanoflows share the same @@ -623,6 +645,9 @@ func (b *Builder) buildReferences() error { for _, m := range microflowMappingRefs(act.Action) { emit(m.targetType, m.name, RefKindMappingCall) } + for _, r := range codeActionParameterRefs(act.Action, resolveDoc) { + emit(r.targetType, r.target, r.kind) + } } } @@ -980,6 +1005,10 @@ func (b *Builder) buildReferences() error { // "Remove if unused" — on a microflow that runs nightly in production. refCount += b.extractScheduledEventRefs(stmt, projectID, snapshotID) + // Agent-editor documents call their tools and use their model, knowledge + // bases and MCP services (ako/mxcli#2074). + refCount += b.extractAgentEditorRefs(stmt, resolveDoc, projectID, snapshotID) + // Attribute validation rules use a named regular expression, so // `show references to ` can answer which entities depend on a shared // pattern. diff --git a/mdl/catalog/builder_strings.go b/mdl/catalog/builder_strings.go index 4490b8f3d..9f332f230 100644 --- a/mdl/catalog/builder_strings.go +++ b/mdl/catalog/builder_strings.go @@ -30,6 +30,15 @@ func (b *Builder) buildStrings() error { } defer stmt.Close() + widgetStmt, err := b.tx.Prepare(` + INSERT INTO strings (QualifiedName, ObjectType, StringValue, StringContext, Language, ElementId, ModuleName, WidgetId) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `) + if err != nil { + return err + } + defer widgetStmt.Close() + count := 0 insert := func(qn, objType, value, ctx, lang, elementID, module string) { if value == "" { @@ -38,6 +47,14 @@ func (b *Builder) buildStrings() error { stmt.Exec(qn, objType, value, ctx, lang, elementID, module) count++ } + // The translatable walk knows the widget around each text (ako/mxcli#1303). + insertText := func(r stringRow) { + if r.StringValue == "" { + return + } + widgetStmt.Exec(r.QualifiedName, r.ObjectType, r.StringValue, r.StringContext, r.Language, r.ElementID, r.ModuleName, r.WidgetID) + count++ + } // Every TRANSLATABLE string comes from the walk below, not from here. What // remains in the typed extractions is the strings that are not Texts$Text @@ -146,7 +163,17 @@ func (b *Builder) buildStrings() error { b.disown() } - b.buildTranslatableStrings(insert) + // Agent prompts and tool descriptions, from the documents the agent pass + // decoded; each row belongs to its agent's unit (ako/mxcli#2075). + for _, d := range b.agentDocRefs { + b.as(d.owner) + for _, r := range agentStrings(d) { + insert(r.QualifiedName, r.ObjectType, r.StringValue, r.StringContext, r.Language, r.ElementID, r.ModuleName) + } + } + b.disown() + + b.buildTranslatableStrings(insertText) docRows, err := b.buildDocumentationStrings() if err != nil { @@ -242,7 +269,7 @@ func (b *Builder) buildDocumentationStrings() (int, error) { // changes them — the same split this is closing. // The caller's insert closure counts the rows it writes, so nothing is counted // here — doing both reported twice the rows the table actually holds. -func (b *Builder) buildTranslatableStrings(insert func(string, string, string, string, string, string, string)) { +func (b *Builder) buildTranslatableStrings(insert func(stringRow)) { units, err := b.lister().ListRawUnitsByType("") if err != nil { return @@ -267,7 +294,7 @@ func (b *Builder) buildTranslatableStrings(insert func(string, string, string, s } for _, r := range translatableRows(u.Type, qn, moduleName, u.Contents) { - insert(r.QualifiedName, r.ObjectType, r.StringValue, r.StringContext, r.Language, r.ElementID, r.ModuleName) + insert(r) } } b.disown() @@ -282,6 +309,7 @@ type stringRow struct { Language string ElementID string ModuleName string + WidgetID string // the widget the text belongs to; "" outside a widget } // translatableRows turns one unit's stored bytes into index rows, one per @@ -310,6 +338,7 @@ func translatableRows(unitType, qualifiedName, moduleName string, raw []byte) [] Language: lang, ElementID: site.ElementID, ModuleName: moduleName, + WidgetID: site.WidgetID, }) } } diff --git a/mdl/catalog/builder_widget_caption_test.go b/mdl/catalog/builder_widget_caption_test.go new file mode 100644 index 000000000..08b8c8298 --- /dev/null +++ b/mdl/catalog/builder_widget_caption_test.go @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "reflect" + "testing" +) + +// ako/mxcli#1303: a widgets() row had no caption or tooltip, so a rule could +// not tell which button's caption was "Delete" -- describe page prints it, the +// row did not. The keys are Studio Pro's, measured on CitrusGrove: a button's +// caption is CaptionTemplate (a Forms$ClientTemplate around a Texts$Text), a +// label's is Caption (a Texts$Text), a tooltip is a Texts$Text. +func rawText(langText ...string) map[string]any { + items := []any{int32(3)} + for i := 0; i < len(langText); i += 2 { + items = append(items, map[string]any{"$Type": "Texts$Translation", "LanguageCode": langText[i], "Text": langText[i+1]}) + } + return map[string]any{"$Type": "Texts$Text", "Items": items} +} + +func clientTemplate(t map[string]any) map[string]any { + return map[string]any{"$Type": "Forms$ClientTemplate", "Template": t, "Parameters": []any{int32(2)}} +} + +func TestWidgetCaptionAndTooltip_Issue1303(t *testing.T) { + page := wid("dv", "dvSale", "Forms$DataView", map[string]any{ + "Widgets": bsonList( + wid("b1", "btnDelete", "Forms$ActionButton", map[string]any{ + "CaptionTemplate": clientTemplate(rawText("en_US", "Delete sale", "nl_NL", "Verkoop verwijderen")), + "Tooltip": rawText("en_US", "Deletes it", "nl_NL", "Verwijdert hem"), + }), + wid("b2", "btnRemove", "Forms$ActionButton", map[string]any{ + "CaptionTemplate": clientTemplate(rawText("en_US", "Remove")), + "Tooltip": rawText(), // a tooltip never set: no translations + }), + wid("t1", "text1", "Forms$Label", map[string]any{ + "Caption": rawText("en_US", "Total"), + }), + // control: an input's label is not its caption + wid("tb", "textBox1", "Forms$TextBox", map[string]any{ + "LabelTemplate": clientTemplate(rawText("en_US", "Amount")), + }), + ), + }) + rows := extractWidgetsRecursive(page) + for _, c := range []struct { + name string + caption, tooltip map[string]string + }{ + {"btnDelete", map[string]string{"en_US": "Delete sale", "nl_NL": "Verkoop verwijderen"}, + map[string]string{"en_US": "Deletes it", "nl_NL": "Verwijdert hem"}}, + {"btnRemove", map[string]string{"en_US": "Remove"}, nil}, + {"text1", map[string]string{"en_US": "Total"}, nil}, + {"textBox1", nil, nil}, + } { + w := widgetByName(rows, c.name) + if w == nil { + t.Fatalf("%s not indexed", c.name) + } + if !reflect.DeepEqual(w.CaptionTexts, c.caption) || !reflect.DeepEqual(w.TooltipTexts, c.tooltip) { + t.Errorf("%s: caption %v tooltip %v, want %v %v", c.name, w.CaptionTexts, w.TooltipTexts, c.caption, c.tooltip) + } + } + // The row holds the project's default language, as pages.title does. + if got := pickTranslation(widgetByName(rows, "btnDelete").CaptionTexts, "nl_NL"); got != "Verkoop verwijderen" { + t.Errorf("nl_NL caption = %q", got) + } +} diff --git a/mdl/catalog/catalog.go b/mdl/catalog/catalog.go index 257867d75..f6ba63d1f 100644 --- a/mdl/catalog/catalog.go +++ b/mdl/catalog/catalog.go @@ -104,6 +104,8 @@ func (c *Catalog) SetBuilt(built bool) { func (c *Catalog) Tables() []string { return []string{ "CATALOG.MODULES", + "CATALOG.FOLDERS", + "CATALOG.ANNOTATIONS", "CATALOG.ENTITIES", "CATALOG.ASSOCIATIONS", "CATALOG.ATTRIBUTES", @@ -134,6 +136,10 @@ func (c *Catalog) Tables() []string { "CATALOG.AI_MODELS", "CATALOG.KNOWLEDGE_BASES", "CATALOG.CONSUMED_MCP_SERVICES", + "CATALOG.PUBLISHED_MCP_SERVICES", + "CATALOG.AGENT_TOOLS", + "CATALOG.AGENT_MCP_SERVICES", + "CATALOG.AGENT_KNOWLEDGE_BASES", "CATALOG.NAVIGATION_PROFILES", "CATALOG.OFFLINE_ENTITY_CONFIGS", "CATALOG.ENTITY_EVENT_HANDLERS", diff --git a/mdl/catalog/tables.go b/mdl/catalog/tables.go index 02d72a9f5..304cb3964 100644 --- a/mdl/catalog/tables.go +++ b/mdl/catalog/tables.go @@ -7,6 +7,38 @@ package catalog // // History: // +// 43 (agent prompts): strings gains an agent's description, system and +// user prompts, and its tool and knowledge base descriptions +// (ako/mxcli#2075). Without the bump a cached catalog keeps answering +// no hits for a prompt's text. +// 42 (agent details): agents, ai_models, knowledge_bases and +// consumed_mcp_services gain the settings their JSON Contents holds; +// new published_mcp_services, agent_tools (with the agent_mcp_services +// view) and agent_knowledge_bases (ako/mxcli#2075). Without the bump a +// cached catalog lacks the columns and every query naming them fails. +// 41 (agent and code action edges): refs gains edges from agent-editor +// documents (an agent's tool `call`s, `tool`, `model`, `type`; a model's +// or knowledge base's `key`) and a `call` from a flow to a microflow or +// agent document it passes as a Java action parameter (ako/mxcli#2074). +// Without the bump a cached catalog keeps answering "(no callers +// found)" for every agent tool. (40 is the widget-texts schema of +// ako/mxcli#1303, merged separately.) +// 40 (widget texts): widgets gains Caption and Tooltip (default language), +// strings gains WidgetId, the widget a caption, tooltip or confirmation +// text belongs to (ako/mxcli#1303). Without the bump a cached catalog +// lacks the columns and every query naming them fails. +// +// 39 (domain model annotations): annotations_data + view, one row per +// domain model annotation with its full caption and canvas position +// (ako/mxcli#1300). Without the bump a cached catalog has the table but +// no rows, and annotations() answers [] -- a "no TODO notes" rule passes. +// +// 38 (folders): folders_data + view, one row per folder with its path, +// parent and direct document and subfolder counts, including empty +// folders, which no document's Folder column can show +// (ako/mxcli#1436). Without the bump a cached catalog has the table +// but no rows, and folders() answers [] with no warning. +// // 37 (entity type edges): refs gains entity targets for a result variable's // type, a created list, an entity-type argument, a snippet parameter and // the entity an association step reaches, and a value edge to an @@ -203,7 +235,7 @@ package catalog // SnapshotSource / SourceId / SourceBranch / SourceRevision columns // from every row (issue #576). // 1 — initial flat schema with denormalized snapshot columns on every row. -const CatalogSchemaVersion = "37" +const CatalogSchemaVersion = "43" // MetaSchemaVersion is the catalog_meta key that records the schema version // the cache was built against. @@ -282,6 +314,40 @@ func (c *Catalog) createTables() error { )`, viewWithFullSnapshot("modules"), + // folders: one row per Projects$Folder, empty ones included. A + // document's Folder column names only the folders that hold a + // document, so an empty folder appeared nowhere else + // (ako/mxcli#1436). Path is spelled as that column spells it. + `CREATE TABLE IF NOT EXISTS folders_data ( + Id TEXT PRIMARY KEY, + Name TEXT, + ModuleName TEXT, + Path TEXT, + ParentPath TEXT DEFAULT '', + Depth INTEGER DEFAULT 1, + DocumentCount INTEGER DEFAULT 0, + FolderCount INTEGER DEFAULT 0, + ProjectId TEXT, + SnapshotId TEXT + )`, + viewWithFullSnapshot("folders"), + + // annotations: one row per domain model annotation. A microflow's + // annotations are activities (activities_data); a domain model's were + // in no table (ako/mxcli#1300). Caption is the full text, newlines kept. + `CREATE TABLE IF NOT EXISTS annotations_data ( + Id TEXT PRIMARY KEY, + DomainModelId TEXT, + ModuleName TEXT, + Caption TEXT, + X INTEGER DEFAULT 0, + Y INTEGER DEFAULT 0, + Width INTEGER DEFAULT 0, + ProjectId TEXT, + SnapshotId TEXT + )`, + viewWithFullSnapshot("annotations"), + // entities `CREATE TABLE IF NOT EXISTS entities_data ( Id TEXT PRIMARY KEY, @@ -740,26 +806,74 @@ func (c *Catalog) createTables() error { // agent-editor documents (CustomBlobDocuments, discriminated by // CustomDocumentType into agent / model / knowledge base / MCP service). + // The columns after Description come from the document's JSON + // Contents (ako/mxcli#2075); a boolean is NULL when the document does + // not store it, as an older Agent Editor does not. `CREATE TABLE IF NOT EXISTS agents_data ( Id TEXT PRIMARY KEY, Name TEXT, QualifiedName TEXT, ModuleName TEXT, - Folder TEXT, Description TEXT, ProjectId TEXT, SnapshotId TEXT + Folder TEXT, Description TEXT, + UsageType TEXT, Model TEXT, SelectedModel TEXT, SelectedModelName TEXT, + ToolChoice TEXT, MaxTokens INTEGER, Temperature REAL, TopP REAL, + EnableBackendStreaming INTEGER, EnableUIStreaming INTEGER, Entity TEXT, + Variables TEXT, ToolCount INTEGER DEFAULT 0, + KnowledgeBaseCount INTEGER DEFAULT 0, MCPServiceCount INTEGER DEFAULT 0, + ProjectId TEXT, SnapshotId TEXT )`, viewWithFullSnapshot("agents"), `CREATE TABLE IF NOT EXISTS ai_models_data ( Id TEXT PRIMARY KEY, Name TEXT, QualifiedName TEXT, ModuleName TEXT, - Folder TEXT, Description TEXT, ProjectId TEXT, SnapshotId TEXT + Folder TEXT, Description TEXT, + Provider TEXT, ModelType TEXT, DisplayName TEXT, KeyConstant TEXT, + ProjectId TEXT, SnapshotId TEXT )`, viewWithFullSnapshot("ai_models"), `CREATE TABLE IF NOT EXISTS knowledge_bases_data ( Id TEXT PRIMARY KEY, Name TEXT, QualifiedName TEXT, ModuleName TEXT, - Folder TEXT, Description TEXT, ProjectId TEXT, SnapshotId TEXT + Folder TEXT, Description TEXT, + Provider TEXT, KeyConstant TEXT, + ProjectId TEXT, SnapshotId TEXT )`, viewWithFullSnapshot("knowledge_bases"), `CREATE TABLE IF NOT EXISTS consumed_mcp_services_data ( Id TEXT PRIMARY KEY, Name TEXT, QualifiedName TEXT, ModuleName TEXT, - Folder TEXT, Description TEXT, ProjectId TEXT, SnapshotId TEXT + Folder TEXT, Description TEXT, + ProtocolVersion TEXT, Version TEXT, ConnectionTimeoutSeconds INTEGER, + ProjectId TEXT, SnapshotId TEXT )`, viewWithFullSnapshot("consumed_mcp_services"), + `CREATE TABLE IF NOT EXISTS published_mcp_services_data ( + Id TEXT PRIMARY KEY, Name TEXT, QualifiedName TEXT, ModuleName TEXT, + Folder TEXT, Description TEXT, + ServerName TEXT, EndpointPath TEXT, Version TEXT, ProtocolVersion TEXT, + AuthenticationEnabled INTEGER, AuthenticationMicroflow TEXT, + ToolCount INTEGER DEFAULT 0, + ProjectId TEXT, SnapshotId TEXT + )`, + viewWithFullSnapshot("published_mcp_services"), + // One row per entry of an agent's tools list: a microflow, nanoflow + // or agent it calls, or a consumed MCP service (ToolType 'MCP'). + `CREATE TABLE IF NOT EXISTS agent_tools_data ( + Id TEXT, AgentId TEXT, Agent TEXT, ModuleName TEXT, + Name TEXT, ToolType TEXT, Document TEXT, Enabled INTEGER, + UserAccessApproval TEXT, Description TEXT, + DisplayTitle TEXT, DisplayDescription TEXT, + ProjectId TEXT, SnapshotId TEXT + )`, + viewWithFullSnapshot("agent_tools"), + // The MCP service tools, under the names the Agent Editor shows them. + `CREATE VIEW IF NOT EXISTS agent_mcp_services AS + SELECT Id, AgentId, Agent, ModuleName, Name, Document AS Service, + Enabled, UserAccessApproval, Description, + ProjectId, ProjectName, SnapshotId, SnapshotDate, SnapshotSource + FROM agent_tools WHERE ToolType = 'MCP'`, + // One row per entry of an agent's knowledge base tools list. + `CREATE TABLE IF NOT EXISTS agent_knowledge_bases_data ( + Id TEXT, AgentId TEXT, Agent TEXT, ModuleName TEXT, + Name TEXT, KnowledgeBase TEXT, Collection TEXT, MaxResults INTEGER, + MinSimilarity REAL, IsVisible INTEGER, Enabled INTEGER, Description TEXT, + ProjectId TEXT, SnapshotId TEXT + )`, + viewWithFullSnapshot("agent_knowledge_bases"), // activities `CREATE TABLE IF NOT EXISTS activities_data ( @@ -871,6 +985,11 @@ func (c *Catalog) createTables() error { -- only microflow, nanoflow and workflow calls can. ActionType TEXT, HasConfirmation INTEGER DEFAULT 0, + -- The widget's own Caption and Tooltip in the project's default + -- language (ako/mxcli#1303); a button's caption was otherwise only + -- in strings, under its Forms$ClientTemplate's id. + Caption TEXT DEFAULT '', + Tooltip TEXT DEFAULT '', ProjectId TEXT, SnapshotId TEXT )`, @@ -1514,6 +1633,10 @@ func (c *Catalog) createTables() error { ProjectId, ProjectName, SnapshotId, SnapshotDate, SnapshotSource FROM consumed_mcp_services UNION ALL + SELECT Id, 'PUBLISHED_MCP_SERVICE' as ObjectType, Name, QualifiedName, ModuleName, Folder, Description, + ProjectId, ProjectName, SnapshotId, SnapshotDate, SnapshotSource + FROM published_mcp_services + UNION ALL SELECT Id, 'CONSUMED_ODATA_SERVICE' as ObjectType, Name, QualifiedName, ModuleName, '' as Folder, '' as Description, ProjectId, ProjectName, SnapshotId, SnapshotDate, SnapshotSource FROM consumed_odata_services @@ -1582,7 +1705,10 @@ func (c *Catalog) createTables() error { ProjectId, ProjectName, SnapshotId, SnapshotDate, SnapshotSource FROM jar_dependencies`, - // FTS5 virtual tables for full-text search + // FTS5 virtual tables for full-text search. WidgetId is the widget a + // text belongs to (ako/mxcli#1303): ElementId is the text's direct + // owner, which for a button caption is its Forms$ClientTemplate. + // UNINDEXED, since an id is no search term. `CREATE VIRTUAL TABLE IF NOT EXISTS strings USING fts5( QualifiedName, ObjectType, @@ -1590,7 +1716,8 @@ func (c *Catalog) createTables() error { StringContext, Language, ElementId, - ModuleName + ModuleName, + WidgetId UNINDEXED )`, // What the source rows were described from: every unit's content @@ -1642,6 +1769,11 @@ func (c *Catalog) createTables() error { // Indexes for common queries — target the underlying *_data tables. `CREATE INDEX IF NOT EXISTS idx_modules_name ON modules_data(Name)`, + `CREATE INDEX IF NOT EXISTS idx_folders_module ON folders_data(ModuleName)`, + `CREATE INDEX IF NOT EXISTS idx_agent_tools_agent ON agent_tools_data(AgentId)`, + `CREATE INDEX IF NOT EXISTS idx_agent_tools_document ON agent_tools_data(Document)`, + `CREATE INDEX IF NOT EXISTS idx_agent_kbs_agent ON agent_knowledge_bases_data(AgentId)`, + `CREATE INDEX IF NOT EXISTS idx_annotations_module ON annotations_data(ModuleName)`, `CREATE INDEX IF NOT EXISTS idx_entities_name ON entities_data(Name)`, `CREATE INDEX IF NOT EXISTS idx_entities_module ON entities_data(ModuleName)`, `CREATE INDEX IF NOT EXISTS idx_microflows_name ON microflows_data(Name)`, diff --git a/mdl/executor/catalog_incremental_budget_pedapp_test.go b/mdl/executor/catalog_incremental_budget_pedapp_test.go index f2b126757..947a31b5e 100644 --- a/mdl/executor/catalog_incremental_budget_pedapp_test.go +++ b/mdl/executor/catalog_incremental_budget_pedapp_test.go @@ -26,6 +26,7 @@ var projectWideTables = map[string]string{ "projects": "the build's own record", "snapshots": "the build's own record", "modules_data": "a module is no per-document pass's unit; a module change rebuilds everything", + "folders_data": "a folder's counts change when a document moves in or out, which rebuilds only that document", "navigation_profiles_data": "the navigation document; a change to it rebuilds everything", "navigation_menu_items": "the navigation document", "navigation_role_homes": "the navigation document", diff --git a/mdl/executor/cmd_agenteditor_agents.go b/mdl/executor/cmd_agenteditor_agents.go index a0a46c5b2..fe4d74a4d 100644 --- a/mdl/executor/cmd_agenteditor_agents.go +++ b/mdl/executor/cmd_agenteditor_agents.go @@ -99,6 +99,12 @@ func describeAgentEditorAgent(ctx *ExecContext, name ast.QualifiedName) error { if a.Model != nil && a.Model.QualifiedName != "" { lines = append(lines, fmt.Sprintf(" Model: %s", a.Model.QualifiedName)) } + if a.SelectedModel != nil { + lines = append(lines, " SelectedModel: "+mdlQuoted(a.SelectedModel.ID)) + if a.SelectedModel.Name != "" && a.SelectedModel.Name != a.SelectedModel.ID { + lines = append(lines, " SelectedModelName: "+mdlQuoted(a.SelectedModel.Name)) + } + } if a.Entity != nil && a.Entity.QualifiedName != "" { lines = append(lines, fmt.Sprintf(" Entity: %s", a.Entity.QualifiedName)) } diff --git a/mdl/executor/cmd_agenteditor_alter.go b/mdl/executor/cmd_agenteditor_alter.go index 542f8ab00..4e3389f7f 100644 --- a/mdl/executor/cmd_agenteditor_alter.go +++ b/mdl/executor/cmd_agenteditor_alter.go @@ -7,6 +7,7 @@ package executor import ( "fmt" + "sort" "strconv" "strings" @@ -139,7 +140,7 @@ func execAlterConsumedMCPService(ctx *ExecContext, s *ast.AlterConsumedMCPServic return mdlerrors.NewNotFound("consumed mcp service", s.Name.String()) } for key, val := range s.Changes { - if err := applyConsumedMCPServiceChange(c, key, val); err != nil { + if err := applyConsumedMCPServiceChange(ctx, c, key, val); err != nil { return err } } @@ -151,8 +152,28 @@ func execAlterConsumedMCPService(ctx *ExecContext, s *ast.AlterConsumedMCPServic return nil } -func applyConsumedMCPServiceChange(c *agenteditor.ConsumedMCPService, key, val string) error { +func applyConsumedMCPServiceChange(ctx *ExecContext, c *agenteditor.ConsumedMCPService, key, val string) error { switch strings.ToLower(key) { + case "endpoint": + if val == "" { + c.Endpoint = nil + return nil + } + ref, err := consumedMCPEndpointRef(ctx, parseQualifiedNameStr(val)) + if err != nil { + return fmt.Errorf("alter consumed mcp service %s: %w", c.Name, err) + } + c.Endpoint = ref + case "authenticationmicroflow": + if val == "" { + c.AuthenticationMicroflow = nil + return nil + } + ref, err := publishedMCPMicroflowRef(ctx, parseQualifiedNameStr(val)) + if err != nil { + return fmt.Errorf("alter consumed mcp service %s: AuthenticationMicroflow: %w", c.Name, err) + } + c.AuthenticationMicroflow = ref case "protocolversion": c.ProtocolVersion = val case "version": @@ -188,8 +209,14 @@ func execAlterAgent(ctx *ExecContext, s *ast.AlterAgentStmt) error { return mdlerrors.NewNotFound("agent", s.Name.String()) } - for key, val := range s.Sets { - if err := applyAgentChange(ctx, a, key, val); err != nil { + // In key order, so SelectedModel is applied before SelectedModelName. + keys := make([]string, 0, len(s.Sets)) + for key := range s.Sets { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { return strings.ToLower(keys[i]) < strings.ToLower(keys[j]) }) + for _, key := range keys { + if err := applyAgentChange(ctx, a, key, s.Sets[key]); err != nil { return err } } @@ -284,6 +311,13 @@ func applyAgentChange(ctx *ExecContext, a *agenteditor.Agent, key, val string) e } else { a.EnableUIStreaming = &b } + case "selectedmodel": + a.SelectedModel = selectedModelFrom(&val, nil, a.SelectedModel) + case "selectedmodelname": + if a.SelectedModel == nil { + return fmt.Errorf("alter agent %s: SelectedModelName needs a SelectedModel; set both", a.Name) + } + a.SelectedModel.Name = val case "entity": qn := parseQualifiedNameStr(val) a.Entity = carryEntityRef(&agenteditor.DocRef{QualifiedName: qn.String()}, a.Entity) diff --git a/mdl/executor/cmd_agenteditor_carry.go b/mdl/executor/cmd_agenteditor_carry.go index e50797649..441e66749 100644 --- a/mdl/executor/cmd_agenteditor_carry.go +++ b/mdl/executor/cmd_agenteditor_carry.go @@ -95,3 +95,33 @@ func sameAgentTool(t, stored agenteditor.AgentTool) bool { } return t.Name == stored.Name } + +// selectedModelFrom is the agent's selectedModel after a statement (#2031). +// id is the SelectedModel key and name SelectedModelName, nil when the +// statement leaves them out. Left out, the stored value stays; an empty id +// clears it. A name left out is the stored one while the id is the same, and +// otherwise the id: the stored display name belongs to the stored model, and +// the runtime reads only the id. +func selectedModelFrom(id, name *string, stored *agenteditor.SelectedModel) *agenteditor.SelectedModel { + if id == nil { + if stored == nil { + return nil + } + sm := *stored + if name != nil { + sm.Name = *name + } + return &sm + } + if *id == "" { + return nil + } + sm := agenteditor.SelectedModel{ID: *id, Name: *id} + switch { + case name != nil: + sm.Name = *name + case stored != nil && stored.ID == *id: + sm.Name = stored.Name + } + return &sm +} diff --git a/mdl/executor/cmd_agenteditor_consumedmcp_endpoint_test.go b/mdl/executor/cmd_agenteditor_consumedmcp_endpoint_test.go new file mode 100644 index 000000000..0b1a5531e --- /dev/null +++ b/mdl/executor/cmd_agenteditor_consumedmcp_endpoint_test.go @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/agenteditor" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// consumedMCPFixture is a module M holding a String constant Url, an Integer +// constant Port, a credentials microflow GetCredentials, and optionally a +// stored consumed MCP service Svc. +type consumedMCPFixture struct { + mod *model.Module + url *model.Constant + port *model.Constant + creds *microflows.Microflow + // badReturn and withParam break the credentials contract. + badReturn, withParam *microflows.Microflow + stored *agenteditor.ConsumedMCPService + written *agenteditor.ConsumedMCPService + mb *mock.MockBackend + h *ContainerHierarchy +} + +func newConsumedMCPFixture(stored bool) *consumedMCPFixture { + f := &consumedMCPFixture{mod: mkModule("M")} + f.url = mkConstant(f.mod.ID, "Url", "String", "http://127.0.0.1:8765/mcp") + f.port = mkConstant(f.mod.ID, "Port", "Integer", "8765") + f.creds = mkMicroflow(f.mod.ID, "GetCredentials") + f.creds.ReturnType = µflows.ListType{EntityQualifiedName: "System.HttpHeader"} + f.badReturn = mkMicroflow(f.mod.ID, "ReturnsString") + f.badReturn.ReturnType = µflows.StringType{} + f.withParam = mkMicroflow(f.mod.ID, "TakesParam") + f.withParam.ReturnType = µflows.ListType{EntityQualifiedName: "System.HttpHeader"} + f.withParam.Parameters = []*microflows.MicroflowParameter{{Name: "Token", Type: µflows.StringType{}}} + f.h = mkHierarchy(f.mod) + withContainer(f.h, f.mod.ID, f.mod.ID) + if stored { + f.stored = &agenteditor.ConsumedMCPService{BaseElement: model.BaseElement{ID: "svc-1"}, ContainerID: f.mod.ID, Name: "Svc", + ProtocolVersion: "v2025_03_26", + Endpoint: &agenteditor.DocRef{DocumentID: string(f.url.ID), QualifiedName: "M.Url"}, + AuthenticationMicroflow: &agenteditor.DocRef{DocumentID: string(f.creds.ID), QualifiedName: "M.GetCredentials"}} + } + record := func(c *agenteditor.ConsumedMCPService) error { f.written = c; return nil } + f.mb = &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{f.mod}, nil }, + ListConstantsFunc: func() ([]*model.Constant, error) { return []*model.Constant{f.url, f.port}, nil }, + ListMicroflowsFunc: func() ([]*microflows.Microflow, error) { + return []*microflows.Microflow{f.creds, f.badReturn, f.withParam}, nil + }, + ListAgentEditorConsumedMCPServicesFunc: func() ([]*agenteditor.ConsumedMCPService, error) { + if f.stored == nil { + return nil, nil + } + return []*agenteditor.ConsumedMCPService{f.stored}, nil + }, + CreateAgentEditorConsumedMCPServiceFunc: record, + UpdateAgentEditorConsumedMCPServiceFunc: record, + } + return f +} + +func (f *consumedMCPFixture) exec(t *testing.T, src string) error { + t.Helper() + ctx, _ := newMockCtx(t, withBackend(f.mb), withHierarchy(f.h)) + switch s := parseMDL(t, src).Statements[0].(type) { + case *ast.CreateConsumedMCPServiceStmt: + return execCreateConsumedMCPService(ctx, s) + case *ast.AlterConsumedMCPServiceStmt: + return execAlterConsumedMCPService(ctx, s) + default: + t.Fatalf("unexpected statement %T", s) + return nil + } +} + +// ako/mxcli#2060: a consumed MCP service had no endpoint and no credentials +// microflow, so AgentEditorCommons skipped it at import ("No value could be +// found for the local configuration for the constant") and the agent's MCP +// tools never appeared. +func TestCreateConsumedMCPService_EndpointAndCredentials(t *testing.T) { + f := newConsumedMCPFixture(false) + assertNoError(t, f.exec(t, `create consumed mcp service M.Svc (ProtocolVersion: v2025_03_26, Endpoint: @M.Url, AuthenticationMicroflow: M.GetCredentials)`)) + if f.written == nil { + t.Fatal("not written") + } + if ep := f.written.Endpoint; ep == nil || ep.QualifiedName != "M.Url" || ep.DocumentID != string(f.url.ID) { + t.Errorf("endpoint %+v, want M.Url with the constant's unit id %s", ep, f.url.ID) + } + if am := f.written.AuthenticationMicroflow; am == nil || am.QualifiedName != "M.GetCredentials" || am.DocumentID != string(f.creds.ID) { + t.Errorf("credentials microflow %+v, want M.GetCredentials with id %s", am, f.creds.ID) + } +} + +func TestCreateConsumedMCPService_EndpointRefusals(t *testing.T) { + for src, want := range map[string]string{ + `create consumed mcp service M.Svc (ProtocolVersion: v2025_03_26, Endpoint: @M.Port)`: "String constant", + `create consumed mcp service M.Svc (ProtocolVersion: v2025_03_26, Endpoint: @M.Missing)`: "constant not found", + `create consumed mcp service M.Svc (ProtocolVersion: v2025_03_26, Endpoint: @M.Url, AuthenticationMicroflow: M.Nope)`: "M.Nope", + } { + f := newConsumedMCPFixture(false) + err := f.exec(t, src) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%s:\n error %v, want it to mention %q", src, err, want) + } + if f.written != nil { + t.Errorf("%s: written despite the error", src) + } + } +} + +func TestCreateOrModifyConsumedMCPService_KeepsEndpoint(t *testing.T) { + f := newConsumedMCPFixture(true) + assertNoError(t, f.exec(t, `create or modify consumed mcp service M.Svc (ProtocolVersion: v2025_03_26, Version: '2.0')`)) + if f.written == nil || f.written.Endpoint == nil || f.written.Endpoint.QualifiedName != "M.Url" || + f.written.AuthenticationMicroflow == nil || f.written.AuthenticationMicroflow.QualifiedName != "M.GetCredentials" { + t.Errorf("left out, the stored endpoint and credentials microflow are lost: %+v", f.written) + } +} + +func TestAlterConsumedMCPService_EndpointAndCredentials(t *testing.T) { + f := newConsumedMCPFixture(true) + f.stored.Endpoint = nil + assertNoError(t, f.exec(t, `alter consumed mcp service M.Svc set Endpoint = @M.Url, AuthenticationMicroflow = ''`)) + if f.written == nil || f.written.Endpoint == nil || f.written.Endpoint.DocumentID != string(f.url.ID) { + t.Errorf("alter set Endpoint: %+v", f.written) + } + if f.written.AuthenticationMicroflow != nil { + t.Errorf("AuthenticationMicroflow = '' left %+v", f.written.AuthenticationMicroflow) + } + f2 := newConsumedMCPFixture(true) + if err := f2.exec(t, `alter consumed mcp service M.Svc set Endpoint = @M.Port`); err == nil || !strings.Contains(err.Error(), "String constant") { + t.Errorf("alter to a non-String constant: %v", err) + } +} + +// describe prints both, and its output reads back as them. +func TestDescribeConsumedMCPService_EndpointAndCredentials(t *testing.T) { + f := newConsumedMCPFixture(true) + ctx, buf := newMockCtx(t, withBackend(f.mb), withHierarchy(f.h)) + assertNoError(t, describeAgentEditorConsumedMCPService(ctx, ast.QualifiedName{Module: "M", Name: "Svc"})) + out := buf.String() + assertContainsStr(t, out, "Endpoint: @M.Url") + assertContainsStr(t, out, "AuthenticationMicroflow: M.GetCredentials") + s := parseMDL(t, out).Statements[0].(*ast.CreateConsumedMCPServiceStmt) + if s.Endpoint == nil || s.Endpoint.String() != "M.Url" || s.AuthenticationMicroflow == nil || s.AuthenticationMicroflow.String() != "M.GetCredentials" { + t.Errorf("describe output does not read back:\n%s", out) + } + // Control: a service without them prints neither. + f.stored.Endpoint, f.stored.AuthenticationMicroflow = nil, nil + buf.Reset() + assertNoError(t, describeAgentEditorConsumedMCPService(ctx, ast.QualifiedName{Module: "M", Name: "Svc"})) + if strings.Contains(buf.String(), "Endpoint") || strings.Contains(buf.String(), "AuthenticationMicroflow") { + t.Errorf("a service without them prints them:\n%s", buf.String()) + } +} + +// check warns on a create without an endpoint: the import skips such a +// service, so it can never connect. +func TestValidateConsumedMCPEndpoint(t *testing.T) { + prog := parseMDL(t, ` +create consumed mcp service M.NoEndpoint (ProtocolVersion: v2025_03_26); +create consumed mcp service M.Good (ProtocolVersion: v2025_03_26, Endpoint: @M.Url); +create or modify consumed mcp service M.Kept (ProtocolVersion: v2025_03_26); +`) + got := map[string]string{} + for _, v := range ValidateConsumedMCPEndpoint(prog) { + if v.RuleID != consumedMCPEndpointRule { + t.Errorf("rule %s", v.RuleID) + } + got[v.Location.DocumentName] = v.Message + } + if !strings.Contains(got["M.NoEndpoint"], "Endpoint") { + t.Errorf("M.NoEndpoint: %q, want an Endpoint warning", got["M.NoEndpoint"]) + } + // Controls: a stated endpoint, and a create or modify (which keeps the + // stored one), say nothing. + for _, name := range []string{"M.Good", "M.Kept"} { + if got[name] != "" { + t.Errorf("%s warned: %s", name, got[name]) + } + } +} + +// check --references predicts what exec and the runtime refuse: a missing or +// non-String endpoint constant, and a credentials microflow MCPClient cannot +// call. MCPClient runs it with no arguments and reads its result as a list of +// System.HttpHeader (ConsumedMCPService_CreateMCPClient, MCP Client 4.4). +func TestReferences_ConsumedMCPServiceEndpoint(t *testing.T) { + for _, c := range []struct{ src, want string }{ + {`create consumed mcp service M.Svc (ProtocolVersion: v2025_03_26, Endpoint: @M.Url, AuthenticationMicroflow: M.GetCredentials);`, ""}, + {`create consumed mcp service M.Svc (ProtocolVersion: v2025_03_26, Endpoint: @M.Port);`, "String constant"}, + {`create consumed mcp service M.Svc (ProtocolVersion: v2025_03_26, Endpoint: @M.Missing);`, "not found"}, + {`create consumed mcp service M.Svc (ProtocolVersion: v2025_03_26, Endpoint: @M.Url, AuthenticationMicroflow: M.Nope);`, "not found"}, + {`create consumed mcp service M.Svc (ProtocolVersion: v2025_03_26, Endpoint: @M.Url, AuthenticationMicroflow: M.ReturnsString);`, "List of System.HttpHeader"}, + {`create consumed mcp service M.Svc (ProtocolVersion: v2025_03_26, Endpoint: @M.Url, AuthenticationMicroflow: M.TakesParam);`, "no parameters"}, + {`create constant M.Later type String default 'http://x/mcp'; +create consumed mcp service M.Svc (ProtocolVersion: v2025_03_26, Endpoint: @M.Later);`, ""}, + } { + t.Run(c.want+"/"+c.src[len(c.src)-40:], func(t *testing.T) { + f := newConsumedMCPFixture(false) + ctx, _ := newMockCtx(t, withBackend(f.mb), withHierarchy(f.h)) + errs := enumErrors(validateProgram(ctx, parseMDL(t, c.src))) + if c.want == "" { + if errs != "" { + t.Fatalf("unexpected: %s", errs) + } + return + } + if !strings.Contains(errs, c.want) { + t.Fatalf("errors %q, want one mentioning %q", errs, c.want) + } + }) + } +} diff --git a/mdl/executor/cmd_agenteditor_mcpservices.go b/mdl/executor/cmd_agenteditor_mcpservices.go index aa4adfaea..341e9f8a9 100644 --- a/mdl/executor/cmd_agenteditor_mcpservices.go +++ b/mdl/executor/cmd_agenteditor_mcpservices.go @@ -14,6 +14,7 @@ import ( "github.com/mendixlabs/mxcli/mdl/ast" mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" "github.com/mendixlabs/mxcli/sdk/agenteditor" + "github.com/mendixlabs/mxcli/sdk/microflows" ) // listAgentEditorConsumedMCPServices handles SHOW CONSUMED MCP SERVICES [IN module]. @@ -85,6 +86,12 @@ func describeAgentEditorConsumedMCPService(ctx *ExecContext, name ast.QualifiedN if c.ProtocolVersion != "" { lines = append(lines, fmt.Sprintf(" ProtocolVersion: '%s'", escapeSQLString(c.ProtocolVersion))) } + if c.Endpoint != nil && c.Endpoint.QualifiedName != "" { + lines = append(lines, " Endpoint: @"+c.Endpoint.QualifiedName) + } + if c.AuthenticationMicroflow != nil && c.AuthenticationMicroflow.QualifiedName != "" { + lines = append(lines, " AuthenticationMicroflow: "+c.AuthenticationMicroflow.QualifiedName) + } if c.Version != "" { lines = append(lines, fmt.Sprintf(" Version: '%s'", escapeSQLString(c.Version))) } @@ -128,3 +135,40 @@ func findAgentEditorConsumedMCPService(ctx *ExecContext, moduleName, svcName str } // --- Executor method wrappers for backward compatibility --- + +// consumedMCPEndpointRef resolves a consumed MCP service's Endpoint (#2060). +// AgentEditorCommons reads the constant's value with GetConstantValue and +// imports the service only when it is not empty, so the reference must be a +// String constant: the document has no other way to hold the URL. +func consumedMCPEndpointRef(ctx *ExecContext, qn ast.QualifiedName) (*agenteditor.DocRef, error) { + consts, err := ctx.Backend.ListConstants() + if err != nil { + return nil, mdlerrors.NewBackend("list constants", err) + } + h, err := getHierarchy(ctx) + if err != nil { + return nil, err + } + for _, c := range consts { + if c.Name != qn.Name || h.GetModuleName(h.FindModuleID(c.ContainerID)) != qn.Module { + continue + } + if c.Type.Kind != "String" { + return nil, fmt.Errorf("endpoint must be a String constant holding the server URL; %s is %s", qn, c.Type.Kind) + } + return &agenteditor.DocRef{DocumentID: string(c.ID), QualifiedName: qn.String()}, nil + } + return nil, fmt.Errorf("endpoint: constant not found: %s", qn) +} + +// consumedMCPCredentialsProblem is why MCPClient cannot use mf as a consumed +// MCP service's credentials microflow, or "". +func consumedMCPCredentialsProblem(mf *microflows.Microflow) string { + if len(mf.Parameters) > 0 { + return fmt.Sprintf("must take no parameters: MCPClient calls it without arguments (it has $%s)", mf.Parameters[0].Name) + } + if lt, ok := mf.ReturnType.(*microflows.ListType); !ok || lt.EntityQualifiedName != "System.HttpHeader" { + return "must return List of System.HttpHeader: MCPClient sends that list as the request headers" + } + return "" +} diff --git a/mdl/executor/cmd_agenteditor_selectedmodel_test.go b/mdl/executor/cmd_agenteditor_selectedmodel_test.go new file mode 100644 index 000000000..19c8ed690 --- /dev/null +++ b/mdl/executor/cmd_agenteditor_selectedmodel_test.go @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/agenteditor" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// ako/mxcli#2031: Agent Editor 2.2+ imports an agent only with a selected +// model (AgentFromModel_Validate requires SelectedModelID, and the deployed +// model is retrieved by Model = SelectedModelID). mxcli could not write one. +func TestCreateOrModifyAgent_SelectedModel(t *testing.T) { + run := func(t *testing.T, src string, stored *agenteditor.SelectedModel) *agenteditor.Agent { + t.Helper() + mod := mkModule("M") + mf := mkMicroflow(mod.ID, "Tool_ExecuteSparql") + kb := &agenteditor.KnowledgeBase{BaseElement: model.BaseElement{ID: "kb-1"}, ContainerID: mod.ID, Name: "KB_WorkInstructions"} + a := storedGraphExplorer(mod, mf, kb) + a.SelectedModel = stored + h := mkHierarchy(mod) + withContainer(h, mf.ContainerID, mod.ID) + var written *agenteditor.Agent + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{mod}, nil }, + ListMicroflowsFunc: func() ([]*microflows.Microflow, error) { return []*microflows.Microflow{mf}, nil }, + ListAgentEditorAgentsFunc: func() ([]*agenteditor.Agent, error) { return []*agenteditor.Agent{a}, nil }, + ListAgentEditorKnowledgeBasesFunc: func() ([]*agenteditor.KnowledgeBase, error) { return []*agenteditor.KnowledgeBase{kb}, nil }, + UpdateAgentEditorAgentFunc: func(ag *agenteditor.Agent) error { + written = ag + return nil + }, + } + ctx, _ := newMockCtx(t, withBackend(mb), withHierarchy(h)) + prog := parseMDL(t, src) + assertNoError(t, execCreateAgent(ctx, prog.Statements[0].(*ast.CreateAgentStmt))) + if written == nil { + t.Fatal("the agent was not written") + } + return written + } + sonnet := &agenteditor.SelectedModel{ID: "eu.anthropic.claude-sonnet-5", Name: "Anthropic Claude Sonnet 5"} + + t.Run("stated", func(t *testing.T) { + w := run(t, `create or modify agent M.GraphExplorer (UsageType: Chat, SelectedModel: 'eu.anthropic.claude-haiku-5')`, sonnet) + // A new id with no name takes the id as its name: the stored display + // name belongs to the old model. + if w.SelectedModel == nil || *w.SelectedModel != (agenteditor.SelectedModel{ID: "eu.anthropic.claude-haiku-5", Name: "eu.anthropic.claude-haiku-5"}) { + t.Errorf("selected model %+v", w.SelectedModel) + } + }) + t.Run("stated with name", func(t *testing.T) { + w := run(t, `create or modify agent M.GraphExplorer (UsageType: Chat, SelectedModel: 'x', SelectedModelName: 'Model X')`, nil) + if w.SelectedModel == nil || *w.SelectedModel != (agenteditor.SelectedModel{ID: "x", Name: "Model X"}) { + t.Errorf("selected model %+v", w.SelectedModel) + } + }) + t.Run("same id keeps the stored name", func(t *testing.T) { + w := run(t, `create or modify agent M.GraphExplorer (UsageType: Chat, SelectedModel: 'eu.anthropic.claude-sonnet-5')`, sonnet) + if w.SelectedModel == nil || *w.SelectedModel != *sonnet { + t.Errorf("selected model %+v, want the stored %+v", w.SelectedModel, sonnet) + } + }) + t.Run("left out keeps the stored one", func(t *testing.T) { + w := run(t, `create or modify agent M.GraphExplorer (UsageType: Chat)`, sonnet) + if w.SelectedModel == nil || *w.SelectedModel != *sonnet { + t.Errorf("selected model %+v, want the stored %+v", w.SelectedModel, sonnet) + } + }) + t.Run("empty clears it", func(t *testing.T) { + w := run(t, `create or modify agent M.GraphExplorer (UsageType: Chat, SelectedModel: '')`, sonnet) + if w.SelectedModel != nil { + t.Errorf("selected model %+v, want none", w.SelectedModel) + } + }) +} + +func TestAlterAgent_SelectedModel(t *testing.T) { + mod := mkModule("M") + a := &agenteditor.Agent{BaseElement: model.BaseElement{ID: "agent-1"}, ContainerID: mod.ID, Name: "A", UsageType: "Task", + SelectedModel: &agenteditor.SelectedModel{ID: "old", Name: "Old Model"}} + h := mkHierarchy(mod) + withContainer(h, a.ContainerID, mod.ID) + var updated *agenteditor.Agent + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListAgentEditorAgentsFunc: func() ([]*agenteditor.Agent, error) { return []*agenteditor.Agent{a}, nil }, + UpdateAgentEditorAgentFunc: func(ag *agenteditor.Agent) error { + updated = ag + return nil + }, + } + ctx, _ := newMockCtx(t, withBackend(mb), withHierarchy(h)) + // Both in one statement: the name must survive whichever key is applied first. + assertNoError(t, execAlterAgent(ctx, &ast.AlterAgentStmt{Name: ast.QualifiedName{Module: "M", Name: "A"}, + Sets: map[string]string{"SelectedModelName": "New Model", "SelectedModel": "new"}})) + if updated == nil || updated.SelectedModel == nil || *updated.SelectedModel != (agenteditor.SelectedModel{ID: "new", Name: "New Model"}) { + t.Fatalf("selected model %+v", updated.SelectedModel) + } + ctx, _ = newMockCtx(t, withBackend(mb), withHierarchy(h)) + assertNoError(t, execAlterAgent(ctx, &ast.AlterAgentStmt{Name: ast.QualifiedName{Module: "M", Name: "A"}, + Sets: map[string]string{"SelectedModel": ""}})) + if updated.SelectedModel != nil { + t.Errorf("SelectedModel = '' left %+v", updated.SelectedModel) + } +} + +// describe prints the selected model, the name only when it is not the id, and +// what it prints reads back as the stored value. +func TestDescribeAgent_SelectedModel(t *testing.T) { + mod := mkModule("M") + a := &agenteditor.Agent{BaseElement: model.BaseElement{ID: "agent-1"}, ContainerID: mod.ID, Name: "A", UsageType: "Task", + SelectedModel: &agenteditor.SelectedModel{ID: "eu.anthropic.claude-sonnet-5", Name: "Anthropic Claude Sonnet 5"}} + h := mkHierarchy(mod) + withContainer(h, a.ContainerID, mod.ID) + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListAgentEditorAgentsFunc: func() ([]*agenteditor.Agent, error) { return []*agenteditor.Agent{a}, nil }, + } + ctx, buf := newMockCtx(t, withBackend(mb), withHierarchy(h)) + assertNoError(t, describeAgentEditorAgent(ctx, ast.QualifiedName{Module: "M", Name: "A"})) + s := parseMDL(t, buf.String()).Statements[0].(*ast.CreateAgentStmt) + if s.SelectedModel == nil || *s.SelectedModel != "eu.anthropic.claude-sonnet-5" || + s.SelectedModelName == nil || *s.SelectedModelName != "Anthropic Claude Sonnet 5" { + t.Errorf("describe does not read back as the stored selected model:\n%s", buf.String()) + } + // A name equal to the id is not repeated. + a.SelectedModel.Name = a.SelectedModel.ID + buf.Reset() + assertNoError(t, describeAgentEditorAgent(ctx, ast.QualifiedName{Module: "M", Name: "A"})) + if strings.Contains(buf.String(), "SelectedModelName") || !strings.Contains(buf.String(), "SelectedModel: 'eu.anthropic.claude-sonnet-5'") { + t.Errorf("name equal to id:\n%s", buf.String()) + } + // Control: an agent without one prints none. + a.SelectedModel = nil + buf.Reset() + assertNoError(t, describeAgentEditorAgent(ctx, ast.QualifiedName{Module: "M", Name: "A"})) + if strings.Contains(buf.String(), "SelectedModel") { + t.Errorf("an agent without a selected model prints one:\n%s", buf.String()) + } +} diff --git a/mdl/executor/cmd_agenteditor_write.go b/mdl/executor/cmd_agenteditor_write.go index 01013376d..c865bf0a0 100644 --- a/mdl/executor/cmd_agenteditor_write.go +++ b/mdl/executor/cmd_agenteditor_write.go @@ -61,6 +61,17 @@ func execCreateConsumedMCPService(ctx *ExecContext, s *ast.CreateConsumedMCPServ // A rewrite that carried no doc comment keeps the stored one (#1018). if existing != nil { c.Documentation = carriedDocumentation(s.DocumentationSet, s.OuterDocumentation, existing.Documentation) + c.Endpoint, c.AuthenticationMicroflow = existing.Endpoint, existing.AuthenticationMicroflow + } + if s.Endpoint != nil { + if c.Endpoint, err = consumedMCPEndpointRef(ctx, *s.Endpoint); err != nil { + return fmt.Errorf("consumed mcp service %s: %w", s.Name, err) + } + } + if s.AuthenticationMicroflow != nil { + if c.AuthenticationMicroflow, err = publishedMCPMicroflowRef(ctx, *s.AuthenticationMicroflow); err != nil { + return fmt.Errorf("consumed mcp service %s: AuthenticationMicroflow: %w", s.Name, err) + } } if existing != nil { @@ -312,9 +323,12 @@ func execCreateAgent(ctx *ExecContext, s *ast.CreateAgentStmt) error { a.KBTools = append(a.KBTools, *akt) } + var storedModel *agenteditor.SelectedModel if existingAgent != nil { carryAgentIdentity(a, existingAgent) + storedModel = existingAgent.SelectedModel } + a.SelectedModel = selectedModelFrom(s.SelectedModel, s.SelectedModelName, storedModel) if existingAgent != nil { a.ID = existingAgent.ID diff --git a/mdl/executor/cmd_microflows_builder_entity_type_arg_test.go b/mdl/executor/cmd_microflows_builder_entity_type_arg_test.go index e649051bc..b73af4d64 100644 --- a/mdl/executor/cmd_microflows_builder_entity_type_arg_test.go +++ b/mdl/executor/cmd_microflows_builder_entity_type_arg_test.go @@ -176,3 +176,34 @@ end;`) } t.Fatal("no call java action in body") } + +// TestBuildJavaScriptAction_QuotedEntityTypeArgument is mendixlabs/mxcli#1785 +// (ako/mxcli#2057) verbatim: quoted identifiers, which the report said were +// stored with their MDL quotes (`"CustomModule"."NPActivity"`), giving CE1613 +// at mxbuild on mxcli 0.26.0. Measured on main: the stored Entity is the bare +// qualified name and `mx check` (11.14, TestApp) reports 0 errors. This pins +// it — a quoted name must never reach storage with its quotes. +func TestBuildJavaScriptAction_QuotedEntityTypeArgument(t *testing.T) { + for name, src := range map[string]string{ + "reporter's MDL": `CREATE OR MODIFY NANOFLOW "CustomModule"."TEST_RefreshEntity_NF" ( + $Obj: "CustomModule"."NPActivity" +) +BEGIN + $r = CALL JAVASCRIPT ACTION "NanoflowCommons"."RefreshEntity" ( + EntityToRefresh = "CustomModule"."NPActivity" + ); + RETURN; +END;`, + "quoted, one line": `create nanoflow CustomModule.ACT_Example () +begin + call javascript action NanoflowCommons.RefreshEntity(EntityToRefresh = "CustomModule"."NPActivity"); +end;`, + } { + t.Run(name, func(t *testing.T) { + fb := &flowBuilder{posX: 100, posY: 100, spacing: HorizontalSpacing, backend: refreshEntityBackend()} + if got := jsEntityArgument(t, fb, src); got != "CustomModule.NPActivity" { + t.Errorf("Entity = %q, want CustomModule.NPActivity", got) + } + }) + } +} diff --git a/mdl/executor/cmd_move.go b/mdl/executor/cmd_move.go index dcad68922..33c8349ba 100644 --- a/mdl/executor/cmd_move.go +++ b/mdl/executor/cmd_move.go @@ -5,6 +5,7 @@ package executor import ( "fmt" + "strings" "github.com/mendixlabs/mxcli/mdl/ast" mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" @@ -102,7 +103,7 @@ func execMove(ctx *ExecContext, s *ast.MoveStmt) (moveErr error) { return err } case ast.DocumentTypeNanoflow: - if err := moveNanoflow(ctx, s.Name, targetContainerID); err != nil { + if err := moveNanoflow(ctx, s.Name, targetContainerID, targetModule, isCrossModuleMove); err != nil { return err } case ast.DocumentTypeRule: @@ -180,7 +181,7 @@ func moveActionSource(ctx *ExecContext, docType ast.DocumentType, fromModule, to // the target container. Anything it cannot establish — no such document, a // different doctype under that name — is left to the mover, which reports it. func alreadyPlaced(ctx *ExecContext, s *ast.MoveStmt, targetContainerID model.ID) bool { - doc, err := ctx.Backend.FindDocumentUnit(s.Name.Module, s.Name.Name) + doc, err := findDocumentOfKind(ctx, s.DocumentType.CanonicalSpelling(), s.Name) if err != nil || doc == nil || doc.ContainerID != targetContainerID { return false } @@ -338,7 +339,7 @@ func moveRule(ctx *ExecContext, name ast.QualifiedName, targetContainerID model. return mdlerrors.NewNotFound("rule", name.String()) } -func moveNanoflow(ctx *ExecContext, name ast.QualifiedName, targetContainerID model.ID) error { +func moveNanoflow(ctx *ExecContext, name ast.QualifiedName, targetContainerID model.ID, targetModule *model.Module, isCrossModuleMove bool) error { // Find the nanoflow nfs, err := ctx.Backend.ListNanoflows() if err != nil { @@ -359,6 +360,15 @@ func moveNanoflow(ctx *ExecContext, name ast.QualifiedName, targetContainerID mo if err := ctx.Backend.MoveNanoflow(nf); err != nil { return mdlerrors.NewBackend("move nanoflow", err) } + // A nanoflow's grants map to the target's same-named roles, as a + // microflow's do; left naming the old module they were CE0148 + // (ako/mxcli#1836). + if isCrossModuleMove { + nf.AllowedModuleRoles = remapDocumentAccessRoles(ctx, targetModule, nf.AllowedModuleRoles) + if err := ctx.Backend.UpdateAllowedRoles(nf.ID, documentRoleStrings(nf.AllowedModuleRoles)); err != nil { + return mdlerrors.NewBackend("remap nanoflow access", err) + } + } fmt.Fprintf(ctx.Output, "Moved nanoflow %s to new location\n", name.String()) return nil } @@ -396,6 +406,12 @@ func moveEntity(ctx *ExecContext, name ast.QualifiedName, sourceModule, targetMo return mdlerrors.NewBackend("get target domain model", err) } + // An entity's access rules may only name its own module's roles, so each + // moves to the target's role of the same name; one the target lacks is kept, + // and said, rather than dropped — a rule that silently lost its role would + // hide the entity from it (ako/mxcli#1836). + remapEntityAccessRoles(ctx, entity, sourceModule, targetModule) + // Move entity via writer (converts associations to CrossAssociations, updates // validation rule and access rule refs) convertedAssocs, err := ctx.Backend.MoveEntity(entity, sourceDM.ID, targetDM.ID, sourceModule.Name, targetModule.Name) @@ -634,3 +650,45 @@ func movePublishedODataService(ctx *ExecContext, name ast.QualifiedName, targetC return mdlerrors.NewNotFound("odata service", name.String()) } + +// remapEntityAccessRoles points each of the entity's access rules at the target +// module's role of the same name, and warns of a role the target lacks. +func remapEntityAccessRoles(ctx *ExecContext, entity *domainmodel.Entity, sourceModule, targetModule *model.Module) { + if len(entity.AccessRules) == 0 { + return + } + targetRoles := map[string]bool{} + if ms, err := ctx.Backend.GetModuleSecurity(targetModule.ID); err == nil && ms != nil { + for _, r := range ms.ModuleRoles { + targetRoles[r.Name] = true + } + } + oldPrefix := sourceModule.Name + "." + var missing []string + seen := map[string]bool{} + for _, ar := range entity.AccessRules { + names := append([]string(nil), ar.ModuleRoleNames...) + changed := false + for i, qn := range names { + if !strings.HasPrefix(qn, oldPrefix) { + continue + } + role := qn[len(oldPrefix):] + if targetRoles[role] { + names[i] = targetModule.Name + "." + role + changed = true + } else if !seen[qn] { + seen[qn] = true + missing = append(missing, qn) + } + } + if changed { + ar.ModuleRoleNames = names + } + } + for _, qn := range missing { + fmt.Fprintf(ctx.Output, "Warning: entity %s.%s keeps access-rule role %s: module %s has no role named %s. "+ + "An entity's access rules may only use its own module's roles — create %s.%s and grant again.\n", + targetModule.Name, entity.Name, qn, targetModule.Name, qn[len(oldPrefix):], targetModule.Name, qn[len(oldPrefix):]) + } +} diff --git a/mdl/executor/cmd_move_document.go b/mdl/executor/cmd_move_document.go index 4805deeef..62b74d7b8 100644 --- a/mdl/executor/cmd_move_document.go +++ b/mdl/executor/cmd_move_document.go @@ -17,6 +17,7 @@ import ( "github.com/mendixlabs/mxcli/mdl/ast" mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" + "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/model" ) @@ -32,7 +33,7 @@ func moveDocumentUnit(ctx *ExecContext, docType ast.DocumentType, name ast.Quali if !ast.IsMoveDocumentType(asked) { return mdlerrors.NewUnsupported("unsupported document type: " + string(docType)) } - doc, err := ctx.Backend.FindDocumentUnit(name.Module, name.Name) + doc, err := findDocumentOfKind(ctx, asked, name) if err != nil { return mdlerrors.NewBackend("find "+asked, err) } @@ -52,6 +53,25 @@ func moveDocumentUnit(ctx *ExecContext, docType ast.DocumentType, name ast.Quali return nil } +// findDocumentOfKind resolves name to the document of the kind the statement +// asked for. A name is unique per document type, not per module — a microflow +// and a scheduled event may both be M.Nightly (mendixlabs/mxcli#1792) — so a +// name-only lookup returned whichever it met first and refused the move as a +// mistyped doctype. With no document of the asked kind, the first one is +// returned, so checkMovedDocumentType can say what the name really is. +func findDocumentOfKind(ctx *ExecContext, asked string, name ast.QualifiedName) (*types.DocumentUnit, error) { + docs, err := ctx.Backend.FindDocumentUnits(name.Module, name.Name) + if err != nil || len(docs) == 0 { + return nil, err + } + for _, d := range docs { + if strings.EqualFold(d.Kind, asked) { + return d, nil + } + } + return docs[0], nil +} + // checkMovedDocumentType refuses a document whose stored type is not the one // the statement named, and says what it actually is. // @@ -66,9 +86,10 @@ func moveDocumentUnit(ctx *ExecContext, docType ast.DocumentType, name ast.Quali // Hence the deliberate asymmetry: refuse only when the derived kind is itself a // doctype MOVE knows how to spell. A kind mxcli has no MDL word for — a page // template, a document type a future Mendix version adds — is a derivation this -// code cannot vouch for, so it defers to the name lookup. Document names are -// unique within a module, so that lookup is the authority; this check is here to -// catch a mistyped doctype, not to second-guess the model. +// code cannot vouch for, so it defers to the name lookup. A name is unique per +// document type, and findDocumentOfKind has already preferred the asked kind, +// so reaching a mismatch here means nothing of that kind has the name; this +// check is here to catch a mistyped doctype, not to second-guess the model. func checkMovedDocumentType(asked, found string, name ast.QualifiedName) error { if found == "" || strings.EqualFold(found, asked) || !ast.IsMoveDocumentType(found) { return nil diff --git a/mdl/executor/cmd_move_same_name_1792_test.go b/mdl/executor/cmd_move_same_name_1792_test.go new file mode 100644 index 000000000..333b304ec --- /dev/null +++ b/mdl/executor/cmd_move_same_name_1792_test.go @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/types" +) + +// mendixlabs/mxcli#1792: a microflow M.Nightly and a scheduled event M.Nightly +// share a name. `move scheduled event M.Nightly to folder Jobs` was refused +// with "M.Nightly is a microflow, not a scheduled event": the lookup was by +// name only and returned whichever document it met first. +func TestMoveSameNamedDocumentPicksTheAskedKind(t *testing.T) { + mod := mkModule("M") + mf := &types.DocumentUnit{ID: nextID("unit"), ContainerID: mod.ID, Name: "Nightly", + Type: "Microflows$Microflow", Kind: "microflow"} + se := &types.DocumentUnit{ID: nextID("unit"), ContainerID: mod.ID, Name: "Nightly", + Type: "ScheduledEvents$ScheduledEvent", Kind: "scheduled event"} + + var probe placementProbe + mb, h := moveMockBackend(t, mod, mf, &probe) // the single lookup meets the microflow first + mb.FindDocumentUnitsFunc = func(moduleName, name string) ([]*types.DocumentUnit, error) { + if moduleName == "M" && name == "Nightly" { + return []*types.DocumentUnit{mf, se}, nil + } + return nil, nil + } + + ctx, buf := newMockCtx(t, withBackend(mb), withHierarchy(h)) + err := execMove(ctx, &ast.MoveStmt{ + DocumentType: ast.DocumentTypeScheduledEvent, + Name: ast.QualifiedName{Module: "M", Name: "Nightly"}, + Folder: "Jobs", + }) + assertNoError(t, err) + if !probe.moved || probe.unitID != se.ID { + t.Fatalf("moved %q (moved=%v), want the scheduled event %q", probe.unitID, probe.moved, se.ID) + } + assertContainsStr(t, buf.String(), "Moved scheduled event M.Nightly") +} + +// Control: with only the microflow under that name, a mistyped doctype is +// still refused, naming what the document is. +func TestMoveSameNameControlStillRefusesAMistypedDoctype(t *testing.T) { + mod := mkModule("M") + mf := &types.DocumentUnit{ID: nextID("unit"), ContainerID: mod.ID, Name: "Nightly", + Type: "Microflows$Microflow", Kind: "microflow"} + var probe placementProbe + mb, h := moveMockBackend(t, mod, mf, &probe) + + ctx, _ := newMockCtx(t, withBackend(mb), withHierarchy(h)) + err := execMove(ctx, &ast.MoveStmt{ + DocumentType: ast.DocumentTypeScheduledEvent, + Name: ast.QualifiedName{Module: "M", Name: "Nightly"}, + Folder: "Jobs", + }) + if err == nil || probe.moved { + t.Fatalf("err = %v, moved = %v; want a refusal and nothing moved", err, probe.moved) + } + assertContainsStr(t, err.Error(), "is a microflow, not a scheduled event") +} diff --git a/mdl/executor/cmd_nanoflows_mock_test.go b/mdl/executor/cmd_nanoflows_mock_test.go index 36bec4e54..f9b4089ae 100644 --- a/mdl/executor/cmd_nanoflows_mock_test.go +++ b/mdl/executor/cmd_nanoflows_mock_test.go @@ -213,7 +213,7 @@ func TestMoveNanoflow_Mock(t *testing.T) { withContainer(h, nf.ContainerID, mod.ID) ctx, _ := newMockCtx(t, withBackend(mb), withHierarchy(h)) - assertNoError(t, moveNanoflow(ctx, ast.QualifiedName{Module: "MyModule", Name: "NF_Move"}, folderID)) + assertNoError(t, moveNanoflow(ctx, ast.QualifiedName{Module: "MyModule", Name: "NF_Move"}, folderID, nil, false)) if movedNF == nil { t.Fatal("Expected MoveNanoflow to be called") @@ -597,7 +597,7 @@ func TestMoveNanoflow_Mock_NotFound(t *testing.T) { } ctx, _ := newMockCtx(t, withBackend(mb), withHierarchy(h)) - err := moveNanoflow(ctx, ast.QualifiedName{Module: "MyModule", Name: "Missing"}, "some-folder-id") + err := moveNanoflow(ctx, ast.QualifiedName{Module: "MyModule", Name: "Missing"}, "some-folder-id", nil, false) assertError(t, err) } diff --git a/mdl/executor/cmd_pages_builder.go b/mdl/executor/cmd_pages_builder.go index 89bc98a5c..fd248f426 100644 --- a/mdl/executor/cmd_pages_builder.go +++ b/mdl/executor/cmd_pages_builder.go @@ -65,6 +65,10 @@ type pageBuilder struct { // Entity context for resolving short attribute names inside DataViews entityContext string // Qualified entity name (e.g., "Module.Entity") + // outerEntityContexts are the entities of the data containers enclosing the + // current one, outermost first: a snippet parameter a call omits is taken + // from any of them, not only the nearest (ako/mxcli#2055). + outerEntityContexts []string // What this point of the walk knows about the object a SHOW_PAGE widget // action would bind its argument to: whether it is knowable at all, whether diff --git a/mdl/executor/cmd_pages_builder_v3_widgets.go b/mdl/executor/cmd_pages_builder_v3_widgets.go index 134ffe6cf..28a232459 100644 --- a/mdl/executor/cmd_pages_builder_v3_widgets.go +++ b/mdl/executor/cmd_pages_builder_v3_widgets.go @@ -95,9 +95,11 @@ func (pb *pageBuilder) buildDataViewV3(w *ast.WidgetV3) (*pages.DataView, error) // Save and restore entity context so nested DataViews work correctly oldContext := pb.entityContext oldArgCtx := pb.argCtx + pb.enterOuterEntityContext(oldContext) pb.entityContext = entityName pb.argCtx = enteringDataWidget(ds, entityName) defer func() { + pb.leaveOuterEntityContext() pb.entityContext = oldContext pb.argCtx = oldArgCtx }() @@ -371,9 +373,11 @@ func (pb *pageBuilder) buildListViewV3(w *ast.WidgetV3) (*pages.ListView, error) // Save and restore entity context so nested containers work correctly oldContext := pb.entityContext oldArgCtx := pb.argCtx + pb.enterOuterEntityContext(oldContext) pb.entityContext = entityName pb.argCtx = enteringDataWidget(ds, entityName) defer func() { + pb.leaveOuterEntityContext() pb.entityContext = oldContext pb.argCtx = oldArgCtx }() @@ -1618,8 +1622,7 @@ func (pb *pageBuilder) buildSnippetCallParams(sc *pages.SnippetCallWidget, snipp // Omitted. Legal exactly when the surrounding data context supplies // the parameter's entity; otherwise nothing can satisfy it and the // author gets the guidance rather than a build error later. - if pb.entityContext != "" && declared.EntityName != "" && - strings.EqualFold(pb.entityContext, declared.EntityName) { + if pb.contextSupplies(declared.EntityName) { continue } return mdlerrors.NewValidationf( @@ -2003,3 +2006,44 @@ func widgetIconSpec(w *ast.WidgetV3) *ast.WidgetIcon { } return nil } + +// enterOuterEntityContext records the context a data container is entered +// from, so a snippet call below it can see every enclosing container's entity; +// leaveOuterEntityContext undoes it. +func (pb *pageBuilder) enterOuterEntityContext(entity string) { + pb.outerEntityContexts = append(pb.outerEntityContexts, entity) +} + +func (pb *pageBuilder) leaveOuterEntityContext() { + if n := len(pb.outerEntityContexts); n > 0 { + pb.outerEntityContexts = pb.outerEntityContexts[:n-1] + } +} + +// contextSupplies reports whether a data container around the current widget +// — the nearest or any enclosing one — supplies an object for a snippet +// parameter of entity want. Studio Pro stores such a call with no parameter +// mapping and mx check accepts it; measured on 11.14 with TestApp's +// WorkflowCommons.ManageTaskAssignments, whose $Account (Administration.Account) +// comes from a System.User data view two containers out (ako/mxcli#2055). An +// entity in want's generalisation chain or one specialising it counts. +func (pb *pageBuilder) contextSupplies(want string) bool { + if want == "" { + return false + } + related := func(ctx string) bool { + if ctx == "" { + return false + } + return strings.EqualFold(ctx, want) || pb.entityInChain(ctx, want) || pb.entityInChain(want, ctx) + } + if related(pb.entityContext) { + return true + } + for _, ctx := range pb.outerEntityContexts { + if related(ctx) { + return true + } + } + return false +} diff --git a/mdl/executor/cmd_pages_describe_datasource.go b/mdl/executor/cmd_pages_describe_datasource.go index 45683f3f3..cdc54d1d7 100644 --- a/mdl/executor/cmd_pages_describe_datasource.go +++ b/mdl/executor/cmd_pages_describe_datasource.go @@ -540,11 +540,33 @@ func dataSourceExpr(ctx *ExecContext, ds *rawDataSource) string { if ds.Reference == "" { return "" } - return associationDataSourceExpr(ds) + return associationDataSourceExpr(withSpecialisedDestination(ctx, ds)) } return "" } +// withSpecialisedDestination returns ds with its single hop spelled +// `Assoc/Destination` when the stored destination is neither end of the +// association — a specialisation of the end, which re-deriving from the +// association alone loses. Measured on TestApp's +// WorkflowCommons.ManageTaskAssignments (11.14): `TaskAssignmentHelper_User` +// stored with DestinationEntity Administration.Account was described as the +// bare association, rebuilt to System.User, and the labels bound to Account's +// FullName became CE1613 (ako/mxcli#2055). A hop that arrives at an end keeps +// its short spelling. +func withSpecialisedDestination(ctx *ExecContext, ds *rawDataSource) *rawDataSource { + if ds.Destination == "" || strings.Contains(ds.Reference, "/") || ctx == nil || ctx.Backend == nil { + return ds + } + ends, ok := projectAssociationEnds(ctx)[ds.Reference] + if !ok || ends[0] == "" || ends[1] == "" || ds.Destination == ends[0] || ds.Destination == ends[1] { + return ds + } + cp := *ds + cp.Reference = ds.Reference + "/" + ds.Destination + return &cp +} + // dataSourceProp renders the whole `DataSource: …` property, or "" when the // datasource cannot be expressed. func dataSourceProp(ctx *ExecContext, ds *rawDataSource) string { diff --git a/mdl/executor/cmd_search.go b/mdl/executor/cmd_search.go index 427ed2971..252dc124a 100644 --- a/mdl/executor/cmd_search.go +++ b/mdl/executor/cmd_search.go @@ -59,6 +59,10 @@ var callerRefKinds = []string{ RefKindCallerPublish, // published REST operation: the microflow behind the endpoint RefKindCallerSettings, // project setting: after-startup, before-shutdown, health check RefKindCallerEvent, // entity event handler: the microflow it runs + // An agent's knowledge bases, MCP services and model are used when it + // runs, by no flow (ako/mxcli#2074); its tool microflows are calls. + RefKindCallerTool, + RefKindCallerModel, } // Kind literals, kept next to the set that uses them so the SQL below cannot @@ -77,6 +81,8 @@ const ( RefKindCallerPublish = "publish" RefKindCallerSettings = "settings" RefKindCallerEvent = "event" + RefKindCallerTool = "tool" + RefKindCallerModel = "model" ) // callerRefKindsSQL renders callerRefKinds as a SQL IN list. diff --git a/mdl/executor/describe_auto.go b/mdl/executor/describe_auto.go index 1e431fb4e..0fca3cfbb 100644 --- a/mdl/executor/describe_auto.go +++ b/mdl/executor/describe_auto.go @@ -61,6 +61,7 @@ var objectTypeToDescribeKind = map[string]ast.DescribeObjectType{ "AI_MODEL": ast.DescribeModel, "KNOWLEDGE_BASE": ast.DescribeKnowledgeBase, "CONSUMED_MCP_SERVICE": ast.DescribeConsumedMCPService, + "PUBLISHED_MCP_SERVICE": ast.DescribePublishedMCPService, } // DescribeKindFor maps a catalog `objects` view ObjectType to the DESCRIBE kind diff --git a/mdl/executor/move_module_split_test.go b/mdl/executor/move_module_split_test.go new file mode 100644 index 000000000..9c0505ee4 --- /dev/null +++ b/mdl/executor/move_module_split_test.go @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" +) + +// ako/mxcli#1836 (mendixlabs/mxcli#1687): splitting a module with MOVE left +// three things naming the old module, all of which mx check reports: +// - the access-rule XPath `[Src.Order_Customer/Src.Customer/Name = 'x']` — +// path steps, which the reference sweep left alone; +// - the entity access rule's module role (an entity's rules may only use its +// own module's roles); +// - a nanoflow's grant, which was not mapped to the target's same-named role +// as a microflow's is (CE0148). +// +// Enumeration and constant references inside expressions, also in the report, +// were already rewritten (#1671, #1674); the test keeps them covered. +func TestMoveModuleSplit_RewritesXPathsRolesAndNanoflowGrants(t *testing.T) { + exec, out, _ := openPedAppCopy(t) + setup := `create module Src1836; +create module Dst1836; +create module role Src1836.User; +create module role Dst1836.User; +create enumeration Src1836.Kind (A 'A', B 'B'); +create constant Src1836.Limit ( Type: Integer, DefaultValue: 5 ); +create persistent entity Src1836.Customer (Name: String(50)); +create persistent entity Src1836.Order (Number: String(20), K: enum Src1836.Kind); +create association Src1836.Order_Customer from Src1836.Order to Src1836.Customer; +grant read * on entity Src1836.Order to Src1836.User where [Src1836.Order_Customer/Src1836.Customer/Name = 'x']; +grant read * on entity Src1836.Customer to Src1836.User; +create microflow Src1836.MF ($O: Src1836.Order) returns Boolean +begin + if $O/K = Src1836.Kind.A and @Src1836.Limit > 3 then + return true; + end if; + return false; +end; +grant execute on microflow Src1836.MF to Src1836.User; +create nanoflow Src1836.NF () returns Boolean +begin + return @Src1836.Limit > 1; +end; +grant execute on nanoflow Src1836.NF to Src1836.User;` + if err := afRun(t, exec, setup); err != nil { + t.Fatalf("setup: %v\n%s", err, out.String()) + } + out.Reset() + moves := `move entity Src1836.Customer to Dst1836; +move entity Src1836.Order to Dst1836; +move enumeration Src1836.Kind to Dst1836; +move constant Src1836.Limit to Dst1836; +move microflow Src1836.MF to Dst1836; +move nanoflow Src1836.NF to Dst1836;` + if err := afRun(t, exec, moves); err != nil { + t.Fatalf("move: %v\n%s", err, out.String()) + } + + describe := func(stmt string) string { + t.Helper() + out.Reset() + if err := afRun(t, exec, stmt); err != nil { + t.Fatalf("%s: %v\n%s", stmt, err, out.String()) + } + return out.String() + } + + order := describe("describe entity Dst1836.Order;") + if !strings.Contains(order, "where [Dst1836.Order_Customer/Dst1836.Customer/Name = 'x']") { + t.Errorf("the access rule's XPath still names the old module:\n%s", order) + } + if !strings.Contains(order, "to Dst1836.User") || strings.Contains(order, "Src1836.User") { + t.Errorf("the access rule keeps the old module's role:\n%s", order) + } + + nf := describe("describe nanoflow Dst1836.NF;") + if !strings.Contains(nf, "grant execute on nanoflow Dst1836.NF to Dst1836.User") { + t.Errorf("the nanoflow's grant was not mapped to the target's role:\n%s", nf) + } + + // Controls: the microflow's grant was always mapped, and its expressions + // rewritten. + mf := describe("describe microflow Dst1836.MF;") + for _, want := range []string{"grant execute on microflow Dst1836.MF to Dst1836.User", "Dst1836.Kind.A", "@Dst1836.Limit"} { + if !strings.Contains(mf, want) { + t.Errorf("control: microflow lacks %q:\n%s", want, mf) + } + } +} + +// A role with no same-named role in the target module is kept, and said: an +// access rule that silently lost its role would hide the entity from it. +func TestMoveEntity_WarnsOfARoleTheTargetLacks(t *testing.T) { + exec, out, _ := openPedAppCopy(t) + setup := `create module SrcW1836; +create module DstW1836; +create module role SrcW1836.Clerk; +create persistent entity SrcW1836.Note (Text: String(50)); +grant read * on entity SrcW1836.Note to SrcW1836.Clerk;` + if err := afRun(t, exec, setup); err != nil { + t.Fatalf("setup: %v\n%s", err, out.String()) + } + out.Reset() + if err := afRun(t, exec, "move entity SrcW1836.Note to DstW1836;"); err != nil { + t.Fatalf("move: %v\n%s", err, out.String()) + } + if s := out.String(); !strings.Contains(s, "SrcW1836.Clerk") || !strings.Contains(s, "DstW1836") || !strings.Contains(strings.ToLower(s), "warning") { + t.Errorf("no warning about the role the target lacks:\n%s", s) + } +} diff --git a/mdl/executor/snippet_call_outer_context_test.go b/mdl/executor/snippet_call_outer_context_test.go new file mode 100644 index 000000000..2a37c7848 --- /dev/null +++ b/mdl/executor/snippet_call_outer_context_test.go @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" +) + +// ako/mxcli#2055: describe → exec of TestApp's WorkflowCommons.ManageTaskAssignments +// was refused — "snippet …Snip_TaskAssignment_UserTask_Assignees requires +// parameter $Account". The call sits in a Data grid 2 of System.WorkflowUserTask +// inside a data view of System.User; Studio Pro stores it with NO parameter +// mappings, and `mx check` (11.14) reports 0 errors: a snippet parameter the +// call omits is taken from an enclosing data container, not only the nearest +// one. The builder looked at the nearest only. PedApp has no Data grid 2 +// package, so the test nests a list view; the TestApp page round-trips too. + +const outerContextSetup = `create module Snip2055; +create persistent entity Snip2055.Item (Name: String(50)); +create snippet Snip2055.ItemAndAccount (Params: ( $Item: Snip2055.Item, $Account: Administration.Account )) { + dynamictext txt1 (Content: 'x') +} +` + +func snippetCallPage(name, outerParamEntity string) string { + return `create page Snip2055.` + name + ` ( + Title: 'x', + Layout: Atlas_Core.Atlas_Default, + Params: ( $Outer: ` + outerParamEntity + ` ) +) { + dataview dvOuter (DataSource: $Outer) { + listview list1 (DataSource: database from Snip2055.Item) { + snippetcall sc1 (Snippet: Snip2055.ItemAndAccount) + } + } +}` +} + +func TestSnippetCall_ParameterFromAnEnclosingDataContainer(t *testing.T) { + exec, out, _ := openPedAppCopy(t) + if err := afRun(t, exec, outerContextSetup); err != nil { + t.Fatalf("setup: %v\n%s", err, out.String()) + } + // The enclosing data view supplies $Account itself, and — as in + // WorkflowCommons — its generalisation System.User. + for name, outer := range map[string]string{"ByAccount": "Administration.Account", "ByUser": "System.User"} { + out.Reset() + if err := afRun(t, exec, snippetCallPage(name, outer)); err != nil { + t.Errorf("%s: a snippet parameter an enclosing %s data view supplies was refused: %v\n%s", name, outer, err, out.String()) + } + } + + // Control: nothing around the grid supplies $Account — still refused. + out.Reset() + err := afRun(t, exec, snippetCallPage("NoAccount", "Snip2055.Item")) + if err == nil || !strings.Contains(err.Error()+out.String(), "requires parameter $Account") { + t.Errorf("control: a call with no context for $Account was accepted: %v\n%s", err, out.String()) + } +} + +// The same page then failed mx check once it built: dataView2's source, +// `$currentObject/WorkflowCommons.TaskAssignmentHelper_User`, is stored with +// the DestinationEntity Administration.Account — a specialisation of the +// association's System.User end — and describe printed the single hop without +// it. The rebuild arrived at System.User, and the labels inside, bound to +// Administration.Account's FullName, became CE1613 "System.User.FullName no +// longer exists" (11.14). A single hop whose stored destination is neither end +// of its association prints the destination. +func TestDescribeAssociationSource_KeepsASpecialisedDestination(t *testing.T) { + exec, out, _ := openPedAppCopy(t) + setup := `create module Spec2055; +create persistent entity Spec2055.Holder (Name: String(20)); +create association Spec2055.Holder_User from Spec2055.Holder to System.User; +create page Spec2055.HolderPage ( + Title: 'x', + Layout: Atlas_Core.Atlas_Default, + Params: ( $Holder: Spec2055.Holder ) +) { + dataview dvHolder (DataSource: $Holder) { + dataview dvAccount (DataSource: $currentObject/Spec2055.Holder_User/Administration.Account) { + dynamictext t1 (Content: '{1}', ContentParams: ({1} = FullName)) + } + dataview dvUser (DataSource: $currentObject/Spec2055.Holder_User) { + dynamictext t2 (Content: '{1}', ContentParams: ({1} = Name)) + } + } +}` + if err := afRun(t, exec, setup); err != nil { + t.Fatalf("setup: %v\n%s", err, out.String()) + } + out.Reset() + if err := afRun(t, exec, "describe page Spec2055.HolderPage;"); err != nil { + t.Fatalf("describe: %v\n%s", err, out.String()) + } + got := out.String() + if !strings.Contains(got, "DataSource: $currentObject/Spec2055.Holder_User/Administration.Account") { + t.Errorf("the specialised destination was dropped:\n%s", got) + } + // Control: a hop that arrives at the association's own end stays short. + if !strings.Contains(got, "DataSource: $currentObject/Spec2055.Holder_User,") && !strings.Contains(got, "DataSource: $currentObject/Spec2055.Holder_User\n") && !strings.Contains(got, "DataSource: $currentObject/Spec2055.Holder_User)") { + t.Errorf("control: the plain hop is not printed short:\n%s", got) + } +} diff --git a/mdl/executor/studiopro_roundtrip_test.go b/mdl/executor/studiopro_roundtrip_test.go index aa0bccef2..538481edb 100644 --- a/mdl/executor/studiopro_roundtrip_test.go +++ b/mdl/executor/studiopro_roundtrip_test.go @@ -58,11 +58,9 @@ import ( // fails in both directions: a check that starts passing must be struck off, and // one that starts failing must be explained. var studioProKnownLossy = map[string]string{ - // Losses this suite found beyond the ones ako/mxcli#705 reported. They are - // the whole-document verdict only; the #705 slices are checked separately. - "page Administration.Account_Overview|document": "DataGrid2 object rebuilt from the " + - "widget template (property order, column texts, filter captions), layout-grid " + - "column weights 12 -> -1, tab-container nulls", + // Empty. The last entry, page Administration.Account_Overview|document, + // passes since its tab container's null default tab is carried + // (ako/mxcli#1897). } type studioProCase struct { diff --git a/mdl/executor/validate.go b/mdl/executor/validate.go index 976fd56d1..93298bce0 100644 --- a/mdl/executor/validate.go +++ b/mdl/executor/validate.go @@ -327,14 +327,14 @@ func (sc *scriptContext) collectSingle(stmt ast.Statement) { if s.ObjectType == "module" && s.NewName != "" { sc.renamedModules[s.NewName] = true } else if s.Name.Module != "" && s.NewName != "" && !s.DryRun { - sc.indirectDocs[s.ObjectType+":"+s.Name.Module+"."+s.NewName] = true + sc.markIndirectDoc(s.ObjectType, s.Name.Module+"."+s.NewName) } case *ast.MoveStmt: if s.DocumentType == ast.DocumentTypeEntity && s.TargetModule != "" { sc.indirectEntities[s.TargetModule+"."+s.Name.Name] = true } if s.TargetModule != "" { - sc.indirectDocs[moveTargetKind(s.DocumentType)+":"+s.TargetModule+"."+s.Name.Name] = true + sc.markIndirectDoc(moveTargetKind(s.DocumentType), s.TargetModule+"."+s.Name.Name) } case *ast.ExecuteScriptStmt: sc.opaqueScript = true @@ -1122,6 +1122,23 @@ func validateWithContext(ctx *ExecContext, stmt ast.Statement, sc *scriptContext } case *ast.CreatePublishedRestServiceStmt: return validatePublishedRestAuthMicroflow(ctx, s.Authentication, sc) + case *ast.CreateConsumedMCPServiceStmt: + return validateConsumedMCPRefs(ctx, sc, s.Name, s.Endpoint, s.AuthenticationMicroflow) + case *ast.AlterConsumedMCPServiceStmt: + var endpoint, mf *ast.QualifiedName + for k, v := range s.Changes { + if v == "" { + continue + } + qn := parseQualifiedNameStr(v) + switch strings.ToLower(k) { + case "endpoint": + endpoint = &qn + case "authenticationmicroflow": + mf = &qn + } + } + return validateConsumedMCPRefs(ctx, sc, s.Name, endpoint, mf) case *ast.CreateMessageDefinitionCollectionStmt: return messageDefinitionMemberErrors(ctx, sc, s.Name.String(), s.Definitions...) case *ast.CreateMessageDefinitionStmt: @@ -1500,6 +1517,38 @@ func carriedExclusion(ctx *ExecContext, kind string, qn ast.QualifiedName, rewri return excluded } +// markIndirectDoc records a document an earlier statement of the script puts +// under qn without creating it — a RENAME, or a MOVE to another module. +// +// It goes into indirectDocs, and into the kind's own script-defined set as +// well: every reference check (show page, call microflow, a page's snippet +// call, …) consults that set, so a document known only to indirectDocs was +// "not found" by the name exec would find it under (mendixlabs/mxcli#1790). +func (sc *scriptContext) markIndirectDoc(kind, qn string) { + if kind == "" { + return + } + sc.indirectDocs[kind+":"+qn] = true + switch kind { + case "page": + sc.pages[qn] = true + case "snippet": + sc.snippets[qn] = true + case "layout": + sc.layouts[qn] = true + case "microflow": + sc.microflows[qn] = true + case "nanoflow": + sc.nanoflows[qn] = true + case "enumeration": + sc.enumerations[qn] = true + case "constant": + sc.constants[qn] = true + case "workflow": + sc.workflows[qn] = true + } +} + // validateFlowBodyReferences validates references in any flow body (microflow or nanoflow). func validateFlowBodyReferences(ctx *ExecContext, body []ast.MicroflowStatement, sc *scriptContext) []string { if !ctx.Connected() || len(body) == 0 { @@ -2383,3 +2432,30 @@ func validatePublishedRestAuthMicroflow(ctx *ExecContext, auth *ast.PublishedRes } return mdlerrors.NewValidationf("MDL-REST04: authentication microflow not found: %s", auth.Microflow) } + +// validateConsumedMCPRefs checks a consumed MCP service's Endpoint and +// AuthenticationMicroflow (#2060). The endpoint must be a String constant: +// AgentEditorCommons reads its value as the server URL. MCPClient calls the +// credentials microflow with no arguments and reads its result as a list of +// System.HttpHeader (ConsumedMCPService_CreateMCPClient), so a parameter or +// another return type fails at runtime. A constant or microflow the script +// itself creates is known by name only, so its type is left to exec. +func validateConsumedMCPRefs(ctx *ExecContext, sc *scriptContext, name ast.QualifiedName, endpoint, mf *ast.QualifiedName) error { + var problems []string + if endpoint != nil && !sc.constants[endpoint.String()] { + if _, err := consumedMCPEndpointRef(ctx, *endpoint); err != nil { + problems = append(problems, err.Error()) + } + } + if mf != nil && !sc.microflows[mf.String()] { + if live, ok := liveMicroflowsByQualifiedName(ctx)[mf.String()]; !ok { + problems = append(problems, "AuthenticationMicroflow not found: "+mf.String()) + } else if p := consumedMCPCredentialsProblem(live); p != "" { + problems = append(problems, "AuthenticationMicroflow "+mf.String()+" "+p) + } + } + if len(problems) == 0 { + return nil + } + return mdlerrors.NewValidationf("consumed mcp service %s: %s", name, strings.Join(problems, "; ")) +} diff --git a/mdl/executor/validate_agent_selected_model.go b/mdl/executor/validate_agent_selected_model.go new file mode 100644 index 000000000..0d2e4d0d8 --- /dev/null +++ b/mdl/executor/validate_agent_selected_model.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/linter" +) + +// agentSelectedModelRule warns on an agent created without a selected model +// (ako/mxcli#2031). +// +// Agent Editor 2.2+ imports an agent only with one: AgentFromModel_Validate +// requires SelectedModelID, and AgentFromModel_CreateUpdateAgent retrieves the +// deployed model where Model = SelectedModelID (measured in AgentEditorCommons +// 2.2.1 and 2.6.0; 1.1.0 does not read it). The refusal comes at startup, from +// ASU_AgentEditor, and it stops the import of every agent document, not just +// this one. A create or modify is not flagged: left out, it keeps the stored +// selected model. +const agentSelectedModelRule = "MDL-AGENT02" + +// ValidateAgentSelectedModel warns on a create agent with no SelectedModel. +func ValidateAgentSelectedModel(prog *ast.Program) []linter.Violation { + var out []linter.Violation + for _, stmt := range prog.Statements { + s, ok := stmt.(*ast.CreateAgentStmt) + if !ok || s.CreateOrModify || (s.SelectedModel != nil && *s.SelectedModel != "") { + continue + } + out = append(out, linter.Violation{ + RuleID: agentSelectedModelRule, + Severity: linter.SeverityWarning, + Message: fmt.Sprintf("agent %s has no SelectedModel: Agent Editor 2.2+ refuses to import it at startup "+ + "(\"Model with name is referenced but could not be found\"), and that stops the import of every agent", s.Name), + Location: linter.Location{DocumentType: "agent", DocumentName: s.Name.String()}, + Suggestion: "Add SelectedModel: '', the id of a model the Model document's resource offers " + + "(e.g. 'eu.anthropic.claude-sonnet-5'). With the app running, MxGenAIConnector.MxCloudDeployedModel " + + "lists them. Only Agent Editor 1.x ignores it.", + }) + } + return out +} diff --git a/mdl/executor/validate_agent_selected_model_test.go b/mdl/executor/validate_agent_selected_model_test.go new file mode 100644 index 000000000..8885e7fe1 --- /dev/null +++ b/mdl/executor/validate_agent_selected_model_test.go @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" +) + +// ako/mxcli#2031: an agent created without a selected model is refused by +// ASU_AgentEditor (Agent Editor 2.2+), and one refused agent stops the import +// of every agent document. check says so before exec. +func TestValidateAgentSelectedModel(t *testing.T) { + prog := parseMDL(t, ` +create agent M.Missing (UsageType: Task, Model: M.Model); +create agent M.Empty (UsageType: Task, Model: M.Model, SelectedModel: ''); +create agent M.Good (UsageType: Task, Model: M.Model, SelectedModel: 'eu.anthropic.claude-sonnet-5'); +create or modify agent M.Kept (UsageType: Task, Model: M.Model); +`) + got := map[string]string{} + for _, v := range ValidateAgentSelectedModel(prog) { + if v.RuleID != agentSelectedModelRule { + t.Errorf("rule %s", v.RuleID) + } + got[v.Location.DocumentName] = v.Message + " | " + v.Suggestion + } + for _, name := range []string{"M.Missing", "M.Empty"} { + if !strings.Contains(got[name], "SelectedModel") { + t.Errorf("%s: %q, want a SelectedModel warning", name, got[name]) + } + } + // Controls: a stated model, and a create or modify (which keeps the stored + // one when the statement leaves it out), say nothing. + for _, name := range []string{"M.Good", "M.Kept"} { + if got[name] != "" { + t.Errorf("%s warned: %s", name, got[name]) + } + } +} diff --git a/mdl/executor/validate_consumed_mcp_endpoint.go b/mdl/executor/validate_consumed_mcp_endpoint.go new file mode 100644 index 000000000..a8909537f --- /dev/null +++ b/mdl/executor/validate_consumed_mcp_endpoint.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/linter" +) + +// consumedMCPEndpointRule warns on a consumed MCP service created without an +// endpoint (ako/mxcli#2060). +// +// AgentEditorCommons' ConsumedMCPService_CreateUpdateFromModel reads the +// endpoint constant's value and, when there is none, logs "No value could be +// found for the local configuration for the constant" and skips the service: +// an agent that uses it gets none of its tools. Measured in AgentEditorCommons +// 1.1.0, 2.2.1 and 2.6.0, which all map endpoint/qualifiedName to +// MCPEndpointConstant. A create or modify is not flagged: left out, it keeps +// the stored endpoint. +const consumedMCPEndpointRule = "MDL-AGENT03" + +// ValidateConsumedMCPEndpoint warns on a create consumed mcp service with no +// Endpoint. +func ValidateConsumedMCPEndpoint(prog *ast.Program) []linter.Violation { + var out []linter.Violation + for _, stmt := range prog.Statements { + s, ok := stmt.(*ast.CreateConsumedMCPServiceStmt) + if !ok || s.CreateOrModify || s.Endpoint != nil { + continue + } + out = append(out, linter.Violation{ + RuleID: consumedMCPEndpointRule, + Severity: linter.SeverityWarning, + Message: fmt.Sprintf("consumed mcp service %s has no Endpoint: Agent Editor skips it at startup "+ + "(\"No value could be found for the local configuration for the constant\"), so it connects nowhere", s.Name), + Location: linter.Location{DocumentType: "consumed mcp service", DocumentName: s.Name.String()}, + Suggestion: "Add Endpoint: @Module.UrlConstant, a String constant holding the server URL " + + "(e.g. 'https://example.com/mcp'); set its value per environment.", + }) + } + return out +} diff --git a/mdl/executor/validate_program.go b/mdl/executor/validate_program.go index dcdb5457f..f1587ef60 100644 --- a/mdl/executor/validate_program.go +++ b/mdl/executor/validate_program.go @@ -333,6 +333,8 @@ func ValidateProgram(prog *ast.Program, projectPath string) []linter.Violation { // been seen to hold. mxbuild never reads it, so a typo builds and only // shows in Studio Pro (ako/mxcli#2023). violations = append(violations, ValidateAgentProviders(prog)...) + violations = append(violations, ValidateAgentSelectedModel(prog)...) + violations = append(violations, ValidateConsumedMCPEndpoint(prog)...) // Flag an after-startup microflow that does not return Boolean (CE0142). // The reference RESOLVES — the name exists — so #274's existence check has diff --git a/mdl/executor/validate_renamed_doc_refs_1790_test.go b/mdl/executor/validate_renamed_doc_refs_1790_test.go new file mode 100644 index 000000000..2c19ca239 --- /dev/null +++ b/mdl/executor/validate_renamed_doc_refs_1790_test.go @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: Apache-2.0 + +// ako/mxcli#2070 (mendixlabs/mxcli#1790): `rename page M.Old to New;` followed +// in the same script by `show page M.New` was refused by check --references +// ("page not found: M.New"), while exec ran it: the rename was recorded only +// where the alter-target check looks, not in the script's page set every +// reference check consults. +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/pages" +) + +func TestValidate_ReferenceToADocumentRenamedEarlierInTheScript(t *testing.T) { + mod := &model.Module{Name: "M"} + mod.ID = "mod1" + other := &model.Module{Name: "N"} + other.ID = "mod2" + stored := &pages.Page{Name: "Old_Page"} + stored.ContainerID = "mod1" + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{mod, other}, nil }, + ListPagesFunc: func() ([]*pages.Page, error) { return []*pages.Page{stored}, nil }, + } + showPage := func(module, page string) ast.MicroflowStatement { + return &ast.ShowPageStmt{PageName: qn(module, page)} + } + cases := []struct { + name string + before []ast.Statement + call ast.MicroflowStatement + wantErr string + }{ + {"the reported script: rename page, then show it by its new name", + []ast.Statement{&ast.RenameStmt{ObjectType: "page", Name: qn("M", "Old_Page"), NewName: "New_Page"}}, + showPage("M", "New_Page"), ""}, + {"a page moved to another module, then shown there", + []ast.Statement{&ast.MoveStmt{DocumentType: ast.DocumentTypePage, Name: qn("M", "Old_Page"), TargetModule: "N"}}, + showPage("N", "Old_Page"), ""}, + {"control: without the rename the new name is not found", + nil, showPage("M", "New_Page"), "page not found: M.New_Page"}, + {"control: a dry-run rename provides nothing", + []ast.Statement{&ast.RenameStmt{ObjectType: "page", Name: qn("M", "Old_Page"), NewName: "New_Page", DryRun: true}}, + showPage("M", "New_Page"), "page not found: M.New_Page"}, + {"control: a renamed microflow does not provide a page", + []ast.Statement{&ast.RenameStmt{ObjectType: "microflow", Name: qn("M", "Old_Page"), NewName: "New_Page"}}, + showPage("M", "New_Page"), "page not found: M.New_Page"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx, _ := newMockCtx(t, withBackend(mb)) + body := []ast.MicroflowStatement{tc.call} + stmts := append(append([]ast.Statement{}, tc.before...), &ast.CreateMicroflowStmt{Name: qn("M", "Caller"), Body: body}) + sc := newScriptContext() + sc.collectDefinitions(&ast.Program{Statements: stmts}) + errs := validateFlowBodyReferences(ctx, body, sc) + if tc.wantErr == "" { + if len(errs) != 0 { + t.Fatalf("refused: %v", errs) + } + return + } + if len(errs) != 1 || !strings.Contains(errs[0], tc.wantErr) { + t.Fatalf("errs = %v, want one containing %q", errs, tc.wantErr) + } + }) + } +} + +// Every kind markIndirectDoc names must have its set allocated, or a rename of +// that kind panics the check. +func TestMarkIndirectDocCoversEveryKindWithoutPanicking(t *testing.T) { + sc := newScriptContext() + for _, k := range []string{"page", "snippet", "layout", "microflow", "nanoflow", "enumeration", "constant", "workflow", "javaaction", ""} { + sc.markIndirectDoc(k, "M.X") + } + if !sc.pages["M.X"] || !sc.workflows["M.X"] || !sc.indirectDocs["javaaction:M.X"] { + t.Fatal("kinds not recorded") + } +} diff --git a/mdl/executor/validate_slot_hidden_1862_test.go b/mdl/executor/validate_slot_hidden_1862_test.go new file mode 100644 index 000000000..3c68cc607 --- /dev/null +++ b/mdl/executor/validate_slot_hidden_1862_test.go @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/types" +) + +// ako/mxcli#1862 (mendixlabs/mxcli#1704): a Data grid 2 `emptyplaceholder { … }` +// block was accepted by check and exec, and the page then failed mx check with +// CE0463 on the grid (measured on TestApp, 11.14). The slot is hidden while +// showEmptyPlaceholder is "none", its default, and a hidden slot holding widgets +// is a definition mismatch. Studio Pro's own grid with a placeholder +// (WorkflowCommons.ConflictResolution) stores showEmptyPlaceholder "custom", and +// a script that says so checks clean. +// +// MDL-WIDGET10 never saw it: a slot is filled by a child block, not by a +// `key: value` property, so it never counted as set. Gallery shares the slot +// and the rule; the rule is supplied here as the installed .mpk supplies it. + +func galleryWithPlaceholderRule(t *testing.T) *WidgetRegistry { + t.Helper() + reg, err := NewWidgetRegistry() + if err != nil { + t.Fatalf("registry: %v", err) + } + real, ok := reg.Get("GALLERY") + if !ok { + t.Fatal("no GALLERY definition") + } + def := *real + def.PropertyVisibility = []types.WidgetVisibilityRule{{ + PropertyKey: "emptyPlaceholder", + HiddenWhen: &types.WidgetVisibilityCondition{PropertyKey: "showEmptyPlaceholder", Operator: "eq", Value: "none"}, + }} + return &WidgetRegistry{byMDLName: map[string]*WidgetDefinition{"GALLERY": &def}} +} + +func TestCheck1862_PlaceholderUnderNoneIsRefused(t *testing.T) { + reg := galleryWithPlaceholderRule(t) + w := parseSnippetWidget(t, `gallery g1 (DataSource: database from M.E) { + emptyplaceholder { dynamictext t1 (Content: 'No items found.') } +}`) + v := validateWidgetVisibility(w, reg, "page P") + if len(v) != 1 || v[0].RuleID != "MDL-WIDGET10" { + t.Fatalf("got %d violation(s) %+v, want 1 MDL-WIDGET10 for the hidden slot", len(v), v) + } + if v[0].Severity.String() != "error" { + t.Errorf("severity = %v, want error: it is CE0463 at mx check", v[0].Severity) + } +} + +// Controls: showEmptyPlaceholder 'custom' shows the slot, and an empty grid +// with no placeholder block sets nothing. +func TestCheck1862_PlaceholderUnderCustomAndNoPlaceholderAreClean(t *testing.T) { + reg := galleryWithPlaceholderRule(t) + for name, src := range map[string]string{ + "custom": `gallery g1 (DataSource: database from M.E, showEmptyPlaceholder: 'custom') { + emptyplaceholder { dynamictext t1 (Content: 'No items found.') } +}`, + "no placeholder": `gallery g1 (DataSource: database from M.E)`, + } { + if v := validateWidgetVisibility(parseSnippetWidget(t, src), reg, "page P"); len(v) != 0 { + t.Errorf("%s: got %+v, want no violation", name, v) + } + } +} diff --git a/mdl/executor/validate_widget_hidden.go b/mdl/executor/validate_widget_hidden.go index 96ef78648..72de68443 100644 --- a/mdl/executor/validate_widget_hidden.go +++ b/mdl/executor/validate_widget_hidden.go @@ -49,6 +49,14 @@ func hiddenPropertySeverity(value, def, operation string) (linter.Severity, stri // page with the slot unset builds clean. (#956) return linter.SeverityError, "an action there fails the build with CE0463" } + if value == slotFilledValue { + // A widgets slot holding widgets, like an action, has no default to + // compare against. Measured on a Data grid 2 at 11.14: widgets in + // `emptyPlaceholder` while showEmptyPlaceholder is "none" fail mx check + // with CE0463; with "custom", or with the slot empty, the page is clean + // (ako/mxcli#1862). + return linter.SeverityError, "widgets there fail the build with CE0463; set the condition's property so the slot shows" + } if def == "" || strings.EqualFold(value, def) { return linter.SeverityWarning, "the value will be ignored" } diff --git a/mdl/executor/validate_widgets.go b/mdl/executor/validate_widgets.go index 8b8bd4b78..189caad47 100644 --- a/mdl/executor/validate_widgets.go +++ b/mdl/executor/validate_widgets.go @@ -867,6 +867,29 @@ func widgetValueMap(w *ast.WidgetV3, def *WidgetDefinition) (values map[string]s values[key] = val } } + // A widgets slot filled by a child block (`emptyplaceholder { … }`) is set as + // surely as a `key: value` property, but has no property to read. Counting + // it lets MDL-WIDGET10 see a slot the configuration hides: a Data grid 2 + // placeholder under showEmptyPlaceholder "none" built and then failed mx + // check with CE0463 (ako/mxcli#1862). + if slots, _ := selectedChildSlots(def, w); len(slots) > 0 { + for _, c := range w.Children { + if c == nil || len(c.Children) == 0 { + continue + } + for _, cs := range slots { + if strings.EqualFold(cs.MDLContainer, defaultSlotContainer) { + continue // the catch-all template is not a choice the script made + } + if strings.EqualFold(cs.MDLContainer, c.Type) || strings.EqualFold(cs.PropertyKey, c.Name) || + (strings.EqualFold(c.Type, "container") && strings.EqualFold(cs.MDLContainer, c.Name)) { + key := strings.ToLower(cs.PropertyKey) + values[key] = slotFilledValue + explicit[key] = true + } + } + } + } // Direct MDL properties named after a widget key (not covered by a mapping). for k, raw := range w.Properties { lk := strings.ToLower(k) @@ -881,6 +904,10 @@ func widgetValueMap(w *ast.WidgetV3, def *WidgetDefinition) (values map[string]s return values, explicit } +// slotFilledValue stands for a widgets slot holding widgets in the value map; +// a slot's default is empty, so a hidden filled slot is reported as an error. +const slotFilledValue = "(widgets)" + // lookupWidgetProp reads a widget property from the MDL by name (case-insensitive) // as a string, reporting whether it was present. func lookupWidgetProp(w *ast.WidgetV3, name string) (string, bool) { diff --git a/mdl/executor/widget_engine.go b/mdl/executor/widget_engine.go index 70ee8237d..b867974c9 100644 --- a/mdl/executor/widget_engine.go +++ b/mdl/executor/widget_engine.go @@ -357,7 +357,9 @@ func (e *PluggableWidgetEngine) Build(def *WidgetDefinition, w *ast.WidgetV3) (* // Save and restore entity context (DataSource mappings may change it) oldEntityContext := e.pageBuilder.entityContext oldArgCtx := e.pageBuilder.argCtx + e.pageBuilder.enterOuterEntityContext(oldEntityContext) defer func() { + e.pageBuilder.leaveOuterEntityContext() e.pageBuilder.entityContext = oldEntityContext e.pageBuilder.argCtx = oldArgCtx }() diff --git a/mdl/grammar/domains/MDLCatalog.g4 b/mdl/grammar/domains/MDLCatalog.g4 index d05f08d16..cfe03c06f 100644 --- a/mdl/grammar/domains/MDLCatalog.g4 +++ b/mdl/grammar/domains/MDLCatalog.g4 @@ -268,6 +268,9 @@ catalogTableName | OBJECTS // keyword token — must be listed explicitly | COMMUNITIES // keyword token (SHOW COMMUNITIES) — must be listed explicitly for CATALOG.COMMUNITIES | QUEUES // keyword token (SHOW QUEUES) — must be listed explicitly for CATALOG.QUEUES + | FOLDERS // keyword token (LIST FOLDERS) — must be listed explicitly for CATALOG.FOLDERS (ako/mxcli#1436) + | AGENTS // keyword token (SHOW AGENTS) — CATALOG.AGENTS parsed to nothing until TestEveryCatalogTableParses caught it + | ANNOTATIONS // keyword token (LIST ANNOTATIONS) — for CATALOG.ANNOTATIONS (ako/mxcli#1300) | SOURCE_KW // For CATALOG.SOURCE FTS table | ODATA // For CATALOG.ODATA_CLIENTS and CATALOG.ODATA_SERVICES (via IDENTIFIER) | IDENTIFIER // For tables like activities, xpath_expressions, projects, snapshots, refs, strings, odata_clients, odata_services, java_actions diff --git a/mdl/linter/context.go b/mdl/linter/context.go index b906c2ad5..abe8a1504 100644 --- a/mdl/linter/context.go +++ b/mdl/linter/context.go @@ -793,6 +793,9 @@ type Enumeration struct { // EnumerationValue is one value of an enumeration, with its caption in the // project's default language. type EnumerationValue struct { + // ID is the value's $ID: the element_id of its captions in strings(), one + // row per language (ako/mxcli#1301). + ID string Name string QualifiedName string // Module.Enumeration.Value, as refs_from() names it Caption string @@ -805,7 +808,7 @@ type EnumerationValue struct { func (ctx *LintContext) enumerationValues() map[string][]EnumerationValue { out := map[string][]EnumerationValue{} rows, err := ctx.db.Query(` - SELECT EnumerationQualifiedName, Name, COALESCE(Caption, ''), COALESCE(Ordinal, 0) + SELECT EnumerationQualifiedName, COALESCE(Id, ''), Name, COALESCE(Caption, ''), COALESCE(Ordinal, 0) FROM enumeration_values ORDER BY EnumerationQualifiedName, Ordinal, Name`) if err != nil { @@ -816,7 +819,7 @@ func (ctx *LintContext) enumerationValues() map[string][]EnumerationValue { for rows.Next() { var enum string var v EnumerationValue - if err := rows.Scan(&enum, &v.Name, &v.Caption, &v.Ordinal); err != nil { + if err := rows.Scan(&enum, &v.ID, &v.Name, &v.Caption, &v.Ordinal); err != nil { ctx.recordQueryError("EnumerationValues (row scan)", err) continue } @@ -948,6 +951,10 @@ type Widget struct { DynamicClasses string ActionType string // stored $Type of the primary action, e.g. "Forms$DeleteClientAction" HasConfirmation bool // only microflow / nanoflow / workflow calls can have one + // Caption and Tooltip in the project's default language (ako/mxcli#1303). + // Other languages: strings() rows whose widget_id is this widget's ID. + Caption string + Tooltip string } // Widgets returns an iterator over all widgets (excluding system modules). @@ -958,7 +965,7 @@ func (ctx *LintContext) Widgets() iter.Seq[Widget] { w.ContainerType, w.ModuleName, w.EntityRef, w.AttributeRef, w.MicroflowRef, w.NanoflowRef, w.PageRef, w.SnippetRef, w.ParentWidgetId, w.Depth, w.Class, w.Style, w.DynamicClasses, - w.ActionType, w.HasConfirmation + w.ActionType, w.HasConfirmation, w.Caption, w.Tooltip FROM widgets w LEFT JOIN modules m ON w.ModuleName = m.Name WHERE %s AND %s @@ -973,11 +980,11 @@ func (ctx *LintContext) Widgets() iter.Seq[Widget] { for rows.Next() { var w Widget var containerID, containerQName, containerType, entityRef, attrRef, mfRef, nfRef, pageRef, snippetRef sql.NullString - var parentID, class, style, dynClasses, actionType sql.NullString + var parentID, class, style, dynClasses, actionType, caption, tooltip sql.NullString var depth, hasConfirmation sql.NullInt64 err := rows.Scan(&w.ID, &w.Name, &w.WidgetType, &containerID, &containerQName, &containerType, &w.ModuleName, &entityRef, &attrRef, &mfRef, &nfRef, &pageRef, &snippetRef, - &parentID, &depth, &class, &style, &dynClasses, &actionType, &hasConfirmation) + &parentID, &depth, &class, &style, &dynClasses, &actionType, &hasConfirmation, &caption, &tooltip) if err != nil { ctx.recordQueryError("Widgets (row scan)", err) continue @@ -998,6 +1005,8 @@ func (ctx *LintContext) Widgets() iter.Seq[Widget] { w.DynamicClasses = dynClasses.String w.ActionType = actionType.String w.HasConfirmation = hasConfirmation.Int64 != 0 + w.Caption = caption.String + w.Tooltip = tooltip.String if ctx.IsExcluded(w.ModuleName) { continue diff --git a/mdl/linter/context_catalog_tables.go b/mdl/linter/context_catalog_tables.go index ea468ca7d..bc3ab3585 100644 --- a/mdl/linter/context_catalog_tables.go +++ b/mdl/linter/context_catalog_tables.go @@ -232,6 +232,7 @@ type CatalogString struct { Language string // "" for text that is not translatable ElementID string ModuleName string + WidgetID string // the widget the text belongs to (ako/mxcli#1303); "" outside a widget } // Strings iterates the catalog's strings outside System and Marketplace modules, @@ -249,7 +250,8 @@ func (ctx *LintContext) Strings(language string) iter.Seq[CatalogString] { rows, err := ctx.db.Query(fmt.Sprintf(` SELECT COALESCE(s.QualifiedName, ''), COALESCE(s.ObjectType, ''), COALESCE(s.StringValue, ''), COALESCE(s.StringContext, ''), - COALESCE(s.Language, ''), COALESCE(s.ElementId, ''), COALESCE(s.ModuleName, '') + COALESCE(s.Language, ''), COALESCE(s.ElementId, ''), COALESCE(s.ModuleName, ''), + COALESCE(s.WidgetId, '') FROM strings s LEFT JOIN modules m ON s.ModuleName = m.Name WHERE %s AND %s AND %s @@ -264,7 +266,7 @@ func (ctx *LintContext) Strings(language string) iter.Seq[CatalogString] { for rows.Next() { var s CatalogString if err := rows.Scan(&s.QualifiedName, &s.ObjectType, &s.Value, &s.Context, - &s.Language, &s.ElementID, &s.ModuleName); err != nil { + &s.Language, &s.ElementID, &s.ModuleName, &s.WidgetID); err != nil { ctx.recordQueryError("Strings (row scan)", err) continue } @@ -460,6 +462,99 @@ func (ctx *LintContext) Modules() iter.Seq[Module] { } } +// Annotation is one domain model annotation (ako/mxcli#1300). A microflow's +// annotations are activities, read through activities_for(). +type Annotation struct { + ID string + ModuleName string + Caption string // the full text, newlines kept + X, Y int + Width int +} + +// Annotations iterates the domain model annotations outside System and +// Marketplace modules. +func (ctx *LintContext) Annotations() iter.Seq[Annotation] { + return func(yield func(Annotation) bool) { + rows, err := ctx.db.Query(fmt.Sprintf(` + SELECT a.Id, COALESCE(a.ModuleName, ''), COALESCE(a.Caption, ''), + COALESCE(a.X, 0), COALESCE(a.Y, 0), COALESCE(a.Width, 0) + FROM annotations a + LEFT JOIN modules m ON a.ModuleName = m.Name + WHERE %s + ORDER BY a.ModuleName, a.Y, a.X, a.Id + `, notPlatformModule("m"))) + if err != nil { + ctx.recordQueryError("Annotations", err) + return + } + defer rows.Close() + + for rows.Next() { + var a Annotation + if err := rows.Scan(&a.ID, &a.ModuleName, &a.Caption, &a.X, &a.Y, &a.Width); err != nil { + ctx.recordQueryError("Annotations (row scan)", err) + continue + } + if ctx.IsExcluded(a.ModuleName) { + continue + } + if !yield(a) { + return + } + } + } +} + +// Folder is one folder of a module, empty folders included (ako/mxcli#1436). +// A document's Folder field names only folders that hold a document, so an +// empty folder appears nowhere else. +type Folder struct { + ID string + Name string + ModuleName string + Path string // "Objects/Order", spelled as a document's Folder field + ParentPath string // "" for a folder directly in the module + Depth int // 1 for a folder directly in the module + DocumentCount int // documents directly in this folder + FolderCount int // folders directly in this folder +} + +// Folders iterates the folders outside System and Marketplace modules. +func (ctx *LintContext) Folders() iter.Seq[Folder] { + return func(yield func(Folder) bool) { + rows, err := ctx.db.Query(fmt.Sprintf(` + SELECT f.Id, COALESCE(f.Name, ''), COALESCE(f.ModuleName, ''), COALESCE(f.Path, ''), + COALESCE(f.ParentPath, ''), COALESCE(f.Depth, 1), + COALESCE(f.DocumentCount, 0), COALESCE(f.FolderCount, 0) + FROM folders f + LEFT JOIN modules m ON f.ModuleName = m.Name + WHERE %s + ORDER BY f.ModuleName, f.Path + `, notPlatformModule("m"))) + if err != nil { + ctx.recordQueryError("Folders", err) + return + } + defer rows.Close() + + for rows.Next() { + var f Folder + if err := rows.Scan(&f.ID, &f.Name, &f.ModuleName, &f.Path, &f.ParentPath, + &f.Depth, &f.DocumentCount, &f.FolderCount); err != nil { + ctx.recordQueryError("Folders (row scan)", err) + continue + } + if ctx.IsExcluded(f.ModuleName) { + continue + } + if !yield(f) { + return + } + } + } +} + // ConstantValue is a constant's value in one run configuration // (Settings → Configurations), which overrides the constant's default there. type ConstantValue struct { diff --git a/mdl/linter/jar_files.go b/mdl/linter/jar_files.go new file mode 100644 index 000000000..35cea8038 --- /dev/null +++ b/mdl/linter/jar_files.go @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: Apache-2.0 + +package linter + +import ( + "os" + "path/filepath" + "sort" + "strings" + + "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" +) + +// JarFile is one .jar file in the project's userlib/ or vendorlib/ +// (ako/mxcli#1304). jar_dependencies() lists the Maven dependencies module +// settings declare; this lists what is on disk, so a rule can find one library +// on the classpath twice, in two versions. +type JarFile struct { + Folder string // "userlib" or "vendorlib" + FileName string // "commons-lang3-3.12.0.jar" + Name string // "commons-lang3" + Version string // "3.12.0"; "" when the file name carries none + RequiredBy []string // modules with a ..RequiredLib marker beside it +} + +// jarFolders are the folders the Mendix runtime puts on the classpath. Only +// their top level is read: a jar in a subfolder is not on it. +var jarFolders = []string{"userlib", "vendorlib"} + +// JarFiles reads the .jar files from disk each time it is called. They are +// not in the .mpr, so the catalog does not hold them and no refresh is needed. +// It returns nothing when the reader does not know the project's path. +func (ctx *LintContext) JarFiles() []JarFile { + p, ok := ctx.reader.(interface{ Path() string }) + if !ok || p.Path() == "" { + return nil + } + projectDir := filepath.Dir(p.Path()) + + var out []JarFile + for _, folder := range jarFolders { + entries, err := os.ReadDir(filepath.Join(projectDir, folder)) + if err != nil { + continue // no such folder: no jars in it + } + required := map[string][]string{} + for _, e := range entries { + // commons-lang3-3.12.0.jar.MyFirstModule.RequiredLib + n := e.Name() + if e.IsDir() || !strings.HasSuffix(n, ".RequiredLib") { + continue + } + rest := strings.TrimSuffix(n, ".RequiredLib") + i := strings.LastIndex(rest, ".jar.") + if i < 0 { + continue + } + jar := rest[:i+len(".jar")] + required[jar] = append(required[jar], rest[i+len(".jar."):]) + } + for _, e := range entries { + n := e.Name() + if e.IsDir() || !strings.EqualFold(filepath.Ext(n), ".jar") { + continue + } + name, version := splitJarName(strings.TrimSuffix(n, filepath.Ext(n))) + by := required[n] + sort.Strings(by) + out = append(out, JarFile{Folder: folder, FileName: n, Name: name, Version: version, RequiredBy: by}) + } + } + return out +} + +// splitJarName splits "commons-lang3-3.12.0" into "commons-lang3" and +// "3.12.0" at the LAST dash followed by a digit, so a name that itself holds +// a version-like part keeps it: "log4j-1.2-api-2.17.1" is log4j-1.2-api 2.17.1, +// and "guava-31.1-jre" is guava 31.1-jre. +func splitJarName(base string) (name, version string) { + for i := len(base) - 2; i > 0; i-- { + if base[i] == '-' && base[i+1] >= '0' && base[i+1] <= '9' { + return base[:i], base[i+1:] + } + } + return base, "" +} + +func (r *StarlarkRule) builtinJarFiles(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if r.ctx == nil { + return starlark.NewList(nil), nil + } + var out []starlark.Value + for _, j := range r.ctx.JarFiles() { + by := make([]starlark.Value, 0, len(j.RequiredBy)) + for _, m := range j.RequiredBy { + by = append(by, starlark.String(m)) + } + out = append(out, starlarkstruct.FromStringDict(starlark.String("jar_file"), starlark.StringDict{ + "folder": starlark.String(j.Folder), + "file_name": starlark.String(j.FileName), + "name": starlark.String(j.Name), + "version": starlark.String(j.Version), + "required_by": starlark.NewList(by), + })) + } + return starlark.NewList(out), nil +} diff --git a/mdl/linter/rules/conv_error_handling.go b/mdl/linter/rules/conv_error_handling.go index 2ff94035c..e39dccc81 100644 --- a/mdl/linter/rules/conv_error_handling.go +++ b/mdl/linter/rules/conv_error_handling.go @@ -4,6 +4,7 @@ package rules import ( "fmt" + "strings" "github.com/mendixlabs/mxcli/mdl/linter" "github.com/mendixlabs/mxcli/model" @@ -60,12 +61,15 @@ func findUnhandledCalls(objects []microflows.MicroflowObject, mf linter.Microflo continue } actionName := "" - switch act.Action.(type) { + switch a := act.Action.(type) { case *microflows.RestCallAction: actionName = "REST call" case *microflows.WebServiceCallAction: actionName = "Web service call" case *microflows.JavaActionCallAction: + if isThrowingJavaAction(a.JavaAction) { + continue + } actionName = "Java action call" default: continue @@ -96,13 +100,28 @@ func findUnhandledCalls(objects []microflows.MicroflowObject, mf linter.Microflo }) } case *microflows.LoopedActivity: - if act.ObjectCollection != nil { - findUnhandledCalls(act.ObjectCollection.Objects, mf, r, violations) - } + // Not descended into: inside a loop Mendix allows only Rollback + // (CE0644, which check reports as MDL006), so a finding there could + // not be acted on in place (ako/mxcli#1838). Custom handling for a + // call in a loop means moving it into a sub-microflow — a design + // choice, not a defect of the call. } } } +// isThrowingJavaAction reports whether a Java action exists to throw, such as +// CommunityCommons.ThrowException and ThrowWebserviceException. A flow calls +// one to roll itself back — MDL refuses `raise error` in the main flow +// (MDL084/CE0710) — so Rollback is the point of the call, not an omission +// (ako/mxcli#1838). +func isThrowingJavaAction(qualifiedName string) bool { + name := qualifiedName + if i := strings.LastIndex(name, "."); i >= 0 { + name = name[i+1:] + } + return strings.HasPrefix(strings.ToLower(name), "throw") +} + // --- CONV014: NoContinueErrorHandling --- // NoContinueErrorHandlingRule flags activities with "Continue" error handling, which silently swallows errors. diff --git a/mdl/linter/rules/conv_error_handling_test.go b/mdl/linter/rules/conv_error_handling_test.go index c0c15bc4a..56c6c6920 100644 --- a/mdl/linter/rules/conv_error_handling_test.go +++ b/mdl/linter/rules/conv_error_handling_test.go @@ -130,6 +130,10 @@ func TestFindUnhandledCalls_NonExternalAction(t *testing.T) { } } +// ako/mxcli#1838 (mendixlabs/mxcli#1689): inside a loop Mendix allows only +// Rollback (CE0644, which check reports as MDL006), so a CONV013 finding there +// could not be fixed in place — one rule or the other always fired. A call in +// a loop is not reported; its handling is the loop's. func TestFindUnhandledCalls_InsideLoop(t *testing.T) { loopBody := µflows.MicroflowObjectCollection{ Objects: []microflows.MicroflowObject{ @@ -151,8 +155,34 @@ func TestFindUnhandledCalls_InsideLoop(t *testing.T) { r := NewErrorHandlingOnCallsRule() findUnhandledCalls(objects, testMicroflow(), r, &violations) + if len(violations) != 0 { + t.Errorf("a call inside a loop can only roll back (CE0644); want no finding, got %d", len(violations)) + } +} + +// ako/mxcli#1838: a call to an action that always throws, such as +// CommunityCommons.ThrowException, is how a flow rolls itself back — MDL +// refuses `raise error` in the main flow (MDL084/CE0710). Its Rollback is the +// point of the call; a custom handler beside it is what another rule flags. +func TestFindUnhandledCalls_ThrowingJavaActionIsDeliberate(t *testing.T) { + call := func(javaAction string) []microflows.MicroflowObject { + return []microflows.MicroflowObject{µflows.ActionActivity{ + Action: µflows.JavaActionCallAction{JavaAction: javaAction}, + }} + } + r := NewErrorHandlingOnCallsRule() + for _, ja := range []string{"CommunityCommons.ThrowException", "CommunityCommons.ThrowWebserviceException"} { + var violations []linter.Violation + findUnhandledCalls(call(ja), testMicroflow(), r, &violations) + if len(violations) != 0 { + t.Errorf("%s: want no finding for a deliberate throw, got %v", ja, violations) + } + } + // Control: any other Java action with Rollback handling is still reported. + var violations []linter.Violation + findUnhandledCalls(call("CommunityCommons.Base64Encode"), testMicroflow(), r, &violations) if len(violations) != 1 { - t.Errorf("expected 1 violation inside loop, got %d", len(violations)) + t.Errorf("control: want 1 finding for a Java action that does not throw, got %d", len(violations)) } } diff --git a/mdl/linter/rules/react_unsupported_widget_test.go b/mdl/linter/rules/react_unsupported_widget_test.go index 4b6310e3a..21049214c 100644 --- a/mdl/linter/rules/react_unsupported_widget_test.go +++ b/mdl/linter/rules/react_unsupported_widget_test.go @@ -112,7 +112,7 @@ func TestReactUnsupportedWidgetRule_SkipsNativePages(t *testing.T) { `CREATE TABLE widgets (Id TEXT, Name TEXT, WidgetType TEXT, ContainerId TEXT, ContainerQualifiedName TEXT, ContainerType TEXT, ModuleName TEXT, EntityRef TEXT, AttributeRef TEXT, MicroflowRef TEXT, NanoflowRef TEXT, PageRef TEXT, SnippetRef TEXT, ParentWidgetId TEXT, Depth INTEGER, Class TEXT, Style TEXT, DynamicClasses TEXT, - ActionType TEXT, HasConfirmation INTEGER)`, + ActionType TEXT, HasConfirmation INTEGER, Caption TEXT, Tooltip TEXT)`, `INSERT INTO widgets (Id, Name, WidgetType, ContainerId, ContainerQualifiedName, ContainerType, ModuleName, EntityRef, AttributeRef, MicroflowRef, NanoflowRef) VALUES ('w1', 'imgWith', 'Forms$StaticImageViewer', 'p1', 'MyFirstModule.Logboek_Images', 'PAGE', 'MyFirstModule', '', '', '', ''), @@ -154,7 +154,7 @@ func TestReactUnsupportedWidgetRule_LayoutWidgets(t *testing.T) { `CREATE TABLE widgets (Id TEXT, Name TEXT, WidgetType TEXT, ContainerId TEXT, ContainerQualifiedName TEXT, ContainerType TEXT, ModuleName TEXT, EntityRef TEXT, AttributeRef TEXT, MicroflowRef TEXT, NanoflowRef TEXT, PageRef TEXT, SnippetRef TEXT, ParentWidgetId TEXT, Depth INTEGER, Class TEXT, Style TEXT, DynamicClasses TEXT, - ActionType TEXT, HasConfirmation INTEGER)`, + ActionType TEXT, HasConfirmation INTEGER, Caption TEXT, Tooltip TEXT)`, `INSERT INTO widgets (Id, Name, WidgetType, ContainerId, ContainerQualifiedName, ContainerType, ModuleName) VALUES ('w1', 'imgLogoWeb', 'Forms$StaticImageViewer', 'l1', 'MyFirstModule.Lay_Web', 'LAYOUT', 'MyFirstModule'), ('w2', 'imgLogoNative', 'Forms$StaticImageViewer', 'l2', 'MyFirstModule.Lay_Native', 'LAYOUT', 'MyFirstModule')`, diff --git a/mdl/linter/starlark.go b/mdl/linter/starlark.go index af42930ca..78511e851 100644 --- a/mdl/linter/starlark.go +++ b/mdl/linter/starlark.go @@ -428,6 +428,9 @@ func (r *StarlarkRule) buildPredeclared() starlark.StringDict { "layouts": starlark.NewBuiltin("layouts", r.builtinLayouts), "published_rest_operations": starlark.NewBuiltin("published_rest_operations", r.builtinPublishedRestOperations), "modules": starlark.NewBuiltin("modules", r.builtinModules), + "folders": starlark.NewBuiltin("folders", r.builtinFolders), + "annotations": starlark.NewBuiltin("annotations", r.builtinAnnotations), + "jar_files": starlark.NewBuiltin("jar_files", r.builtinJarFiles), // Graph-analysis facts (populated by `refresh catalog communities`). "community_of": starlark.NewBuiltin("community_of", r.builtinCommunityOf), @@ -1276,6 +1279,7 @@ func enumerationValuesToStarlark(values []EnumerationValue) *starlark.List { out := make([]starlark.Value, 0, len(values)) for _, v := range values { out = append(out, starlarkstruct.FromStringDict(starlark.String("enumeration_value"), starlark.StringDict{ + "id": starlark.String(v.ID), "name": starlark.String(v.Name), "qualified_name": starlark.String(v.QualifiedName), "caption": starlark.String(v.Caption), @@ -1328,6 +1332,8 @@ func widgetToStarlark(w Widget) starlark.Value { "dynamic_classes": starlark.String(w.DynamicClasses), "action_type": starlark.String(w.ActionType), "has_confirmation": starlark.Bool(w.HasConfirmation), + "caption": starlark.String(w.Caption), + "tooltip": starlark.String(w.Tooltip), }) } diff --git a/mdl/linter/starlark_arch001_test.go b/mdl/linter/starlark_arch001_test.go index a22ee1214..83f5238c6 100644 --- a/mdl/linter/starlark_arch001_test.go +++ b/mdl/linter/starlark_arch001_test.go @@ -56,3 +56,64 @@ func TestARCH001ExemptsSystemWorkflowEntities(t *testing.T) { }) } } + +// ako/mxcli#2069: ARCH001 flagged the data view on ConversationalUI.ChatContext +// that Conversational UI's chat snippet needs — the snippet takes that entity as +// its parameter, so the module's own entity is the only thing that can feed it, +// and its own full-screen chat page does the same. A cross-module entity is +// exempt when the page calls a snippet (or links to a page) of that entity's +// module that takes the entity as a parameter. Controls: the same entity on a +// page that calls no such snippet, and a snippet of that module whose parameter +// is a different entity. +func TestARCH001ExemptsAnEntityFedToItsOwnModulesSnippet(t *testing.T) { + cat, err := catalog.NewFromFile(filepath.Join(t.TempDir(), "cat.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { cat.Close() }) + db := cat.CatalogDB() + for _, q := range []string{ + `INSERT INTO modules_data (Id, Name, Source, ProjectId, SnapshotId) VALUES ('m', 'ServiceDesk', '', 'p', 's')`, + `INSERT INTO widgets_data (Id, Name, WidgetType, ContainerId, ContainerQualifiedName, ContainerType, ModuleName, EntityRef, ProjectId, SnapshotId) VALUES + ('w1', 'dvChat', 'Forms$DataView', 'pg1', 'ServiceDesk.Chat', 'PAGE', 'ServiceDesk', 'ConversationalUI.ChatContext', 'p', 's'), + ('w2', 'dvLinked', 'Forms$DataView', 'pg2', 'ServiceDesk.ChatLink', 'PAGE', 'ServiceDesk', 'ConversationalUI.ChatContext', 'p', 's'), + ('w3', 'dvBare', 'Forms$DataView', 'pg3', 'ServiceDesk.ChatBare', 'PAGE', 'ServiceDesk', 'ConversationalUI.ChatContext', 'p', 's'), + ('w4', 'dvOther', 'Forms$DataView', 'pg4', 'ServiceDesk.Msg', 'PAGE', 'ServiceDesk', 'ConversationalUI.Message', 'p', 's')`, + `INSERT INTO refs (SourceType, SourceId, SourceName, TargetType, TargetName, RefKind) VALUES + ('PAGE', 'pg1', 'ServiceDesk.Chat', 'SNIPPET', 'ConversationalUI.Snippet_ChatContext_ConversationalUI', 'snippet_call'), + ('SNIPPET', 'sn1', 'ConversationalUI.Snippet_ChatContext_ConversationalUI', 'ENTITY', 'ConversationalUI.ChatContext', 'parameter'), + ('PAGE', 'pg2', 'ServiceDesk.ChatLink', 'PAGE', 'ConversationalUI.ConversationalUI_FullScreenChat', 'action'), + ('PAGE', 'pgx', 'ConversationalUI.ConversationalUI_FullScreenChat', 'ENTITY', 'ConversationalUI.ChatContext', 'parameter'), + ('PAGE', 'pg4', 'ServiceDesk.Msg', 'SNIPPET', 'ConversationalUI.Snippet_ChatContext_ConversationalUI', 'snippet_call')`, + } { + if _, err := db.Exec(q); err != nil { + t.Fatalf("%s: %v", q, err) + } + } + + for _, dir := range []string{".claude/lint-rules", "cmd/mxcli/lint-rules"} { + t.Run(dir, func(t *testing.T) { + r, err := linter.LoadStarlarkRule(filepath.Join("..", "..", dir, "cross_module_data.star")) + if err != nil { + t.Fatal(err) + } + got := r.Check(linter.NewLintContext(cat, &minimalReader{})) + flagged := map[string]bool{} + var msgs []string + for _, v := range got { + msgs = append(msgs, v.Message) + for _, w := range []string{"dvChat", "dvLinked", "dvBare", "dvOther"} { + if strings.Contains(v.Message, "'"+w+"'") { + flagged[w] = true + } + } + } + if flagged["dvChat"] || flagged["dvLinked"] { + t.Errorf("an entity fed to its own module's snippet or page was flagged:\n%s", strings.Join(msgs, "\n")) + } + if !flagged["dvBare"] || !flagged["dvOther"] { + t.Errorf("controls not flagged (want dvBare and dvOther):\n%s", strings.Join(msgs, "\n")) + } + }) + } +} diff --git a/mdl/linter/starlark_catalog_data_1453_test.go b/mdl/linter/starlark_catalog_data_1453_test.go index db8f39432..fb55299b5 100644 --- a/mdl/linter/starlark_catalog_data_1453_test.go +++ b/mdl/linter/starlark_catalog_data_1453_test.go @@ -71,6 +71,46 @@ func TestStarlarkEnumerationCarriesItsValuesAndCaptions(t *testing.T) { } } +// ako/mxcli#1301, the half #1453 left: values() carries the caption in the +// default language only. A caption in another language is a strings() row whose +// element_id is the value's $ID, and a rule had no id on the value to match it +// with. Measured on CitrusGrove, all 144 enumeration caption rows match a value +// by id. The control row has the same text under another value's id, so a match +// on anything but the id picks the wrong one. +func TestStarlarkEnumerationValueIdMatchesItsTranslations(t *testing.T) { + cat := emptyCatalog(t) + for _, stmt := range []string{ + `INSERT INTO modules_data (Id, Name, QualifiedName, ModuleName, Source) VALUES ('m1', 'Shop', 'Shop', 'Shop', '')`, + `INSERT INTO enumerations_data (Id, Name, QualifiedName, ModuleName, ValueCount) VALUES + ('e1', 'ApiTimeout', 'Shop.ApiTimeout', 'Shop', 2)`, + `INSERT INTO enumeration_values_data (Id, EnumerationId, EnumerationQualifiedName, ModuleName, Name, Caption, Ordinal) VALUES + ('v1', 'e1', 'Shop.ApiTimeout', 'Shop', 'Quick', '30', 0), + ('v2', 'e1', 'Shop.ApiTimeout', 'Shop', 'Slow', '400', 1)`, + `INSERT INTO strings (QualifiedName, ObjectType, StringValue, StringContext, Language, ElementId, ModuleName) VALUES + ('Shop.ApiTimeout', 'ENUMERATION', '30', 'Enumerations$EnumerationValue.Caption', 'en_US', 'v1', 'Shop'), + ('Shop.ApiTimeout', 'ENUMERATION', '400', 'Enumerations$EnumerationValue.Caption', 'en_US', 'v2', 'Shop'), + ('Shop.ApiTimeout', 'ENUMERATION', 'traag', 'Enumerations$EnumerationValue.Caption', 'nl_NL', 'v2', 'Shop'), + ('Shop.ApiTimeout', 'ENUMERATION', 'snel', 'Enumerations$EnumerationValue.Caption', 'nl_NL', 'v1', 'Shop')`, + } { + if _, err := cat.CatalogDB().Exec(stmt); err != nil { + t.Fatalf("%s: %v", stmt, err) + } + } + got := runDump(t, linter.NewLintContext(cat, nil), ` nl = {} + for s in strings(language = "nl_NL"): + if s.context == "Enumerations$EnumerationValue.Caption": + nl[s.element_id] = s.value + out = [] + for e in enumerations(): + for v in e.values: + out.append(violation(message = "%s id=%s nl=%s" % (v.qualified_name, v.id, nl.get(v.id, "?")))) + return out`) + want := "Shop.ApiTimeout.Quick id=v1 nl=snel|Shop.ApiTimeout.Slow id=v2 nl=traag" + if strings.Join(got, "|") != want { + t.Errorf("value ids = %v, want %s", got, want) + } +} + func TestStarlarkConstantValuesAndProjectSettings(t *testing.T) { cat := emptyCatalog(t) if _, err := cat.CatalogDB().Exec(`INSERT INTO constant_values (ConstantName, ConfigurationName, Value) VALUES diff --git a/mdl/linter/starlark_catalog_mode_guard_test.go b/mdl/linter/starlark_catalog_mode_guard_test.go index 0e2f918cc..9e6a7be1e 100644 --- a/mdl/linter/starlark_catalog_mode_guard_test.go +++ b/mdl/linter/starlark_catalog_mode_guard_test.go @@ -61,6 +61,9 @@ var builtinModes = map[string]builtinProbe{ "project_settings": {CatalogFast, "[project_settings()]"}, "attributes_for": {CatalogFast, "[a for n in ENTITY_NAMES for a in attributes_for(n)]"}, "modules": {CatalogFast, "modules()"}, + "folders": {CatalogFast, "folders()"}, + "annotations": {CatalogFast, "annotations()"}, + "jar_files": {CatalogFast, "jar_files()"}, "associations": {CatalogFast, "associations()"}, "entity_event_handlers": {CatalogFast, "entity_event_handlers()"}, "navigation_menu_items": {CatalogFast, "navigation_menu_items()"}, diff --git a/mdl/linter/starlark_catalog_tables.go b/mdl/linter/starlark_catalog_tables.go index 9b95479f2..cb40e8adc 100644 --- a/mdl/linter/starlark_catalog_tables.go +++ b/mdl/linter/starlark_catalog_tables.go @@ -115,6 +115,52 @@ func (r *StarlarkRule) builtinModules(_ *starlark.Thread, _ *starlark.Builtin, a return starlark.NewList(out), nil } +func (r *StarlarkRule) builtinAnnotations(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if r.ctx == nil { + return starlark.NewList(nil), nil + } + var out []starlark.Value + for a := range r.ctx.Annotations() { + out = append(out, annotationToStarlark(a)) + } + return starlark.NewList(out), nil +} + +func annotationToStarlark(a Annotation) starlark.Value { + return starlarkstruct.FromStringDict(starlark.String("annotation"), starlark.StringDict{ + "id": starlark.String(a.ID), + "module_name": starlark.String(a.ModuleName), + "caption": starlark.String(a.Caption), + "x": starlark.MakeInt(a.X), + "y": starlark.MakeInt(a.Y), + "width": starlark.MakeInt(a.Width), + }) +} + +func (r *StarlarkRule) builtinFolders(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if r.ctx == nil { + return starlark.NewList(nil), nil + } + var out []starlark.Value + for f := range r.ctx.Folders() { + out = append(out, folderToStarlark(f)) + } + return starlark.NewList(out), nil +} + +func folderToStarlark(f Folder) starlark.Value { + return starlarkstruct.FromStringDict(starlark.String("folder"), starlark.StringDict{ + "id": starlark.String(f.ID), + "name": starlark.String(f.Name), + "module_name": starlark.String(f.ModuleName), + "path": starlark.String(f.Path), + "parent_path": starlark.String(f.ParentPath), + "depth": starlark.MakeInt(f.Depth), + "document_count": starlark.MakeInt(f.DocumentCount), + "folder_count": starlark.MakeInt(f.FolderCount), + }) +} + func associationToStarlark(a Association) starlark.Value { return starlarkstruct.FromStringDict(starlark.String("association"), starlark.StringDict{ "name": starlark.String(a.Name), @@ -176,6 +222,7 @@ func catalogStringToStarlark(s CatalogString) starlark.Value { "context": starlark.String(s.Context), "language": starlark.String(s.Language), "element_id": starlark.String(s.ElementID), + "widget_id": starlark.String(s.WidgetID), "module_name": starlark.String(s.ModuleName), }) } diff --git a/mdl/linter/starlark_catalog_tables_test.go b/mdl/linter/starlark_catalog_tables_test.go index aea023303..660f8bd9c 100644 --- a/mdl/linter/starlark_catalog_tables_test.go +++ b/mdl/linter/starlark_catalog_tables_test.go @@ -61,6 +61,16 @@ func catalogTablesFixture(t *testing.T) *catalog.Catalog { IsIncluded, ProjectId, SnapshotId) VALUES (?,?,?,?,?,?,?,?,?)`, "j-"+mod, mod, "org.example", "lib", "org.example:lib", "1.2.3", 0, "p", "s") + // A domain model annotation (ako/mxcli#1300); the newline must survive. + exec(`INSERT INTO annotations_data (Id, DomainModelId, ModuleName, Caption, X, Y, Width, + ProjectId, SnapshotId) + VALUES (?,?,?,?,?,?,?,?,?)`, + "an-"+mod, "dm-"+mod, mod, "TODO split\nper region", 60, 40, 400, "p", "s") + // An empty folder: nothing else in the catalog names it (ako/mxcli#1436). + exec(`INSERT INTO folders_data (Id, Name, ModuleName, Path, ParentPath, Depth, DocumentCount, + FolderCount, ProjectId, SnapshotId) + VALUES (?,?,?,?,?,?,?,?,?,?)`, + "fo-"+mod, "UI", mod, "UI", "", 1, 0, 0, "p", "s") exec(`INSERT INTO layouts_data (Id, Name, QualifiedName, ModuleName, Folder, LayoutType, Platform, Description, ProjectId, SnapshotId) VALUES (?,?,?,?,?,?,?,?,?,?)`, @@ -71,9 +81,9 @@ func catalogTablesFixture(t *testing.T) *catalog.Catalog { "o-"+mod, "svc-"+mod, mod+".OrderApi", "orders", "Get", "/{id}", "Get one order", mod+".PRS_GetOrder", 1, mod, "p", "s") for _, lang := range []string{"en_US", "nl_NL"} { - exec(`INSERT INTO strings (QualifiedName, ObjectType, StringValue, StringContext, Language, ElementId, ModuleName) - VALUES (?,?,?,?,?,?,?)`, - mod+".Order_Overview", "PAGE", "Orders "+lang, "Forms$Page.Title", lang, "w-"+mod, mod) + exec(`INSERT INTO strings (QualifiedName, ObjectType, StringValue, StringContext, Language, ElementId, ModuleName, WidgetId) + VALUES (?,?,?,?,?,?,?,?)`, + mod+".Order_Overview", "PAGE", "Orders "+lang, "Forms$ClientTemplate.Template", lang, "tpl-"+mod, mod, "btn-"+mod) } } // Navigation belongs to the project: no module, no module filter. @@ -153,14 +163,14 @@ func TestCatalogTableBuiltins(t *testing.T) { "module_name=Sales|version=1.2.3", }}, {"strings()", []string{ - "context=Forms$Page.Title|element_id=w-Sales|language=en_US|module_name=Sales|object_type=PAGE|" + - "qualified_name=Sales.Order_Overview|value=Orders en_US", - "context=Forms$Page.Title|element_id=w-Sales|language=nl_NL|module_name=Sales|object_type=PAGE|" + - "qualified_name=Sales.Order_Overview|value=Orders nl_NL", + "context=Forms$ClientTemplate.Template|element_id=tpl-Sales|language=en_US|module_name=Sales|object_type=PAGE|" + + "qualified_name=Sales.Order_Overview|value=Orders en_US|widget_id=btn-Sales", + "context=Forms$ClientTemplate.Template|element_id=tpl-Sales|language=nl_NL|module_name=Sales|object_type=PAGE|" + + "qualified_name=Sales.Order_Overview|value=Orders nl_NL|widget_id=btn-Sales", }}, {`strings(language = "nl_NL")`, []string{ - "context=Forms$Page.Title|element_id=w-Sales|language=nl_NL|module_name=Sales|object_type=PAGE|" + - "qualified_name=Sales.Order_Overview|value=Orders nl_NL", + "context=Forms$ClientTemplate.Template|element_id=tpl-Sales|language=nl_NL|module_name=Sales|object_type=PAGE|" + + "qualified_name=Sales.Order_Overview|value=Orders nl_NL|widget_id=btn-Sales", }}, // An untranslated language has no row, rather than an empty one. {`strings("de_DE")`, nil}, @@ -172,6 +182,12 @@ func TestCatalogTableBuiltins(t *testing.T) { "deprecated=True|http_method=Get|microflow=Sales.PRS_GetOrder|module_name=Sales|path=/{id}|" + "resource=orders|service=Sales.OrderApi|summary=Get one order", }}, + {"annotations()", []string{ + "caption=TODO split\nper region|id=an-Sales|module_name=Sales|width=400|x=60|y=40", + }}, + {"folders()", []string{ + "depth=1|document_count=0|folder_count=0|id=fo-Sales|module_name=Sales|name=UI|parent_path=|path=UI", + }}, {"modules()", []string{ "domain_model_documentation=Orders and customers.|id=mod-sales|name=Sales", }}, diff --git a/mdl/linter/starlark_jar_files_test.go b/mdl/linter/starlark_jar_files_test.go new file mode 100644 index 000000000..e06bd012c --- /dev/null +++ b/mdl/linter/starlark_jar_files_test.go @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: Apache-2.0 + +package linter_test + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "github.com/mendixlabs/mxcli/mdl/linter" +) + +// pathReader is a reader that knows the project file, as the MPR backend does. +type pathReader struct { + minimalReader + path string +} + +func (r *pathReader) Path() string { return r.path } + +// ako/mxcli#1304: a rule could not see the .jar files in userlib/ and +// vendorlib/. jar_dependencies() lists the managed Maven dependencies from +// module settings only, so two versions of one library on the classpath -- a +// hand-added jar in userlib/ and the managed one sync-java-deps put in +// vendorlib/ -- went unseen. The project below is the issue's repro. +func TestJarFiles_Issue1304(t *testing.T) { + dir := t.TempDir() + write := func(rel string) { + t.Helper() + p := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, nil, 0o644); err != nil { + t.Fatal(err) + } + } + write("App.mpr") + write("userlib/commons-lang3-3.12.0.jar") + write("userlib/commons-lang3-3.12.0.jar.MyFirstModule.RequiredLib") + write("userlib/commons-lang3-3.12.0.jar.Sales.RequiredLib") + write("userlib/log4j-1.2-api-2.17.1.jar") // a name with a version-like part + write("userlib/README.txt") // control: not a jar + write("userlib/sub/nested-1.0.jar") // control: the runtime does not read subfolders + write("vendorlib/commons-lang3-3.14.0.jar") + write("vendorlib/unversioned.jar") + + ctx := linter.NewLintContext(emptyCatalog(t), &pathReader{path: filepath.Join(dir, "App.mpr")}) + vs, _ := runSrc(t, ctx, dumpRule("jar_files()")) + var got []string + for _, v := range vs { + got = append(got, v.Message) + } + want := []string{ + "file_name=commons-lang3-3.12.0.jar|folder=userlib|name=commons-lang3|required_by=[\"MyFirstModule\", \"Sales\"]|version=3.12.0", + "file_name=log4j-1.2-api-2.17.1.jar|folder=userlib|name=log4j-1.2-api|required_by=[]|version=2.17.1", + "file_name=commons-lang3-3.14.0.jar|folder=vendorlib|name=commons-lang3|required_by=[]|version=3.14.0", + "file_name=unversioned.jar|folder=vendorlib|name=unversioned|required_by=[]|version=", + } + if !slices.Equal(got, want) { + t.Errorf("jar_files():\n got %q\n want %q", got, want) + } + + // Without a project directory there is nothing to read: [] rather than an + // error, as the other reader-backed builtins answer without a reader. + vs, _ = runSrc(t, linter.NewLintContext(emptyCatalog(t), &minimalReader{}), dumpRule("jar_files()")) + if len(vs) != 0 { + t.Errorf("jar_files() without a project path: %d rows, want 0", len(vs)) + } +} diff --git a/mdl/linter/widgets_projection_test.go b/mdl/linter/widgets_projection_test.go index fdec911eb..3e7743378 100644 --- a/mdl/linter/widgets_projection_test.go +++ b/mdl/linter/widgets_projection_test.go @@ -4,6 +4,7 @@ package linter_test import ( "path/filepath" + "strings" "testing" "github.com/mendixlabs/mxcli/mdl/catalog" @@ -107,3 +108,52 @@ def check(): t.Errorf("projection = %q\nwant %q", vs[0].Message, want) } } + +// ako/mxcli#1303: with two buttons on a page a rule could not tell which +// caption belonged to the one without a confirmation: widgets() had no caption, +// and the strings() caption row's element_id is the button's +// Forms$ClientTemplate, not the button. A row now carries caption and tooltip +// in the default language, and each strings() row the widget_id, so the +// Dutch caption is found by the widget's id. +func TestWidgets_CaptionTooltipAndTheirTranslations(t *testing.T) { + cat, err := catalog.NewFromFile(filepath.Join(t.TempDir(), "cat.db")) + if err != nil { + t.Fatalf("NewFromFile: %v", err) + } + defer cat.Close() + db := cat.CatalogDB() + for _, q := range []string{ + `INSERT INTO modules_data (Id, Name, ProjectId, SnapshotId) VALUES ('mod-1', 'Sales', 'default', 's1')`, + `INSERT INTO widgets_data (Id, Name, WidgetType, ContainerId, ContainerQualifiedName, ContainerType, + ModuleName, ActionType, HasConfirmation, Caption, Tooltip, ProjectId, SnapshotId) VALUES + ('b1', 'btnDelete', 'Forms$ActionButton', 'p', 'Sales.Sale_Edit', 'PAGE', 'Sales', 'Forms$MicroflowAction', 0, 'Delete sale', 'Deletes it', 'default', 's1'), + ('b2', 'btnRemove', 'Forms$ActionButton', 'p', 'Sales.Sale_Edit', 'PAGE', 'Sales', 'Forms$MicroflowAction', 1, 'Remove', '', 'default', 's1')`, + `INSERT INTO strings (QualifiedName, ObjectType, StringValue, StringContext, Language, ElementId, ModuleName, WidgetId) VALUES + ('Sales.Sale_Edit', 'PAGE', 'Verkoop verwijderen', 'Forms$ClientTemplate.Template', 'nl_NL', 'tpl1', 'Sales', 'b1'), + ('Sales.Sale_Edit', 'PAGE', 'Weghalen', 'Forms$ClientTemplate.Template', 'nl_NL', 'tpl2', 'Sales', 'b2'), + ('Sales.Sale_Edit', 'PAGE', 'Deze verkoop verwijderen?', 'Forms$ConfirmationInfo.Question', 'nl_NL', 'conf2', 'Sales', 'b2')`, + } { + if _, err := db.Exec(q); err != nil { + t.Fatalf("%s: %v", q, err) + } + } + vs, _ := runSrc(t, linter.NewLintContext(cat, nil), ` +def check(): + nl = {} + for s in strings(language = "nl_NL"): + nl.setdefault(s.widget_id, []).append(s.value) + out = [] + for w in widgets(): + out.append(violation(message = "%s|%s|%s|%s|%s" % (w.name, w.caption, w.tooltip, w.has_confirmation, ",".join(sorted(nl.get(w.id, [])))))) + return out +`) + var got []string + for _, v := range vs { + got = append(got, v.Message) + } + want := "btnDelete|Delete sale|Deletes it|False|Verkoop verwijderen;" + + "btnRemove|Remove||True|Deze verkoop verwijderen?,Weghalen" + if strings.Join(got, ";") != want { + t.Errorf("got %q\nwant %q", strings.Join(got, ";"), want) + } +} diff --git a/mdl/roundtrip/allowlist_test.go b/mdl/roundtrip/allowlist_test.go index 6dbc6b479..6bcd67d25 100644 --- a/mdl/roundtrip/allowlist_test.go +++ b/mdl/roundtrip/allowlist_test.go @@ -75,5 +75,4 @@ var knownFailures = map[string]knownFailure{ // AlternativeText and Items). // Pages: #705 item 1 plus #721 C. - "page Administration.Account_Overview": {laws: []law{lawGetPut}, issue: "#705 #721", why: "texts and translations (#705 item 1); widget properties (#721 C)"}, } diff --git a/mdl/roundtrip/testapp_allowlist_test.go b/mdl/roundtrip/testapp_allowlist_test.go index 70f706bfa..2c19395f3 100644 --- a/mdl/roundtrip/testapp_allowlist_test.go +++ b/mdl/roundtrip/testapp_allowlist_test.go @@ -64,116 +64,90 @@ package roundtrip // PageVariable naming only the widget, which describe dropped and exec could // not write back — the rebuilt button called its microflow unbound (CE1571). var testAppKnownFailures = map[string]knownFailure{ - "building block Atlas_Web_Content.Alert": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.AlertIcon": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.AlertIcon_WithAction": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Alert_WithAction": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Breadcrumb": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Breadcrumb_Underline": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Card": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Card_Action": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Card_ActionWithImage": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Card_Background": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Card_WithImage": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Form_Horizontal": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Form_Horizontal_WithAction": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Form_Horizontal_WithTitle": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Form_Vertical": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Form_Vertical_WithAction": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Form_Vertical_WithTitle": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Heroheader": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Heroheader_Background": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Heroheader_WithAction": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.ListItem_DoubleLine": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.ListItem_SingleLine": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.ListItem_WithImage": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.List_Cards": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.List_WithImage": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Master_Detail": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Pageheader": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.PageheaderImage": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.PageheaderImage_WithBack": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.PageheaderImage_WithControls": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Pageheader_WithBack": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Pageheader_WithControls": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Pageheader_WithSearch": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Timeline": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Timeline_WithImage": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Wizard_Arrow": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Wizard_Arrow_Step": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Wizard_Circle": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block Atlas_Web_Content.Wizard_Circle_Step": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "building block FeedbackModule.FeedbackWidget": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "entity WorkflowCommons.ConflictedWorkflowView": {laws: []law{lawExec}, issue: "#721", why: "entity: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "entity WorkflowCommons.MyInitiatedWorkflowView": {laws: []law{lawExec}, issue: "#721", why: "entity: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "entity WorkflowCommons.UserTaskView": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "entity: executes since a Studio Pro index keeps its GUID on CREATE OR MODIFY (mendixlabs/mxcli#1470); re-executing adds a fourth access rule (the XPath-scoped grant is not matched to its stored rule) and the documentation's trailing whitespace drifts — not yet triaged into #721's classes"}, - "entity WorkflowCommons.WorkflowAttachment": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "entity: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "entity WorkflowCommons.WorkflowComment": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "entity: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "entity WorkflowCommons.WorkflowGroupView": {laws: []law{lawParse}, issue: "#721", why: "entity: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "entity WorkflowCommons.WorkflowUserTaskView": {laws: []law{lawExec}, issue: "#721", why: "entity: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "entity WorkflowCommons.WorkflowView": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "entity: executes since a Studio Pro index keeps its GUID on CREATE OR MODIFY (mendixlabs/mxcli#1470); re-executing adds a fourth access rule (the XPath-scoped grant is not matched to its stored rule) and the documentation's trailing whitespace drifts — not yet triaged into #721's classes"}, - "enumeration WorkflowCommons.Enum_NotificationArea_RenderAs": {laws: []law{lawGetPut}, issue: "#721", why: "enumeration: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "export mapping Services.OrdersExportMapping": {laws: []law{lawGetPut}, issue: "#721", why: "export mapping: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page Administration.Account_Overview": {laws: []law{lawGetPut}, issue: "#721", why: "page: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.DefaultWorkflowAdmin": {laws: []law{lawGetPut}, issue: "#721", why: "page: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.ManageTaskAssignments": {laws: []law{lawExec}, issue: "#721", why: "page: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.MyInitiatedWorkflows": {laws: []law{lawGetPut}, issue: "#721", why: "page: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.TaskAssignmentHelper_UserTask_Reassign_TargetUserOptions": {laws: []law{lawGetPut}, issue: "#721", why: "page: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.TaskAssignmentHelper_UserTask_Retarget_TargetUserOptions": {laws: []law{lawGetPut}, issue: "#721", why: "page: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.TaskAssignmentHelper_UserTask_Unassign_TargetUserOptions": {laws: []law{lawGetPut}, issue: "#721", why: "page: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.TaskDashboard": {laws: []law{lawGetPut}, issue: "#721", why: "page: breaks getput on TestApp (putget struck by #721 L4), measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.TaskInbox": {laws: []law{lawGetPut}, issue: "#721", why: "page: breaks getput on TestApp (putget struck by #721 L4), measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.UserTask_Assign": {laws: []law{lawGetPut}, issue: "#721", why: "page: breaks getput on TestApp (putget since this branch's widget fixes), measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.UserTask_Target": {laws: []law{lawGetPut}, issue: "#721", why: "page: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.UserTask_TargetGroups": {laws: []law{lawGetPut}, issue: "#721", why: "page: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.WorkflowAttachment_View": {laws: []law{lawExec}, issue: "#721", why: "page: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.WorkflowCommentHelper_Edit": {laws: []law{lawGetPut}, issue: "#721", why: "page: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.WorkflowComment_Edit_Admin": {laws: []law{lawGetPut}, issue: "#721", why: "page: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.WorkflowDefinition_CleanUp": {laws: []law{lawGetPut}, issue: "#721", why: "page: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.WorkflowDefinition_View": {laws: []law{lawExec}, issue: "#721", why: "page: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.WorkflowSubProcess_View": {laws: []law{lawExec}, issue: "#721", why: "page: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "page WorkflowCommons.Workflow_Dashboard": {laws: []law{lawGetPut}, issue: "#721", why: "page: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "rest client Clients.OrdersRestClient": {laws: []law{lawParse}, issue: "#721", why: "rest client: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "rule Rules.Rule1": {laws: []law{lawGetPut}, issue: "#721", why: "rule: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "rule Rules.Rule2": {laws: []law{lawGetPut}, issue: "#721", why: "rule: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "scheduled event WorkflowCommons.SE_WorkflowAuditTrailRecord_CleanUp": {laws: []law{lawGetPut}, issue: "#721", why: "scheduled event: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_Assignee_Search": {laws: []law{lawGetPut}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes; executes since a widget bound to a snippet parameter is described as $Param.Attr — getput not yet triaged"}, - "snippet WorkflowCommons.Snip_AuditTrailViewer_Toolbar": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "snippet: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_MyInitiatedWorkflowView_NotificationArea": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "snippet: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_MyInitiatedWorkflowView_StateCircleOnly": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "snippet: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_TaskAssignmentHelper_Assignee_Search": {laws: []law{lawGetPut}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes; executes since a widget bound to a snippet parameter is described as $Param.Attr — getput not yet triaged"}, - "snippet WorkflowCommons.Snip_TaskAssignmentHelper_TargetUser_Search": {laws: []law{lawGetPut}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes; executes since a widget bound to a snippet parameter is described as $Param.Attr — getput not yet triaged"}, - "snippet WorkflowCommons.Snip_TaskAssignment_UserTask_Assignees": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "snippet: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_TaskAssignment_UserTask_TargetUsers": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "snippet: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_TimelineViewer_Timeline_Full": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_TimelineViewer_Timeline_Tasks": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_UserTask_DueDate_Warning": {laws: []law{lawGetPut}, issue: "#721", why: "snippet: breaks getput on TestApp (#743); putget passes since a template parameter written as an expression is stored as one (#823 option C)"}, - "snippet WorkflowCommons.Snip_UserTask_NameColumn": {laws: []law{lawGetPut, lawPutGet}, issue: "#721 #826", why: "executes since describe prints snippet call Params (#826); an association-path template parameter read from a snippet parameter loses its AttributeRef and SourceVariable (#721)"}, - "snippet WorkflowCommons.Snip_UserTask_NotificationArea": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "snippet: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_WorkflowActivityRecord_ActivityIcon": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_WorkflowActivityRecord_Details": {laws: []law{lawGetPut}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes; executes since a widget bound to a snippet parameter is described as $Param.Attr — getput not yet triaged"}, - "snippet WorkflowCommons.Snip_WorkflowActivityRecord_Header": {laws: []law{lawGetPut, lawPutGet}, issue: "#721 #826", why: "executes since describe prints snippet call Params (#826); association-path template parameters and an expression parameter drift (#721)"}, - "snippet WorkflowCommons.Snip_WorkflowActivityRecord_TaskDueDate_Warning": {laws: []law{lawGetPut}, issue: "#721", why: "snippet: breaks getput on TestApp (#743); putget passes since a template parameter written as an expression is stored as one (#823 option C)"}, - "snippet WorkflowCommons.Snip_WorkflowActivity_Details_ActivityType": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_WorkflowAuditTrailRecord_Overview": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_WorkflowDashboard_TaskNumbers": {laws: []law{lawGetPut}, issue: "#721", why: "snippet: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_WorkflowDashboard_WorkflowNumbers": {laws: []law{lawGetPut}, issue: "#721", why: "snippet: breaks getput on TestApp (#743); putget passes since a template parameter written as an expression is stored as one (#823 option C)"}, - "snippet WorkflowCommons.Snip_WorkflowDefinition_AuditTrail": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_WorkflowSubProcess_Details": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_WorkflowSubProcess_IncompatibleWarning": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_WorkflowSubProcess_State": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_WorkflowUserTaskView_Details": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_WorkflowUserTaskView_DueDate_Warning": {laws: []law{lawGetPut}, issue: "#721", why: "snippet: breaks getput on TestApp (#743); putget passes since a template parameter written as an expression is stored as one (#823 option C)"}, - "snippet WorkflowCommons.Snip_WorkflowUserTaskView_NameColumn": {laws: []law{lawGetPut, lawPutGet}, issue: "#721 #826", why: "executes since describe prints snippet call Params (#826); an association-path template parameter read from a snippet parameter loses its AttributeRef and SourceVariable (#721)"}, - "snippet WorkflowCommons.Snip_Workflow_AuditTrail": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_Workflow_CommentsAndAttachments": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_Workflow_CommentsAndAttachments_Admin": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_Workflow_Detail": {laws: []law{lawGetPut}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes; executes since a widget bound to a snippet parameter is described as $Param.Attr — getput not yet triaged"}, - "snippet WorkflowCommons.Snip_Workflow_DueDate_Warning": {laws: []law{lawGetPut}, issue: "#721", why: "snippet: breaks getput on TestApp (#743); putget passes since a template parameter written as an expression is stored as one (#823 option C)"}, - "snippet WorkflowCommons.Snip_Workflow_Header": {laws: []law{lawGetPut, lawPutGet}, issue: "#721 #826", why: "executes since describe prints snippet call Params (#826); an association-path caption parameter read from a snippet parameter loses its AttributeRef and SourceVariable (#721)"}, - "snippet WorkflowCommons.Snip_Workflow_NotificationArea": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "snippet: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_Workflow_StateCircleOnly": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "snippet: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "snippet WorkflowCommons.Snip_Workflow_SubProcessDetails": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Alert": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.AlertIcon": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.AlertIcon_WithAction": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Alert_WithAction": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Breadcrumb": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Breadcrumb_Underline": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Card": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Card_Action": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Card_ActionWithImage": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Card_Background": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Card_WithImage": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Form_Horizontal": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Form_Horizontal_WithAction": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Form_Horizontal_WithTitle": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Form_Vertical": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Form_Vertical_WithAction": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Form_Vertical_WithTitle": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Heroheader": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Heroheader_Background": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Heroheader_WithAction": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.ListItem_DoubleLine": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.ListItem_SingleLine": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.ListItem_WithImage": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.List_Cards": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.List_WithImage": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Master_Detail": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Pageheader": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.PageheaderImage": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.PageheaderImage_WithBack": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.PageheaderImage_WithControls": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Pageheader_WithBack": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Pageheader_WithControls": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Pageheader_WithSearch": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Timeline": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Timeline_WithImage": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Wizard_Arrow": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Wizard_Arrow_Step": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Wizard_Circle": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block Atlas_Web_Content.Wizard_Circle_Step": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "building block FeedbackModule.FeedbackWidget": {laws: []law{lawParse}, issue: "#721", why: "building block: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "entity WorkflowCommons.ConflictedWorkflowView": {laws: []law{lawExec}, issue: "#721", why: "entity: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "entity WorkflowCommons.MyInitiatedWorkflowView": {laws: []law{lawExec}, issue: "#721", why: "entity: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "entity WorkflowCommons.UserTaskView": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "entity: executes since a Studio Pro index keeps its GUID on CREATE OR MODIFY (mendixlabs/mxcli#1470); re-executing adds a fourth access rule (the XPath-scoped grant is not matched to its stored rule) and the documentation's trailing whitespace drifts — not yet triaged into #721's classes"}, + "entity WorkflowCommons.WorkflowAttachment": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "entity: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "entity WorkflowCommons.WorkflowComment": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "entity: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "entity WorkflowCommons.WorkflowGroupView": {laws: []law{lawParse}, issue: "#721", why: "entity: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "entity WorkflowCommons.WorkflowUserTaskView": {laws: []law{lawExec}, issue: "#721", why: "entity: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "entity WorkflowCommons.WorkflowView": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "entity: executes since a Studio Pro index keeps its GUID on CREATE OR MODIFY (mendixlabs/mxcli#1470); re-executing adds a fourth access rule (the XPath-scoped grant is not matched to its stored rule) and the documentation's trailing whitespace drifts — not yet triaged into #721's classes"}, + "enumeration WorkflowCommons.Enum_NotificationArea_RenderAs": {laws: []law{lawGetPut}, issue: "#721", why: "enumeration: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "export mapping Services.OrdersExportMapping": {laws: []law{lawGetPut}, issue: "#721", why: "export mapping: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "page WorkflowCommons.WorkflowAttachment_View": {laws: []law{lawExec}, issue: "#721", why: "page: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "page WorkflowCommons.WorkflowDefinition_View": {laws: []law{lawExec}, issue: "#721", why: "page: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "page WorkflowCommons.WorkflowSubProcess_View": {laws: []law{lawExec}, issue: "#721", why: "page: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "rest client Clients.OrdersRestClient": {laws: []law{lawParse}, issue: "#721", why: "rest client: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "rule Rules.Rule1": {laws: []law{lawGetPut}, issue: "#721", why: "rule: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "rule Rules.Rule2": {laws: []law{lawGetPut}, issue: "#721", why: "rule: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "scheduled event WorkflowCommons.SE_WorkflowAuditTrailRecord_CleanUp": {laws: []law{lawGetPut}, issue: "#721", why: "scheduled event: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_Assignee_Search": {laws: []law{lawGetPut}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes; executes since a widget bound to a snippet parameter is described as $Param.Attr — getput not yet triaged"}, + "snippet WorkflowCommons.Snip_AuditTrailViewer_Toolbar": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "snippet: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_MyInitiatedWorkflowView_NotificationArea": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "snippet: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_MyInitiatedWorkflowView_StateCircleOnly": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "snippet: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_TaskAssignmentHelper_Assignee_Search": {laws: []law{lawGetPut}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes; executes since a widget bound to a snippet parameter is described as $Param.Attr — getput not yet triaged"}, + "snippet WorkflowCommons.Snip_TaskAssignmentHelper_TargetUser_Search": {laws: []law{lawGetPut}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes; executes since a widget bound to a snippet parameter is described as $Param.Attr — getput not yet triaged"}, + "snippet WorkflowCommons.Snip_TaskAssignment_UserTask_Assignees": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "snippet: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_TaskAssignment_UserTask_TargetUsers": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "snippet: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_TimelineViewer_Timeline_Full": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_TimelineViewer_Timeline_Tasks": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_UserTask_NameColumn": {laws: []law{lawGetPut, lawPutGet}, issue: "#721 #826", why: "executes since describe prints snippet call Params (#826); an association-path template parameter read from a snippet parameter loses its AttributeRef and SourceVariable (#721)"}, + "snippet WorkflowCommons.Snip_UserTask_NotificationArea": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "snippet: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_WorkflowActivityRecord_ActivityIcon": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_WorkflowActivityRecord_Details": {laws: []law{lawGetPut}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes; executes since a widget bound to a snippet parameter is described as $Param.Attr — getput not yet triaged"}, + "snippet WorkflowCommons.Snip_WorkflowActivityRecord_Header": {laws: []law{lawGetPut, lawPutGet}, issue: "#721 #826", why: "executes since describe prints snippet call Params (#826); association-path template parameters and an expression parameter drift (#721)"}, + "snippet WorkflowCommons.Snip_WorkflowActivity_Details_ActivityType": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_WorkflowSubProcess_Details": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_WorkflowSubProcess_IncompatibleWarning": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_WorkflowSubProcess_State": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_WorkflowUserTaskView_Details": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_WorkflowUserTaskView_NameColumn": {laws: []law{lawGetPut, lawPutGet}, issue: "#721 #826", why: "executes since describe prints snippet call Params (#826); an association-path template parameter read from a snippet parameter loses its AttributeRef and SourceVariable (#721)"}, + "snippet WorkflowCommons.Snip_Workflow_CommentsAndAttachments": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_Workflow_CommentsAndAttachments_Admin": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_Workflow_Header": {laws: []law{lawGetPut, lawPutGet}, issue: "#721 #826", why: "executes since describe prints snippet call Params (#826); an association-path caption parameter read from a snippet parameter loses its AttributeRef and SourceVariable (#721)"}, + "snippet WorkflowCommons.Snip_Workflow_NotificationArea": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "snippet: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_Workflow_StateCircleOnly": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "snippet: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, + "snippet WorkflowCommons.Snip_Workflow_SubProcessDetails": {laws: []law{lawExec}, issue: "#721", why: "snippet: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, } // createOrModifyProbeKnownFailures is the allowlist of diff --git a/mdl/translations/sites.go b/mdl/translations/sites.go index b343ac462..343992f60 100644 --- a/mdl/translations/sites.go +++ b/mdl/translations/sites.go @@ -3,6 +3,8 @@ package translations import ( + "strings" + "go.mongodb.org/mongo-driver/v2/bson" "github.com/mendixlabs/mxcli/modelsdk/codec" @@ -32,6 +34,14 @@ type Site struct { ElementID string // OwnerName is the owner's Name ("tabPage2"), empty when it has none. OwnerName string + // WidgetID is the $ID of the nearest named Forms$ or CustomWidgets$ + // element around the text below the document root: the widget a caption, + // tooltip or confirmation text belongs to. ElementID is the text's direct + // owner, which for a button's caption is its Forms$ClientTemplate and for + // its confirmation the Forms$ConfirmationInfo, neither of them the button + // (ako/mxcli#1303). Empty for text that is not inside a widget, such as a + // page's own title. + WidgetID string // Targets is language code → text, exactly as stored. A language present // with an empty string is a text that exists but is not translated yet. Targets map[string]string @@ -41,8 +51,8 @@ type Site struct { // order. Order is stable so a caller writing rows gets a deterministic result. func SitesIn(doc bson.D) []Site { var out []Site - var walk func(v any, ownerType, ownerID, ownerName, prop string) - walk = func(v any, ownerType, ownerID, ownerName, prop string) { + var walk func(v any, ownerType, ownerID, ownerName, prop, widgetID string, root bool) + walk = func(v any, ownerType, ownerID, ownerName, prop, widgetID string, root bool) { switch n := v.(type) { case bson.D: if ty, _ := lookup(n, "$Type").(string); ty == "Texts$Text" { @@ -51,6 +61,7 @@ func SitesIn(doc bson.D) []Site { Property: prop, ElementID: ownerID, OwnerName: ownerName, + WidgetID: widgetID, Targets: translationsOf(n), }) return @@ -59,24 +70,27 @@ func SitesIn(doc bson.D) []Site { // it. A node without one (an anonymous sub-document) leaves the // owner as it was, so the text is still attributed to a real // element rather than to nothing. - nt, nid, nname := ownerType, ownerID, ownerName + nt, nid, nname, nwidget := ownerType, ownerID, ownerName, widgetID if ty, _ := lookup(n, "$Type").(string); ty != "" { nt = ty nid = elementIDOf(n) nname, _ = lookup(n, "Name").(string) + if !root && nname != "" && (strings.HasPrefix(ty, "Forms$") || strings.HasPrefix(ty, "CustomWidgets$")) { + nwidget = nid + } } for _, e := range n { - walk(e.Value, nt, nid, nname, e.Key) + walk(e.Value, nt, nid, nname, e.Key, nwidget, false) } case bson.A: // An array element inherits the property its array hangs off, so a // text inside `Widgets` is not reported as living at "Widgets". for _, e := range n { - walk(e, ownerType, ownerID, ownerName, prop) + walk(e, ownerType, ownerID, ownerName, prop, widgetID, false) } } } - walk(doc, "", "", "", "") + walk(doc, "", "", "", "", "", true) return out } diff --git a/mdl/translations/sites_test.go b/mdl/translations/sites_test.go index ad0841df4..d61a015d7 100644 --- a/mdl/translations/sites_test.go +++ b/mdl/translations/sites_test.go @@ -127,3 +127,45 @@ func siteKeys(m map[string]Site) []string { } return out } + +// ako/mxcli#1303: a button's caption is a Texts$Text inside a +// Forms$ClientTemplate, and its confirmation texts sit in a ConfirmationInfo +// inside the action, so ElementID is the template's or the confirmation's id, +// never the button's. WidgetID is the nearest named widget around the text. +// The page's own title has none: its owner is the document. +func TestSitesIn_WidgetID_Issue1303(t *testing.T) { + txt := func(s string) bson.D { + return bson.D{{Key: "$Type", Value: "Texts$Text"}, {Key: "Items", Value: bson.A{int32(3), + bson.D{{Key: "$Type", Value: "Texts$Translation"}, {Key: "LanguageCode", Value: "en_US"}, {Key: "Text", Value: s}}}}} + } + doc := bson.D{ + {Key: "$Type", Value: "Forms$Page"}, {Key: "$ID", Value: "page"}, {Key: "Name", Value: "Sale_Edit"}, + {Key: "Title", Value: txt("Edit sale")}, + {Key: "Widgets", Value: bson.A{int32(3), bson.D{ + {Key: "$Type", Value: "Forms$DataView"}, {Key: "$ID", Value: "dv"}, {Key: "Name", Value: "dvSale"}, + {Key: "Widgets", Value: bson.A{int32(3), bson.D{ + {Key: "$Type", Value: "Forms$ActionButton"}, {Key: "$ID", Value: "btn"}, {Key: "Name", Value: "btnRemove"}, + {Key: "Caption", Value: bson.D{{Key: "$Type", Value: "Forms$ClientTemplate"}, {Key: "$ID", Value: "tpl"}, + {Key: "Template", Value: txt("Remove")}}}, + {Key: "Action", Value: bson.D{{Key: "$Type", Value: "Forms$MicroflowAction"}, {Key: "$ID", Value: "act"}, + {Key: "MicroflowSettings", Value: bson.D{{Key: "$Type", Value: "Forms$MicroflowSettings"}, {Key: "$ID", Value: "set"}, + {Key: "ConfirmationInfo", Value: bson.D{{Key: "$Type", Value: "Forms$ConfirmationInfo"}, {Key: "$ID", Value: "conf"}, + {Key: "Question", Value: txt("Delete this sale?")}}}}}}}, + }}}, + }}}, + } + got := map[string][2]string{} + for _, s := range SitesIn(doc) { + got[s.Targets["en_US"]] = [2]string{s.ElementID, s.WidgetID} + } + want := map[string][2]string{ + "Edit sale": {"page", ""}, + "Remove": {"tpl", "btn"}, + "Delete this sale?": {"conf", "btn"}, + } + for text, w := range want { + if got[text] != w { + t.Errorf("%q: (element, widget) = %v, want %v", text, got[text], w) + } + } +} diff --git a/mdl/types/agenteditor.go b/mdl/types/agenteditor.go index c1b29711b..d61536a4e 100644 --- a/mdl/types/agenteditor.go +++ b/mdl/types/agenteditor.go @@ -150,6 +150,12 @@ type ConsumedMCPService struct { Version string `json:"version,omitempty"` InnerDocumentation string `json:"innerDocumentation,omitempty"` // Contents.documentation ConnectionTimeoutSeconds int `json:"connectionTimeoutSeconds,omitempty"` + // Endpoint is the String constant holding the server URL; without one + // AgentEditorCommons does not import the service (#2060). + Endpoint *DocRef `json:"endpoint,omitempty"` + // AuthenticationMicroflow returns the HTTP headers sent to the server + // (no parameters, List of System.HttpHeader). + AuthenticationMicroflow *DocRef `json:"authenticationMicroflow,omitempty"` } // GetName returns the MCP service's name. @@ -244,6 +250,17 @@ type Agent struct { // not the model. nil when the document holds none. EnableBackendStreaming *bool `json:"enableBackendStreaming,omitempty"` EnableUIStreaming *bool `json:"enableUIStreaming,omitempty"` + // SelectedModel is the model the agent uses on a multi-model resource: its + // id, which the import matches against DeployedModel.Model, and the display + // name Studio Pro shows. Agent Editor 2.2+ refuses to import an agent + // without one (#2031). nil when the document holds none. + SelectedModel *AgentSelectedModel `json:"selectedModel,omitempty"` +} + +// AgentSelectedModel is the agent's `selectedModel` JSON object. +type AgentSelectedModel struct { + ID string `json:"id"` + Name string `json:"name"` } // GetName returns the agent's name. diff --git a/mdl/types/expression_rename.go b/mdl/types/expression_rename.go index 97f9c71e9..918240f27 100644 --- a/mdl/types/expression_rename.go +++ b/mdl/types/expression_rename.go @@ -106,3 +106,66 @@ func isNameStartByte(c byte) bool { func isNameByte(c byte) bool { return isNameStartByte(c) || c >= '0' && c <= '9' } + +// RewritePathSteps renames oldQN where it is a step of a path — in an XPath +// constraint (`[Src.Order_Customer/Src.Customer/Name = 'x']`) or an expression +// (`$O/Src.Order_Customer/Name`) — returning the rewritten text and how many +// steps it rewrote. +// +// RewriteExpressionReferences deliberately leaves path steps alone: an +// enumeration renamed must not rewrite an association that shares its name. +// This is the rewrite for when the renamed element IS an entity or an +// association, which is the caller's to establish (ako/mxcli#1836: a MOVE of +// entities between modules left every access-rule XPath naming the old module). +// +// A step is oldQN as a whole token: not preceded by a name character, `.`, `$` +// or `@`, and not followed by a name character or `.` — so neither a longer +// name, another module's element, a variable, a constant nor a member access. +// Text inside a string literal is data and is left alone. +func RewritePathSteps(text, oldQN, newQN string) (string, int) { + if !isQualifiedName(oldQN) || !isQualifiedName(newQN) || !strings.Contains(text, oldQN) { + return text, 0 + } + var b strings.Builder + count := 0 + inLiteral := false + for i := 0; i < len(text); { + c := text[i] + if c == '\'' { + if inLiteral && i+1 < len(text) && text[i+1] == '\'' { + b.WriteString("''") + i += 2 + continue + } + inLiteral = !inLiteral + b.WriteByte(c) + i++ + continue + } + if !inLiteral && strings.HasPrefix(text[i:], oldQN) && startsPathStep(text, i) { + if next := byteAt(text, i+len(oldQN)); !isNameByte(next) && next != '.' { + b.WriteString(newQN) + i += len(oldQN) + count++ + continue + } + } + b.WriteByte(c) + i++ + } + if count == 0 { + return text, 0 + } + return b.String(), count +} + +// startsPathStep reports whether a path step may begin at text[i]: after a +// step separator or an operator, not in the middle of a name, a variable or a +// constant. +func startsPathStep(text string, i int) bool { + if i == 0 { + return true + } + p := text[i-1] + return !isNameByte(p) && p != '.' && p != '$' && p != '@' +} diff --git a/mdl/types/expression_rename_test.go b/mdl/types/expression_rename_test.go index ca452f645..d4c2cb092 100644 --- a/mdl/types/expression_rename_test.go +++ b/mdl/types/expression_rename_test.go @@ -55,3 +55,35 @@ func TestRewriteExpressionReferences(t *testing.T) { }) } } + +// ako/mxcli#1836 (mendixlabs/mxcli#1687): a MOVE of entities between modules +// left the access-rule XPath `[Src.Order_Customer/Src.Customer/Name = 'x']` +// naming the old module — its steps are path steps, which +// RewriteExpressionReferences leaves alone on purpose (an enumeration renamed +// must not rewrite an association of the same name). RewritePathSteps is the +// rewrite for when the renamed element IS an entity or an association. +func TestRewritePathSteps(t *testing.T) { + cases := []struct { + name, in, old, new, want string + n int + }{ + {"xpath, first step", `[Src.Order_Customer/Src.Customer/Name = 'x']`, "Src.Order_Customer", "Dst.Order_Customer", `[Dst.Order_Customer/Src.Customer/Name = 'x']`, 1}, + {"xpath, entity step", `[Src.Order_Customer/Src.Customer/Name = 'x']`, "Src.Customer", "Dst.Customer", `[Src.Order_Customer/Dst.Customer/Name = 'x']`, 1}, + {"microflow path", `$O/Src.Order_Customer/Name != empty`, "Src.Order_Customer", "Dst.Order_Customer", `$O/Dst.Order_Customer/Name != empty`, 1}, + {"specialisation in a path", `[Src.Order_Customer/Src.Customer]`, "Src.Customer", "Dst.Customer", `[Src.Order_Customer/Dst.Customer]`, 1}, + {"inside parentheses", `[(Src.A_B/Src.B/X = 1)]`, "Src.A_B", "Dst.A_B", `[(Dst.A_B/Src.B/X = 1)]`, 1}, + {"a longer name is another element", `[Src.Order_CustomerX/Src.CustomerY/N = 1]`, "Src.Customer", "Dst.Customer", `[Src.Order_CustomerX/Src.CustomerY/N = 1]`, 0}, + {"another module's element", `[Other.Customer/Name = 1]`, "Src.Customer", "Dst.Customer", `[Other.Customer/Name = 1]`, 0}, + {"inside a literal", `[Name = 'Src.Customer/x']`, "Src.Customer", "Dst.Customer", `[Name = 'Src.Customer/x']`, 0}, + {"a member access is RewriteExpressionReferences' case", `Src.Customer.Name`, "Src.Customer", "Dst.Customer", `Src.Customer.Name`, 0}, + {"a variable's attribute", `$Src.Customer`, "Src.Customer", "Dst.Customer", `$Src.Customer`, 0}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, n := RewritePathSteps(c.in, c.old, c.new) + if got != c.want || n != c.n { + t.Errorf("RewritePathSteps(%q, %q, %q) = %q, %d; want %q, %d", c.in, c.old, c.new, got, n, c.want, c.n) + } + }) + } +} diff --git a/mdl/visitor/visitor_agent_settings_test.go b/mdl/visitor/visitor_agent_settings_test.go index 77cae100e..674736ac9 100644 --- a/mdl/visitor/visitor_agent_settings_test.go +++ b/mdl/visitor/visitor_agent_settings_test.go @@ -72,3 +72,46 @@ func TestAgentSettings_AlterAdd(t *testing.T) { t.Errorf("alter add %+v %+v", s.AddTools[0], s.AddKBs[0]) } } + +// ako/mxcli#2031: the model an agent uses on a multi-model resource. +func TestAgentSettings_SelectedModel(t *testing.T) { + prog := mustBuild(t, `create agent M.A ( + UsageType: Task, + Model: M.Model, + SelectedModel: 'eu.anthropic.claude-sonnet-5', + SelectedModelName: 'Anthropic Claude Sonnet 5' +); +create or modify agent M.B (UsageType: Task); +alter agent M.A set SelectedModel = 'eu.anthropic.claude-haiku-5';`) + a := prog.Statements[0].(*ast.CreateAgentStmt) + if a.SelectedModel == nil || *a.SelectedModel != "eu.anthropic.claude-sonnet-5" || + a.SelectedModelName == nil || *a.SelectedModelName != "Anthropic Claude Sonnet 5" { + t.Errorf("selected model %v / %v", a.SelectedModel, a.SelectedModelName) + } + // Control: left out, nothing is said, so a create or modify keeps the stored value. + if b := prog.Statements[1].(*ast.CreateAgentStmt); b.SelectedModel != nil || b.SelectedModelName != nil { + t.Errorf("unstated selected model set: %v %v", b.SelectedModel, b.SelectedModelName) + } + if alt := prog.Statements[2].(*ast.AlterAgentStmt); alt.Sets["selectedmodel"] != "eu.anthropic.claude-haiku-5" { + t.Errorf("alter set SelectedModel: %+v", alt.Sets) + } +} + +// ako/mxcli#2060: a consumed MCP service's endpoint constant and credentials +// microflow. +func TestConsumedMCPService_EndpointAndCredentials(t *testing.T) { + prog := mustBuild(t, `create consumed mcp service M.Svc ( + ProtocolVersion: v2025_03_26, + Endpoint: @M.Url, + AuthenticationMicroflow: M.GetCredentials +); +create or modify consumed mcp service M.Kept (ProtocolVersion: v2025_03_26);`) + s := prog.Statements[0].(*ast.CreateConsumedMCPServiceStmt) + if s.Endpoint == nil || s.Endpoint.String() != "M.Url" || s.AuthenticationMicroflow == nil || s.AuthenticationMicroflow.String() != "M.GetCredentials" { + t.Errorf("endpoint %v, credentials %v", s.Endpoint, s.AuthenticationMicroflow) + } + // Control: left out, nothing is said. + if k := prog.Statements[1].(*ast.CreateConsumedMCPServiceStmt); k.Endpoint != nil || k.AuthenticationMicroflow != nil { + t.Errorf("unstated references set: %v %v", k.Endpoint, k.AuthenticationMicroflow) + } +} diff --git a/mdl/visitor/visitor_agenteditor.go b/mdl/visitor/visitor_agenteditor.go index 0369118f7..86a52fe4f 100644 --- a/mdl/visitor/visitor_agenteditor.go +++ b/mdl/visitor/visitor_agenteditor.go @@ -172,6 +172,14 @@ func (b *Builder) ExitCreateConsumedMCPServiceStatement(ctx *parser.CreateConsum if v, ok := props["connectiontimeoutseconds"]; ok { fmt.Sscanf(v, "%d", &stmt.ConnectionTimeoutSeconds) } + if v, ok := props["endpoint"]; ok { + qn := parseQualifiedNameString(v) + stmt.Endpoint = &qn + } + if v, ok := props["authenticationmicroflow"]; ok { + qn := parseQualifiedNameString(v) + stmt.AuthenticationMicroflow = &qn + } if createStmt := findParentCreateStatement(ctx); createStmt != nil { if createStmt.OR() != nil && (createStmt.MODIFY() != nil || createStmt.REPLACE() != nil) { @@ -240,6 +248,12 @@ func (b *Builder) ExitCreateAgentStatement(ctx *parser.CreateAgentStatementConte stmt.ToolChoice = props["toolchoice"] stmt.EnableBackendStreaming = boolProp(props, "enablebackendstreaming") stmt.EnableUIStreaming = boolProp(props, "enableuistreaming") + if v, ok := props["selectedmodel"]; ok { + stmt.SelectedModel = &v + } + if v, ok := props["selectedmodelname"]; ok { + stmt.SelectedModelName = &v + } if v, ok := props["maxtokens"]; ok { var n int diff --git a/mdl/visitor/visitor_catalog_tables_parse_test.go b/mdl/visitor/visitor_catalog_tables_parse_test.go new file mode 100644 index 000000000..aab9611d2 --- /dev/null +++ b/mdl/visitor/visitor_catalog_tables_parse_test.go @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 + +package visitor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/catalog" +) + +// Every table SHOW CATALOG TABLES names must parse as CATALOG.. A table +// whose name is also a lexer keyword (COMMUNITIES, QUEUES, and FOLDERS for +// ako/mxcli#1436) parsed to no statement at all unless catalogTableName listed +// it: no error, no output, exit 0. Each was found by hand after it shipped; this +// enumerates the catalog's own list, so the next one fails here instead. +func TestEveryCatalogTableParses(t *testing.T) { + cat, err := catalog.New() + if err != nil { + t.Fatal(err) + } + defer cat.Close() + tables := cat.Tables() + if len(tables) < 30 { + t.Fatalf("control: the catalog lists %d tables", len(tables)) + } + for _, name := range tables { + for _, q := range []string{ + "SELECT * FROM " + name + ";", + "select * from " + strings.ToLower(name) + ";", + } { + prog, errs := Build(q) + if len(errs) > 0 { + t.Errorf("%s: parse error: %v", q, errs[0]) + continue + } + if len(prog.Statements) != 1 { + t.Errorf("%s: %d statements, want 1 -- add the keyword to catalogTableName", q, len(prog.Statements)) + continue + } + if _, ok := prog.Statements[0].(*ast.SelectStmt); !ok { + t.Errorf("%s: parsed as %T, want a SelectStmt", q, prog.Statements[0]) + } + } + } +} diff --git a/mdl/visitor/visitor_strict_properties.go b/mdl/visitor/visitor_strict_properties.go index be60e88e3..52070577a 100644 --- a/mdl/visitor/visitor_strict_properties.go +++ b/mdl/visitor/visitor_strict_properties.go @@ -236,6 +236,8 @@ var consumedMCPServiceSchema = propSchema{on: "a consumed MCP service", keys: [] {"Version", []propShape{shapeString}}, {"ConnectionTimeoutSeconds", []propShape{shapeNumber}}, {"Documentation", textShapes}, + {"Endpoint", []propShape{shapeConstant, shapeQName}}, + {"AuthenticationMicroflow", []propShape{shapeQName}}, }} // ServerName is stored but not in Studio Pro's editor: a new document gets its @@ -273,6 +275,8 @@ var agentSchema = propSchema{on: "an agent", keys: []propKey{ {"UserPrompt", textShapes}, {"EnableBackendStreaming", []propShape{shapeBool}}, {"EnableUIStreaming", []propShape{shapeBool}}, + {"SelectedModel", []propShape{shapeString}}, + {"SelectedModelName", []propShape{shapeString}}, }} var agentMCPServiceSchema = propSchema{on: "an agent's mcp service block", keys: []propKey{ diff --git a/modelsdk/canon/identity.go b/modelsdk/canon/identity.go index 8a8cb0c96..33c4dff79 100644 --- a/modelsdk/canon/identity.go +++ b/modelsdk/canon/identity.go @@ -141,6 +141,17 @@ func Reconcile(contents, stored []byte, opts ...Option) (out []byte, unchanged b // no-op elision could never fire for one (issue #949). contents = CarryPersistentIDs(contents, stored) + // A property MDL cannot state: a tab container's default tab. The writer + // points it at the first tab, so a round trip rewrote a page whose stored + // default was null and reset any other choice (ako/mxcli#1897). Needs the + // transplanted $IDs, so it runs after TransplantIDs. + contents = CarryDefaultTabPages(contents, stored) + + // The rest of what MDL cannot state: an expression's surrounding whitespace + // and an empty screen-reader label Studio Pro stores as a ClientTemplate + // (ako/mxcli#1897). + contents = CarryUnexpressed(contents, stored) + if alwaysWrite() { return contents, false } diff --git a/modelsdk/canon/tabdefault.go b/modelsdk/canon/tabdefault.go new file mode 100644 index 000000000..6a335a27e --- /dev/null +++ b/modelsdk/canon/tabdefault.go @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: Apache-2.0 + +package canon + +import ( + "bytes" + + "go.mongodb.org/mongo-driver/v2/x/bsonx/bsoncore" +) + +// tabControlType is the storage name of a page's tab container. +const tabControlType = "Forms$TabControl" + +// CarryDefaultTabPages returns contents with each tab container's +// DefaultPagePointer set to the value the stored container of the same $ID +// holds, when that value still means something in the written document: null +// (Studio Pro's "no default set"), or the $ID of a tab page the written +// container still has. +// +// MDL has no syntax for a tab container's default tab and DESCRIBE prints none, +// so the writer points it at the first tab page. A describe → create or +// replace round trip therefore rewrote a page whose stored pointer was null — +// the write elision saw a difference (ako/mxcli#1897) — and reset a stored +// default other than the first on every pass. Run after TransplantIDs, which is +// what makes the written container and its tab pages carry their stored $IDs. +// +// The patch is in place over a fixed-width 16-byte binary, like the $ID +// transplant; nothing is re-marshalled. A document that cannot be read passes +// through untouched. +func CarryDefaultTabPages(contents, stored []byte) []byte { + want := storedDefaultTabs(bsoncore.Document(stored)) + if len(want) == 0 { + return contents + } + out := append([]byte(nil), contents...) + if !carryDefaultTabs(bsoncore.Document(out), want) { + return contents + } + return out +} + +// storedDefaultTabs maps each stored tab container's $ID to its +// DefaultPagePointer. +func storedDefaultTabs(doc bsoncore.Document) map[string][]byte { + m := map[string][]byte{} + walkDocs(doc, func(d bsoncore.Document) { + if docType(d) != tabControlType { + return + } + id, ok := binaryField(d, "$ID") + if !ok { + return + } + if def, ok := binaryField(d, "DefaultPagePointer"); ok { + m[string(id)] = append([]byte(nil), def...) + } + }) + return m +} + +// carryDefaultTabs patches each written tab container that has a stored +// counterpart; false when the document cannot be read. +func carryDefaultTabs(doc bsoncore.Document, want map[string][]byte) bool { + if _, err := doc.Elements(); err != nil { + return false + } + walkDocs(doc, func(d bsoncore.Document) { + if docType(d) != tabControlType { + return + } + id, found := binaryField(d, "$ID") + if !found { + return + } + stored, found := want[string(id)] + if !found { + return + } + def, found := binaryField(d, "DefaultPagePointer") + if !found || bytes.Equal(def, stored) { + return + } + if !isNullID(stored) && !hasTabPage(d, stored) { + return // the stored default is gone; carrying it would dangle + } + copy(def, stored) // aliases the buffer: an in-place patch + }) + return true +} + +func isNullID(b []byte) bool { + for _, c := range b { + if c != 0 { + return false + } + } + return true +} + +// hasTabPage reports whether the container lists a tab page with this $ID. +func hasTabPage(ctl bsoncore.Document, id []byte) bool { + v, err := ctl.LookupErr("TabPages") + if err != nil { + return false + } + arr, ok := v.ArrayOK() + if !ok { + return false + } + vals, err := arr.Values() + if err != nil { + return false + } + for _, e := range vals { + if sub, ok := e.DocumentOK(); ok { + if pid, ok := binaryField(sub, "$ID"); ok && bytes.Equal(pid, id) { + return true + } + } + } + return false +} + +// walkDocs calls fn on doc and on every document nested in it, depth first. +func walkDocs(doc bsoncore.Document, fn func(bsoncore.Document)) { + fn(doc) + elems, err := doc.Elements() + if err != nil { + return + } + for _, e := range elems { + v := e.Value() + switch v.Type { + case bsoncore.TypeEmbeddedDocument: + if sub, ok := v.DocumentOK(); ok { + walkDocs(sub, fn) + } + case bsoncore.TypeArray: + if arr, ok := v.ArrayOK(); ok { + walkDocs(bsoncore.Document(arr), fn) + } + } + } +} + +func docType(d bsoncore.Document) string { + v, err := d.LookupErr("$Type") + if err != nil { + return "" + } + s, _ := v.StringValueOK() + return s +} + +// binaryField returns a 16-byte binary property's bytes, aliasing the +// document's buffer so a caller can patch it in place. +func binaryField(d bsoncore.Document, key string) ([]byte, bool) { + v, err := d.LookupErr(key) + if err != nil { + return nil, false + } + _, data, ok := v.BinaryOK() + if !ok || len(data) != 16 { + return nil, false + } + return data, true +} diff --git a/modelsdk/canon/tabdefault_test.go b/modelsdk/canon/tabdefault_test.go new file mode 100644 index 000000000..b30e6bf7e --- /dev/null +++ b/modelsdk/canon/tabdefault_test.go @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: Apache-2.0 + +package canon + +import ( + "bytes" + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +// ako/mxcli#1897: a describe → create or replace round trip rewrote a page +// although nothing changed. Measured on TestApp's Administration.Account_Overview: +// the one difference was its tab container's DefaultPagePointer — stored null +// (Studio Pro's "no default set"), written as the first tab page. MDL has no +// syntax for the default tab and DESCRIBE prints none, so a stored default +// other than the first was reset on every round trip too. The stored pointer is +// carried while it still means something: null, or a tab page the rebuilt +// control still has. + +func id16(b byte) bson.Binary { + d := bytes.Repeat([]byte{b}, 16) + return bson.Binary{Subtype: 0, Data: d} +} + +var zero16 = bson.Binary{Subtype: 0, Data: make([]byte, 16)} + +func tabControl(id bson.Binary, def bson.Binary, tabs ...bson.Binary) bson.D { + var pages bson.A + pages = append(pages, int32(2)) + for _, t := range tabs { + pages = append(pages, bson.D{{Key: "$ID", Value: t}, {Key: "$Type", Value: "Forms$TabPage"}}) + } + return bson.D{ + {Key: "$ID", Value: id}, + {Key: "$Type", Value: "Forms$TabControl"}, + {Key: "DefaultPagePointer", Value: def}, + {Key: "TabPages", Value: pages}, + } +} + +func pageWith(t *testing.T, ctl bson.D) []byte { + return mustMarshal(t, bson.D{ + {Key: "$ID", Value: id16(0xAA)}, + {Key: "$Type", Value: "Forms$Page"}, + {Key: "Widgets", Value: bson.A{int32(2), ctl}}, + }) +} + +func defaultPointerOf(t *testing.T, doc []byte) []byte { + t.Helper() + var d struct { + Widgets []any `bson:"Widgets"` + } + if err := bson.Unmarshal(doc, &d); err != nil { + t.Fatal(err) + } + ctl := d.Widgets[1].(bson.D) + for _, e := range ctl { + if e.Key == "DefaultPagePointer" { + return e.Value.(bson.Binary).Data + } + } + t.Fatal("no DefaultPagePointer") + return nil +} + +func TestCarryDefaultTab_NullStaysNullAndTheWriteIsElided(t *testing.T) { + stored := pageWith(t, tabControl(id16(1), zero16, id16(2), id16(3))) + written := pageWith(t, tabControl(id16(1), id16(2), id16(2), id16(3))) + + // Control: without the carry the two differ — the reported rewrite. + if eq, _ := Equal(written, stored); eq { + t.Fatal("control: the documents should differ before the carry") + } + out, unchanged := Reconcile(written, stored) + if !unchanged { + t.Errorf("a round trip that changed nothing was not elided; pointer now %x", defaultPointerOf(t, out)) + } +} + +func TestCarryDefaultTab_KeepsAStoredSecondTab(t *testing.T) { + stored := pageWith(t, tabControl(id16(1), id16(3), id16(2), id16(3))) + written := pageWith(t, tabControl(id16(1), id16(2), id16(2), id16(3))) + out := CarryDefaultTabPages(written, stored) + if got := defaultPointerOf(t, out); !bytes.Equal(got, id16(3).Data) { + t.Errorf("default tab = %x, want the stored second tab %x", got, id16(3).Data) + } +} + +// A stored default naming a tab the rebuild no longer has is not carried: it +// would dangle. +func TestCarryDefaultTab_DoesNotCarryARemovedTab(t *testing.T) { + stored := pageWith(t, tabControl(id16(1), id16(3), id16(2), id16(3))) + written := pageWith(t, tabControl(id16(1), id16(2), id16(2))) + out := CarryDefaultTabPages(written, stored) + if got := defaultPointerOf(t, out); !bytes.Equal(got, id16(2).Data) { + t.Errorf("default tab = %x, want the written first tab kept", got) + } +} + +// A tab control that is not the stored one (another $ID) gets nothing. +func TestCarryDefaultTab_OnlyTheSameControl(t *testing.T) { + stored := pageWith(t, tabControl(id16(1), zero16, id16(2))) + written := pageWith(t, tabControl(id16(9), id16(2), id16(2))) + out := CarryDefaultTabPages(written, stored) + if got := defaultPointerOf(t, out); !bytes.Equal(got, id16(2).Data) { + t.Errorf("a different control took the stored default: %x", got) + } +} diff --git a/modelsdk/canon/unexpressed.go b/modelsdk/canon/unexpressed.go new file mode 100644 index 000000000..837877627 --- /dev/null +++ b/modelsdk/canon/unexpressed.go @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: Apache-2.0 + +package canon + +import ( + "strings" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +// CarryUnexpressed returns contents with two values no MDL statement can state +// put back from the stored element of the same $ID, so a describe → create or +// replace round trip that changed nothing writes nothing (ako/mxcli#1897): +// +// - An Expression that differs from the stored one only in leading or +// trailing whitespace. MDL writes an expression bare, so a stored "true\n" +// came back as "true" — measured on TestApp's WorkflowCommons pages, in +// visibility conditions and page-call parameter mappings. +// - A property written null where the stored element holds an EMPTY +// Forms$ClientTemplate — no text in any language and no parameters. Studio +// Pro stores a widget's unset screen-reader label that way; the writer +// writes null. Both mean "no label". +// +// Only elements whose $ID the stored document already has take anything, which +// after TransplantIDs means the elements that correspond; a real change — other +// expression text, a label with text — is not touched and still lands. +func CarryUnexpressed(contents, stored []byte) []byte { + if len(contents) == 0 || len(stored) == 0 { + return contents + } + var storedDoc, newDoc bson.D + if bson.Unmarshal(stored, &storedDoc) != nil || bson.Unmarshal(contents, &newDoc) != nil { + return contents + } + byID := map[string]bson.D{} + walkElements(storedDoc, func(d bson.D) { + if id, ok := docLookup(d, "$ID").(bson.Binary); ok { + byID[string(id.Data)] = d + } + }) + used := elementIDs(newDoc) + + changed := false + walkElements(newDoc, func(d bson.D) { + id, ok := docLookup(d, "$ID").(bson.Binary) + if !ok { + return + } + old, ok := byID[string(id.Data)] + if !ok { + return + } + for i, e := range d { + ov := docLookup(old, e.Key) + switch v := e.Value.(type) { + case string: + s, isStr := ov.(string) + if e.Key == "Expression" && isStr && s != v && strings.TrimSpace(s) == strings.TrimSpace(v) { + d[i].Value = s + changed = true + } + case nil: + if t, isDoc := ov.(bson.D); isDoc && isEmptyClientTemplate(t) && !anyIDUsed(t, used) { + d[i].Value = t + changed = true + } + } + } + }) + if !changed { + return contents + } + out, err := bson.Marshal(newDoc) + if err != nil { + return contents + } + return out +} + +// walkElements calls fn on every embedded document, depth first. The bson.D +// values are live: a change through one changes the document. +func walkElements(v any, fn func(bson.D)) { + switch n := v.(type) { + case bson.D: + fn(n) + for _, e := range n { + walkElements(e.Value, fn) + } + case bson.A: + for _, e := range n { + walkElements(e, fn) + } + } +} + +// isEmptyClientTemplate reports whether t is a Forms$ClientTemplate with no +// parameters and no translation in its Template or Fallback. +func isEmptyClientTemplate(t bson.D) bool { + if ty, _ := docLookup(t, "$Type").(string); ty != "Forms$ClientTemplate" { + return false + } + if hasItems(docLookup(t, "Parameters")) { + return false + } + for _, k := range []string{"Template", "Fallback"} { + if txt, ok := docLookup(t, k).(bson.D); ok && hasItems(docLookup(txt, "Items")) { + return false + } + } + return true +} + +// hasItems reports whether a Mendix list holds anything past its leading +// type marker. +func hasItems(v any) bool { + a, ok := v.(bson.A) + return ok && len(a) > 1 +} + +// anyIDUsed reports whether any element id inside d is already in the +// document: carrying it in would make two elements share an identity. +func anyIDUsed(d bson.D, used map[string]bool) bool { + clash := false + walkElements(d, func(e bson.D) { + if id, ok := docLookup(e, "$ID").(bson.Binary); ok && used[string(id.Data)] { + clash = true + } + }) + return clash +} diff --git a/modelsdk/canon/unexpressed_test.go b/modelsdk/canon/unexpressed_test.go new file mode 100644 index 000000000..39af625bc --- /dev/null +++ b/modelsdk/canon/unexpressed_test.go @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: Apache-2.0 + +package canon + +import ( + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +// ako/mxcli#1897, measured on TestApp's WorkflowCommons pages: two more +// differences no MDL statement can state made a describe → create or replace +// round trip rewrite an unchanged page. +// - An expression stored with a trailing newline ("true\n", a visibility +// condition ending in "\n"): MDL writes expressions bare, so the +// whitespace cannot be expressed and came back trimmed. +// - An empty screen-reader label: Studio Pro stores a ClientTemplate with no +// text and no parameters, the writer null. Both mean "no label". +// Each is carried from the stored element of the same $ID, so the write is +// elided; a real change still lands. + +func textEmpty(id byte) bson.D { + return bson.D{{Key: "$ID", Value: id16(id)}, {Key: "$Type", Value: "Texts$Text"}, {Key: "Items", Value: bson.A{int32(3)}}} +} + +func emptyTemplate() bson.D { + return bson.D{ + {Key: "$ID", Value: id16(0x30)}, + {Key: "$Type", Value: "Forms$ClientTemplate"}, + {Key: "Fallback", Value: textEmpty(0x31)}, + {Key: "Parameters", Value: bson.A{int32(2)}}, + {Key: "Template", Value: textEmpty(0x32)}, + } +} + +func widget(expr string, label any) []byte { + w := bson.D{ + {Key: "$ID", Value: id16(0x10)}, + {Key: "$Type", Value: "Forms$TextArea"}, + {Key: "ConditionalVisibilitySettings", Value: bson.D{ + {Key: "$ID", Value: id16(0x11)}, + {Key: "$Type", Value: "Forms$ConditionalVisibilitySettings"}, + {Key: "Expression", Value: expr}, + }}, + {Key: "ScreenReaderLabel", Value: label}, + } + b, _ := bson.Marshal(bson.D{{Key: "$ID", Value: id16(0xAA)}, {Key: "$Type", Value: "Forms$Page"}, {Key: "Widgets", Value: bson.A{int32(2), w}}}) + return b +} + +func TestUnexpressed_RoundTripIsElided(t *testing.T) { + stored := widget("$currentObject/TotalCount > 0\n", emptyTemplate()) + written := widget("$currentObject/TotalCount > 0", nil) + if eq, _ := Equal(written, stored); eq { + t.Fatal("control: the documents should differ before the carries") + } + if _, unchanged := Reconcile(written, stored); !unchanged { + t.Error("a round trip that changed nothing was not elided") + } +} + +// Controls: a real change to either property still lands. +func TestUnexpressed_ARealChangeLands(t *testing.T) { + stored := widget("$currentObject/TotalCount > 0\n", emptyTemplate()) + if _, unchanged := Reconcile(widget("$currentObject/TotalCount > 1", nil), stored); unchanged { + t.Error("a changed expression was elided") + } + label := emptyTemplate() + label[2].Value = bson.D{{Key: "$ID", Value: id16(0x31)}, {Key: "$Type", Value: "Texts$Text"}, + {Key: "Items", Value: bson.A{int32(3), bson.D{{Key: "$ID", Value: id16(0x40)}, {Key: "$Type", Value: "Texts$Translation"}, {Key: "LanguageCode", Value: "en_US"}, {Key: "Text", Value: "Comment"}}}}} + if _, unchanged := Reconcile(widget("$currentObject/TotalCount > 0", nil), widget("$currentObject/TotalCount > 0\n", label)); unchanged { + t.Error("removing a label that has text was elided") + } +} diff --git a/modelsdk/mprread/reader_documents.go b/modelsdk/mprread/reader_documents.go index fcad22e7e..7f88250d0 100644 --- a/modelsdk/mprread/reader_documents.go +++ b/modelsdk/mprread/reader_documents.go @@ -550,14 +550,18 @@ func decodeAgentEditorConsumedMCPService(unitID, containerID string, wrap *custo return c, nil } var payload struct { - ProtocolVersion string `json:"protocolVersion"` - Documentation string `json:"documentation"` - Version string `json:"version"` - ConnectionTimeoutSeconds int `json:"connectionTimeoutSeconds"` + ProtocolVersion string `json:"protocolVersion"` + Documentation string `json:"documentation"` + Version string `json:"version"` + ConnectionTimeoutSeconds int `json:"connectionTimeoutSeconds"` + Endpoint *types.DocRef `json:"endpoint"` + AuthenticationMicroflow *types.DocRef `json:"authenticationMicroflow"` } if err := json.Unmarshal([]byte(wrap.Contents), &payload); err != nil { return nil, fmt.Errorf("unmarshal ConsumedMCPService Contents JSON: %w", err) } + c.Endpoint = payload.Endpoint + c.AuthenticationMicroflow = payload.AuthenticationMicroflow c.ProtocolVersion = payload.ProtocolVersion c.InnerDocumentation = payload.Documentation c.Version = payload.Version @@ -585,21 +589,22 @@ func decodeAgentEditorAgent(unitID, containerID string, wrap *customBlobWrapper) return a, nil } var payload struct { - Description string `json:"description"` - SystemPrompt string `json:"systemPrompt"` - UserPrompt string `json:"userPrompt"` - UsageType string `json:"usageType"` - Variables []types.AgentVar `json:"variables"` - Tools []types.AgentTool `json:"tools"` - KnowledgebaseTools []types.AgentKBTool `json:"knowledgebaseTools"` - Model *types.DocRef `json:"model"` - Entity *types.DocRef `json:"entity"` - MaxTokens *int `json:"maxTokens"` - ToolChoice string `json:"toolChoice"` - Temperature *float64 `json:"temperature"` - TopP *float64 `json:"topP"` - EnableBackend *bool `json:"enableBackendStreaming"` - EnableUI *bool `json:"enableUIStreaming"` + Description string `json:"description"` + SystemPrompt string `json:"systemPrompt"` + UserPrompt string `json:"userPrompt"` + UsageType string `json:"usageType"` + Variables []types.AgentVar `json:"variables"` + Tools []types.AgentTool `json:"tools"` + KnowledgebaseTools []types.AgentKBTool `json:"knowledgebaseTools"` + Model *types.DocRef `json:"model"` + Entity *types.DocRef `json:"entity"` + MaxTokens *int `json:"maxTokens"` + ToolChoice string `json:"toolChoice"` + Temperature *float64 `json:"temperature"` + TopP *float64 `json:"topP"` + EnableBackend *bool `json:"enableBackendStreaming"` + EnableUI *bool `json:"enableUIStreaming"` + SelectedModel *types.AgentSelectedModel `json:"selectedModel"` } if err := json.Unmarshal([]byte(wrap.Contents), &payload); err != nil { return nil, fmt.Errorf("unmarshal Agent Contents JSON: %w", err) @@ -619,9 +624,51 @@ func decodeAgentEditorAgent(unitID, containerID string, wrap *customBlobWrapper) a.TopP = payload.TopP a.EnableBackendStreaming = payload.EnableBackend a.EnableUIStreaming = payload.EnableUI + a.SelectedModel = payload.SelectedModel return a, nil } +func decodeAgentEditorPublishedMCPService(unitID, containerID string, wrap *customBlobWrapper) (*types.PublishedMCPService, error) { + p := &types.PublishedMCPService{} + p.ID = model.ID(unitID) + p.TypeName = customBlobDocBsonType + p.ContainerID = model.ID(containerID) + p.Name = wrap.Name + p.Documentation = wrap.Documentation + p.Excluded = wrap.Excluded + p.ExportLevel = wrap.ExportLevel + + if wrap.Contents == "" { + return p, nil + } + var payload struct { + ServerName string `json:"serverName"` + EndpointPath string `json:"endpointPath"` + Version string `json:"version"` + ProtocolVersion string `json:"protocolVersion"` + AuthenticationEnabled bool `json:"authenticationEnabled"` + AuthenticationMicroflow *types.DocRef `json:"authenticationMicroflow"` + AuthorizationServerURL string `json:"authorizationServerUrl"` + Scopes string `json:"scopes"` + Tools []types.PublishedMCPTool `json:"tools"` + Documentation string `json:"documentation"` + } + if err := json.Unmarshal([]byte(wrap.Contents), &payload); err != nil { + return nil, fmt.Errorf("unmarshal PublishedMCPService Contents JSON: %w", err) + } + p.ServerName = payload.ServerName + p.EndpointPath = payload.EndpointPath + p.Version = payload.Version + p.ProtocolVersion = payload.ProtocolVersion + p.AuthenticationEnabled = payload.AuthenticationEnabled + p.AuthenticationMicroflow = payload.AuthenticationMicroflow + p.AuthorizationServerURL = payload.AuthorizationServerURL + p.Scopes = payload.Scopes + p.Tools = payload.Tools + p.InnerDocumentation = payload.Documentation + return p, nil +} + // --------------------------------------------------------------------------- // Pages, Snippets, Layouts, BuildingBlocks, PageTemplates (Forms domain) // --------------------------------------------------------------------------- @@ -730,3 +777,28 @@ func ListModules(r *mmpr.Reader) ([]Unit[*genProj.Module], error) { func ListFolders(r *mmpr.Reader) ([]Unit[*genProj.Folder], error) { return ListUnitsWithContainer[*genProj.Folder](r) } + +// DecodeAgentEditorUnit decodes one agent-editor CustomBlobDocument from its +// stored bytes into a *types.Agent, *types.Model, *types.KnowledgeBase or +// *types.ConsumedMCPService, chosen by its CustomDocumentType. It returns nil +// for any other custom document type. The catalog reads the raw units, and +// uses this to see what an agent document references (ako/mxcli#2074). +func DecodeAgentEditorUnit(unitID, containerID string, contents []byte) (any, error) { + wrap, err := parseCustomBlobWrapper(contents) + if err != nil { + return nil, err + } + switch wrap.CustomDocumentType { + case types.CustomTypeAgent: + return decodeAgentEditorAgent(unitID, containerID, wrap) + case types.CustomTypeModel: + return decodeAgentEditorModel(unitID, containerID, wrap) + case types.CustomTypeKnowledgeBase: + return decodeAgentEditorKnowledgeBase(unitID, containerID, wrap) + case types.CustomTypeConsumedMCPService: + return decodeAgentEditorConsumedMCPService(unitID, containerID, wrap) + case types.CustomTypePublishedMCPService: + return decodeAgentEditorPublishedMCPService(unitID, containerID, wrap) + } + return nil, nil +} diff --git a/sdk/agenteditor/types.go b/sdk/agenteditor/types.go index 087db9116..b0f9dfdb8 100644 --- a/sdk/agenteditor/types.go +++ b/sdk/agenteditor/types.go @@ -150,6 +150,12 @@ type ConsumedMCPService struct { Version string `json:"version,omitempty"` InnerDocumentation string `json:"innerDocumentation,omitempty"` // Contents.documentation ConnectionTimeoutSeconds int `json:"connectionTimeoutSeconds,omitempty"` + // Endpoint is the String constant holding the server URL; without one + // AgentEditorCommons does not import the service (#2060). + Endpoint *DocRef `json:"endpoint,omitempty"` + // AuthenticationMicroflow returns the HTTP headers sent to the server + // (no parameters, List of System.HttpHeader). + AuthenticationMicroflow *DocRef `json:"authenticationMicroflow,omitempty"` } // GetName returns the MCP service's name. @@ -244,6 +250,17 @@ type Agent struct { // not the model. nil when the document holds none. EnableBackendStreaming *bool `json:"enableBackendStreaming,omitempty"` EnableUIStreaming *bool `json:"enableUIStreaming,omitempty"` + // SelectedModel is the model the agent uses on a multi-model resource. + // Agent Editor 2.2+ refuses to import an agent without one (#2031). + SelectedModel *SelectedModel `json:"selectedModel,omitempty"` +} + +// SelectedModel is an agent's `selectedModel`: the deployed model's id, which +// the import matches against DeployedModel.Model, and the display name Studio +// Pro shows, which nothing at runtime reads. +type SelectedModel struct { + ID string `json:"id"` + Name string `json:"name"` } // GetName returns the agent's name.