From 70d5e273724688da7b46a68af9e9ecb0278e9afb Mon Sep 17 00:00:00 2001 From: Ako Date: Thu, 12 Nov 2026 06:04:18 +0000 Subject: [PATCH] feat(catalog): workflow activities with their kind and slots for lint rules (#1295) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A lint rule could not ask which kind of workflow activity runs a microflow — "a system task's microflow is prefixed WF_SYS_", "a user task's targeting microflow WF_TGT_" — because refs records only that the workflow calls the microflow, and workflows_data only counts activities. CATALOG.WORKFLOW_ACTIVITIES holds one row per activity of a workflow, boundary-event paths and event sub-processes included, with its kind (MultiUserTask and AIAgentTask told apart from UserTask and CallMicroflow) and the microflow, page or workflow each slot names; an empty slot is NULL. It is written by every build. workflows() and workflow_activities() expose workflows_data and the new table to Starlark rules. Catalog schema 48. Co-Authored-By: Claude Opus 5.5 --- .../skills/mendix/write-lint-rules/SKILL.md | 17 +-- .../mendix/write-lint-rules/catalog-tables.md | 35 ++++++ CHANGELOG.md | 1 + docs-site/src/tools/catalog-tables.md | 31 +++++ mdl/catalog/builder_workflow_activities.go | 104 ++++++++++++++++ .../builder_workflow_activities_test.go | 87 ++++++++++++++ mdl/catalog/builder_workflows.go | 6 + mdl/catalog/catalog.go | 1 + mdl/catalog/tables.go | 43 ++++++- mdl/linter/context_catalog_tables.go | 111 ++++++++++++++++++ mdl/linter/context_systemmodule_test.go | 10 ++ mdl/linter/starlark.go | 2 + .../starlark_catalog_mode_guard_test.go | 2 + mdl/linter/starlark_catalog_tables.go | 55 +++++++++ .../starlark_workflow_activities_test.go | 93 +++++++++++++++ 15 files changed, 590 insertions(+), 8 deletions(-) create mode 100644 mdl/catalog/builder_workflow_activities.go create mode 100644 mdl/catalog/builder_workflow_activities_test.go create mode 100644 mdl/linter/starlark_workflow_activities_test.go diff --git a/.claude/skills/mendix/write-lint-rules/SKILL.md b/.claude/skills/mendix/write-lint-rules/SKILL.md index 05a9fcc1b8..0b73dbabff 100644 --- a/.claude/skills/mendix/write-lint-rules/SKILL.md +++ b/.claude/skills/mendix/write-lint-rules/SKILL.md @@ -11,11 +11,11 @@ description: "Write custom Starlark lint rules in .claude/lint-rules/ that run b | lines | section | |---|---| | 25-67 | Rule File Structure | -| 68-167 | Available Query Functions | -| 168-574 | Object Properties | -| 575-588 | Helper Functions | -| 589-675 | Common Patterns | -| 676-684 | Validation | +| 68-170 | Available Query Functions | +| 171-577 | Object Properties | +| 578-591 | Helper Functions | +| 592-678 | Common Patterns | +| 679-687 | Validation | # Writing Custom Starlark Lint Rules @@ -111,10 +111,13 @@ Without this, a rule that reads a full-only table under a fast build gets | `strings(language = None)` | list of catalog_string | User-facing and documentary text, one row per text and language; pass `language` (`"nl_NL"`) to narrow. An untranslated language has **no row**. Needs a FULL catalog, which `mxcli lint` builds automatically for a rule that calls it | | `layouts()` | list of layout | All non-system layouts | | `published_rest_operations()` | list of published_rest_operation | Operations of published REST services, with the microflow behind each | +| `workflows()` | list of workflow | The user's workflows, with their parameter entity and activity counts | +| `workflow_activities()` | list of workflow_activity | Every activity of the user's workflows — boundary-event paths and event sub-processes included — with its kind and the microflow, page or workflow each slot names. Use it for "a system task's microflow is prefixed `WF_`": `refs_to()` says only that the workflow calls a microflow, not from which kind of activity | The fields of `module`, `folder`, `annotation`, `association`, `entity_event_handler`, -`navigation_menu_item`, `jar_dependency`, `jar_file`, `catalog_string`, `layout` and -`published_rest_operation` are in [catalog-tables.md](catalog-tables.md). +`navigation_menu_item`, `jar_dependency`, `jar_file`, `catalog_string`, `layout`, +`published_rest_operation`, `workflow` and `workflow_activity` are in +[catalog-tables.md](catalog-tables.md). ### Graph-analysis functions (architecture rules) diff --git a/.claude/skills/mendix/write-lint-rules/catalog-tables.md b/.claude/skills/mendix/write-lint-rules/catalog-tables.md index 4d42623818..0a3a686b87 100644 --- a/.claude/skills/mendix/write-lint-rules/catalog-tables.md +++ b/.claude/skills/mendix/write-lint-rules/catalog-tables.md @@ -177,6 +177,41 @@ Returned by `published_rest_operations()`. | `deprecated` | bool | Marked deprecated | | `module_name` | string | `"Sales"` | +### workflow +Returned by `workflows()`. + +| Property | Type | Example | +|----------|------|---------| +| `id` | string | Workflow ID | +| `qualified_name` | string | `"Sales.ApproveOrder"` | +| `name` | string | `"ApproveOrder"` | +| `module_name` | string | `"Sales"` | +| `description` | string | Documentation text | +| `parameter_entity` | string | Qualified name of the context entity: `"Sales.Order"` | +| `activity_count` | int | Activities in every flow of the workflow, boundary-event paths and event sub-processes included | +| `user_task_count` | int | User tasks and multi-user tasks | +| `microflow_call_count` | int | System tasks and call-microflow activities | +| `decision_count` | int | Decisions | + +### workflow_activity +Returned by `workflow_activities()`, each workflow's activities in walk order: the main flow, each activity followed by the paths of its boundary events, then the event sub-processes. A slot that names nothing is `""`. + +| Property | Type | Example | +|----------|------|---------| +| `id` | string | Activity ID | +| `workflow_qualified_name` | string | `"Sales.ApproveOrder"` | +| `module_name` | string | `"Sales"` | +| `name` | string | `"ReviewOrder"` | +| `caption` | string | `"Review the order"` | +| `activity_type` | string | `"UserTask"`, `"MultiUserTask"`, `"SystemTask"`, `"CallMicroflow"`, `"AIAgentTask"`, `"CallWorkflow"`, `"Decision"`, `"ParallelSplit"`, `"JumpTo"`, `"WaitForTimer"`, `"WaitForNotification"`, `"StartWorkflow"`, `"EndWorkflow"`, … | +| `microflow` | string | The microflow a `SystemTask`, `CallMicroflow` or `AIAgentTask` runs | +| `user_source_microflow` | string | A user task's targeting microflow (users or groups) | +| `on_created_microflow` | string | A user task's on-created microflow | +| `completion_microflow` | string | A multi-user task's completion microflow | +| `page` | string | A user task's page | +| `user_task_entity` | string | A user task's task entity | +| `called_workflow` | string | The workflow a `CallWorkflow` activity calls | + ### widget Returned by `widgets()` (full catalog — auto-detected). diff --git a/CHANGELOG.md b/CHANGELOG.md index 52aab82761..ab7903eb53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added +- **`CATALOG.WORKFLOW_ACTIVITIES`, and `workflows()` / `workflow_activities()` for lint rules** (ako/mxcli#1295, mendixlabs/mxcli#1269) — one row per workflow activity, boundary-event paths and event sub-processes included, with its kind (user task, multi-user task, system task, call microflow, AI agent task, call workflow, …) and the microflow, page or workflow each of its slots names. A rule can now check "a system task's microflow is prefixed `WF_SYS_`": `refs` records only that the workflow calls the microflow, not from which activity. Catalog schema 48. - **`make bench`: a benchmark of what an agent waits for, and `MXCLI_STATS_FILE`** — a fixed session of mxcli commands (open, author entities, microflows and pages, describe, catalog refresh, check, a one-document edit, a bulk change, a re-run, a rename), one process per command, on PedApp and on PedApp with a generated app seeded into it. `make bench BENCH_RUNTIME=` adds a session that validates and builds the project with mx and then runs it — `docker check`, `docker build`, boot, a hot change, an entity change, a page check, an attached and a cold test run — on a project `mxcli new` creates, with mxbuild and the runtime installed by mxcli when they are not cached. With `BASELINE=` the two binaries run interleaved on the same machine and each step reports the ratio; the nightly `bench` job does that against the previous nightly and commits the record to the `benchmarks` branch. It also records counts that do not depend on the machine: unit files read and unit files changed. `MXCLI_STATS_FILE=` makes any mxcli command append one JSON line per closed project connection with its unit-file reads and its writes offered and landed. See "Agent Dev-Loop Benchmark" in the docs. - **`mxcli test --local --reuse-build` skips the injection and the build when nothing changed** (mendixlabs/mxcli#1783, closes mendixlabs/mxcli#1782) — a local test run injects its test microflows, builds and boots; when the project, the test files and mxcli are unchanged, the deployment the previous run left is already this run's build, so `--reuse-build` boots it directly (about 12 s against 60 s, measured by the author). The decision is a content hash of the project folder (outputs, caches, `theme-cache/`, and any `.git`/`node_modules` left out), the generated test MDL, the mxcli binary and the mxbuild override, pinned to the identity of `deployment/model/model.mdp`, so a deployment rebuilt by anything else is never taken for the test build. Not under `--watch` or `--skip-build`. - **A `genai-runtime` skill for the runtime side of Agents Kit 2** (ako/mxcli#2068) — registering a Mendix Cloud GenAI key or an OpenAI-compatible endpoint, seeding AgentCommons agents from JSON (matched by `Title` only, so a changed JSON never reaches a stored agent), resuming Conversational UI chats, loading PgVector and Mendix Cloud knowledge bases, and traces. It collects the gotchas that `check` and `mx check` cannot see: the encryption key both the OpenAI and PgVector setup need, the `vector` extension only the UI's Test button creates, the 60–120 s indexing delay, the chat that collapses outside its page, and `StoreTraces` defaulting to false. Its examples pass `check --references` and build with 0 errors against the Agent Builder Starter App's modules. diff --git a/docs-site/src/tools/catalog-tables.md b/docs-site/src/tools/catalog-tables.md index 4572bcb129..eb6cb462be 100644 --- a/docs-site/src/tools/catalog-tables.md +++ b/docs-site/src/tools/catalog-tables.md @@ -250,6 +250,37 @@ JOIN CATALOG.ACTIVITIES a ON a.Id = f.DestinationId WHERE f.FlowType = 'AnnotationFlow'; ``` +### CATALOG.WORKFLOW_ACTIVITIES + +One row per activity of a workflow, in walk order: the main flow, each +activity followed by the paths of its boundary events, then the event +sub-processes. Populated by every catalog build. A slot that names nothing is +NULL. + +| Column | Description | +|--------|-------------| +| `Id` | The activity's ID | +| `WorkflowId`, `WorkflowQualifiedName`, `ModuleName` | The workflow it belongs to (`CATALOG.WORKFLOWS.Id`) | +| `Sequence` | Walk order within the workflow | +| `Name`, `Caption` | As in Studio Pro | +| `ActivityType` | `UserTask`, `MultiUserTask`, `SystemTask`, `CallMicroflow`, `AIAgentTask`, `CallWorkflow`, `Decision`, `ParallelSplit`, `JumpTo`, `WaitForTimer`, `WaitForNotification`, `StartWorkflow`, `EndWorkflow`, and the event sub-process and path-end kinds | +| `Microflow` | The microflow a system task, call-microflow activity or AI agent task runs | +| `UserSourceMicroflow` | A user task's targeting microflow (users or groups) | +| `OnCreatedMicroflow` | A user task's on-created microflow | +| `CompletionMicroflow` | A multi-user task's completion microflow | +| `Page`, `UserTaskEntity` | A user task's page and task entity | +| `CalledWorkflow` | The workflow a call-workflow activity calls | + +`CATALOG.REFS` records that a workflow calls a microflow; this table says from +which kind of activity and which slot. + +```sql +-- Microflows that system tasks run, without the WF_SYS_ prefix +SELECT WorkflowQualifiedName, Name, Microflow +FROM CATALOG.WORKFLOW_ACTIVITIES +WHERE ActivityType = 'SystemTask' AND Microflow NOT LIKE '%.WF_SYS_%'; +``` + ### CATALOG.PAGES Information about pages and their properties. diff --git a/mdl/catalog/builder_workflow_activities.go b/mdl/catalog/builder_workflow_activities.go new file mode 100644 index 0000000000..e7b10d5136 --- /dev/null +++ b/mdl/catalog/builder_workflow_activities.go @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "github.com/mendixlabs/mxcli/mdl/backend/wfnames" + "github.com/mendixlabs/mxcli/sdk/workflows" +) + +// workflowActivityRow is one row of workflow_activities (ako/mxcli#1295). +type workflowActivityRow struct { + ID, Name, Caption, ActivityType string + Microflow, UserSourceMicroflow, OnCreatedMicroflow, CompletionMicroflow string + Page, UserTaskEntity, CalledWorkflow string +} + +// workflowActivityRows lists a workflow's activities in walk order with the +// documents each one names. A lint rule that wants "every microflow a system +// task runs" or "every page a user task opens" reads it, where refs says only +// that the workflow calls the microflow, not from which kind of activity. +func workflowActivityRows(wf *workflows.Workflow) []workflowActivityRow { + var out []workflowActivityRow + wfnames.WalkActivities(wf, func(act workflows.WorkflowActivity) { + r := workflowActivityRow{ + ID: string(act.GetID()), + Name: act.GetName(), + Caption: act.GetCaption(), + ActivityType: act.ActivityType(), + } + switch a := act.(type) { + case *workflows.UserTask: + if a.IsMulti { + r.ActivityType = "MultiUserTask" + } + r.Page, r.UserTaskEntity, r.OnCreatedMicroflow = a.Page, a.UserTaskEntity, a.OnCreated + switch us := a.UserSource.(type) { + case *workflows.MicroflowBasedUserSource: + r.UserSourceMicroflow = us.Microflow + case *workflows.MicroflowGroupSource: + r.UserSourceMicroflow = us.Microflow + } + if a.CompletionCriteria != nil { + r.CompletionMicroflow = a.CompletionCriteria.Microflow + } + case *workflows.CallMicroflowTask: + if a.IsAgent { + r.ActivityType = "AIAgentTask" + } + r.Microflow = a.Microflow + case *workflows.SystemTask: + r.Microflow = a.Microflow + case *workflows.CallWorkflowActivity: + r.CalledWorkflow = a.Workflow + } + out = append(out, r) + }) + return out +} + +// buildWorkflowActivities writes workflow_activities. +func (b *Builder) buildWorkflowActivities(wfs []*workflows.Workflow) (int, error) { + stmt, err := b.tx.Prepare(` + INSERT INTO workflow_activities_data (Id, Sequence, WorkflowId, WorkflowQualifiedName, ModuleName, + Name, Caption, ActivityType, Microflow, UserSourceMicroflow, OnCreatedMicroflow, + CompletionMicroflow, Page, UserTaskEntity, CalledWorkflow, ProjectId, SnapshotId) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `) + if err != nil { + return 0, err + } + defer stmt.Close() + projectID, snapshotID := b.snapshotMeta() + + count := 0 + for _, wf := range wfs { + if !b.own(string(wf.ID)) { + continue + } + moduleName := b.hierarchy.getModuleName(b.hierarchy.findModuleID(wf.ContainerID)) + qualifiedName := moduleName + "." + wf.Name + for i, r := range workflowActivityRows(wf) { + if _, err := stmt.Exec(r.ID, i, string(wf.ID), qualifiedName, moduleName, + r.Name, r.Caption, r.ActivityType, + nullIfEmpty(r.Microflow), nullIfEmpty(r.UserSourceMicroflow), nullIfEmpty(r.OnCreatedMicroflow), + nullIfEmpty(r.CompletionMicroflow), nullIfEmpty(r.Page), nullIfEmpty(r.UserTaskEntity), + nullIfEmpty(r.CalledWorkflow), projectID, snapshotID); err != nil { + return count, err + } + count++ + } + } + b.disown() + return count, nil +} + +// nullIfEmpty is a reference column: the referenced document's qualified +// name, or NULL when the slot is empty — one rule since schema 47: absent is +// NULL (ako/mxcli#2093, docRefName). +func nullIfEmpty(s string) any { + if s == "" { + return nil + } + return s +} diff --git a/mdl/catalog/builder_workflow_activities_test.go b/mdl/catalog/builder_workflow_activities_test.go new file mode 100644 index 0000000000..445e30e212 --- /dev/null +++ b/mdl/catalog/builder_workflow_activities_test.go @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "fmt" + "strings" + "testing" + + modelsdkbackend "github.com/mendixlabs/mxcli/mdl/backend/modelsdk" +) + +// ako/mxcli#1295 (mendixlabs/mxcli#1269): a lint rule could not ask which kind +// of workflow activity runs a microflow — "a system task's microflow is +// prefixed WF_SYS_", "a user task's targeting microflow WF_TGT_". refs records +// only that the WORKFLOW calls the MICROFLOW, and workflows_data only counts +// activities. + +func TestWorkflowActivityRows_KindAndSlots_Issue1295(t *testing.T) { + var got []string + for _, r := range workflowActivityRows(everySlotWorkflow()) { + got = append(got, fmt.Sprintf("%s mf=%s src=%s created=%s done=%s page=%s wf=%s", + r.ActivityType, r.Microflow, r.UserSourceMicroflow, r.OnCreatedMicroflow, + r.CompletionMicroflow, r.Page, r.CalledWorkflow)) + } + want := []string{ + // The main flow, each activity followed by its boundary-event paths. + "UserTask mf= src=M.TargetUsers created=M.OnCreated done= page=M.TaskPage wf=", + "CallMicroflow mf=M.InUserTaskBoundary src= created= done= page= wf=", + "MultiUserTask mf= src=M.TargetGroups created= done=M.Completion page= wf=", + "CallMicroflow mf=M.MainCall src= created= done= page= wf=", + "SystemTask mf=M.InCallBoundary src= created= done= page= wf=", + "CallWorkflow mf= src= created= done= page= wf=M.SubWorkflow", + "CallMicroflow mf=M.InCallWorkflowBoundary src= created= done= page= wf=", + "WaitForNotification mf= src= created= done= page= wf=", + "CallMicroflow mf=M.InWaitBoundary src= created= done= page= wf=", + // Then the event sub-process. + "InterruptingNotificationEventSubProcessStart mf= src= created= done= page= wf=", + "CallMicroflow mf=M.InEventSubProcess src= created= done= page= wf=", + "UserTask mf= src= created= done= page=M.SubProcessTaskPage wf=", + } + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Errorf("rows:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +} + +// The table is written by a fast build: model-quality rules run on one. +func TestTestApp_WorkflowActivitiesInFastBuild_Issue1295(t *testing.T) { + be := modelsdkbackend.New() + if err := be.Connect(copyTestApp(t)); err != nil { + t.Fatalf("connect: %v", err) + } + t.Cleanup(func() { be.Disconnect() }) + cat, err := New() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { cat.Close() }) + if err := NewBuilder(cat, be).Build(nil); err != nil { + t.Fatalf("catalog build: %v", err) + } + + // workflow.UserTaskEventHandle is a user task's on-created microflow + // (TestTestApp_WorkflowHandlersAreNotDead). + rows := queryRows(t, cat, `SELECT WorkflowQualifiedName, ActivityType FROM workflow_activities + WHERE OnCreatedMicroflow = 'workflow.UserTaskEventHandle'`) + if len(rows) == 0 { + t.Fatal("no activity has workflow.UserTaskEventHandle as its on-created microflow") + } + for _, r := range rows { + if typ := r[1].(string); typ != "UserTask" && typ != "MultiUserTask" { + t.Errorf("%v: on-created microflow on a %s", r[0], typ) + } + } + // An empty slot is NULL, not ''. + if n := queryRows(t, cat, `SELECT COUNT(*) FROM workflow_activities WHERE Microflow = '' OR Page = ''`); n[0][0].(int64) != 0 { + t.Errorf("%v rows store an empty slot as ''", n[0][0]) + } + // Every workflow's activity count matches its rows. + for _, r := range queryRows(t, cat, `SELECT w.QualifiedName, w.ActivityCount, + (SELECT COUNT(*) FROM workflow_activities a WHERE a.WorkflowId = w.Id) + FROM workflows w`) { + if r[1].(int64) != r[2].(int64) { + t.Errorf("%v: ActivityCount %v but %v activity rows", r[0], r[1], r[2]) + } + } +} diff --git a/mdl/catalog/builder_workflows.go b/mdl/catalog/builder_workflows.go index ec1cd00bf8..e904a7bb33 100644 --- a/mdl/catalog/builder_workflows.go +++ b/mdl/catalog/builder_workflows.go @@ -72,6 +72,12 @@ func (b *Builder) buildWorkflows() error { b.disown() b.report("Workflows", count) + + activities, err := b.buildWorkflowActivities(wfs) + if err != nil { + return err + } + b.report("Workflow activities", activities) return nil } diff --git a/mdl/catalog/catalog.go b/mdl/catalog/catalog.go index 47ee873f45..9528df35c0 100644 --- a/mdl/catalog/catalog.go +++ b/mdl/catalog/catalog.go @@ -157,6 +157,7 @@ func (c *Catalog) Tables() []string { "CATALOG.SNAPSHOTS", "CATALOG.OBJECTS", "CATALOG.WORKFLOWS", + "CATALOG.WORKFLOW_ACTIVITIES", "CATALOG.CONSUMED_ODATA_SERVICES", "CATALOG.PUBLISHED_ODATA_SERVICES", "CATALOG.BUSINESS_EVENT_SERVICES", diff --git a/mdl/catalog/tables.go b/mdl/catalog/tables.go index e01478b591..04bc71e795 100644 --- a/mdl/catalog/tables.go +++ b/mdl/catalog/tables.go @@ -7,6 +7,11 @@ package catalog // // History: // +// 48 (workflow activities): new workflow_activities table, one row per +// activity of a workflow — boundary-event paths and event sub-processes +// included — with its kind and the microflow, page or workflow each of +// its slots names (ako/mxcli#1295). Without the bump a cached catalog +// lacks the table and workflow_activities() fails with "no such table". // 47 (absent references are NULL): the agent-editor tables (agents, // agent_tools, agent_knowledge_bases, ai_models, knowledge_bases, // consumed_mcp_services, published_mcp_services) store an absent @@ -257,7 +262,7 @@ package catalog // SnapshotSource / SourceId / SourceBranch / SourceRevision columns // from every row (issue #576). // 1 — initial flat schema with denormalized snapshot columns on every row. -const CatalogSchemaVersion = "47" +const CatalogSchemaVersion = "48" // MetaSchemaVersion is the catalog_meta key that records the schema version // the cache was built against. @@ -1135,6 +1140,41 @@ func (c *Catalog) createTables() error { )`, viewWithFullSnapshot("workflows"), + // workflow_activities — one row per activity of a workflow, in the + // order wfnames.WalkActivities visits them: the main flow, the paths + // of its boundary events, then its event sub-processes + // (ako/mxcli#1295). ActivityType is the activity's kind as + // workflows_data counts it, with the two kinds that share a Go type + // told apart: MultiUserTask from UserTask, AIAgentTask from + // CallMicroflow. Each slot that names another document is a column of + // its own, so a rule can ask which microflow runs where: + // Microflow a system task's, call-microflow or AI agent task's + // UserSourceMicroflow a user task's targeting microflow (users or groups) + // OnCreatedMicroflow a user task's on-created microflow + // CompletionMicroflow a multi-user task's completion microflow + // Page a user task's page + // CalledWorkflow a call-workflow activity's workflow + `CREATE TABLE IF NOT EXISTS workflow_activities_data ( + Id TEXT PRIMARY KEY, + Sequence INTEGER DEFAULT 0, + WorkflowId TEXT, + WorkflowQualifiedName TEXT, + ModuleName TEXT, + Name TEXT, + Caption TEXT, + ActivityType TEXT, + Microflow TEXT, + UserSourceMicroflow TEXT, + OnCreatedMicroflow TEXT, + CompletionMicroflow TEXT, + Page TEXT, + UserTaskEntity TEXT, + CalledWorkflow TEXT, + ProjectId TEXT, + SnapshotId TEXT + )`, + viewWithFullSnapshot("workflow_activities"), + // business_event_services `CREATE TABLE IF NOT EXISTS business_event_services_data ( Id TEXT PRIMARY KEY, @@ -1868,6 +1908,7 @@ func (c *Catalog) createTables() error { `CREATE INDEX IF NOT EXISTS idx_odata_services_module ON odata_services_data(ModuleName)`, `CREATE INDEX IF NOT EXISTS idx_workflows_name ON workflows_data(QualifiedName)`, `CREATE INDEX IF NOT EXISTS idx_workflows_module ON workflows_data(ModuleName)`, + `CREATE INDEX IF NOT EXISTS idx_workflow_activities_workflow ON workflow_activities_data(WorkflowQualifiedName)`, `CREATE INDEX IF NOT EXISTS idx_be_services_name ON business_event_services_data(QualifiedName)`, `CREATE INDEX IF NOT EXISTS idx_be_services_module ON business_event_services_data(ModuleName)`, `CREATE INDEX IF NOT EXISTS idx_role_mappings_user_role ON role_mappings(UserRoleName)`, diff --git a/mdl/linter/context_catalog_tables.go b/mdl/linter/context_catalog_tables.go index f3a55f452e..4cfbfa5600 100644 --- a/mdl/linter/context_catalog_tables.go +++ b/mdl/linter/context_catalog_tables.go @@ -662,3 +662,114 @@ func (ctx *LintContext) MarketplaceModuleVersion(name string) string { } return v.String } + +// Workflow is one workflow the user owns (ako/mxcli#1295). +type Workflow struct { + ID string + QualifiedName string + Name string + ModuleName string + Description string + ParameterEntity string + ActivityCount int + UserTaskCount int + MicroflowCallCount int + DecisionCount int +} + +// Workflows iterates the workflows outside System and Marketplace modules. +// --document names the workflow. +func (ctx *LintContext) Workflows() iter.Seq[Workflow] { + return func(yield func(Workflow) bool) { + rows, err := ctx.db.Query(fmt.Sprintf(` + SELECT w.Id, COALESCE(w.QualifiedName, ''), COALESCE(w.Name, ''), COALESCE(w.ModuleName, ''), + COALESCE(w.Description, ''), COALESCE(w.ParameterEntity, ''), + COALESCE(w.ActivityCount, 0), COALESCE(w.UserTaskCount, 0), + COALESCE(w.MicroflowCallCount, 0), COALESCE(w.DecisionCount, 0) + FROM workflows w + LEFT JOIN modules m ON w.ModuleName = m.Name + WHERE %s AND %s + ORDER BY w.QualifiedName + `, notPlatformModule("m"), ctx.documentFilterSQL("w.QualifiedName"))) + if err != nil { + ctx.recordQueryError("Workflows", err) + return + } + defer rows.Close() + for rows.Next() { + var w Workflow + if err := rows.Scan(&w.ID, &w.QualifiedName, &w.Name, &w.ModuleName, &w.Description, + &w.ParameterEntity, &w.ActivityCount, &w.UserTaskCount, &w.MicroflowCallCount, + &w.DecisionCount); err != nil { + ctx.recordQueryError("Workflows (row scan)", err) + continue + } + if ctx.IsExcluded(w.ModuleName) { + continue + } + if !yield(w) { + return + } + } + } +} + +// WorkflowActivity is one activity of a workflow, boundary-event paths and +// event sub-processes included (ako/mxcli#1295). ActivityType is its kind — +// UserTask, MultiUserTask, SystemTask, CallMicroflow, AIAgentTask, +// CallWorkflow, Decision, … — and each slot naming another document is a field +// of its own, "" when empty. +type WorkflowActivity struct { + ID string + WorkflowQualifiedName string + ModuleName string + Name string + Caption string + ActivityType string + Microflow string // a system task's, call-microflow or AI agent task's + UserSourceMicroflow string // a user task's targeting microflow + OnCreatedMicroflow string // a user task's on-created microflow + CompletionMicroflow string // a multi-user task's completion microflow + Page string // a user task's page + UserTaskEntity string // a user task's task entity + CalledWorkflow string // a call-workflow activity's workflow +} + +// WorkflowActivities iterates the activities of every workflow outside System +// and Marketplace modules, each workflow's in walk order. --document names +// the workflow. +func (ctx *LintContext) WorkflowActivities() iter.Seq[WorkflowActivity] { + return func(yield func(WorkflowActivity) bool) { + rows, err := ctx.db.Query(fmt.Sprintf(` + SELECT a.Id, COALESCE(a.WorkflowQualifiedName, ''), COALESCE(a.ModuleName, ''), + COALESCE(a.Name, ''), COALESCE(a.Caption, ''), COALESCE(a.ActivityType, ''), + COALESCE(a.Microflow, ''), COALESCE(a.UserSourceMicroflow, ''), + COALESCE(a.OnCreatedMicroflow, ''), COALESCE(a.CompletionMicroflow, ''), + COALESCE(a.Page, ''), COALESCE(a.UserTaskEntity, ''), COALESCE(a.CalledWorkflow, '') + FROM workflow_activities a + LEFT JOIN modules m ON a.ModuleName = m.Name + WHERE %s AND %s + ORDER BY a.WorkflowQualifiedName, a.Sequence + `, notPlatformModule("m"), ctx.documentFilterSQL("a.WorkflowQualifiedName"))) + if err != nil { + ctx.recordQueryError("WorkflowActivities", err) + return + } + defer rows.Close() + for rows.Next() { + var a WorkflowActivity + if err := rows.Scan(&a.ID, &a.WorkflowQualifiedName, &a.ModuleName, &a.Name, &a.Caption, + &a.ActivityType, &a.Microflow, &a.UserSourceMicroflow, &a.OnCreatedMicroflow, + &a.CompletionMicroflow, &a.Page, &a.UserTaskEntity, &a.CalledWorkflow); err != nil { + ctx.recordQueryError("WorkflowActivities (row scan)", err) + continue + } + if ctx.IsExcluded(a.ModuleName) { + continue + } + if !yield(a) { + return + } + } + } +} diff --git a/mdl/linter/context_systemmodule_test.go b/mdl/linter/context_systemmodule_test.go index d6d13275a0..98cdbda237 100644 --- a/mdl/linter/context_systemmodule_test.go +++ b/mdl/linter/context_systemmodule_test.go @@ -66,6 +66,10 @@ func systemLeakFixture(t *testing.T) *catalog.Catalog { VALUES (?,?,?,?,?,?)`, "ja-"+mod, "JA"+mod, mod+".JA"+mod, mod, "default", "s1") exec(`INSERT INTO widgets_data (Id, Name, WidgetType, ModuleName, ProjectId, SnapshotId) VALUES (?,?,?,?,?,?)`, "w-"+mod, "Widget"+mod, "TextBox", mod, "default", "s1") + exec(`INSERT INTO workflows_data (Id, Name, QualifiedName, ModuleName, ProjectId, SnapshotId) + VALUES (?,?,?,?,?,?)`, "wf-"+mod, "WF"+mod, mod+".WF"+mod, mod, "default", "s1") + exec(`INSERT INTO workflow_activities_data (Id, WorkflowId, WorkflowQualifiedName, ModuleName, Name, ActivityType, ProjectId, SnapshotId) + VALUES (?,?,?,?,?,?,?,?)`, "wa-"+mod, "wf-"+mod, mod+".WF"+mod, mod, "Task", "SystemTask", "default", "s1") } return cat @@ -107,6 +111,12 @@ func TestIterators_ExcludePlatformModules(t *testing.T) { for w := range ctx.Widgets() { collect("Widgets", w.ModuleName+".w") } + for w := range ctx.Workflows() { + collect("Workflows", w.QualifiedName) + } + for a := range ctx.WorkflowActivities() { + collect("WorkflowActivities", a.WorkflowQualifiedName) + } for d := range ctx.DocumentableElements() { collect("DocumentableElements", d.QualifiedName) } diff --git a/mdl/linter/starlark.go b/mdl/linter/starlark.go index 920599b9cb..8abfe2c018 100644 --- a/mdl/linter/starlark.go +++ b/mdl/linter/starlark.go @@ -427,6 +427,8 @@ func (r *StarlarkRule) buildPredeclared() starlark.StringDict { "strings": starlark.NewBuiltin("strings", r.builtinStrings), "layouts": starlark.NewBuiltin("layouts", r.builtinLayouts), "published_rest_operations": starlark.NewBuiltin("published_rest_operations", r.builtinPublishedRestOperations), + "workflows": starlark.NewBuiltin("workflows", r.builtinWorkflows), + "workflow_activities": starlark.NewBuiltin("workflow_activities", r.builtinWorkflowActivities), "modules": starlark.NewBuiltin("modules", r.builtinModules), "folders": starlark.NewBuiltin("folders", r.builtinFolders), "annotations": starlark.NewBuiltin("annotations", r.builtinAnnotations), diff --git a/mdl/linter/starlark_catalog_mode_guard_test.go b/mdl/linter/starlark_catalog_mode_guard_test.go index 84e9c82a6f..f110e89246 100644 --- a/mdl/linter/starlark_catalog_mode_guard_test.go +++ b/mdl/linter/starlark_catalog_mode_guard_test.go @@ -70,6 +70,8 @@ var builtinModes = map[string]builtinProbe{ "jar_dependencies": {CatalogFast, "jar_dependencies()"}, "layouts": {CatalogFast, "layouts()"}, "published_rest_operations": {CatalogFast, "published_rest_operations()"}, + "workflows": {CatalogFast, "workflows()"}, + "workflow_activities": {CatalogFast, "workflow_activities()"}, "widgets": {CatalogFull, "widgets()"}, "xpath_expressions": {CatalogFull, "xpath_expressions()"}, diff --git a/mdl/linter/starlark_catalog_tables.go b/mdl/linter/starlark_catalog_tables.go index cb40e8adce..ef1ee80664 100644 --- a/mdl/linter/starlark_catalog_tables.go +++ b/mdl/linter/starlark_catalog_tables.go @@ -263,3 +263,58 @@ func moduleToStarlark(m Module) starlark.Value { "domain_model_documentation": starlark.String(m.DomainModelDocumentation), }) } + +func (r *StarlarkRule) builtinWorkflows(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if r.ctx == nil { + return starlark.NewList(nil), nil + } + var out []starlark.Value + for w := range r.ctx.Workflows() { + out = append(out, workflowToStarlark(w)) + } + return starlark.NewList(out), nil +} + +func (r *StarlarkRule) builtinWorkflowActivities(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if r.ctx == nil { + return starlark.NewList(nil), nil + } + var out []starlark.Value + for a := range r.ctx.WorkflowActivities() { + out = append(out, workflowActivityToStarlark(a)) + } + return starlark.NewList(out), nil +} + +func workflowToStarlark(w Workflow) starlark.Value { + return starlarkstruct.FromStringDict(starlark.String("workflow"), starlark.StringDict{ + "id": starlark.String(w.ID), + "qualified_name": starlark.String(w.QualifiedName), + "name": starlark.String(w.Name), + "module_name": starlark.String(w.ModuleName), + "description": starlark.String(w.Description), + "parameter_entity": starlark.String(w.ParameterEntity), + "activity_count": starlark.MakeInt(w.ActivityCount), + "user_task_count": starlark.MakeInt(w.UserTaskCount), + "microflow_call_count": starlark.MakeInt(w.MicroflowCallCount), + "decision_count": starlark.MakeInt(w.DecisionCount), + }) +} + +func workflowActivityToStarlark(a WorkflowActivity) starlark.Value { + return starlarkstruct.FromStringDict(starlark.String("workflow_activity"), starlark.StringDict{ + "id": starlark.String(a.ID), + "workflow_qualified_name": starlark.String(a.WorkflowQualifiedName), + "module_name": starlark.String(a.ModuleName), + "name": starlark.String(a.Name), + "caption": starlark.String(a.Caption), + "activity_type": starlark.String(a.ActivityType), + "microflow": starlark.String(a.Microflow), + "user_source_microflow": starlark.String(a.UserSourceMicroflow), + "on_created_microflow": starlark.String(a.OnCreatedMicroflow), + "completion_microflow": starlark.String(a.CompletionMicroflow), + "page": starlark.String(a.Page), + "user_task_entity": starlark.String(a.UserTaskEntity), + "called_workflow": starlark.String(a.CalledWorkflow), + }) +} diff --git a/mdl/linter/starlark_workflow_activities_test.go b/mdl/linter/starlark_workflow_activities_test.go new file mode 100644 index 0000000000..b4ce7e6389 --- /dev/null +++ b/mdl/linter/starlark_workflow_activities_test.go @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: Apache-2.0 + +package linter_test + +import ( + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/catalog" + "github.com/mendixlabs/mxcli/mdl/linter" +) + +// ako/mxcli#1295 (mendixlabs/mxcli#1269): "microflows used in workflows must +// use a prefix per role (user task, system task, ...)". A rule needs the kind +// of activity each microflow runs from, which refs (WORKFLOW -> MICROFLOW +// `call`) does not record. +func TestWorkflowActivities_PrefixPerRole_Issue1295(t *testing.T) { + cat, err := catalog.New() + if err != nil { + t.Fatal(err) + } + defer cat.Close() + db := cat.CatalogDB() + for _, q := range []string{ + `INSERT INTO modules_data (Id, Name, QualifiedName, ModuleName, Source) VALUES ('m', 'Sales', 'Sales', 'Sales', '')`, + `INSERT INTO workflows_data (Id, Name, QualifiedName, ModuleName, ParameterEntity, ActivityCount) + VALUES ('wf', 'Approve', 'Sales.Approve', 'Sales', 'Sales.Order', 4)`, + `INSERT INTO workflow_activities_data (Id, Sequence, WorkflowId, WorkflowQualifiedName, ModuleName, Name, + ActivityType, Microflow, UserSourceMicroflow, OnCreatedMicroflow, Page) + VALUES + ('a1', 0, 'wf', 'Sales.Approve', 'Sales', 'Start', 'StartWorkflow', NULL, NULL, NULL, NULL), + ('a2', 1, 'wf', 'Sales.Approve', 'Sales', 'Review', 'UserTask', NULL, 'Sales.WF_TGT_Managers', 'Sales.Notify', 'Sales.Review_Page'), + ('a3', 2, 'wf', 'Sales.Approve', 'Sales', 'Book', 'SystemTask', 'Sales.BookOrder', NULL, NULL, NULL), + ('a4', 3, 'wf', 'Sales.Approve', 'Sales', 'Mail', 'SystemTask', 'Sales.WF_SYS_Mail', NULL, NULL, NULL)`, + } { + if _, err := db.Exec(q); err != nil { + t.Fatalf("fixture: %v", err) + } + } + + rule := `RULE_ID = "TEST01" +RULE_NAME = "Workflow microflow prefixes" +DESCRIPTION = "probe" +CATEGORY = "naming" +SEVERITY = "info" + +PREFIX = {"SystemTask": "WF_SYS_", "user_source": "WF_TGT_", "on_created": "WF_EVT_"} + +def bad(mf, prefix): + return mf != "" and not mf.split(".")[-1].startswith(prefix) + +def check(): + out = [] + for wf in workflows(): + out.append(violation(message = "%s: %d activities, context %s" % (wf.qualified_name, wf.activity_count, wf.parameter_entity))) + for a in workflow_activities(): + if a.activity_type == "SystemTask" and bad(a.microflow, PREFIX["SystemTask"]): + out.append(violation(message = "%s/%s runs %s" % (a.workflow_qualified_name, a.name, a.microflow))) + if bad(a.user_source_microflow, PREFIX["user_source"]): + out.append(violation(message = "%s/%s targets with %s" % (a.workflow_qualified_name, a.name, a.user_source_microflow))) + if bad(a.on_created_microflow, PREFIX["on_created"]): + out.append(violation(message = "%s/%s on created %s" % (a.workflow_qualified_name, a.name, a.on_created_microflow))) + return out +` + path := filepath.Join(t.TempDir(), "wf.star") + if err := os.WriteFile(path, []byte(rule), 0o644); err != nil { + t.Fatal(err) + } + r, err := linter.LoadStarlarkRule(path) + if err != nil { + t.Fatalf("LoadStarlarkRule: %v", err) + } + ctx := linter.NewLintContextFromDB(db) + var got []string + for _, v := range r.Check(ctx) { + got = append(got, v.Message) + } + if errs := ctx.QueryErrors(); len(errs) > 0 { + t.Fatalf("query errors: %v -- the query names a column the catalog does not have", errs) + } + sort.Strings(got) + want := []string{ + "Sales.Approve/Book runs Sales.BookOrder", + "Sales.Approve/Review on created Sales.Notify", + "Sales.Approve: 4 activities, context Sales.Order", + } + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Errorf("findings:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +}