From 092f90240cabb1c0dfb66dea1c208b34153b1e84 Mon Sep 17 00:00:00 2001 From: Grant Minor Date: Fri, 18 Sep 2026 16:11:16 -0500 Subject: [PATCH 1/4] Initial add of TechSoup import: Work in progress --- docs/techsoup.md | 264 ++++++++++++++++++++++++ package-lock.json | 44 +++- package.json | 5 +- src/index.ts | 2 + src/okf.ts | 416 ++++++++++++++++++++++++++++++++++++++ src/techsoup.ts | 387 +++++++++++++++++++++++++++++++++++ src/techsoup.unit.test.ts | 225 +++++++++++++++++++++ 7 files changed, 1336 insertions(+), 7 deletions(-) create mode 100644 docs/techsoup.md create mode 100644 src/okf.ts create mode 100644 src/techsoup.ts create mode 100644 src/techsoup.unit.test.ts diff --git a/docs/techsoup.md b/docs/techsoup.md new file mode 100644 index 0000000..08db3a8 --- /dev/null +++ b/docs/techsoup.md @@ -0,0 +1,264 @@ +# TechSoup Integration — Technical Report + +**Source file:** [`../src/techsoup.ts`](../src/techsoup.ts) +**Generated:** 2026-09-18 +**Purpose:** Read nonprofit ("changemaker") data from TechSoup [OKF](https://okf.md/spec/)-format bundles delivered as a zip file or folder, and import the mapped organization fields into the Philanthropy Data Commons (PDC) as changemaker field values. + +--- + +## 1. Overview + +`techsoup.ts` defines a [yargs](https://yargs.js.org/) command module (`techsoup`) registered in [`../src/index.ts`](../src/index.ts) as part of the `data-scripts` CLI. Unlike the Charity Navigator and GivingTuesday integrations, the **source is not a REST API** — it is a set of local files (an OKF bundle, or "package") that TechSoup produces. The destination is the same PDC REST API used by the other integrations. + +It exposes three subcommands: + +| Subcommand | Purpose | Writes to PDC? | Needs PDC auth? | +| --------------- | --------------------------------------------------------------------------------------------------------- | -------------- | -------------------------- | +| `inventory` | Examine a zip/folder, discover the OKF packages inside, and print/save the mapped fields available | No | No | +| `lookupFromPdc` | Discover the packages, extract each EIN, and report which are present in PDC (dry-run style) | No | No (reads PDC anonymously) | +| `updateAll` | Full sync: discover packages → match EIN to a PDC changemaker → write mapped organization fields into PDC | **Yes**\* | **Yes (OIDC)**\* | + +\* `updateAll --dry-run` performs the full discovery and mapping and logs exactly what it _would_ write, without authenticating or contacting the PDC for writes. + +Invocation examples: + +```bash +data-scripts techsoup inventory --source C:\Data\TechSoup\organizations +``` + +```bash +data-scripts techsoup updateAll --source techsoup-bundles.zip --dry-run --pdc-api-base-url --oidc-base-url --oidc-client-id --oidc-client-secret +``` + +```bash +data-scripts techsoup updateAll --source C:\Data\TechSoup\organizations --pdc-api-base-url --oidc-base-url --oidc-client-id --oidc-client-secret +``` + +> Because the source is a local bundle, no source API key is required. Authentication (OIDC) is only needed to _write_ into PDC via `updateAll` (and is skipped entirely under `--dry-run`). + +--- + +## 2. External Systems + +### TechSoup OKF bundles (source) + +- **Format:** [OKF](https://okf.md/spec/) — each organization is a directory ("package") of markdown files with YAML **frontmatter**. The package root is `README.md` with frontmatter `type: org`; sibling files (`impact.md`, `population.md`, `programs.md`, `what_i_need_funding_for.md`, `technical-volunteers/*.md`, …) carry the rest. +- **Delivery:** a single `.zip`, a folder of extracted packages, or a folder containing several `.zip` files. See §6. +- **Parsing:** frontmatter is parsed with [`js-yaml`](https://github.com/nodeca/js-yaml) (`JSON_SCHEMA`, so timestamp-like scalars stay strings). Zips are read with [`adm-zip`](https://github.com/cthackers/adm-zip) — no shell dependency and cross-platform. +- **Mapping origin:** [`docs/mappings/techsoup-okf-field-gap-analysis.verbose.md`](mappings/techsoup-okf-field-gap-analysis.verbose.md). + +### PDC API (destination) + +- **Client:** Axios wrapper in [`../src/pdc-api.ts`](../src/pdc-api.ts) / [`../src/client.ts`](../src/client.ts) +- **Auth:** OIDC `client_credentials` grant via [`../src/oidc.ts`](../src/oidc.ts) (`getToken`). Only `updateAll` (non-dry-run) authenticates; reads of `/changemakers` are anonymous. +- **Relevant endpoints:** + - `GET /changemakers` — list all changemakers (shallow, anonymous) + - `GET /sources` — find the TechSoup source + - `POST /sources` — create the source (admin-only; typically fails for non-admins) + - `POST /changemakerFieldValueBatches` — open a batch + - `POST /changemakerFieldValues` — write one field value + +--- + +## 3. Data Flow (the `updateAll` path) + +``` +--source (zip | folder | folder-of-zips) + │ + ▼ +prepareSourceRoot ──► extract any .zip(s) to a temp dir (adm-zip), else use folder as-is + │ + ▼ +findOrgBundles ──► walk the tree; every dir whose README.md frontmatter is `type: org` + is one OKF package (1 package if the root itself is a bundle, else many) + │ + ▼ +GET PDC /changemakers ──► (optionally scoped to one via --changemaker-id) + │ + ▼ (if --dry-run: log intended writes and stop here) +Authenticate to PDC (OIDC client_credentials) ──► access token + │ + ▼ +Find or create PDC Source (dataProviderShortCode = "techsoup") + │ + ▼ +Open a ChangemakerFieldValueBatch (sourceId + notes with timestamp) + │ + ▼ +For each OKF package (one at a time): + extract EIN from README frontmatter (x-civic.registration.id, IRS-EIN only) + └─ no valid EIN ──► skip package + match EIN to a PDC changemaker (getChangemakerByEin) + └─ no unique match ──► skip package + derive goodAsOf from README generated.at + collect organization fields (organizationFieldMap) and proposal fields + └─ proposal fields are reported but NOT written (stubbed — see §4.2) + For each organization field ──► POST /changemakerFieldValues + (batchId, changemakerId, baseFieldShortCode, value, goodAsOf) + └─ 403 Forbidden ──► warn + record changemakerId, continue + │ + ▼ +Clean up any temp extraction dir +``` + +### Key processing details + +- **Package detection (1 vs many):** `findOrgBundles` walks the resolved root and treats every directory whose `README.md` frontmatter declares `type: org` as one package (it does not descend into a package once found). If the root _itself_ is a package, exactly one is returned; otherwise every nested package is collected. `describeBundleCount` logs which case occurred. Either way, packages are processed **one at a time**. +- **EIN extraction (`extractEin`):** reads `x-civic.registration.id` and only accepts it when `x-civic.registration.scheme` is `IRS-EIN` (or absent) **and** the id passes `isValidEin` (`^\d{2}-?\d{7}$`). Non-US bundles (Polish `KRS`, Kenyan PBO, …) therefore yield no EIN and are skipped — they cannot match a US-EIN-keyed changemaker. +- **EIN → changemaker matching (`getChangemakerByEin`):** compares hyphen-stripped EINs and returns a changemaker only when **exactly one** matches. Zero → logged at info and skipped; more than one → warning and skipped (ambiguous, nothing written). This is the "if it doesn't exist in the PDC, skip it" rule. +- **`goodAsOf` derivation (`extractGoodAsOf`):** takes the date portion (`YYYY-MM-DD`) of the README's `generated.at`, or `null` if missing/unparseable (`goodAsOf` is nullable in PDC). +- **Value coercion (`toFieldValueString`):** trims strings (dropping empties), stringifies numbers/booleans, and joins arrays (e.g. `aliases`, `ntee`) with `"; "`. Objects are skipped. +- **Sequential writes:** field values are POSTed one at a time (not `Promise.all`) because the PDC API times out under concurrent POSTs to `/changemakerFieldValues`. +- **403 handling:** `postChangemakerFieldValueWarnOnForbidden` swallows HTTP 403 (logs a warning, records the `changemakerId` for an end-of-run summary) and re-throws any other error. +- **Source resolution:** `getOrCreateSource` finds an existing source with `dataProviderShortCode === "techsoup"`; otherwise attempts to create one, warning that this usually requires a `pdc-admin`. +- **Temp cleanup:** any temporary extraction directory is removed in a `finally` block, whether the run succeeds or throws. + +--- + +## 4. Field Mapping: TechSoup OKF → PDC + +The mapping mirrors the original OKF exporter (gap analysis §2) and is defined by two constants in [`../src/techsoup.ts`](../src/techsoup.ts). Each entry reads one dotted frontmatter path from one file in the package and targets a PDC **base field** short code. + +### 4.1 Organization fields — written to PDC (`organizationFieldMap`) + +| OKF source (`file#frontmatter.path`) | PDC base field short code | +| ---------------------------------------- | ------------------------------- | +| `README.md#title` | `organization_name` | +| `README.md#description` | `organization_overview` | +| `README.md#aliases` | `organization_dba_name` | +| `README.md#status` | `organization_status` | +| `README.md#x-civic.registration_country` | `organization_country` | +| `README.md#x-civic.registration.id` | `organization_tax_id` | +| `README.md#x-civic.ntee` | `organization_ntee_code` | +| `README.md#x-civic.situation` | `organization_geographic_area` | +| `population.md#description` | `community_definition_overview` | + +Each becomes a **`ChangemakerFieldValue`**: + +| PDC field value property | Value / source | +| ------------------------ | ------------------------------------------------------------ | +| `changemakerId` | Resolved via `getChangemakerByEin` (normalized EIN match) | +| `batchId` | ID of the batch opened for this run | +| `baseFieldShortCode` | From the mapping table above | +| `value` | Coerced via `toFieldValueString` (arrays joined with `"; "`) | +| `goodAsOf` | Derived from README `generated.at` (ISO date or `null`) | + +> **Known data-quality caveat (gap analysis §5.1):** `README#status` is an OKF _document_ lifecycle value (`stable`/`draft`), not an organizational status. It is mapped to `organization_status` to match the existing exporter output; the code carries a `NOTE` to revisit this once the correct source is settled. `organization_dba_name` and `organization_ntee_code` originate as arrays and are flattened to a `"; "`-joined string (§5.5). + +### 4.2 Proposal fields — collected and reported, **not yet written** (`proposalFieldMap`) + +| OKF source (`file#frontmatter.path`) | PDC base field short code | +| --------------------------------------------------------- | --------------------------------- | +| `impact.md#description` | `proposal_project_outcomes` | +| `programs.md#description` | `proposal_related_programs` | +| `what_i_need_funding_for.md#title` | `proposal_name` | +| `what_i_need_funding_for.md#description` | `proposal_funding_summary` | +| `technical-volunteers/.md#description` | `proposal_learning_and_evalution` | + +**Why stubbed:** PDC proposal data is not a flat set of values on a changemaker. It requires an **Opportunity + ApplicationForm + ProposalVersion** scaffold (see [`../src/postProposalVersions.ts`](../src/postProposalVersions.ts)) whose form/opportunity identity is not derivable from an OKF bundle alone. Until that model is decided, `updateAll` **collects** proposal fields and logs them via `warnProposalsStubbed` (a `TODO(techsoup-proposals)` marks the write site) but does not POST them. `inventory` still reports them so their availability is visible. + +> The volunteer-request markdown file is named differently per package (e.g. `cohort-outcome-reporting.md`, `intake-triage-assist.md`), so `resolveVolunteerRequestFile` finds it by frontmatter `type: volunteer-request` rather than by a fixed filename. + +### 4.3 EIN — used for matching, not stored as a field + +`x-civic.registration.id` is the join key. It _is_ also mapped to `organization_tax_id` (per the exporter), but its primary role is to match the package to a PDC changemaker; a package with no matching changemaker is skipped entirely. + +--- + +## 5. Command Reference + +### `inventory` + +- **Args:** `--source` (required), `--output-file`/`--write` (optional), `--show-fields`/`--fields` (optional boolean, default `false`). +- **Behavior:** Resolves the source (extracting zips as needed), discovers OKF packages, and for each reports its EIN, `goodAsOf`, and the count of mapped organization/proposal fields. Writes a full JSON inventory to `--output-file` if given, otherwise logs a summary. **No PDC interaction whatsoever** — this is the way to preview the mapping fully offline. +- **`--show-fields`:** Instead of per-package counts, prints the complete field-by-field mapping to the console: each organization field as a `WOULD POST changemakerFieldValue: = ` line, and each proposal field as a `proposal field (collected, NOT written)` line, with its `file#anchor` provenance. Nothing is sent to or read from the PDC; the changemaker match is only noted as happening at run time. This is the answer to "show me exactly what would go into PDC without running anything." + +```bash +data-scripts techsoup inventory --source C:\Data\TechSoup\organizations --show-fields +``` + +### `lookupFromPdc` + +- **Args:** `--source` (required), `--pdc-api-base-url` (required), `--output-file` (optional). +- **Behavior:** Discovers packages, extracts each EIN, reads all PDC changemakers, and reports per package whether a matching changemaker exists (and its ID). **Read-only** with respect to PDC. + +### `updateAll` + +- **Args:** `--source` (required), `--pdc-api-base-url` (required), all `oidcOptions` (`--oidc-base-url`, `--oidc-client-id`, `--oidc-client-secret`), `--changemaker-id` (optional; scope to one changemaker), `--dry-run` (optional boolean, default `false`). +- **Behavior:** The full sync described in §3. The only subcommand that writes to PDC. `--dry-run` performs discovery and mapping and logs each intended `POST` without authenticating or writing. + +--- + +## 6. Source resolution & package detection + +`prepareSourceRoot` accepts three shapes for `--source` and normalizes them to a single directory to walk: + +| `--source` | Handling | +| -------------------------------- | ---------------------------------------------------------------------------------------- | +| A `.zip` file | Extracted (adm-zip) to a temp dir; that dir becomes the root. | +| A folder with no `.zip` files | Used directly as the root (extracted packages already on disk). | +| A folder containing `.zip` files | Each `.zip` is extracted into a temp dir (one subfolder per archive); that becomes root. | + +The **number of OKF packages** is then a property of the tree, determined by `findOrgBundles` (§3): a root that is itself a `type: org` bundle yields one package; otherwise every nested bundle is collected. `describeBundleCount` logs "single OKF package" vs "N OKF packages … processing one at a time". A zip that wraps everything in a top-level folder is handled transparently because the walk is recursive. Any temp directory created for extraction is deleted after the command finishes. + +--- + +## 7. Error Handling & Resilience Summary + +| Concern | Handling | +| --------------------------------------- | ------------------------------------------------------------------------------ | +| Source is not a `.zip` or directory | Throws a clear error naming the offending path | +| Unreadable / missing markdown file | `parseFrontmatterFile` logs at debug and returns `null`; that field is skipped | +| Malformed YAML frontmatter | Caught; treated as empty frontmatter (package may then be skipped) | +| No frontmatter at all | Treated as an empty object | +| Non-US / non-EIN registration | `extractEin` returns `null`; package skipped | +| EIN not present in PDC | `getChangemakerByEin` returns `null`; package skipped | +| Ambiguous EIN → changemaker (>1 match) | Skipped with a warning; nothing written | +| Null / empty / object-valued attributes | Skipped (not posted) | +| Multi-value attributes (arrays) | Flattened to a `"; "`-joined string | +| Missing / unparseable `generated.at` | `goodAsOf` set to `null` | +| HTTP 403 on write | Warned + changemakerId recorded; run continues; summary warning at end | +| PDC concurrency timeouts | Field values POSTed sequentially | +| Missing TechSoup source | Attempt to create (usually admin-only); warns it may fail | +| Proposal fields | Collected and reported, **not written** (stubbed — §4.2) | +| Temp extraction directory | Always removed in a `finally` block | + +--- + +## 8. Testing + +Unit tests in [`../src/techsoup.unit.test.ts`](../src/techsoup.unit.test.ts) cover the exported pure helpers: + +- **`parseFrontmatter`** — parses top-level scalars, flow sequences, an inline flow map, a nested block map under `x-civic`, and a block sequence of mappings (`sources`) without disturbing later keys; strips trailing comments; returns `{}` when no frontmatter is present. +- **`getFrontmatterValue`** — follows a dotted path; returns `undefined` for a missing key, when descending into a non-object, or when descending into an array. +- **`toFieldValueString`** — trims strings, returns `null` for empty/whitespace/`null`/`undefined`/empty-array/plain-object, stringifies numbers/booleans, and joins arrays with `"; "` (dropping empties). +- **`extractEin`** — returns a valid EIN for `IRS-EIN` (or scheme-absent) registrations, and `null` for a non-IRS scheme (e.g. Polish KRS), an invalid id, or a missing id. +- **`extractGoodAsOf`** — extracts `YYYY-MM-DD` from `generated.at`; returns `null` when missing or unparseable. +- **`getChangemakerByEin`** — matches ignoring hyphens; returns `null` when there is no match or when more than one changemaker matches. +- **`selectChangemakers`** — returns the full bundle when no ID is given; scopes to a single changemaker and updates `total`; returns an empty bundle for an unknown ID. + +The file-system and network-facing functions (`prepareSourceRoot`, `extractZip`, `findOrgBundles`, `collectFieldsFromMap`, the command handlers, and all PDC writes) are not directly unit-tested; testability is achieved by extracting the pure parsing/mapping/matching logic into the exported helpers above (the same approach as the GivingTuesday integration). + +--- + +## 9. Notable Constants & TODOs in the Code + +- `TS_SHORT_CODE = 'techsoup'` — the PDC data provider short code for TechSoup. +- `ARRAY_VALUE_SEPARATOR = '; '` — how multi-valued frontmatter is flattened into a scalar field value. +- `FRONTMATTER_PATTERN` — isolates the YAML frontmatter block so the markdown body is never handed to the YAML parser. +- `HTTP_STATUS_FORBIDDEN = 403` — with a comment noting a shared `@pdc/http-status-codes` package should replace this once available. +- `TODO(techsoup-proposals)` — the write site for proposal data, pending the Opportunity/ApplicationForm/ProposalVersion model (§4.2). + +--- + +## 10. Comparison with the other integrations + +| Aspect | Charity Navigator | GivingTuesday | TechSoup | +| ----------------- | --------------------------------- | ------------------------------ | ------------------------------------------------------- | +| Source | GraphQL API (Apollo) | REST API (Axios GET) | Local OKF bundle (zip/folder) | +| Source auth | Bearer API key | None | None (local files) | +| Source parsing | GraphQL response | JSON response | YAML frontmatter (`js-yaml`) + `adm-zip` | +| EIN normalization | Strip hyphen | Strip hyphen + zero-pad to 9 | Strip hyphen; IRS-EIN scheme only | +| `goodAsOf` source | `updatedAt` | `Date_Released` (`YYYY_MM_DD`) | README `generated.at` (date portion) | +| Fields written | 4 (name, website, phone, mission) | 14 (IRS BMF) | 9 organization fields (proposals stubbed) | +| Dry run | via `lookup`/`lookupFromPdc` | via `lookup`/`lookupFromPdc` | `inventory`/`lookupFromPdc` **+ `updateAll --dry-run`** | diff --git a/package-lock.json b/package-lock.json index 4141393..bd2b7eb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,9 +10,11 @@ "license": "AGPL-3.0-or-later", "dependencies": { "@apollo/client": "^4.2.12", + "adm-zip": "^0.6.1", "axios": "^1.20.0", "csv-parse": "^5.5.6", "dotenv": "^17.4.2", + "js-yaml": "^5.4.2", "jwt-decode": "^4.0.0", "openid-client": "^5.6.5", "pino": "^10.3.1", @@ -807,6 +809,29 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@eslint/eslintrc/node_modules/js-yaml": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/@eslint/js": { "version": "9.39.5", "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.5.tgz", @@ -2435,6 +2460,15 @@ "node": ">=0.4.0" } }, + "node_modules/adm-zip": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.6.1.tgz", + "integrity": "sha512-Xwrja8nx9e5o2N1my4DsKCeKpdrnACyr1wtbPxBDgGzKzKyE9kRtBFA8mWldI+RVlD7CBZNWY/wQ2+ydwOR6kQ==", + "license": "MIT", + "engines": { + "node": ">=14.0" + } + }, "node_modules/agent-base": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", @@ -2545,7 +2579,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, "license": "Python-2.0" }, "node_modules/array-back": { @@ -6444,10 +6477,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.0.tgz", - "integrity": "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==", - "dev": true, + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.4.2.tgz", + "integrity": "sha512-m+aqu+LwO1O6sIopafj8HUVl5aawITwZQe/yHpMCKjaWBaA/d07B/QdMb3529REftiU+RMMHL3Vlsw3hON7vWg==", "funding": [ { "type": "github", @@ -6463,7 +6495,7 @@ "argparse": "^2.0.1" }, "bin": { - "js-yaml": "bin/js-yaml.js" + "js-yaml": "bin/js-yaml.mjs" } }, "node_modules/jsesc": { diff --git a/package.json b/package.json index 9ed1beb..b313e49 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "postProposalVersions": "node --import ./register.mjs src/postProposalVersions.ts", "charityNavigator": "node --import ./register.mjs src/index.ts charityNavigator", "givingTuesday": "node --import ./register.mjs src/index.ts givingTuesday", - "getMetrics": "node --import ./register.mjs src/index.ts getMetrics" + "getMetrics": "node --import ./register.mjs src/index.ts getMetrics", + "techsoup": "node --import ./register.mjs src/index.ts techsoup" }, "author": "Open Tech Strategies", "license": "AGPL-3.0-or-later", @@ -51,9 +52,11 @@ }, "dependencies": { "@apollo/client": "^4.2.12", + "adm-zip": "^0.6.1", "axios": "^1.20.0", "csv-parse": "^5.5.6", "dotenv": "^17.4.2", + "js-yaml": "^5.4.2", "jwt-decode": "^4.0.0", "openid-client": "^5.6.5", "pino": "^10.3.1", diff --git a/src/index.ts b/src/index.ts index 96e61c8..77cf4bc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,7 @@ import { getMetrics } from './getMetrics.js'; import { givingTuesday } from './givingTuesday.js'; import { logger } from './logger.js'; import { getTokenCommand } from './oidc.js'; +import { techsoup } from './techsoup.js'; config(); @@ -37,6 +38,7 @@ const main = async (argv: string[]): Promise => { .command(charityNavigator) .command(getMetrics) .command(givingTuesday) + .command(techsoup) .demandCommand() .parse(); }; diff --git a/src/okf.ts b/src/okf.ts new file mode 100644 index 0000000..6dad970 --- /dev/null +++ b/src/okf.ts @@ -0,0 +1,416 @@ +import { mkdtemp, readFile, readdir, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { basename, join } from 'node:path'; +import AdmZip from 'adm-zip'; +import { JSON_SCHEMA, load as loadYaml } from 'js-yaml'; +import { isValidEin } from './ein.js'; +import { logger } from './logger.js'; + +// Multi-valued frontmatter (e.g. `aliases`, `ntee`) is flattened into a single +// PDC string value with this separator, since a ChangemakerFieldValue.value is a +// scalar string. (The upstream OKF export emits a raw array; see docs/techsoup.md.) +const ARRAY_VALUE_SEPARATOR = '; '; +// A leading UTF-8 byte-order mark is not stripped by `readFile(..., 'utf8')`; +// remove it before matching frontmatter. +const BYTE_ORDER_MARK = 0xfeff; +// Isolate the YAML frontmatter block (between the leading and next `---`) so the +// markdown body that follows is never handed to the YAML parser. +const FRONTMATTER_PATTERN = /^---\r?\n(?[\s\S]*?)\r?\n---\s*(?:\r?\n|$)/v; + +/** + * A discovered OKF organization bundle ("package"): a directory whose + * `README.md` frontmatter declares `type: org`. `readme` holds that parsed + * frontmatter. + */ +interface OkfBundle { + slug: string; + path: string; + readme: Record; +} + +/** One base-field mapping: read `frontmatterPath` from `sourceFile`, write to `baseFieldShortCode`. */ +interface FieldMapEntry { + sourceFile: string; + frontmatterPath: string; + baseFieldShortCode: string; +} + +/** A resolved value ready to be posted as a PDC field, with a provenance note. */ +interface MappedField { + baseFieldShortCode: string; + value: string; + sourceNote: string; +} + +/** Inventory summary for a single bundle (produced by the `inventory` subcommand). */ +interface BundleSummary { + slug: string; + path: string; + ein: string | null; + goodAsOf: string | null; + organizationFields: MappedField[]; + proposalFields: MappedField[]; +} + +/** + * Organization-scoped OKF frontmatter → PDC base field short code. These are + * written as ChangemakerFieldValues, exactly like the Charity Navigator and + * GivingTuesday integrations. The mapping mirrors the original TechSoup OKF + * exporter (see docs/mappings/techsoup-okf-field-gap-analysis.verbose.md §2). + */ +const organizationFieldMap: FieldMapEntry[] = [ + { sourceFile: 'README.md', frontmatterPath: 'title', baseFieldShortCode: 'organization_name' }, + { sourceFile: 'README.md', frontmatterPath: 'description', baseFieldShortCode: 'organization_overview' }, + { sourceFile: 'README.md', frontmatterPath: 'aliases', baseFieldShortCode: 'organization_dba_name' }, + // NOTE: the OKF `status` is a *document* lifecycle value (stable/draft), not an + // organizational status; the gap analysis (§5.1) flags mapping it to + // `organization_status` as a data-quality bug. It is retained here to match the + // existing exporter output — revisit once the org status source is settled. + { sourceFile: 'README.md', frontmatterPath: 'status', baseFieldShortCode: 'organization_status' }, + { + sourceFile: 'README.md', + frontmatterPath: 'x-civic.registration_country', + baseFieldShortCode: 'organization_country', + }, + { sourceFile: 'README.md', frontmatterPath: 'x-civic.registration.id', baseFieldShortCode: 'organization_tax_id' }, + { sourceFile: 'README.md', frontmatterPath: 'x-civic.ntee', baseFieldShortCode: 'organization_ntee_code' }, + { sourceFile: 'README.md', frontmatterPath: 'x-civic.situation', baseFieldShortCode: 'organization_geographic_area' }, + { sourceFile: 'population.md', frontmatterPath: 'description', baseFieldShortCode: 'community_definition_overview' }, +]; + +/** + * Proposal-scoped OKF frontmatter → PDC base field short code. These are + * COLLECTED and reported but NOT written yet: PDC proposal data requires an + * Opportunity + ApplicationForm + ProposalVersion scaffold that is out of scope + * for this pass (see docs/techsoup.md, "Proposals"). The variable-named + * volunteer-request file contributes `proposal_learning_and_evalution` and is + * resolved dynamically in `collectProposalFields`. + */ +const proposalFieldMap: FieldMapEntry[] = [ + { sourceFile: 'impact.md', frontmatterPath: 'description', baseFieldShortCode: 'proposal_project_outcomes' }, + { sourceFile: 'programs.md', frontmatterPath: 'description', baseFieldShortCode: 'proposal_related_programs' }, + { sourceFile: 'what_i_need_funding_for.md', frontmatterPath: 'title', baseFieldShortCode: 'proposal_name' }, + { + sourceFile: 'what_i_need_funding_for.md', + frontmatterPath: 'description', + baseFieldShortCode: 'proposal_funding_summary', + }, +]; + +// --------------------------------------------------------------------------- +// OKF (YAML) frontmatter reading +// --------------------------------------------------------------------------- + +const stripBom = (text: string): string => (text.charCodeAt(0) === BYTE_ORDER_MARK ? text.slice(1) : text); + +/** + * Parse a markdown document's YAML frontmatter into an object (empty object if + * none, malformed, or not a mapping). Uses `JSON_SCHEMA` so that timestamp-like + * scalars (e.g. `generated.at`) stay strings rather than being coerced to Date. + */ +const parseFrontmatter = (content: string): Record => { + const body = FRONTMATTER_PATTERN.exec(stripBom(content))?.groups?.body; + if (body === undefined) { + return {}; + } + const parsed = ((): unknown => { + try { + return loadYaml(body, { schema: JSON_SCHEMA }); + } catch (error: unknown) { + logger.debug({ error }, 'Failed to parse OKF frontmatter YAML'); + return null; + } + })(); + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { + return {}; + } + /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- js-yaml load returns unknown; a top-level YAML mapping is a string-keyed record */ + return parsed as Record; +}; + +/** Read a markdown file and return its parsed frontmatter, or null if unreadable/missing. */ +const parseFrontmatterFile = async (filePath: string): Promise | null> => { + try { + const content = await readFile(filePath, 'utf8'); + return parseFrontmatter(content); + } catch (error: unknown) { + logger.debug({ error, filePath }, 'Could not read frontmatter file'); + return null; + } +}; + +/** Follow a dotted path (e.g. `x-civic.registration.id`) into a parsed frontmatter object. */ +const getFrontmatterValue = (frontmatter: Record, dottedPath: string): unknown => { + let current: unknown = frontmatter; + for (const key of dottedPath.split('.')) { + if (current === null || typeof current !== 'object' || Array.isArray(current)) { + return undefined; + } + /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- narrowing a parsed-YAML node to a record to index it by key */ + const record = current as Record; + if (!(key in record)) { + return undefined; + } + const { [key]: next } = record; + current = next; + } + return current; +}; + +/** Coerce a parsed frontmatter value to a single trimmed PDC field string (arrays joined), or null. */ +const toFieldValueString = (value: unknown): string | null => { + if (value === null || value === undefined) { + return null; + } + if (typeof value === 'string') { + const trimmed = value.trim(); + return trimmed === '' ? null : trimmed; + } + if (typeof value === 'number' || typeof value === 'boolean') { + return String(value); + } + if (Array.isArray(value)) { + const parts = value.map((item) => toFieldValueString(item)).filter((item): item is string => item !== null); + return parts.length === 0 ? null : parts.join(ARRAY_VALUE_SEPARATOR); + } + return null; +}; + +/** + * Extract the organization's EIN from README frontmatter. Only returns a value + * when the registration scheme is IRS-EIN (or absent) and the id is a valid EIN; + * non-US bundles (KRS, PBO, ...) therefore yield null and are skipped downstream. + */ +const extractEin = (readmeFrontmatter: Record): string | null => { + const id = toFieldValueString(getFrontmatterValue(readmeFrontmatter, 'x-civic.registration.id')); + if (id === null) { + return null; + } + const scheme = toFieldValueString(getFrontmatterValue(readmeFrontmatter, 'x-civic.registration.scheme')); + if (scheme !== null && scheme.toUpperCase() !== 'IRS-EIN') { + return null; + } + return isValidEin(id) ? id : null; +}; + +/** Derive a `goodAsOf` ISO date (YYYY-MM-DD) from the bundle's `generated.at`, or null. */ +const extractGoodAsOf = (readmeFrontmatter: Record): string | null => { + const at = toFieldValueString(getFrontmatterValue(readmeFrontmatter, 'generated.at')); + if (at === null) { + return null; + } + const match = /^(?\d{4}-\d{2}-\d{2})/v.exec(at); + return match?.groups?.date ?? null; +}; + +// --------------------------------------------------------------------------- +// Source (zip / folder) inventory +// --------------------------------------------------------------------------- + +/** Extract a .zip into `destDir` using adm-zip (cross-platform, no shell dependency). */ +const extractZip = async (zipPath: string, destDir: string): Promise => { + const zip = new AdmZip(zipPath); + await zip.extractAllToAsync(destDir, true, false); +}; + +/** + * Resolve a `--source` argument to a directory tree to walk. Accepts a single + * .zip file, a folder of extracted OKF bundles, or a folder containing .zip + * files (each is extracted). Returns the root plus a cleanup for any temp dir. + * Whether the source holds one OKF package or several is determined afterwards + * by `findOrgBundles`. + */ +const prepareSourceRoot = async (source: string): Promise<{ root: string; cleanup: () => Promise }> => { + const stats = await stat(source); + const noop = async (): Promise => { + /* nothing to clean up when the source directory is walked in place */ + }; + if (stats.isFile()) { + if (!source.toLowerCase().endsWith('.zip')) { + throw new Error(`Source file is not a .zip: ${source}`); + } + const temp = await mkdtemp(join(tmpdir(), 'techsoup-')); + logger.info(`Extracting ${source} to ${temp}`); + await extractZip(source, temp); + return { + root: temp, + cleanup: async () => { + await rm(temp, { recursive: true, force: true }); + }, + }; + } + if (!stats.isDirectory()) { + throw new Error(`Source is neither a .zip file nor a directory: ${source}`); + } + const entries = await readdir(source, { withFileTypes: true }); + const zips = entries.filter((entry) => entry.isFile() && entry.name.toLowerCase().endsWith('.zip')); + if (zips.length === 0) { + return { root: source, cleanup: noop }; + } + const temp = await mkdtemp(join(tmpdir(), 'techsoup-')); + logger.info(`Found ${zips.length} zip file(s) in ${source}; extracting to ${temp}`); + for (const zip of zips) { + // eslint-disable-next-line no-await-in-loop -- extract archives sequentially to bound disk/CPU pressure + await extractZip(join(source, zip.name), join(temp, zip.name.replace(/\.zip$/iv, ''))); + } + return { + root: temp, + cleanup: async () => { + await rm(temp, { recursive: true, force: true }); + }, + }; +}; + +/** + * Walk `root` and return every OKF organization package (a directory whose + * README.md frontmatter declares `type: org`). This is what determines whether + * the source held a single package or many: a `root` that is itself a bundle + * yields one; otherwise every nested bundle is collected. Bundle subdirectories + * are not descended into. Results are sorted by slug for stable output. + */ +const findOrgBundles = async (root: string): Promise => { + const bundles: OkfBundle[] = []; + const walk = async (dir: string): Promise => { + const entries = await readdir(dir, { withFileTypes: true }); + const readmeEntry = entries.find((entry) => entry.isFile() && entry.name.toLowerCase() === 'readme.md'); + if (readmeEntry !== undefined) { + const frontmatter = await parseFrontmatterFile(join(dir, readmeEntry.name)); + if (frontmatter !== null && toFieldValueString(frontmatter.type) === 'org') { + bundles.push({ slug: basename(dir), path: dir, readme: frontmatter }); + return; + } + } + for (const entry of entries) { + if (entry.isDirectory()) { + // eslint-disable-next-line no-await-in-loop -- sequential recursion bounds open file handles + await walk(join(dir, entry.name)); + } + } + }; + await walk(root); + return bundles.sort((a, b) => a.slug.localeCompare(b.slug)); +}; + +/** Report whether the source resolved to a single OKF package or several (all processed one at a time). */ +const describeBundleCount = (bundles: OkfBundle[], source: string): void => { + if (bundles.length === 0) { + logger.warn(`No OKF organization packages found under ${source}`); + } else if (bundles.length === 1) { + logger.info(`Detected a single OKF package under ${source}: ${bundles[0]?.slug ?? '(unknown)'}`); + } else { + logger.info(`Detected ${bundles.length} OKF packages under ${source}; processing one at a time`); + } +}; + +/** Read each mapping's source file frontmatter and resolve the mapped, non-empty values. */ +const collectFieldsFromMap = async (bundlePath: string, fieldMap: FieldMapEntry[]): Promise => { + const cache = new Map | null>(); + const result: MappedField[] = []; + /* eslint-disable no-await-in-loop -- files are read on demand and memoized in `cache` */ + for (const entry of fieldMap) { + let frontmatter = cache.get(entry.sourceFile); + if (frontmatter === undefined) { + frontmatter = await parseFrontmatterFile(join(bundlePath, entry.sourceFile)); + cache.set(entry.sourceFile, frontmatter); + } + if (frontmatter === null) { + continue; + } + const value = toFieldValueString(getFrontmatterValue(frontmatter, entry.frontmatterPath)); + if (value === null) { + continue; + } + result.push({ + baseFieldShortCode: entry.baseFieldShortCode, + value, + sourceNote: `${basename(bundlePath)}/${entry.sourceFile}#${entry.frontmatterPath}`, + }); + } + /* eslint-enable no-await-in-loop */ + return result; +}; + +/** Locate the (variably named) volunteer-request markdown file within a bundle, relative to it. */ +const resolveVolunteerRequestFile = async (bundlePath: string): Promise => { + const dir = join(bundlePath, 'technical-volunteers'); + const entries = await readdir(dir, { withFileTypes: true }).catch(() => null); + if (entries === null) { + return null; + } + /* eslint-disable no-await-in-loop -- scan candidate files sequentially until the request file is found */ + for (const entry of entries) { + if ( + !entry.isFile() || + !entry.name.endsWith('.md') || + entry.name === 'index.md' || + entry.name === 'constraints.md' + ) { + continue; + } + const frontmatter = await parseFrontmatterFile(join(dir, entry.name)); + if (frontmatter !== null && toFieldValueString(frontmatter.type) === 'volunteer-request') { + // Forward slash (not path.join) so the provenance note matches the OKF + // `file#anchor` convention; join() below accepts it on all platforms. + return `technical-volunteers/${entry.name}`; + } + } + /* eslint-enable no-await-in-loop */ + return null; +}; + +/** Collect proposal-scoped fields (fixed files plus the dynamic volunteer-request file). */ +const collectProposalFields = async (bundlePath: string): Promise => { + const fields = await collectFieldsFromMap(bundlePath, proposalFieldMap); + const volunteerFile = await resolveVolunteerRequestFile(bundlePath); + if (volunteerFile !== null) { + const frontmatter = await parseFrontmatterFile(join(bundlePath, volunteerFile)); + if (frontmatter !== null) { + const value = toFieldValueString(getFrontmatterValue(frontmatter, 'description')); + if (value !== null) { + fields.push({ + baseFieldShortCode: 'proposal_learning_and_evalution', + value, + sourceNote: `${basename(bundlePath)}/${volunteerFile}#description`, + }); + } + } + } + return fields; +}; + +const summarizeBundles = async (bundles: OkfBundle[]): Promise => { + const summaries: BundleSummary[] = []; + /* eslint-disable no-await-in-loop -- bundles are summarized sequentially to bound concurrent file reads */ + for (const bundle of bundles) { + const organizationFields = await collectFieldsFromMap(bundle.path, organizationFieldMap); + const proposalFields = await collectProposalFields(bundle.path); + summaries.push({ + slug: bundle.slug, + path: bundle.path, + ein: extractEin(bundle.readme), + goodAsOf: extractGoodAsOf(bundle.readme), + organizationFields, + proposalFields, + }); + } + /* eslint-enable no-await-in-loop */ + return summaries; +}; + +export { + type BundleSummary, + type MappedField, + type OkfBundle, + collectFieldsFromMap, + collectProposalFields, + describeBundleCount, + extractEin, + extractGoodAsOf, + findOrgBundles, + getFrontmatterValue, + organizationFieldMap, + parseFrontmatter, + prepareSourceRoot, + summarizeBundles, + toFieldValueString, +}; diff --git a/src/techsoup.ts b/src/techsoup.ts new file mode 100644 index 0000000..e6d8d71 --- /dev/null +++ b/src/techsoup.ts @@ -0,0 +1,387 @@ +import { writeFile } from 'node:fs/promises'; +import { AxiosError } from 'axios'; +import { logger } from './logger.js'; +import { + type BundleSummary, + type MappedField, + type OkfBundle, + collectFieldsFromMap, + collectProposalFields, + describeBundleCount, + extractEin, + extractGoodAsOf, + findOrgBundles, + organizationFieldMap, + prepareSourceRoot, + summarizeBundles, +} from './okf.js'; +import { type AccessTokenSet, getToken, oidcOptions } from './oidc.js'; +import { + getChangemakers, + getSources, + postChangemakerFieldValue, + postChangemakerFieldValueBatch, + postSource, + type WritableChangemakerFieldValue, +} from './pdc-api.js'; +import type { CommandModule } from 'yargs'; +import type { Changemaker, ChangemakerBundle, Source } from '@pdc/sdk'; + +const TS_SHORT_CODE = 'techsoup'; +const JSON_SPACES = 2; +// When `@pdc/http-status-codes` is ready (issues 18-20 solved), use it instead. +const HTTP_STATUS_FORBIDDEN = 403; + +const getChangemakerByEin = (ein: string, changemakers: ChangemakerBundle): Changemaker | null => { + const normalized = ein.replace('-', ''); + const matches = changemakers.entries.filter((c) => c.taxId.replace('-', '') === normalized); + if (matches.length > 1) { + logger.warn(`Found multiple changemakers with EIN ${ein}, not returning any.`); + return null; + } + if (matches.length < 1) { + logger.info(`Found no changemaker with EIN ${ein}`); + return null; + } + const [match] = matches; + if (match !== undefined) { + return match; + } + throw new Error('How could this have happened?'); +}; + +/** + * Return the changemaker bundle scoped to a single ID when `changemakerId` is + * supplied, or the full bundle otherwise. + */ +const selectChangemakers = (changemakers: ChangemakerBundle, changemakerId: number | undefined): ChangemakerBundle => { + if (changemakerId === undefined) { + return changemakers; + } + const entries = changemakers.entries.filter((c) => c.id === changemakerId); + return { ...changemakers, entries, total: entries.length }; +}; + +/** Light wrapper around `postChangemakerFieldValue` that logs a warning on HTTP 403. */ +const postChangemakerFieldValueWarnOnForbidden = async ( + baseUrl: string, + token: AccessTokenSet, + data: WritableChangemakerFieldValue, + warnedChangemakers: Set, // Mutated! This is for observation/logs, not control! +): Promise => { + try { + const fieldValue = await postChangemakerFieldValue(baseUrl, token, data); + logger.info(`Added changemaker field value: ${JSON.stringify(fieldValue)}`); + } catch (e: unknown) { + if (e instanceof AxiosError && e.status === HTTP_STATUS_FORBIDDEN) { + logger.warn(`No permission (403) to create ${JSON.stringify(data)}`); + warnedChangemakers.add(data.changemakerId); + } else { + throw e; + } + } +}; + +const getOrCreateSource = async (baseUrl: string, token: AccessTokenSet): Promise => { + const sources = await getSources(baseUrl, token); + const filteredSources = sources.entries.filter((s) => s.dataProviderShortCode === TS_SHORT_CODE); + if (filteredSources.length === 1 && filteredSources[0] !== undefined) { + return filteredSources[0]; + } + logger.warn('Have a `pdc-admin` create a source because only administrators may be able.'); + return await postSource(baseUrl, token, { + dataProviderShortCode: TS_SHORT_CODE, + label: 'TechSoup', + }); +}; + +/** + * Proposal fields are collected but not yet written. PDC proposal data requires + * an Opportunity + ApplicationForm + ProposalVersion scaffold that is out of + * scope for this pass; surface what would be written so it is visible. + */ +const warnProposalsStubbed = (slug: string, proposalFields: MappedField[]): void => { + if (proposalFields.length === 0) { + return; + } + // TODO(techsoup-proposals): write these as PDC proposal data (Opportunity + + // ApplicationForm + ProposalVersion, cf. postProposalVersions.ts). Until the + // proposal model is decided, proposal fields are reported but NOT written. + logger.warn( + { slug, proposalFields: proposalFields.map((f) => f.baseFieldShortCode) }, + `Collected ${proposalFields.length} proposal field(s) — NOT written (stubbed, pending proposal model)`, + ); +}; + +/** + * Print the full field-by-field mapping for one package to the console, showing + * exactly which changemaker field values WOULD be written and which proposal + * fields are collected-but-not-written. Reads only local files — nothing is sent + * to (or read from) the PDC. + */ +const logMappedFields = (summary: BundleSummary): void => { + logger.info( + { ein: summary.ein, goodAsOf: summary.goodAsOf }, + `${summary.slug} — mapping preview (nothing sent to PDC; changemaker match happens only at run time)`, + ); + for (const field of summary.organizationFields) { + logger.info( + ` WOULD POST changemakerFieldValue: ${field.baseFieldShortCode} = ${JSON.stringify(field.value)} (goodAsOf=${summary.goodAsOf ?? 'null'}) [${field.sourceNote}]`, + ); + } + for (const field of summary.proposalFields) { + logger.info( + ` proposal field (collected, NOT written): ${field.baseFieldShortCode} = ${JSON.stringify(field.value)} [${field.sourceNote}]`, + ); + } +}; + +// --------------------------------------------------------------------------- +// CLI command modules +// --------------------------------------------------------------------------- + +interface InventoryCommandArgs { + source: string; + outputFile?: string; + 'show-fields': boolean; +} + +interface LookupFromPdcCommandArgs { + source: string; + 'pdc-api-base-url': string; + outputFile?: string; +} + +interface UpdateAllCommandArgs { + source: string; + 'pdc-api-base-url': string; + 'changemaker-id'?: number; + 'dry-run': boolean; + 'oidc-base-url': string; + 'oidc-client-id': string; + 'oidc-client-secret': string; +} + +const SOURCE_DESCRIBE = + 'Path to a TechSoup OKF .zip file, a folder of extracted OKF packages, or a folder of .zip files'; + +const inventoryCommand: CommandModule = { + command: 'inventory', + describe: 'Examine a TechSoup OKF zip/folder and inventory the available packages and mapped fields', + builder: (y) => + y + .option('source', { describe: SOURCE_DESCRIBE, demandOption: true, normalize: true, type: 'string' }) + .option('output-file', { + alias: 'write', + describe: 'Write the inventory to the specified JSON file', + normalize: true, + type: 'string', + }) + .option('show-fields', { + alias: 'fields', + describe: 'Print the full field-by-field mapping that would be sent to PDC (offline; nothing is sent)', + demandOption: false, + default: false, + type: 'boolean', + }), + handler: async (args): Promise => { + const prepared = await prepareSourceRoot(args.source); + try { + const bundles = await findOrgBundles(prepared.root); + describeBundleCount(bundles, args.source); + const summaries = await summarizeBundles(bundles); + if (args.outputFile === undefined || args.outputFile === '') { + for (const summary of summaries) { + if (args.showFields) { + logMappedFields(summary); + } else { + logger.info( + { + ein: summary.ein, + goodAsOf: summary.goodAsOf, + organizationFields: summary.organizationFields.length, + proposalFields: summary.proposalFields.length, + }, + summary.slug, + ); + } + } + } else { + await writeFile(args.outputFile, JSON.stringify(summaries, null, JSON_SPACES)); + logger.info(`Wrote TechSoup inventory (${summaries.length} package(s)) to ${JSON.stringify(args.outputFile)}`); + } + } finally { + await prepared.cleanup(); + } + }, +}; + +const lookupFromPdcCommand: CommandModule = { + command: 'lookupFromPdc', + describe: 'Match TechSoup package EINs against PDC changemakers and report which are present (read-only)', + builder: (y) => + y + .option('source', { describe: SOURCE_DESCRIBE, demandOption: true, normalize: true, type: 'string' }) + .option('pdc-api-base-url', { describe: 'Location of PDC API', demandOption: true, type: 'string' }) + .option('output-file', { + alias: 'write', + describe: 'Write the match report to the specified JSON file', + normalize: true, + type: 'string', + }), + handler: async (args): Promise => { + const prepared = await prepareSourceRoot(args.source); + try { + const bundles = await findOrgBundles(prepared.root); + describeBundleCount(bundles, args.source); + const changemakers = await getChangemakers(args.pdcApiBaseUrl); + const rows = bundles.map((bundle) => { + const ein = extractEin(bundle.readme); + const changemaker = ein === null ? null : getChangemakerByEin(ein, changemakers); + return { slug: bundle.slug, ein, changemakerId: changemaker?.id ?? null, presentInPdc: changemaker !== null }; + }); + const { length: presentCount } = rows.filter((row) => row.presentInPdc); + if (args.outputFile === undefined || args.outputFile === '') { + logger.info({ total: rows.length, present: presentCount }, 'TechSoup packages matched against PDC'); + for (const row of rows) { + logger.info(row, row.slug); + } + } else { + await writeFile(args.outputFile, JSON.stringify(rows, null, JSON_SPACES)); + logger.info( + `Wrote TechSoup PDC match report (${rows.length} package(s)) to ${JSON.stringify(args.outputFile)}`, + ); + } + } finally { + await prepared.cleanup(); + } + }, +}; + +/** Dry-run of `updateAll`: log every intended write without contacting the PDC for auth or POSTs. */ +const runUpdateAllDryRun = async (bundles: OkfBundle[], changemakers: ChangemakerBundle): Promise => { + logger.info('[dry-run] No data will be written to the PDC.'); + /* eslint-disable no-await-in-loop -- packages are processed sequentially, mirroring the real write path */ + for (const bundle of bundles) { + const ein = extractEin(bundle.readme); + if (ein === null) { + logger.info(`[dry-run] Skipping ${bundle.slug}: no IRS-EIN in package`); + continue; + } + const changemaker = getChangemakerByEin(ein, changemakers); + if (changemaker === null) { + continue; + } + const goodAsOf = extractGoodAsOf(bundle.readme); + const organizationFields = await collectFieldsFromMap(bundle.path, organizationFieldMap); + for (const field of organizationFields) { + logger.info( + `[dry-run] would POST changemakerFieldValue changemakerId=${changemaker.id} baseFieldShortCode=${field.baseFieldShortCode} goodAsOf=${goodAsOf ?? 'null'} value=${JSON.stringify(field.value)}`, + ); + } + warnProposalsStubbed(bundle.slug, await collectProposalFields(bundle.path)); + } + /* eslint-enable no-await-in-loop */ +}; + +const updateAllCommand: CommandModule = { + command: 'updateAll', + describe: 'Load organization fields from a TechSoup OKF zip/folder into PDC for each matching changemaker', + builder: { + ...oidcOptions, + source: { describe: SOURCE_DESCRIBE, demandOption: true, normalize: true, type: 'string' }, + 'pdc-api-base-url': { describe: 'Location of PDC API', demandOption: true, type: 'string' }, + 'changemaker-id': { + describe: 'Only load the single PDC changemaker with this ID (default: all changemakers in the PDC)', + demandOption: false, + type: 'number', + }, + 'dry-run': { + describe: 'Report what would be written without contacting the PDC for authentication or writes', + demandOption: false, + default: false, + type: 'boolean', + }, + }, + handler: async (args): Promise => { + const prepared = await prepareSourceRoot(args.source); + try { + const bundles = await findOrgBundles(prepared.root); + describeBundleCount(bundles, args.source); + if (bundles.length === 0) { + return; + } + const allChangemakers = await getChangemakers(args.pdcApiBaseUrl); + const changemakers = selectChangemakers(allChangemakers, args.changemakerId); + if (changemakers.entries.length === 0) { + logger.warn({ changemakerId: args.changemakerId }, 'No matching changemakers found in PDC; nothing to load.'); + return; + } + + if (args.dryRun) { + await runUpdateAllDryRun(bundles, changemakers); + return; + } + + // Up to this point we didn't need PDC authentication. Now we do. + const token = await getToken(args.oidcBaseUrl, args.oidcClientId, args.oidcClientSecret); + const source = await getOrCreateSource(args.pdcApiBaseUrl, token); + logger.info(source, 'The PDC Source for TechSoup was found'); + const fieldBatch = await postChangemakerFieldValueBatch(args.pdcApiBaseUrl, token, { + sourceId: source.id, + notes: `data-scripts techsoup.ts execution ${Date.now()}`, + }); + const missingPermissionChangemakerIds = new Set(); + // Each OKF package is processed one at a time, and field values are POSTed + // one at a time (not Promise.all) because the PDC API times out under + // concurrent POSTs to /changemakerFieldValues. + /* eslint-disable no-await-in-loop -- sequential POSTs avoid PDC API connection saturation. */ + for (const bundle of bundles) { + const ein = extractEin(bundle.readme); + if (ein === null) { + logger.info(`Skipping ${bundle.slug}: no IRS-EIN in package`); + continue; + } + const changemaker = getChangemakerByEin(ein, changemakers); + if (changemaker === null) { + continue; + } + const goodAsOf = extractGoodAsOf(bundle.readme); + const organizationFields = await collectFieldsFromMap(bundle.path, organizationFieldMap); + warnProposalsStubbed(bundle.slug, await collectProposalFields(bundle.path)); + for (const field of organizationFields) { + await postChangemakerFieldValueWarnOnForbidden( + args.pdcApiBaseUrl, + token, + { + changemakerId: changemaker.id, + batchId: fieldBatch.id, + baseFieldShortCode: field.baseFieldShortCode, + value: field.value, + goodAsOf, + }, + missingPermissionChangemakerIds, + ); + } + } + /* eslint-enable no-await-in-loop */ + if (missingPermissionChangemakerIds.size > 0) { + logger.warn( + `No permission for at least one field in each of these changemakers (so not updated): ${JSON.stringify([...missingPermissionChangemakerIds])}`, + ); + } + } finally { + await prepared.cleanup(); + } + }, +}; + +const techsoup: CommandModule = { + command: 'techsoup', + describe: 'Load TechSoup OKF package data (from a zip or folder) into the PDC', + builder: (y) => y.command(inventoryCommand).command(lookupFromPdcCommand).command(updateAllCommand).demandCommand(1), + /* eslint-disable-next-line @typescript-eslint/no-empty-function -- yargs demandCommand handles routing to subcommands */ + handler: () => {}, +}; + +export { getChangemakerByEin, selectChangemakers, techsoup }; diff --git a/src/techsoup.unit.test.ts b/src/techsoup.unit.test.ts new file mode 100644 index 0000000..e3ba35f --- /dev/null +++ b/src/techsoup.unit.test.ts @@ -0,0 +1,225 @@ +import { describe, expect, it } from '@jest/globals'; +import { extractEin, extractGoodAsOf, getFrontmatterValue, parseFrontmatter, toFieldValueString } from './okf.js'; +import { getChangemakerByEin, selectChangemakers } from './techsoup.js'; +import type { Changemaker, ChangemakerBundle } from '@pdc/sdk'; + +// Minimal Changemaker fixture; the helpers under test only read `id`/`taxId`, +// but the SDK type requires the full shape, so fill the rest with inert defaults. +const makeChangemaker = (id: number, taxId: string): Changemaker => ({ + id, + taxId, + name: `Org ${id}`, + createdAt: '2024-01-01T00:00:00Z', + createdBy: 'test', + fiscalSponsors: [], + fields: [], +}); + +// A trimmed-down but structurally faithful OKF README frontmatter (nested block +// map, flow sequences, an inline flow map, a block sequence of mappings, and a +// trailing comment) to exercise the whole minimal parser. +const README_FRONTMATTER = `--- +type: org +title: "synthetic-Black Mountain Workforce Partnership" +description: "A fabricated post-coal workforce training organization." +aliases: ["synthetic-Black Mountain Workforce", "Black Mountain Workforce Partnership"] +synthetic: true +status: stable +generated: { by: claude-code/claude-opus-5, at: 2026-07-29T00:00:00Z } +sources: + - id: org-site + resource: "https://example.org" + last_modified: 2026-03-02 + - id: registry + resource: "simulated registry extract" + last_modified: 2026-01-15 +x-civic: + # ---- REQUIRED by civic/0.6. ---- + profile: civic/0.6 + registration_country: US + registration: + scheme: "IRS-EIN" + id: "00-1000008" + tax_status: "501(c)(3)" + ntee: ["J22"] # optional US-only layer + situation: US-KY-letcher + verifiable_by: [techsoup] +--- + +# Body content that must be ignored by the parser. +`; + +describe('parseFrontmatter', () => { + const fm = parseFrontmatter(README_FRONTMATTER); + + it('parses top-level scalars', () => { + expect(fm.type).toBe('org'); + expect(fm.title).toBe('synthetic-Black Mountain Workforce Partnership'); + expect(fm.status).toBe('stable'); + }); + + it('parses a flow sequence of quoted strings', () => { + expect(fm.aliases).toStrictEqual(['synthetic-Black Mountain Workforce', 'Black Mountain Workforce Partnership']); + }); + + it('parses an inline flow mapping', () => { + expect(fm.generated).toStrictEqual({ by: 'claude-code/claude-opus-5', at: '2026-07-29T00:00:00Z' }); + }); + + it('parses a nested block mapping under x-civic', () => { + expect(getFrontmatterValue(fm, 'x-civic.registration.scheme')).toBe('IRS-EIN'); + expect(getFrontmatterValue(fm, 'x-civic.registration.id')).toBe('00-1000008'); + expect(getFrontmatterValue(fm, 'x-civic.registration_country')).toBe('US'); + expect(getFrontmatterValue(fm, 'x-civic.situation')).toBe('US-KY-letcher'); + }); + + it('consumes a block sequence of mappings without disturbing later keys', () => { + expect(Array.isArray(fm.sources)).toBe(true); + expect(getFrontmatterValue(fm, 'x-civic.ntee')).toStrictEqual(['J22']); + expect(getFrontmatterValue(fm, 'x-civic.verifiable_by')).toStrictEqual(['techsoup']); + }); + + it('strips a trailing comment from a value', () => { + expect(getFrontmatterValue(fm, 'x-civic.ntee')).toStrictEqual(['J22']); + }); + + it('returns an empty object when there is no frontmatter', () => { + expect(parseFrontmatter('# Just a heading\n\nSome text.')).toStrictEqual({}); + }); +}); + +describe('getFrontmatterValue', () => { + const fm = { a: { b: { c: 'deep' } }, list: [1, 2] }; + + it('follows a dotted path', () => { + expect(getFrontmatterValue(fm, 'a.b.c')).toBe('deep'); + }); + + it('returns undefined for a missing key', () => { + expect(getFrontmatterValue(fm, 'a.b.x')).toBeUndefined(); + }); + + it('returns undefined when descending into a non-object', () => { + expect(getFrontmatterValue(fm, 'a.b.c.d')).toBeUndefined(); + }); + + it('returns undefined when descending into an array', () => { + expect(getFrontmatterValue(fm, 'list.0')).toBeUndefined(); + }); +}); + +describe('toFieldValueString', () => { + it('trims a string', () => { + expect(toFieldValueString(' hello ')).toBe('hello'); + }); + + it('returns null for an empty or whitespace-only string', () => { + expect(toFieldValueString(' ')).toBeNull(); + }); + + it('returns null for null and undefined', () => { + expect(toFieldValueString(null)).toBeNull(); + expect(toFieldValueString(undefined)).toBeNull(); + }); + + it('stringifies numbers and booleans', () => { + expect(toFieldValueString(42)).toBe('42'); + expect(toFieldValueString(true)).toBe('true'); + }); + + it('joins array values with a separator, dropping empties', () => { + expect(toFieldValueString(['a', '', 'b'])).toBe('a; b'); + }); + + it('returns null for an empty array', () => { + expect(toFieldValueString([])).toBeNull(); + }); + + it('returns null for a plain object', () => { + expect(toFieldValueString({ a: 1 })).toBeNull(); + }); +}); + +describe('extractEin', () => { + it('returns a valid EIN when the scheme is IRS-EIN', () => { + expect(extractEin({ 'x-civic': { registration: { scheme: 'IRS-EIN', id: '00-1000008' } } })).toBe('00-1000008'); + }); + + it('returns the EIN when no scheme is declared but the id is a valid EIN', () => { + expect(extractEin({ 'x-civic': { registration: { id: '12-3456789' } } })).toBe('12-3456789'); + }); + + it('returns null for a non-IRS scheme (e.g. Polish KRS)', () => { + expect(extractEin({ 'x-civic': { registration: { scheme: 'KRS', id: '0000000000' } } })).toBeNull(); + }); + + it('returns null when the id is not a valid EIN', () => { + expect(extractEin({ 'x-civic': { registration: { scheme: 'IRS-EIN', id: 'not-an-ein' } } })).toBeNull(); + }); + + it('returns null when there is no registration id', () => { + expect(extractEin({ 'x-civic': { registration: {} } })).toBeNull(); + }); +}); + +describe('extractGoodAsOf', () => { + it('extracts the date from a generated.at timestamp', () => { + expect(extractGoodAsOf({ generated: { at: '2026-07-29T00:00:00Z' } })).toBe('2026-07-29'); + }); + + it('returns null when generated.at is missing', () => { + expect(extractGoodAsOf({ generated: {} })).toBeNull(); + }); + + it('returns null when generated.at is not a parseable date', () => { + expect(extractGoodAsOf({ generated: { at: 'sometime' } })).toBeNull(); + }); +}); + +describe('getChangemakerByEin', () => { + const bundle: ChangemakerBundle = { + entries: [makeChangemaker(1, '00-1000008'), makeChangemaker(2, '223456789')], + total: 2, + }; + + it('matches ignoring a hyphen', () => { + expect(getChangemakerByEin('001000008', bundle)?.id).toBe(1); + expect(getChangemakerByEin('22-3456789', bundle)?.id).toBe(2); + }); + + it('returns null when there is no match', () => { + expect(getChangemakerByEin('99-9999999', bundle)).toBeNull(); + }); + + it('returns null when more than one changemaker matches', () => { + const ambiguous: ChangemakerBundle = { + entries: [makeChangemaker(1, '00-1000008'), makeChangemaker(2, '001000008')], + total: 2, + }; + expect(getChangemakerByEin('00-1000008', ambiguous)).toBeNull(); + }); +}); + +describe('selectChangemakers', () => { + const bundle: ChangemakerBundle = { + entries: [makeChangemaker(1, '11-1111111'), makeChangemaker(2, '22-2222222')], + total: 2, + }; + + it('returns the full bundle unchanged when no ID is supplied', () => { + expect(selectChangemakers(bundle, undefined)).toBe(bundle); + }); + + it('scopes to the single matching changemaker and updates total', () => { + const result = selectChangemakers(bundle, 2); + expect(result.entries).toHaveLength(1); + expect(result.entries[0]?.id).toBe(2); + expect(result.total).toBe(1); + }); + + it('returns an empty bundle with total 0 for an unknown ID', () => { + const result = selectChangemakers(bundle, 999); + expect(result.entries).toStrictEqual([]); + expect(result.total).toBe(0); + }); +}); From 49df47d40e381cb1e72743f6ad970c9753deb8c6 Mon Sep 17 00:00:00 2001 From: Grant Minor Date: Thu, 24 Sep 2026 11:19:23 -0500 Subject: [PATCH 2/4] Changing functionality of TechSoup to allow creation of new ChangeMakers/Orgs --- docs/techsoup.md | 33 ++-- src/changemakers.ts | 354 ++++++++++++++++++++++++++++++++++++++ src/okf.ts | 5 + src/pdc-api.ts | 28 ++- src/techsoup.ts | 201 +++------------------- src/techsoup.unit.test.ts | 52 +++++- 6 files changed, 483 insertions(+), 190 deletions(-) create mode 100644 src/changemakers.ts diff --git a/docs/techsoup.md b/docs/techsoup.md index 08db3a8..f576db4 100644 --- a/docs/techsoup.md +++ b/docs/techsoup.md @@ -88,8 +88,11 @@ Open a ChangemakerFieldValueBatch (sourceId + notes with timestamp) For each OKF package (one at a time): extract EIN from README frontmatter (x-civic.registration.id, IRS-EIN only) └─ no valid EIN ──► skip package - match EIN to a PDC changemaker (getChangemakerByEin) - └─ no unique match ──► skip package + resolve EIN against PDC changemakers (resolveChangemaker) + ├─ exactly one match ──► use it + ├─ >1 match (ambiguous) ──► skip package (warn) + └─ no match ──► --add-changemakers ? POST /changemakers (taxId=EIN, name=README title) + : skip package derive goodAsOf from README generated.at collect organization fields (organizationFieldMap) and proposal fields └─ proposal fields are reported but NOT written (stubbed — see §4.2) @@ -105,7 +108,8 @@ Clean up any temp extraction dir - **Package detection (1 vs many):** `findOrgBundles` walks the resolved root and treats every directory whose `README.md` frontmatter declares `type: org` as one package (it does not descend into a package once found). If the root _itself_ is a package, exactly one is returned; otherwise every nested package is collected. `describeBundleCount` logs which case occurred. Either way, packages are processed **one at a time**. - **EIN extraction (`extractEin`):** reads `x-civic.registration.id` and only accepts it when `x-civic.registration.scheme` is `IRS-EIN` (or absent) **and** the id passes `isValidEin` (`^\d{2}-?\d{7}$`). Non-US bundles (Polish `KRS`, Kenyan PBO, …) therefore yield no EIN and are skipped — they cannot match a US-EIN-keyed changemaker. -- **EIN → changemaker matching (`getChangemakerByEin`):** compares hyphen-stripped EINs and returns a changemaker only when **exactly one** matches. Zero → logged at info and skipped; more than one → warning and skipped (ambiguous, nothing written). This is the "if it doesn't exist in the PDC, skip it" rule. +- **EIN → changemaker matching (`resolveChangemaker`):** compares hyphen-stripped EINs and classifies the result as `matched` (exactly one), `ambiguous` (more than one — always skipped with a warning, never auto-created), or `missing` (none). By default `missing` is skipped ("if it doesn't exist in the PDC, skip it"); with `--add-changemakers` a `missing` package instead **creates** a changemaker (see below). +- **Creating changemakers (`--add-changemakers`, `createChangemakerForBundle`):** off by default. When on, a `missing` package is created via `POST /changemakers` with `taxId` = the package EIN (stored **verbatim, hyphenated**, e.g. `00-1000008`) and `name` = the README `title`. A package with **no README title is skipped** (a changemaker needs a name), and a **403** on create is treated like other privileged writes (warned and skipped). Newly created changemakers are tracked in-memory so a second package with the same EIN in the same run reuses the new record rather than creating a duplicate. The flag is **ignored when `--changemaker-id` is set** (that scopes the run to one existing changemaker), and it does not apply to `ambiguous` matches. - **`goodAsOf` derivation (`extractGoodAsOf`):** takes the date portion (`YYYY-MM-DD`) of the README's `generated.at`, or `null` if missing/unparseable (`goodAsOf` is nullable in PDC). - **Value coercion (`toFieldValueString`):** trims strings (dropping empties), stringifies numbers/booleans, and joins arrays (e.g. `aliases`, `ntee`) with `"; "`. Objects are skipped. - **Sequential writes:** field values are POSTed one at a time (not `Promise.all`) because the PDC API times out under concurrent POSTs to `/changemakerFieldValues`. @@ -184,8 +188,13 @@ data-scripts techsoup inventory --source C:\Data\TechSoup\organizations --show-f ### `updateAll` -- **Args:** `--source` (required), `--pdc-api-base-url` (required), all `oidcOptions` (`--oidc-base-url`, `--oidc-client-id`, `--oidc-client-secret`), `--changemaker-id` (optional; scope to one changemaker), `--dry-run` (optional boolean, default `false`). -- **Behavior:** The full sync described in §3. The only subcommand that writes to PDC. `--dry-run` performs discovery and mapping and logs each intended `POST` without authenticating or writing. +- **Args:** `--source` (required), `--pdc-api-base-url` (required), all `oidcOptions` (`--oidc-base-url`, `--oidc-client-id`, `--oidc-client-secret`), `--changemaker-id` (optional; scope to one changemaker), `--dry-run` (optional boolean, default `false`), `--add-changemakers` (optional boolean, default `false`). +- **Behavior:** The full sync described in §3. The only subcommand that writes to PDC. `--dry-run` performs discovery and mapping and logs each intended `POST` (including `would CREATE changemaker …` lines when `--add-changemakers` is set) without authenticating or writing. +- **`--add-changemakers`:** When a package's EIN has **no** match in the PDC, create a new changemaker for it (`taxId` = EIN, `name` = README `title`) instead of skipping it. Requires a valid IRS-EIN and a README title; ambiguous matches are never auto-created; ignored when `--changemaker-id` is set. See §3 "Creating changemakers". + +```bash +data-scripts techsoup updateAll --source C:\Data\TechSoup\organizations --add-changemakers --dry-run --pdc-api-base-url --oidc-base-url --oidc-client-id --oidc-client-secret +``` --- @@ -211,13 +220,15 @@ The **number of OKF packages** is then a property of the tree, determined by `fi | Unreadable / missing markdown file | `parseFrontmatterFile` logs at debug and returns `null`; that field is skipped | | Malformed YAML frontmatter | Caught; treated as empty frontmatter (package may then be skipped) | | No frontmatter at all | Treated as an empty object | -| Non-US / non-EIN registration | `extractEin` returns `null`; package skipped | -| EIN not present in PDC | `getChangemakerByEin` returns `null`; package skipped | -| Ambiguous EIN → changemaker (>1 match) | Skipped with a warning; nothing written | +| Non-US / non-EIN registration | `extractEin` returns `null`; package skipped (never created, even with `--add-changemakers`) | +| EIN not present in PDC | Default: package skipped. With `--add-changemakers`: a changemaker is created (see below) | +| Ambiguous EIN → changemaker (>1 match) | Skipped with a warning; never auto-created | +| Create requested but no README title | Skipped with a warning (`--add-changemakers` needs a name) | +| HTTP 403 on changemaker create | Warned; that package skipped; run continues | | Null / empty / object-valued attributes | Skipped (not posted) | | Multi-value attributes (arrays) | Flattened to a `"; "`-joined string | | Missing / unparseable `generated.at` | `goodAsOf` set to `null` | -| HTTP 403 on write | Warned + changemakerId recorded; run continues; summary warning at end | +| HTTP 403 on field-value write | Warned + changemakerId recorded; run continues; summary warning at end | | PDC concurrency timeouts | Field values POSTed sequentially | | Missing TechSoup source | Attempt to create (usually admin-only); warns it may fail | | Proposal fields | Collected and reported, **not written** (stubbed — §4.2) | @@ -234,10 +245,12 @@ Unit tests in [`../src/techsoup.unit.test.ts`](../src/techsoup.unit.test.ts) cov - **`toFieldValueString`** — trims strings, returns `null` for empty/whitespace/`null`/`undefined`/empty-array/plain-object, stringifies numbers/booleans, and joins arrays with `"; "` (dropping empties). - **`extractEin`** — returns a valid EIN for `IRS-EIN` (or scheme-absent) registrations, and `null` for a non-IRS scheme (e.g. Polish KRS), an invalid id, or a missing id. - **`extractGoodAsOf`** — extracts `YYYY-MM-DD` from `generated.at`; returns `null` when missing or unparseable. +- **`extractOrganizationName`** — returns the README `title` (the name used to create a changemaker); `null` when absent or blank. - **`getChangemakerByEin`** — matches ignoring hyphens; returns `null` when there is no match or when more than one changemaker matches. +- **`resolveChangemaker`** — classifies an EIN as `matched` (unique), `missing` (none), or `ambiguous` (>1); this is what drives the `--add-changemakers` decision. - **`selectChangemakers`** — returns the full bundle when no ID is given; scopes to a single changemaker and updates `total`; returns an empty bundle for an unknown ID. -The file-system and network-facing functions (`prepareSourceRoot`, `extractZip`, `findOrgBundles`, `collectFieldsFromMap`, the command handlers, and all PDC writes) are not directly unit-tested; testability is achieved by extracting the pure parsing/mapping/matching logic into the exported helpers above (the same approach as the GivingTuesday integration). +The parsing/OKF helpers live in [`../src/okf.ts`](../src/okf.ts) and the changemaker matching/write helpers in [`../src/changemakers.ts`](../src/changemakers.ts); [`../src/techsoup.ts`](../src/techsoup.ts) is the CLI wiring. The file-system and network-facing functions (`prepareSourceRoot`, `extractZip`, `findOrgBundles`, `collectFieldsFromMap`, `createChangemakerForBundle`, `runUpdateAllWrite`, the command handlers, and all PDC writes) are not directly unit-tested; testability is achieved by extracting the pure parsing/mapping/matching logic into the exported helpers above (the same approach as the GivingTuesday integration). --- diff --git a/src/changemakers.ts b/src/changemakers.ts new file mode 100644 index 0000000..e350ee8 --- /dev/null +++ b/src/changemakers.ts @@ -0,0 +1,354 @@ +import { AxiosError } from 'axios'; +import { logger } from './logger.js'; +import { + type MappedField, + type OkfBundle, + collectFieldsFromMap, + collectProposalFields, + extractEin, + extractGoodAsOf, + extractOrganizationName, + organizationFieldMap, +} from './okf.js'; +import { type AccessTokenSet, getToken } from './oidc.js'; +import { + getSources, + postChangemaker, + postChangemakerFieldValue, + postChangemakerFieldValueBatch, + postSource, + type WritableChangemakerFieldValue, +} from './pdc-api.js'; +import type { Changemaker, ChangemakerBundle, Source } from '@pdc/sdk'; + +const TS_SHORT_CODE = 'techsoup'; +// When `@pdc/http-status-codes` is ready (issues 18-20 solved), use it instead. +const HTTP_STATUS_FORBIDDEN = 403; + +// --------------------------------------------------------------------------- +// EIN → changemaker matching +// --------------------------------------------------------------------------- + +/** All changemakers whose (hyphen-insensitive) taxId equals the given EIN. */ +const findChangemakersByEin = (ein: string, changemakers: ChangemakerBundle): Changemaker[] => { + const normalized = ein.replace('-', ''); + return changemakers.entries.filter((c) => c.taxId.replace('-', '') === normalized); +}; + +/** + * The outcome of resolving an EIN against the PDC changemakers: a unique match + * (`matched`), several matches (`ambiguous` — never auto-created), or none + * (`missing` — a create candidate when `--add-changemakers` is set). + */ +type ChangemakerResolution = + { kind: 'matched'; changemaker: Changemaker } | { kind: 'ambiguous' } | { kind: 'missing' }; + +const resolveChangemaker = (ein: string, changemakers: ChangemakerBundle): ChangemakerResolution => { + const matches = findChangemakersByEin(ein, changemakers); + if (matches.length > 1) { + return { kind: 'ambiguous' }; + } + const [match] = matches; + if (match !== undefined) { + return { kind: 'matched', changemaker: match }; + } + return { kind: 'missing' }; +}; + +const getChangemakerByEin = (ein: string, changemakers: ChangemakerBundle): Changemaker | null => { + const resolution = resolveChangemaker(ein, changemakers); + if (resolution.kind === 'ambiguous') { + logger.warn(`Found multiple changemakers with EIN ${ein}, not returning any.`); + return null; + } + if (resolution.kind === 'missing') { + logger.info(`Found no changemaker with EIN ${ein}`); + return null; + } + return resolution.changemaker; +}; + +/** + * Return the changemaker bundle scoped to a single ID when `changemakerId` is + * supplied, or the full bundle otherwise. + */ +const selectChangemakers = (changemakers: ChangemakerBundle, changemakerId: number | undefined): ChangemakerBundle => { + if (changemakerId === undefined) { + return changemakers; + } + const entries = changemakers.entries.filter((c) => c.id === changemakerId); + return { ...changemakers, entries, total: entries.length }; +}; + +// --------------------------------------------------------------------------- +// PDC writes (source, changemaker creation, field values) +// --------------------------------------------------------------------------- + +/** Light wrapper around `postChangemakerFieldValue` that logs a warning on HTTP 403. */ +const postChangemakerFieldValueWarnOnForbidden = async ( + baseUrl: string, + token: AccessTokenSet, + data: WritableChangemakerFieldValue, + warnedChangemakers: Set, // Mutated! This is for observation/logs, not control! +): Promise => { + try { + const fieldValue = await postChangemakerFieldValue(baseUrl, token, data); + logger.info(`Added changemaker field value: ${JSON.stringify(fieldValue)}`); + } catch (e: unknown) { + if (e instanceof AxiosError && e.status === HTTP_STATUS_FORBIDDEN) { + logger.warn(`No permission (403) to create ${JSON.stringify(data)}`); + warnedChangemakers.add(data.changemakerId); + } else { + throw e; + } + } +}; + +const getOrCreateSource = async (baseUrl: string, token: AccessTokenSet): Promise => { + const sources = await getSources(baseUrl, token); + const filteredSources = sources.entries.filter((s) => s.dataProviderShortCode === TS_SHORT_CODE); + if (filteredSources.length === 1 && filteredSources[0] !== undefined) { + return filteredSources[0]; + } + logger.warn('Have a `pdc-admin` create a source because only administrators may be able.'); + return await postSource(baseUrl, token, { + dataProviderShortCode: TS_SHORT_CODE, + label: 'TechSoup', + }); +}; + +/** + * Create a new PDC changemaker for a package whose EIN is not yet in the PDC. + * Requires an organization name (README `title`); the EIN is stored verbatim + * (hyphenated) as the taxId. Returns null (with a warning) when the name is + * missing or the create is forbidden (403 — creating changemakers is privileged). + */ +const createChangemakerForBundle = async ( + baseUrl: string, + token: AccessTokenSet, + ein: string, + bundle: OkfBundle, +): Promise => { + const name = extractOrganizationName(bundle.readme); + if (name === null) { + logger.warn(`Cannot create a changemaker for ${bundle.slug}: no organization name (README title). Skipping.`); + return null; + } + try { + const changemaker = await postChangemaker(baseUrl, token, { taxId: ein, name }); + logger.info({ id: changemaker.id, taxId: ein, name }, `Created changemaker for ${bundle.slug}`); + return changemaker; + } catch (e: unknown) { + if (e instanceof AxiosError && e.status === HTTP_STATUS_FORBIDDEN) { + logger.warn(`No permission (403) to create a changemaker for ${bundle.slug} (taxId ${ein}). Skipping.`); + return null; + } + throw e; + } +}; + +/** + * Proposal fields are collected but not yet written. PDC proposal data requires + * an Opportunity + ApplicationForm + ProposalVersion scaffold that is out of + * scope for this pass; surface what would be written so it is visible. + */ +const warnProposalsStubbed = (slug: string, proposalFields: MappedField[]): void => { + if (proposalFields.length === 0) { + return; + } + // TODO(techsoup-proposals): write these as PDC proposal data (Opportunity + + // ApplicationForm + ProposalVersion, cf. postProposalVersions.ts). Until the + // proposal model is decided, proposal fields are reported but NOT written. + logger.warn( + { slug, proposalFields: proposalFields.map((f) => f.baseFieldShortCode) }, + `Collected ${proposalFields.length} proposal field(s) — NOT written (stubbed, pending proposal model)`, + ); +}; + +// --------------------------------------------------------------------------- +// updateAll write path +// --------------------------------------------------------------------------- + +/** Everything one package's writes need, gathered once so per-package helpers stay small. */ +interface WriteRun { + baseUrl: string; + token: AccessTokenSet; + batchId: number; + changemakers: ChangemakerBundle; + addChangemakers: boolean; + warnedChangemakers: Set; +} + +/** Base URL + OIDC credentials needed to start a write run (before authentication). */ +interface UpdateAllWriteContext { + pdcApiBaseUrl: string; + oidcBaseUrl: string; + oidcClientId: string; + oidcClientSecret: string; +} + +/** + * Resolve the changemaker to load a package into, creating one when the EIN is + * missing and `--add-changemakers` is set. Returns null when the package should + * be skipped (ambiguous match, missing without the flag, or a missing-name / + * forbidden create) — the reason is logged. Mutates `run.changemakers.entries` + * when a new record is created so later packages with the same EIN reuse it. + */ +const resolveOrCreateChangemaker = async ( + run: WriteRun, + ein: string, + bundle: OkfBundle, +): Promise => { + const resolution = resolveChangemaker(ein, run.changemakers); + if (resolution.kind === 'matched') { + return resolution.changemaker; + } + if (resolution.kind === 'ambiguous') { + logger.warn(`Skipping ${bundle.slug}: EIN ${ein} matches multiple changemakers`); + return null; + } + if (!run.addChangemakers) { + logger.info(`Skipping ${bundle.slug}: EIN ${ein} not found in PDC`); + return null; + } + const created = await createChangemakerForBundle(run.baseUrl, run.token, ein, bundle); + if (created !== null) { + run.changemakers.entries.push(created); + } + return created; +}; + +/** POST every mapped organization field for one package's changemaker (proposals stay stubbed). */ +const postOrganizationFields = async (run: WriteRun, changemaker: Changemaker, bundle: OkfBundle): Promise => { + const goodAsOf = extractGoodAsOf(bundle.readme); + const organizationFields = await collectFieldsFromMap(bundle.path, organizationFieldMap); + warnProposalsStubbed(bundle.slug, await collectProposalFields(bundle.path)); + /* eslint-disable no-await-in-loop -- sequential POSTs avoid PDC API connection saturation. */ + for (const field of organizationFields) { + await postChangemakerFieldValueWarnOnForbidden( + run.baseUrl, + run.token, + { + changemakerId: changemaker.id, + batchId: run.batchId, + baseFieldShortCode: field.baseFieldShortCode, + value: field.value, + goodAsOf, + }, + run.warnedChangemakers, + ); + } + /* eslint-enable no-await-in-loop */ +}; + +/** The live (non-dry-run) `updateAll` path: authenticate, open a batch, and load each package. */ +const runUpdateAllWrite = async ( + bundles: OkfBundle[], + changemakers: ChangemakerBundle, + addChangemakers: boolean, + ctx: UpdateAllWriteContext, +): Promise => { + // Up to this point we didn't need PDC authentication. Now we do. + const token = await getToken(ctx.oidcBaseUrl, ctx.oidcClientId, ctx.oidcClientSecret); + const source = await getOrCreateSource(ctx.pdcApiBaseUrl, token); + logger.info(source, 'The PDC Source for TechSoup was found'); + const fieldBatch = await postChangemakerFieldValueBatch(ctx.pdcApiBaseUrl, token, { + sourceId: source.id, + notes: `data-scripts techsoup.ts execution ${Date.now()}`, + }); + const run: WriteRun = { + baseUrl: ctx.pdcApiBaseUrl, + token, + batchId: fieldBatch.id, + changemakers, + addChangemakers, + warnedChangemakers: new Set(), + }; + /* eslint-disable no-await-in-loop -- sequential per-package processing avoids PDC API saturation. */ + for (const bundle of bundles) { + const ein = extractEin(bundle.readme); + if (ein === null) { + logger.info(`Skipping ${bundle.slug}: no IRS-EIN in package`); + continue; + } + const changemaker = await resolveOrCreateChangemaker(run, ein, bundle); + if (changemaker === null) { + continue; + } + await postOrganizationFields(run, changemaker, bundle); + } + /* eslint-enable no-await-in-loop */ + if (run.warnedChangemakers.size > 0) { + logger.warn( + `No permission for at least one field in each of these changemakers (so not updated): ${JSON.stringify([...run.warnedChangemakers])}`, + ); + } +}; + +// --------------------------------------------------------------------------- +// updateAll dry-run path +// --------------------------------------------------------------------------- + +/** + * Resolve the label a dry-run should print for a package's changemaker, or null + * when the package would be skipped (logging the reason). `(new)` is returned + * when `--add-changemakers` would create one. + */ +const dryRunChangemakerLabel = ( + bundle: OkfBundle, + ein: string, + changemakers: ChangemakerBundle, + addChangemakers: boolean, +): string | null => { + const resolution = resolveChangemaker(ein, changemakers); + if (resolution.kind === 'ambiguous') { + logger.warn(`[dry-run] Skipping ${bundle.slug}: EIN ${ein} matches multiple changemakers`); + return null; + } + if (resolution.kind === 'matched') { + return String(resolution.changemaker.id); + } + if (!addChangemakers) { + logger.info(`[dry-run] Skipping ${bundle.slug}: EIN ${ein} not found in PDC`); + return null; + } + const name = extractOrganizationName(bundle.readme); + if (name === null) { + logger.warn(`[dry-run] Skipping ${bundle.slug}: would create a changemaker but README has no title (name)`); + return null; + } + logger.info(`[dry-run] would CREATE changemaker taxId=${ein} name=${JSON.stringify(name)}`); + return '(new)'; +}; + +/** Dry-run of `updateAll`: log every intended write without contacting the PDC for auth or POSTs. */ +const runUpdateAllDryRun = async ( + bundles: OkfBundle[], + changemakers: ChangemakerBundle, + addChangemakers: boolean, +): Promise => { + logger.info('[dry-run] No data will be written to the PDC.'); + /* eslint-disable no-await-in-loop -- packages are processed sequentially, mirroring the real write path */ + for (const bundle of bundles) { + const ein = extractEin(bundle.readme); + if (ein === null) { + logger.info(`[dry-run] Skipping ${bundle.slug}: no IRS-EIN in package`); + continue; + } + const changemakerLabel = dryRunChangemakerLabel(bundle, ein, changemakers, addChangemakers); + if (changemakerLabel === null) { + continue; + } + const goodAsOf = extractGoodAsOf(bundle.readme); + const organizationFields = await collectFieldsFromMap(bundle.path, organizationFieldMap); + for (const field of organizationFields) { + logger.info( + `[dry-run] would POST changemakerFieldValue changemakerId=${changemakerLabel} baseFieldShortCode=${field.baseFieldShortCode} goodAsOf=${goodAsOf ?? 'null'} value=${JSON.stringify(field.value)}`, + ); + } + warnProposalsStubbed(bundle.slug, await collectProposalFields(bundle.path)); + } + /* eslint-enable no-await-in-loop */ +}; + +export { getChangemakerByEin, resolveChangemaker, runUpdateAllDryRun, runUpdateAllWrite, selectChangemakers }; diff --git a/src/okf.ts b/src/okf.ts index 6dad970..6481621 100644 --- a/src/okf.ts +++ b/src/okf.ts @@ -193,6 +193,10 @@ const extractEin = (readmeFrontmatter: Record): string | null = return isValidEin(id) ? id : null; }; +/** The organization's display name (README `title`), used when creating a changemaker; null if absent. */ +const extractOrganizationName = (readmeFrontmatter: Record): string | null => + toFieldValueString(getFrontmatterValue(readmeFrontmatter, 'title')); + /** Derive a `goodAsOf` ISO date (YYYY-MM-DD) from the bundle's `generated.at`, or null. */ const extractGoodAsOf = (readmeFrontmatter: Record): string | null => { const at = toFieldValueString(getFrontmatterValue(readmeFrontmatter, 'generated.at')); @@ -406,6 +410,7 @@ export { describeBundleCount, extractEin, extractGoodAsOf, + extractOrganizationName, findOrgBundles, getFrontmatterValue, organizationFieldMap, diff --git a/src/pdc-api.ts b/src/pdc-api.ts index 3c83750..027a58c 100644 --- a/src/pdc-api.ts +++ b/src/pdc-api.ts @@ -1,6 +1,14 @@ import { client } from './client.js'; import type { AccessTokenSet } from './oidc.js'; -import type { BaseField, ProposalBundle, ChangemakerBundle, SourceBundle, Source, BaseFieldBundle } from '@pdc/sdk'; +import type { + BaseField, + Changemaker, + ProposalBundle, + ChangemakerBundle, + SourceBundle, + Source, + BaseFieldBundle, +} from '@pdc/sdk'; const callPdcApi = async ( baseUrl: string, @@ -71,6 +79,23 @@ export interface WritableSource { const postSource = async (baseUrl: string, token: AccessTokenSet, data: WritableSource): Promise => await callPdcApi(baseUrl, '/sources', 'post', { params: {}, token, data }); +/** + * The POST /changemakers body. The SDK's `Writable` also pulls in + * `fields`/`fiscalSponsors`, which the create endpoint does not accept; only + * `taxId` and `name` are required (`keycloakOrganizationId` is optional). + */ +export interface WritableChangemaker { + taxId: string; + name: string; + keycloakOrganizationId?: string; +} + +const postChangemaker = async ( + baseUrl: string, + token: AccessTokenSet, + data: WritableChangemaker, +): Promise => await callPdcApi(baseUrl, '/changemakers', 'post', { params: {}, token, data }); + // TODO: use the SDK, delete these temp types copied from the service repo interface ChangemakerFieldValueBatch { readonly id: number; @@ -138,6 +163,7 @@ export { getChangemakers, getProposals, getSources, + postChangemaker, postChangemakerFieldValueBatch, postChangemakerFieldValue, postPlatformProviderData, diff --git a/src/techsoup.ts b/src/techsoup.ts index e6d8d71..c035c0d 100644 --- a/src/techsoup.ts +++ b/src/techsoup.ts @@ -1,117 +1,19 @@ import { writeFile } from 'node:fs/promises'; -import { AxiosError } from 'axios'; +import { getChangemakerByEin, runUpdateAllDryRun, runUpdateAllWrite, selectChangemakers } from './changemakers.js'; import { logger } from './logger.js'; import { type BundleSummary, - type MappedField, - type OkfBundle, - collectFieldsFromMap, - collectProposalFields, describeBundleCount, extractEin, - extractGoodAsOf, findOrgBundles, - organizationFieldMap, prepareSourceRoot, summarizeBundles, } from './okf.js'; -import { type AccessTokenSet, getToken, oidcOptions } from './oidc.js'; -import { - getChangemakers, - getSources, - postChangemakerFieldValue, - postChangemakerFieldValueBatch, - postSource, - type WritableChangemakerFieldValue, -} from './pdc-api.js'; +import { oidcOptions } from './oidc.js'; +import { getChangemakers } from './pdc-api.js'; import type { CommandModule } from 'yargs'; -import type { Changemaker, ChangemakerBundle, Source } from '@pdc/sdk'; -const TS_SHORT_CODE = 'techsoup'; const JSON_SPACES = 2; -// When `@pdc/http-status-codes` is ready (issues 18-20 solved), use it instead. -const HTTP_STATUS_FORBIDDEN = 403; - -const getChangemakerByEin = (ein: string, changemakers: ChangemakerBundle): Changemaker | null => { - const normalized = ein.replace('-', ''); - const matches = changemakers.entries.filter((c) => c.taxId.replace('-', '') === normalized); - if (matches.length > 1) { - logger.warn(`Found multiple changemakers with EIN ${ein}, not returning any.`); - return null; - } - if (matches.length < 1) { - logger.info(`Found no changemaker with EIN ${ein}`); - return null; - } - const [match] = matches; - if (match !== undefined) { - return match; - } - throw new Error('How could this have happened?'); -}; - -/** - * Return the changemaker bundle scoped to a single ID when `changemakerId` is - * supplied, or the full bundle otherwise. - */ -const selectChangemakers = (changemakers: ChangemakerBundle, changemakerId: number | undefined): ChangemakerBundle => { - if (changemakerId === undefined) { - return changemakers; - } - const entries = changemakers.entries.filter((c) => c.id === changemakerId); - return { ...changemakers, entries, total: entries.length }; -}; - -/** Light wrapper around `postChangemakerFieldValue` that logs a warning on HTTP 403. */ -const postChangemakerFieldValueWarnOnForbidden = async ( - baseUrl: string, - token: AccessTokenSet, - data: WritableChangemakerFieldValue, - warnedChangemakers: Set, // Mutated! This is for observation/logs, not control! -): Promise => { - try { - const fieldValue = await postChangemakerFieldValue(baseUrl, token, data); - logger.info(`Added changemaker field value: ${JSON.stringify(fieldValue)}`); - } catch (e: unknown) { - if (e instanceof AxiosError && e.status === HTTP_STATUS_FORBIDDEN) { - logger.warn(`No permission (403) to create ${JSON.stringify(data)}`); - warnedChangemakers.add(data.changemakerId); - } else { - throw e; - } - } -}; - -const getOrCreateSource = async (baseUrl: string, token: AccessTokenSet): Promise => { - const sources = await getSources(baseUrl, token); - const filteredSources = sources.entries.filter((s) => s.dataProviderShortCode === TS_SHORT_CODE); - if (filteredSources.length === 1 && filteredSources[0] !== undefined) { - return filteredSources[0]; - } - logger.warn('Have a `pdc-admin` create a source because only administrators may be able.'); - return await postSource(baseUrl, token, { - dataProviderShortCode: TS_SHORT_CODE, - label: 'TechSoup', - }); -}; - -/** - * Proposal fields are collected but not yet written. PDC proposal data requires - * an Opportunity + ApplicationForm + ProposalVersion scaffold that is out of - * scope for this pass; surface what would be written so it is visible. - */ -const warnProposalsStubbed = (slug: string, proposalFields: MappedField[]): void => { - if (proposalFields.length === 0) { - return; - } - // TODO(techsoup-proposals): write these as PDC proposal data (Opportunity + - // ApplicationForm + ProposalVersion, cf. postProposalVersions.ts). Until the - // proposal model is decided, proposal fields are reported but NOT written. - logger.warn( - { slug, proposalFields: proposalFields.map((f) => f.baseFieldShortCode) }, - `Collected ${proposalFields.length} proposal field(s) — NOT written (stubbed, pending proposal model)`, - ); -}; /** * Print the full field-by-field mapping for one package to the console, showing @@ -157,6 +59,7 @@ interface UpdateAllCommandArgs { 'pdc-api-base-url': string; 'changemaker-id'?: number; 'dry-run': boolean; + 'add-changemakers': boolean; 'oidc-base-url': string; 'oidc-client-id': string; 'oidc-client-secret': string; @@ -258,32 +161,6 @@ const lookupFromPdcCommand: CommandModule = { }, }; -/** Dry-run of `updateAll`: log every intended write without contacting the PDC for auth or POSTs. */ -const runUpdateAllDryRun = async (bundles: OkfBundle[], changemakers: ChangemakerBundle): Promise => { - logger.info('[dry-run] No data will be written to the PDC.'); - /* eslint-disable no-await-in-loop -- packages are processed sequentially, mirroring the real write path */ - for (const bundle of bundles) { - const ein = extractEin(bundle.readme); - if (ein === null) { - logger.info(`[dry-run] Skipping ${bundle.slug}: no IRS-EIN in package`); - continue; - } - const changemaker = getChangemakerByEin(ein, changemakers); - if (changemaker === null) { - continue; - } - const goodAsOf = extractGoodAsOf(bundle.readme); - const organizationFields = await collectFieldsFromMap(bundle.path, organizationFieldMap); - for (const field of organizationFields) { - logger.info( - `[dry-run] would POST changemakerFieldValue changemakerId=${changemaker.id} baseFieldShortCode=${field.baseFieldShortCode} goodAsOf=${goodAsOf ?? 'null'} value=${JSON.stringify(field.value)}`, - ); - } - warnProposalsStubbed(bundle.slug, await collectProposalFields(bundle.path)); - } - /* eslint-enable no-await-in-loop */ -}; - const updateAllCommand: CommandModule = { command: 'updateAll', describe: 'Load organization fields from a TechSoup OKF zip/folder into PDC for each matching changemaker', @@ -302,6 +179,13 @@ const updateAllCommand: CommandModule = { default: false, type: 'boolean', }, + 'add-changemakers': { + describe: + 'Create a new changemaker when a package EIN has no match in the PDC (default: skip unmatched packages). Ignored when --changemaker-id is set.', + demandOption: false, + default: false, + type: 'boolean', + }, }, handler: async (args): Promise => { const prepared = await prepareSourceRoot(args.source); @@ -313,63 +197,26 @@ const updateAllCommand: CommandModule = { } const allChangemakers = await getChangemakers(args.pdcApiBaseUrl); const changemakers = selectChangemakers(allChangemakers, args.changemakerId); - if (changemakers.entries.length === 0) { + // --changemaker-id targets one existing changemaker, which is incompatible + // with creating missing ones; when both are given, honor the scope and + // disable creation (decision 3). + const addChangemakers = args.addChangemakers && args.changemakerId === undefined; + if (args.addChangemakers && args.changemakerId !== undefined) { + logger.warn('--add-changemakers is ignored because --changemaker-id targets a single existing changemaker.'); + } + // With --add-changemakers we can proceed even against an empty scope (we + // create as we go); otherwise there is nothing to match and we stop. + if (changemakers.entries.length === 0 && !addChangemakers) { logger.warn({ changemakerId: args.changemakerId }, 'No matching changemakers found in PDC; nothing to load.'); return; } if (args.dryRun) { - await runUpdateAllDryRun(bundles, changemakers); + await runUpdateAllDryRun(bundles, changemakers, addChangemakers); return; } - // Up to this point we didn't need PDC authentication. Now we do. - const token = await getToken(args.oidcBaseUrl, args.oidcClientId, args.oidcClientSecret); - const source = await getOrCreateSource(args.pdcApiBaseUrl, token); - logger.info(source, 'The PDC Source for TechSoup was found'); - const fieldBatch = await postChangemakerFieldValueBatch(args.pdcApiBaseUrl, token, { - sourceId: source.id, - notes: `data-scripts techsoup.ts execution ${Date.now()}`, - }); - const missingPermissionChangemakerIds = new Set(); - // Each OKF package is processed one at a time, and field values are POSTed - // one at a time (not Promise.all) because the PDC API times out under - // concurrent POSTs to /changemakerFieldValues. - /* eslint-disable no-await-in-loop -- sequential POSTs avoid PDC API connection saturation. */ - for (const bundle of bundles) { - const ein = extractEin(bundle.readme); - if (ein === null) { - logger.info(`Skipping ${bundle.slug}: no IRS-EIN in package`); - continue; - } - const changemaker = getChangemakerByEin(ein, changemakers); - if (changemaker === null) { - continue; - } - const goodAsOf = extractGoodAsOf(bundle.readme); - const organizationFields = await collectFieldsFromMap(bundle.path, organizationFieldMap); - warnProposalsStubbed(bundle.slug, await collectProposalFields(bundle.path)); - for (const field of organizationFields) { - await postChangemakerFieldValueWarnOnForbidden( - args.pdcApiBaseUrl, - token, - { - changemakerId: changemaker.id, - batchId: fieldBatch.id, - baseFieldShortCode: field.baseFieldShortCode, - value: field.value, - goodAsOf, - }, - missingPermissionChangemakerIds, - ); - } - } - /* eslint-enable no-await-in-loop */ - if (missingPermissionChangemakerIds.size > 0) { - logger.warn( - `No permission for at least one field in each of these changemakers (so not updated): ${JSON.stringify([...missingPermissionChangemakerIds])}`, - ); - } + await runUpdateAllWrite(bundles, changemakers, addChangemakers, args); } finally { await prepared.cleanup(); } @@ -384,4 +231,4 @@ const techsoup: CommandModule = { handler: () => {}, }; -export { getChangemakerByEin, selectChangemakers, techsoup }; +export { techsoup }; diff --git a/src/techsoup.unit.test.ts b/src/techsoup.unit.test.ts index e3ba35f..01f266a 100644 --- a/src/techsoup.unit.test.ts +++ b/src/techsoup.unit.test.ts @@ -1,6 +1,13 @@ import { describe, expect, it } from '@jest/globals'; -import { extractEin, extractGoodAsOf, getFrontmatterValue, parseFrontmatter, toFieldValueString } from './okf.js'; -import { getChangemakerByEin, selectChangemakers } from './techsoup.js'; +import { getChangemakerByEin, resolveChangemaker, selectChangemakers } from './changemakers.js'; +import { + extractEin, + extractGoodAsOf, + extractOrganizationName, + getFrontmatterValue, + parseFrontmatter, + toFieldValueString, +} from './okf.js'; import type { Changemaker, ChangemakerBundle } from '@pdc/sdk'; // Minimal Changemaker fixture; the helpers under test only read `id`/`taxId`, @@ -176,6 +183,47 @@ describe('extractGoodAsOf', () => { }); }); +describe('extractOrganizationName', () => { + it('returns the README title', () => { + expect(extractOrganizationName({ title: 'synthetic-Black Mountain Workforce Partnership' })).toBe( + 'synthetic-Black Mountain Workforce Partnership', + ); + }); + + it('returns null when the title is absent', () => { + expect(extractOrganizationName({ description: 'no title here' })).toBeNull(); + }); + + it('returns null when the title is blank', () => { + expect(extractOrganizationName({ title: ' ' })).toBeNull(); + }); +}); + +describe('resolveChangemaker', () => { + const bundle: ChangemakerBundle = { + entries: [makeChangemaker(1, '00-1000008'), makeChangemaker(2, '22-3456789')], + total: 2, + }; + + it('resolves a unique match (hyphen-insensitive)', () => { + const resolution = resolveChangemaker('001000008', bundle); + expect(resolution.kind).toBe('matched'); + expect(resolution.kind === 'matched' ? resolution.changemaker.id : null).toBe(1); + }); + + it('resolves to missing when there is no match', () => { + expect(resolveChangemaker('99-9999999', bundle).kind).toBe('missing'); + }); + + it('resolves to ambiguous when more than one changemaker matches', () => { + const ambiguous: ChangemakerBundle = { + entries: [makeChangemaker(1, '00-1000008'), makeChangemaker(2, '001000008')], + total: 2, + }; + expect(resolveChangemaker('00-1000008', ambiguous).kind).toBe('ambiguous'); + }); +}); + describe('getChangemakerByEin', () => { const bundle: ChangemakerBundle = { entries: [makeChangemaker(1, '00-1000008'), makeChangemaker(2, '223456789')], From a6c37821c07e73a92dbbe07f86718ccf27890561 Mon Sep 17 00:00:00 2001 From: Grant Minor Date: Thu, 12 Nov 2026 14:00:03 -0600 Subject: [PATCH 3/4] Updated techsoup import to focus on json summaries only --- src/fields.ts | 50 +++++++++++++++ src/pdc-json.ts | 167 ++++++++++++++++++++++++++++++++++++++++++++++++ src/source.ts | 106 ++++++++++++++++++++++++++++++ 3 files changed, 323 insertions(+) create mode 100644 src/fields.ts create mode 100644 src/pdc-json.ts create mode 100644 src/source.ts diff --git a/src/fields.ts b/src/fields.ts new file mode 100644 index 0000000..f98bc05 --- /dev/null +++ b/src/fields.ts @@ -0,0 +1,50 @@ +// Multi-valued source data (e.g. OKF `aliases`/`ntee`, or an array-valued field +// in a `_pdc.json`) is flattened into a single PDC string with this separator, +// since a ChangemakerFieldValue.value is a scalar string. +const ARRAY_VALUE_SEPARATOR = '; '; + +/** A resolved value ready to be posted as a PDC field, with a provenance note. */ +interface MappedField { + baseFieldShortCode: string; + value: string; + sourceNote: string; +} + +/** + * The source-agnostic unit of work: the normalized package that both the OKF + * bundle reader and the `_pdc.json` reader produce, and what the importer loads + * into the PDC. `ein`/`name`/`goodAsOf` may be null (e.g. `goodAsOf` is always + * null for `_pdc.json` sources, which carry no generation date). + */ +interface TechSoupPackage { + slug: string; + ein: string | null; + name: string | null; + goodAsOf: string | null; + organizationFields: MappedField[]; + proposalFields: MappedField[]; +} + +/** Coerce an arbitrary source value to a single trimmed PDC field string (arrays joined), or null. */ +const toFieldValueString = (value: unknown): string | null => { + if (value === null || value === undefined) { + return null; + } + if (typeof value === 'string') { + const trimmed = value.trim(); + return trimmed === '' ? null : trimmed; + } + if (typeof value === 'number' || typeof value === 'boolean') { + return String(value); + } + if (Array.isArray(value)) { + const parts = value.map((item) => toFieldValueString(item)).filter((item): item is string => item !== null); + return parts.length === 0 ? null : parts.join(ARRAY_VALUE_SEPARATOR); + } + return null; +}; + +/** Proposal-scoped base fields are those whose short code starts with `proposal_`. */ +const isProposalField = (baseFieldShortCode: string): boolean => baseFieldShortCode.startsWith('proposal_'); + +export { type MappedField, type TechSoupPackage, isProposalField, toFieldValueString }; diff --git a/src/pdc-json.ts b/src/pdc-json.ts new file mode 100644 index 0000000..47c8878 --- /dev/null +++ b/src/pdc-json.ts @@ -0,0 +1,167 @@ +import { readFile, readdir } from 'node:fs/promises'; +import { basename, join } from 'node:path'; +import { isValidEin } from './ein.js'; +import { type MappedField, type TechSoupPackage, isProposalField, toFieldValueString } from './fields.js'; +import { logger } from './logger.js'; + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === 'object' && !Array.isArray(value); + +/** The provenance note for a `_pdc.json` field: its `batch.notes`, or null. */ +const extractFieldNote = (field: Record): string | null => { + const { batch } = field; + if (isRecord(batch) && typeof batch.notes === 'string' && batch.notes.trim() !== '') { + return batch.notes; + } + return null; +}; + +/** The changemaker name carried by a `_pdc.json` document (`changemaker.name`), or null. */ +const extractChangemakerName = (raw: Record): string | null => { + const { changemaker } = raw; + if (isRecord(changemaker)) { + return toFieldValueString(changemaker.name); + } + return null; +}; + +/** Convert one `_pdc.json` field entry to a `MappedField`, or null when malformed/empty. */ +const toMappedField = (field: unknown): MappedField | null => { + if (!isRecord(field) || typeof field.baseFieldShortCode !== 'string') { + return null; + } + const value = toFieldValueString(field.value); + if (value === null) { + return null; + } + return { + baseFieldShortCode: field.baseFieldShortCode, + value, + sourceNote: extractFieldNote(field) ?? field.baseFieldShortCode, + }; +}; + +interface ParsedFields { + organizationFields: MappedField[]; + proposalFields: MappedField[]; + taxIdValue: string | null; +} + +/** Split a `_pdc.json` `fields[]` array into organization/proposal `MappedField`s and capture the tax id. */ +const parsePdcJsonFields = (raw: unknown): ParsedFields => { + const organizationFields: MappedField[] = []; + const proposalFields: MappedField[] = []; + if (isRecord(raw) && Array.isArray(raw.fields)) { + for (const field of raw.fields) { + const mapped = toMappedField(field); + if (mapped === null) { + continue; + } + if (isProposalField(mapped.baseFieldShortCode)) { + proposalFields.push(mapped); + } else { + organizationFields.push(mapped); + } + } + } + const taxIdValue = organizationFields.find((f) => f.baseFieldShortCode === 'organization_tax_id')?.value ?? null; + return { organizationFields, proposalFields, taxIdValue }; +}; + +/** + * Convert one parsed `_pdc.json` document into a normalized package. The EIN is + * taken from the `organization_tax_id` field and only kept when it is a valid + * US EIN (so non-US ids like Poland's `0000000000` yield null, as with OKF). + * `goodAsOf` is always null — the exporter output carries no generation date. + */ +const pdcJsonToPackage = (raw: unknown, slug: string): TechSoupPackage => { + const { organizationFields, proposalFields, taxIdValue } = parsePdcJsonFields(raw); + return { + slug, + ein: taxIdValue !== null && isValidEin(taxIdValue) ? taxIdValue : null, + name: isRecord(raw) ? extractChangemakerName(raw) : null, + goodAsOf: null, + organizationFields, + proposalFields, + }; +}; + +/** The slug for a `_pdc.json` file: its basename with the `_pdc.json`/`.json` suffix removed. */ +const slugForJsonFile = (filePath: string): string => + basename(filePath) + .replace(/_pdc\.json$/iv, '') + .replace(/\.json$/iv, ''); + +/** Recursively collect every `.json` file under `root`. */ +const findJsonFiles = async (root: string): Promise => { + const files: string[] = []; + const walk = async (dir: string): Promise => { + const entries = await readdir(dir, { withFileTypes: true }).catch(() => null); + if (entries === null) { + return; + } + for (const entry of entries) { + const full = join(dir, entry.name); + if (entry.isDirectory()) { + // eslint-disable-next-line no-await-in-loop -- sequential recursion bounds open file handles + await walk(full); + } else if (entry.isFile() && entry.name.toLowerCase().endsWith('.json')) { + files.push(full); + } + } + }; + await walk(root); + return files; +}; + +/** Read one JSON file, returning its parsed content or null when unreadable/not JSON. */ +const readJsonFile = async (filePath: string): Promise => { + try { + const content = await readFile(filePath, 'utf8'); + return JSON.parse(content); + } catch (error: unknown) { + logger.debug({ error, filePath }, 'Could not read/parse JSON file'); + return null; + } +}; + +/** + * Read every TechSoup exporter `_pdc.json` document under `root` and return the + * normalized packages. `.json` files that are not PDC-export shaped (no `fields` + * array) are skipped, so unrelated JSON (package.json, config, …) is ignored. + * Results are sorted by slug for stable output. + */ +const readPdcJsonPackages = async (root: string): Promise => { + const files = await findJsonFiles(root); + const packages: TechSoupPackage[] = []; + /* eslint-disable no-await-in-loop -- sequential reads bound concurrent file handles */ + for (const file of files) { + const raw = await readJsonFile(file); + if (!isRecord(raw) || !Array.isArray(raw.fields)) { + continue; + } + packages.push(pdcJsonToPackage(raw, slugForJsonFile(file))); + } + /* eslint-enable no-await-in-loop */ + return packages.sort((a, b) => a.slug.localeCompare(b.slug)); +}; + +/** + * Validate one parsed JSON document as a single PDC-export package and normalize + * it. Throws when the document is not `{ changemaker?, fields: [...] }` shaped, + * so a wrong file fails loudly instead of importing nothing. + */ +const parsePdcJsonPackage = (raw: unknown, slug: string): TechSoupPackage => { + if (!isRecord(raw) || !Array.isArray(raw.fields)) { + throw new Error(`${slug} is not a TechSoup PDC JSON document (expected an object with a "fields" array)`); + } + return pdcJsonToPackage(raw, slug); +}; + +/** Read and normalize the single TechSoup PDC JSON file at `filePath` (the import's only input). */ +const readPdcJsonFile = async (filePath: string): Promise => { + const content = await readFile(filePath, 'utf8'); + return parsePdcJsonPackage(JSON.parse(content), slugForJsonFile(filePath)); +}; + +export { parsePdcJsonPackage, pdcJsonToPackage, readPdcJsonFile, readPdcJsonPackages }; diff --git a/src/source.ts b/src/source.ts new file mode 100644 index 0000000..026e478 --- /dev/null +++ b/src/source.ts @@ -0,0 +1,106 @@ +// DISABLED: not wired into the `techsoup` command, which now imports a single +// PDC JSON file (see techsoup.ts). Kept for a future step that turns OKF +// markdown bundles (zip/folder) into that single JSON file. +import { mkdtemp, readdir, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import AdmZip from 'adm-zip'; +import { logger } from './logger.js'; +import { readOkfBundlePackages } from './okf.js'; +import { readPdcJsonPackages } from './pdc-json.js'; +import type { TechSoupPackage } from './fields.js'; + +/** How `--source` is interpreted: auto-detect, raw OKF bundles, or exporter `_pdc.json` files. */ +type SourceFormat = 'auto' | 'okf' | 'json'; + +/** Narrow a raw CLI string (constrained by yargs `choices`) to a `SourceFormat`. */ +const asSourceFormat = (value: string): SourceFormat => (value === 'okf' || value === 'json' ? value : 'auto'); + +/** Extract a .zip into `destDir` using adm-zip (cross-platform, no shell dependency). */ +const extractZip = async (zipPath: string, destDir: string): Promise => { + const zip = new AdmZip(zipPath); + await zip.extractAllToAsync(destDir, true, false); +}; + +/** + * Resolve a `--source` argument (a `.zip` file, a folder, or a folder containing + * `.zip` files) to a single directory tree to read. Returns the root plus a + * cleanup for any temp dir created for extraction. The resolved root is read by + * either the OKF or the `_pdc.json` reader depending on `--source-format`. + */ +const prepareSourceRoot = async (source: string): Promise<{ root: string; cleanup: () => Promise }> => { + const stats = await stat(source); + const noop = async (): Promise => { + /* nothing to clean up when the source directory is read in place */ + }; + if (stats.isFile()) { + if (!source.toLowerCase().endsWith('.zip')) { + throw new Error(`Source file is not a .zip (pass a .zip or a folder): ${source}`); + } + const temp = await mkdtemp(join(tmpdir(), 'techsoup-')); + logger.info(`Extracting ${source} to ${temp}`); + await extractZip(source, temp); + return { + root: temp, + cleanup: async () => { + await rm(temp, { recursive: true, force: true }); + }, + }; + } + if (!stats.isDirectory()) { + throw new Error(`Source is neither a .zip file nor a directory: ${source}`); + } + const entries = await readdir(source, { withFileTypes: true }); + const zips = entries.filter((entry) => entry.isFile() && entry.name.toLowerCase().endsWith('.zip')); + if (zips.length === 0) { + return { root: source, cleanup: noop }; + } + const temp = await mkdtemp(join(tmpdir(), 'techsoup-')); + logger.info(`Found ${zips.length} zip file(s) in ${source}; extracting to ${temp}`); + for (const zip of zips) { + // eslint-disable-next-line no-await-in-loop -- extract archives sequentially to bound disk/CPU pressure + await extractZip(join(source, zip.name), join(temp, zip.name.replace(/\.zip$/iv, ''))); + } + return { + root: temp, + cleanup: async () => { + await rm(temp, { recursive: true, force: true }); + }, + }; +}; + +/** + * Read the packages under `root` in the requested format. `auto` prefers raw OKF + * bundles and falls back to `_pdc.json` files when none are found. Returns the + * packages plus the format actually used (for logging). + */ +const readPackages = async ( + root: string, + format: SourceFormat, +): Promise<{ packages: TechSoupPackage[]; format: 'okf' | 'json' }> => { + if (format === 'json') { + return { packages: await readPdcJsonPackages(root), format: 'json' }; + } + if (format === 'okf') { + return { packages: await readOkfBundlePackages(root), format: 'okf' }; + } + const okfPackages = await readOkfBundlePackages(root); + if (okfPackages.length > 0) { + return { packages: okfPackages, format: 'okf' }; + } + return { packages: await readPdcJsonPackages(root), format: 'json' }; +}; + +/** Report whether the source resolved to a single package or several (all processed one at a time). */ +const describePackageCount = (packages: TechSoupPackage[], source: string, format: 'okf' | 'json'): void => { + const label = format === 'okf' ? 'OKF' : 'PDC-JSON'; + if (packages.length === 0) { + logger.warn(`No ${label} packages found under ${source}`); + } else if (packages.length === 1) { + logger.info(`Detected a single ${label} package under ${source}: ${packages[0]?.slug ?? '(unknown)'}`); + } else { + logger.info(`Detected ${packages.length} ${label} packages under ${source}; processing one at a time`); + } +}; + +export { type SourceFormat, asSourceFormat, describePackageCount, prepareSourceRoot, readPackages }; From 8e2090a63dd2dee03514e81a838518446139b040 Mon Sep 17 00:00:00 2001 From: Grant Minor Date: Thu, 12 Nov 2026 14:01:34 -0600 Subject: [PATCH 4/4] techsoup tested in sandbox --- docs/techsoup.md | 336 ++++++++--------------- package-lock.json | 34 +-- package.json | 2 - src/changemakers.ts | 489 +++++++++++++++++++-------------- src/fields.ts | 28 +- src/index.ts | 6 +- src/logger.ts | 32 ++- src/okf.ts | 421 ---------------------------- src/pdc-api.ts | 3 +- src/pdc-json.ts | 118 ++------ src/source.ts | 106 ------- src/techsoup.ts | 244 +++-------------- src/techsoup.unit.test.ts | 562 ++++++++++++++++++++++++++------------ 13 files changed, 902 insertions(+), 1479 deletions(-) delete mode 100644 src/okf.ts delete mode 100644 src/source.ts diff --git a/docs/techsoup.md b/docs/techsoup.md index f576db4..47a5337 100644 --- a/docs/techsoup.md +++ b/docs/techsoup.md @@ -1,155 +1,118 @@ # TechSoup Integration — Technical Report **Source file:** [`../src/techsoup.ts`](../src/techsoup.ts) -**Generated:** 2026-09-18 -**Purpose:** Read nonprofit ("changemaker") data from TechSoup [OKF](https://okf.md/spec/)-format bundles delivered as a zip file or folder, and import the mapped organization fields into the Philanthropy Data Commons (PDC) as changemaker field values. +**Generated:** 2026-09-18 (updated 2026-11-11: single-JSON-file import only; zip and markdown readers removed) +**Purpose:** Import one TechSoup organization, delivered as a single already-mapped PDC JSON file, into the Philanthropy Data Commons (PDC) as changemaker field values — either into an existing changemaker or a newly created one. + +**Modules:** [`../src/techsoup.ts`](../src/techsoup.ts) (CLI) · [`../src/pdc-json.ts`](../src/pdc-json.ts) (JSON reader) · [`../src/fields.ts`](../src/fields.ts) (`TechSoupDocument` as read, `TechSoupPackage` once sorted by category) · [`../src/changemakers.ts`](../src/changemakers.ts) (choosing the changemaker, preview, PDC writes). --- ## 1. Overview -`techsoup.ts` defines a [yargs](https://yargs.js.org/) command module (`techsoup`) registered in [`../src/index.ts`](../src/index.ts) as part of the `data-scripts` CLI. Unlike the Charity Navigator and GivingTuesday integrations, the **source is not a REST API** — it is a set of local files (an OKF bundle, or "package") that TechSoup produces. The destination is the same PDC REST API used by the other integrations. - -It exposes three subcommands: - -| Subcommand | Purpose | Writes to PDC? | Needs PDC auth? | -| --------------- | --------------------------------------------------------------------------------------------------------- | -------------- | -------------------------- | -| `inventory` | Examine a zip/folder, discover the OKF packages inside, and print/save the mapped fields available | No | No | -| `lookupFromPdc` | Discover the packages, extract each EIN, and report which are present in PDC (dry-run style) | No | No (reads PDC anonymously) | -| `updateAll` | Full sync: discover packages → match EIN to a PDC changemaker → write mapped organization fields into PDC | **Yes**\* | **Yes (OIDC)**\* | - -\* `updateAll --dry-run` performs the full discovery and mapping and logs exactly what it _would_ write, without authenticating or contacting the PDC for writes. - -Invocation examples: +`techsoup` is a single [yargs](https://yargs.js.org/) command registered in [`../src/index.ts`](../src/index.ts). It takes **one JSON file** as its argument and loads that file's organization fields into **one** PDC changemaker. ```bash -data-scripts techsoup inventory --source C:\Data\TechSoup\organizations +npm run techsoup -- [--create-changemaker | --changemaker-id ] [--dry-run] --pdc-api-base-url --oidc-base-url --oidc-client-id --oidc-client-secret ``` -```bash -data-scripts techsoup updateAll --source techsoup-bundles.zip --dry-run --pdc-api-base-url --oidc-base-url --oidc-client-id --oidc-client-secret -``` +How the target changemaker is chosen: -```bash -data-scripts techsoup updateAll --source C:\Data\TechSoup\organizations --pdc-api-base-url --oidc-base-url --oidc-client-id --oidc-client-secret -``` +| Flags | Target | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| `--create-changemaker` | Create a **new** changemaker (`taxId` = file EIN, `name` = `changemaker.name`). Refused if the EIN already exists in PDC. | +| `--changemaker-id ` | Load into the existing changemaker with that ID (the file EIN is not required; a mismatch is warned about). | +| neither | Match the file EIN (`organization_tax_id`) against PDC changemakers; exactly one must match. | -> Because the source is a local bundle, no source API key is required. Authentication (OIDC) is only needed to _write_ into PDC via `updateAll` (and is skipped entirely under `--dry-run`). +`--create-changemaker` and `--changemaker-id` are mutually exclusive. ---- +`--dry-run` prints the target changemaker (ID, name, taxId — or `NEW changemaker`, or why nothing would be imported) and every field that would be imported, then stops. It reads the PDC changemaker list anonymously but does not authenticate or write. + +> The OIDC options are declared as required by the shared `oidcOptions`, so supply them (or the `DS_*` environment variables) even with `--dry-run`; they are only used for a real import. -## 2. External Systems +--- -### TechSoup OKF bundles (source) +## 2. Input file -- **Format:** [OKF](https://okf.md/spec/) — each organization is a directory ("package") of markdown files with YAML **frontmatter**. The package root is `README.md` with frontmatter `type: org`; sibling files (`impact.md`, `population.md`, `programs.md`, `what_i_need_funding_for.md`, `technical-volunteers/*.md`, …) carry the rest. -- **Delivery:** a single `.zip`, a folder of extracted packages, or a folder containing several `.zip` files. See §6. -- **Parsing:** frontmatter is parsed with [`js-yaml`](https://github.com/nodeca/js-yaml) (`JSON_SCHEMA`, so timestamp-like scalars stay strings). Zips are read with [`adm-zip`](https://github.com/cthackers/adm-zip) — no shell dependency and cross-platform. -- **Mapping origin:** [`docs/mappings/techsoup-okf-field-gap-analysis.verbose.md`](mappings/techsoup-okf-field-gap-analysis.verbose.md). +One JSON document per organization — the TechSoup exporter's `_pdc.json` shape: -### PDC API (destination) +```json +{ + "changemaker": { "name": "Example Org" }, + "fields": [ + { "baseFieldShortCode": "organization_name", "value": "Example Org", "batch": { "notes": "README.md#title" } }, + { "baseFieldShortCode": "organization_tax_id", "value": "12-3456789" }, + { "baseFieldShortCode": "organization_dba_name", "value": ["Alias One", "Alias Two"] }, + { "baseFieldShortCode": "proposal_name", "value": "A funding ask" } + ] +} +``` -- **Client:** Axios wrapper in [`../src/pdc-api.ts`](../src/pdc-api.ts) / [`../src/client.ts`](../src/client.ts) -- **Auth:** OIDC `client_credentials` grant via [`../src/oidc.ts`](../src/oidc.ts) (`getToken`). Only `updateAll` (non-dry-run) authenticates; reads of `/changemakers` are anonymous. -- **Relevant endpoints:** - - `GET /changemakers` — list all changemakers (shallow, anonymous) - - `GET /sources` — find the TechSoup source - - `POST /sources` — create the source (admin-only; typically fails for non-admins) - - `POST /changemakerFieldValueBatches` — open a batch - - `POST /changemakerFieldValues` — write one field value +- **EIN** — the `organization_tax_id` field value, kept only when it passes `isValidEin` (`^\d{2}-?\d{7}$`). Non-US ids yield no EIN. +- **Name** — `changemaker.name` (needed only when creating a changemaker). +- **Values** — coerced by `toFieldValueString`: strings trimmed (blank dropped), numbers/booleans stringified, arrays joined with `"; "`, objects dropped. +- **Provenance** — `batch.notes` (or the short code) is shown in the dry-run output. +- **`goodAsOf`** — always `null` (the file carries no generation date). +- **Split** — fields are classified by the PDC's own base-field categories (`GET /baseFields`, which needs no token). Only `organization` fields are posted to the changemaker; fields of any other category (`project`, `budget`, …) are proposal fields and are listed but not imported. Short codes that are not base fields in the target PDC are skipped with a warning. +- A file without a `fields` array, or that is not valid JSON, fails with an error. --- -## 3. Data Flow (the `updateAll` path) +## 3. Data Flow ``` ---source (zip | folder | folder-of-zips) - │ - ▼ -prepareSourceRoot ──► extract any .zip(s) to a temp dir (adm-zip), else use folder as-is - │ - ▼ -findOrgBundles ──► walk the tree; every dir whose README.md frontmatter is `type: org` - is one OKF package (1 package if the root itself is a bundle, else many) - │ - ▼ -GET PDC /changemakers ──► (optionally scoped to one via --changemaker-id) - │ - ▼ (if --dry-run: log intended writes and stop here) -Authenticate to PDC (OIDC client_credentials) ──► access token - │ - ▼ -Find or create PDC Source (dataProviderShortCode = "techsoup") - │ - ▼ -Open a ChangemakerFieldValueBatch (sourceId + notes with timestamp) - │ - ▼ -For each OKF package (one at a time): - extract EIN from README frontmatter (x-civic.registration.id, IRS-EIN only) - └─ no valid EIN ──► skip package - resolve EIN against PDC changemakers (resolveChangemaker) - ├─ exactly one match ──► use it - ├─ >1 match (ambiguous) ──► skip package (warn) - └─ no match ──► --add-changemakers ? POST /changemakers (taxId=EIN, name=README title) - : skip package - derive goodAsOf from README generated.at - collect organization fields (organizationFieldMap) and proposal fields - └─ proposal fields are reported but NOT written (stubbed — see §4.2) - For each organization field ──► POST /changemakerFieldValues - (batchId, changemakerId, baseFieldShortCode, value, goodAsOf) - └─ 403 Forbidden ──► warn + record changemakerId, continue - │ - ▼ -Clean up any temp extraction dir + + │ + ▼ +readPdcJsonFile ──► TechSoupDocument { slug, ein, name, goodAsOf, fields } + │ + ▼ +GET PDC /baseFields (anonymous) ──► applyBaseFieldCategories ──► TechSoupPackage { …, organizationFields, proposalFields } + │ + ▼ +GET PDC /changemakers (anonymous) + │ + ▼ +planImport ──► update (existing changemaker) | create (new changemaker) | skip (reason) + │ + ├─ --dry-run ──► log formatImportPreview lines and stop + ├─ skip ──────► fail with the reason + ▼ +Authenticate (OIDC client_credentials) + │ + ▼ +find PDC Source "techsoup" (oldest if several; create only if none — 403/404 → stop, nothing written) + │ + ▼ +create changemaker (if planned; 403 → warn and stop) ──► open ChangemakerFieldValueBatch + │ + ▼ +POST each organization field to /changemakerFieldValues (sequentially; 403 → warn and continue) +proposal fields: reported, NOT written (stubbed — see §4.2) ``` -### Key processing details - -- **Package detection (1 vs many):** `findOrgBundles` walks the resolved root and treats every directory whose `README.md` frontmatter declares `type: org` as one package (it does not descend into a package once found). If the root _itself_ is a package, exactly one is returned; otherwise every nested package is collected. `describeBundleCount` logs which case occurred. Either way, packages are processed **one at a time**. -- **EIN extraction (`extractEin`):** reads `x-civic.registration.id` and only accepts it when `x-civic.registration.scheme` is `IRS-EIN` (or absent) **and** the id passes `isValidEin` (`^\d{2}-?\d{7}$`). Non-US bundles (Polish `KRS`, Kenyan PBO, …) therefore yield no EIN and are skipped — they cannot match a US-EIN-keyed changemaker. -- **EIN → changemaker matching (`resolveChangemaker`):** compares hyphen-stripped EINs and classifies the result as `matched` (exactly one), `ambiguous` (more than one — always skipped with a warning, never auto-created), or `missing` (none). By default `missing` is skipped ("if it doesn't exist in the PDC, skip it"); with `--add-changemakers` a `missing` package instead **creates** a changemaker (see below). -- **Creating changemakers (`--add-changemakers`, `createChangemakerForBundle`):** off by default. When on, a `missing` package is created via `POST /changemakers` with `taxId` = the package EIN (stored **verbatim, hyphenated**, e.g. `00-1000008`) and `name` = the README `title`. A package with **no README title is skipped** (a changemaker needs a name), and a **403** on create is treated like other privileged writes (warned and skipped). Newly created changemakers are tracked in-memory so a second package with the same EIN in the same run reuses the new record rather than creating a duplicate. The flag is **ignored when `--changemaker-id` is set** (that scopes the run to one existing changemaker), and it does not apply to `ambiguous` matches. -- **`goodAsOf` derivation (`extractGoodAsOf`):** takes the date portion (`YYYY-MM-DD`) of the README's `generated.at`, or `null` if missing/unparseable (`goodAsOf` is nullable in PDC). -- **Value coercion (`toFieldValueString`):** trims strings (dropping empties), stringifies numbers/booleans, and joins arrays (e.g. `aliases`, `ntee`) with `"; "`. Objects are skipped. -- **Sequential writes:** field values are POSTed one at a time (not `Promise.all`) because the PDC API times out under concurrent POSTs to `/changemakerFieldValues`. -- **403 handling:** `postChangemakerFieldValueWarnOnForbidden` swallows HTTP 403 (logs a warning, records the `changemakerId` for an end-of-run summary) and re-throws any other error. -- **Source resolution:** `getOrCreateSource` finds an existing source with `dataProviderShortCode === "techsoup"`; otherwise attempts to create one, warning that this usually requires a `pdc-admin`. -- **Temp cleanup:** any temporary extraction directory is removed in a `finally` block, whether the run succeeds or throws. - --- -## 4. Field Mapping: TechSoup OKF → PDC +## 4. Field Mapping -The mapping mirrors the original OKF exporter (gap analysis §2) and is defined by two constants in [`../src/techsoup.ts`](../src/techsoup.ts). Each entry reads one dotted frontmatter path from one file in the package and targets a PDC **base field** short code. +The JSON file is already mapped to PDC base-field short codes, so no mapping happens at import time. The tables below record the mapping TechSoup's exporter applies (from its OKF markdown profile), to document what the JSON is expected to contain. Which fields are written is decided by each base field's PDC category, not by these tables. -### 4.1 Organization fields — written to PDC (`organizationFieldMap`) +### 4.1 Organization fields — written to PDC -| OKF source (`file#frontmatter.path`) | PDC base field short code | -| ---------------------------------------- | ------------------------------- | -| `README.md#title` | `organization_name` | -| `README.md#description` | `organization_overview` | -| `README.md#aliases` | `organization_dba_name` | -| `README.md#status` | `organization_status` | -| `README.md#x-civic.registration_country` | `organization_country` | -| `README.md#x-civic.registration.id` | `organization_tax_id` | -| `README.md#x-civic.ntee` | `organization_ntee_code` | -| `README.md#x-civic.situation` | `organization_geographic_area` | -| `population.md#description` | `community_definition_overview` | +| OKF source (`file#frontmatter.path`) | PDC base field short code | +| ---------------------------------------- | ------------------------------ | +| `README.md#title` | `organization_name` | +| `README.md#description` | `organization_overview` | +| `README.md#aliases` | `organization_dba_name` | +| `README.md#status` | `organization_status` | +| `README.md#x-civic.registration_country` | `organization_country` | +| `README.md#x-civic.registration.id` | `organization_tax_id` | +| `README.md#x-civic.ntee` | `organization_ntee_code` | +| `README.md#x-civic.situation` | `organization_geographic_area` | -Each becomes a **`ChangemakerFieldValue`**: +> **Known data-quality caveat (gap analysis §5.1):** `README#status` is an OKF _document_ lifecycle value (`stable`/`draft`), not an organizational status. It is mapped to `organization_status` to match the existing exporter output. -| PDC field value property | Value / source | -| ------------------------ | ------------------------------------------------------------ | -| `changemakerId` | Resolved via `getChangemakerByEin` (normalized EIN match) | -| `batchId` | ID of the batch opened for this run | -| `baseFieldShortCode` | From the mapping table above | -| `value` | Coerced via `toFieldValueString` (arrays joined with `"; "`) | -| `goodAsOf` | Derived from README `generated.at` (ISO date or `null`) | - -> **Known data-quality caveat (gap analysis §5.1):** `README#status` is an OKF _document_ lifecycle value (`stable`/`draft`), not an organizational status. It is mapped to `organization_status` to match the existing exporter output; the code carries a `NOTE` to revisit this once the correct source is settled. `organization_dba_name` and `organization_ntee_code` originate as arrays and are flattened to a `"; "`-joined string (§5.5). - -### 4.2 Proposal fields — collected and reported, **not yet written** (`proposalFieldMap`) +### 4.2 Proposal fields — reported, **not yet written** | OKF source (`file#frontmatter.path`) | PDC base field short code | | --------------------------------------------------------- | --------------------------------- | @@ -158,120 +121,45 @@ Each becomes a **`ChangemakerFieldValue`**: | `what_i_need_funding_for.md#title` | `proposal_name` | | `what_i_need_funding_for.md#description` | `proposal_funding_summary` | | `technical-volunteers/.md#description` | `proposal_learning_and_evalution` | +| `population.md#description` | `community_definition_overview` | -**Why stubbed:** PDC proposal data is not a flat set of values on a changemaker. It requires an **Opportunity + ApplicationForm + ProposalVersion** scaffold (see [`../src/postProposalVersions.ts`](../src/postProposalVersions.ts)) whose form/opportunity identity is not derivable from an OKF bundle alone. Until that model is decided, `updateAll` **collects** proposal fields and logs them via `warnProposalsStubbed` (a `TODO(techsoup-proposals)` marks the write site) but does not POST them. `inventory` still reports them so their availability is visible. - -> The volunteer-request markdown file is named differently per package (e.g. `cohort-outcome-reporting.md`, `intake-triage-assist.md`), so `resolveVolunteerRequestFile` finds it by frontmatter `type: volunteer-request` rather than by a fixed filename. +`community_definition_overview` is a `project`-category base field in the PDC, so it can only be stored with a proposal even though it does not start with `proposal_`. -### 4.3 EIN — used for matching, not stored as a field - -`x-civic.registration.id` is the join key. It _is_ also mapped to `organization_tax_id` (per the exporter), but its primary role is to match the package to a PDC changemaker; a package with no matching changemaker is skipped entirely. +PDC proposal data needs an **Opportunity + ApplicationForm + ProposalVersion** scaffold (see [`../src/postProposalVersions.ts`](../src/postProposalVersions.ts)) that a single organization file cannot supply. Until that model is decided, proposal fields are listed by `--dry-run` and warned about by `warnProposalsStubbed` (`TODO(techsoup-proposals)`), but not POSTed. --- -## 5. Command Reference - -### `inventory` - -- **Args:** `--source` (required), `--output-file`/`--write` (optional), `--show-fields`/`--fields` (optional boolean, default `false`). -- **Behavior:** Resolves the source (extracting zips as needed), discovers OKF packages, and for each reports its EIN, `goodAsOf`, and the count of mapped organization/proposal fields. Writes a full JSON inventory to `--output-file` if given, otherwise logs a summary. **No PDC interaction whatsoever** — this is the way to preview the mapping fully offline. -- **`--show-fields`:** Instead of per-package counts, prints the complete field-by-field mapping to the console: each organization field as a `WOULD POST changemakerFieldValue: = ` line, and each proposal field as a `proposal field (collected, NOT written)` line, with its `file#anchor` provenance. Nothing is sent to or read from the PDC; the changemaker match is only noted as happening at run time. This is the answer to "show me exactly what would go into PDC without running anything." - -```bash -data-scripts techsoup inventory --source C:\Data\TechSoup\organizations --show-fields -``` - -### `lookupFromPdc` - -- **Args:** `--source` (required), `--pdc-api-base-url` (required), `--output-file` (optional). -- **Behavior:** Discovers packages, extracts each EIN, reads all PDC changemakers, and reports per package whether a matching changemaker exists (and its ID). **Read-only** with respect to PDC. - -### `updateAll` - -- **Args:** `--source` (required), `--pdc-api-base-url` (required), all `oidcOptions` (`--oidc-base-url`, `--oidc-client-id`, `--oidc-client-secret`), `--changemaker-id` (optional; scope to one changemaker), `--dry-run` (optional boolean, default `false`), `--add-changemakers` (optional boolean, default `false`). -- **Behavior:** The full sync described in §3. The only subcommand that writes to PDC. `--dry-run` performs discovery and mapping and logs each intended `POST` (including `would CREATE changemaker …` lines when `--add-changemakers` is set) without authenticating or writing. -- **`--add-changemakers`:** When a package's EIN has **no** match in the PDC, create a new changemaker for it (`taxId` = EIN, `name` = README `title`) instead of skipping it. Requires a valid IRS-EIN and a README title; ambiguous matches are never auto-created; ignored when `--changemaker-id` is set. See §3 "Creating changemakers". - -```bash -data-scripts techsoup updateAll --source C:\Data\TechSoup\organizations --add-changemakers --dry-run --pdc-api-base-url --oidc-base-url --oidc-client-id --oidc-client-secret -``` +## 5. Error Handling + +| Concern | Handling | +| ------------------------------------------------ | ------------------------------------------------------------------------------------------------- | +| File missing / not JSON / no `fields` array | Error; nothing is read from or written to the PDC | +| No valid EIN and no `--changemaker-id` | Skipped with reason (dry-run prints it; real run fails) | +| EIN not in PDC (default mode) | Skipped; reason suggests `--create-changemaker` | +| EIN matches several changemakers | Skipped; reason suggests `--changemaker-id` | +| `--changemaker-id` not found | Skipped with reason | +| `--changemaker-id` taxId differs from file EIN | Warning; import proceeds into the given ID | +| `--create-changemaker` but EIN already in PDC | Skipped (no duplicate); reason names the existing changemaker | +| `--create-changemaker` but no `changemaker.name` | Skipped with reason | +| HTTP 403 creating the changemaker | Warned; nothing imported | +| HTTP 403 on a field-value write | Warned; remaining fields continue; summary warning at end | +| PDC concurrency timeouts | Field values POSTed sequentially | +| No TechSoup source | Attempt to create (usually admin-only). 403/404 → clear "ask a pdc-admin" error, before any write | --- -## 6. Source resolution & package detection - -`prepareSourceRoot` accepts three shapes for `--source` and normalizes them to a single directory to walk: - -| `--source` | Handling | -| -------------------------------- | ---------------------------------------------------------------------------------------- | -| A `.zip` file | Extracted (adm-zip) to a temp dir; that dir becomes the root. | -| A folder with no `.zip` files | Used directly as the root (extracted packages already on disk). | -| A folder containing `.zip` files | Each `.zip` is extracted into a temp dir (one subfolder per archive); that becomes root. | - -The **number of OKF packages** is then a property of the tree, determined by `findOrgBundles` (§3): a root that is itself a `type: org` bundle yields one package; otherwise every nested bundle is collected. `describeBundleCount` logs "single OKF package" vs "N OKF packages … processing one at a time". A zip that wraps everything in a top-level folder is handled transparently because the walk is recursive. Any temp directory created for extraction is deleted after the command finishes. - ---- +## 6. Testing -## 7. Error Handling & Resilience Summary - -| Concern | Handling | -| --------------------------------------- | ------------------------------------------------------------------------------ | -| Source is not a `.zip` or directory | Throws a clear error naming the offending path | -| Unreadable / missing markdown file | `parseFrontmatterFile` logs at debug and returns `null`; that field is skipped | -| Malformed YAML frontmatter | Caught; treated as empty frontmatter (package may then be skipped) | -| No frontmatter at all | Treated as an empty object | -| Non-US / non-EIN registration | `extractEin` returns `null`; package skipped (never created, even with `--add-changemakers`) | -| EIN not present in PDC | Default: package skipped. With `--add-changemakers`: a changemaker is created (see below) | -| Ambiguous EIN → changemaker (>1 match) | Skipped with a warning; never auto-created | -| Create requested but no README title | Skipped with a warning (`--add-changemakers` needs a name) | -| HTTP 403 on changemaker create | Warned; that package skipped; run continues | -| Null / empty / object-valued attributes | Skipped (not posted) | -| Multi-value attributes (arrays) | Flattened to a `"; "`-joined string | -| Missing / unparseable `generated.at` | `goodAsOf` set to `null` | -| HTTP 403 on field-value write | Warned + changemakerId recorded; run continues; summary warning at end | -| PDC concurrency timeouts | Field values POSTed sequentially | -| Missing TechSoup source | Attempt to create (usually admin-only); warns it may fail | -| Proposal fields | Collected and reported, **not written** (stubbed — §4.2) | -| Temp extraction directory | Always removed in a `finally` block | +Unit tests in [`../src/techsoup.unit.test.ts`](../src/techsoup.unit.test.ts) cover: ---- - -## 8. Testing - -Unit tests in [`../src/techsoup.unit.test.ts`](../src/techsoup.unit.test.ts) cover the exported pure helpers: - -- **`parseFrontmatter`** — parses top-level scalars, flow sequences, an inline flow map, a nested block map under `x-civic`, and a block sequence of mappings (`sources`) without disturbing later keys; strips trailing comments; returns `{}` when no frontmatter is present. -- **`getFrontmatterValue`** — follows a dotted path; returns `undefined` for a missing key, when descending into a non-object, or when descending into an array. -- **`toFieldValueString`** — trims strings, returns `null` for empty/whitespace/`null`/`undefined`/empty-array/plain-object, stringifies numbers/booleans, and joins arrays with `"; "` (dropping empties). -- **`extractEin`** — returns a valid EIN for `IRS-EIN` (or scheme-absent) registrations, and `null` for a non-IRS scheme (e.g. Polish KRS), an invalid id, or a missing id. -- **`extractGoodAsOf`** — extracts `YYYY-MM-DD` from `generated.at`; returns `null` when missing or unparseable. -- **`extractOrganizationName`** — returns the README `title` (the name used to create a changemaker); `null` when absent or blank. -- **`getChangemakerByEin`** — matches ignoring hyphens; returns `null` when there is no match or when more than one changemaker matches. -- **`resolveChangemaker`** — classifies an EIN as `matched` (unique), `missing` (none), or `ambiguous` (>1); this is what drives the `--add-changemakers` decision. -- **`selectChangemakers`** — returns the full bundle when no ID is given; scopes to a single changemaker and updates `total`; returns an empty bundle for an unknown ID. - -The parsing/OKF helpers live in [`../src/okf.ts`](../src/okf.ts) and the changemaker matching/write helpers in [`../src/changemakers.ts`](../src/changemakers.ts); [`../src/techsoup.ts`](../src/techsoup.ts) is the CLI wiring. The file-system and network-facing functions (`prepareSourceRoot`, `extractZip`, `findOrgBundles`, `collectFieldsFromMap`, `createChangemakerForBundle`, `runUpdateAllWrite`, the command handlers, and all PDC writes) are not directly unit-tested; testability is achieved by extracting the pure parsing/mapping/matching logic into the exported helpers above (the same approach as the GivingTuesday integration). - ---- - -## 9. Notable Constants & TODOs in the Code - -- `TS_SHORT_CODE = 'techsoup'` — the PDC data provider short code for TechSoup. -- `ARRAY_VALUE_SEPARATOR = '; '` — how multi-valued frontmatter is flattened into a scalar field value. -- `FRONTMATTER_PATTERN` — isolates the YAML frontmatter block so the markdown body is never handed to the YAML parser. -- `HTTP_STATUS_FORBIDDEN = 403` — with a comment noting a shared `@pdc/http-status-codes` package should replace this once available. -- `TODO(techsoup-proposals)` — the write site for proposal data, pending the Opportunity/ApplicationForm/ProposalVersion model (§4.2). - ---- +- **`readPdcJsonFile` / `parsePdcJsonDocument`** — reads one file (slug from the file name), keeps every usable field in file order, and rejects non-PDC-shaped or invalid JSON. +- **`pdcJsonToDocument`** — EIN from `organization_tax_id` (null for non-US), name from `changemaker.name`, array flattening, `batch.notes` provenance, blank values dropped. +- **`toChangemakerTarget`** — maps `--create-changemaker` / `--changemaker-id` / neither to a target. +- **`planImport`** — every branch: EIN match/missing/ambiguous/absent; explicit ID found/missing/without EIN; create new/duplicate/ambiguous/no name/no EIN. +- **`formatImportPreview`** — the dry-run output names the target changemaker (existing or new), lists each field to import, lists proposal fields separately, and prints only the reason on skip. +- **`applyBaseFieldCategories`** — keeps only `organization`-category fields, moves other categories to the proposal fields, drops unknown short codes. +- **`selectTechSoupSource`** — one, none, or several TechSoup sources. +- **`toLoggableError`** — HTTP errors are logged without request headers or tokens. +- **`resolveChangemaker`**, **`toFieldValueString`**. -## 10. Comparison with the other integrations - -| Aspect | Charity Navigator | GivingTuesday | TechSoup | -| ----------------- | --------------------------------- | ------------------------------ | ------------------------------------------------------- | -| Source | GraphQL API (Apollo) | REST API (Axios GET) | Local OKF bundle (zip/folder) | -| Source auth | Bearer API key | None | None (local files) | -| Source parsing | GraphQL response | JSON response | YAML frontmatter (`js-yaml`) + `adm-zip` | -| EIN normalization | Strip hyphen | Strip hyphen + zero-pad to 9 | Strip hyphen; IRS-EIN scheme only | -| `goodAsOf` source | `updatedAt` | `Date_Released` (`YYYY_MM_DD`) | README `generated.at` (date portion) | -| Fields written | 4 (name, website, phone, mission) | 14 (IRS BMF) | 9 organization fields (proposals stubbed) | -| Dry run | via `lookup`/`lookupFromPdc` | via `lookup`/`lookupFromPdc` | `inventory`/`lookupFromPdc` **+ `updateAll --dry-run`** | +The PDC-facing write path (`runImportWrite`) and the command handler are not unit-tested; the decision logic is extracted into the pure `planImport` / `formatImportPreview` helpers above. diff --git a/package-lock.json b/package-lock.json index bd2b7eb..4226a59 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,11 +10,9 @@ "license": "AGPL-3.0-or-later", "dependencies": { "@apollo/client": "^4.2.12", - "adm-zip": "^0.6.1", "axios": "^1.20.0", "csv-parse": "^5.5.6", "dotenv": "^17.4.2", - "js-yaml": "^5.4.2", "jwt-decode": "^4.0.0", "openid-client": "^5.6.5", "pino": "^10.3.1", @@ -2460,15 +2458,6 @@ "node": ">=0.4.0" } }, - "node_modules/adm-zip": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.6.1.tgz", - "integrity": "sha512-Xwrja8nx9e5o2N1my4DsKCeKpdrnACyr1wtbPxBDgGzKzKyE9kRtBFA8mWldI+RVlD7CBZNWY/wQ2+ydwOR6kQ==", - "license": "MIT", - "engines": { - "node": ">=14.0" - } - }, "node_modules/agent-base": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", @@ -2579,6 +2568,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, "license": "Python-2.0" }, "node_modules/array-back": { @@ -6476,28 +6466,6 @@ "dev": true, "license": "MIT" }, - "node_modules/js-yaml": { - "version": "5.4.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.4.2.tgz", - "integrity": "sha512-m+aqu+LwO1O6sIopafj8HUVl5aawITwZQe/yHpMCKjaWBaA/d07B/QdMb3529REftiU+RMMHL3Vlsw3hON7vWg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/puzrin" - }, - { - "type": "github", - "url": "https://github.com/sponsors/nodeca" - } - ], - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.mjs" - } - }, "node_modules/jsesc": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", diff --git a/package.json b/package.json index b313e49..141f8cd 100644 --- a/package.json +++ b/package.json @@ -52,11 +52,9 @@ }, "dependencies": { "@apollo/client": "^4.2.12", - "adm-zip": "^0.6.1", "axios": "^1.20.0", "csv-parse": "^5.5.6", "dotenv": "^17.4.2", - "js-yaml": "^5.4.2", "jwt-decode": "^4.0.0", "openid-client": "^5.6.5", "pino": "^10.3.1", diff --git a/src/changemakers.ts b/src/changemakers.ts index e350ee8..9ced5fa 100644 --- a/src/changemakers.ts +++ b/src/changemakers.ts @@ -1,15 +1,5 @@ import { AxiosError } from 'axios'; import { logger } from './logger.js'; -import { - type MappedField, - type OkfBundle, - collectFieldsFromMap, - collectProposalFields, - extractEin, - extractGoodAsOf, - extractOrganizationName, - organizationFieldMap, -} from './okf.js'; import { type AccessTokenSet, getToken } from './oidc.js'; import { getSources, @@ -19,11 +9,13 @@ import { postSource, type WritableChangemakerFieldValue, } from './pdc-api.js'; -import type { Changemaker, ChangemakerBundle, Source } from '@pdc/sdk'; +import type { MappedField, TechSoupDocument, TechSoupPackage } from './fields.js'; +import type { Changemaker, ChangemakerBundle, Source, SourceBundle } from '@pdc/sdk'; const TS_SHORT_CODE = 'techsoup'; // When `@pdc/http-status-codes` is ready (issues 18-20 solved), use it instead. const HTTP_STATUS_FORBIDDEN = 403; +const HTTP_STATUS_NOT_FOUND = 404; // --------------------------------------------------------------------------- // EIN → changemaker matching @@ -38,7 +30,7 @@ const findChangemakersByEin = (ein: string, changemakers: ChangemakerBundle): Ch /** * The outcome of resolving an EIN against the PDC changemakers: a unique match * (`matched`), several matches (`ambiguous` — never auto-created), or none - * (`missing` — a create candidate when `--add-changemakers` is set). + * (`missing` — a create candidate when `--create-changemaker` is set). */ type ChangemakerResolution = { kind: 'matched'; changemaker: Changemaker } | { kind: 'ambiguous' } | { kind: 'missing' }; @@ -55,31 +47,6 @@ const resolveChangemaker = (ein: string, changemakers: ChangemakerBundle): Chang return { kind: 'missing' }; }; -const getChangemakerByEin = (ein: string, changemakers: ChangemakerBundle): Changemaker | null => { - const resolution = resolveChangemaker(ein, changemakers); - if (resolution.kind === 'ambiguous') { - logger.warn(`Found multiple changemakers with EIN ${ein}, not returning any.`); - return null; - } - if (resolution.kind === 'missing') { - logger.info(`Found no changemaker with EIN ${ein}`); - return null; - } - return resolution.changemaker; -}; - -/** - * Return the changemaker bundle scoped to a single ID when `changemakerId` is - * supplied, or the full bundle otherwise. - */ -const selectChangemakers = (changemakers: ChangemakerBundle, changemakerId: number | undefined): ChangemakerBundle => { - if (changemakerId === undefined) { - return changemakers; - } - const entries = changemakers.entries.filter((c) => c.id === changemakerId); - return { ...changemakers, entries, total: entries.length }; -}; - // --------------------------------------------------------------------------- // PDC writes (source, changemaker creation, field values) // --------------------------------------------------------------------------- @@ -104,43 +71,87 @@ const postChangemakerFieldValueWarnOnForbidden = async ( } }; +/** + * Pick the TechSoup source (`dataProviderShortCode === 'techsoup'`) from the PDC + * sources, or null when there is none. When several exist, the oldest (lowest + * ID) is used, with a warning, rather than creating yet another. + */ +const selectTechSoupSource = (sources: SourceBundle): Source | null => { + const techSoupSources = sources.entries + .filter((s) => s.dataProviderShortCode === TS_SHORT_CODE) + .sort((a, b) => a.id - b.id); + const [oldest] = techSoupSources; + if (oldest === undefined) { + return null; + } + if (techSoupSources.length > 1) { + logger.warn( + `Found ${techSoupSources.length} TechSoup sources (${techSoupSources.map((s) => s.id).join(', ')}); using source ${oldest.id}`, + ); + } + return oldest; +}; + +/** + * Find the TechSoup source, creating it only when none exists. Creating needs + * the `techsoup` data provider to exist and permission to add sources to it + * (typically a `pdc-admin`); a 403/404 from the PDC is reported as that setup + * problem instead of a raw HTTP error. + */ const getOrCreateSource = async (baseUrl: string, token: AccessTokenSet): Promise => { const sources = await getSources(baseUrl, token); - const filteredSources = sources.entries.filter((s) => s.dataProviderShortCode === TS_SHORT_CODE); - if (filteredSources.length === 1 && filteredSources[0] !== undefined) { - return filteredSources[0]; + logger.debug( + { sources: sources.entries.map((s) => ({ id: s.id, dataProviderShortCode: s.dataProviderShortCode })) }, + `GET /sources returned ${sources.entries.length} source(s)`, + ); + const existing = selectTechSoupSource(sources); + if (existing !== null) { + return existing; + } + logger.warn('No TechSoup source found in the PDC; trying to create one (usually requires a `pdc-admin`).'); + try { + return await postSource(baseUrl, token, { + dataProviderShortCode: TS_SHORT_CODE, + label: 'TechSoup', + }); + } catch (e: unknown) { + if (e instanceof AxiosError && (e.status === HTTP_STATUS_NOT_FOUND || e.status === HTTP_STATUS_FORBIDDEN)) { + const problem = + e.status === HTTP_STATUS_NOT_FOUND + ? `the "${TS_SHORT_CODE}" data provider does not exist in this PDC` + : `this account may not create sources for the "${TS_SHORT_CODE}" data provider`; + throw new Error( + `Cannot create the TechSoup source (HTTP ${e.status}): ${problem}. Ask a pdc-admin to create the "${TS_SHORT_CODE}" data provider and its source, then re-run.`, + { cause: e }, + ); + } + throw e; } - logger.warn('Have a `pdc-admin` create a source because only administrators may be able.'); - return await postSource(baseUrl, token, { - dataProviderShortCode: TS_SHORT_CODE, - label: 'TechSoup', - }); }; /** * Create a new PDC changemaker for a package whose EIN is not yet in the PDC. - * Requires an organization name (README `title`); the EIN is stored verbatim - * (hyphenated) as the taxId. Returns null (with a warning) when the name is - * missing or the create is forbidden (403 — creating changemakers is privileged). + * Requires the package name; the EIN is stored verbatim (hyphenated) as the + * taxId. Returns null (with a warning) when the name is missing or the create is + * forbidden (403 — creating changemakers is privileged). */ -const createChangemakerForBundle = async ( +const createChangemakerForPackage = async ( baseUrl: string, token: AccessTokenSet, ein: string, - bundle: OkfBundle, + pkg: TechSoupPackage, ): Promise => { - const name = extractOrganizationName(bundle.readme); - if (name === null) { - logger.warn(`Cannot create a changemaker for ${bundle.slug}: no organization name (README title). Skipping.`); + if (pkg.name === null) { + logger.warn(`Cannot create a changemaker for ${pkg.slug}: no organization name. Skipping.`); return null; } try { - const changemaker = await postChangemaker(baseUrl, token, { taxId: ein, name }); - logger.info({ id: changemaker.id, taxId: ein, name }, `Created changemaker for ${bundle.slug}`); + const changemaker = await postChangemaker(baseUrl, token, { taxId: ein, name: pkg.name }); + logger.info({ id: changemaker.id, taxId: ein, name: pkg.name }, `Created changemaker for ${pkg.slug}`); return changemaker; } catch (e: unknown) { if (e instanceof AxiosError && e.status === HTTP_STATUS_FORBIDDEN) { - logger.warn(`No permission (403) to create a changemaker for ${bundle.slug} (taxId ${ein}). Skipping.`); + logger.warn(`No permission (403) to create a changemaker for ${pkg.slug} (taxId ${ein}). Skipping.`); return null; } throw e; @@ -165,190 +176,258 @@ const warnProposalsStubbed = (slug: string, proposalFields: MappedField[]): void ); }; +/** + * Sort the document's fields by the PDC's own base-field categories. Only + * `organization` fields may be posted to `/changemakerFieldValues`; fields of + * any other category (`project`, `budget`, …) belong to a proposal and go to + * `proposalFields`. Fields with no matching base field in this PDC are dropped + * with a warning, since the PDC would reject them. + */ +const applyBaseFieldCategories = ( + doc: TechSoupDocument, + baseFields: { entries: ReadonlyArray<{ shortCode: string; category: string }> }, +): TechSoupPackage => { + const { fields, ...pkg } = doc; + const categoryByShortCode = new Map(baseFields.entries.map((b) => [b.shortCode, b.category])); + const organizationFields: MappedField[] = []; + const proposalFields: MappedField[] = []; + const unknown: string[] = []; + for (const field of fields) { + const category = categoryByShortCode.get(field.baseFieldShortCode); + if (category === undefined) { + unknown.push(field.baseFieldShortCode); + } else if (category === 'organization') { + organizationFields.push(field); + } else { + proposalFields.push(field); + } + } + if (unknown.length > 0) { + logger.warn( + `${pkg.slug}: ${unknown.length} field(s) are not base fields in this PDC; skipping ${unknown.join(', ')}`, + ); + } + return { ...pkg, organizationFields, proposalFields }; +}; + // --------------------------------------------------------------------------- -// updateAll write path +// Import plan: which changemaker the package is loaded into // --------------------------------------------------------------------------- -/** Everything one package's writes need, gathered once so per-package helpers stay small. */ -interface WriteRun { - baseUrl: string; - token: AccessTokenSet; - batchId: number; - changemakers: ChangemakerBundle; - addChangemakers: boolean; - warnedChangemakers: Set; -} +/** + * How the target changemaker is chosen: create a new one (`create`), use an + * explicit PDC changemaker ID (`id`), or match the package EIN against the + * existing changemakers (`ein`). + */ +type ChangemakerTarget = { kind: 'create' } | { kind: 'id'; changemakerId: number } | { kind: 'ein' }; -/** Base URL + OIDC credentials needed to start a write run (before authentication). */ -interface UpdateAllWriteContext { - pdcApiBaseUrl: string; - oidcBaseUrl: string; - oidcClientId: string; - oidcClientSecret: string; -} +/** Map the CLI flags to a `ChangemakerTarget` (the CLI rejects combining the two flags). */ +const toChangemakerTarget = (createChangemaker: boolean, changemakerId: number | undefined): ChangemakerTarget => { + if (createChangemaker) { + return { kind: 'create' }; + } + if (changemakerId !== undefined) { + return { kind: 'id', changemakerId }; + } + return { kind: 'ein' }; +}; /** - * Resolve the changemaker to load a package into, creating one when the EIN is - * missing and `--add-changemakers` is set. Returns null when the package should - * be skipped (ambiguous match, missing without the flag, or a missing-name / - * forbidden create) — the reason is logged. Mutates `run.changemakers.entries` - * when a new record is created so later packages with the same EIN reuse it. + * What an import will do: load fields into an existing changemaker (`update`), + * create a changemaker and then load fields into it (`create`), or do nothing + * (`skip`, with the reason). */ -const resolveOrCreateChangemaker = async ( - run: WriteRun, - ein: string, - bundle: OkfBundle, -): Promise => { - const resolution = resolveChangemaker(ein, run.changemakers); +type ImportPlan = + | { action: 'update'; changemaker: Changemaker } + | { action: 'create'; taxId: string; name: string } + | { action: 'skip'; reason: string }; + +const planForExplicitId = ( + pkg: TechSoupPackage, + changemakers: ChangemakerBundle, + changemakerId: number, +): ImportPlan => { + const changemaker = changemakers.entries.find((c) => c.id === changemakerId); + if (changemaker === undefined) { + return { action: 'skip', reason: `No changemaker with ID ${changemakerId} exists in the PDC` }; + } + if (pkg.ein !== null && pkg.ein.replace('-', '') !== changemaker.taxId.replace('-', '')) { + logger.warn( + `${pkg.slug}: file EIN ${pkg.ein} differs from changemaker ${changemakerId} taxId ${changemaker.taxId}; using --changemaker-id as given`, + ); + } + return { action: 'update', changemaker }; +}; + +const planForCreate = (pkg: TechSoupPackage, ein: string, resolution: ChangemakerResolution): ImportPlan => { if (resolution.kind === 'matched') { - return resolution.changemaker; + return { + action: 'skip', + reason: `EIN ${ein} already belongs to changemaker ${resolution.changemaker.id}; rerun without --create-changemaker to load into it`, + }; } if (resolution.kind === 'ambiguous') { - logger.warn(`Skipping ${bundle.slug}: EIN ${ein} matches multiple changemakers`); - return null; + return { action: 'skip', reason: `EIN ${ein} already matches multiple changemakers; not creating another` }; } - if (!run.addChangemakers) { - logger.info(`Skipping ${bundle.slug}: EIN ${ein} not found in PDC`); - return null; + if (pkg.name === null) { + return { action: 'skip', reason: `${pkg.slug} has no organization name, which a new changemaker requires` }; + } + return { action: 'create', taxId: ein, name: pkg.name }; +}; + +const planForEin = (ein: string, resolution: ChangemakerResolution): ImportPlan => { + if (resolution.kind === 'matched') { + return { action: 'update', changemaker: resolution.changemaker }; + } + if (resolution.kind === 'ambiguous') { + return { action: 'skip', reason: `EIN ${ein} matches multiple changemakers; pass --changemaker-id to choose one` }; } - const created = await createChangemakerForBundle(run.baseUrl, run.token, ein, bundle); - if (created !== null) { - run.changemakers.entries.push(created); + return { + action: 'skip', + reason: `EIN ${ein} not found in the PDC; pass --create-changemaker to create it or --changemaker-id to choose one`, + }; +}; + +/** + * Decide which changemaker the package goes to, without writing anything. + * `--create-changemaker` refuses to create a duplicate of an EIN already in the + * PDC; `--changemaker-id` must name an existing changemaker; otherwise the + * package EIN must match exactly one changemaker. + */ +const planImport = (pkg: TechSoupPackage, changemakers: ChangemakerBundle, target: ChangemakerTarget): ImportPlan => { + if (target.kind === 'id') { + return planForExplicitId(pkg, changemakers, target.changemakerId); + } + if (pkg.ein === null) { + return { + action: 'skip', + reason: `${pkg.slug} has no valid IRS EIN (organization_tax_id); pass --changemaker-id to choose a changemaker`, + }; + } + const resolution = resolveChangemaker(pkg.ein, changemakers); + return target.kind === 'create' ? planForCreate(pkg, pkg.ein, resolution) : planForEin(pkg.ein, resolution); +}; + +/** One line naming the changemaker the plan targets (or why nothing will be imported). */ +const describeTarget = (plan: ImportPlan): string => { + if (plan.action === 'update') { + return `changemaker ${plan.changemaker.id} (${JSON.stringify(plan.changemaker.name)}, taxId ${plan.changemaker.taxId})`; + } + if (plan.action === 'create') { + return `NEW changemaker (${JSON.stringify(plan.name)}, taxId ${plan.taxId})`; } - return created; + return `nothing — ${plan.reason}`; }; -/** POST every mapped organization field for one package's changemaker (proposals stay stubbed). */ -const postOrganizationFields = async (run: WriteRun, changemaker: Changemaker, bundle: OkfBundle): Promise => { - const goodAsOf = extractGoodAsOf(bundle.readme); - const organizationFields = await collectFieldsFromMap(bundle.path, organizationFieldMap); - warnProposalsStubbed(bundle.slug, await collectProposalFields(bundle.path)); +/** + * The `--dry-run` report: the target changemaker, every organization field that + * would be imported into it, and the proposal fields that are present but not + * imported. Pure, so it can be unit-tested and logged line by line. + */ +const formatImportPreview = (pkg: TechSoupPackage, plan: ImportPlan): string[] => { + const lines = [`${pkg.slug} → ${describeTarget(plan)}`]; + if (plan.action === 'skip') { + return lines; + } + lines.push(`Fields that would be imported (${pkg.organizationFields.length}, goodAsOf=${pkg.goodAsOf ?? 'null'}):`); + for (const field of pkg.organizationFields) { + lines.push(` ${field.baseFieldShortCode} = ${JSON.stringify(field.value)} [${field.sourceNote}]`); + } + if (pkg.proposalFields.length > 0) { + lines.push(`Proposal (non-organization) fields NOT imported (${pkg.proposalFields.length}):`); + for (const field of pkg.proposalFields) { + lines.push(` ${field.baseFieldShortCode} = ${JSON.stringify(field.value)} [${field.sourceNote}]`); + } + } + return lines; +}; + +// --------------------------------------------------------------------------- +// Import write path +// --------------------------------------------------------------------------- + +/** Base URL + OIDC credentials needed to start a write run (before authentication). */ +interface ImportWriteContext { + pdcApiBaseUrl: string; + oidcBaseUrl: string; + oidcClientId: string; + oidcClientSecret: string; +} + +/** An authenticated write session: where to POST, with what token, into which field-value batch. */ +interface WriteSession { + baseUrl: string; + token: AccessTokenSet; + batchId: number; +} + +/** POST every organization field of the package to one changemaker, sequentially (proposals stay stubbed). */ +const postOrganizationFields = async ( + session: WriteSession, + changemaker: Changemaker, + pkg: TechSoupPackage, +): Promise => { + warnProposalsStubbed(pkg.slug, pkg.proposalFields); + const warnedChangemakers = new Set(); /* eslint-disable no-await-in-loop -- sequential POSTs avoid PDC API connection saturation. */ - for (const field of organizationFields) { + for (const field of pkg.organizationFields) { await postChangemakerFieldValueWarnOnForbidden( - run.baseUrl, - run.token, + session.baseUrl, + session.token, { changemakerId: changemaker.id, - batchId: run.batchId, + batchId: session.batchId, baseFieldShortCode: field.baseFieldShortCode, value: field.value, - goodAsOf, + goodAsOf: pkg.goodAsOf, }, - run.warnedChangemakers, + warnedChangemakers, ); } /* eslint-enable no-await-in-loop */ + if (warnedChangemakers.size > 0) { + logger.warn(`No permission for at least one field of changemaker ${changemaker.id} (so not fully updated)`); + } }; -/** The live (non-dry-run) `updateAll` path: authenticate, open a batch, and load each package. */ -const runUpdateAllWrite = async ( - bundles: OkfBundle[], - changemakers: ChangemakerBundle, - addChangemakers: boolean, - ctx: UpdateAllWriteContext, +/** + * Carry out an `update`/`create` plan: authenticate, resolve the TechSoup + * source, create the changemaker if planned, open a field-value batch, then + * POST the organization fields. The source is resolved first so a missing + * source fails the run before anything (such as a new changemaker) is written. + */ +const runImportWrite = async ( + pkg: TechSoupPackage, + plan: Exclude, + ctx: ImportWriteContext, ): Promise => { // Up to this point we didn't need PDC authentication. Now we do. const token = await getToken(ctx.oidcBaseUrl, ctx.oidcClientId, ctx.oidcClientSecret); const source = await getOrCreateSource(ctx.pdcApiBaseUrl, token); logger.info(source, 'The PDC Source for TechSoup was found'); + const changemaker = + plan.action === 'create' + ? await createChangemakerForPackage(ctx.pdcApiBaseUrl, token, plan.taxId, pkg) + : plan.changemaker; + if (changemaker === null) { + return; + } const fieldBatch = await postChangemakerFieldValueBatch(ctx.pdcApiBaseUrl, token, { sourceId: source.id, - notes: `data-scripts techsoup.ts execution ${Date.now()}`, + notes: `data-scripts techsoup.ts import of ${pkg.slug} ${Date.now()}`, }); - const run: WriteRun = { - baseUrl: ctx.pdcApiBaseUrl, - token, - batchId: fieldBatch.id, - changemakers, - addChangemakers, - warnedChangemakers: new Set(), - }; - /* eslint-disable no-await-in-loop -- sequential per-package processing avoids PDC API saturation. */ - for (const bundle of bundles) { - const ein = extractEin(bundle.readme); - if (ein === null) { - logger.info(`Skipping ${bundle.slug}: no IRS-EIN in package`); - continue; - } - const changemaker = await resolveOrCreateChangemaker(run, ein, bundle); - if (changemaker === null) { - continue; - } - await postOrganizationFields(run, changemaker, bundle); - } - /* eslint-enable no-await-in-loop */ - if (run.warnedChangemakers.size > 0) { - logger.warn( - `No permission for at least one field in each of these changemakers (so not updated): ${JSON.stringify([...run.warnedChangemakers])}`, - ); - } + await postOrganizationFields({ baseUrl: ctx.pdcApiBaseUrl, token, batchId: fieldBatch.id }, changemaker, pkg); + logger.info(`Imported ${pkg.organizationFields.length} field(s) into changemaker ${changemaker.id}`); }; -// --------------------------------------------------------------------------- -// updateAll dry-run path -// --------------------------------------------------------------------------- - -/** - * Resolve the label a dry-run should print for a package's changemaker, or null - * when the package would be skipped (logging the reason). `(new)` is returned - * when `--add-changemakers` would create one. - */ -const dryRunChangemakerLabel = ( - bundle: OkfBundle, - ein: string, - changemakers: ChangemakerBundle, - addChangemakers: boolean, -): string | null => { - const resolution = resolveChangemaker(ein, changemakers); - if (resolution.kind === 'ambiguous') { - logger.warn(`[dry-run] Skipping ${bundle.slug}: EIN ${ein} matches multiple changemakers`); - return null; - } - if (resolution.kind === 'matched') { - return String(resolution.changemaker.id); - } - if (!addChangemakers) { - logger.info(`[dry-run] Skipping ${bundle.slug}: EIN ${ein} not found in PDC`); - return null; - } - const name = extractOrganizationName(bundle.readme); - if (name === null) { - logger.warn(`[dry-run] Skipping ${bundle.slug}: would create a changemaker but README has no title (name)`); - return null; - } - logger.info(`[dry-run] would CREATE changemaker taxId=${ein} name=${JSON.stringify(name)}`); - return '(new)'; -}; - -/** Dry-run of `updateAll`: log every intended write without contacting the PDC for auth or POSTs. */ -const runUpdateAllDryRun = async ( - bundles: OkfBundle[], - changemakers: ChangemakerBundle, - addChangemakers: boolean, -): Promise => { - logger.info('[dry-run] No data will be written to the PDC.'); - /* eslint-disable no-await-in-loop -- packages are processed sequentially, mirroring the real write path */ - for (const bundle of bundles) { - const ein = extractEin(bundle.readme); - if (ein === null) { - logger.info(`[dry-run] Skipping ${bundle.slug}: no IRS-EIN in package`); - continue; - } - const changemakerLabel = dryRunChangemakerLabel(bundle, ein, changemakers, addChangemakers); - if (changemakerLabel === null) { - continue; - } - const goodAsOf = extractGoodAsOf(bundle.readme); - const organizationFields = await collectFieldsFromMap(bundle.path, organizationFieldMap); - for (const field of organizationFields) { - logger.info( - `[dry-run] would POST changemakerFieldValue changemakerId=${changemakerLabel} baseFieldShortCode=${field.baseFieldShortCode} goodAsOf=${goodAsOf ?? 'null'} value=${JSON.stringify(field.value)}`, - ); - } - warnProposalsStubbed(bundle.slug, await collectProposalFields(bundle.path)); - } - /* eslint-enable no-await-in-loop */ +export { + type ChangemakerTarget, + type ImportPlan, + applyBaseFieldCategories, + formatImportPreview, + planImport, + resolveChangemaker, + runImportWrite, + selectTechSoupSource, + toChangemakerTarget, }; - -export { getChangemakerByEin, resolveChangemaker, runUpdateAllDryRun, runUpdateAllWrite, selectChangemakers }; diff --git a/src/fields.ts b/src/fields.ts index f98bc05..a66d365 100644 --- a/src/fields.ts +++ b/src/fields.ts @@ -1,6 +1,5 @@ -// Multi-valued source data (e.g. OKF `aliases`/`ntee`, or an array-valued field -// in a `_pdc.json`) is flattened into a single PDC string with this separator, -// since a ChangemakerFieldValue.value is a scalar string. +// An array-valued field in a `_pdc.json` is flattened into a single PDC string +// with this separator, since a ChangemakerFieldValue.value is a scalar string. const ARRAY_VALUE_SEPARATOR = '; '; /** A resolved value ready to be posted as a PDC field, with a provenance note. */ @@ -11,16 +10,24 @@ interface MappedField { } /** - * The source-agnostic unit of work: the normalized package that both the OKF - * bundle reader and the `_pdc.json` reader produce, and what the importer loads - * into the PDC. `ein`/`name`/`goodAsOf` may be null (e.g. `goodAsOf` is always - * null for `_pdc.json` sources, which carry no generation date). + * One organization as read from a `_pdc.json` file, before its fields are + * sorted by PDC base-field category. `ein`/`name` may be null; `goodAsOf` is + * currently always null because the JSON carries no generation date. */ -interface TechSoupPackage { +interface TechSoupDocument { slug: string; ein: string | null; name: string | null; goodAsOf: string | null; + fields: MappedField[]; +} + +/** + * A document whose fields have been sorted by the PDC's base-field categories + * (see `applyBaseFieldCategories`): what the importer loads into the PDC. + * Only `organizationFields` are written; `proposalFields` are reported. + */ +interface TechSoupPackage extends Omit { organizationFields: MappedField[]; proposalFields: MappedField[]; } @@ -44,7 +51,4 @@ const toFieldValueString = (value: unknown): string | null => { return null; }; -/** Proposal-scoped base fields are those whose short code starts with `proposal_`. */ -const isProposalField = (baseFieldShortCode: string): boolean => baseFieldShortCode.startsWith('proposal_'); - -export { type MappedField, type TechSoupPackage, isProposalField, toFieldValueString }; +export { type MappedField, type TechSoupDocument, type TechSoupPackage, toFieldValueString }; diff --git a/src/index.ts b/src/index.ts index 77cf4bc..0b86349 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,7 +5,7 @@ import { candid } from './candid.js'; import { charityNavigator } from './charityNavigator.js'; import { getMetrics } from './getMetrics.js'; import { givingTuesday } from './givingTuesday.js'; -import { logger } from './logger.js'; +import { logger, toLoggableError } from './logger.js'; import { getTokenCommand } from './oidc.js'; import { techsoup } from './techsoup.js'; @@ -20,7 +20,7 @@ const main = async (argv: string[]): Promise => { .env('DS') .config('config') .fail((msg, err, y) => { - logger.fatal({ m: msg, err, yargs: y }, 'Error encountered'); + logger.fatal({ m: msg, err: toLoggableError(err), yargs: y }, 'Error encountered'); }) .command( ['show-args', 'args'], @@ -44,6 +44,6 @@ const main = async (argv: string[]): Promise => { }; main(process.argv).catch((err: unknown) => { - logger.fatal({ err }, 'Error encountered while running `main`'); + logger.fatal({ err: toLoggableError(err) }, 'Error encountered while running `main`'); process.exit(1); }); diff --git a/src/logger.ts b/src/logger.ts index ffc4e53..83edc24 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,3 +1,4 @@ +import { AxiosError } from 'axios'; import pino from 'pino'; const logger = pino({ @@ -11,4 +12,33 @@ const logger = pino({ }, }); -export { logger }; +/** + * Reduce an error to what is useful in a log. An AxiosError otherwise serializes + * its whole request — including the `Authorization: Bearer …` header — so only + * its status, method, URL, and response body are kept. Plain errors keep their + * name, message, and stack, and any `cause` is reduced the same way. + */ +const toLoggableError = (err: unknown): unknown => { + if (err instanceof AxiosError) { + return { + name: err.name, + message: err.message, + code: err.code, + status: err.status, + method: err.config?.method, + url: err.config?.url, + responseData: err.response?.data as unknown, + }; + } + if (err instanceof Error) { + return { + name: err.name, + message: err.message, + stack: err.stack, + ...(err.cause === undefined ? {} : { cause: toLoggableError(err.cause) }), + }; + } + return err; +}; + +export { logger, toLoggableError }; diff --git a/src/okf.ts b/src/okf.ts deleted file mode 100644 index 6481621..0000000 --- a/src/okf.ts +++ /dev/null @@ -1,421 +0,0 @@ -import { mkdtemp, readFile, readdir, rm, stat } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { basename, join } from 'node:path'; -import AdmZip from 'adm-zip'; -import { JSON_SCHEMA, load as loadYaml } from 'js-yaml'; -import { isValidEin } from './ein.js'; -import { logger } from './logger.js'; - -// Multi-valued frontmatter (e.g. `aliases`, `ntee`) is flattened into a single -// PDC string value with this separator, since a ChangemakerFieldValue.value is a -// scalar string. (The upstream OKF export emits a raw array; see docs/techsoup.md.) -const ARRAY_VALUE_SEPARATOR = '; '; -// A leading UTF-8 byte-order mark is not stripped by `readFile(..., 'utf8')`; -// remove it before matching frontmatter. -const BYTE_ORDER_MARK = 0xfeff; -// Isolate the YAML frontmatter block (between the leading and next `---`) so the -// markdown body that follows is never handed to the YAML parser. -const FRONTMATTER_PATTERN = /^---\r?\n(?[\s\S]*?)\r?\n---\s*(?:\r?\n|$)/v; - -/** - * A discovered OKF organization bundle ("package"): a directory whose - * `README.md` frontmatter declares `type: org`. `readme` holds that parsed - * frontmatter. - */ -interface OkfBundle { - slug: string; - path: string; - readme: Record; -} - -/** One base-field mapping: read `frontmatterPath` from `sourceFile`, write to `baseFieldShortCode`. */ -interface FieldMapEntry { - sourceFile: string; - frontmatterPath: string; - baseFieldShortCode: string; -} - -/** A resolved value ready to be posted as a PDC field, with a provenance note. */ -interface MappedField { - baseFieldShortCode: string; - value: string; - sourceNote: string; -} - -/** Inventory summary for a single bundle (produced by the `inventory` subcommand). */ -interface BundleSummary { - slug: string; - path: string; - ein: string | null; - goodAsOf: string | null; - organizationFields: MappedField[]; - proposalFields: MappedField[]; -} - -/** - * Organization-scoped OKF frontmatter → PDC base field short code. These are - * written as ChangemakerFieldValues, exactly like the Charity Navigator and - * GivingTuesday integrations. The mapping mirrors the original TechSoup OKF - * exporter (see docs/mappings/techsoup-okf-field-gap-analysis.verbose.md §2). - */ -const organizationFieldMap: FieldMapEntry[] = [ - { sourceFile: 'README.md', frontmatterPath: 'title', baseFieldShortCode: 'organization_name' }, - { sourceFile: 'README.md', frontmatterPath: 'description', baseFieldShortCode: 'organization_overview' }, - { sourceFile: 'README.md', frontmatterPath: 'aliases', baseFieldShortCode: 'organization_dba_name' }, - // NOTE: the OKF `status` is a *document* lifecycle value (stable/draft), not an - // organizational status; the gap analysis (§5.1) flags mapping it to - // `organization_status` as a data-quality bug. It is retained here to match the - // existing exporter output — revisit once the org status source is settled. - { sourceFile: 'README.md', frontmatterPath: 'status', baseFieldShortCode: 'organization_status' }, - { - sourceFile: 'README.md', - frontmatterPath: 'x-civic.registration_country', - baseFieldShortCode: 'organization_country', - }, - { sourceFile: 'README.md', frontmatterPath: 'x-civic.registration.id', baseFieldShortCode: 'organization_tax_id' }, - { sourceFile: 'README.md', frontmatterPath: 'x-civic.ntee', baseFieldShortCode: 'organization_ntee_code' }, - { sourceFile: 'README.md', frontmatterPath: 'x-civic.situation', baseFieldShortCode: 'organization_geographic_area' }, - { sourceFile: 'population.md', frontmatterPath: 'description', baseFieldShortCode: 'community_definition_overview' }, -]; - -/** - * Proposal-scoped OKF frontmatter → PDC base field short code. These are - * COLLECTED and reported but NOT written yet: PDC proposal data requires an - * Opportunity + ApplicationForm + ProposalVersion scaffold that is out of scope - * for this pass (see docs/techsoup.md, "Proposals"). The variable-named - * volunteer-request file contributes `proposal_learning_and_evalution` and is - * resolved dynamically in `collectProposalFields`. - */ -const proposalFieldMap: FieldMapEntry[] = [ - { sourceFile: 'impact.md', frontmatterPath: 'description', baseFieldShortCode: 'proposal_project_outcomes' }, - { sourceFile: 'programs.md', frontmatterPath: 'description', baseFieldShortCode: 'proposal_related_programs' }, - { sourceFile: 'what_i_need_funding_for.md', frontmatterPath: 'title', baseFieldShortCode: 'proposal_name' }, - { - sourceFile: 'what_i_need_funding_for.md', - frontmatterPath: 'description', - baseFieldShortCode: 'proposal_funding_summary', - }, -]; - -// --------------------------------------------------------------------------- -// OKF (YAML) frontmatter reading -// --------------------------------------------------------------------------- - -const stripBom = (text: string): string => (text.charCodeAt(0) === BYTE_ORDER_MARK ? text.slice(1) : text); - -/** - * Parse a markdown document's YAML frontmatter into an object (empty object if - * none, malformed, or not a mapping). Uses `JSON_SCHEMA` so that timestamp-like - * scalars (e.g. `generated.at`) stay strings rather than being coerced to Date. - */ -const parseFrontmatter = (content: string): Record => { - const body = FRONTMATTER_PATTERN.exec(stripBom(content))?.groups?.body; - if (body === undefined) { - return {}; - } - const parsed = ((): unknown => { - try { - return loadYaml(body, { schema: JSON_SCHEMA }); - } catch (error: unknown) { - logger.debug({ error }, 'Failed to parse OKF frontmatter YAML'); - return null; - } - })(); - if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { - return {}; - } - /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- js-yaml load returns unknown; a top-level YAML mapping is a string-keyed record */ - return parsed as Record; -}; - -/** Read a markdown file and return its parsed frontmatter, or null if unreadable/missing. */ -const parseFrontmatterFile = async (filePath: string): Promise | null> => { - try { - const content = await readFile(filePath, 'utf8'); - return parseFrontmatter(content); - } catch (error: unknown) { - logger.debug({ error, filePath }, 'Could not read frontmatter file'); - return null; - } -}; - -/** Follow a dotted path (e.g. `x-civic.registration.id`) into a parsed frontmatter object. */ -const getFrontmatterValue = (frontmatter: Record, dottedPath: string): unknown => { - let current: unknown = frontmatter; - for (const key of dottedPath.split('.')) { - if (current === null || typeof current !== 'object' || Array.isArray(current)) { - return undefined; - } - /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- narrowing a parsed-YAML node to a record to index it by key */ - const record = current as Record; - if (!(key in record)) { - return undefined; - } - const { [key]: next } = record; - current = next; - } - return current; -}; - -/** Coerce a parsed frontmatter value to a single trimmed PDC field string (arrays joined), or null. */ -const toFieldValueString = (value: unknown): string | null => { - if (value === null || value === undefined) { - return null; - } - if (typeof value === 'string') { - const trimmed = value.trim(); - return trimmed === '' ? null : trimmed; - } - if (typeof value === 'number' || typeof value === 'boolean') { - return String(value); - } - if (Array.isArray(value)) { - const parts = value.map((item) => toFieldValueString(item)).filter((item): item is string => item !== null); - return parts.length === 0 ? null : parts.join(ARRAY_VALUE_SEPARATOR); - } - return null; -}; - -/** - * Extract the organization's EIN from README frontmatter. Only returns a value - * when the registration scheme is IRS-EIN (or absent) and the id is a valid EIN; - * non-US bundles (KRS, PBO, ...) therefore yield null and are skipped downstream. - */ -const extractEin = (readmeFrontmatter: Record): string | null => { - const id = toFieldValueString(getFrontmatterValue(readmeFrontmatter, 'x-civic.registration.id')); - if (id === null) { - return null; - } - const scheme = toFieldValueString(getFrontmatterValue(readmeFrontmatter, 'x-civic.registration.scheme')); - if (scheme !== null && scheme.toUpperCase() !== 'IRS-EIN') { - return null; - } - return isValidEin(id) ? id : null; -}; - -/** The organization's display name (README `title`), used when creating a changemaker; null if absent. */ -const extractOrganizationName = (readmeFrontmatter: Record): string | null => - toFieldValueString(getFrontmatterValue(readmeFrontmatter, 'title')); - -/** Derive a `goodAsOf` ISO date (YYYY-MM-DD) from the bundle's `generated.at`, or null. */ -const extractGoodAsOf = (readmeFrontmatter: Record): string | null => { - const at = toFieldValueString(getFrontmatterValue(readmeFrontmatter, 'generated.at')); - if (at === null) { - return null; - } - const match = /^(?\d{4}-\d{2}-\d{2})/v.exec(at); - return match?.groups?.date ?? null; -}; - -// --------------------------------------------------------------------------- -// Source (zip / folder) inventory -// --------------------------------------------------------------------------- - -/** Extract a .zip into `destDir` using adm-zip (cross-platform, no shell dependency). */ -const extractZip = async (zipPath: string, destDir: string): Promise => { - const zip = new AdmZip(zipPath); - await zip.extractAllToAsync(destDir, true, false); -}; - -/** - * Resolve a `--source` argument to a directory tree to walk. Accepts a single - * .zip file, a folder of extracted OKF bundles, or a folder containing .zip - * files (each is extracted). Returns the root plus a cleanup for any temp dir. - * Whether the source holds one OKF package or several is determined afterwards - * by `findOrgBundles`. - */ -const prepareSourceRoot = async (source: string): Promise<{ root: string; cleanup: () => Promise }> => { - const stats = await stat(source); - const noop = async (): Promise => { - /* nothing to clean up when the source directory is walked in place */ - }; - if (stats.isFile()) { - if (!source.toLowerCase().endsWith('.zip')) { - throw new Error(`Source file is not a .zip: ${source}`); - } - const temp = await mkdtemp(join(tmpdir(), 'techsoup-')); - logger.info(`Extracting ${source} to ${temp}`); - await extractZip(source, temp); - return { - root: temp, - cleanup: async () => { - await rm(temp, { recursive: true, force: true }); - }, - }; - } - if (!stats.isDirectory()) { - throw new Error(`Source is neither a .zip file nor a directory: ${source}`); - } - const entries = await readdir(source, { withFileTypes: true }); - const zips = entries.filter((entry) => entry.isFile() && entry.name.toLowerCase().endsWith('.zip')); - if (zips.length === 0) { - return { root: source, cleanup: noop }; - } - const temp = await mkdtemp(join(tmpdir(), 'techsoup-')); - logger.info(`Found ${zips.length} zip file(s) in ${source}; extracting to ${temp}`); - for (const zip of zips) { - // eslint-disable-next-line no-await-in-loop -- extract archives sequentially to bound disk/CPU pressure - await extractZip(join(source, zip.name), join(temp, zip.name.replace(/\.zip$/iv, ''))); - } - return { - root: temp, - cleanup: async () => { - await rm(temp, { recursive: true, force: true }); - }, - }; -}; - -/** - * Walk `root` and return every OKF organization package (a directory whose - * README.md frontmatter declares `type: org`). This is what determines whether - * the source held a single package or many: a `root` that is itself a bundle - * yields one; otherwise every nested bundle is collected. Bundle subdirectories - * are not descended into. Results are sorted by slug for stable output. - */ -const findOrgBundles = async (root: string): Promise => { - const bundles: OkfBundle[] = []; - const walk = async (dir: string): Promise => { - const entries = await readdir(dir, { withFileTypes: true }); - const readmeEntry = entries.find((entry) => entry.isFile() && entry.name.toLowerCase() === 'readme.md'); - if (readmeEntry !== undefined) { - const frontmatter = await parseFrontmatterFile(join(dir, readmeEntry.name)); - if (frontmatter !== null && toFieldValueString(frontmatter.type) === 'org') { - bundles.push({ slug: basename(dir), path: dir, readme: frontmatter }); - return; - } - } - for (const entry of entries) { - if (entry.isDirectory()) { - // eslint-disable-next-line no-await-in-loop -- sequential recursion bounds open file handles - await walk(join(dir, entry.name)); - } - } - }; - await walk(root); - return bundles.sort((a, b) => a.slug.localeCompare(b.slug)); -}; - -/** Report whether the source resolved to a single OKF package or several (all processed one at a time). */ -const describeBundleCount = (bundles: OkfBundle[], source: string): void => { - if (bundles.length === 0) { - logger.warn(`No OKF organization packages found under ${source}`); - } else if (bundles.length === 1) { - logger.info(`Detected a single OKF package under ${source}: ${bundles[0]?.slug ?? '(unknown)'}`); - } else { - logger.info(`Detected ${bundles.length} OKF packages under ${source}; processing one at a time`); - } -}; - -/** Read each mapping's source file frontmatter and resolve the mapped, non-empty values. */ -const collectFieldsFromMap = async (bundlePath: string, fieldMap: FieldMapEntry[]): Promise => { - const cache = new Map | null>(); - const result: MappedField[] = []; - /* eslint-disable no-await-in-loop -- files are read on demand and memoized in `cache` */ - for (const entry of fieldMap) { - let frontmatter = cache.get(entry.sourceFile); - if (frontmatter === undefined) { - frontmatter = await parseFrontmatterFile(join(bundlePath, entry.sourceFile)); - cache.set(entry.sourceFile, frontmatter); - } - if (frontmatter === null) { - continue; - } - const value = toFieldValueString(getFrontmatterValue(frontmatter, entry.frontmatterPath)); - if (value === null) { - continue; - } - result.push({ - baseFieldShortCode: entry.baseFieldShortCode, - value, - sourceNote: `${basename(bundlePath)}/${entry.sourceFile}#${entry.frontmatterPath}`, - }); - } - /* eslint-enable no-await-in-loop */ - return result; -}; - -/** Locate the (variably named) volunteer-request markdown file within a bundle, relative to it. */ -const resolveVolunteerRequestFile = async (bundlePath: string): Promise => { - const dir = join(bundlePath, 'technical-volunteers'); - const entries = await readdir(dir, { withFileTypes: true }).catch(() => null); - if (entries === null) { - return null; - } - /* eslint-disable no-await-in-loop -- scan candidate files sequentially until the request file is found */ - for (const entry of entries) { - if ( - !entry.isFile() || - !entry.name.endsWith('.md') || - entry.name === 'index.md' || - entry.name === 'constraints.md' - ) { - continue; - } - const frontmatter = await parseFrontmatterFile(join(dir, entry.name)); - if (frontmatter !== null && toFieldValueString(frontmatter.type) === 'volunteer-request') { - // Forward slash (not path.join) so the provenance note matches the OKF - // `file#anchor` convention; join() below accepts it on all platforms. - return `technical-volunteers/${entry.name}`; - } - } - /* eslint-enable no-await-in-loop */ - return null; -}; - -/** Collect proposal-scoped fields (fixed files plus the dynamic volunteer-request file). */ -const collectProposalFields = async (bundlePath: string): Promise => { - const fields = await collectFieldsFromMap(bundlePath, proposalFieldMap); - const volunteerFile = await resolveVolunteerRequestFile(bundlePath); - if (volunteerFile !== null) { - const frontmatter = await parseFrontmatterFile(join(bundlePath, volunteerFile)); - if (frontmatter !== null) { - const value = toFieldValueString(getFrontmatterValue(frontmatter, 'description')); - if (value !== null) { - fields.push({ - baseFieldShortCode: 'proposal_learning_and_evalution', - value, - sourceNote: `${basename(bundlePath)}/${volunteerFile}#description`, - }); - } - } - } - return fields; -}; - -const summarizeBundles = async (bundles: OkfBundle[]): Promise => { - const summaries: BundleSummary[] = []; - /* eslint-disable no-await-in-loop -- bundles are summarized sequentially to bound concurrent file reads */ - for (const bundle of bundles) { - const organizationFields = await collectFieldsFromMap(bundle.path, organizationFieldMap); - const proposalFields = await collectProposalFields(bundle.path); - summaries.push({ - slug: bundle.slug, - path: bundle.path, - ein: extractEin(bundle.readme), - goodAsOf: extractGoodAsOf(bundle.readme), - organizationFields, - proposalFields, - }); - } - /* eslint-enable no-await-in-loop */ - return summaries; -}; - -export { - type BundleSummary, - type MappedField, - type OkfBundle, - collectFieldsFromMap, - collectProposalFields, - describeBundleCount, - extractEin, - extractGoodAsOf, - extractOrganizationName, - findOrgBundles, - getFrontmatterValue, - organizationFieldMap, - parseFrontmatter, - prepareSourceRoot, - summarizeBundles, - toFieldValueString, -}; diff --git a/src/pdc-api.ts b/src/pdc-api.ts index 027a58c..20734c0 100644 --- a/src/pdc-api.ts +++ b/src/pdc-api.ts @@ -28,7 +28,8 @@ const callPdcApi = async ( return response.data; }; -const getBaseFields = async (baseUrl: string, token: AccessTokenSet): Promise => +/** Get all base fields. `GET /baseFields` is public, so the token is optional. */ +const getBaseFields = async (baseUrl: string, token?: AccessTokenSet): Promise => await callPdcApi(baseUrl, '/baseFields', 'get', { params: { _page: '1', diff --git a/src/pdc-json.ts b/src/pdc-json.ts index 47c8878..7edaa91 100644 --- a/src/pdc-json.ts +++ b/src/pdc-json.ts @@ -1,8 +1,7 @@ -import { readFile, readdir } from 'node:fs/promises'; -import { basename, join } from 'node:path'; +import { readFile } from 'node:fs/promises'; +import { basename } from 'node:path'; import { isValidEin } from './ein.js'; -import { type MappedField, type TechSoupPackage, isProposalField, toFieldValueString } from './fields.js'; -import { logger } from './logger.js'; +import { type MappedField, type TechSoupDocument, toFieldValueString } from './fields.js'; const isRecord = (value: unknown): value is Record => value !== null && typeof value === 'object' && !Array.isArray(value); @@ -41,48 +40,29 @@ const toMappedField = (field: unknown): MappedField | null => { }; }; -interface ParsedFields { - organizationFields: MappedField[]; - proposalFields: MappedField[]; - taxIdValue: string | null; -} - -/** Split a `_pdc.json` `fields[]` array into organization/proposal `MappedField`s and capture the tax id. */ -const parsePdcJsonFields = (raw: unknown): ParsedFields => { - const organizationFields: MappedField[] = []; - const proposalFields: MappedField[] = []; - if (isRecord(raw) && Array.isArray(raw.fields)) { - for (const field of raw.fields) { - const mapped = toMappedField(field); - if (mapped === null) { - continue; - } - if (isProposalField(mapped.baseFieldShortCode)) { - proposalFields.push(mapped); - } else { - organizationFields.push(mapped); - } - } +/** The usable entries of a `_pdc.json` `fields[]` array (malformed or empty ones are dropped). */ +const parsePdcJsonFields = (raw: unknown): MappedField[] => { + if (!isRecord(raw) || !Array.isArray(raw.fields)) { + return []; } - const taxIdValue = organizationFields.find((f) => f.baseFieldShortCode === 'organization_tax_id')?.value ?? null; - return { organizationFields, proposalFields, taxIdValue }; + return raw.fields.map(toMappedField).filter((field): field is MappedField => field !== null); }; /** - * Convert one parsed `_pdc.json` document into a normalized package. The EIN is + * Convert one parsed `_pdc.json` document into a `TechSoupDocument`. The EIN is * taken from the `organization_tax_id` field and only kept when it is a valid - * US EIN (so non-US ids like Poland's `0000000000` yield null, as with OKF). + * US EIN (so non-US ids like Poland's `0000000000` yield null). * `goodAsOf` is always null — the exporter output carries no generation date. */ -const pdcJsonToPackage = (raw: unknown, slug: string): TechSoupPackage => { - const { organizationFields, proposalFields, taxIdValue } = parsePdcJsonFields(raw); +const pdcJsonToDocument = (raw: unknown, slug: string): TechSoupDocument => { + const fields = parsePdcJsonFields(raw); + const taxIdValue = fields.find((f) => f.baseFieldShortCode === 'organization_tax_id')?.value ?? null; return { slug, ein: taxIdValue !== null && isValidEin(taxIdValue) ? taxIdValue : null, name: isRecord(raw) ? extractChangemakerName(raw) : null, goodAsOf: null, - organizationFields, - proposalFields, + fields, }; }; @@ -92,76 +72,22 @@ const slugForJsonFile = (filePath: string): string => .replace(/_pdc\.json$/iv, '') .replace(/\.json$/iv, ''); -/** Recursively collect every `.json` file under `root`. */ -const findJsonFiles = async (root: string): Promise => { - const files: string[] = []; - const walk = async (dir: string): Promise => { - const entries = await readdir(dir, { withFileTypes: true }).catch(() => null); - if (entries === null) { - return; - } - for (const entry of entries) { - const full = join(dir, entry.name); - if (entry.isDirectory()) { - // eslint-disable-next-line no-await-in-loop -- sequential recursion bounds open file handles - await walk(full); - } else if (entry.isFile() && entry.name.toLowerCase().endsWith('.json')) { - files.push(full); - } - } - }; - await walk(root); - return files; -}; - -/** Read one JSON file, returning its parsed content or null when unreadable/not JSON. */ -const readJsonFile = async (filePath: string): Promise => { - try { - const content = await readFile(filePath, 'utf8'); - return JSON.parse(content); - } catch (error: unknown) { - logger.debug({ error, filePath }, 'Could not read/parse JSON file'); - return null; - } -}; - -/** - * Read every TechSoup exporter `_pdc.json` document under `root` and return the - * normalized packages. `.json` files that are not PDC-export shaped (no `fields` - * array) are skipped, so unrelated JSON (package.json, config, …) is ignored. - * Results are sorted by slug for stable output. - */ -const readPdcJsonPackages = async (root: string): Promise => { - const files = await findJsonFiles(root); - const packages: TechSoupPackage[] = []; - /* eslint-disable no-await-in-loop -- sequential reads bound concurrent file handles */ - for (const file of files) { - const raw = await readJsonFile(file); - if (!isRecord(raw) || !Array.isArray(raw.fields)) { - continue; - } - packages.push(pdcJsonToPackage(raw, slugForJsonFile(file))); - } - /* eslint-enable no-await-in-loop */ - return packages.sort((a, b) => a.slug.localeCompare(b.slug)); -}; - /** - * Validate one parsed JSON document as a single PDC-export package and normalize - * it. Throws when the document is not `{ changemaker?, fields: [...] }` shaped, - * so a wrong file fails loudly instead of importing nothing. + * Validate one parsed JSON value as a TechSoup PDC JSON document and normalize + * it. Throws when it is not `{ changemaker?, fields: [...] }` shaped, so a wrong + * file fails loudly instead of importing nothing. */ -const parsePdcJsonPackage = (raw: unknown, slug: string): TechSoupPackage => { +const parsePdcJsonDocument = (raw: unknown, slug: string): TechSoupDocument => { if (!isRecord(raw) || !Array.isArray(raw.fields)) { throw new Error(`${slug} is not a TechSoup PDC JSON document (expected an object with a "fields" array)`); } - return pdcJsonToPackage(raw, slug); + return pdcJsonToDocument(raw, slug); }; /** Read and normalize the single TechSoup PDC JSON file at `filePath` (the import's only input). */ -const readPdcJsonFile = async (filePath: string): Promise => { +const readPdcJsonFile = async (filePath: string): Promise => { const content = await readFile(filePath, 'utf8'); - return parsePdcJsonPackage(JSON.parse(content), slugForJsonFile(filePath)); + return parsePdcJsonDocument(JSON.parse(content), slugForJsonFile(filePath)); }; -export { parsePdcJsonPackage, pdcJsonToPackage, readPdcJsonFile, readPdcJsonPackages }; +export { parsePdcJsonDocument, pdcJsonToDocument, readPdcJsonFile }; diff --git a/src/source.ts b/src/source.ts deleted file mode 100644 index 026e478..0000000 --- a/src/source.ts +++ /dev/null @@ -1,106 +0,0 @@ -// DISABLED: not wired into the `techsoup` command, which now imports a single -// PDC JSON file (see techsoup.ts). Kept for a future step that turns OKF -// markdown bundles (zip/folder) into that single JSON file. -import { mkdtemp, readdir, rm, stat } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import AdmZip from 'adm-zip'; -import { logger } from './logger.js'; -import { readOkfBundlePackages } from './okf.js'; -import { readPdcJsonPackages } from './pdc-json.js'; -import type { TechSoupPackage } from './fields.js'; - -/** How `--source` is interpreted: auto-detect, raw OKF bundles, or exporter `_pdc.json` files. */ -type SourceFormat = 'auto' | 'okf' | 'json'; - -/** Narrow a raw CLI string (constrained by yargs `choices`) to a `SourceFormat`. */ -const asSourceFormat = (value: string): SourceFormat => (value === 'okf' || value === 'json' ? value : 'auto'); - -/** Extract a .zip into `destDir` using adm-zip (cross-platform, no shell dependency). */ -const extractZip = async (zipPath: string, destDir: string): Promise => { - const zip = new AdmZip(zipPath); - await zip.extractAllToAsync(destDir, true, false); -}; - -/** - * Resolve a `--source` argument (a `.zip` file, a folder, or a folder containing - * `.zip` files) to a single directory tree to read. Returns the root plus a - * cleanup for any temp dir created for extraction. The resolved root is read by - * either the OKF or the `_pdc.json` reader depending on `--source-format`. - */ -const prepareSourceRoot = async (source: string): Promise<{ root: string; cleanup: () => Promise }> => { - const stats = await stat(source); - const noop = async (): Promise => { - /* nothing to clean up when the source directory is read in place */ - }; - if (stats.isFile()) { - if (!source.toLowerCase().endsWith('.zip')) { - throw new Error(`Source file is not a .zip (pass a .zip or a folder): ${source}`); - } - const temp = await mkdtemp(join(tmpdir(), 'techsoup-')); - logger.info(`Extracting ${source} to ${temp}`); - await extractZip(source, temp); - return { - root: temp, - cleanup: async () => { - await rm(temp, { recursive: true, force: true }); - }, - }; - } - if (!stats.isDirectory()) { - throw new Error(`Source is neither a .zip file nor a directory: ${source}`); - } - const entries = await readdir(source, { withFileTypes: true }); - const zips = entries.filter((entry) => entry.isFile() && entry.name.toLowerCase().endsWith('.zip')); - if (zips.length === 0) { - return { root: source, cleanup: noop }; - } - const temp = await mkdtemp(join(tmpdir(), 'techsoup-')); - logger.info(`Found ${zips.length} zip file(s) in ${source}; extracting to ${temp}`); - for (const zip of zips) { - // eslint-disable-next-line no-await-in-loop -- extract archives sequentially to bound disk/CPU pressure - await extractZip(join(source, zip.name), join(temp, zip.name.replace(/\.zip$/iv, ''))); - } - return { - root: temp, - cleanup: async () => { - await rm(temp, { recursive: true, force: true }); - }, - }; -}; - -/** - * Read the packages under `root` in the requested format. `auto` prefers raw OKF - * bundles and falls back to `_pdc.json` files when none are found. Returns the - * packages plus the format actually used (for logging). - */ -const readPackages = async ( - root: string, - format: SourceFormat, -): Promise<{ packages: TechSoupPackage[]; format: 'okf' | 'json' }> => { - if (format === 'json') { - return { packages: await readPdcJsonPackages(root), format: 'json' }; - } - if (format === 'okf') { - return { packages: await readOkfBundlePackages(root), format: 'okf' }; - } - const okfPackages = await readOkfBundlePackages(root); - if (okfPackages.length > 0) { - return { packages: okfPackages, format: 'okf' }; - } - return { packages: await readPdcJsonPackages(root), format: 'json' }; -}; - -/** Report whether the source resolved to a single package or several (all processed one at a time). */ -const describePackageCount = (packages: TechSoupPackage[], source: string, format: 'okf' | 'json'): void => { - const label = format === 'okf' ? 'OKF' : 'PDC-JSON'; - if (packages.length === 0) { - logger.warn(`No ${label} packages found under ${source}`); - } else if (packages.length === 1) { - logger.info(`Detected a single ${label} package under ${source}: ${packages[0]?.slug ?? '(unknown)'}`); - } else { - logger.info(`Detected ${packages.length} ${label} packages under ${source}; processing one at a time`); - } -}; - -export { type SourceFormat, asSourceFormat, describePackageCount, prepareSourceRoot, readPackages }; diff --git a/src/techsoup.ts b/src/techsoup.ts index c035c0d..c1646a6 100644 --- a/src/techsoup.ts +++ b/src/techsoup.ts @@ -1,234 +1,78 @@ -import { writeFile } from 'node:fs/promises'; -import { getChangemakerByEin, runUpdateAllDryRun, runUpdateAllWrite, selectChangemakers } from './changemakers.js'; -import { logger } from './logger.js'; import { - type BundleSummary, - describeBundleCount, - extractEin, - findOrgBundles, - prepareSourceRoot, - summarizeBundles, -} from './okf.js'; + applyBaseFieldCategories, + formatImportPreview, + planImport, + runImportWrite, + toChangemakerTarget, +} from './changemakers.js'; +import { logger } from './logger.js'; import { oidcOptions } from './oidc.js'; -import { getChangemakers } from './pdc-api.js'; +import { getBaseFields, getChangemakers } from './pdc-api.js'; +import { readPdcJsonFile } from './pdc-json.js'; import type { CommandModule } from 'yargs'; -const JSON_SPACES = 2; - -/** - * Print the full field-by-field mapping for one package to the console, showing - * exactly which changemaker field values WOULD be written and which proposal - * fields are collected-but-not-written. Reads only local files — nothing is sent - * to (or read from) the PDC. - */ -const logMappedFields = (summary: BundleSummary): void => { - logger.info( - { ein: summary.ein, goodAsOf: summary.goodAsOf }, - `${summary.slug} — mapping preview (nothing sent to PDC; changemaker match happens only at run time)`, - ); - for (const field of summary.organizationFields) { - logger.info( - ` WOULD POST changemakerFieldValue: ${field.baseFieldShortCode} = ${JSON.stringify(field.value)} (goodAsOf=${summary.goodAsOf ?? 'null'}) [${field.sourceNote}]`, - ); - } - for (const field of summary.proposalFields) { - logger.info( - ` proposal field (collected, NOT written): ${field.baseFieldShortCode} = ${JSON.stringify(field.value)} [${field.sourceNote}]`, - ); - } -}; - -// --------------------------------------------------------------------------- -// CLI command modules -// --------------------------------------------------------------------------- - -interface InventoryCommandArgs { - source: string; - outputFile?: string; - 'show-fields': boolean; -} - -interface LookupFromPdcCommandArgs { - source: string; - 'pdc-api-base-url': string; - outputFile?: string; -} +// The importer reads exactly one TechSoup PDC JSON file per run. -interface UpdateAllCommandArgs { - source: string; +interface ImportCommandArgs { + file: string; 'pdc-api-base-url': string; + 'create-changemaker'?: boolean; 'changemaker-id'?: number; 'dry-run': boolean; - 'add-changemakers': boolean; 'oidc-base-url': string; 'oidc-client-id': string; 'oidc-client-secret': string; } -const SOURCE_DESCRIBE = - 'Path to a TechSoup OKF .zip file, a folder of extracted OKF packages, or a folder of .zip files'; - -const inventoryCommand: CommandModule = { - command: 'inventory', - describe: 'Examine a TechSoup OKF zip/folder and inventory the available packages and mapped fields', - builder: (y) => - y - .option('source', { describe: SOURCE_DESCRIBE, demandOption: true, normalize: true, type: 'string' }) - .option('output-file', { - alias: 'write', - describe: 'Write the inventory to the specified JSON file', - normalize: true, - type: 'string', - }) - .option('show-fields', { - alias: 'fields', - describe: 'Print the full field-by-field mapping that would be sent to PDC (offline; nothing is sent)', - demandOption: false, - default: false, - type: 'boolean', - }), - handler: async (args): Promise => { - const prepared = await prepareSourceRoot(args.source); - try { - const bundles = await findOrgBundles(prepared.root); - describeBundleCount(bundles, args.source); - const summaries = await summarizeBundles(bundles); - if (args.outputFile === undefined || args.outputFile === '') { - for (const summary of summaries) { - if (args.showFields) { - logMappedFields(summary); - } else { - logger.info( - { - ein: summary.ein, - goodAsOf: summary.goodAsOf, - organizationFields: summary.organizationFields.length, - proposalFields: summary.proposalFields.length, - }, - summary.slug, - ); - } - } - } else { - await writeFile(args.outputFile, JSON.stringify(summaries, null, JSON_SPACES)); - logger.info(`Wrote TechSoup inventory (${summaries.length} package(s)) to ${JSON.stringify(args.outputFile)}`); - } - } finally { - await prepared.cleanup(); - } - }, -}; - -const lookupFromPdcCommand: CommandModule = { - command: 'lookupFromPdc', - describe: 'Match TechSoup package EINs against PDC changemakers and report which are present (read-only)', - builder: (y) => - y - .option('source', { describe: SOURCE_DESCRIBE, demandOption: true, normalize: true, type: 'string' }) - .option('pdc-api-base-url', { describe: 'Location of PDC API', demandOption: true, type: 'string' }) - .option('output-file', { - alias: 'write', - describe: 'Write the match report to the specified JSON file', - normalize: true, - type: 'string', - }), - handler: async (args): Promise => { - const prepared = await prepareSourceRoot(args.source); - try { - const bundles = await findOrgBundles(prepared.root); - describeBundleCount(bundles, args.source); - const changemakers = await getChangemakers(args.pdcApiBaseUrl); - const rows = bundles.map((bundle) => { - const ein = extractEin(bundle.readme); - const changemaker = ein === null ? null : getChangemakerByEin(ein, changemakers); - return { slug: bundle.slug, ein, changemakerId: changemaker?.id ?? null, presentInPdc: changemaker !== null }; - }); - const { length: presentCount } = rows.filter((row) => row.presentInPdc); - if (args.outputFile === undefined || args.outputFile === '') { - logger.info({ total: rows.length, present: presentCount }, 'TechSoup packages matched against PDC'); - for (const row of rows) { - logger.info(row, row.slug); - } - } else { - await writeFile(args.outputFile, JSON.stringify(rows, null, JSON_SPACES)); - logger.info( - `Wrote TechSoup PDC match report (${rows.length} package(s)) to ${JSON.stringify(args.outputFile)}`, - ); - } - } finally { - await prepared.cleanup(); - } - }, -}; - -const updateAllCommand: CommandModule = { - command: 'updateAll', - describe: 'Load organization fields from a TechSoup OKF zip/folder into PDC for each matching changemaker', +const techsoup: CommandModule = { + command: 'techsoup ', + describe: 'Import one TechSoup PDC JSON file into a PDC changemaker', builder: { ...oidcOptions, - source: { describe: SOURCE_DESCRIBE, demandOption: true, normalize: true, type: 'string' }, + // Filled from the `` positional in `command`. + file: { describe: 'Path to the TechSoup PDC JSON file to import', normalize: true, type: 'string' }, 'pdc-api-base-url': { describe: 'Location of PDC API', demandOption: true, type: 'string' }, + 'create-changemaker': { + describe: 'Create a new changemaker from the file (EIN + name); fails if the EIN already exists in the PDC', + // No `default`: yargs counts a defaulted flag as "set" and would trip `conflicts`. + type: 'boolean', + conflicts: 'changemaker-id', + }, 'changemaker-id': { - describe: 'Only load the single PDC changemaker with this ID (default: all changemakers in the PDC)', + describe: 'Load into the existing changemaker with this ID (default: match the file EIN)', demandOption: false, type: 'number', }, 'dry-run': { - describe: 'Report what would be written without contacting the PDC for authentication or writes', - demandOption: false, - default: false, - type: 'boolean', - }, - 'add-changemakers': { - describe: - 'Create a new changemaker when a package EIN has no match in the PDC (default: skip unmatched packages). Ignored when --changemaker-id is set.', - demandOption: false, + describe: 'Print the target changemaker and the fields that would be imported; write nothing', default: false, type: 'boolean', }, }, handler: async (args): Promise => { - const prepared = await prepareSourceRoot(args.source); - try { - const bundles = await findOrgBundles(prepared.root); - describeBundleCount(bundles, args.source); - if (bundles.length === 0) { - return; - } - const allChangemakers = await getChangemakers(args.pdcApiBaseUrl); - const changemakers = selectChangemakers(allChangemakers, args.changemakerId); - // --changemaker-id targets one existing changemaker, which is incompatible - // with creating missing ones; when both are given, honor the scope and - // disable creation (decision 3). - const addChangemakers = args.addChangemakers && args.changemakerId === undefined; - if (args.addChangemakers && args.changemakerId !== undefined) { - logger.warn('--add-changemakers is ignored because --changemaker-id targets a single existing changemaker.'); - } - // With --add-changemakers we can proceed even against an empty scope (we - // create as we go); otherwise there is nothing to match and we stop. - if (changemakers.entries.length === 0 && !addChangemakers) { - logger.warn({ changemakerId: args.changemakerId }, 'No matching changemakers found in PDC; nothing to load.'); - return; - } + // Split fields by the PDC's base-field categories: only `organization` + // fields can be posted to a changemaker. + const pkg = applyBaseFieldCategories(await readPdcJsonFile(args.file), await getBaseFields(args.pdcApiBaseUrl)); + const changemakers = await getChangemakers(args.pdcApiBaseUrl); + const plan = planImport( + pkg, + changemakers, + toChangemakerTarget(args.createChangemaker === true, args.changemakerId), + ); - if (args.dryRun) { - await runUpdateAllDryRun(bundles, changemakers, addChangemakers); - return; + if (args.dryRun) { + logger.info('[dry-run] No data will be written to the PDC.'); + for (const line of formatImportPreview(pkg, plan)) { + logger.info(line); } + return; + } - await runUpdateAllWrite(bundles, changemakers, addChangemakers, args); - } finally { - await prepared.cleanup(); + if (plan.action === 'skip') { + throw new Error(`Not importing ${args.file}: ${plan.reason}`); } + await runImportWrite(pkg, plan, args); }, }; -const techsoup: CommandModule = { - command: 'techsoup', - describe: 'Load TechSoup OKF package data (from a zip or folder) into the PDC', - builder: (y) => y.command(inventoryCommand).command(lookupFromPdcCommand).command(updateAllCommand).demandCommand(1), - /* eslint-disable-next-line @typescript-eslint/no-empty-function -- yargs demandCommand handles routing to subcommands */ - handler: () => {}, -}; - export { techsoup }; diff --git a/src/techsoup.unit.test.ts b/src/techsoup.unit.test.ts index 01f266a..a5d913f 100644 --- a/src/techsoup.unit.test.ts +++ b/src/techsoup.unit.test.ts @@ -1,14 +1,21 @@ -import { describe, expect, it } from '@jest/globals'; -import { getChangemakerByEin, resolveChangemaker, selectChangemakers } from './changemakers.js'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from '@jest/globals'; +import { AxiosError, AxiosHeaders } from 'axios'; import { - extractEin, - extractGoodAsOf, - extractOrganizationName, - getFrontmatterValue, - parseFrontmatter, - toFieldValueString, -} from './okf.js'; -import type { Changemaker, ChangemakerBundle } from '@pdc/sdk'; + applyBaseFieldCategories, + formatImportPreview, + planImport, + resolveChangemaker, + selectTechSoupSource, + toChangemakerTarget, +} from './changemakers.js'; +import { toFieldValueString } from './fields.js'; +import { toLoggableError } from './logger.js'; +import { parsePdcJsonDocument, pdcJsonToDocument, readPdcJsonFile } from './pdc-json.js'; +import type { TechSoupDocument, TechSoupPackage } from './fields.js'; +import type { Changemaker, ChangemakerBundle, Source } from '@pdc/sdk'; // Minimal Changemaker fixture; the helpers under test only read `id`/`taxId`, // but the SDK type requires the full shape, so fill the rest with inert defaults. @@ -22,97 +29,25 @@ const makeChangemaker = (id: number, taxId: string): Changemaker => ({ fields: [], }); -// A trimmed-down but structurally faithful OKF README frontmatter (nested block -// map, flow sequences, an inline flow map, a block sequence of mappings, and a -// trailing comment) to exercise the whole minimal parser. -const README_FRONTMATTER = `--- -type: org -title: "synthetic-Black Mountain Workforce Partnership" -description: "A fabricated post-coal workforce training organization." -aliases: ["synthetic-Black Mountain Workforce", "Black Mountain Workforce Partnership"] -synthetic: true -status: stable -generated: { by: claude-code/claude-opus-5, at: 2026-07-29T00:00:00Z } -sources: - - id: org-site - resource: "https://example.org" - last_modified: 2026-03-02 - - id: registry - resource: "simulated registry extract" - last_modified: 2026-01-15 -x-civic: - # ---- REQUIRED by civic/0.6. ---- - profile: civic/0.6 - registration_country: US - registration: - scheme: "IRS-EIN" - id: "00-1000008" - tax_status: "501(c)(3)" - ntee: ["J22"] # optional US-only layer - situation: US-KY-letcher - verifiable_by: [techsoup] ---- - -# Body content that must be ignored by the parser. -`; - -describe('parseFrontmatter', () => { - const fm = parseFrontmatter(README_FRONTMATTER); - - it('parses top-level scalars', () => { - expect(fm.type).toBe('org'); - expect(fm.title).toBe('synthetic-Black Mountain Workforce Partnership'); - expect(fm.status).toBe('stable'); - }); - - it('parses a flow sequence of quoted strings', () => { - expect(fm.aliases).toStrictEqual(['synthetic-Black Mountain Workforce', 'Black Mountain Workforce Partnership']); - }); - - it('parses an inline flow mapping', () => { - expect(fm.generated).toStrictEqual({ by: 'claude-code/claude-opus-5', at: '2026-07-29T00:00:00Z' }); - }); - - it('parses a nested block mapping under x-civic', () => { - expect(getFrontmatterValue(fm, 'x-civic.registration.scheme')).toBe('IRS-EIN'); - expect(getFrontmatterValue(fm, 'x-civic.registration.id')).toBe('00-1000008'); - expect(getFrontmatterValue(fm, 'x-civic.registration_country')).toBe('US'); - expect(getFrontmatterValue(fm, 'x-civic.situation')).toBe('US-KY-letcher'); - }); - - it('consumes a block sequence of mappings without disturbing later keys', () => { - expect(Array.isArray(fm.sources)).toBe(true); - expect(getFrontmatterValue(fm, 'x-civic.ntee')).toStrictEqual(['J22']); - expect(getFrontmatterValue(fm, 'x-civic.verifiable_by')).toStrictEqual(['techsoup']); - }); - - it('strips a trailing comment from a value', () => { - expect(getFrontmatterValue(fm, 'x-civic.ntee')).toStrictEqual(['J22']); - }); - - it('returns an empty object when there is no frontmatter', () => { - expect(parseFrontmatter('# Just a heading\n\nSome text.')).toStrictEqual({}); - }); -}); - -describe('getFrontmatterValue', () => { - const fm = { a: { b: { c: 'deep' } }, list: [1, 2] }; - - it('follows a dotted path', () => { - expect(getFrontmatterValue(fm, 'a.b.c')).toBe('deep'); - }); - - it('returns undefined for a missing key', () => { - expect(getFrontmatterValue(fm, 'a.b.x')).toBeUndefined(); - }); - - it('returns undefined when descending into a non-object', () => { - expect(getFrontmatterValue(fm, 'a.b.c.d')).toBeUndefined(); - }); - - it('returns undefined when descending into an array', () => { - expect(getFrontmatterValue(fm, 'list.0')).toBeUndefined(); - }); +// Minimal Source fixture; `selectTechSoupSource` only reads `id` and +// `dataProviderShortCode`, the rest are inert defaults the SDK type requires. +const makeSource = (id: number, dataProviderShortCode: string): Source => ({ + id, + label: `Source ${id}`, + createdAt: '2024-01-01T00:00:00Z', + createdBy: 'test', + dataProviderShortCode, + dataProvider: { + shortCode: dataProviderShortCode, + name: dataProviderShortCode, + keycloakOrganizationId: '', + createdAt: '2024-01-01T00:00:00Z', + createdBy: 'test', + }, + funderShortCode: '', + funder: { shortCode: '', name: '', createdAt: '2024-01-01T00:00:00Z', createdBy: 'test', isCollaborative: false }, + changemakerId: 0, + changemaker: makeChangemaker(0, ''), }); describe('toFieldValueString', () => { @@ -147,127 +82,404 @@ describe('toFieldValueString', () => { }); }); -describe('extractEin', () => { - it('returns a valid EIN when the scheme is IRS-EIN', () => { - expect(extractEin({ 'x-civic': { registration: { scheme: 'IRS-EIN', id: '00-1000008' } } })).toBe('00-1000008'); +describe('resolveChangemaker', () => { + const bundle: ChangemakerBundle = { + entries: [makeChangemaker(1, '00-1000008'), makeChangemaker(2, '22-3456789')], + total: 2, + }; + + it('resolves a unique match (hyphen-insensitive)', () => { + const resolution = resolveChangemaker('001000008', bundle); + expect(resolution.kind).toBe('matched'); + expect(resolution.kind === 'matched' ? resolution.changemaker.id : null).toBe(1); }); - it('returns the EIN when no scheme is declared but the id is a valid EIN', () => { - expect(extractEin({ 'x-civic': { registration: { id: '12-3456789' } } })).toBe('12-3456789'); + it('resolves to missing when there is no match', () => { + expect(resolveChangemaker('99-9999999', bundle).kind).toBe('missing'); }); - it('returns null for a non-IRS scheme (e.g. Polish KRS)', () => { - expect(extractEin({ 'x-civic': { registration: { scheme: 'KRS', id: '0000000000' } } })).toBeNull(); + it('resolves to ambiguous when more than one changemaker matches', () => { + const ambiguous: ChangemakerBundle = { + entries: [makeChangemaker(1, '00-1000008'), makeChangemaker(2, '001000008')], + total: 2, + }; + expect(resolveChangemaker('00-1000008', ambiguous).kind).toBe('ambiguous'); + }); +}); + +describe('toChangemakerTarget', () => { + it('targets a new changemaker when --create-changemaker is set', () => { + expect(toChangemakerTarget(true, undefined)).toStrictEqual({ kind: 'create' }); }); - it('returns null when the id is not a valid EIN', () => { - expect(extractEin({ 'x-civic': { registration: { scheme: 'IRS-EIN', id: 'not-an-ein' } } })).toBeNull(); + it('targets the explicit ID when --changemaker-id is given', () => { + expect(toChangemakerTarget(false, 7)).toStrictEqual({ kind: 'id', changemakerId: 7 }); }); - it('returns null when there is no registration id', () => { - expect(extractEin({ 'x-civic': { registration: {} } })).toBeNull(); + it('falls back to EIN matching when neither flag is given', () => { + expect(toChangemakerTarget(false, undefined)).toStrictEqual({ kind: 'ein' }); }); }); -describe('extractGoodAsOf', () => { - it('extracts the date from a generated.at timestamp', () => { - expect(extractGoodAsOf({ generated: { at: '2026-07-29T00:00:00Z' } })).toBe('2026-07-29'); +// A normalized package as read from a single PDC JSON file. +const makePackage = (overrides: Partial = {}): TechSoupPackage => ({ + slug: 'synthetic-org', + ein: '00-1000008', + name: 'Synthetic Org', + goodAsOf: null, + organizationFields: [ + { baseFieldShortCode: 'organization_name', value: 'Synthetic Org', sourceNote: 'README.md#title' }, + { baseFieldShortCode: 'organization_tax_id', value: '00-1000008', sourceNote: 'organization_tax_id' }, + ], + proposalFields: [], + ...overrides, +}); + +describe('planImport', () => { + const bundle: ChangemakerBundle = { + entries: [makeChangemaker(1, '001000008'), makeChangemaker(2, '22-3456789')], + total: 2, + }; + const ambiguous: ChangemakerBundle = { + entries: [makeChangemaker(1, '00-1000008'), makeChangemaker(2, '001000008')], + total: 2, + }; + const empty: ChangemakerBundle = { entries: [], total: 0 }; + + describe('matching by EIN (default)', () => { + it('updates the changemaker whose taxId matches the file EIN', () => { + const plan = planImport(makePackage(), bundle, { kind: 'ein' }); + expect(plan.action).toBe('update'); + expect(plan.action === 'update' ? plan.changemaker.id : null).toBe(1); + }); + + it('skips when the EIN is not in the PDC, pointing at --create-changemaker', () => { + const plan = planImport(makePackage({ ein: '99-9999999' }), bundle, { kind: 'ein' }); + expect(plan.action).toBe('skip'); + expect(plan.action === 'skip' ? plan.reason : '').toContain('--create-changemaker'); + }); + + it('skips when the EIN matches more than one changemaker', () => { + const plan = planImport(makePackage(), ambiguous, { kind: 'ein' }); + expect(plan.action).toBe('skip'); + expect(plan.action === 'skip' ? plan.reason : '').toContain('multiple'); + }); + + it('skips when the file has no valid EIN', () => { + const plan = planImport(makePackage({ ein: null }), bundle, { kind: 'ein' }); + expect(plan.action).toBe('skip'); + expect(plan.action === 'skip' ? plan.reason : '').toContain('--changemaker-id'); + }); + }); + + describe('explicit --changemaker-id', () => { + it('updates the changemaker with that ID, regardless of EIN', () => { + const plan = planImport(makePackage(), bundle, { kind: 'id', changemakerId: 2 }); + expect(plan.action === 'update' ? plan.changemaker.id : null).toBe(2); + }); + + it('works when the file has no EIN', () => { + const plan = planImport(makePackage({ ein: null }), bundle, { kind: 'id', changemakerId: 1 }); + expect(plan.action === 'update' ? plan.changemaker.id : null).toBe(1); + }); + + it('skips when no changemaker has that ID', () => { + const plan = planImport(makePackage(), bundle, { kind: 'id', changemakerId: 999 }); + expect(plan.action).toBe('skip'); + expect(plan.action === 'skip' ? plan.reason : '').toContain('999'); + }); + }); + + describe('--create-changemaker', () => { + it('plans a create with the file EIN (verbatim) and name when the EIN is new', () => { + expect(planImport(makePackage(), empty, { kind: 'create' })).toStrictEqual({ + action: 'create', + taxId: '00-1000008', + name: 'Synthetic Org', + }); + }); + + it('refuses to create a duplicate when the EIN already exists', () => { + const plan = planImport(makePackage(), bundle, { kind: 'create' }); + expect(plan.action).toBe('skip'); + expect(plan.action === 'skip' ? plan.reason : '').toContain('changemaker 1'); + }); + + it('refuses to create when the EIN already matches several changemakers', () => { + expect(planImport(makePackage(), ambiguous, { kind: 'create' }).action).toBe('skip'); + }); + + it('skips when the file has no name', () => { + const plan = planImport(makePackage({ name: null }), empty, { kind: 'create' }); + expect(plan.action).toBe('skip'); + expect(plan.action === 'skip' ? plan.reason : '').toContain('name'); + }); + + it('skips when the file has no valid EIN', () => { + expect(planImport(makePackage({ ein: null }), empty, { kind: 'create' }).action).toBe('skip'); + }); + }); +}); + +describe('applyBaseFieldCategories', () => { + // Only `shortCode` and `category` are read. + const makeBaseField = (shortCode: string, category: string): { shortCode: string; category: string } => ({ + shortCode, + category, + }); + const field = (baseFieldShortCode: string): { baseFieldShortCode: string; value: string; sourceNote: string } => ({ + baseFieldShortCode, + value: 'v', + sourceNote: baseFieldShortCode, + }); + const baseFields = { + entries: [ + makeBaseField('organization_name', 'organization'), + makeBaseField('community_definition_overview', 'project'), + makeBaseField('proposal_name', 'project'), + ], + total: 3, + }; + const doc: TechSoupDocument = { + slug: 'example', + ein: null, + name: null, + goodAsOf: null, + fields: [ + field('organization_name'), + field('proposal_name'), + field('community_definition_overview'), + field('not_a_base_field'), + ], + }; + + it('keeps only organization-category fields for the changemaker', () => { + expect(applyBaseFieldCategories(doc, baseFields).organizationFields.map((f) => f.baseFieldShortCode)).toStrictEqual( + ['organization_name'], + ); }); - it('returns null when generated.at is missing', () => { - expect(extractGoodAsOf({ generated: {} })).toBeNull(); + it('sends fields of other categories to the proposal fields, whatever their prefix', () => { + expect(applyBaseFieldCategories(doc, baseFields).proposalFields.map((f) => f.baseFieldShortCode)).toStrictEqual([ + 'proposal_name', + 'community_definition_overview', + ]); }); - it('returns null when generated.at is not a parseable date', () => { - expect(extractGoodAsOf({ generated: { at: 'sometime' } })).toBeNull(); + it('drops fields that are not base fields in the PDC', () => { + const result = applyBaseFieldCategories(doc, baseFields); + const codes = [...result.organizationFields, ...result.proposalFields].map((f) => f.baseFieldShortCode); + expect(codes).not.toContain('not_a_base_field'); }); }); -describe('extractOrganizationName', () => { - it('returns the README title', () => { - expect(extractOrganizationName({ title: 'synthetic-Black Mountain Workforce Partnership' })).toBe( - 'synthetic-Black Mountain Workforce Partnership', - ); +describe('formatImportPreview', () => { + const pkg = makePackage({ + proposalFields: [{ baseFieldShortCode: 'proposal_name', value: 'A funding ask', sourceNote: 'proposal_name' }], + }); + + it('names the existing target changemaker and lists every field to import', () => { + const lines = formatImportPreview(pkg, { action: 'update', changemaker: makeChangemaker(1, '00-1000008') }); + expect(lines[0]).toBe('synthetic-org → changemaker 1 ("Org 1", taxId 00-1000008)'); + expect(lines).toContain('Fields that would be imported (2, goodAsOf=null):'); + expect(lines).toContain(' organization_name = "Synthetic Org" [README.md#title]'); + expect(lines).toContain(' organization_tax_id = "00-1000008" [organization_tax_id]'); }); - it('returns null when the title is absent', () => { - expect(extractOrganizationName({ description: 'no title here' })).toBeNull(); + it('names a new changemaker when the plan creates one', () => { + const lines = formatImportPreview(pkg, { action: 'create', taxId: '00-1000008', name: 'Synthetic Org' }); + expect(lines[0]).toBe('synthetic-org → NEW changemaker ("Synthetic Org", taxId 00-1000008)'); }); - it('returns null when the title is blank', () => { - expect(extractOrganizationName({ title: ' ' })).toBeNull(); + it('lists proposal fields separately as not imported', () => { + const lines = formatImportPreview(pkg, { action: 'update', changemaker: makeChangemaker(1, '00-1000008') }); + expect(lines).toContain('Proposal (non-organization) fields NOT imported (1):'); + expect(lines).toContain(' proposal_name = "A funding ask" [proposal_name]'); + }); + + it('prints only the skip reason when nothing would be imported', () => { + expect(formatImportPreview(pkg, { action: 'skip', reason: 'no match' })).toStrictEqual([ + 'synthetic-org → nothing — no match', + ]); }); }); -describe('resolveChangemaker', () => { - const bundle: ChangemakerBundle = { - entries: [makeChangemaker(1, '00-1000008'), makeChangemaker(2, '22-3456789')], - total: 2, +describe('pdcJsonToDocument', () => { + const raw = { + changemaker: { name: 'synthetic-Black Mountain Workforce Partnership' }, + fields: [ + { + baseFieldShortCode: 'organization_name', + value: 'synthetic-Black Mountain Workforce Partnership', + batch: { notes: 'slug/README.md#title' }, + }, + { baseFieldShortCode: 'organization_tax_id', value: '00-1000008' }, + { baseFieldShortCode: 'organization_dba_name', value: ['Alias One', 'Alias Two'] }, + { baseFieldShortCode: 'proposal_name', value: 'A funding ask' }, + { baseFieldShortCode: 'organization_overview', value: ' ' }, + ], }; + const doc = pdcJsonToDocument(raw, 'synthetic-black-mountain-workforce-partnership'); - it('resolves a unique match (hyphen-insensitive)', () => { - const resolution = resolveChangemaker('001000008', bundle); - expect(resolution.kind).toBe('matched'); - expect(resolution.kind === 'matched' ? resolution.changemaker.id : null).toBe(1); + it('takes the EIN from organization_tax_id (hyphenated, verbatim)', () => { + expect(doc.ein).toBe('00-1000008'); }); - it('resolves to missing when there is no match', () => { - expect(resolveChangemaker('99-9999999', bundle).kind).toBe('missing'); + it('takes the name from changemaker.name and leaves goodAsOf null', () => { + expect(doc.name).toBe('synthetic-Black Mountain Workforce Partnership'); + expect(doc.goodAsOf).toBeNull(); }); - it('resolves to ambiguous when more than one changemaker matches', () => { - const ambiguous: ChangemakerBundle = { - entries: [makeChangemaker(1, '00-1000008'), makeChangemaker(2, '001000008')], - total: 2, - }; - expect(resolveChangemaker('00-1000008', ambiguous).kind).toBe('ambiguous'); + it('keeps every usable field in file order, without sorting them by prefix', () => { + expect(doc.fields.map((f) => f.baseFieldShortCode)).toStrictEqual([ + 'organization_name', + 'organization_tax_id', + 'organization_dba_name', + 'proposal_name', + ]); + }); + + it('flattens array values and uses batch.notes as the provenance note', () => { + const dba = doc.fields.find((f) => f.baseFieldShortCode === 'organization_dba_name'); + expect(dba?.value).toBe('Alias One; Alias Two'); + const name = doc.fields.find((f) => f.baseFieldShortCode === 'organization_name'); + expect(name?.sourceNote).toBe('slug/README.md#title'); + }); + + it('drops blank-valued fields', () => { + expect(doc.fields.some((f) => f.baseFieldShortCode === 'organization_overview')).toBe(false); + }); + + it('yields a null EIN for a non-US tax id', () => { + const polish = pdcJsonToDocument( + { + changemaker: { name: 'Fundacja' }, + fields: [{ baseFieldShortCode: 'organization_tax_id', value: '0000000000' }], + }, + 'synthetic-fundacja', + ); + expect(polish.ein).toBeNull(); }); }); -describe('getChangemakerByEin', () => { - const bundle: ChangemakerBundle = { - entries: [makeChangemaker(1, '00-1000008'), makeChangemaker(2, '223456789')], - total: 2, - }; +describe('parsePdcJsonDocument', () => { + it('normalizes a PDC JSON document', () => { + const doc = parsePdcJsonDocument( + { changemaker: { name: 'Org' }, fields: [{ baseFieldShortCode: 'organization_tax_id', value: '12-3456789' }] }, + 'org', + ); + expect(doc.ein).toBe('12-3456789'); + expect(doc.name).toBe('Org'); + }); - it('matches ignoring a hyphen', () => { - expect(getChangemakerByEin('001000008', bundle)?.id).toBe(1); - expect(getChangemakerByEin('22-3456789', bundle)?.id).toBe(2); + it('throws when the document has no fields array', () => { + expect(() => parsePdcJsonDocument({ changemaker: { name: 'Org' } }, 'org')).toThrow('"fields" array'); }); - it('returns null when there is no match', () => { - expect(getChangemakerByEin('99-9999999', bundle)).toBeNull(); + it('throws when the document is not an object', () => { + expect(() => parsePdcJsonDocument([], 'org')).toThrow('not a TechSoup PDC JSON document'); }); +}); - it('returns null when more than one changemaker matches', () => { - const ambiguous: ChangemakerBundle = { - entries: [makeChangemaker(1, '00-1000008'), makeChangemaker(2, '001000008')], - total: 2, - }; - expect(getChangemakerByEin('00-1000008', ambiguous)).toBeNull(); +describe('readPdcJsonFile', () => { + let dir = ''; + + beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'techsoup-test-')); + }); + + afterAll(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('reads one file and derives the slug from its name', async () => { + const filePath = join(dir, 'synthetic-org_pdc.json'); + await writeFile( + filePath, + JSON.stringify({ + changemaker: { name: 'Synthetic Org' }, + fields: [ + { baseFieldShortCode: 'organization_name', value: 'Synthetic Org' }, + { baseFieldShortCode: 'organization_tax_id', value: '00-1000008' }, + { baseFieldShortCode: 'proposal_name', value: 'A funding ask' }, + ], + }), + ); + const doc = await readPdcJsonFile(filePath); + expect(doc.slug).toBe('synthetic-org'); + expect(doc.ein).toBe('00-1000008'); + expect(doc.fields).toHaveLength(3); + }); + + it('rejects a JSON file that is not PDC-export shaped', async () => { + const filePath = join(dir, 'package.json'); + await writeFile(filePath, JSON.stringify({ name: 'not-a-pdc-file' })); + await expect(readPdcJsonFile(filePath)).rejects.toThrow('"fields" array'); + }); + + it('rejects a file that is not valid JSON', async () => { + const filePath = join(dir, 'broken.json'); + await writeFile(filePath, '{ not json'); + await expect(readPdcJsonFile(filePath)).rejects.toThrow(SyntaxError); }); }); -describe('selectChangemakers', () => { - const bundle: ChangemakerBundle = { - entries: [makeChangemaker(1, '11-1111111'), makeChangemaker(2, '22-2222222')], - total: 2, +describe('selectTechSoupSource', () => { + it('returns the single TechSoup source', () => { + const sources = { entries: [makeSource(1, 'charitynavigator'), makeSource(2, 'techsoup')], total: 2 }; + expect(selectTechSoupSource(sources)?.id).toBe(2); + }); + + it('returns null when there is no TechSoup source', () => { + expect(selectTechSoupSource({ entries: [makeSource(1, 'charitynavigator')], total: 1 })).toBeNull(); + }); + + it('uses the oldest (lowest ID) when several TechSoup sources exist', () => { + const sources = { entries: [makeSource(9, 'techsoup'), makeSource(4, 'techsoup')], total: 2 }; + expect(selectTechSoupSource(sources)?.id).toBe(4); + }); +}); + +describe('toLoggableError', () => { + const NOT_FOUND = 404; + const config = { + method: 'post', + url: 'https://api.example.org/sources', + headers: new AxiosHeaders({ Authorization: 'Bearer secret-token' }), }; + const axiosError = new AxiosError('Request failed with status code 404', 'ERR_BAD_REQUEST', config, null, { + data: { name: 'NotFoundError', message: 'Entity not found' }, + status: NOT_FOUND, + statusText: 'Not Found', + headers: {}, + config, + }); + + it('keeps the status, method, URL, and response body of an AxiosError', () => { + expect(toLoggableError(axiosError)).toStrictEqual({ + name: 'AxiosError', + message: 'Request failed with status code 404', + code: 'ERR_BAD_REQUEST', + status: NOT_FOUND, + method: 'post', + url: 'https://api.example.org/sources', + responseData: { name: 'NotFoundError', message: 'Entity not found' }, + }); + }); - it('returns the full bundle unchanged when no ID is supplied', () => { - expect(selectChangemakers(bundle, undefined)).toBe(bundle); + it('drops the request headers, so the bearer token is never logged', () => { + expect(JSON.stringify(toLoggableError(axiosError))).not.toContain('secret-token'); }); - it('scopes to the single matching changemaker and updates total', () => { - const result = selectChangemakers(bundle, 2); - expect(result.entries).toHaveLength(1); - expect(result.entries[0]?.id).toBe(2); - expect(result.total).toBe(1); + it('reduces an AxiosError found in an Error cause', () => { + const wrapped = new Error('Cannot create the TechSoup source', { cause: axiosError }); + const loggable = toLoggableError(wrapped); + expect(JSON.stringify(loggable)).not.toContain('secret-token'); + expect(loggable).toMatchObject({ message: 'Cannot create the TechSoup source', cause: { status: NOT_FOUND } }); }); - it('returns an empty bundle with total 0 for an unknown ID', () => { - const result = selectChangemakers(bundle, 999); - expect(result.entries).toStrictEqual([]); - expect(result.total).toBe(0); + it('passes non-Error values through unchanged', () => { + expect(toLoggableError('plain string')).toBe('plain string'); + expect(toLoggableError(undefined)).toBeUndefined(); }); });