diff --git a/README.md b/README.md index 23ae7f3..d9ca80f 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,11 @@ Each fund carries a derived `metrics` object that powers the catalog columns sha - `volatility1y` - annualized volatility of the last 252 daily log returns, in percent (needs 253 closes) - `sigmaFromMean60d` - distance of the last close from the mean of the last 60 closes, in sample standard deviations - `high60d`, `low60d` - highest and lowest of the last 60 closes +- Dividend metrics (shared `dividends` block, the formulas of the Stocks feed) from the Yahoo Finance dividend events of the daily chart since listing (split adjusted; the published distribution table is not), one request per fund the run updates; a failed or truncated chart keeps the published values; `null` (never `0`) when unknown: + - `dividendRate` - sum of the distributions with ex-dates in the last 12 months + - `dividendGrowth`, `dividendGrowth3y`, `dividendGrowth5y` - last complete calendar year against 1, 3, 5 years earlier (3 and 5 as a yearly rate), the listing year is never a base + - `dividendStreak` - complete calendar years in a row with a higher sum + - `dividendCut10y` - `true` when a year of the last 10 paid more than 20% less than its base year and the next year did not come back - `returnsBasis` - mandatory non-empty text saying how the returns were computed: official WisdomTree month-end Market Price Returns (gaps filled with Yahoo estimates), or estimates derived from Yahoo adjusted closes when the product page is unavailable - `performanceAsOf` - mandatory ISO `YYYY-MM-DD` date the returns are as of: the WisdomTree month-end performance table date, or the last Yahoo close date when derived (not the NAV date); `null` only when truly unknown - NAV and the catalog figures come from WisdomTree; history and returns derived from Yahoo adjusted closes are market-price estimates, not official NAV returns diff --git a/scripts/update-data.test.ts b/scripts/update-data.test.ts index eae3b5f..9372255 100644 --- a/scripts/update-data.test.ts +++ b/scripts/update-data.test.ts @@ -483,10 +483,16 @@ const fundListRows = (tickers: string[]): any[] => tickers.map((t) => ({ /** As r.jina.ai returns it for Accept: application/json. */ const fundListFor = (tickers: string[]): string => JSON.stringify({ code: 200, status: 20000, data: { title: '', url: 'https://dataspanapi.wisdomtree.com/fundlist/data', content: JSON.stringify(fundListRows(tickers)) } }); -function chartPayload(days = 400, last = Date.UTC(2026, 8, 25)): any { +const DIVIDEND_KEYS = ['dividendRate', 'dividendGrowth', 'dividendGrowth3y', 'dividendGrowth5y', 'dividendStreak', 'dividendCut10y']; + +function chartPayload(days = 400, last = Date.UTC(2026, 8, 25), dividend = false): any { const timestamps = Array.from({ length: days }, (_, i) => Math.floor((last - (days - 1 - i) * 86_400_000) / 1000)); const close = timestamps.map((_, i) => 100 + i * 0.1); - return { chart: { result: [{ timestamp: timestamps, indicators: { quote: [{ close, volume: close.map(() => 1000) }], adjclose: [{ adjclose: close }] }, events: { dividends: {} }, meta: { exchangeName: 'NGM', regularMarketPrice: 140, regularMarketTime: Math.floor(last / 1000) } }] } }; + // the dividends block asks for its own chart (includePrePost=false): one 0.25 event on the last bar, listing on the first bar + const lastBar = timestamps[timestamps.length - 1]; + const dividends = dividend ? { [String(lastBar)]: { amount: 0.25, date: lastBar } } : {}; + const meta = { exchangeName: 'NGM', regularMarketPrice: 140, regularMarketTime: Math.floor(last / 1000), ...(dividend ? { dataGranularity: '1d', firstTradeDate: timestamps[0] } : {}) }; + return { chart: { result: [{ timestamp: timestamps, indicators: { quote: [{ close, volume: close.map(() => 1000) }], adjclose: [{ adjclose: close }] }, events: { dividends }, meta }] } }; } /** SEC world shared by every EDGAR test (each run starts with a fresh SEC state, so a test may change it). */ @@ -514,7 +520,7 @@ class Scenario { if (url.startsWith('https://r.jina.ai/https://dataspanapi.wisdomtree.com/')) return new Response(fundListFor(this.tickers)); if (url.startsWith('https://r.jina.ai/')) return url.endsWith('/us/products') ? new Response(catalogFor(this.tickers)) : new Response(productPage(url.split('/').pop()!.toUpperCase())); if (url.includes('wisdomtree.com')) return new Response('blocked', { status: 403 }); - if (url.includes('finance.yahoo.com')) return new Response(JSON.stringify(chartPayload())); + if (url.includes('finance.yahoo.com')) return new Response(JSON.stringify(chartPayload(400, undefined, url.includes('includePrePost=false')))); return new Response('nope', { status: 404 }); }; async run(env: Record = {}, overrides: Record = {}, deadlineMs?: number): Promise { @@ -561,7 +567,16 @@ describe('pipeline', () => { const RANGE_KEYS = ['week52Low', 'week52High', 'range52w', 'range52wClose', 'range52wAsOf', 'range52wSince', 'range52wBasis']; for (const f of [row, ...rest]) expect(Object.keys(f.metrics)).toEqual(expect.arrayContaining(RANGE_KEYS)); // the swing keys from the Close column of the same pages (shared swing block): ATH and its basis from the first close - for (const f of [row, ...rest]) expect(Object.keys(f.metrics).slice(-11, -2)).toEqual(['pctFrom52wHigh', 'athPrice', 'athDate', 'athBasis', 'pctFromAth', 'volatility1y', 'sigmaFromMean60d', 'high60d', 'low60d']); + for (const f of [row, ...rest]) expect(Object.keys(f.metrics).slice(-17, -8)).toEqual(['pctFrom52wHigh', 'athPrice', 'athDate', 'athBasis', 'pctFromAth', 'volatility1y', 'sigmaFromMean60d', 'high60d', 'low60d']); + // the dividend keys (shared dividends block): the mocked 0.25 event on the last bar is the trailing sum, the rest is unknown + for (const f of [row, ...rest]) expect(Object.keys(f.metrics).slice(-8, -2)).toEqual(DIVIDEND_KEYS); + for (const f of [row, ...rest]) expect(DIVIDEND_KEYS.map((key) => f.metrics[key])).toEqual([0.25, null, null, null, null, null]); + // one dividend chart per updated fund (it differs from the history chart by includePrePost=false), none for an unselected one + const dividendTickers = (from: number) => s.calls.slice(from).filter((call) => /finance\.yahoo\.com.*includePrePost=false/.test(call.url)).map((call) => /chart\/(\w+)\?/.exec(call.url)![1]).sort(); + expect(dividendTickers(0)).toEqual(['AAA', 'BBB', 'CCC']); + const seen = s.calls.length; + await s.run({ TICKERS: 'AAA' }); + expect(dividendTickers(seen)).toEqual(['AAA']); expect([typeof row.metrics.athPrice, row.metrics.athBasis]).toEqual(['number', expect.stringMatching(/^daily closes since \d{4}-\d{2}-\d{2}$/)]); for (const f of [row, ...rest]) expect([f.metrics.dividendYield, f.metrics.dividendYieldBasis]).toEqual([4.72, 'official-trailing-12m']); expect(s.meta('AAA').yields.dividendYieldBasis).toBe(row.metrics.dividendYieldBasis); @@ -976,7 +991,7 @@ describe('pipeline 7', () => { const base = s.handler; const listing = (inceptionDt: string) => JSON.stringify({ data: { content: JSON.stringify(fundListRows(['AAA']).map((row) => ({ ...row, inceptionDt }))) } }); let inception = '2025-09-11'; - s.handler = (url, init) => (url.includes('dataspanapi.wisdomtree.com') ? new Response(listing(inception)) : url.includes('finance.yahoo.com') ? new Response(JSON.stringify(chartPayload(380))) : base(url, init)); + s.handler = (url, init) => (url.includes('dataspanapi.wisdomtree.com') ? new Response(listing(inception)) : url.includes('finance.yahoo.com') ? new Response(JSON.stringify(chartPayload(380, undefined, url.includes('includePrePost=false')))) : base(url, init)); await s.run(); const metrics = s.index().funds[0].metrics; expect([metrics.ytd, metrics.tr1y, metrics.cagr3y, metrics.siAnn, metrics.performanceAsOf]).toEqual([2.59, null, null, null, '2026-08-31']); @@ -1010,7 +1025,7 @@ describe('pipeline 7', () => { // one of a new day that does not join keeps the fund as published await s.run(); expect(s.snapshot()).toEqual(first); - s.handler = (url, init) => (url.includes('finance.yahoo.com') ? new Response(JSON.stringify(chartPayload(1, Date.UTC(2026, 9, 25)))) : base(url, init)); + s.handler = (url, init) => (url.includes('finance.yahoo.com') ? new Response(JSON.stringify(chartPayload(1, Date.UTC(2026, 9, 25), url.includes('includePrePost=false')))) : base(url, init)); await s.run(); expect(s.meta('AAA').history.totalRows).toBe(400); }); @@ -1092,7 +1107,7 @@ describe('network', () => { test('a reachable direct page is used and the proxy is never called', async () => { const s = new Scenario(); - s.handler = (url) => secResponse(url, s.nportAsOf) ?? (url.startsWith('https://r.jina.ai/') ? new Response('should not be used', { status: 500 }) : url.startsWith('https://dataspanapi.wisdomtree.com/') ? new Response(JSON.stringify(fundListRows(s.tickers))) : url.includes('wisdomtree.com') ? new Response(productPage(url.split('/').pop()!.toUpperCase())) : url.includes('yahoo') ? new Response(JSON.stringify(chartPayload())) : new Response('no', { status: 404 })); + s.handler = (url) => secResponse(url, s.nportAsOf) ?? (url.startsWith('https://r.jina.ai/') ? new Response('should not be used', { status: 500 }) : url.startsWith('https://dataspanapi.wisdomtree.com/') ? new Response(JSON.stringify(fundListRows(s.tickers))) : url.includes('wisdomtree.com') ? new Response(productPage(url.split('/').pop()!.toUpperCase())) : url.includes('yahoo') ? new Response(JSON.stringify(chartPayload(400, undefined, url.includes('includePrePost=false')))) : new Response('no', { status: 404 })); await s.run(); expect(s.calls.some((c) => c.url.startsWith('https://r.jina.ai/'))).toBe(false); expect(s.meta('AAA').source.productPageStatus).toContain('directly'); diff --git a/scripts/update-data.ts b/scripts/update-data.ts index 7d5f135..fae7ab8 100755 --- a/scripts/update-data.ts +++ b/scripts/update-data.ts @@ -2460,8 +2460,14 @@ export async function runUpdate(config: UpdaterConfig, deadlineMs: number = SOFT for (const row of [...mergedRows.values()].sort((a, b) => String(a.ticker).localeCompare(String(b.ticker)))) funds.push(normalizeIndexRow(row, await hasMetaFile(String(row.ticker)))); const counts = { funds: funds.length, holdings: funds.reduce((sum, row) => sum + (numberOrNull(row.holdings) || 0), 0), history: funds.reduce((sum, row) => sum + (numberOrNull(row.history) || 0), 0) }; // Closed funds keep their published row as it is, marked closed (written even when this run updated nothing). - // the 52-week range and the swing metrics of every row from its published history pages (shared range-52w and swing blocks) - await writeIndex(await withSwing(API_ROOT, await withRange52w(API_ROOT, markClosedFunds(funds, CLOSED_FUNDS)))); + // the 52-week range and the swing metrics of every row from its published history pages (shared range-52w and swing blocks), + // the dividend metrics from one Yahoo daily chart per fund this run updated (shared dividends block) + await writeIndex(await withDividends(API_ROOT, await withSwing(API_ROOT, await withRange52w(API_ROOT, markClosedFunds(funds, CLOSED_FUNDS))), { + fetchChart: (url, ticker) => fetchJson(url, `[ dividend ] ${ticker}`, config, { 'User-Agent': 'Mozilla/5.0' }), + tickers: config.skipYahoo ? [] : results.map((row) => String(row.ticker)), + previous: Array.from(previous.values()), concurrency: config.concurrency, + onError: (ticker, error) => outputNote(`[ dividend ] ${ticker}: ${error instanceof Error ? error.message : String(error)}; published dividend metrics kept`), + })); // Q28: a fund is announced on its first successful publish (a catalog-only row is not published), new catalog-only rows as pending const newFunds = newFundsLines(newFundsReport(Array.from(previous.values()), funds)); for (const line of newFunds) console.log(line); @@ -3844,11 +3850,12 @@ export function swingDaysFromChart(payload: unknown): SwingDay[] | null { /** * The Yahoo chart request of one fund: the whole daily history since listing. Explicit period1=0 and period2 (tomorrow), never - * `range=max`: Yahoo answers `range=max&interval=1d` with weekly bars for a long history. + * `range=max`: Yahoo answers `range=max&interval=1d` with weekly bars for a long history. The answer also carries the dividend + * events (`events=div%7Csplit`) for the dividends block, so a tap on the same request serves both blocks. */ export function swingChartUrl(ticker: string, nowMs?: number): string { const period2 = Math.floor((nowMs === undefined ? Date.now() : nowMs) / 1000) + 86400; - return `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(ticker)}?period1=0&period2=${period2}&interval=1d&includePrePost=false&events=split`; + return `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(ticker)}?period1=0&period2=${period2}&interval=1d&includePrePost=false&events=div%7Csplit`; } export type SwingOptions = { @@ -3968,3 +3975,342 @@ export async function withSwing(feedRoot: URL | string, funds: T[], options?: return out; } // + +// shared block of .claude/tools/etfs/etf-std/dividends.block.ts, applied by apply-block.ts: never edit it in one repo +// The formulas are those of investments/Stocks/scripts/update-data.ts `dividendMetrics` and scripts/metrics/dividends.ts +// `dividendHistory`, rewritten without top-level consts (the block sits after `await main()`). Two ETF differences: the +// listing year (year of `meta.firstTradeDate`) is dropped from the yearly sums, and the cut tolerance is 20% instead of 10%. +// Source: the dividend events of the Yahoo daily chart (swingChartUrl, split adjusted), never the brand distributions. +// etf-std/blocks.test.ts runs this block and the Stocks functions on the same data and expects identical output. +export type DividendEvent = { date: string; amount: number }; + +/** asOf: last daily bar; firstDay: first daily bar; listed: meta.firstTradeDate (null only in tests, then no year is dropped) */ +export type DividendsChart = { asOf: string; firstDay: string; listed: string | null; events: DividendEvent[] }; + +/** The six dividend keys of an index row's `metrics`, in key order */ +export type DividendRow = { + dividendRate: number | null; // sum of the amounts with ex-dates in the 365.25 days up to asOf, 4 decimals + dividendGrowth: number | null; // last complete calendar year against the year before, percent + dividendGrowth3y: number | null; // yearly rate over 3 years, percent + dividendGrowth5y: number | null; // yearly rate over 5 years, percent + dividendStreak: number | null; // complete calendar years in a row with a higher sum than the base year + dividendCut10y: boolean | null; // a confirmed cut in the last 10 complete calendar years +}; + +/** A drop smaller than this share of the base year is ordinary fluctuation, not a cut (Stocks: 0.1) */ +export function dividendCutTolerance(): number { + return 0.2; +} + +/** A year more than this factor above both neighbours is a special dividend or a shifted payment */ +export function dividendSpikeFactor(): number { + return 1.2; +} + +/** Window of the cut flag in complete calendar years */ +export function dividendCutWindowYears(): number { + return 10; +} + +/** Complete calendar years kept for the streak (plus the current partial year) */ +export function dividendAnnualYears(): number { + return 30; +} + +/** The six dividend keys, all null: a catalog-only row and a fund that never paid carry this */ +export function dividendsEmpty(): DividendRow { + return { dividendRate: null, dividendGrowth: null, dividendGrowth3y: null, dividendGrowth5y: null, dividendStreak: null, dividendCut10y: null }; +} + +function dividendRoundTo(value: number, digits: number): number | null { + if (!Number.isFinite(value)) return null; + const f = Math.pow(10, digits); + return Math.round(value * f) / f; +} + +function dividendIsoDay(seconds: number): string { + return new Date(seconds * 1000).toISOString().slice(0, 10); +} + +/** Unrounded sums by calendar year, years after `listedYear` only; `years` newest first */ +function dividendYearSums(events: DividendEvent[], listedYear?: number | null): { years: number[]; sums: Record } { + const sums: Record = {}; + const years: number[] = []; + const list = Array.isArray(events) ? events : []; + for (let i = 0; i < list.length; i += 1) { + const event = list[i]; + if (!event || typeof event.date !== 'string' || typeof event.amount !== 'number' || !Number.isFinite(event.amount) || event.amount <= 0) continue; + const year = Number(event.date.slice(0, 4)); + if (!Number.isFinite(year)) continue; + if (typeof listedYear === 'number' && Number.isFinite(listedYear) && year <= listedYear) continue; + if (!(year in sums)) { sums[year] = 0; years.push(year); } + sums[year] += event.amount; + } + years.sort((a, b) => b - a); + return { years, sums }; +} + +/** + * The Yahoo daily chart answer (text or parsed JSON) as dividend events. Null (a failed source) when there is no result, the + * bars are not daily, there is no finite timestamp, `meta.firstTradeDate` is missing (completeness unproven) or the first bar + * is more than 31 days after it (a truncated answer). Missing `events.dividends` is an honest empty list + */ +export function dividendsFromChart(payload: unknown): DividendsChart | null { + let json: unknown = payload; + if (typeof payload === 'string') { + try { json = JSON.parse(payload); } catch (_) { return null; } + } + const chart = json && typeof json === 'object' ? (json as { chart?: { result?: unknown } }).chart : undefined; + const results = chart && Array.isArray(chart.result) ? chart.result : null; + const result = results && results[0] && typeof results[0] === 'object' ? (results[0] as Record) : null; + if (!result) return null; + const meta = result.meta && typeof result.meta === 'object' ? (result.meta as Record) : {}; + if (typeof meta.dataGranularity === 'string' && meta.dataGranularity !== '1d') return null; + const stamps = Array.isArray(result.timestamp) ? (result.timestamp as unknown[]) : []; + let first = Infinity, last = -Infinity; + for (let i = 0; i < stamps.length; i += 1) { + const stamp = stamps[i]; + if (typeof stamp !== 'number' || !Number.isFinite(stamp)) continue; + if (stamp < first) first = stamp; + if (stamp > last) last = stamp; + } + if (first === Infinity) return null; + const listed = meta.firstTradeDate; + if (typeof listed !== 'number' || !Number.isFinite(listed)) return null; + if (first - listed > 31 * 86400) return null; + const events: DividendEvent[] = []; + const raw = result.events && typeof result.events === 'object' ? (result.events as Record).dividends : null; + if (raw && typeof raw === 'object') { + const keys = Object.keys(raw); + for (let i = 0; i < keys.length; i += 1) { + const item = (raw as Record)[keys[i]]; + if (!item || typeof item !== 'object') continue; + const amount = (item as Record).amount, when = (item as Record).date; + if (typeof amount !== 'number' || !Number.isFinite(amount) || amount <= 0) continue; + const seconds = typeof when === 'number' && Number.isFinite(when) ? when : Number(keys[i]); + if (!Number.isFinite(seconds)) continue; + events.push({ date: dividendIsoDay(seconds), amount }); + } + } + events.sort((a, b) => (a.date < b.date ? -1 : a.date > b.date ? 1 : 0)); + return { asOf: dividendIsoDay(last), firstDay: dividendIsoDay(first), listed: dividendIsoDay(listed), events }; +} + +/** Calendar-year sums newest first, years after the listing year and within the last 30 complete years, 4 decimals (the current partial year included) */ +export function dividendAnnual(events: DividendEvent[], asOf: string, listedYear?: number | null): Array<{ year: number; dps: number }> { + const complete = Number(String(asOf).slice(0, 4)) - 1; + const totals = dividendYearSums(events, listedYear); + const out: Array<{ year: number; dps: number }> = []; + for (let i = 0; i < totals.years.length; i += 1) { + const year = totals.years[i]; + if (year > complete - dividendAnnualYears()) out.push({ year, dps: dividendRoundTo(totals.sums[year], 4) || 0 }); + } + return out; +} + +/** + * Streak of yearly increases and the 10-year cut flag (Stocks scripts/metrics/dividends.ts, line by line): a year is compared + * with its base (the year before, or the year before a spike), a cut is a drop of more than `tolerance` that the next complete + * year does not recover, a year without payment after a paying year is a cut, years before the first payment are unknown + */ +export function dividendHistory(annual: Array<{ year: number; dps: number }>, asOf: string, tolerance?: number): { dividendStreak: number | null; dividendCut10y: boolean | null } { + const cutTolerance = tolerance === undefined ? dividendCutTolerance() : tolerance; + const spike = dividendSpikeFactor(); + const none = { dividendStreak: null, dividendCut10y: null }; + const lastYear = Number(String(asOf).slice(0, 4)); + if (!Number.isFinite(lastYear) || !annual.length) return none; + const complete = lastYear - 1; // the current calendar year is partial + const byYear: Record = {}; + let first = Infinity; + annual.forEach((a) => { + if (!Number.isFinite(a.year) || !Number.isFinite(a.dps) || a.dps <= 0 || a.year > complete) return; + byYear[a.year] = (byYear[a.year] === undefined ? 0 : byYear[a.year]) + a.dps; + first = Math.min(first, a.year); + }); + if (first === Infinity) return none; + /** dps of a year, 0 for a year without payments after the first one, null before it */ + const dps = (y: number): number | null => (y < first ? null : byYear[y] === undefined ? 0 : byYear[y]); + const base = (y: number): number | null => { + const prev = dps(y - 1), before = dps(y - 2), cur = dps(y); + if (prev === null) return null; + if (before !== null && before > 0 && cur !== null && prev > spike * before && prev > spike * cur) return before; + return prev; + }; + const isCut = (y: number): boolean => { + const b = base(y), cur = dps(y); + if (b === null || cur === null || b <= 0 || cur >= (1 - cutTolerance) * b) return false; + const next = y < complete ? dps(y + 1) : null; + return next === null || next < (1 - cutTolerance) * b; + }; + let paidInWindow = false; + let cut = false; + for (let y = complete - dividendCutWindowYears() + 1; y <= complete; y += 1) { + const now = dps(y), prev = dps(y - 1); + if ((now === null ? 0 : now) > 0 || (prev === null ? 0 : prev) > 0) paidInWindow = true; + if (isCut(y)) cut = true; + } + let streak: number | null = null; + const last = dps(complete); + if ((last === null ? 0 : last) > 0) { + streak = 0; + for (let y = complete; ; y -= 1) { + const b = base(y), cur = dps(y); + if (b === null || cur === null || !(b > 0) || !(cur > b)) break; + streak += 1; + } + } + return { dividendStreak: streak, dividendCut10y: paidInWindow ? cut : null }; +} + +/** The six keys of one fund from its chart: no wall clock, so a rerun on the same answer gives the same values */ +export function dividendRow(chart: DividendsChart, tolerance?: number): DividendRow { + const listedYear = chart.listed ? Number(chart.listed.slice(0, 4)) : null; + const events = Array.isArray(chart.events) ? chart.events : []; + const complete = Number(String(chart.asOf).slice(0, 4)) - 1; // the current calendar year is partial + const totals = dividendYearSums(events, listedYear); + const dps = (y: number): number | null => (y in totals.sums ? totals.sums[y] : null); + const growth = (back: number): number | null => { + const a = dps(complete), b = dps(complete - back); + if (a === null || b === null || b <= 0) return null; + return dividendRoundTo((back === 1 ? a / b - 1 : Math.pow(a / b, 1 / back) - 1) * 100, 2); + }; + const cutoff = new Date(Date.parse(`${chart.asOf}T00:00:00Z`) - 365.25 * 86400000).toISOString().slice(0, 10); + let ttm = 0; + for (let i = 0; i < events.length; i += 1) { + const event = events[i]; + if (event && event.date > cutoff && typeof event.amount === 'number' && Number.isFinite(event.amount) && event.amount > 0) ttm += event.amount; + } + const history = dividendHistory(dividendAnnual(events, chart.asOf, listedYear), chart.asOf, tolerance); + return { + dividendRate: ttm > 0 ? dividendRoundTo(ttm, 4) : null, + dividendGrowth: growth(1), dividendGrowth3y: growth(3), dividendGrowth5y: growth(5), + dividendStreak: history.dividendStreak, dividendCut10y: history.dividendCut10y, + }; +} + +/** + * Wraps the brand fetch that withSwing uses (Yahoo-mode brands): every answer also fills `seen[ticker]` with its parsed + * dividend events (null for an incomplete chart), the payload goes back untouched; a rejected fetch stores nothing + */ +export function dividendsTap(fetchChart: (url: string, ticker: string) => Promise): { fetchChart: (url: string, ticker: string) => Promise; seen: Record } { + const seen: Record = {}; + return { + seen, + fetchChart: async (url: string, ticker: string): Promise => { + const payload = await fetchChart(url, ticker); + seen[ticker] = dividendsFromChart(payload); + return payload; + }, + }; +} + +export type DividendsOptions = { + /** Tap mode: `dividendsTap(...).seen` of the swing request */ + charts?: Record; + /** Own-request mode: GETs swingChartUrl(ticker) through the brand's own Yahoo fetch, returns the body text or parsed JSON */ + fetchChart?: (url: string, ticker: string) => Promise; + /** Own-request mode: the funds this run updated; always pass it (`[]` under SKIP_YAHOO) */ + tickers?: string[] | null; + /** The previously published index rows: a fund that is not recomputed keeps their dividend keys */ + previous?: unknown[] | null; + /** Own-request mode: parallel chart requests, default 4 */ + concurrency?: number; + /** Own-request mode: time for all requests, default 150000 ms; funds left after it keep their published keys */ + budgetMs?: number; + /** One call per failed chart (and `*` once for the budget), for the brand's log */ + onError?: (ticker: string, error: unknown) => void; +}; + +/** + * The index rows with the six dividend keys in every `metrics`, after the swing keys and before `returnsBasis` and + * `performanceAsOf`. Wired once: `withDividends(feedRoot, await withSwing(...), options)`. + * - Tap mode (`charts`): a fund with a complete chart gets fresh keys, an incomplete chart is reported and keeps the published keys + * - Own-request mode (`fetchChart`): one swingChartUrl request per fund of `tickers`, funds without published keys first, within `budgetMs` + * - A failed source never changes a published fund (`previous`, else the row's own keys); a row without `dataFile` gets + * dividendsEmpty(); a row without a `metrics` object is left as it is + */ +export async function withDividends(feedRoot: URL | string, funds: T[], options?: DividendsOptions): Promise { + const opts: DividendsOptions = options || {}; + const list = Array.isArray(funds) ? funds : []; + const rows = list.map((fund) => fund as unknown as Record); + const keys = Object.keys(dividendsEmpty()); + const tickerOf = (row: unknown): string => (row && typeof row === 'object' && typeof (row as Record).ticker === 'string' ? String((row as Record).ticker) : ''); + const hasData = (row: Record): boolean => !!row && typeof row === 'object' && !!row.metrics && typeof row.metrics === 'object' && typeof row.dataFile === 'string' && row.dataFile !== ''; + const own = (row: unknown): DividendRow | null => { + const metrics = row && typeof row === 'object' ? (row as Record).metrics : null; + if (!metrics || typeof metrics !== 'object') return null; + const out = dividendsEmpty() as unknown as Record; + for (let k = 0; k < keys.length; k += 1) { + if (!(keys[k] in (metrics as Record))) return null; + out[keys[k]] = (metrics as Record)[keys[k]]; + } + return out as unknown as DividendRow; + }; + const before: Record = {}; + const previous = Array.isArray(opts.previous) ? opts.previous : []; + for (let i = 0; i < previous.length; i += 1) if (tickerOf(previous[i])) before[tickerOf(previous[i])] = previous[i]; + const published = (row: Record): DividendRow => own(before[tickerOf(row)]) || own(row) || dividendsEmpty(); + const computed: Array = rows.map(() => null); + const report = (ticker: string, error: unknown): void => { if (opts.onError) opts.onError(ticker, error); }; + if (opts.charts && typeof opts.charts === 'object') { + const charts = opts.charts; + for (let i = 0; i < rows.length; i += 1) { + if (!hasData(rows[i])) continue; + const ticker = tickerOf(rows[i]); + if (!Object.prototype.hasOwnProperty.call(charts, ticker)) continue; + const chart = charts[ticker]; + if (chart) computed[i] = dividendRow(chart); + else report(ticker, new Error('Yahoo chart is not a complete daily history')); + } + } else if (opts.fetchChart) { + const fetchChart = opts.fetchChart; + const wanted = Array.isArray(opts.tickers) ? opts.tickers.map((t) => String(t)) : null; + const missing: number[] = [], rest: number[] = []; + for (let i = 0; i < rows.length; i += 1) { + if (!hasData(rows[i]) || (wanted && wanted.indexOf(tickerOf(rows[i])) < 0)) continue; + const old = own(before[tickerOf(rows[i])]) || own(rows[i]); + let known = false; + if (old) for (let k = 0; k < keys.length; k += 1) if ((old as unknown as Record)[keys[k]] !== null) known = true; + (known ? rest : missing).push(i); + } + const queue = missing.concat(rest); + const budget = opts.budgetMs === undefined || !Number.isFinite(Number(opts.budgetMs)) ? 150000 : Math.max(0, Number(opts.budgetMs)); + const started = Date.now(); + let next = 0, spent = false; + const worker = async (): Promise => { + while (next < queue.length) { + if (Date.now() - started >= budget) { spent = true; return; } + const index = queue[next]; + next += 1; + const ticker = tickerOf(rows[index]); + try { + const chart = dividendsFromChart(await fetchChart(swingChartUrl(ticker), ticker)); + if (!chart) throw new Error('Yahoo chart is not a complete daily history'); + computed[index] = dividendRow(chart); + } catch (error) { + report(ticker, error); + } + } + }; + const lanes = Math.max(1, Math.min(queue.length || 1, Math.floor(Number(opts.concurrency) || 4))); + const workers: Array> = []; + for (let i = 0; i < lanes; i += 1) workers.push(worker()); + await Promise.all(workers); + if (spent) report('*', new Error(`dividends budget of ${Math.round(budget / 1000)} s reached, ${queue.length - next} funds keep their published dividend metrics`)); + } + const out: T[] = []; + for (let i = 0; i < rows.length; i += 1) { + const row = rows[i]; + if (!row || typeof row !== 'object' || !row.metrics || typeof row.metrics !== 'object') { out.push(list[i]); continue; } + const values = (hasData(row) ? computed[i] || published(row) : dividendsEmpty()) as unknown as Record; + // the dividend keys go in front of returnsBasis and performanceAsOf, which stay the last two keys (metrics contract) + const metrics: Record = {}, old = row.metrics as Record, tail = ['returnsBasis', 'performanceAsOf']; + const oldKeys = Object.keys(old); + for (let k = 0; k < oldKeys.length; k += 1) if (tail.indexOf(oldKeys[k]) < 0 && keys.indexOf(oldKeys[k]) < 0) metrics[oldKeys[k]] = old[oldKeys[k]]; + for (let k = 0; k < keys.length; k += 1) metrics[keys[k]] = values[keys[k]]; + for (let k = 0; k < tail.length; k += 1) if (tail[k] in old) metrics[tail[k]] = old[tail[k]]; + out.push({ ...row, metrics } as unknown as T); + } + return out; +} +//