Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ Each fund carries a derived `metrics` object that powers the catalog columns sha
- `volatility1y` - annualized volatility of the last 252 daily log returns, in percent (needs 253 closes)
- `sigmaFromMean60d` - distance of the last close from the mean of the last 60 closes, in sample standard deviations
- `high60d`, `low60d` - highest and lowest of the last 60 closes
- Dividend metrics (shared `dividends` block, the formulas of the Stocks feed) from the Yahoo Finance dividend events of the daily chart since listing (split adjusted; the published distribution table is not), one request per fund the run updates; a failed or truncated chart keeps the published values; `null` (never `0`) when unknown:
- Dividend metrics (shared `dividends` block, the formulas of the Stocks feed) from the Yahoo Finance dividend events of the daily chart since listing (split adjusted; the published distribution table is not), read from the history chart request of each fund the run updates (one extra chart request per fund only when `HISTORY_RANGE` is not `max`); a failed or truncated chart keeps the published values; `null` (never `0`) when unknown:
- `dividendRate` - sum of the distributions with ex-dates in the last 12 months
- `dividendGrowth`, `dividendGrowth3y`, `dividendGrowth5y` - last complete calendar year against 1, 3, 5 years earlier (3 and 5 as a yearly rate), the listing year is never a base
- `dividendStreak` - complete calendar years in a row with a higher sum
Expand Down Expand Up @@ -118,7 +118,7 @@ Every control is a key in `scripts/update-data.config.json`; values are strings.
| --- | --: | --- |
| `MAX_FETCHES` | `0` | Batch size over the funds that pass `TICKERS`/`AUM`/`TER`/`DIVIDEND_YIELD`: with a positive value the updater continues after the committed cursor in `api/wisdomtree/update-state.json` (in ticker order, wrapping around; the cursor moves past every fund taken, whatever its outcome); `0` is a full pass. A `TICKERS` run never touches the cursor. |
| `REQUEST_SLEEP` | `2` | Minimum delay in seconds between outgoing request starts, including retries. |
| `CONCURRENCY` | `2` | Number of parallel fund update workers, each with its own request pacing lane. |
| `CONCURRENCY` | `4` | Number of parallel fund update workers, each with its own request pacing lane. |
| `AUM` | `:` | Net Assets range. Each bound may be a USD amount or `K`/`M`/`B`/`T`, or one of `nano`, `micro`, `small`, `mid`, `large`. |
| `TER` | `:` | Expense ratio range in % (strict `min:max`). |
| `DIVIDEND_YIELD` | `:` | Dividend-yield percentage range. |
Expand Down
2 changes: 1 addition & 1 deletion scripts/update-data.config.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"REQUEST_SLEEP": "2",
"AUM": ":",
"TER": ":",
"CONCURRENCY": "2",
"CONCURRENCY": "4",
"HOLDINGS_PAGE_SIZE": "250",
"HISTORY_PAGE_SIZE": "1000",
"STORE_RAW_DOWNLOADS": "false",
Expand Down
26 changes: 15 additions & 11 deletions scripts/update-data.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ describe('controls 2', () => {
expect(help).toContain(t ? `${t[1]}_YTD|1Y|3Y|5Y|10Y` : ` ${name}=`);
}
const config = readConfig(resolveControls(file));
expect([config.maxFetches, config.requestSleep, config.concurrency, config.maxRetries, config.tickers, config.historyRange]).toEqual([0, 2, 2, 2, null, 'max']);
expect([config.maxFetches, config.requestSleep, config.concurrency, config.maxRetries, config.tickers, config.historyRange]).toEqual([0, 2, 4, 2, null, 'max']);
expect([config.edgarFallback, config.skipYahoo, config.skipWisdomTree, config.storeRawDownloads, config.aum, config.secYield]).toEqual([true, false, false, false, undefined, undefined]);
});

Expand Down Expand Up @@ -492,7 +492,7 @@ const BENCHMARK_KEYS = ['perf1y', 'perf3y', 'perf5y', 'perf10y', 'perfSi', 'maxD
function chartPayload(days = 400, last = Date.UTC(2026, 8, 25), dividend = false): any {
const timestamps = Array.from({ length: days }, (_, i) => Math.floor((last - (days - 1 - i) * 86_400_000) / 1000));
const close = timestamps.map((_, i) => 100 + i * 0.1);
// the dividends block asks for its own chart (includePrePost=false): one 0.25 event on the last bar, listing on the first bar
// a full daily chart (period1=0, as Yahoo answers it): one 0.25 event on the last bar, listing on the first bar
const lastBar = timestamps[timestamps.length - 1];
const dividends = dividend ? { [String(lastBar)]: { amount: 0.25, date: lastBar } } : {};
const meta = { exchangeName: 'NGM', regularMarketPrice: 140, regularMarketTime: Math.floor(last / 1000), ...(dividend ? { dataGranularity: '1d', firstTradeDate: timestamps[0] } : {}) };
Expand Down Expand Up @@ -524,7 +524,7 @@ class Scenario {
if (url.startsWith('https://r.jina.ai/https://dataspanapi.wisdomtree.com/')) return new Response(fundListFor(this.tickers));
if (url.startsWith('https://r.jina.ai/')) return url.endsWith('/us/products') ? new Response(catalogFor(this.tickers)) : new Response(productPage(url.split('/').pop()!.toUpperCase()));
if (url.includes('wisdomtree.com')) return new Response('blocked', { status: 403 });
if (url.includes('finance.yahoo.com')) return new Response(JSON.stringify(chartPayload(400, undefined, url.includes('includePrePost=false'))));
if (url.includes('finance.yahoo.com')) return new Response(JSON.stringify(chartPayload(400, undefined, url.includes('period1=0&'))));
return new Response('nope', { status: 404 });
};
async run(env: Record<string, string> = {}, overrides: Record<string, unknown> = {}, deadlineMs?: number): Promise<void> {
Expand Down Expand Up @@ -578,12 +578,16 @@ describe('pipeline', () => {
for (const f of [row, ...rest]) expect(DIVIDEND_KEYS.map((key) => f.metrics[key])).toEqual([0.25, null, null, null, null, null, 1, 'annual']);
// the mock gives SPY and QQQ the same bars as every fund: the fund equals both benchmarks (no difference, beta and correlation 1)
expect([row.metrics.vsSpy1y, row.metrics.vsQqq1y, row.metrics.beta1ySpy, row.metrics.corr1yQqq, row.metrics.relStrength3mSpy, typeof row.metrics.benchmarkAsOf]).toEqual([0, 0, 1, 1, 0, 'string']);
// one chart per updated fund feeds swing, dividends and benchmark; SPY and QQQ are read once more per run (it differs from the history chart by includePrePost=false), none for an unselected one
const dividendTickers = (from: number) => s.calls.slice(from).filter((call) => /finance\.yahoo\.com.*includePrePost=false/.test(call.url)).map((call) => /chart\/(\w+)\?/.exec(call.url)![1]).sort();
expect(dividendTickers(0)).toEqual(['AAA', 'BBB', 'CCC', 'QQQ', 'SPY']);
const seen = s.calls.length;
// one full daily chart per updated fund feeds the history, dividends and benchmark (the blocks reuse the history request); SPY and
// QQQ are read once per run, none for an unselected fund; a HISTORY_RANGE window is no full chart, the blocks ask for their own
const chartTickers = (from: number) => s.calls.slice(from).filter((call) => /finance\.yahoo\.com/.test(call.url)).map((call) => /chart\/(\w+)\?/.exec(call.url)![1]).sort();
expect(chartTickers(0)).toEqual(['AAA', 'BBB', 'CCC', 'QQQ', 'SPY']);
let seen = s.calls.length;
await s.run({ TICKERS: 'AAA' });
expect(dividendTickers(seen)).toEqual(['AAA', 'QQQ', 'SPY']);
expect(chartTickers(seen)).toEqual(['AAA', 'QQQ', 'SPY']);
seen = s.calls.length;
await s.run({ TICKERS: 'AAA', HISTORY_RANGE: '1y' });
expect(chartTickers(seen)).toEqual(['AAA', 'AAA', 'QQQ', 'SPY']);
expect([typeof row.metrics.athPrice, row.metrics.athBasis]).toEqual(['number', expect.stringMatching(/^daily closes since \d{4}-\d{2}-\d{2}$/)]);
for (const f of [row, ...rest]) expect([f.metrics.dividendYield, f.metrics.dividendYieldBasis]).toEqual([4.72, 'official-trailing-12m']);
expect(s.meta('AAA').yields.dividendYieldBasis).toBe(row.metrics.dividendYieldBasis);
Expand Down Expand Up @@ -998,7 +1002,7 @@ describe('pipeline 7', () => {
const base = s.handler;
const listing = (inceptionDt: string) => JSON.stringify({ data: { content: JSON.stringify(fundListRows(['AAA']).map((row) => ({ ...row, inceptionDt }))) } });
let inception = '2025-09-11';
s.handler = (url, init) => (url.includes('dataspanapi.wisdomtree.com') ? new Response(listing(inception)) : url.includes('finance.yahoo.com') ? new Response(JSON.stringify(chartPayload(380, undefined, url.includes('includePrePost=false')))) : base(url, init));
s.handler = (url, init) => (url.includes('dataspanapi.wisdomtree.com') ? new Response(listing(inception)) : url.includes('finance.yahoo.com') ? new Response(JSON.stringify(chartPayload(380, undefined, url.includes('period1=0&')))) : base(url, init));
await s.run();
const metrics = s.index().funds[0].metrics;
expect([metrics.ytd, metrics.tr1y, metrics.cagr3y, metrics.siAnn, metrics.performanceAsOf]).toEqual([2.59, null, null, null, '2026-08-31']);
Expand Down Expand Up @@ -1032,7 +1036,7 @@ describe('pipeline 7', () => {
// one of a new day that does not join keeps the fund as published
await s.run();
expect(s.snapshot()).toEqual(first);
s.handler = (url, init) => (url.includes('finance.yahoo.com') ? new Response(JSON.stringify(chartPayload(1, Date.UTC(2026, 9, 25), url.includes('includePrePost=false')))) : base(url, init));
s.handler = (url, init) => (url.includes('finance.yahoo.com') ? new Response(JSON.stringify(chartPayload(1, Date.UTC(2026, 9, 25), url.includes('period1=0&')))) : base(url, init));
await s.run();
expect(s.meta('AAA').history.totalRows).toBe(400);
});
Expand Down Expand Up @@ -1114,7 +1118,7 @@ describe('network', () => {

test('a reachable direct page is used and the proxy is never called', async () => {
const s = new Scenario();
s.handler = (url) => secResponse(url, s.nportAsOf) ?? (url.startsWith('https://r.jina.ai/') ? new Response('should not be used', { status: 500 }) : url.startsWith('https://dataspanapi.wisdomtree.com/') ? new Response(JSON.stringify(fundListRows(s.tickers))) : url.includes('wisdomtree.com') ? new Response(productPage(url.split('/').pop()!.toUpperCase())) : url.includes('yahoo') ? new Response(JSON.stringify(chartPayload(400, undefined, url.includes('includePrePost=false')))) : new Response('no', { status: 404 }));
s.handler = (url) => secResponse(url, s.nportAsOf) ?? (url.startsWith('https://r.jina.ai/') ? new Response('should not be used', { status: 500 }) : url.startsWith('https://dataspanapi.wisdomtree.com/') ? new Response(JSON.stringify(fundListRows(s.tickers))) : url.includes('wisdomtree.com') ? new Response(productPage(url.split('/').pop()!.toUpperCase())) : url.includes('yahoo') ? new Response(JSON.stringify(chartPayload(400, undefined, url.includes('period1=0&')))) : new Response('no', { status: 404 }));
await s.run();
expect(s.calls.some((c) => c.url.startsWith('https://r.jina.ai/'))).toBe(false);
expect(s.meta('AAA').source.productPageStatus).toContain('directly');
Expand Down
15 changes: 14 additions & 1 deletion scripts/update-data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,8 @@ const EMPTY_PRICE_RETURNS: PriceReturns = { asOfDate: '', mo1: null, qtd: null,
let requestGates: number[] = [0];
let requestSleepSeconds = 1.5;
/** One run's SEC state: the two SEC ticker tables (each loaded once) and the N-PORT reader cache (submissions and documents). */
/** This run's full daily Yahoo chart answers of the funds processFund refreshed, read once by the dividends and benchmark blocks. */
const fundCharts = new Map<string, JsonRecord>();
let secState: { funds: Promise<Map<string, NportSeriesRef>> | null; names: Promise<Map<string, string>> | null; reader: Map<string, unknown> } = { funds: null, names: null, reader: new Map() };

function sleep(ms: number): Promise<void> {
Expand Down Expand Up @@ -1805,6 +1807,9 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J
chart = mergeWindowedChart(fetched, previousRows, publishedDistributions);
if (!chart) throw new Error(`chart does not join the published history (${config.historyRange}, ${fetched.days.length} days)`);
days = chart.days;
// A full daily chart (period1=0) is the same answer as the dividends block's swingChartUrl request: kept for that block, so a
// fund costs one Yahoo chart request instead of two. A HISTORY_RANGE window is not kept (the block ignores HISTORY_RANGE).
if (query.get('period1') === '0') fundCharts.set(fund.ticker, payload);
fresh = fresh || days.length > 0;
} catch (error) {
outputNote(`[ ${'chart'.padEnd(8)} ] ${fund.ticker}: ${error instanceof Error ? error.message : String(error)}`);
Expand Down Expand Up @@ -2357,6 +2362,7 @@ export async function runUpdate(config: UpdaterConfig, deadlineMs: number = SOFT
requestGates = new Array(Math.max(1, config.concurrency)).fill(0);
proxyNextStart = 0;
directBlocked = false;
fundCharts.clear();
resetSecState();
outputPrintConfig('WisdomTree', config);

Expand Down Expand Up @@ -2465,7 +2471,14 @@ export async function runUpdate(config: UpdaterConfig, deadlineMs: number = SOFT
// benchmark blocks); the SPY and QQQ charts are read once per run before any fund chart
const chartFetch = (url: string, ticker: string) => fetchJson(url, `[ dividend ] ${ticker}`, config, { 'User-Agent': 'Mozilla/5.0' });
const benchmarks = config.skipYahoo ? {} : await benchmarkLoad(chartFetch, (ticker, error) => outputNote(`[ bench ] ${ticker}: ${error instanceof Error ? error.message : String(error)}; published ${ticker} benchmark metrics kept`));
const benchRows = benchmarkTap(chartFetch, benchmarks);
// A fund chart processFund already got in full is taken from memory (once); only the others are requested again.
const fundChartFetch = (url: string, ticker: string): Promise<unknown> => {
const kept = fundCharts.get(ticker);
if (!kept) return chartFetch(url, ticker);
fundCharts.delete(ticker);
return Promise.resolve(kept);
};
const benchRows = benchmarkTap(fundChartFetch, benchmarks);
await writeIndex(await withBenchmark(API_ROOT, await withDividends(API_ROOT, await withSwing(API_ROOT, await withRange52w(API_ROOT, markClosedFunds(funds, CLOSED_FUNDS))), {
fetchChart: benchRows.fetchChart,
tickers: config.skipYahoo ? [] : results.map((row) => String(row.ticker)),
Expand Down