diff --git a/README.md b/README.md index 9a9aced..bbc5667 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ Each fund carries a derived `metrics` object that powers the catalog columns sha - `volatility1y` - annualized volatility of the last 252 daily log returns, in percent (needs 253 closes) - `sigmaFromMean60d` - distance of the last close from the mean of the last 60 closes, in sample standard deviations - `high60d`, `low60d` - highest and lowest of the last 60 closes -- Dividend metrics (shared `dividends` block, the formulas of the Stocks feed) from the Yahoo Finance dividend events of the daily chart since listing (split adjusted; the published distribution table is not), one request per fund the run updates; a failed or truncated chart keeps the published values; `null` (never `0`) when unknown: +- Dividend metrics (shared `dividends` block, the formulas of the Stocks feed) from the Yahoo Finance dividend events of the daily chart since listing (split adjusted; the published distribution table is not), read from the history chart request of each fund the run updates (one extra chart request per fund only when `HISTORY_RANGE` is not `max`); a failed or truncated chart keeps the published values; `null` (never `0`) when unknown: - `dividendRate` - sum of the distributions with ex-dates in the last 12 months - `dividendGrowth`, `dividendGrowth3y`, `dividendGrowth5y` - last complete calendar year against 1, 3, 5 years earlier (3 and 5 as a yearly rate), the listing year is never a base - `dividendStreak` - complete calendar years in a row with a higher sum @@ -118,7 +118,7 @@ Every control is a key in `scripts/update-data.config.json`; values are strings. | --- | --: | --- | | `MAX_FETCHES` | `0` | Batch size over the funds that pass `TICKERS`/`AUM`/`TER`/`DIVIDEND_YIELD`: with a positive value the updater continues after the committed cursor in `api/wisdomtree/update-state.json` (in ticker order, wrapping around; the cursor moves past every fund taken, whatever its outcome); `0` is a full pass. A `TICKERS` run never touches the cursor. | | `REQUEST_SLEEP` | `2` | Minimum delay in seconds between outgoing request starts, including retries. | -| `CONCURRENCY` | `2` | Number of parallel fund update workers, each with its own request pacing lane. | +| `CONCURRENCY` | `4` | Number of parallel fund update workers, each with its own request pacing lane. | | `AUM` | `:` | Net Assets range. Each bound may be a USD amount or `K`/`M`/`B`/`T`, or one of `nano`, `micro`, `small`, `mid`, `large`. | | `TER` | `:` | Expense ratio range in % (strict `min:max`). | | `DIVIDEND_YIELD` | `:` | Dividend-yield percentage range. | diff --git a/scripts/update-data.config.json b/scripts/update-data.config.json index 601dd46..5b2f760 100644 --- a/scripts/update-data.config.json +++ b/scripts/update-data.config.json @@ -3,7 +3,7 @@ "REQUEST_SLEEP": "2", "AUM": ":", "TER": ":", - "CONCURRENCY": "2", + "CONCURRENCY": "4", "HOLDINGS_PAGE_SIZE": "250", "HISTORY_PAGE_SIZE": "1000", "STORE_RAW_DOWNLOADS": "false", diff --git a/scripts/update-data.test.ts b/scripts/update-data.test.ts index 3930cc3..5a8af90 100644 --- a/scripts/update-data.test.ts +++ b/scripts/update-data.test.ts @@ -130,7 +130,7 @@ describe('controls 2', () => { expect(help).toContain(t ? `${t[1]}_YTD|1Y|3Y|5Y|10Y` : ` ${name}=`); } const config = readConfig(resolveControls(file)); - expect([config.maxFetches, config.requestSleep, config.concurrency, config.maxRetries, config.tickers, config.historyRange]).toEqual([0, 2, 2, 2, null, 'max']); + expect([config.maxFetches, config.requestSleep, config.concurrency, config.maxRetries, config.tickers, config.historyRange]).toEqual([0, 2, 4, 2, null, 'max']); expect([config.edgarFallback, config.skipYahoo, config.skipWisdomTree, config.storeRawDownloads, config.aum, config.secYield]).toEqual([true, false, false, false, undefined, undefined]); }); @@ -492,7 +492,7 @@ const BENCHMARK_KEYS = ['perf1y', 'perf3y', 'perf5y', 'perf10y', 'perfSi', 'maxD function chartPayload(days = 400, last = Date.UTC(2026, 8, 25), dividend = false): any { const timestamps = Array.from({ length: days }, (_, i) => Math.floor((last - (days - 1 - i) * 86_400_000) / 1000)); const close = timestamps.map((_, i) => 100 + i * 0.1); - // the dividends block asks for its own chart (includePrePost=false): one 0.25 event on the last bar, listing on the first bar + // a full daily chart (period1=0, as Yahoo answers it): one 0.25 event on the last bar, listing on the first bar const lastBar = timestamps[timestamps.length - 1]; const dividends = dividend ? { [String(lastBar)]: { amount: 0.25, date: lastBar } } : {}; const meta = { exchangeName: 'NGM', regularMarketPrice: 140, regularMarketTime: Math.floor(last / 1000), ...(dividend ? { dataGranularity: '1d', firstTradeDate: timestamps[0] } : {}) }; @@ -524,7 +524,7 @@ class Scenario { if (url.startsWith('https://r.jina.ai/https://dataspanapi.wisdomtree.com/')) return new Response(fundListFor(this.tickers)); if (url.startsWith('https://r.jina.ai/')) return url.endsWith('/us/products') ? new Response(catalogFor(this.tickers)) : new Response(productPage(url.split('/').pop()!.toUpperCase())); if (url.includes('wisdomtree.com')) return new Response('blocked', { status: 403 }); - if (url.includes('finance.yahoo.com')) return new Response(JSON.stringify(chartPayload(400, undefined, url.includes('includePrePost=false')))); + if (url.includes('finance.yahoo.com')) return new Response(JSON.stringify(chartPayload(400, undefined, url.includes('period1=0&')))); return new Response('nope', { status: 404 }); }; async run(env: Record = {}, overrides: Record = {}, deadlineMs?: number): Promise { @@ -578,12 +578,16 @@ describe('pipeline', () => { for (const f of [row, ...rest]) expect(DIVIDEND_KEYS.map((key) => f.metrics[key])).toEqual([0.25, null, null, null, null, null, 1, 'annual']); // the mock gives SPY and QQQ the same bars as every fund: the fund equals both benchmarks (no difference, beta and correlation 1) expect([row.metrics.vsSpy1y, row.metrics.vsQqq1y, row.metrics.beta1ySpy, row.metrics.corr1yQqq, row.metrics.relStrength3mSpy, typeof row.metrics.benchmarkAsOf]).toEqual([0, 0, 1, 1, 0, 'string']); - // one chart per updated fund feeds swing, dividends and benchmark; SPY and QQQ are read once more per run (it differs from the history chart by includePrePost=false), none for an unselected one - const dividendTickers = (from: number) => s.calls.slice(from).filter((call) => /finance\.yahoo\.com.*includePrePost=false/.test(call.url)).map((call) => /chart\/(\w+)\?/.exec(call.url)![1]).sort(); - expect(dividendTickers(0)).toEqual(['AAA', 'BBB', 'CCC', 'QQQ', 'SPY']); - const seen = s.calls.length; + // one full daily chart per updated fund feeds the history, dividends and benchmark (the blocks reuse the history request); SPY and + // QQQ are read once per run, none for an unselected fund; a HISTORY_RANGE window is no full chart, the blocks ask for their own + const chartTickers = (from: number) => s.calls.slice(from).filter((call) => /finance\.yahoo\.com/.test(call.url)).map((call) => /chart\/(\w+)\?/.exec(call.url)![1]).sort(); + expect(chartTickers(0)).toEqual(['AAA', 'BBB', 'CCC', 'QQQ', 'SPY']); + let seen = s.calls.length; await s.run({ TICKERS: 'AAA' }); - expect(dividendTickers(seen)).toEqual(['AAA', 'QQQ', 'SPY']); + expect(chartTickers(seen)).toEqual(['AAA', 'QQQ', 'SPY']); + seen = s.calls.length; + await s.run({ TICKERS: 'AAA', HISTORY_RANGE: '1y' }); + expect(chartTickers(seen)).toEqual(['AAA', 'AAA', 'QQQ', 'SPY']); expect([typeof row.metrics.athPrice, row.metrics.athBasis]).toEqual(['number', expect.stringMatching(/^daily closes since \d{4}-\d{2}-\d{2}$/)]); for (const f of [row, ...rest]) expect([f.metrics.dividendYield, f.metrics.dividendYieldBasis]).toEqual([4.72, 'official-trailing-12m']); expect(s.meta('AAA').yields.dividendYieldBasis).toBe(row.metrics.dividendYieldBasis); @@ -998,7 +1002,7 @@ describe('pipeline 7', () => { const base = s.handler; const listing = (inceptionDt: string) => JSON.stringify({ data: { content: JSON.stringify(fundListRows(['AAA']).map((row) => ({ ...row, inceptionDt }))) } }); let inception = '2025-09-11'; - s.handler = (url, init) => (url.includes('dataspanapi.wisdomtree.com') ? new Response(listing(inception)) : url.includes('finance.yahoo.com') ? new Response(JSON.stringify(chartPayload(380, undefined, url.includes('includePrePost=false')))) : base(url, init)); + s.handler = (url, init) => (url.includes('dataspanapi.wisdomtree.com') ? new Response(listing(inception)) : url.includes('finance.yahoo.com') ? new Response(JSON.stringify(chartPayload(380, undefined, url.includes('period1=0&')))) : base(url, init)); await s.run(); const metrics = s.index().funds[0].metrics; expect([metrics.ytd, metrics.tr1y, metrics.cagr3y, metrics.siAnn, metrics.performanceAsOf]).toEqual([2.59, null, null, null, '2026-08-31']); @@ -1032,7 +1036,7 @@ describe('pipeline 7', () => { // one of a new day that does not join keeps the fund as published await s.run(); expect(s.snapshot()).toEqual(first); - s.handler = (url, init) => (url.includes('finance.yahoo.com') ? new Response(JSON.stringify(chartPayload(1, Date.UTC(2026, 9, 25), url.includes('includePrePost=false')))) : base(url, init)); + s.handler = (url, init) => (url.includes('finance.yahoo.com') ? new Response(JSON.stringify(chartPayload(1, Date.UTC(2026, 9, 25), url.includes('period1=0&')))) : base(url, init)); await s.run(); expect(s.meta('AAA').history.totalRows).toBe(400); }); @@ -1114,7 +1118,7 @@ describe('network', () => { test('a reachable direct page is used and the proxy is never called', async () => { const s = new Scenario(); - s.handler = (url) => secResponse(url, s.nportAsOf) ?? (url.startsWith('https://r.jina.ai/') ? new Response('should not be used', { status: 500 }) : url.startsWith('https://dataspanapi.wisdomtree.com/') ? new Response(JSON.stringify(fundListRows(s.tickers))) : url.includes('wisdomtree.com') ? new Response(productPage(url.split('/').pop()!.toUpperCase())) : url.includes('yahoo') ? new Response(JSON.stringify(chartPayload(400, undefined, url.includes('includePrePost=false')))) : new Response('no', { status: 404 })); + s.handler = (url) => secResponse(url, s.nportAsOf) ?? (url.startsWith('https://r.jina.ai/') ? new Response('should not be used', { status: 500 }) : url.startsWith('https://dataspanapi.wisdomtree.com/') ? new Response(JSON.stringify(fundListRows(s.tickers))) : url.includes('wisdomtree.com') ? new Response(productPage(url.split('/').pop()!.toUpperCase())) : url.includes('yahoo') ? new Response(JSON.stringify(chartPayload(400, undefined, url.includes('period1=0&')))) : new Response('no', { status: 404 })); await s.run(); expect(s.calls.some((c) => c.url.startsWith('https://r.jina.ai/'))).toBe(false); expect(s.meta('AAA').source.productPageStatus).toContain('directly'); diff --git a/scripts/update-data.ts b/scripts/update-data.ts index cc97265..a8b72e1 100755 --- a/scripts/update-data.ts +++ b/scripts/update-data.ts @@ -386,6 +386,8 @@ const EMPTY_PRICE_RETURNS: PriceReturns = { asOfDate: '', mo1: null, qtd: null, let requestGates: number[] = [0]; let requestSleepSeconds = 1.5; /** One run's SEC state: the two SEC ticker tables (each loaded once) and the N-PORT reader cache (submissions and documents). */ +/** This run's full daily Yahoo chart answers of the funds processFund refreshed, read once by the dividends and benchmark blocks. */ +const fundCharts = new Map(); let secState: { funds: Promise> | null; names: Promise> | null; reader: Map } = { funds: null, names: null, reader: new Map() }; function sleep(ms: number): Promise { @@ -1805,6 +1807,9 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J chart = mergeWindowedChart(fetched, previousRows, publishedDistributions); if (!chart) throw new Error(`chart does not join the published history (${config.historyRange}, ${fetched.days.length} days)`); days = chart.days; + // A full daily chart (period1=0) is the same answer as the dividends block's swingChartUrl request: kept for that block, so a + // fund costs one Yahoo chart request instead of two. A HISTORY_RANGE window is not kept (the block ignores HISTORY_RANGE). + if (query.get('period1') === '0') fundCharts.set(fund.ticker, payload); fresh = fresh || days.length > 0; } catch (error) { outputNote(`[ ${'chart'.padEnd(8)} ] ${fund.ticker}: ${error instanceof Error ? error.message : String(error)}`); @@ -2357,6 +2362,7 @@ export async function runUpdate(config: UpdaterConfig, deadlineMs: number = SOFT requestGates = new Array(Math.max(1, config.concurrency)).fill(0); proxyNextStart = 0; directBlocked = false; + fundCharts.clear(); resetSecState(); outputPrintConfig('WisdomTree', config); @@ -2465,7 +2471,14 @@ export async function runUpdate(config: UpdaterConfig, deadlineMs: number = SOFT // benchmark blocks); the SPY and QQQ charts are read once per run before any fund chart const chartFetch = (url: string, ticker: string) => fetchJson(url, `[ dividend ] ${ticker}`, config, { 'User-Agent': 'Mozilla/5.0' }); const benchmarks = config.skipYahoo ? {} : await benchmarkLoad(chartFetch, (ticker, error) => outputNote(`[ bench ] ${ticker}: ${error instanceof Error ? error.message : String(error)}; published ${ticker} benchmark metrics kept`)); - const benchRows = benchmarkTap(chartFetch, benchmarks); + // A fund chart processFund already got in full is taken from memory (once); only the others are requested again. + const fundChartFetch = (url: string, ticker: string): Promise => { + const kept = fundCharts.get(ticker); + if (!kept) return chartFetch(url, ticker); + fundCharts.delete(ticker); + return Promise.resolve(kept); + }; + const benchRows = benchmarkTap(fundChartFetch, benchmarks); await writeIndex(await withBenchmark(API_ROOT, await withDividends(API_ROOT, await withSwing(API_ROOT, await withRange52w(API_ROOT, markClosedFunds(funds, CLOSED_FUNDS))), { fetchChart: benchRows.fetchChart, tickers: config.skipYahoo ? [] : results.map((row) => String(row.ticker)),