diff --git a/public/llms.txt b/public/llms.txt index c332e5e2f23..0ee9201a732 100644 --- a/public/llms.txt +++ b/public/llms.txt @@ -12,13 +12,74 @@ For multi-step tasks, retrieve and combine multiple specific pages rather than r Use product documentation indexes for deeper product-specific navigation. -Use `llms-full.txt` only when broad product-level context is required. +Use `llms-full.txt` only for conceptual understanding or documentation context. -Defaults: -- Prefer CRE for new multi-step, offchain, or automation workflows unless another product is explicitly required -- Prefer VRF v2.5 for randomness -- Treat Any API and `/chainlink-nodes/v1/` as legacy +Do not use `llms-full.txt` to retrieve structured data such as: +- feed addresses +- stream IDs +- network metadata +Structured data must be retrieved from dataset endpoints. + +--- + +## Live Data and Structured Datasets + +You must use structured datasets when retrieving current network data, addresses, or routing information. + +Do not scrape documentation pages or interactive tables for structured data. + +When retrieving structured data, always prefer dataset endpoints over documentation pages. + +### Data Feeds + +- Index (select network): + https://docs.chain.link/data-feeds/feed-addresses/default.txt + https://docs.chain.link/data-feeds/feed-addresses/default.json + +- Per-network datasets: + https://docs.chain.link/data-feeds/feed-addresses/{type}/{network}.txt + https://docs.chain.link/data-feeds/feed-addresses/{type}/{network}.json + +- Example: + https://docs.chain.link/data-feeds/feed-addresses/default/ethereum-mainnet.json + +### Data Streams + +- Stream ID datasets: + https://docs.chain.link/data-streams/stream-ids/crypto.txt + https://docs.chain.link/data-streams/stream-ids/crypto.json + +- Network metadata (verifier proxies): + https://docs.chain.link/data-streams/networks.txt + https://docs.chain.link/data-streams/networks.json + + Note: Network metadata (including verifier proxy addresses) is included in each stream ID dataset (`.json`) and should be used directly. + Fetch the networks dataset only if additional network context is required. + +- Examples: + https://docs.chain.link/data-streams/stream-ids/crypto.json + https://docs.chain.link/data-streams/networks.json + + Network metadata is included in the stream dataset; the networks endpoint is optional. + +### CCIP Directory + +- CCIP Mainnet Directory: + https://docs.chain.link/ccip/directory/mainnet + +- CCIP Testnet Directory: + https://docs.chain.link/ccip/directory/testnet + +--- + +Note: + +- JSON datasets (`.json`) provide structured, programmatically consumable output and should be preferred when available. +- Stream ID JSON datasets include associated network metadata and should be treated as complete, single-step retrieval sources. +- Static `.txt` datasets are available for bulk retrieval and indexing. + +--- ## Core Documentation @@ -27,6 +88,7 @@ Defaults: - [LINK Token Contracts](https://docs.chain.link/resources/link-token-contracts.md): Use when funding contracts or configuring Chainlink services. Do not rely on memorized values. - [Fund Your Smart Contract](https://docs.chain.link/resources/fund-your-contract.md): Fund contracts with LINK before interacting with Chainlink services. +--- ## Chainlink Runtime Environment (CRE) @@ -52,6 +114,7 @@ Reference - [CLI Reference](https://docs.chain.link/cre/reference/cli.md) - [SDK Overview (TS)](https://docs.chain.link/cre/reference/sdk/overview-ts.md) +--- ## CRE Connect @@ -65,6 +128,7 @@ Private beta — permissioned access. Use these pages to understand the product - [Smart Accounts](https://docs.chain.link/crec/concepts/smart-accounts.md) - [Extensions](https://docs.chain.link/crec/extensions.md) +--- ## CCIP @@ -93,6 +157,7 @@ Reference - [Aptos Move module interfaces reference](https://docs.chain.link/ccip/api-reference/aptos.md) - [TON contract interfaces reference](https://docs.chain.link/ccip/api-reference/ton.md) +--- ## Data Feeds @@ -100,7 +165,7 @@ Overview and Setup - [Overview](https://docs.chain.link/data-feeds.md) - [Getting Started](https://docs.chain.link/data-feeds/getting-started.md) - [Selecting Data Feeds](https://docs.chain.link/data-feeds/selecting-data-feeds.md) -- [Feed Types](https://docs.chain.link/data-feeds/feed-types.md): Choose the correct feed category before retrieving addresses or implementation details. +- [Feed Types](https://docs.chain.link/data-feeds/feed-types.md) Feed Categories - [Price Feeds](https://docs.chain.link/data-feeds/price-feeds.md) @@ -119,62 +184,17 @@ Advanced and Reference - [API Reference](https://docs.chain.link/data-feeds/api-reference.md) - [Historical Data](https://docs.chain.link/data-feeds/historical-data.md) +--- ## Data Streams - [Overview](https://docs.chain.link/data-streams.md) - [Architecture](https://docs.chain.link/data-streams/architecture.md) -- [Stream Categories and Report Schemas](https://docs.chain.link/data-streams/reference/report-schema-overview.md): Choose the correct stream category and schema before implementation. +- [Stream Categories and Report Schemas](https://docs.chain.link/data-streams/reference/report-schema-overview.md) - [Supported Networks](https://docs.chain.link/data-streams/supported-networks.md) - [API Reference](https://docs.chain.link/data-streams/reference/data-streams-api.md) - -## DataLink - -- [Overview](https://docs.chain.link/datalink.md) -- [Pull Delivery Overview](https://docs.chain.link/datalink/pull-delivery/overview.md) -- [Architecture](https://docs.chain.link/datalink/pull-delivery/architecture.md) - - -## Automated Compliance Engine (ACE) - -- [Overview](https://docs.chain.link/ace.md) -- [Getting Started](https://docs.chain.link/ace/getting-started.md) -- [Concepts](https://docs.chain.link/ace/concepts/architecture.md) -- [API Reference](https://docs.chain.link/ace/reference/api/coordinator.md) - - -## DTA Technical Standard - -- [Overview](https://docs.chain.link/dta-technical-standard.md) -- [How It Works](https://docs.chain.link/dta-technical-standard/how-it-works.md) -- [Architecture](https://docs.chain.link/dta-technical-standard/concepts/architecture.md) -- [Actors](https://docs.chain.link/dta-technical-standard/actors.md) - - -## Chainlink Automation - -- [Overview](https://docs.chain.link/chainlink-automation.md) -- [Getting Started](https://docs.chain.link/chainlink-automation/overview/getting-started.md) -- [Automation Architecture](https://docs.chain.link/chainlink-automation/concepts/automation-architecture.md) -- [Service Limits](https://docs.chain.link/chainlink-automation/overview/service-limits.md) - - -## Chainlink Functions - -- [Overview](https://docs.chain.link/chainlink-functions.md) -- [Getting Started](https://docs.chain.link/chainlink-functions/getting-started.md) -- [Architecture](https://docs.chain.link/chainlink-functions/resources/architecture.md) -- [API Reference](https://docs.chain.link/chainlink-functions/api-reference/functions-client.md) - - -## VRF - -- [Overview](https://docs.chain.link/vrf.md) -- [Getting Started v2.5](https://docs.chain.link/vrf/v2-5/getting-started.md) -- [Best Practices](https://docs.chain.link/vrf/v2-5/best-practices.md) -- [Migration Guides](https://docs.chain.link/vrf/v2-5/migration-from-v2.md) -- [Supported Networks](https://docs.chain.link/vrf/v2-5/supported-networks.md) +--- ## Coverage model @@ -184,6 +204,8 @@ For full coverage: - Use product documentation indexes for structured navigation - Use full documentation bundles when broad context is required +--- + ## Product Documentation Indexes Use these for deeper product-level navigation. These indexes provide broader coverage than the curated sections above while preserving page-level retrieval. @@ -199,21 +221,13 @@ Use these for deeper product-level navigation. These indexes provide broader cov - [DTA Technical Standard Documentation Index](https://docs.chain.link/dta-technical-standard/llms.txt) - [VRF Documentation Index](https://docs.chain.link/vrf/llms.txt) -## Live Data and Directories - -Use these only when retrieving current network data, addresses, or routing information. - -If `.md` is unavailable or incomplete, use the HTML page as the source of truth. Do not infer or reconstruct missing values. - -- [CCIP Mainnet Directory](https://docs.chain.link/ccip/directory/mainnet) -- [CCIP Testnet Directory](https://docs.chain.link/ccip/directory/testnet) -- [Price Feed Addresses](https://docs.chain.link/data-feeds/price-feeds/addresses) -- [Cryptocurrency Streams](https://docs.chain.link/data-streams/crypto-streams) - +--- ## Full Documentation Bundles -Use `llms-full.txt` only when broad product-level context is required or when the relevant page cannot be identified from the curated index. +Use only for conceptual understanding or when no dataset or specific page can be identified. + +Do not use `llms-full.txt` for retrieving structured data such as feed addresses, stream IDs, or network metadata. - [CRE Full](https://docs.chain.link/cre/llms-full-ts.txt) - [CRE Connect Full](https://docs.chain.link/crec/llms-full.txt) diff --git a/src/content/data-feeds/llms-full.txt b/src/content/data-feeds/llms-full.txt index 272a908d29e..3f2d5e5f706 100644 --- a/src/content/data-feeds/llms-full.txt +++ b/src/content/data-feeds/llms-full.txt @@ -3559,7 +3559,18 @@ Check our [implementation guides](/data-feeds/mvr-feeds/guides) for detailed ins # Price Feed Contract Addresses Source: https://docs.chain.link/data-feeds/price-feeds/addresses - +
+ {`DATA ACCESS (REQUIRED): + You must retrieve feed addresses from structured datasets. + + Use: + /data-feeds/feed-addresses/default/{network}.json + + Example: + https://docs.chain.link/data-feeds/feed-addresses/default/ethereum-mainnet.json + + Do not extract feed addresses from this page.`} +
--- @@ -3573,7 +3584,18 @@ Chainlink Data Feeds provide data that is aggregated from many data sources by a # Rate and Volatility Feed Addresses Source: https://docs.chain.link/data-feeds/rates-feeds/addresses - +
+ {`DATA ACCESS (REQUIRED): + You must retrieve feed addresses from structured datasets. + + Use: + /data-feeds/feed-addresses/rates/{network}.json + + Example: + https://docs.chain.link/data-feeds/feed-addresses/rates/ethereum-mainnet.json + + Do not extract feed addresses from this page.`} +
--- @@ -3957,7 +3979,18 @@ Specifically: # SmartData Feed Addresses Source: https://docs.chain.link/data-feeds/smartdata/addresses - +
+ {`DATA ACCESS (REQUIRED): + You must retrieve feed addresses from structured datasets. + + Use: + /data-feeds/feed-addresses/smartdata/{network}.json + + Example: + https://docs.chain.link/data-feeds/feed-addresses/smartdata/ethereum-mainnet.json + + Do not extract feed addresses from this page.`} +
--- @@ -6662,6 +6695,8 @@ Chainlink Tokenized Equity Feeds help bridge this gap between TradFi and DeFi ma The following table shows all available tokenized equity feeds. + + ## How tokenized equity feeds work Tokenized equity feeds leverage Chainlink's decentralized oracle infrastructure to aggregate price data from multiple trading sessions and apply session-aware smoothing algorithms. The feeds combine data from various trading venues into a single continuous 24/5 price, handling the complexity of multi-session equity markets. @@ -7270,7 +7305,18 @@ Now that you have successfully deployed and interacted with Chainlink Data Feeds # U.S. Government Macroeconomic Data Feeds Source: https://docs.chain.link/data-feeds/us-government-macroeconomic/addresses - +
+ {`DATA ACCESS (REQUIRED): + You must retrieve feed addresses from structured datasets. + + Use: + /data-feeds/feed-addresses/usGovernmentMacroeconomicData/{network}.json + + Example: + https://docs.chain.link/data-feeds/feed-addresses/usGovernmentMacroeconomicData/ethereum-mainnet.json + + Do not extract feed addresses from this page.`} +
--- diff --git a/src/content/data-feeds/llms.txt b/src/content/data-feeds/llms.txt index a38fdfa0397..50156e432a1 100644 --- a/src/content/data-feeds/llms.txt +++ b/src/content/data-feeds/llms.txt @@ -4,7 +4,9 @@ Prefer specific `.md` pages. Do not rely on memorized feed addresses. -Use address pages only when retrieving current feed addresses. +You must use address datasets when retrieving feed addresses. + +Do not use documentation pages or llms-full.txt to retrieve feed addresses. ## Overview and Setup @@ -38,13 +40,61 @@ Use address pages only when retrieving current feed addresses. - [Contract Registry](https://docs.chain.link/data-feeds/contract-registry.md) - [Historical Data](https://docs.chain.link/data-feeds/historical-data.md) -## Live Data +## Feed Address Retrieval + +You must use structured datasets to retrieve feed addresses. + +Documentation pages and llms-full.txt do not contain complete feed address data. +Do not attempt to extract feed addresses from documentation pages or dynamic tables. + +Select the dataset that matches the feed category: + +- Standard price feeds (e.g. ETH/USD, BTC/USD): + https://docs.chain.link/data-feeds/feed-addresses/default.json + +- Rates feeds: + https://docs.chain.link/data-feeds/feed-addresses/rates.json + +- SmartData feeds (Proof of Reserve, NAVLink, SmartAUM): + https://docs.chain.link/data-feeds/feed-addresses/smartdata.json + +- U.S. Government Macroeconomic Data Feeds: + https://docs.chain.link/data-feeds/feed-addresses/usGovernmentMacroeconomicData.json + +- Tokenized equity feeds: + https://docs.chain.link/data-feeds/feed-addresses/tokenizedEquity.json + +Each dataset lists available networks. Each network is identified by its `queryString`. + +Retrieve a specific dataset: + +- https://docs.chain.link/data-feeds/feed-addresses/{type}/{network}.json + +For a specific network, you should directly retrieve the per-network dataset instead of scanning the full index. + +You must first retrieve the dataset index to determine valid network queryStrings before requesting a per-network dataset. +Usage pattern: + +1. Select the dataset that matches the feed category +2. Retrieve the dataset index: + https://docs.chain.link/data-feeds/feed-addresses/{type}.json +3. Use this dataset to determine the correct network `queryString` +4. Retrieve the corresponding per-network dataset (`.json`) +5. Filter by feed name by matching the `name` field + +Static `.txt` datasets are available for bulk retrieval and indexing only: + +- https://docs.chain.link/data-feeds/feed-addresses/default.txt +- https://docs.chain.link/data-feeds/feed-addresses/rates.txt +- https://docs.chain.link/data-feeds/feed-addresses/smartdata.txt +- https://docs.chain.link/data-feeds/feed-addresses/tokenizedEquity.txt +- https://docs.chain.link/data-feeds/feed-addresses/usGovernmentMacroeconomicData.txt -Use only when retrieving current addresses. If `.md` output is incomplete, use the HTML page as the source of truth. +Constraints: -- [Price Feed Addresses](https://docs.chain.link/data-feeds/price-feeds) -- [Rates Feed Addresses](https://docs.chain.link/data-feeds/rates-feeds/addresses) -- [SmartData Addresses](https://docs.chain.link/data-feeds/smartdata/addresses) -- [U.S. Government Macroeconomic Data Feeds](https://docs.chain.link/data-feeds/us-government-macroeconomic/addresses) -- [Rate and Volatility Feeds](https://docs.chain.link/data-feeds/rates-feeds/addresses) -- [Deprecating Feeds](https://docs.chain.link/data-feeds/deprecating-feeds) +- Each network dataset contains the complete feed list for that network +- JSON datasets are structured and should be parsed programmatically +- Loading all networks will significantly increase response size and latency +- Do not load multiple networks unless required +- Do not scrape documentation pages or interactive tables for feed addresses +- Do not rely on memorized or cached addresses \ No newline at end of file diff --git a/src/content/data-feeds/price-feeds/addresses.mdx b/src/content/data-feeds/price-feeds/addresses.mdx index 52d5e6f4cdd..352950800b5 100644 --- a/src/content/data-feeds/price-feeds/addresses.mdx +++ b/src/content/data-feeds/price-feeds/addresses.mdx @@ -9,4 +9,18 @@ date: Last Modified import FeedPage from "@features/feeds/components/FeedPage.astro" +
+{`DATA ACCESS (REQUIRED): +You must retrieve feed addresses from structured datasets. + +Use: +/data-feeds/feed-addresses/default/{network}.json + +Example: +https://docs.chain.link/data-feeds/feed-addresses/default/ethereum-mainnet.json + +Do not extract feed addresses from this page.`} + +
+ diff --git a/src/content/data-feeds/rates-feeds/addresses.mdx b/src/content/data-feeds/rates-feeds/addresses.mdx index 214c4d67c99..ad229bfac41 100644 --- a/src/content/data-feeds/rates-feeds/addresses.mdx +++ b/src/content/data-feeds/rates-feeds/addresses.mdx @@ -10,4 +10,18 @@ date: Last Modified import FeedPage from "@features/feeds/components/FeedPage.astro" +
+{`DATA ACCESS (REQUIRED): +You must retrieve feed addresses from structured datasets. + +Use: +/data-feeds/feed-addresses/rates/{network}.json + +Example: +https://docs.chain.link/data-feeds/feed-addresses/rates/ethereum-mainnet.json + +Do not extract feed addresses from this page.`} + +
+ diff --git a/src/content/data-feeds/smartdata/addresses.mdx b/src/content/data-feeds/smartdata/addresses.mdx index 97228dd7648..a8bf81d0d63 100644 --- a/src/content/data-feeds/smartdata/addresses.mdx +++ b/src/content/data-feeds/smartdata/addresses.mdx @@ -10,4 +10,18 @@ date: Last Modified import FeedPage from "@features/feeds/components/FeedPage.astro" +
+{`DATA ACCESS (REQUIRED): +You must retrieve feed addresses from structured datasets. + +Use: +/data-feeds/feed-addresses/smartdata/{network}.json + +Example: +https://docs.chain.link/data-feeds/feed-addresses/smartdata/ethereum-mainnet.json + +Do not extract feed addresses from this page.`} + +
+ diff --git a/src/content/data-feeds/tokenized-equity-feeds/index.mdx b/src/content/data-feeds/tokenized-equity-feeds/index.mdx index 78867d9d650..61e7eeda37f 100644 --- a/src/content/data-feeds/tokenized-equity-feeds/index.mdx +++ b/src/content/data-feeds/tokenized-equity-feeds/index.mdx @@ -63,6 +63,20 @@ Chainlink Tokenized Equity Feeds help bridge this gap between TradFi and DeFi ma The following table shows all available tokenized equity feeds. +
+{`DATA ACCESS (REQUIRED): +You must retrieve feed addresses from structured datasets. + +Use: +/data-feeds/feed-addresses/tokenizedEquity/{network}.json + +Example: +https://docs.chain.link/data-feeds/feed-addresses/tokenizedEquity/ethereum-mainnet.json + +Do not extract feed addresses from this page.`} + +
+ ## How tokenized equity feeds work diff --git a/src/content/data-feeds/us-government-macroeconomic/addresses.mdx b/src/content/data-feeds/us-government-macroeconomic/addresses.mdx index 2b865c41d26..099d86901cd 100644 --- a/src/content/data-feeds/us-government-macroeconomic/addresses.mdx +++ b/src/content/data-feeds/us-government-macroeconomic/addresses.mdx @@ -9,4 +9,18 @@ date: Last Modified import FeedPage from "@features/feeds/components/FeedPage.astro" +
+{`DATA ACCESS (REQUIRED): +You must retrieve feed addresses from structured datasets. + +Use: +/data-feeds/feed-addresses/usGovernmentMacroeconomicData/{network}.json + +Example: +https://docs.chain.link/data-feeds/feed-addresses/usGovernmentMacroeconomicData/ethereum-mainnet.json + +Do not extract feed addresses from this page.`} + +
+ diff --git a/src/content/data-streams/crypto-streams/index.mdx b/src/content/data-streams/crypto-streams/index.mdx index 2078a12fa91..d63aa442b54 100644 --- a/src/content/data-streams/crypto-streams/index.mdx +++ b/src/content/data-streams/crypto-streams/index.mdx @@ -11,6 +11,25 @@ isIndex: true import FeedPage from "@features/feeds/components/FeedPage.astro" +
+{`DATA ACCESS (REQUIRED): +You must retrieve stream IDs from structured datasets. + +Use: +/data-streams/stream-ids/{type}.json + +Example: +https://docs.chain.link/data-streams/stream-ids/crypto.json + +Network metadata is included in the dataset. + +Do not extract stream IDs from this page. + +IMPORTANT: +Stream IDs are not network-specific.`} + +
+ +{`DATA ACCESS (REQUIRED): +You must retrieve stream IDs from structured datasets. + +Use: +/data-streams/stream-ids/exchangeRate.json + +Example: +https://docs.chain.link/data-streams/stream-ids/exchangeRate.json + +Network metadata is included in the dataset. + +Do not extract stream IDs from this page. + +IMPORTANT: +Stream IDs are not network-specific.`} + + + +
+ {`DATA ACCESS (REQUIRED): + You must retrieve stream IDs from structured datasets. + + Use: + /data-streams/stream-ids/{type}.json + + Example: + https://docs.chain.link/data-streams/stream-ids/crypto.json + + Network metadata is included in the dataset. + + Do not extract stream IDs from this page. + + IMPORTANT: + Stream IDs are not network-specific.`} +
--- @@ -863,7 +879,23 @@ Developers are responsible for understanding and managing all additional risk fa # Exchange Rate Data Streams Source: https://docs.chain.link/data-streams/exchange-rate-streams - +
+ {`DATA ACCESS (REQUIRED): + You must retrieve stream IDs from structured datasets. + + Use: + /data-streams/stream-ids/exchangeRate.json + + Example: + https://docs.chain.link/data-streams/stream-ids/exchangeRate.json + + Network metadata is included in the dataset. + + Do not extract stream IDs from this page. + + IMPORTANT: + Stream IDs are not network-specific.`} +
--- @@ -6289,14 +6321,46 @@ Announcements can cause sharp price spikes or sustained moves. # Real World Asset (RWA) Data Streams Source: https://docs.chain.link/data-streams/rwa-streams - +
+ {`DATA ACCESS (REQUIRED): + You must retrieve stream IDs from structured datasets. + + Use: + /data-streams/stream-ids/rwa.json + + Example: + https://docs.chain.link/data-streams/stream-ids/rwa.json + + Network metadata is included in the dataset. + + Do not extract stream IDs from this page. + + IMPORTANT: + Stream IDs are not network-specific.`} +
--- # SmartData Streams Source: https://docs.chain.link/data-streams/smartdata-streams - +
+ {`DATA ACCESS (REQUIRED): + You must retrieve stream IDs from structured datasets. + + Use: + /data-streams/stream-ids/smartdata.json + + Example: + https://docs.chain.link/data-streams/stream-ids/smartdata.json + + Network metadata is included in the dataset. + + Do not extract stream IDs from this page. + + IMPORTANT: + Stream IDs are not network-specific.`} +
--- @@ -6700,6 +6764,8 @@ Source: https://docs.chain.link/data-streams/tokenized-asset-streams Developers are solely responsible for monitoring and mitigating market integrity risks, as outlined in the [Developer Responsibilities](/data-streams/developer-responsibilities) documentation. + + --- # Verify report data onchain (EVM) diff --git a/src/content/data-streams/llms.txt b/src/content/data-streams/llms.txt index eabc9d5513c..434c9b1a0d8 100644 --- a/src/content/data-streams/llms.txt +++ b/src/content/data-streams/llms.txt @@ -4,7 +4,9 @@ Prefer specific `.md` pages. Choose the stream category and report schema before implementation. -Do not rely on memorized supported networks, stream IDs, verifier addresses, or schema details. +You must use structured datasets to retrieve stream IDs and network metadata. + +Do not use documentation pages or llms-full.txt to retrieve stream IDs, schemas, or verifier addresses. ## Overview and Concepts @@ -17,11 +19,12 @@ Do not rely on memorized supported networks, stream IDs, verifier addresses, or Use these pages to choose the correct stream category and report schema before implementation. -These pages may include data-driven or dynamically rendered content. If a `.md` page is incomplete or unavailable, use the corresponding HTML page as the source of truth. Do not infer stream IDs, schemas, supported networks, or verifier details. +These pages may include data-driven or dynamically rendered content. Do not infer stream IDs, schemas, supported networks, or verifier details from documentation pages. - [Stream Categories and Report Schemas](https://docs.chain.link/data-streams/reference/report-schema-overview.md): Start here to choose the correct category and schema. Primary Categories + - [Cryptocurrency Streams](https://docs.chain.link/data-streams/crypto-streams) - [Exchange Rate Streams](https://docs.chain.link/data-streams/exchange-rate-streams) - [Real World Asset (RWA) Streams](https://docs.chain.link/data-streams/rwa-streams) @@ -42,6 +45,61 @@ Additional Data and Lifecycle - [Fetch Reports with Rust SDK](https://docs.chain.link/data-streams/tutorials/rust-sdk-fetch.md) - [Stream Reports with Rust SDK](https://docs.chain.link/data-streams/tutorials/rust-sdk-stream.md) +## Stream ID Retrieval + +You must use structured datasets to retrieve stream IDs and network metadata. + +Documentation pages and llms-full.txt do not contain complete stream ID or network data. +Do not attempt to extract stream IDs or verifier addresses from documentation pages or dynamic tables. + +First determine the correct stream category and schema: + +- https://docs.chain.link/data-streams/reference/report-schema-overview.md + +Then select the dataset that matches that category: + +- Cryptocurrency streams (e.g. BTC/USD, ETH/USD): + https://docs.chain.link/data-streams/stream-ids/crypto.json + +- Real World Asset (RWA) streams: + https://docs.chain.link/data-streams/stream-ids/rwa.json + +- SmartData streams: + https://docs.chain.link/data-streams/stream-ids/smartdata.json + +- Exchange rate streams: + https://docs.chain.link/data-streams/stream-ids/exchangeRate.json + +- Tokenized asset streams: + https://docs.chain.link/data-streams/stream-ids/tokenizedAsset.json + +Stream IDs are universal and valid across all supported networks. + +Network metadata (including verifier proxy addresses) is included in each stream dataset. + +Do not fetch a separate networks dataset unless explicitly required. + +Static `.txt` datasets are available for bulk retrieval and indexing only: + +- https://docs.chain.link/data-streams/stream-ids/crypto.txt +- https://docs.chain.link/data-streams/networks.txt + +## Retrieval Pattern + +1. Select the correct stream category and schema +2. Retrieve the structured dataset for that category (`.json`) +3. Identify the required stream ID by matching the `name` field +4. Use the embedded network metadata to determine the correct verifier proxy for the target network + +## Constraints + +- Stream IDs are not network-specific +- Datasets may contain multiple schema versions +- JSON datasets are structured and should be parsed programmatically +- Do not duplicate requests per network +- Do not scrape documentation pages or interactive tables +- Do not rely on memorized or cached values + ## Operations and Billing - [Billing](https://docs.chain.link/data-streams/billing.md) @@ -49,4 +107,4 @@ Additional Data and Lifecycle ## Full Context -- [Data Streams Full](https://docs.chain.link/data-streams/llms-full.txt) +- [Data Streams Full](https://docs.chain.link/data-streams/llms-full.txt) \ No newline at end of file diff --git a/src/content/data-streams/rwa-streams/index.mdx b/src/content/data-streams/rwa-streams/index.mdx index 3c7b6719853..70f04cd92d3 100644 --- a/src/content/data-streams/rwa-streams/index.mdx +++ b/src/content/data-streams/rwa-streams/index.mdx @@ -11,6 +11,25 @@ isIndex: true import FeedPage from "@features/feeds/components/FeedPage.astro" +
+{`DATA ACCESS (REQUIRED): +You must retrieve stream IDs from structured datasets. + +Use: +/data-streams/stream-ids/rwa.json + +Example: +https://docs.chain.link/data-streams/stream-ids/rwa.json + +Network metadata is included in the dataset. + +Do not extract stream IDs from this page. + +IMPORTANT: +Stream IDs are not network-specific.`} + +
+ +{`DATA ACCESS (REQUIRED): +You must retrieve stream IDs from structured datasets. + +Use: +/data-streams/stream-ids/smartdata.json + +Example: +https://docs.chain.link/data-streams/stream-ids/smartdata.json + +Network metadata is included in the dataset. + +Do not extract stream IDs from this page. + +IMPORTANT: +Stream IDs are not network-specific.`} + + + +
+{`DATA ACCESS (REQUIRED): +You must retrieve stream IDs from structured datasets. + +Use: +/data-streams/stream-ids/tokenizedAsset.json + +Example: +https://docs.chain.link/data-streams/stream-ids/tokenizedAsset.json + +Network metadata is included in the dataset. + +Do not extract stream IDs from this page. + +IMPORTANT: +Stream IDs are not network-specific.`} + +
+ = { + default: "Standard Price Feeds", + smartdata: "SmartData Feeds (Proof of Reserve, NAVLink, SmartAUM)", + rates: "Rates Feeds", + streamsCrypto: "Data Streams — Crypto", + streamsRwa: "Data Streams — RWA", + streamsNav: "Data Streams — SmartData (NAV)", + streamsExRate: "Data Streams — Exchange Rate", + streamsBacked: "Data Streams — Tokenized Assets", + tokenizedEquity: "Tokenized Equity Feeds", + usGovernmentMacroeconomicData: "US Government Macroeconomic Data Feeds", +} + +export function isStreamsType(type: DataFeedType): boolean { + return ( + type === "streamsCrypto" || + type === "streamsRwa" || + type === "streamsNav" || + type === "streamsExRate" || + type === "streamsBacked" + ) +} + +export function formatHeartbeat(seconds: number): string { + if (!seconds) return "N/A" + if (seconds < 60) return `${seconds}s` + if (seconds < 3600) return `${Math.round(seconds / 60)}m` + return `${Math.round(seconds / 3600)}h` +} + +export function formatDeviation(threshold: number): string { + if (threshold == null || threshold === 0) return "N/A" + return `${threshold}%` +} + +export interface FeedMarkdownOptions { + schemaFilter?: string + publicType?: string + networkType?: "mainnet" | "testnet" +} + +export interface StreamEntry { + name: string + feedId: string + assetClass: string + schema: string + tradingHours?: string +} + +export interface FeedEntry { + name: string + proxyAddress: string + deviation: string + heartbeat: string + network: string + networkName: string + networkType: string + chain: string + svr?: "shared" | "aave" +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function resolveSVR(feed: any): "shared" | "aave" | undefined { + const isShared = typeof feed.path === "string" && /-shared-svr$/.test(feed.path) + if (isShared) return "shared" + if (feed.secondaryProxyAddress) return "aave" + return undefined +} + +export function collectStreamEntries( + type: DataFeedType, + chainCache: Record, + options: FeedMarkdownOptions = {} +): StreamEntry[] { + const visibilityOpts = { + ...(options.schemaFilter ? { schemaFilter: options.schemaFilter } : {}), + } as any + + const seenFeedIds = new Set() + const raw: any[] = [] + + for (const chain of CHAINS) { + const chainData = chainCache[chain.page] + if (!chainData?.networks) continue + + for (const network of chainData.networks as any[]) { + if (!network?.metadata?.length) continue + if (network.networkType !== "mainnet") continue + + for (const feed of network.metadata as any[]) { + if (!isFeedVisible(feed, type, "", visibilityOpts)) continue + if (!feed.feedId || seenFeedIds.has(feed.feedId)) continue + + seenFeedIds.add(feed.feedId) + raw.push(feed) + } + } + } + + raw.sort((a, b) => (resolveStreamPair(a) ?? "").localeCompare(resolveStreamPair(b) ?? "")) + + return raw.flatMap((feed: any) => { + const name = resolveStreamPair(feed) + if (!name) return [] + + const tradingHours = resolveTradingHours(feed) + + const entry: StreamEntry = { + name, + feedId: feed.feedId, + assetClass: resolveAssetClass(feed), + schema: resolveStreamSchema(type, feed), + } + + if (tradingHours && tradingHours !== "—") { + entry.tradingHours = tradingHours + } + + return [entry] + }) +} + +function resolveChainName(chainTitle: string, chainPage: string): string { + const stripped = chainTitle.replace(/\s*Data Feeds$/, "").trim() + return stripped || chainPage.charAt(0).toUpperCase() + chainPage.slice(1) +} + +export function collectFeedEntries( + type: DataFeedType, + networkFilter: string | null, + chainCache: Record, + options: FeedMarkdownOptions = {} +): FeedEntry[] { + const visibilityOpts = { + ...(options.schemaFilter ? { schemaFilter: options.schemaFilter } : {}), + } as any + + const entries: FeedEntry[] = [] + + for (const chain of CHAINS) { + const chainData = chainCache[chain.page] + if (!chainData?.networks) continue + + const chainName = resolveChainName(chain.title, chain.page) + + for (const network of chainData.networks as any[]) { + if (!network?.metadata?.length) continue + if (networkFilter && network.queryString !== networkFilter) continue + if (options.networkType && network.networkType !== options.networkType) continue + + const visibleFeeds = network.metadata.filter((feed: any) => isFeedVisible(feed, type, "", visibilityOpts)) + + if (visibleFeeds.length === 0) continue + + for (const feed of visibleFeeds as any[]) { + const svr = resolveSVR(feed) + + const entry: FeedEntry = { + name: feed.name || feed.assetName || "Unknown", + proxyAddress: feed.proxyAddress || feed.transmissionsAccount || feed.contractAddress || "N/A", + deviation: formatDeviation(feed.threshold), + heartbeat: formatHeartbeat(feed.heartbeat), + network: network.queryString, + networkName: network.name, + networkType: network.networkType, + chain: chainName, + } + + if (svr) entry.svr = svr + + entries.push(entry) + } + } + } + + entries.sort((a, b) => { + if (a.network !== b.network) return a.network.localeCompare(b.network) + return a.name.localeCompare(b.name) + }) + + return entries +} + +export function buildFeedAddressMarkdown( + type: DataFeedType, + networkFilter: string | null, + chainCache: Record, + siteBase = "https://docs.chain.link", + options: FeedMarkdownOptions = {} +): string { + const lines: string[] = [] + const streams = isStreamsType(type) + const label = FEED_TYPE_LABELS[type] + + if (streams) { + lines.push(`# Chainlink Data Streams: ${label}`) + lines.push(`Source: ${siteBase}/data-streams`) + lines.push("") + lines.push(`> Stream IDs for Chainlink Data Streams – ${label.replace("Data Streams — ", "")}.`) + lines.push(`> These IDs are universal and valid across all supported networks.`) + lines.push( + `> To use a stream ID, retrieve the verifier proxy for the target network from /data-streams/networks.txt.` + ) + lines.push(`> Datasets may contain multiple schema versions. Filter by schema if needed.`) + lines.push("") + + const streamEntries = collectStreamEntries(type, chainCache, options) + + if (streamEntries.length > 0) { + lines.push("| Stream | Feed ID | Asset Class | Schema | Trading Hours |") + lines.push("|--------|---------|-------------|--------|---------------|") + + for (const entry of streamEntries) { + lines.push( + `| ${entry.name} | \`${entry.feedId}\` | ${entry.assetClass} | \`${entry.schema}\` | ${entry.tradingHours ?? "—"} |` + ) + } + + lines.push("") + } else { + lines.push(`No stream IDs found for type \`${type}\`.`) + } + } else { + lines.push(`# Chainlink Feed Addresses: ${label}`) + lines.push("") + + const feedEntries = collectFeedEntries(type, networkFilter, chainCache, options) + + if (feedEntries.length === 0) { + lines.push( + networkFilter + ? `No feeds found for type \`${type}\` on network \`${networkFilter}\`.` + : `No feeds found for type \`${type}\`.` + ) + } else { + let currentNetwork = "" + + for (const entry of feedEntries) { + if (entry.network !== currentNetwork) { + currentNetwork = entry.network + const network = feedEntries.find((e) => e.network === currentNetwork) + + if (!network) continue + + lines.push(`## ${entry.chain} — ${network.networkName}`) + lines.push(`- Network type: ${network.networkType}`) + lines.push(`- Query string: \`${network.network}\``) + lines.push("") + lines.push("| Feed Name | Proxy Address | Deviation | Heartbeat |") + lines.push("|-----------|--------------|-----------|-----------|") + } + + const name = escapePipes(entry.name) + + lines.push(`| ${name} | \`${entry.proxyAddress}\` | ${entry.deviation} | ${entry.heartbeat} |`) + } + + lines.push("") + } + } + + return lines.join("\n") +} diff --git a/src/features/feeds/utils/streamMetadata.ts b/src/features/feeds/utils/streamMetadata.ts new file mode 100644 index 00000000000..f93551548fd --- /dev/null +++ b/src/features/feeds/utils/streamMetadata.ts @@ -0,0 +1,130 @@ +/** + * Utilities for resolving display metadata from Data Streams feed entries. + * + * Stream source data is inconsistent in some fields (schema, decimals, pair names). + * The functions here centralize inference logic so it stays easy to update when + * the upstream data improves. + * + * Schema mapping (see /data-streams/reference/report-schema-overview): + * crypto → v3 (Crypto Advanced) or v3-dex (DEX State Price) + * rwa → v8 (RWA Standard) or v11 (RWA Advanced) + * exchangeRate → v7 (Redemption Rates) + * smartdata → v9 (SmartData) + * tokenizedAsset → v10 (Tokenized Asset) + */ + +import type { DataFeedType } from "../components/FeedList.tsx" + +/** + * Maps stable public API type names to internal DataFeedType values. + * Update right-hand values here if internal names change. + * Do not change public keys without a deprecation period. + */ +export const STREAM_CATEGORY_MAP: Record = { + crypto: "streamsCrypto", + rwa: "streamsRwa", + smartdata: "streamsNav", + exchangeRate: "streamsExRate", + tokenizedAsset: "streamsBacked", +} + +/** + * Escapes backslashes and pipe characters for safe rendering in markdown tables. + * Backslashes are escaped first to prevent them from interfering with pipe escaping. + */ +export function escapePipes(value: string): string { + return value.replace(/\\/g, "\\\\").replace(/\|/g, "\\|") +} + +/** + * Returns the display pair name ("BTC/USD") from a stream feed. + * Returns null for incomplete entries that should be skipped. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function resolveStreamPair(feed: any): string | null { + const base = Array.isArray(feed.pair) && feed.pair.length >= 2 ? feed.pair[0] : "" + const quote = Array.isArray(feed.pair) && feed.pair.length >= 2 ? feed.pair[1] : "" + if (base && quote) return `${base}/${quote}` + const fromName = escapePipes((feed.name || "").replace(/-Streams-.*$/, "")) + return fromName || null +} + +// Normalize raw assetClass values from source data to human-readable display strings. +// Some values are internal camelCase identifiers that need to be mapped. +// TODO: remove once source data exposes clean display names. +const ASSET_CLASS_DISPLAY: Record = { + TokenizedEquities: "Tokenized Equities", +} + +/** + * Returns the asset class display string. + * Suppresses redundant subclass when it equals the main class + * (e.g. assetClass="Crypto", assetSubClass="Crypto" → "Crypto"). + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function resolveAssetClass(feed: any): string { + const rawMain = feed.docs?.assetClass || "" + const rawSub = feed.docs?.assetSubClass || "" + const main = ASSET_CLASS_DISPLAY[rawMain] ?? rawMain + const sub = ASSET_CLASS_DISPLAY[rawSub] ?? rawSub + // Compare raw values to catch cases where main and sub are the same internal identifier + if (main && sub && rawSub !== rawMain) return escapePipes(`${main} — ${sub}`) + return escapePipes(main || sub || "—") +} + +/** + * Returns the trading hours label for a stream feed. + * Derived from assetSubClass when present, falling back to clicProductName parsing. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function resolveTradingHours(feed: any): string { + const sub = feed.docs?.assetSubClass || "" + const clic = feed.docs?.clicProductName || "" + if ( + sub === "Regular Hours" || + (clic.includes("RegularHours") && !clic.includes("ExtendedHours") && !clic.includes("OvernightHours")) + ) { + return "Regular Hours (9:30am–4:00pm Mon–Fri ET)" + } + if (sub === "Extended Hours" || clic.includes("ExtendedHours")) { + return "Extended Hours (4:00am–9:30am & 4:00pm–8:00pm Mon–Fri ET)" + } + if (sub === "Overnight Hours" || clic.includes("OvernightHours")) { + return "Overnight Hours (8:00pm–4:00am Sun evening–Fri morning ET)" + } + return "—" +} + +/** + * Returns the report schema version for a stream feed. + * + * Most categories have a fixed schema. RWA is the exception — v8 (Standard) + * and v11 (Advanced) are inferred from the clicProductName suffix until the + * source data exposes a clean schema field on every feed. + * + * TODO: simplify the RWA case once feed.docs.schema is reliable upstream. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function resolveStreamSchema(type: DataFeedType, feed: any): string { + switch (type) { + case "streamsCrypto": + return feed.docs?.feedType === "Crypto-DEX" ? "v3-dex" : "v3" + case "streamsExRate": + return "v7" + case "streamsNav": + return "v9" + case "streamsBacked": + return "v10" + case "streamsRwa": { + // Infer v8 vs v11 from the numeric suffix on clicProductName (e.g. "-011" → v11) + const match = (feed.docs?.clicProductName || "").match(/-0(\d{2})$/) + if (match) { + if (match[1] === "11") return "v11" + if (match[1] === "04" || match[1] === "08") return "v8" + } + return "—" + } + default: + return "—" + } +} diff --git a/src/pages/[...path].md.ts b/src/pages/[...path].md.ts index 93be892199a..7c120721c95 100644 --- a/src/pages/[...path].md.ts +++ b/src/pages/[...path].md.ts @@ -1,30 +1,20 @@ import type { APIRoute } from "astro" import fs from "node:fs/promises" import path from "node:path" + +import { getServerSideChainMetadata } from "~/features/data/api/backend.ts" +import { CHAINS } from "~/features/data/chains.ts" +import { buildFeedAddressMarkdown, collectStreamEntries } from "~/features/feeds/utils/feedOutput.ts" +import { STREAM_CATEGORY_MAP } from "~/features/feeds/utils/streamMetadata.ts" + import { textPlainHeaders } from "@lib/api/cacheHeaders.js" import { transformPageToMarkdown } from "@lib/markdown/transformMarkdown.js" import { extractFrontmatter, getIsoStringOrUndefined, toCanonicalUrl, toContentRelative } from "@lib/markdown/utils.js" -const SITE_BASE = "https://docs.chain.link" const CONTENT_ROOT = path.resolve("src/content") const LLMS_DIRECTIVE = "> For the complete documentation index, see [llms.txt](/llms.txt)." -const MARKDOWN_REDIRECTS: Record = { - "ccip/tutorials/cross-chain-tokens": "ccip/tutorials/evm/cross-chain-tokens", - - // Data Streams - "data-streams/getting-started": "data-streams/tutorials/streams-trade/getting-started", - "data-streams/getting-started-hardhat": "data-streams/tutorials/streams-trade/getting-started-hardhat", - "data-streams/reference/streams-direct/streams-direct-onchain-verification": - "data-streams/reference/onchain-verification", - - // Newly surfaced redirects - "chainlink-functions/resources/concepts": "chainlink-functions/resources", - "cre/getting-started/conclusion": "cre/getting-started", - "data-streams/reference/streams-direct/streams-direct-interface-ws": "data-streams/reference/interface-ws", -} - const markdownHeaders = { ...textPlainHeaders, "Content-Type": "text/markdown; charset=utf-8", @@ -32,6 +22,10 @@ const markdownHeaders = { export const prerender = false +function buildHiddenDirective(content: string): string { + return `
\n${content.trim()}\n
\n` +} + export const GET: APIRoute = async ({ params, request }) => { const cleanPath = normalizeMarkdownPath(params.path) @@ -39,114 +33,25 @@ export const GET: APIRoute = async ({ params, request }) => { return new Response("Page not found.", { status: 404 }) } - const specialResolution = await resolveSpecialCanonicalMarkdownPath(cleanPath) - if (specialResolution) { - return buildMarkdownResponseFromPath(specialResolution.resolvedPath, request, specialResolution.sourceCanonicalPath) - } - - const creResolution = await resolveCreCanonicalMarkdownPath(cleanPath) - - if (creResolution.kind === "selector") { - return new Response(buildCreSelectorMarkdown(cleanPath, creResolution), { - status: 200, - headers: markdownHeaders, - }) - } - - const resolvedPath = creResolution.kind === "resolved" ? creResolution.path : cleanPath - return buildMarkdownResponseFromPath(resolvedPath, request) -} - -type SpecialResolution = { - resolvedPath: string - sourceCanonicalPath: string -} - -async function resolveSpecialCanonicalMarkdownPath(cleanPath: string): Promise { - const specialPathMap: Record = { - "cre-templates": "cre/templates", - } - - const mappedPath = specialPathMap[cleanPath] - if (!mappedPath) return null - - const file = await findContentFile(mappedPath) - if (!file) return null - - return { - resolvedPath: mappedPath, - sourceCanonicalPath: cleanPath, - } -} - -type CreResolution = - | { kind: "none" } - | { kind: "resolved"; path: string } - | { kind: "selector"; goPath: string; tsPath: string } - -async function resolveCreCanonicalMarkdownPath(cleanPath: string): Promise { - if (!cleanPath.startsWith("cre/")) { - return { kind: "none" } - } - - const direct = await findContentFile(cleanPath) - if (direct) { - return { kind: "resolved", path: cleanPath } - } - - const goPath = `${cleanPath}-go` - const tsPath = `${cleanPath}-ts` - - const [goFile, tsFile] = await Promise.all([findContentFile(goPath), findContentFile(tsPath)]) - - if (goFile && tsFile) { - return { kind: "selector", goPath, tsPath } - } - - if (goFile) { - return { kind: "resolved", path: goPath } - } - - if (tsFile) { - return { kind: "resolved", path: tsPath } - } - - return { kind: "none" } -} - -async function buildMarkdownResponseFromPath( - resolvedPath: string, - request: Request, - sourceCanonicalPathOverride?: string -): Promise { - const redirectTarget = MARKDOWN_REDIRECTS[resolvedPath] - - if (redirectTarget) { - return buildMarkdownMovedResponse(resolvedPath, redirectTarget) - } - - const mdxAbsPath = await findContentFile(resolvedPath) + const mdxAbsPath = await findContentFile(cleanPath) if (!mdxAbsPath) { return new Response("Page not found.", { status: 404 }) } - const url = new URL(request.url) - const targetLanguage = url.searchParams.get("lang") || undefined + const origin = new URL(request.url).origin + const targetLanguage = new URL(request.url).searchParams.get("lang") || undefined const raw = await fs.readFile(mdxAbsPath, "utf-8") const { body, fmTitle, fmLastModified } = extractFrontmatter(raw) - const section = resolvedPath.split("/")[0] - - const transformed = await transformPageBodyToMarkdown(body, mdxAbsPath, { - siteBase: SITE_BASE, + const transformed = await transformPageBodyToMarkdown(body, mdxAbsPath, cleanPath, { + siteBase: origin, targetLanguage, }) const relFromContent = toContentRelative(mdxAbsPath) - const derivedSourceUrl = toCanonicalUrl(section, relFromContent, SITE_BASE) - const sourceUrl = sourceCanonicalPathOverride ? `${SITE_BASE}/${sourceCanonicalPathOverride}` : derivedSourceUrl + const sourceUrl = toCanonicalUrl(cleanPath.split("/")[0], relFromContent, origin) const title = fmTitle || path.basename(mdxAbsPath, path.extname(mdxAbsPath)) const lastModified = getIsoStringOrUndefined(fmLastModified) @@ -169,97 +74,116 @@ async function buildMarkdownResponseFromPath( async function transformPageBodyToMarkdown( body: string, mdxAbsPath: string, + routePath: string, options: { siteBase: string targetLanguage?: string } ): Promise { - // Targeted fix for problematic page - if (mdxAbsPath.includes("data-feeds/deprecating-feeds")) { - return ` -## Deprecated Feeds + const BASE_URL = options.siteBase -This page contains dynamically generated or component-heavy content. - -For the full and most up-to-date information, see: -https://docs.chain.link/data-feeds/deprecating-feeds -`.trim() - } + let chainCache: Record = {} try { - return await transformPageToMarkdown(body, mdxAbsPath, options) - } catch { - const sanitizedBody = stripRuntimeMdxSyntax(body) - - try { - return await transformPageToMarkdown(sanitizedBody, mdxAbsPath, options) - } catch { - return buildFallbackMarkdownBody(sanitizedBody) + chainCache = await getServerSideChainMetadata(CHAINS) + } catch (e) { + console.error("Failed to load chain metadata:", e) + } + + // -------------------------------------------------- + // STRIP HIDDEN DIRECTIVE (prevents JSX in markdown) + // -------------------------------------------------- + body = body.replace(/
[\s\S]*?<\/div>/g, "") + + // -------------------------------------------------- + // FEEDS + // -------------------------------------------------- + if (body.includes(" = { + default: "Price feeds", + smartdata: "SmartData feeds", + rates: "Rates feeds", + tokenizedEquity: "Tokenized equity feeds", + usGovernmentMacroeconomicData: "U.S. Government Macroeconomic Data feeds", } - } -} -function buildFallbackMarkdownBody(body: string): string { - return stripRuntimeMdxSyntax(body) - .replace(/<([A-Z][A-Za-z0-9]*)\b[^>]*\/>/g, "") - .replace(/<([A-Z][A-Za-z0-9]*)\b[^>]*>/g, "") - .replace(/<\/[A-Z][A-Za-z0-9]*>/g, "") - .trim() -} + const feedLabel = labelMap[feedType] || feedType -function stripRuntimeMdxSyntax(body: string): string { - const lines = body.split("\n") - const output: string[] = [] + const replacement = ` +## Feed Contract Addresses - let skippingExportBlock = false - let skippingImportBlock = false - let braceDepth = 0 +For programmatic access: +${BASE_URL}/data-feeds/feed-addresses/${feedType}/ethereum-mainnet.json - for (const line of lines) { - const trimmed = line.trim() +First retrieve available networks: +${BASE_URL}/data-feeds/feed-addresses/${feedType}.json - if (skippingImportBlock) { - if (trimmed.includes(" from ") || trimmed.endsWith('"') || trimmed.endsWith("'")) { - skippingImportBlock = false - } - continue - } +Use this dataset to determine the correct network queryString before retrieving a specific network dataset. - if (skippingExportBlock) { - braceDepth += countChar(line, "{") - braceDepth -= countChar(line, "}") +The interactive address table on this page is loaded dynamically and is not included in this markdown export. - if (braceDepth <= 0) { - skippingExportBlock = false - braceDepth = 0 - } - continue - } +- ${feedLabel}${feedType === "default" ? " (default dataset)" : ""}: +/data-feeds/feed-addresses/${feedType}.json - if (/^import\s+/.test(trimmed)) { - if (!trimmed.includes(" from ")) skippingImportBlock = true - continue - } +- Per-network datasets: +/data-feeds/feed-addresses/${feedType}/{network}.json +` - if (/^export\s+(async\s+)?function\s+/.test(trimmed)) { - skippingExportBlock = true - braceDepth = countChar(line, "{") - countChar(line, "}") - continue - } + body = body.replace(//g, replacement) + } + // -------------------------------------------------- + // STREAMS + // -------------------------------------------------- + if (body.includes(" = { + crypto: "Crypto streams", + rwa: "RWA streams", + exchangeRate: "Exchange rate streams", + smartdata: "SmartData streams", + tokenizedAsset: "Tokenized asset streams", } - output.push(line) + const streamLabel = streamLabelMap[rawType] || rawType + + const replacement = ` +## Stream IDs + +For programmatic access: +${BASE_URL}/data-streams/stream-ids/${rawType}.json + +The interactive stream table on this page is loaded dynamically and is not included in this markdown export. + +- ${streamLabel}: +/data-streams/stream-ids/${rawType}.json +` + + body = body.replace(//g, replacement) } - return output.join("\n") + try { + return await transformPageToMarkdown(body, mdxAbsPath, options) + } catch { + return body + } } -function countChar(value: string, char: string): number { - return value.split(char).length - 1 -} +// ----------------------- +// UTILITIES +// ----------------------- function normalizeMarkdownPath(pathParam: string | undefined): string | null { if (!pathParam) return null @@ -294,37 +218,3 @@ async function findContentFile(cleanPath: string): Promise { return null } - -function buildMarkdownMovedResponse(sourcePath: string, targetPath: string): Response { - const sourceUrl = `${SITE_BASE}/${sourcePath}` - const targetUrl = `/${targetPath}.md` - - return new Response( - [ - `# Redirect`, - `Source: ${sourceUrl}`, - "", - LLMS_DIRECTIVE, - "", - "This page has moved.", - "", - `Use the current documentation: [${targetPath}](${targetUrl}).`, - "", - ].join("\n"), - { status: 200, headers: markdownHeaders } - ) -} - -function buildCreSelectorMarkdown(canonicalPath: string, resolution: any): string { - const canonicalUrl = `${SITE_BASE}/${canonicalPath}` - return [ - `# ${canonicalPath}`, - `Source: ${canonicalUrl}`, - "", - LLMS_DIRECTIVE, - "", - `- Go: /${resolution.goPath}.md`, - `- TypeScript: /${resolution.tsPath}.md`, - "", - ].join("\n") -} diff --git a/src/pages/data-feeds/feed-addresses/[type].json.ts b/src/pages/data-feeds/feed-addresses/[type].json.ts new file mode 100644 index 00000000000..2140a45953e --- /dev/null +++ b/src/pages/data-feeds/feed-addresses/[type].json.ts @@ -0,0 +1,26 @@ +import type { APIRoute } from "astro" + +import { getServerSideChainMetadata } from "~/features/data/api/backend.ts" +import { CHAINS } from "~/features/data/chains.ts" +import { collectFeedEntries, VALID_FEED_TYPES } from "~/features/feeds/utils/feedOutput.ts" + +export const prerender = false + +export const GET: APIRoute = async ({ params }) => { + const type = params.type as string + + if (!VALID_FEED_TYPES.includes(type as any)) { + return new Response(`Invalid type "${type}"`, { status: 400 }) + } + + const chainCache = await getServerSideChainMetadata(CHAINS) + + const feeds = collectFeedEntries(type as any, null, chainCache) + + return new Response(JSON.stringify(feeds, null, 2), { + headers: { + "Content-Type": "application/json", + "Cache-Control": "public, max-age=300", + }, + }) +} diff --git a/src/pages/data-feeds/feed-addresses/[type].txt.ts b/src/pages/data-feeds/feed-addresses/[type].txt.ts new file mode 100644 index 00000000000..af3d09ab2dd --- /dev/null +++ b/src/pages/data-feeds/feed-addresses/[type].txt.ts @@ -0,0 +1,98 @@ +import type { APIRoute } from "astro" +import { getServerSideChainMetadata } from "~/features/data/api/backend.ts" +import { CHAINS } from "~/features/data/chains.ts" +import { VALID_FEED_TYPES } from "~/features/feeds/utils/feedOutput.ts" +import type { DataFeedType } from "~/features/feeds/components/FeedList.tsx" + +export function getStaticPaths() { + return VALID_FEED_TYPES.map((type) => ({ + params: { type }, + })) +} + +export const GET: APIRoute = async ({ params }) => { + const type = params.type as DataFeedType + + if (!VALID_FEED_TYPES.includes(type)) { + return new Response(`Invalid type "${type}"`, { status: 400 }) + } + + const chainCache = await getServerSideChainMetadata(CHAINS) + + const networks: { + queryString: string + networkName: string + chain: string + networkType: string + }[] = [] + + const seen = new Set() + + for (const chain of Object.values(chainCache)) { + // ✅ FIX: rename to avoid shadowing + const chainNetworks = (chain as { networks?: any[] }).networks ?? [] + + for (const network of chainNetworks) { + const queryString = network.queryString + if (!queryString) continue + + if (seen.has(queryString)) continue + seen.add(queryString) + + networks.push({ + queryString, + networkName: network.name, + chain: typeof network.chain === "string" ? network.chain : "", + networkType: network.networkType || "mainnet", + }) + } + } + + // Sort: mainnet first, then testnet + networks.sort((a, b) => { + if (a.networkType !== b.networkType) { + return a.networkType === "mainnet" ? -1 : 1 + } + return a.queryString.localeCompare(b.queryString) + }) + + const lines: string[] = [] + + lines.push(`# Chainlink Feed Addresses Index (${type})`) + lines.push("") + lines.push("This document lists all available networks for this feed type.") + lines.push("") + lines.push("Each network has a dedicated dataset.") + lines.push("") + lines.push("Do not load multiple networks unless required.") + lines.push("Each file contains the complete dataset for one network.") + lines.push("") + + lines.push("## Networks") + lines.push("") + + for (const net of networks) { + lines.push(`- ${net.queryString} → /data-feeds/feed-addresses/${type}/${net.queryString}.txt`) + lines.push(` name: ${net.networkName}`) + lines.push(` type: ${net.networkType}`) + if (net.chain) { + lines.push(` chain: ${net.chain}`) + } + lines.push("") + } + + lines.push("## Usage pattern") + lines.push("") + lines.push("1. Select a network from the list above") + lines.push("2. Fetch the corresponding dataset") + lines.push("3. Filter by feed name as needed") + lines.push("") + + return new Response(lines.join("\n"), { + status: 200, + headers: { + "Content-Type": "text/plain; charset=utf-8", + "Cache-Control": "public, max-age=0, s-maxage=86400, stale-while-revalidate=604800", + }, + }) +} diff --git a/src/pages/data-feeds/feed-addresses/[type]/[network].json.ts b/src/pages/data-feeds/feed-addresses/[type]/[network].json.ts new file mode 100644 index 00000000000..821afded784 --- /dev/null +++ b/src/pages/data-feeds/feed-addresses/[type]/[network].json.ts @@ -0,0 +1,31 @@ +import type { APIRoute } from "astro" + +import { getServerSideChainMetadata } from "~/features/data/api/backend.ts" +import { CHAINS } from "~/features/data/chains.ts" +import { collectFeedEntries, VALID_FEED_TYPES } from "~/features/feeds/utils/feedOutput.ts" + +export const prerender = false + +export const GET: APIRoute = async ({ params }) => { + const type = params.type as string + const network = params.network ?? null + + if (!VALID_FEED_TYPES.includes(type as any)) { + return new Response(`Invalid type "${type}"`, { + status: 400, + headers: { "Content-Type": "text/plain; charset=utf-8" }, + }) + } + + const chainCache = await getServerSideChainMetadata(CHAINS) + + const feeds = collectFeedEntries(type as any, network, chainCache) + + return new Response(JSON.stringify(feeds, null, 2), { + status: 200, + headers: { + "Content-Type": "application/json", + "Cache-Control": "public, max-age=300", + }, + }) +} diff --git a/src/pages/data-feeds/feed-addresses/[type]/[network].txt.ts b/src/pages/data-feeds/feed-addresses/[type]/[network].txt.ts new file mode 100644 index 00000000000..d000e440473 --- /dev/null +++ b/src/pages/data-feeds/feed-addresses/[type]/[network].txt.ts @@ -0,0 +1,78 @@ +/** + * Static snapshots of feed addresses per network, pre-rendered at build time. + * Served at /data-feeds/feed-addresses/{type}/{network}.txt + * + * For always-live data use the dynamic endpoint: /api/feeds/addresses + */ + +import type { APIRoute } from "astro" +import { getServerSideChainMetadata } from "~/features/data/api/backend.ts" +import { CHAINS } from "~/features/data/chains.ts" +import { buildFeedAddressMarkdown, VALID_FEED_TYPES } from "~/features/feeds/utils/feedOutput.ts" +import { STREAM_CATEGORY_MAP } from "~/features/feeds/utils/streamMetadata.ts" +import type { DataFeedType } from "~/features/feeds/components/FeedList.tsx" + +// Reverse map: internal → public +const INTERNAL_TO_PUBLIC: Record = Object.fromEntries( + Object.entries(STREAM_CATEGORY_MAP).map(([pub, internal]) => [internal, pub]) +) + +// ✅ Build-time route generation +export async function getStaticPaths() { + const chainCache = await getServerSideChainMetadata(CHAINS) + + const paths: { params: { type: string; network: string } }[] = [] + const seen = new Set() + + for (const type of VALID_FEED_TYPES) { + for (const chain of Object.values(chainCache)) { + const networks = (chain as any).networks ?? [] + + for (const network of networks) { + const queryString = network.queryString + if (!queryString) continue + + const key = `${type}:${queryString}` + if (seen.has(key)) continue + seen.add(key) + + paths.push({ + params: { + type, + network: queryString, + }, + }) + } + } + } + + return paths +} + +// ❌ IMPORTANT: no `prerender = false` here +// this must run at build time + +export const GET: APIRoute = async ({ params }) => { + const type = params.type as DataFeedType + const network = typeof params.network === "string" ? params.network : null + + if (!VALID_FEED_TYPES.includes(type)) { + return new Response(`Invalid type "${type}"`, { status: 400 }) + } + + const publicType = INTERNAL_TO_PUBLIC[type] + + // ✅ Safe at build time (filesystem allowed) + const chainCache = await getServerSideChainMetadata(CHAINS) + + const markdown = buildFeedAddressMarkdown(type, network, chainCache, "https://docs.chain.link", { publicType }) + + return new Response(markdown, { + status: 200, + headers: { + "Content-Type": "text/plain; charset=utf-8", + // Long CDN cache — content only changes on redeploy + "Cache-Control": "public, max-age=0, s-maxage=86400, stale-while-revalidate=604800", + }, + }) +} diff --git a/src/pages/data-streams/networks.txt.ts b/src/pages/data-streams/networks.txt.ts new file mode 100644 index 00000000000..f889453d4e7 --- /dev/null +++ b/src/pages/data-streams/networks.txt.ts @@ -0,0 +1,105 @@ +import type { APIRoute } from "astro" +import { StreamsNetworksData } from "~/features/feeds/data/StreamsNetworksData.ts" +import { textPlainHeaders } from "@lib/api/cacheHeaders.js" + +export const prerender = false + +interface StreamNetworkEntry { + network: string + label: string + verifierProxy: string +} + +function getNetworks(): { + mainnet: StreamNetworkEntry[] + testnet: StreamNetworkEntry[] +} { + const mainnet: StreamNetworkEntry[] = [] + const testnet: StreamNetworkEntry[] = [] + + for (const entry of StreamsNetworksData) { + if (entry.isCanton) continue + + if (entry.mainnet?.verifierProxy) { + mainnet.push({ + network: entry.network, + label: entry.mainnet.label, + verifierProxy: entry.mainnet.verifierProxy, + }) + } + + if (entry.testnet?.verifierProxy) { + testnet.push({ + network: entry.network, + label: entry.testnet.label, + verifierProxy: entry.testnet.verifierProxy, + }) + } + } + + return { mainnet, testnet } +} + +function buildMarkdown(): string { + const { mainnet, testnet } = getNetworks() + + const lines: string[] = [ + "# Chainlink Data Streams Networks", + "", + "Supported networks and verifier proxy addresses.", + "", + "Verifier proxies are shared across all Data Streams categories and schema versions.", + "", + "Use the verifier proxy for the network where your application is deployed.", + "", + "---", + "", + "## Mainnet Networks", + "", + "| Network | Label | Verifier Proxy |", + "|---------|-------|----------------|", + ] + + for (const n of mainnet) { + lines.push(`| ${n.network} | ${n.label} | \`${n.verifierProxy}\` |`) + } + + if (testnet.length > 0) { + lines.push( + "", + "## Testnet Networks", + "", + "| Network | Label | Verifier Proxy |", + "|---------|-------|----------------|" + ) + + for (const n of testnet) { + lines.push(`| ${n.network} | ${n.label} | \`${n.verifierProxy}\` |`) + } + } + + lines.push( + "", + "---", + "", + "Use this file together with stream ID datasets:", + "", + "- /data-streams/stream-ids/{type}.txt", + "", + "Stream IDs are universal. Networks provide verifier proxy addresses." + ) + + return lines.join("\n") +} + +export const GET: APIRoute = async () => { + const markdown = buildMarkdown() + + return new Response(markdown, { + status: 200, + headers: { + ...textPlainHeaders, + "Cache-Control": "public, max-age=3600, s-maxage=86400", + }, + }) +} diff --git a/src/pages/data-streams/stream-ids/[type].json.ts b/src/pages/data-streams/stream-ids/[type].json.ts new file mode 100644 index 00000000000..8981340892e --- /dev/null +++ b/src/pages/data-streams/stream-ids/[type].json.ts @@ -0,0 +1,32 @@ +import type { APIRoute } from "astro" + +import { getServerSideChainMetadata } from "~/features/data/api/backend.ts" +import { CHAINS } from "~/features/data/chains.ts" +import { collectStreamEntries } from "~/features/feeds/utils/feedOutput.ts" +import { STREAM_CATEGORY_MAP } from "~/features/feeds/utils/streamMetadata.ts" + +export const prerender = false + +export const GET: APIRoute = async ({ params }) => { + const rawType = params.type as string + const internalType = STREAM_CATEGORY_MAP[rawType] + + if (!internalType) { + return new Response(`Invalid type "${rawType}"`, { + status: 400, + headers: { "Content-Type": "text/plain; charset=utf-8" }, + }) + } + + const chainCache = await getServerSideChainMetadata(CHAINS) + + const streams = collectStreamEntries(internalType, chainCache, { publicType: rawType } as any) + + return new Response(JSON.stringify(streams, null, 2), { + status: 200, + headers: { + "Content-Type": "application/json", + "Cache-Control": "public, max-age=300", + }, + }) +} diff --git a/src/pages/data-streams/stream-ids/[type].txt.ts b/src/pages/data-streams/stream-ids/[type].txt.ts new file mode 100644 index 00000000000..12f5688eff8 --- /dev/null +++ b/src/pages/data-streams/stream-ids/[type].txt.ts @@ -0,0 +1,113 @@ +import type { APIRoute } from "astro" +import { getServerSideChainMetadata } from "~/features/data/api/backend.ts" +import { CHAINS } from "~/features/data/chains.ts" +import { buildFeedAddressMarkdown, type FeedMarkdownOptions } from "~/features/feeds/utils/feedOutput.ts" +import { STREAM_CATEGORY_MAP } from "~/features/feeds/utils/streamMetadata.ts" +import { textPlainHeaders } from "@lib/api/cacheHeaders.js" + +// ✅ Build-time route generation +export function getStaticPaths() { + return Object.keys(STREAM_CATEGORY_MAP).map((type) => ({ + params: { type }, + })) +} + +export const GET: APIRoute = async ({ params }) => { + const rawType = params.type as string + + const internalType = STREAM_CATEGORY_MAP[rawType] + if (!internalType) { + const valid = Object.keys(STREAM_CATEGORY_MAP).join(", ") + return new Response(`Invalid type "${rawType}". Valid values: ${valid}`, { + status: 400, + headers: { "Content-Type": "text/plain; charset=utf-8" }, + }) + } + + // ✅ Safe at build time + const chainCache = await getServerSideChainMetadata(CHAINS) + + const options: FeedMarkdownOptions = { + publicType: rawType, + } + + let markdown = buildFeedAddressMarkdown(internalType, null, chainCache, "https://docs.chain.link", options) + + // -------------------------------------------------- + // CLEANUP: remove API + cross-product references + // -------------------------------------------------- + + markdown = markdown + .replace(/Machine-readable endpoint:[^\n]*\n?/g, "") + .replace(/Supported networks and verifier proxy addresses:[^\n]*\n?/g, "") + .replace(/Static snapshot:[^\n]*\n?/g, "") + + // -------------------------------------------------- + // SCHEMA DETECTION + // -------------------------------------------------- + + const schemaMatches = [...markdown.matchAll(/\|\s*[^|]+\s*\|\s*`?0x[a-fA-F0-9]+`?\s*\|\s*[^|]*\|\s*(v\d+)/g)] + + const schemas = Array.from(new Set(schemaMatches.map((m) => m[1]))) + + // -------------------------------------------------- + // SCHEMA → DOCS MAP + // -------------------------------------------------- + + const SCHEMA_DOCS: Record = { + v3: "https://docs.chain.link/data-streams/reference/report-schema-v3", + v7: "https://docs.chain.link/data-streams/reference/report-schema-v7", + v8: "https://docs.chain.link/data-streams/reference/report-schema-v8", + v9: "https://docs.chain.link/data-streams/reference/report-schema-v9", + v10: "https://docs.chain.link/data-streams/reference/report-schema-v10", + v11: "https://docs.chain.link/data-streams/reference/report-schema-v11", + } + + // -------------------------------------------------- + // BUILD INTRO BLOCK + // -------------------------------------------------- + + const introLines = [ + `> Stream IDs for Chainlink Data Streams – ${capitalize(rawType)}.`, + `> These IDs are universal and valid across all supported networks.`, + `> To use a stream ID, retrieve the verifier proxy for the target network from /data-streams/networks.txt.`, + `> Datasets may contain multiple schema versions. Filter by schema if needed.`, + ] + + if (schemas.length > 0) { + introLines.push(`> Schemas present in this dataset:`) + + for (const s of schemas) { + const url = SCHEMA_DOCS[s] + if (url) { + introLines.push(`> - \`${s}\` → ${url}`) + } else { + introLines.push(`> - ${s}`) + } + } + } + + const introBlock = introLines.join("\n") + "\n" + + // -------------------------------------------------- + // REPLACE ORIGINAL INTRO + // -------------------------------------------------- + + markdown = markdown.replace(/> Stream IDs[\s\S]*?\n/, introBlock) + + return new Response(markdown.trim(), { + status: 200, + headers: { + ...textPlainHeaders, + "Cache-Control": "public, max-age=3600, s-maxage=86400", + }, + }) +} + +// ----------------------- +// Utility +// ----------------------- + +function capitalize(value: string): string { + return value.charAt(0).toUpperCase() + value.slice(1) +}