Skip to content

Commit 86cf83f

Browse files
Bud-MacaulayBud-Macaulay
committed
bugfix - non-indexed dbs now can go next page.
1 parent 4d297b0 commit 86cf83f

File tree

6 files changed

+356
-7
lines changed

6 files changed

+356
-7
lines changed

‎public/cachedElementCounts.json‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
[
2+
{
3+
"providerUrl": "https://aflow.org/API/optimade/",
4+
"min": null,
5+
"max": null
6+
},
7+
{
8+
"providerUrl": "https://alexandria.icams.rub.de/pbesol",
9+
"min": 1,
10+
"max": 8
11+
},
12+
{
13+
"providerUrl": "https://alexandria.icams.rub.de/pbe",
14+
"min": 1,
15+
"max": 9
16+
},
17+
{
18+
"providerUrl": "https://www.crystallography.net/cod/optimade",
19+
"min": 6,
20+
"max": 6
21+
},
22+
{
23+
"providerUrl": "https://cmr-optimade.fysik.dtu.dk",
24+
"min": 1,
25+
"max": 1
26+
},
27+
{
28+
"providerUrl": "https://optimade.materialscloud.org/main/mc3d-pbe-v1",
29+
"min": 1,
30+
"max": 8
31+
},
32+
{
33+
"providerUrl": "https://optimade.materialscloud.org/main/mc3d-pbesol-v1",
34+
"min": 1,
35+
"max": 8
36+
},
37+
{
38+
"providerUrl": "https://optimade.materialscloud.org/main/mc3d-pbesol-v2",
39+
"min": 1,
40+
"max": 8
41+
},
42+
{
43+
"providerUrl": "https://optimade.materialscloud.org/main/mc2d",
44+
"min": 1,
45+
"max": 6
46+
},
47+
{
48+
"providerUrl": "https://optimade.materialscloud.org/main/2dtopo",
49+
"min": 1,
50+
"max": 5
51+
},
52+
{
53+
"providerUrl": "https://optimade.materialscloud.org/main/pyrene-mofs",
54+
"min": 4,
55+
"max": 7
56+
},
57+
{
58+
"providerUrl": "https://optimade.materialscloud.org/main/curated-cofs",
59+
"min": 2,
60+
"max": 7
61+
},
62+
{
63+
"providerUrl": "https://optimade.materialscloud.org/main/stoceriaitf",
64+
"min": 4,
65+
"max": 4
66+
},
67+
{
68+
"providerUrl": "https://optimade.materialscloud.org/main/autowannier",
69+
"min": 1,
70+
"max": 3
71+
},
72+
{
73+
"providerUrl": "https://optimade.materialscloud.org/main/tin-antimony-sulfoiodide",
74+
"min": 2,
75+
"max": 4
76+
},
77+
{
78+
"providerUrl": "https://optimade.materialscloud.org/archive/1z-pd",
79+
"min": 1,
80+
"max": 6
81+
},
82+
{
83+
"providerUrl": "https://optimade.materialscloud.org/archive/m0-zg",
84+
"min": 1,
85+
"max": 8
86+
},
87+
{
88+
"providerUrl": "https://optimade.materialscloud.org/archive/ed-g2",
89+
"min": 1,
90+
"max": 6
91+
},
92+
{
93+
"providerUrl": "https://optimade.materialscloud.org/archive/jk-9v",
94+
"min": 1,
95+
"max": 10
96+
},
97+
{
98+
"providerUrl": "https://optimade.materialscloud.org/archive/zt-z4",
99+
"min": 3,
100+
"max": 3
101+
},
102+
{
103+
"providerUrl": "https://optimade.materialsproject.org",
104+
"min": 1,
105+
"max": 9
106+
},
107+
{
108+
"providerUrl": "http://mpddoptimade.phaseslab.org",
109+
"min": 1,
110+
"max": 9
111+
},
112+
{
113+
"providerUrl": "https://api.mpds.io",
114+
"min": null,
115+
"max": null
116+
},
117+
{
118+
"providerUrl": "https://optimade.matterverse.ai",
119+
"min": null,
120+
"max": null
121+
},
122+
{
123+
"providerUrl": "http://mpod_optimade.cimav.edu.mx",
124+
"min": null,
125+
"max": null
126+
},
127+
{
128+
"providerUrl": "https://nomad-lab.eu/prod/v1/optimade/",
129+
"min": null,
130+
"max": null
131+
},
132+
{
133+
"providerUrl": "https://optimade.odbx.science",
134+
"min": 1,
135+
"max": 2
136+
},
137+
{
138+
"providerUrl": "https://optimade-misc.odbx.science",
139+
"min": 1,
140+
"max": 7
141+
},
142+
{
143+
"providerUrl": "https://optimade-gnome.odbx.science",
144+
"min": 2,
145+
"max": 6
146+
},
147+
{
148+
"providerUrl": "http://optimade.openmaterialsdb.se",
149+
"min": 6,
150+
"max": 6
151+
},
152+
{
153+
"providerUrl": "https://oqmd.org/optimade/",
154+
"min": null,
155+
"max": null
156+
},
157+
{
158+
"providerUrl": "https://jarvis.nist.gov/optimade/jarvisdft",
159+
"min": null,
160+
"max": null
161+
},
162+
{
163+
"providerUrl": "https://www.crystallography.net/tcod/optimade",
164+
"min": 3,
165+
"max": 3
166+
},
167+
{
168+
"providerUrl": "http://optimade.2dmatpedia.org",
169+
"min": 1,
170+
"max": 4
171+
}
172+
]

‎scripts/build-counts-cache.mjs‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
// build-counts-cache.mjs
2+
import fs from "fs";
3+
import path from "path";
4+
import {
5+
getProvidersList,
6+
getProviderLinks,
7+
getElementsCount,
8+
} from "../src/api.js";
9+
10+
const OUTPUT_PATH = path.resolve("./public/cachedElementCounts.json");
11+
12+
// load old cache if available
13+
function loadCache() {
14+
if (!fs.existsSync(OUTPUT_PATH)) return {};
15+
try {
16+
const data = JSON.parse(fs.readFileSync(OUTPUT_PATH, "utf-8"));
17+
return Object.fromEntries(data.map((e) => [e.providerUrl, e]));
18+
} catch (err) {
19+
console.warn("Failed to parse existing cache:", err);
20+
return {};
21+
}
22+
}
23+
24+
function saveCache(map) {
25+
const arr = Object.values(map);
26+
fs.writeFileSync(OUTPUT_PATH, JSON.stringify(arr, null, 2));
27+
}
28+
29+
async function processProvider(provider, cacheMap) {
30+
const baseUrl = provider.attributes.base_url;
31+
if (!baseUrl || !baseUrl.startsWith("http")) {
32+
console.log(`Skipping provider ${provider} (invalid base_url):`, baseUrl);
33+
return;
34+
}
35+
36+
console.log(`\n→ Provider: ${baseUrl}`);
37+
38+
let links;
39+
try {
40+
links = await getProviderLinks(baseUrl);
41+
} catch (err) {
42+
console.log(` Failed to fetch provider links:`, err.message);
43+
return;
44+
}
45+
46+
if (links.error) {
47+
console.log(` Provider returned error`);
48+
return;
49+
}
50+
51+
console.log(` Children: ${links.children.length}`);
52+
53+
for (const child of links.children) {
54+
const url = child.attributes.base_url;
55+
if (!url || !url.startsWith("http")) {
56+
console.log(` Skipping child (invalid URL):`, url);
57+
continue;
58+
}
59+
60+
console.log(` → Child: ${url}`);
61+
62+
// Check for cache hit
63+
if (cacheMap[url]?.min != null && cacheMap[url]?.max != null) {
64+
console.log(
65+
` Cached ✓ (min=${cacheMap[url].min}, max=${cacheMap[url].max})`
66+
);
67+
continue;
68+
}
69+
70+
console.log(` Fetching element counts...`);
71+
72+
const { min, max, durationMs } = await getElementsCount({
73+
providerUrl: url,
74+
});
75+
76+
console.log(
77+
` Result: min=${min}, max=${max} fetchTime=${durationMs.toFixed(
78+
0
79+
)}ms`
80+
);
81+
82+
cacheMap[url] = { providerUrl: url, min, max };
83+
saveCache(cacheMap);
84+
}
85+
}
86+
87+
async function main() {
88+
const prevCache = loadCache();
89+
const cacheMap = { ...prevCache };
90+
91+
console.log("Loading providers list...");
92+
93+
const { data: providers } = await getProvidersList();
94+
95+
console.log(`Total providers: ${providers.length}`);
96+
97+
for (const provider of providers) {
98+
await processProvider(provider, cacheMap);
99+
}
100+
101+
saveCache(cacheMap);
102+
console.log("\nDone. Cached element ranges written to", OUTPUT_PATH);
103+
}
104+
105+
main().catch((err) => {
106+
console.error("Fatal error:", err);
107+
process.exit(1);
108+
});

‎src/api.js‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,62 @@ export async function getStructures({
126126
throw new Error("All fetch attempts failed for getStructures");
127127
}
128128

129+
export async function getElementsCount({ providerUrl }) {
130+
const start = performance.now();
131+
132+
const minUrl = `${providerUrl}/v1/structures?sort=nelements&response_format=json&response_fields=nelements&page_limit=1`;
133+
const maxUrl = `${providerUrl}/v1/structures?sort=-nelements&response_format=json&response_fields=nelements&page_limit=1`;
134+
135+
async function fetchValue(url) {
136+
try {
137+
const res = await fetch(url);
138+
if (!res.ok) return null;
139+
140+
const json = await res.json();
141+
const value = json?.data?.[0]?.attributes?.nelements;
142+
143+
return typeof value === "number" ? value : null;
144+
} catch {
145+
return null;
146+
}
147+
}
148+
149+
const min = await fetchValue(minUrl);
150+
const max = await fetchValue(maxUrl);
151+
152+
const durationMs = performance.now() - start;
153+
154+
return { min, max, durationMs };
155+
}
156+
157+
export async function getSitesCount({ providerUrl }) {
158+
const attempts = [
159+
{ name: "direct", url: `${providerUrl}/v1/info/structures` },
160+
...corsProxies.map((proxy) => ({
161+
name: proxy.name,
162+
url: proxy.urlRule(`${providerUrl}/v1/info/structures`),
163+
})),
164+
];
165+
166+
for (const { url, name } of attempts) {
167+
try {
168+
const res = await fetch(url);
169+
if (!res.ok) continue;
170+
171+
const json = await res.json();
172+
const filteredProps = Object.fromEntries(
173+
Object.entries(json.data.properties || {}).filter(([key]) =>
174+
key.startsWith("_")
175+
)
176+
);
177+
178+
return { customProps: filteredProps, error: null };
179+
} catch {
180+
continue;
181+
}
182+
}
183+
}
184+
129185
// very messy divide and conquer strategy for returning the elements that exist in the PT.
130186
// this strategy is very bad for very broad providers...
131187
// Divide-and-conquer strategy for determining elements present in the provider

‎src/components/OptimadeClient/OptimadeChildInfo.jsx‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@ import { textNormal, textTiny, textHyperlink } from "../../styles/textStyles";
44
export default function OptimadeChildInfo({ child }) {
55
if (!child) return;
66

7-
console.log("childAttr", child);
8-
97
const { name, description, homepage, base_url, ...otherAttrs } = child;
108

119
return (

0 commit comments

Comments
 (0)