From caf6babdd937e3ada2584fb84b07aaf0cf519785 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sat, 17 Jan 2026 21:45:19 -0800 Subject: [PATCH 01/10] Turn on settings automigration for everyone --- api/routes/update.ts | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/api/routes/update.ts b/api/routes/update.ts index 7ccca44..47f9b51 100644 --- a/api/routes/update.ts +++ b/api/routes/update.ts @@ -46,7 +46,6 @@ import { import { deleteSettings as deleteSettingsInStately, getSettingsForUpdate, - setSetting as setSettingInStately, } from '../stately/settings-queries.js'; import { trackUntrackTriumphs } from '../stately/triumphs-queries.js'; import { @@ -313,26 +312,22 @@ async function statelyUpdate( mergedSettings = { ...mergedSettings, ...update.payload }; } - if (bungieMembershipId === 7094) { - const statelySettings = await getSettingsForUpdate(txn, bungieMembershipId); - - if (statelySettings) { - mergedSettings = { ...statelySettings, ...mergedSettings }; - await transaction(async (pgClient) => { - await replaceSettings( - pgClient, - bungieMembershipId, - subtractObject(mergedSettings, defaultSettings), - ); - }); - await deleteSettingsInStately(bungieMembershipId); - } else { - await transaction(async (pgClient) => { - await setSettingInPostgres(pgClient, bungieMembershipId, mergedSettings); - }); - } + const statelySettings = await getSettingsForUpdate(txn, bungieMembershipId); + + if (statelySettings) { + mergedSettings = { ...statelySettings, ...mergedSettings }; + await transaction(async (pgClient) => { + await replaceSettings( + pgClient, + bungieMembershipId, + subtractObject(mergedSettings, defaultSettings), + ); + }); + await deleteSettingsInStately(bungieMembershipId); } else { - await setSettingInStately(txn, bungieMembershipId, mergedSettings); + await transaction(async (pgClient) => { + await setSettingInPostgres(pgClient, bungieMembershipId, mergedSettings); + }); } break; } From e933cedb3a0b4029269dbe2477d10a61fcb371f5 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sun, 18 Jan 2026 20:33:40 -0800 Subject: [PATCH 02/10] Cleanup --- api/routes/profile.ts | 25 +++++++++++-------------- api/routes/update.ts | 7 ++----- api/utils.ts | 4 ++-- 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/api/routes/profile.ts b/api/routes/profile.ts index a09d213..8329017 100644 --- a/api/routes/profile.ts +++ b/api/routes/profile.ts @@ -172,7 +172,7 @@ async function statelyProfile( }; const timerPrefix = response.sync ? 'profileSync' : 'profileStately'; const counterPrefix = response.sync ? 'sync' : 'stately'; - const syncTokens: { [component: string]: string } = {}; + const syncTokens: { [component: string]: string | number } = {}; const addSyncToken = ( name: string, token: ListToken | { canSync: boolean; tokenData: number }, @@ -181,7 +181,7 @@ async function statelyProfile( syncTokens[name] = token.tokenData instanceof Uint8Array ? Buffer.from(token.tokenData).toString('base64') - : token.tokenData.toString(); + : token.tokenData; } }; const getSyncToken = (name: string) => { @@ -203,30 +203,27 @@ async function statelyProfile( const start = new Date(); const statelySettings = await querySettings(bungieMembershipId); - if (!statelySettings.settings) { + if (statelySettings.settings) { + const tokenData = getSyncToken('settings'); + const { settings: storedSettings, token: settingsToken } = tokenData + ? await syncSettings(tokenData) + : statelySettings; + response.settings = storedSettings; + addSyncToken('settings', settingsToken); + } else { const now = Date.now(); const pgSettings = await readTransaction(async (pgClient) => getSettings(pgClient, bungieMembershipId), ); if (pgSettings) { const tokenData = getSyncToken('s'); - if (tokenData === undefined || pgSettings.lastModifiedAt > tokenData) { + if (tokenData === undefined || pgSettings.lastModifiedAt > Number(tokenData)) { response.settings = { ...defaultSettings, ...pgSettings.settings }; } } else { response.settings = defaultSettings; } addSyncToken('s', { canSync: true, tokenData: pgSettings?.lastModifiedAt ?? now }); - } else { - const tokenData = getSyncToken('settings'); - const { settings: storedSettings, token: settingsToken } = tokenData - ? await syncSettings(tokenData) - : { - settings: statelySettings.settings ?? defaultSettings, - token: statelySettings.token, - }; - response.settings = storedSettings; - addSyncToken('settings', settingsToken); } metrics.timing(`${timerPrefix}.settings`, start); diff --git a/api/routes/update.ts b/api/routes/update.ts index 47f9b51..677e9f2 100644 --- a/api/routes/update.ts +++ b/api/routes/update.ts @@ -43,10 +43,7 @@ import { UpdateSearch, updateSearches, } from '../stately/searches-queries.js'; -import { - deleteSettings as deleteSettingsInStately, - getSettingsForUpdate, -} from '../stately/settings-queries.js'; +import { getSettingsForUpdate, keyFor } from '../stately/settings-queries.js'; import { trackUntrackTriumphs } from '../stately/triumphs-queries.js'; import { badRequest, @@ -323,7 +320,7 @@ async function statelyUpdate( subtractObject(mergedSettings, defaultSettings), ); }); - await deleteSettingsInStately(bungieMembershipId); + await txn.del(keyFor(bungieMembershipId)); } else { await transaction(async (pgClient) => { await setSettingInPostgres(pgClient, bungieMembershipId, mergedSettings); diff --git a/api/utils.ts b/api/utils.ts index cb76cd2..0baed18 100644 --- a/api/utils.ts +++ b/api/utils.ts @@ -1,4 +1,4 @@ -import { camelCase, mapKeys } from 'es-toolkit'; +import { camelCase, isEqual, mapKeys } from 'es-toolkit'; import { Response } from 'express'; /** @@ -141,7 +141,7 @@ export function subtractObject(obj: Partial, defaults: T): const result: Partial = {}; if (obj) { for (const key in defaults) { - if (obj[key] !== undefined && obj[key] !== defaults[key]) { + if (obj[key] !== undefined && !isEqual(obj[key], defaults[key])) { result[key] = obj[key]; } } From aa0643dd9a98caf4457c785b9232e5a85024e351 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sun, 18 Jan 2026 20:54:21 -0800 Subject: [PATCH 03/10] Fix type --- api/db/settings-queries.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/db/settings-queries.ts b/api/db/settings-queries.ts index 8559aed..b809249 100644 --- a/api/db/settings-queries.ts +++ b/api/db/settings-queries.ts @@ -9,7 +9,7 @@ export async function getSettings( bungieMembershipId: number, ): Promise<{ settings: Partial; deleted: boolean; lastModifiedAt: number } | undefined> { const results = await client.query<{ - settings: Settings; + settings: Partial; deleted_at: Date | null; last_updated_at: Date; }>({ From 80b9b4721b192d145a333ba09cc53554509cfada Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sun, 18 Jan 2026 21:08:38 -0800 Subject: [PATCH 04/10] Prefer postgres over stately? --- api/routes/profile.ts | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/api/routes/profile.ts b/api/routes/profile.ts index 8329017..9be080d 100644 --- a/api/routes/profile.ts +++ b/api/routes/profile.ts @@ -202,28 +202,24 @@ async function statelyProfile( // Load settings from Stately. If they're there, you're done. Otherwise load from Postgres. const start = new Date(); - const statelySettings = await querySettings(bungieMembershipId); - if (statelySettings.settings) { + const now = Date.now(); + // TODO: Should add the token to the query to avoid fetching if unchanged + const pgSettings = await readTransaction(async (pgClient) => + getSettings(pgClient, bungieMembershipId), + ); + if (pgSettings) { + const tokenData = getSyncToken('s'); + if (tokenData === undefined || pgSettings.lastModifiedAt > Number(tokenData)) { + response.settings = { ...defaultSettings, ...pgSettings.settings }; + } + addSyncToken('s', { canSync: true, tokenData: pgSettings?.lastModifiedAt ?? now }); + } else { const tokenData = getSyncToken('settings'); const { settings: storedSettings, token: settingsToken } = tokenData ? await syncSettings(tokenData) - : statelySettings; + : await querySettings(bungieMembershipId); response.settings = storedSettings; addSyncToken('settings', settingsToken); - } else { - const now = Date.now(); - const pgSettings = await readTransaction(async (pgClient) => - getSettings(pgClient, bungieMembershipId), - ); - if (pgSettings) { - const tokenData = getSyncToken('s'); - if (tokenData === undefined || pgSettings.lastModifiedAt > Number(tokenData)) { - response.settings = { ...defaultSettings, ...pgSettings.settings }; - } - } else { - response.settings = defaultSettings; - } - addSyncToken('s', { canSync: true, tokenData: pgSettings?.lastModifiedAt ?? now }); } metrics.timing(`${timerPrefix}.settings`, start); From 842c03cc82f7e7f5cc80b255bd63b29ace758e0a Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sun, 18 Jan 2026 21:18:23 -0800 Subject: [PATCH 05/10] Typing --- api/routes/profile.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/routes/profile.ts b/api/routes/profile.ts index 9be080d..8b79416 100644 --- a/api/routes/profile.ts +++ b/api/routes/profile.ts @@ -328,6 +328,6 @@ async function statelyProfile( return response; } -function serializeSyncToken(syncTokens: { [component: string]: string }) { +function serializeSyncToken(syncTokens: { [component: string]: string | number }) { return JSON.stringify(syncTokens); } From 16d131743d0688ab4a8a6320205f46f7d3b2263d Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sun, 18 Jan 2026 21:29:00 -0800 Subject: [PATCH 06/10] Really prefer postgres --- api/routes/update.ts | 43 ++++++++++++++++++++++++------------- api/stately/bulk-queries.ts | 16 ++++++++------ 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/api/routes/update.ts b/api/routes/update.ts index 677e9f2..28a2986 100644 --- a/api/routes/update.ts +++ b/api/routes/update.ts @@ -2,9 +2,13 @@ import { captureMessage } from '@sentry/node'; import { chunk, groupBy, partition, sortBy } from 'es-toolkit'; import express from 'express'; import asyncHandler from 'express-async-handler'; -import { transaction } from '../db/index.js'; +import { readTransaction, transaction } from '../db/index.js'; import { backfillMigrationState } from '../db/migration-state-queries.js'; -import { replaceSettings, setSetting as setSettingInPostgres } from '../db/settings-queries.js'; +import { + getSettings, + replaceSettings, + setSetting as setSettingInPostgres, +} from '../db/settings-queries.js'; import { metrics } from '../metrics/index.js'; import { ApiApp } from '../shapes/app.js'; import { DestinyVersion } from '../shapes/general.js'; @@ -309,22 +313,31 @@ async function statelyUpdate( mergedSettings = { ...mergedSettings, ...update.payload }; } - const statelySettings = await getSettingsForUpdate(txn, bungieMembershipId); - - if (statelySettings) { - mergedSettings = { ...statelySettings, ...mergedSettings }; - await transaction(async (pgClient) => { - await replaceSettings( - pgClient, - bungieMembershipId, - subtractObject(mergedSettings, defaultSettings), - ); - }); - await txn.del(keyFor(bungieMembershipId)); - } else { + // TODO: Remove the check for settings in Postgres once we're fully migrated off Stately + const pgSettings = await readTransaction((client) => + getSettings(client, bungieMembershipId), + ); + if (pgSettings) { await transaction(async (pgClient) => { await setSettingInPostgres(pgClient, bungieMembershipId, mergedSettings); }); + } else { + const statelySettings = await getSettingsForUpdate(txn, bungieMembershipId); + if (statelySettings) { + mergedSettings = { ...statelySettings, ...mergedSettings }; + await transaction(async (pgClient) => { + await replaceSettings( + pgClient, + bungieMembershipId, + subtractObject(mergedSettings, defaultSettings), + ); + }); + await txn.del(keyFor(bungieMembershipId)); + } else { + await transaction(async (pgClient) => { + await setSettingInPostgres(pgClient, bungieMembershipId, mergedSettings); + }); + } } break; } diff --git a/api/stately/bulk-queries.ts b/api/stately/bulk-queries.ts index fb342e1..e2feaf2 100644 --- a/api/stately/bulk-queries.ts +++ b/api/stately/bulk-queries.ts @@ -6,7 +6,7 @@ import { DeleteAllResponse } from '../shapes/delete-all.js'; import { ExportResponse } from '../shapes/export.js'; import { DestinyVersion } from '../shapes/general.js'; import { ProfileResponse } from '../shapes/profile.js'; -import { defaultSettings } from '../shapes/settings.js'; +import { defaultSettings, Settings } from '../shapes/settings.js'; import { delay, subtractObject } from '../utils.js'; import { client } from './client.js'; import { AnyItem } from './generated/index.js'; @@ -115,12 +115,14 @@ export async function exportDataForUser( bungieMembershipId: number, platformMembershipIds: string[], ): Promise { - let settings = await getSettings(bungieMembershipId); - if (!settings) { - const pgSettings = await readTransaction((client) => - getSettingsFromPostgres(client, bungieMembershipId), - ); - settings = { ...defaultSettings, ...pgSettings?.settings }; + let settings: Settings; + const pgSettings = await readTransaction((client) => + getSettingsFromPostgres(client, bungieMembershipId), + ); + if (pgSettings) { + settings = { ...defaultSettings, ...pgSettings.settings }; + } else { + settings = (await getSettings(bungieMembershipId)) ?? defaultSettings; } const responses = await Promise.all(platformMembershipIds.map((p) => exportDataForProfile(p))); From e04569cba6feacdc969b5e5d8c34d176346ee8c7 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sun, 18 Jan 2026 21:46:40 -0800 Subject: [PATCH 07/10] Don't worry about missing tokens --- api/routes/profile.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/routes/profile.ts b/api/routes/profile.ts index 8b79416..3a986a1 100644 --- a/api/routes/profile.ts +++ b/api/routes/profile.ts @@ -186,9 +186,9 @@ async function statelyProfile( }; const getSyncToken = (name: string) => { const tokenData = incomingSyncTokens?.[name]; - if (incomingSyncTokens && !tokenData) { - throw new Error(`Missing sync token: ${name}`); - } + // if (incomingSyncTokens && !tokenData) { + // throw new Error(`Missing sync token: ${name}`); + // } return tokenData as T | undefined; }; From a14e3fd60ead8d99e79f7fc7a46d5b608e79bf9f Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sun, 18 Jan 2026 22:01:30 -0800 Subject: [PATCH 08/10] More test --- api/server.test.ts | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/api/server.test.ts b/api/server.test.ts index 80e30d7..ec6f285 100644 --- a/api/server.test.ts +++ b/api/server.test.ts @@ -210,11 +210,40 @@ describe('profile', () => { expect(profileSyncResponse.syncToken).toBeDefined(); expect(profileSyncResponse.syncToken).not.toBe(profileResponse.syncToken); + expect(profileSyncResponse.syncToken).toContain('"s":'); expect(profileSyncResponse.sync).toBe(true); expect(profileSyncResponse.settings?.showNewItems).toBe(true); expect(profileSyncResponse.tags?.length).toBe(1); expect(profileSyncResponse.tags?.[0].id).toBe('1234'); expect(profileSyncResponse.deletedLoadoutIds?.length).toBe(1); + + const request2: ProfileUpdateRequest = { + platformMembershipId, + destinyVersion: 2, + updates: [ + { + action: 'setting', + payload: { + compareBaseStats: true, + }, + }, + ], + }; + await postRequestAuthed('/profile', request2).expect(200); + + const profileSyncResponse2 = (await getRequestAuthed( + `/profile?components=settings,loadouts,tags,triumphs,searches,hashtags&platformMembershipId=${platformMembershipId}&sync=${encodeURIComponent(profileSyncResponse.syncToken!)}`, + ) + .expect(200) + .json()) as ProfileResponse; + + expect(profileSyncResponse2.syncToken).toBeDefined(); + expect(profileSyncResponse2.syncToken).not.toBe(profileSyncResponse.syncToken); + expect(profileSyncResponse2.syncToken).toContain('"s":'); + expect(profileSyncResponse2.sync).toBe(true); + expect(profileSyncResponse2.settings).toBeDefined(); + expect(profileSyncResponse.settings?.compareBaseStats).toBe(true); + expect(profileSyncResponse2.settings?.showNewItems).toBe(true); }); it('can retrieve only settings, without needing a platform membership ID', async () => { From 0407fea71e72f24c38f9d07568db776747d12810 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sun, 18 Jan 2026 22:12:57 -0800 Subject: [PATCH 09/10] Fix import --- api/routes/import.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/api/routes/import.ts b/api/routes/import.ts index b80d84c..cfed0a4 100644 --- a/api/routes/import.ts +++ b/api/routes/import.ts @@ -1,6 +1,8 @@ import { uniqBy } from 'es-toolkit'; import { isEmpty } from 'es-toolkit/compat'; import asyncHandler from 'express-async-handler'; +import { transaction } from '../db/index.js'; +import { replaceSettings } from '../db/settings-queries.js'; import { ExportResponse } from '../shapes/export.js'; import { DestinyVersion } from '../shapes/general.js'; import { ImportResponse } from '../shapes/import.js'; @@ -15,7 +17,6 @@ import { importTags } from '../stately/item-annotations-queries.js'; import { importHashTags } from '../stately/item-hash-tags-queries.js'; import { importLoadouts } from '../stately/loadouts-queries.js'; import { importSearches } from '../stately/searches-queries.js'; -import { convertToStatelyItem } from '../stately/settings-queries.js'; import { batches } from '../stately/stately-utils.js'; import { importTriumphs } from '../stately/triumphs-queries.js'; import { badRequest, delay, subtractObject } from '../utils.js'; @@ -119,11 +120,6 @@ export async function statelyImport( let numTriumphs = 0; await deleteAllDataForUser(bungieMembershipId, platformMembershipIds); - const settingsItem = convertToStatelyItem( - { ...defaultSettings, ...settings }, - bungieMembershipId, - ); - // The export will have duplicates because import saved to each profile // instead of the one that was exported. itemHashTags = uniqBy(itemHashTags, (a) => a.hash); @@ -152,10 +148,9 @@ export async function statelyImport( items.push(...importSearches(platformMembershipId, searches)); } - // Put the settings in first since it's in a different group - await client.put({ - item: settingsItem, - }); + // Settings live in Postgres now + await transaction(async (client) => replaceSettings(client, bungieMembershipId, settings)); + // OK now put them in as fast as we can for (const batch of batches(items)) { // We shouldn't have any existing items... From 56710c16fa44aec6eadcd153de16f171a71c1836 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sun, 18 Jan 2026 22:30:05 -0800 Subject: [PATCH 10/10] More robust token handling --- api/routes/profile.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/api/routes/profile.ts b/api/routes/profile.ts index 3a986a1..58a7003 100644 --- a/api/routes/profile.ts +++ b/api/routes/profile.ts @@ -147,7 +147,10 @@ function extractSyncToken(syncTokenParam: string | undefined) { const tokenMap = JSON.parse(syncTokenParam) as { [component: string]: string | number }; return Object.entries(tokenMap).reduce<{ [component: string]: Buffer | number }>( (acc, [component, token]) => { - acc[component] = typeof token === 'string' ? Buffer.from(token, 'base64') : token; + acc[component] = + typeof token === 'string' && !/^\d+$/.exec(token) + ? Buffer.from(token, 'base64') + : Number(token); return acc; }, {}, @@ -209,7 +212,7 @@ async function statelyProfile( ); if (pgSettings) { const tokenData = getSyncToken('s'); - if (tokenData === undefined || pgSettings.lastModifiedAt > Number(tokenData)) { + if (tokenData === undefined || pgSettings.lastModifiedAt > tokenData) { response.settings = { ...defaultSettings, ...pgSettings.settings }; } addSyncToken('s', { canSync: true, tokenData: pgSettings?.lastModifiedAt ?? now });