From 640585bbc12ef29b41b50cbab16d3ec1a04f1133 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sat, 17 Jan 2026 18:53:24 -0800 Subject: [PATCH 1/4] Automigrate settings Ben's account only, for now --- api/db/settings-queries.test.ts | 10 ++--- api/db/settings-queries.ts | 18 ++++++-- api/routes/profile.ts | 63 +++++++++++++++++++++------- api/routes/update.ts | 74 +++++++++++++++++++++++++++++++-- api/stately/bulk-queries.ts | 9 +++- api/stately/settings-queries.ts | 43 +++++++++++++------ 6 files changed, 174 insertions(+), 43 deletions(-) diff --git a/api/db/settings-queries.test.ts b/api/db/settings-queries.test.ts index c026d9a..ef8a512 100644 --- a/api/db/settings-queries.test.ts +++ b/api/db/settings-queries.test.ts @@ -11,7 +11,7 @@ it('can insert settings where none exist before', async () => { showNewItems: true, }); - const settings = await getSettings(client, bungieMembershipId); + const settings = (await getSettings(client, bungieMembershipId))!.settings; expect(settings.showNewItems).toBe(true); }); }); @@ -22,14 +22,14 @@ it('can update settings', async () => { showNewItems: true, }); - const settings = await getSettings(client, bungieMembershipId); + const settings = (await getSettings(client, bungieMembershipId))!.settings; expect(settings.showNewItems).toBe(true); await setSetting(client, bungieMembershipId, { showNewItems: false, }); - const settings2 = await getSettings(client, bungieMembershipId); + const settings2 = (await getSettings(client, bungieMembershipId))!.settings; expect(settings2.showNewItems).toBe(false); }); }); @@ -40,14 +40,14 @@ it('can partially update settings', async () => { showNewItems: true, }); - const settings = await getSettings(client, bungieMembershipId); + const settings = (await getSettings(client, bungieMembershipId))!.settings; expect(settings.showNewItems).toBe(true); await setSetting(client, bungieMembershipId, { singleCharacter: true, }); - const settings2 = await getSettings(client, bungieMembershipId); + const settings2 = (await getSettings(client, bungieMembershipId))!.settings; expect(settings2.showNewItems).toBe(true); }); }); diff --git a/api/db/settings-queries.ts b/api/db/settings-queries.ts index 1c30bc6..8559aed 100644 --- a/api/db/settings-queries.ts +++ b/api/db/settings-queries.ts @@ -7,13 +7,23 @@ import { Settings } from '../shapes/settings.js'; export async function getSettings( client: ClientBase, bungieMembershipId: number, -): Promise> { - const results = await client.query<{ settings: Settings }>({ +): Promise<{ settings: Partial; deleted: boolean; lastModifiedAt: number } | undefined> { + const results = await client.query<{ + settings: Settings; + deleted_at: Date | null; + last_updated_at: Date; + }>({ name: 'get_settings', - text: 'SELECT settings FROM settings WHERE membership_id = $1 and deleted_at IS NULL', + text: 'SELECT settings, deleted_at, last_updated_at FROM settings WHERE membership_id = $1', values: [bungieMembershipId], }); - return results.rows.length > 0 ? results.rows[0].settings : {}; + return results.rows.length > 0 + ? { + settings: results.rows[0].settings, + deleted: Boolean(results.rows[0].deleted_at), + lastModifiedAt: results.rows[0].last_updated_at.getTime(), + } + : undefined; } /** diff --git a/api/routes/profile.ts b/api/routes/profile.ts index 718e0dc..a09d213 100644 --- a/api/routes/profile.ts +++ b/api/routes/profile.ts @@ -2,10 +2,13 @@ import * as Sentry from '@sentry/node'; import { ListToken } from '@stately-cloud/client'; import express from 'express'; import asyncHandler from 'express-async-handler'; +import { readTransaction } from '../db/index.js'; +import { getSettings } from '../db/settings-queries.js'; import { metrics } from '../metrics/index.js'; import { ApiApp } from '../shapes/app.js'; import { DestinyVersion } from '../shapes/general.js'; import { ProfileResponse } from '../shapes/profile.js'; +import { defaultSettings } from '../shapes/settings.js'; import { UserInfo } from '../shapes/user.js'; import { getProfile, syncProfile } from '../stately/bulk-queries.js'; import { cannedSearches } from '../stately/searches-queries.js'; @@ -141,10 +144,10 @@ function extractSyncToken(syncTokenParam: string | undefined) { } try { - const tokenMap = JSON.parse(syncTokenParam) as { [component: string]: string }; - return Object.entries(tokenMap).reduce<{ [component: string]: Buffer }>( + const tokenMap = JSON.parse(syncTokenParam) as { [component: string]: string | number }; + return Object.entries(tokenMap).reduce<{ [component: string]: Buffer | number }>( (acc, [component, token]) => { - acc[component] = Buffer.from(token, 'base64'); + acc[component] = typeof token === 'string' ? Buffer.from(token, 'base64') : token; return acc; }, {}, @@ -162,7 +165,7 @@ async function statelyProfile( bungieMembershipId: number, platformMembershipId: string | undefined, destinyVersion: DestinyVersion, - incomingSyncTokens?: { [component: string]: Buffer }, + incomingSyncTokens?: { [component: string]: Buffer | number }, ) { let response: ProfileResponse = { sync: Boolean(incomingSyncTokens), @@ -170,32 +173,62 @@ async function statelyProfile( const timerPrefix = response.sync ? 'profileSync' : 'profileStately'; const counterPrefix = response.sync ? 'sync' : 'stately'; const syncTokens: { [component: string]: string } = {}; - const addSyncToken = (name: string, token: ListToken) => { + const addSyncToken = ( + name: string, + token: ListToken | { canSync: boolean; tokenData: number }, + ) => { if (token.canSync) { - syncTokens[name] = Buffer.from(token.tokenData).toString('base64'); + syncTokens[name] = + token.tokenData instanceof Uint8Array + ? Buffer.from(token.tokenData).toString('base64') + : token.tokenData.toString(); } }; - const getSyncToken = (name: string) => { + const getSyncToken = (name: string) => { const tokenData = incomingSyncTokens?.[name]; if (incomingSyncTokens && !tokenData) { throw new Error(`Missing sync token: ${name}`); } - return tokenData; + return tokenData as T | undefined; }; // We'll accumulate promises and await them all at the end const promises: Promise[] = []; + if (components.includes('settings')) { // TODO: should settings be stored under profile too?? maybe primary profile ID? promises.push( (async () => { + // Load settings from Stately. If they're there, you're done. Otherwise load from Postgres. const start = new Date(); - const tokenData = getSyncToken('settings'); - const { settings: storedSettings, token: settingsToken } = tokenData - ? await syncSettings(tokenData) - : await querySettings(bungieMembershipId); - response.settings = storedSettings; - addSyncToken('settings', settingsToken); + + const statelySettings = await querySettings(bungieMembershipId); + if (!statelySettings.settings) { + const now = Date.now(); + const pgSettings = await readTransaction(async (pgClient) => + getSettings(pgClient, bungieMembershipId), + ); + if (pgSettings) { + const tokenData = getSyncToken('s'); + if (tokenData === undefined || pgSettings.lastModifiedAt > tokenData) { + response.settings = { ...defaultSettings, ...pgSettings.settings }; + } + } else { + response.settings = defaultSettings; + } + addSyncToken('s', { canSync: true, tokenData: pgSettings?.lastModifiedAt ?? now }); + } else { + const tokenData = getSyncToken('settings'); + const { settings: storedSettings, token: settingsToken } = tokenData + ? await syncSettings(tokenData) + : { + settings: statelySettings.settings ?? defaultSettings, + token: statelySettings.token, + }; + response.settings = storedSettings; + addSyncToken('settings', settingsToken); + } + metrics.timing(`${timerPrefix}.settings`, start); })(), ); @@ -225,7 +258,7 @@ async function statelyProfile( promises.push( (async () => { const start = new Date(); - const tokenData = getSyncToken(name); + const tokenData = getSyncToken(name); const { profile, token } = tokenData ? await syncProfile(tokenData) : await getProfile(platformMembershipId, destinyVersion, suffix); diff --git a/api/routes/update.ts b/api/routes/update.ts index bd404de..7ccca44 100644 --- a/api/routes/update.ts +++ b/api/routes/update.ts @@ -4,6 +4,7 @@ import express from 'express'; import asyncHandler from 'express-async-handler'; import { transaction } from '../db/index.js'; import { backfillMigrationState } from '../db/migration-state-queries.js'; +import { replaceSettings, setSetting as setSettingInPostgres } from '../db/settings-queries.js'; import { metrics } from '../metrics/index.js'; import { ApiApp } from '../shapes/app.js'; import { DestinyVersion } from '../shapes/general.js'; @@ -25,7 +26,7 @@ import { UsedSearchUpdate, } from '../shapes/profile.js'; import { SearchType } from '../shapes/search.js'; -import { Settings } from '../shapes/settings.js'; +import { defaultSettings, Settings } from '../shapes/settings.js'; import { UserInfo } from '../shapes/user.js'; import { client } from '../stately/client.js'; import { @@ -42,7 +43,11 @@ import { UpdateSearch, updateSearches, } from '../stately/searches-queries.js'; -import { setSetting as setSettingInStately } from '../stately/settings-queries.js'; +import { + deleteSettings as deleteSettingsInStately, + getSettingsForUpdate, + setSetting as setSettingInStately, +} from '../stately/settings-queries.js'; import { trackUntrackTriumphs } from '../stately/triumphs-queries.js'; import { badRequest, @@ -50,6 +55,7 @@ import { delay, isValidItemId, isValidPlatformMembershipId, + subtractObject, } from '../utils.js'; /** @@ -213,7 +219,6 @@ function validateUpdates( } switch (update.action) { - case 'setting': case 'tag_cleanup': case 'delete_loadout': case 'track_triumph': @@ -221,6 +226,10 @@ function validateUpdates( // no special validation break; + case 'setting': + result = validateUpdateSettings(update.payload); + break; + case 'loadout': result = validateUpdateLoadout(update.payload); break; @@ -303,7 +312,28 @@ async function statelyUpdate( for (const update of group as SettingUpdate[]) { mergedSettings = { ...mergedSettings, ...update.payload }; } - await setSettingInStately(txn, bungieMembershipId, mergedSettings); + + if (bungieMembershipId === 7094) { + const statelySettings = await getSettingsForUpdate(txn, bungieMembershipId); + + if (statelySettings) { + mergedSettings = { ...statelySettings, ...mergedSettings }; + await transaction(async (pgClient) => { + await replaceSettings( + pgClient, + bungieMembershipId, + subtractObject(mergedSettings, defaultSettings), + ); + }); + await deleteSettingsInStately(bungieMembershipId); + } else { + await transaction(async (pgClient) => { + await setSettingInPostgres(pgClient, bungieMembershipId, mergedSettings); + }); + } + } else { + await setSettingInStately(txn, bungieMembershipId, mergedSettings); + } break; } @@ -506,6 +536,42 @@ async function statelyUpdate( // metrics.timing('update.loadout', start); // } +/** Helper function to validate integer ranges */ +function validateIntRange( + value: unknown, + fieldName: string, + min: number, + max: number, +): string | undefined { + if (value === undefined) { + return undefined; + } + if (typeof value !== 'number' || !Number.isInteger(value) || value < min || value > max) { + metrics.increment(`update.validation.${fieldName}OutOfRange.count`); + return `${fieldName} must be an integer between ${min} and ${max}`; + } + return undefined; +} + +function validateUpdateSettings(settings: Partial): ProfileUpdateResult { + const errors = [ + // Validate numeric ranges + validateIntRange(settings.charCol, 'charCol', 2, 5), + validateIntRange(settings.charColMobile, 'charColMobile', 2, 5), + validateIntRange(settings.itemSize, 'itemSize', 0, 66), + validateIntRange(settings.inventoryClearSpaces, 'inventoryClearSpaces', 0, 9), + ].filter((e) => e !== undefined); + + if (errors.length > 0) { + return { + status: 'InvalidArgument', + message: errors.join('; '), + }; + } + + return { status: 'Success' }; +} + function validateUpdateLoadout(loadout: Loadout): ProfileUpdateResult { return validateLoadout('update', loadout) ?? { status: 'Success' }; } diff --git a/api/stately/bulk-queries.ts b/api/stately/bulk-queries.ts index 2858a09..73e9f53 100644 --- a/api/stately/bulk-queries.ts +++ b/api/stately/bulk-queries.ts @@ -1,5 +1,7 @@ import { captureMessage } from '@sentry/node'; import { keyPath, ListToken } from '@stately-cloud/client'; +import { transaction } from '../db/index.js'; +import { deleteSettings } from '../db/settings-queries.js'; import { DeleteAllResponse } from '../shapes/delete-all.js'; import { ExportResponse } from '../shapes/export.js'; import { DestinyVersion } from '../shapes/general.js'; @@ -12,7 +14,7 @@ import { convertItemAnnotation, keyFor as tagKeyFor } from './item-annotations-q import { convertItemHashTag, keyFor as hashTagKeyFor } from './item-hash-tags-queries.js'; import { convertLoadoutFromStately, keyFor as loadoutKeyFor } from './loadouts-queries.js'; import { convertSearchFromStately, keyFor as searchKeyFor } from './searches-queries.js'; -import { deleteSettings, getSettings } from './settings-queries.js'; +import { deleteSettings as deleteSettingsInStately, getSettings } from './settings-queries.js'; import { batches, fromStatelyUUID, parseKeyPath } from './stately-utils.js'; import { keyFor as triumphKeyFor } from './triumphs-queries.js'; @@ -26,7 +28,9 @@ export async function deleteAllDataForUser( const responses = await Promise.all(platformMembershipIds.map((p) => deleteAllDataForProfile(p))); // Also delete settings, which are stored by membershipId - await deleteSettings(bungieMembershipId); + await deleteSettingsInStately(bungieMembershipId); + // And delete from Postgres too + await transaction(async (pgClient) => deleteSettings(pgClient, bungieMembershipId)); const response = responses.reduce( (acc, r) => { @@ -111,6 +115,7 @@ export async function exportDataForUser( bungieMembershipId: number, platformMembershipIds: string[], ): Promise { + // TODO: get settings from Postgres first, then fall back to Stately const settingsPromise = getSettings(bungieMembershipId); const responses = await Promise.all(platformMembershipIds.map((p) => exportDataForProfile(p))); diff --git a/api/stately/settings-queries.ts b/api/stately/settings-queries.ts index 083449f..5d8f736 100644 --- a/api/stately/settings-queries.ts +++ b/api/stately/settings-queries.ts @@ -53,7 +53,7 @@ export async function getSettings(bungieMembershipId: number): Promise */ export async function querySettings( bungieMembershipId: number, -): Promise<{ settings: Settings; token: ListToken }> { +): Promise<{ settings: Settings | undefined; token: ListToken }> { const iter = client.beginList(keyFor(bungieMembershipId)); let settingsItem: StatelySettings | undefined; for await (const item of iter) { @@ -63,7 +63,7 @@ export async function querySettings( } const token = iter.token!; return { - settings: settingsItem ? convertToDimSettings(settingsItem) : defaultSettings, + settings: settingsItem ? convertToDimSettings(settingsItem) : undefined, token, }; } @@ -315,6 +315,28 @@ export function convertToStatelyItem( }); } +/** + * Get existing settings for update purposes. + */ +export async function getSettingsForUpdate( + txn: Transaction, + bungieMembershipId: number, +): Promise { + const storedSettings = await txn.get('Settings', keyFor(bungieMembershipId)); + return storedSettings ? convertToDimSettings(storedSettings) : undefined; +} + +/** + * Update settings by putting the full settings object. + */ +export async function updateSettings( + txn: Transaction, + bungieMembershipId: number, + settings: Settings, +): Promise { + await txn.put(convertToStatelyItem(settings, bungieMembershipId)); +} + /** * Update specific key/value pairs within settings, leaving the rest alone. Creates the settings row if it doesn't exist. */ @@ -323,17 +345,12 @@ export async function setSetting( bungieMembershipId: number, settings: Partial, ): Promise { - const storedSettings = await txn.get('Settings', keyFor(bungieMembershipId)); - await txn.put( - convertToStatelyItem( - // Merge in the partial settings - { - ...(storedSettings ? convertToDimSettings(storedSettings) : defaultSettings), - ...settings, - }, - bungieMembershipId, - ), - ); + const existingSettings = await getSettingsForUpdate(txn, bungieMembershipId); + await updateSettings(txn, bungieMembershipId, { + ...defaultSettings, + ...existingSettings, + ...settings, + }); } /** From 157eb64b6dc27008afea75e54fe351cf0440c96c Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sat, 17 Jan 2026 19:05:35 -0800 Subject: [PATCH 2/4] Missed a TODO --- api/stately/bulk-queries.test.ts | 2 +- api/stately/bulk-queries.ts | 15 ++++++++++----- api/stately/settings-queries.ts | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/api/stately/bulk-queries.test.ts b/api/stately/bulk-queries.test.ts index 126c014..655f658 100644 --- a/api/stately/bulk-queries.test.ts +++ b/api/stately/bulk-queries.test.ts @@ -80,7 +80,7 @@ describe('deleteAllDataForUser', () => { ), ).toEqual([]); expect((await getTrackedTriumphsForProfile(platformMembershipId)).triumphs).toEqual([]); - expect((await getSettings(bungieMembershipId)).showNewItems).toBe(false); + expect((await getSettings(bungieMembershipId))?.showNewItems).toBe(false); }); }); diff --git a/api/stately/bulk-queries.ts b/api/stately/bulk-queries.ts index 73e9f53..fb342e1 100644 --- a/api/stately/bulk-queries.ts +++ b/api/stately/bulk-queries.ts @@ -1,7 +1,7 @@ import { captureMessage } from '@sentry/node'; import { keyPath, ListToken } from '@stately-cloud/client'; -import { transaction } from '../db/index.js'; -import { deleteSettings } from '../db/settings-queries.js'; +import { readTransaction, transaction } from '../db/index.js'; +import { deleteSettings, getSettings as getSettingsFromPostgres } from '../db/settings-queries.js'; import { DeleteAllResponse } from '../shapes/delete-all.js'; import { ExportResponse } from '../shapes/export.js'; import { DestinyVersion } from '../shapes/general.js'; @@ -115,11 +115,16 @@ export async function exportDataForUser( bungieMembershipId: number, platformMembershipIds: string[], ): Promise { - // TODO: get settings from Postgres first, then fall back to Stately - const settingsPromise = getSettings(bungieMembershipId); + let settings = await getSettings(bungieMembershipId); + if (!settings) { + const pgSettings = await readTransaction((client) => + getSettingsFromPostgres(client, bungieMembershipId), + ); + settings = { ...defaultSettings, ...pgSettings?.settings }; + } + const responses = await Promise.all(platformMembershipIds.map((p) => exportDataForProfile(p))); - const settings = await settingsPromise; const initialResponse: ExportResponse = { settings: subtractObject(settings, defaultSettings), loadouts: [], diff --git a/api/stately/settings-queries.ts b/api/stately/settings-queries.ts index 5d8f736..287ef38 100644 --- a/api/stately/settings-queries.ts +++ b/api/stately/settings-queries.ts @@ -43,9 +43,9 @@ export function keyFor(bungieMembershipId: number) { /** * Get settings for a particular account. */ -export async function getSettings(bungieMembershipId: number): Promise { +export async function getSettings(bungieMembershipId: number): Promise { const results = await client.get('Settings', keyFor(bungieMembershipId)); - return results ? convertToDimSettings(results) : defaultSettings; + return results ? convertToDimSettings(results) : undefined; } /** From 0795142d2414e9dc846200177c84f3fd309b37f0 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sat, 17 Jan 2026 19:09:07 -0800 Subject: [PATCH 3/4] Types --- api/stately/settings-queries.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/stately/settings-queries.test.ts b/api/stately/settings-queries.test.ts index 3a08c6b..5c3a109 100644 --- a/api/stately/settings-queries.test.ts +++ b/api/stately/settings-queries.test.ts @@ -33,7 +33,7 @@ it('can insert settings where none exist before', async () => { }); const settings = await getSettings(bungieMembershipId); - expect(settings.showNewItems).toBe(true); + expect(settings?.showNewItems).toBe(true); }); it('can update settings', async () => { @@ -44,7 +44,7 @@ it('can update settings', async () => { }); const settings = await getSettings(bungieMembershipId); - expect(settings.showNewItems).toBe(true); + expect(settings?.showNewItems).toBe(true); await client.transaction(async (txn) => { await setSetting(txn, bungieMembershipId, { @@ -53,7 +53,7 @@ it('can update settings', async () => { }); const settings2 = await getSettings(bungieMembershipId); - expect(settings2.showNewItems).toBe(false); + expect(settings2?.showNewItems).toBe(false); }); it('can partially update settings', async () => { @@ -64,7 +64,7 @@ it('can partially update settings', async () => { }); const settings = await getSettings(bungieMembershipId); - expect(settings.showNewItems).toBe(true); + expect(settings?.showNewItems).toBe(true); await client.transaction(async (txn) => { await setSetting(txn, bungieMembershipId, { @@ -73,5 +73,5 @@ it('can partially update settings', async () => { }); const settings2 = await getSettings(bungieMembershipId); - expect(settings2.showNewItems).toBe(true); + expect(settings2?.showNewItems).toBe(true); }); From 93291d8e8fdcb5863c096b64e96d14dda5240a5e Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sat, 17 Jan 2026 21:01:18 -0800 Subject: [PATCH 4/4] OK --- api/stately/bulk-queries.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/stately/bulk-queries.test.ts b/api/stately/bulk-queries.test.ts index 655f658..d85a720 100644 --- a/api/stately/bulk-queries.test.ts +++ b/api/stately/bulk-queries.test.ts @@ -80,7 +80,7 @@ describe('deleteAllDataForUser', () => { ), ).toEqual([]); expect((await getTrackedTriumphsForProfile(platformMembershipId)).triumphs).toEqual([]); - expect((await getSettings(bungieMembershipId))?.showNewItems).toBe(false); + expect((await getSettings(bungieMembershipId))?.showNewItems).toBe(undefined); }); });