Automigrate settings

api/db/settings-queries.test.ts

@@ -11,7 +11,7 @@ it('can insert settings where none exist before', async () => {
       showNewItems: true,
     });
 
-    const settings = await getSettings(client, bungieMembershipId);
+    const settings = (await getSettings(client, bungieMembershipId))!.settings;
     expect(settings.showNewItems).toBe(true);
   });
 });
@@ -22,14 +22,14 @@ it('can update settings', async () => {
       showNewItems: true,
     });
 
-    const settings = await getSettings(client, bungieMembershipId);
+    const settings = (await getSettings(client, bungieMembershipId))!.settings;
     expect(settings.showNewItems).toBe(true);
 
     await setSetting(client, bungieMembershipId, {
       showNewItems: false,
     });
 
-    const settings2 = await getSettings(client, bungieMembershipId);
+    const settings2 = (await getSettings(client, bungieMembershipId))!.settings;
     expect(settings2.showNewItems).toBe(false);
   });
 });
@@ -40,14 +40,14 @@ it('can partially update settings', async () => {
       showNewItems: true,
     });
 
-    const settings = await getSettings(client, bungieMembershipId);
+    const settings = (await getSettings(client, bungieMembershipId))!.settings;
     expect(settings.showNewItems).toBe(true);
 
     await setSetting(client, bungieMembershipId, {
       singleCharacter: true,
     });
 
-    const settings2 = await getSettings(client, bungieMembershipId);
+    const settings2 = (await getSettings(client, bungieMembershipId))!.settings;
     expect(settings2.showNewItems).toBe(true);
   });
 });

api/db/settings-queries.ts

@@ -7,13 +7,23 @@ import { Settings } from '../shapes/settings.js';
 export async function getSettings(
   client: ClientBase,
   bungieMembershipId: number,
-): Promise<Partial<Settings>> {
-  const results = await client.query<{ settings: Settings }>({
+): Promise<{ settings: Partial<Settings>; deleted: boolean; lastModifiedAt: number } | undefined> {
+  const results = await client.query<{
+    settings: Settings;
+    deleted_at: Date | null;
+    last_updated_at: Date;
+  }>({
     name: 'get_settings',
-    text: 'SELECT settings FROM settings WHERE membership_id = $1 and deleted_at IS NULL',
+    text: 'SELECT settings, deleted_at, last_updated_at FROM settings WHERE membership_id = $1',
     values: [bungieMembershipId],
   });
-  return results.rows.length > 0 ? results.rows[0].settings : {};
+  return results.rows.length > 0
+    ? {
+        settings: results.rows[0].settings,
+        deleted: Boolean(results.rows[0].deleted_at),
+        lastModifiedAt: results.rows[0].last_updated_at.getTime(),
+      }
+    : undefined;
 }
 
 /**

api/routes/profile.ts

@@ -2,10 +2,13 @@ import * as Sentry from '@sentry/node';
 import { ListToken } from '@stately-cloud/client';
 import express from 'express';
 import asyncHandler from 'express-async-handler';
+import { readTransaction } from '../db/index.js';
+import { getSettings } from '../db/settings-queries.js';
 import { metrics } from '../metrics/index.js';
 import { ApiApp } from '../shapes/app.js';
 import { DestinyVersion } from '../shapes/general.js';
 import { ProfileResponse } from '../shapes/profile.js';
+import { defaultSettings } from '../shapes/settings.js';
 import { UserInfo } from '../shapes/user.js';
 import { getProfile, syncProfile } from '../stately/bulk-queries.js';
 import { cannedSearches } from '../stately/searches-queries.js';
@@ -141,10 +144,10 @@ function extractSyncToken(syncTokenParam: string | undefined) {
     }
 
     try {
-      const tokenMap = JSON.parse(syncTokenParam) as { [component: string]: string };
-      return Object.entries(tokenMap).reduce<{ [component: string]: Buffer }>(
+      const tokenMap = JSON.parse(syncTokenParam) as { [component: string]: string | number };
+      return Object.entries(tokenMap).reduce<{ [component: string]: Buffer | number }>(
         (acc, [component, token]) => {
-          acc[component] = Buffer.from(token, 'base64');
+          acc[component] = typeof token === 'string' ? Buffer.from(token, 'base64') : token;
           return acc;
         },
         {},
@@ -162,40 +165,70 @@ async function statelyProfile(
   bungieMembershipId: number,
   platformMembershipId: string | undefined,
   destinyVersion: DestinyVersion,
-  incomingSyncTokens?: { [component: string]: Buffer },
+  incomingSyncTokens?: { [component: string]: Buffer | number },
 ) {
   let response: ProfileResponse = {
     sync: Boolean(incomingSyncTokens),
   };
   const timerPrefix = response.sync ? 'profileSync' : 'profileStately';
   const counterPrefix = response.sync ? 'sync' : 'stately';
   const syncTokens: { [component: string]: string } = {};
-  const addSyncToken = (name: string, token: ListToken) => {
+  const addSyncToken = (
+    name: string,
+    token: ListToken | { canSync: boolean; tokenData: number },
+  ) => {
     if (token.canSync) {
-      syncTokens[name] = Buffer.from(token.tokenData).toString('base64');
+      syncTokens[name] =
+        token.tokenData instanceof Uint8Array
+          ? Buffer.from(token.tokenData).toString('base64')
+          : token.tokenData.toString();
     }
   };
-  const getSyncToken = (name: string) => {
+  const getSyncToken = <T extends number | Buffer>(name: string) => {
     const tokenData = incomingSyncTokens?.[name];
     if (incomingSyncTokens && !tokenData) {
       throw new Error(`Missing sync token: ${name}`);
     }
-    return tokenData;
+    return tokenData as T | undefined;
   };
 
   // We'll accumulate promises and await them all at the end
   const promises: Promise<void>[] = [];
+
   if (components.includes('settings')) {
     // TODO: should settings be stored under profile too?? maybe primary profile ID?
     promises.push(
       (async () => {
+        // Load settings from Stately. If they're there, you're done. Otherwise load from Postgres.
         const start = new Date();
-        const tokenData = getSyncToken('settings');
-        const { settings: storedSettings, token: settingsToken } = tokenData
-          ? await syncSettings(tokenData)
-          : await querySettings(bungieMembershipId);
-        response.settings = storedSettings;
-        addSyncToken('settings', settingsToken);
+
+        const statelySettings = await querySettings(bungieMembershipId);
+        if (!statelySettings.settings) {
+          const now = Date.now();
+          const pgSettings = await readTransaction(async (pgClient) =>
+            getSettings(pgClient, bungieMembershipId),
+          );
+          if (pgSettings) {
+            const tokenData = getSyncToken<number>('s');
+            if (tokenData === undefined || pgSettings.lastModifiedAt > tokenData) {
+              response.settings = { ...defaultSettings, ...pgSettings.settings };
+            }
+          } else {
+            response.settings = defaultSettings;
+          }
+          addSyncToken('s', { canSync: true, tokenData: pgSettings?.lastModifiedAt ?? now });
+        } else {
+          const tokenData = getSyncToken<Buffer>('settings');
+          const { settings: storedSettings, token: settingsToken } = tokenData
+            ? await syncSettings(tokenData)
+            : {
+                settings: statelySettings.settings ?? defaultSettings,
+                token: statelySettings.token,
+              };
+          response.settings = storedSettings;
+          addSyncToken('settings', settingsToken);
+        }
+
         metrics.timing(`${timerPrefix}.settings`, start);
       })(),
     );
@@ -225,7 +258,7 @@ async function statelyProfile(
       promises.push(
         (async () => {
           const start = new Date();
-          const tokenData = getSyncToken(name);
+          const tokenData = getSyncToken<Buffer>(name);
           const { profile, token } = tokenData
             ? await syncProfile(tokenData)
             : await getProfile(platformMembershipId, destinyVersion, suffix);

api/routes/update.ts

@@ -4,6 +4,7 @@ import express from 'express';
 import asyncHandler from 'express-async-handler';
 import { transaction } from '../db/index.js';
 import { backfillMigrationState } from '../db/migration-state-queries.js';
+import { replaceSettings, setSetting as setSettingInPostgres } from '../db/settings-queries.js';
 import { metrics } from '../metrics/index.js';
 import { ApiApp } from '../shapes/app.js';
 import { DestinyVersion } from '../shapes/general.js';
@@ -25,7 +26,7 @@ import {
   UsedSearchUpdate,
 } from '../shapes/profile.js';
 import { SearchType } from '../shapes/search.js';
-import { Settings } from '../shapes/settings.js';
+import { defaultSettings, Settings } from '../shapes/settings.js';
 import { UserInfo } from '../shapes/user.js';
 import { client } from '../stately/client.js';
 import {
@@ -42,14 +43,19 @@ import {
   UpdateSearch,
   updateSearches,
 } from '../stately/searches-queries.js';
-import { setSetting as setSettingInStately } from '../stately/settings-queries.js';
+import {
+  deleteSettings as deleteSettingsInStately,
+  getSettingsForUpdate,
+  setSetting as setSettingInStately,
+} from '../stately/settings-queries.js';
 import { trackUntrackTriumphs } from '../stately/triumphs-queries.js';
 import {
   badRequest,
   checkPlatformMembershipId,
   delay,
   isValidItemId,
   isValidPlatformMembershipId,
+  subtractObject,
 } from '../utils.js';
 
 /**
@@ -213,14 +219,17 @@ function validateUpdates(
     }
 
     switch (update.action) {
-      case 'setting':
       case 'tag_cleanup':
       case 'delete_loadout':
       case 'track_triumph':
       case 'delete_search':
         // no special validation
         break;
 
+      case 'setting':
+        result = validateUpdateSettings(update.payload);
+        break;
+
       case 'loadout':
         result = validateUpdateLoadout(update.payload);
         break;
@@ -303,7 +312,28 @@ async function statelyUpdate(
             for (const update of group as SettingUpdate[]) {
               mergedSettings = { ...mergedSettings, ...update.payload };
             }
-            await setSettingInStately(txn, bungieMembershipId, mergedSettings);
+
+            if (bungieMembershipId === 7094) {
+              const statelySettings = await getSettingsForUpdate(txn, bungieMembershipId);
+
+              if (statelySettings) {
+                mergedSettings = { ...statelySettings, ...mergedSettings };
+                await transaction(async (pgClient) => {
+                  await replaceSettings(
+                    pgClient,
+                    bungieMembershipId,
+                    subtractObject(mergedSettings, defaultSettings),
+                  );
+                });
+                await deleteSettingsInStately(bungieMembershipId);
+              } else {
+                await transaction(async (pgClient) => {
+                  await setSettingInPostgres(pgClient, bungieMembershipId, mergedSettings);
+                });
+              }
+            } else {
+              await setSettingInStately(txn, bungieMembershipId, mergedSettings);
+            }
             break;
           }
 
@@ -506,6 +536,42 @@ async function statelyUpdate(
 //   metrics.timing('update.loadout', start);
 // }
 
+/** Helper function to validate integer ranges */
+function validateIntRange(
+  value: unknown,
+  fieldName: string,
+  min: number,
+  max: number,
+): string | undefined {
+  if (value === undefined) {
+    return undefined;
+  }
+  if (typeof value !== 'number' || !Number.isInteger(value) || value < min || value > max) {
+    metrics.increment(`update.validation.${fieldName}OutOfRange.count`);
+    return `${fieldName} must be an integer between ${min} and ${max}`;
+  }
+  return undefined;
+}
+
+function validateUpdateSettings(settings: Partial<Settings>): ProfileUpdateResult {
+  const errors = [
+    // Validate numeric ranges
+    validateIntRange(settings.charCol, 'charCol', 2, 5),
+    validateIntRange(settings.charColMobile, 'charColMobile', 2, 5),
+    validateIntRange(settings.itemSize, 'itemSize', 0, 66),
+    validateIntRange(settings.inventoryClearSpaces, 'inventoryClearSpaces', 0, 9),
+  ].filter((e) => e !== undefined);
+
+  if (errors.length > 0) {
+    return {
+      status: 'InvalidArgument',
+      message: errors.join('; '),
+    };
+  }
+
+  return { status: 'Success' };
+}
+
 function validateUpdateLoadout(loadout: Loadout): ProfileUpdateResult {
   return validateLoadout('update', loadout) ?? { status: 'Success' };
 }

api/stately/bulk-queries.test.ts

@@ -80,7 +80,7 @@ describe('deleteAllDataForUser', () => {
       ),
     ).toEqual([]);
     expect((await getTrackedTriumphsForProfile(platformMembershipId)).triumphs).toEqual([]);
-    expect((await getSettings(bungieMembershipId)).showNewItems).toBe(false);
+    expect((await getSettings(bungieMembershipId))?.showNewItems).toBe(undefined);
   });
 });
 

api/stately/bulk-queries.ts

@@ -1,5 +1,7 @@
 import { captureMessage } from '@sentry/node';
 import { keyPath, ListToken } from '@stately-cloud/client';
+import { readTransaction, transaction } from '../db/index.js';
+import { deleteSettings, getSettings as getSettingsFromPostgres } from '../db/settings-queries.js';
 import { DeleteAllResponse } from '../shapes/delete-all.js';
 import { ExportResponse } from '../shapes/export.js';
 import { DestinyVersion } from '../shapes/general.js';
@@ -12,7 +14,7 @@ import { convertItemAnnotation, keyFor as tagKeyFor } from './item-annotations-q
 import { convertItemHashTag, keyFor as hashTagKeyFor } from './item-hash-tags-queries.js';
 import { convertLoadoutFromStately, keyFor as loadoutKeyFor } from './loadouts-queries.js';
 import { convertSearchFromStately, keyFor as searchKeyFor } from './searches-queries.js';
-import { deleteSettings, getSettings } from './settings-queries.js';
+import { deleteSettings as deleteSettingsInStately, getSettings } from './settings-queries.js';
 import { batches, fromStatelyUUID, parseKeyPath } from './stately-utils.js';
 import { keyFor as triumphKeyFor } from './triumphs-queries.js';
 
@@ -26,7 +28,9 @@ export async function deleteAllDataForUser(
   const responses = await Promise.all(platformMembershipIds.map((p) => deleteAllDataForProfile(p)));
 
   // Also delete settings, which are stored by membershipId
-  await deleteSettings(bungieMembershipId);
+  await deleteSettingsInStately(bungieMembershipId);
+  // And delete from Postgres too
+  await transaction(async (pgClient) => deleteSettings(pgClient, bungieMembershipId));
 
   const response = responses.reduce<DeleteAllResponse['deleted']>(
     (acc, r) => {
@@ -111,10 +115,16 @@ export async function exportDataForUser(
   bungieMembershipId: number,
   platformMembershipIds: string[],
 ): Promise<ExportResponse> {
-  const settingsPromise = getSettings(bungieMembershipId);
+  let settings = await getSettings(bungieMembershipId);
+  if (!settings) {
+    const pgSettings = await readTransaction((client) =>
+      getSettingsFromPostgres(client, bungieMembershipId),
+    );
+    settings = { ...defaultSettings, ...pgSettings?.settings };
+  }
+
   const responses = await Promise.all(platformMembershipIds.map((p) => exportDataForProfile(p)));
 
-  const settings = await settingsPromise;
   const initialResponse: ExportResponse = {
     settings: subtractObject(settings, defaultSettings),
     loadouts: [],