diff --git a/api/db/settings-queries.ts b/api/db/settings-queries.ts index 8559aed..b809249 100644 --- a/api/db/settings-queries.ts +++ b/api/db/settings-queries.ts @@ -9,7 +9,7 @@ export async function getSettings( bungieMembershipId: number, ): Promise<{ settings: Partial; deleted: boolean; lastModifiedAt: number } | undefined> { const results = await client.query<{ - settings: Settings; + settings: Partial; deleted_at: Date | null; last_updated_at: Date; }>({ diff --git a/api/routes/import.ts b/api/routes/import.ts index b80d84c..cfed0a4 100644 --- a/api/routes/import.ts +++ b/api/routes/import.ts @@ -1,6 +1,8 @@ import { uniqBy } from 'es-toolkit'; import { isEmpty } from 'es-toolkit/compat'; import asyncHandler from 'express-async-handler'; +import { transaction } from '../db/index.js'; +import { replaceSettings } from '../db/settings-queries.js'; import { ExportResponse } from '../shapes/export.js'; import { DestinyVersion } from '../shapes/general.js'; import { ImportResponse } from '../shapes/import.js'; @@ -15,7 +17,6 @@ import { importTags } from '../stately/item-annotations-queries.js'; import { importHashTags } from '../stately/item-hash-tags-queries.js'; import { importLoadouts } from '../stately/loadouts-queries.js'; import { importSearches } from '../stately/searches-queries.js'; -import { convertToStatelyItem } from '../stately/settings-queries.js'; import { batches } from '../stately/stately-utils.js'; import { importTriumphs } from '../stately/triumphs-queries.js'; import { badRequest, delay, subtractObject } from '../utils.js'; @@ -119,11 +120,6 @@ export async function statelyImport( let numTriumphs = 0; await deleteAllDataForUser(bungieMembershipId, platformMembershipIds); - const settingsItem = convertToStatelyItem( - { ...defaultSettings, ...settings }, - bungieMembershipId, - ); - // The export will have duplicates because import saved to each profile // instead of the one that was exported. itemHashTags = uniqBy(itemHashTags, (a) => a.hash); @@ -152,10 +148,9 @@ export async function statelyImport( items.push(...importSearches(platformMembershipId, searches)); } - // Put the settings in first since it's in a different group - await client.put({ - item: settingsItem, - }); + // Settings live in Postgres now + await transaction(async (client) => replaceSettings(client, bungieMembershipId, settings)); + // OK now put them in as fast as we can for (const batch of batches(items)) { // We shouldn't have any existing items... diff --git a/api/routes/profile.ts b/api/routes/profile.ts index a09d213..58a7003 100644 --- a/api/routes/profile.ts +++ b/api/routes/profile.ts @@ -147,7 +147,10 @@ function extractSyncToken(syncTokenParam: string | undefined) { const tokenMap = JSON.parse(syncTokenParam) as { [component: string]: string | number }; return Object.entries(tokenMap).reduce<{ [component: string]: Buffer | number }>( (acc, [component, token]) => { - acc[component] = typeof token === 'string' ? Buffer.from(token, 'base64') : token; + acc[component] = + typeof token === 'string' && !/^\d+$/.exec(token) + ? Buffer.from(token, 'base64') + : Number(token); return acc; }, {}, @@ -172,7 +175,7 @@ async function statelyProfile( }; const timerPrefix = response.sync ? 'profileSync' : 'profileStately'; const counterPrefix = response.sync ? 'sync' : 'stately'; - const syncTokens: { [component: string]: string } = {}; + const syncTokens: { [component: string]: string | number } = {}; const addSyncToken = ( name: string, token: ListToken | { canSync: boolean; tokenData: number }, @@ -181,14 +184,14 @@ async function statelyProfile( syncTokens[name] = token.tokenData instanceof Uint8Array ? Buffer.from(token.tokenData).toString('base64') - : token.tokenData.toString(); + : token.tokenData; } }; const getSyncToken = (name: string) => { const tokenData = incomingSyncTokens?.[name]; - if (incomingSyncTokens && !tokenData) { - throw new Error(`Missing sync token: ${name}`); - } + // if (incomingSyncTokens && !tokenData) { + // throw new Error(`Missing sync token: ${name}`); + // } return tokenData as T | undefined; }; @@ -202,29 +205,22 @@ async function statelyProfile( // Load settings from Stately. If they're there, you're done. Otherwise load from Postgres. const start = new Date(); - const statelySettings = await querySettings(bungieMembershipId); - if (!statelySettings.settings) { - const now = Date.now(); - const pgSettings = await readTransaction(async (pgClient) => - getSettings(pgClient, bungieMembershipId), - ); - if (pgSettings) { - const tokenData = getSyncToken('s'); - if (tokenData === undefined || pgSettings.lastModifiedAt > tokenData) { - response.settings = { ...defaultSettings, ...pgSettings.settings }; - } - } else { - response.settings = defaultSettings; + const now = Date.now(); + // TODO: Should add the token to the query to avoid fetching if unchanged + const pgSettings = await readTransaction(async (pgClient) => + getSettings(pgClient, bungieMembershipId), + ); + if (pgSettings) { + const tokenData = getSyncToken('s'); + if (tokenData === undefined || pgSettings.lastModifiedAt > tokenData) { + response.settings = { ...defaultSettings, ...pgSettings.settings }; } addSyncToken('s', { canSync: true, tokenData: pgSettings?.lastModifiedAt ?? now }); } else { const tokenData = getSyncToken('settings'); const { settings: storedSettings, token: settingsToken } = tokenData ? await syncSettings(tokenData) - : { - settings: statelySettings.settings ?? defaultSettings, - token: statelySettings.token, - }; + : await querySettings(bungieMembershipId); response.settings = storedSettings; addSyncToken('settings', settingsToken); } @@ -335,6 +331,6 @@ async function statelyProfile( return response; } -function serializeSyncToken(syncTokens: { [component: string]: string }) { +function serializeSyncToken(syncTokens: { [component: string]: string | number }) { return JSON.stringify(syncTokens); } diff --git a/api/routes/update.ts b/api/routes/update.ts index 7ccca44..28a2986 100644 --- a/api/routes/update.ts +++ b/api/routes/update.ts @@ -2,9 +2,13 @@ import { captureMessage } from '@sentry/node'; import { chunk, groupBy, partition, sortBy } from 'es-toolkit'; import express from 'express'; import asyncHandler from 'express-async-handler'; -import { transaction } from '../db/index.js'; +import { readTransaction, transaction } from '../db/index.js'; import { backfillMigrationState } from '../db/migration-state-queries.js'; -import { replaceSettings, setSetting as setSettingInPostgres } from '../db/settings-queries.js'; +import { + getSettings, + replaceSettings, + setSetting as setSettingInPostgres, +} from '../db/settings-queries.js'; import { metrics } from '../metrics/index.js'; import { ApiApp } from '../shapes/app.js'; import { DestinyVersion } from '../shapes/general.js'; @@ -43,11 +47,7 @@ import { UpdateSearch, updateSearches, } from '../stately/searches-queries.js'; -import { - deleteSettings as deleteSettingsInStately, - getSettingsForUpdate, - setSetting as setSettingInStately, -} from '../stately/settings-queries.js'; +import { getSettingsForUpdate, keyFor } from '../stately/settings-queries.js'; import { trackUntrackTriumphs } from '../stately/triumphs-queries.js'; import { badRequest, @@ -313,9 +313,16 @@ async function statelyUpdate( mergedSettings = { ...mergedSettings, ...update.payload }; } - if (bungieMembershipId === 7094) { + // TODO: Remove the check for settings in Postgres once we're fully migrated off Stately + const pgSettings = await readTransaction((client) => + getSettings(client, bungieMembershipId), + ); + if (pgSettings) { + await transaction(async (pgClient) => { + await setSettingInPostgres(pgClient, bungieMembershipId, mergedSettings); + }); + } else { const statelySettings = await getSettingsForUpdate(txn, bungieMembershipId); - if (statelySettings) { mergedSettings = { ...statelySettings, ...mergedSettings }; await transaction(async (pgClient) => { @@ -325,14 +332,12 @@ async function statelyUpdate( subtractObject(mergedSettings, defaultSettings), ); }); - await deleteSettingsInStately(bungieMembershipId); + await txn.del(keyFor(bungieMembershipId)); } else { await transaction(async (pgClient) => { await setSettingInPostgres(pgClient, bungieMembershipId, mergedSettings); }); } - } else { - await setSettingInStately(txn, bungieMembershipId, mergedSettings); } break; } diff --git a/api/server.test.ts b/api/server.test.ts index 80e30d7..ec6f285 100644 --- a/api/server.test.ts +++ b/api/server.test.ts @@ -210,11 +210,40 @@ describe('profile', () => { expect(profileSyncResponse.syncToken).toBeDefined(); expect(profileSyncResponse.syncToken).not.toBe(profileResponse.syncToken); + expect(profileSyncResponse.syncToken).toContain('"s":'); expect(profileSyncResponse.sync).toBe(true); expect(profileSyncResponse.settings?.showNewItems).toBe(true); expect(profileSyncResponse.tags?.length).toBe(1); expect(profileSyncResponse.tags?.[0].id).toBe('1234'); expect(profileSyncResponse.deletedLoadoutIds?.length).toBe(1); + + const request2: ProfileUpdateRequest = { + platformMembershipId, + destinyVersion: 2, + updates: [ + { + action: 'setting', + payload: { + compareBaseStats: true, + }, + }, + ], + }; + await postRequestAuthed('/profile', request2).expect(200); + + const profileSyncResponse2 = (await getRequestAuthed( + `/profile?components=settings,loadouts,tags,triumphs,searches,hashtags&platformMembershipId=${platformMembershipId}&sync=${encodeURIComponent(profileSyncResponse.syncToken!)}`, + ) + .expect(200) + .json()) as ProfileResponse; + + expect(profileSyncResponse2.syncToken).toBeDefined(); + expect(profileSyncResponse2.syncToken).not.toBe(profileSyncResponse.syncToken); + expect(profileSyncResponse2.syncToken).toContain('"s":'); + expect(profileSyncResponse2.sync).toBe(true); + expect(profileSyncResponse2.settings).toBeDefined(); + expect(profileSyncResponse.settings?.compareBaseStats).toBe(true); + expect(profileSyncResponse2.settings?.showNewItems).toBe(true); }); it('can retrieve only settings, without needing a platform membership ID', async () => { diff --git a/api/stately/bulk-queries.ts b/api/stately/bulk-queries.ts index fb342e1..e2feaf2 100644 --- a/api/stately/bulk-queries.ts +++ b/api/stately/bulk-queries.ts @@ -6,7 +6,7 @@ import { DeleteAllResponse } from '../shapes/delete-all.js'; import { ExportResponse } from '../shapes/export.js'; import { DestinyVersion } from '../shapes/general.js'; import { ProfileResponse } from '../shapes/profile.js'; -import { defaultSettings } from '../shapes/settings.js'; +import { defaultSettings, Settings } from '../shapes/settings.js'; import { delay, subtractObject } from '../utils.js'; import { client } from './client.js'; import { AnyItem } from './generated/index.js'; @@ -115,12 +115,14 @@ export async function exportDataForUser( bungieMembershipId: number, platformMembershipIds: string[], ): Promise { - let settings = await getSettings(bungieMembershipId); - if (!settings) { - const pgSettings = await readTransaction((client) => - getSettingsFromPostgres(client, bungieMembershipId), - ); - settings = { ...defaultSettings, ...pgSettings?.settings }; + let settings: Settings; + const pgSettings = await readTransaction((client) => + getSettingsFromPostgres(client, bungieMembershipId), + ); + if (pgSettings) { + settings = { ...defaultSettings, ...pgSettings.settings }; + } else { + settings = (await getSettings(bungieMembershipId)) ?? defaultSettings; } const responses = await Promise.all(platformMembershipIds.map((p) => exportDataForProfile(p))); diff --git a/api/utils.ts b/api/utils.ts index cb76cd2..0baed18 100644 --- a/api/utils.ts +++ b/api/utils.ts @@ -1,4 +1,4 @@ -import { camelCase, mapKeys } from 'es-toolkit'; +import { camelCase, isEqual, mapKeys } from 'es-toolkit'; import { Response } from 'express'; /** @@ -141,7 +141,7 @@ export function subtractObject(obj: Partial, defaults: T): const result: Partial = {}; if (obj) { for (const key in defaults) { - if (obj[key] !== undefined && obj[key] !== defaults[key]) { + if (obj[key] !== undefined && !isEqual(obj[key], defaults[key])) { result[key] = obj[key]; } }