diff --git a/jest/setup.ts b/jest/setup.ts index 76b88f58d8e6..519a34e53e75 100644 --- a/jest/setup.ts +++ b/jest/setup.ts @@ -121,7 +121,14 @@ jest.mock('react-native-fs', () => ({ res([]); }), ), - CachesDirectoryPath: jest.fn(), + exists: jest.fn(() => Promise.resolve(false)), + mkdir: jest.fn(() => Promise.resolve()), + moveFile: jest.fn(() => Promise.resolve()), + copyFile: jest.fn(() => Promise.resolve()), + writeFile: jest.fn(() => Promise.resolve()), + DocumentDirectoryPath: '/mock/documents', + CachesDirectoryPath: '/mock/caches', + LibraryDirectoryPath: '/mock/library', })); jest.mock('react-native-share', () => ({ diff --git a/patches/react-native-nitro-sqlite/details.md b/patches/react-native-nitro-sqlite/details.md index fae807b6534d..47198d20deda 100644 --- a/patches/react-native-nitro-sqlite/details.md +++ b/patches/react-native-nitro-sqlite/details.md @@ -1,4 +1,4 @@ -# `react-native-nitro-sqlite` patches +# react-native-nitro-sqlite patches ### [react-native-nitro-sqlite+9.6.0+001+dont-mask-original-error-on-rollback-failure.patch](react-native-nitro-sqlite+9.6.0+001+dont-mask-original-error-on-rollback-failure.patch) @@ -15,3 +15,21 @@ - Upstream PR/issue: Already fixed upstream in 9.7.0 via https://github.com/margelo/react-native-nitro-sqlite/pull/292. We stay on 9.6.0 because the 9.7.0 podspec force-enables `SQLITE_THREADSAFE=0` on iOS and its new per-database queue breaks second opens of the same database (used by `src/libs/ExportOnyxState/index.native.ts`). The patch can be dropped when those are resolved and we bump. - E/App issue: https://github.com/Expensify/App/issues/97908 - PR introducing patch: https://github.com/Expensify/App/pull/97954 + +### [react-native-nitro-sqlite+9.6.0+002+store-database-outside-documents.patch](react-native-nitro-sqlite+9.6.0+002+store-database-outside-documents.patch) + +- Reason: + + ``` + The library stores SQLite databases in the iOS Documents directory, which is exposed to users + via the Files app when file sharing is enabled. This patch stores databases in + Library/Application Support instead (persistent, backed up, never user-visible) and migrates + databases created by older app versions out of Documents on first launch. The database and its + -wal/-shm journal files are copied as a set before the originals are deleted, and if the copy + fails the database keeps being opened from Documents and the migration retries on the next + launch, so committed writes are never separated from their WAL. + ``` + +- Upstream PR/issue: https://github.com/margelo/react-native-nitro-sqlite/issues/289 +- E/App issue: https://github.com/Expensify/App/issues/96649 +- PR introducing patch: https://github.com/Expensify/App/pull/96531 diff --git a/patches/react-native-nitro-sqlite/react-native-nitro-sqlite+9.6.0+002+store-database-outside-documents.patch b/patches/react-native-nitro-sqlite/react-native-nitro-sqlite+9.6.0+002+store-database-outside-documents.patch new file mode 100644 index 000000000000..a98111df979c --- /dev/null +++ b/patches/react-native-nitro-sqlite/react-native-nitro-sqlite+9.6.0+002+store-database-outside-documents.patch @@ -0,0 +1,95 @@ +diff --git a/node_modules/react-native-nitro-sqlite/ios/OnLoad.mm b/node_modules/react-native-nitro-sqlite/ios/OnLoad.mm +index 6ce7258..65c29cb 100644 +--- a/node_modules/react-native-nitro-sqlite/ios/OnLoad.mm ++++ b/node_modules/react-native-nitro-sqlite/ios/OnLoad.mm +@@ -10,6 +10,63 @@ + using namespace margelo::nitro; + using namespace margelo::nitro::rnnitrosqlite; + ++// The Documents directory can be exposed to the user (Files app) when file sharing ++// is enabled, so databases are stored in Library/Application Support instead. ++// Databases created by older app versions are moved out of Documents on first launch. ++// ++// The "OnyxDB" name must match the database the app opens: DEFAULT_DB_NAME in ++// src/CONST/index.ts and DB_NAME in react-native-onyx's SQLiteProvider. A Jest test ++// (MoveFilesOutOfDocumentsTest) fails if the names drift apart. ++static void migrateDatabaseFiles(NSString *fromDirectory, NSString *toDirectory) { ++ NSFileManager *fileManager = [NSFileManager defaultManager]; ++ NSArray *databaseFiles = @[ @"OnyxDB", @"OnyxDB-wal", @"OnyxDB-shm" ]; ++ ++ if (![fileManager fileExistsAtPath:[fromDirectory stringByAppendingPathComponent:@"OnyxDB"]]) { ++ // Nothing to migrate. A previous run may have been interrupted after copying the set but ++ // before removing the journal files, so sweep any leftovers out of Documents. ++ [fileManager removeItemAtPath:[fromDirectory stringByAppendingPathComponent:@"OnyxDB-wal"] error:nil]; ++ [fileManager removeItemAtPath:[fromDirectory stringByAppendingPathComponent:@"OnyxDB-shm"] error:nil]; ++ return; ++ } ++ ++ // A database in Documents means an older build was writing there, so it is the live copy. ++ // Remove whatever sits at the destination (e.g. after a downgrade and re-upgrade) so a -wal ++ // from one database generation is never replayed into a database from another. ++ for (NSString *file in databaseFiles) { ++ [fileManager removeItemAtPath:[toDirectory stringByAppendingPathComponent:file] error:nil]; ++ } ++ ++ // Copy the whole set before deleting anything: committed-but-uncheckpointed writes live in ++ // the -wal file and SQLite only replays it when it sits next to its database, so the files ++ // must never be separated. If any copy fails, the intact originals in Documents keep being ++ // used (see the fallback in +load) and the migration retries on the next launch. ++ for (NSString *file in databaseFiles) { ++ NSString *sourcePath = [fromDirectory stringByAppendingPathComponent:file]; ++ if (![fileManager fileExistsAtPath:sourcePath]) { ++ continue; ++ } ++ ++ NSError *error = nil; ++ if (![fileManager copyItemAtPath:sourcePath toPath:[toDirectory stringByAppendingPathComponent:file] error:&error]) { ++ NSLog(@"Failed to migrate database file %@: %@", file, error.localizedDescription); ++ return; ++ } ++ } ++ ++ // The database file is deleted first, and the journals only once that succeeds: if the ++ // database cannot be removed, the existence check in +load keeps using the source database ++ // on this launch, so its -wal must stay next to it or committed writes would be lost. ++ // An interruption after the first delete can only leave journal files behind, which the ++ // sweep above removes on the next launch. ++ NSError *removeError = nil; ++ if (![fileManager removeItemAtPath:[fromDirectory stringByAppendingPathComponent:@"OnyxDB"] error:&removeError]) { ++ NSLog(@"Failed to remove migrated database from its old location: %@", removeError.localizedDescription); ++ return; ++ } ++ [fileManager removeItemAtPath:[fromDirectory stringByAppendingPathComponent:@"OnyxDB-wal"] error:nil]; ++ [fileManager removeItemAtPath:[fromDirectory stringByAppendingPathComponent:@"OnyxDB-shm"] error:nil]; ++} ++ + + (void)load { + // Get appGroupID value from Info.plist using key "AppGroup" + NSString *appGroupID = [[NSBundle mainBundle] objectForInfoDictionaryKey:@"RNNitroSQLite_AppGroup"]; +@@ -30,9 +80,24 @@ + + documentPath = [storeUrl path]; + } else { +- // Get iOS app's document directory (to safely store database .sqlite3 file) +- NSArray *paths = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, true); ++ // Store databases in Library/Application Support, which is persistent, backed up, ++ // and never exposed to the user via the Files app (unlike the Documents directory) ++ NSArray *paths = NSSearchPathForDirectoriesInDomains(NSApplicationSupportDirectory, NSUserDomainMask, true); + documentPath = [paths objectAtIndex:0]; ++ ++ NSFileManager *fileManager = [NSFileManager defaultManager]; ++ if (![fileManager fileExistsAtPath:documentPath]) { ++ [fileManager createDirectoryAtPath:documentPath withIntermediateDirectories:YES attributes:nil error:nil]; ++ } ++ ++ NSString *documentsDirectory = [NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, true) objectAtIndex:0]; ++ migrateDatabaseFiles(documentsDirectory, documentPath); ++ ++ // If the database could not be moved out of Documents, keep opening it there rather than ++ // creating a fresh empty one; the migration retries on the next launch. ++ if ([fileManager fileExistsAtPath:[documentsDirectory stringByAppendingPathComponent:@"OnyxDB"]]) { ++ documentPath = documentsDirectory; ++ } + } + + HybridNitroSQLite::docPath = [documentPath UTF8String]; diff --git a/src/ONYXKEYS.ts b/src/ONYXKEYS.ts index 08c248fd7e6a..e0cb64f3b105 100755 --- a/src/ONYXKEYS.ts +++ b/src/ONYXKEYS.ts @@ -584,6 +584,9 @@ const ONYXKEYS = { // Max width supported for HTML element MAX_CANVAS_WIDTH: 'maxCanvasWidth', + // Whether the MoveFilesOutOfDocuments migration has rewritten attachment record paths, so later launches skip the collection scan + ATTACHMENT_RECORD_PATHS_MIGRATED: 'attachmentRecordPathsMigrated', + // Stores last visited path LAST_VISITED_PATH: 'lastVisitedPath', @@ -1781,6 +1784,7 @@ type OnyxValuesMapping = { [ONYXKEYS.PRESERVED_ACCOUNT]: OnyxTypes.Account; [ONYXKEYS.NVP_DISMISSED_PRODUCT_TRAINING]: OnyxTypes.DismissedProductTraining; [ONYXKEYS.NVP_SUBMIT_MIGRATION_MODAL_SHOWN]: boolean; + [ONYXKEYS.ATTACHMENT_RECORD_PATHS_MIGRATED]: boolean; [ONYXKEYS.CORPAY_ONBOARDING_FIELDS]: OnyxTypes.CorpayOnboardingFields; [ONYXKEYS.LAST_FULL_RECONNECT_TIME]: string; [ONYXKEYS.TRAVEL_PROVISIONING]: OnyxTypes.TravelProvisioning; diff --git a/src/libs/ExportOnyxState/common.ts b/src/libs/ExportOnyxState/common.ts index 288351f180b0..1dbe9e27dbe9 100644 --- a/src/libs/ExportOnyxState/common.ts +++ b/src/libs/ExportOnyxState/common.ts @@ -166,6 +166,7 @@ const safeOnyxKeys = new Set([ ONYXKEYS.AGENT_TEMPLATES, ONYXKEYS.ARE_AGENTS_LOADED, ONYXKEYS.ARE_POLICY_ROOMS_LOADED, + ONYXKEYS.ATTACHMENT_RECORD_PATHS_MIGRATED, ONYXKEYS.BETAS, ONYXKEYS.BETA_CONFIGURATION, ONYXKEYS.CACHED_PDF_PATHS, diff --git a/src/libs/ExportOnyxState/index.native.ts b/src/libs/ExportOnyxState/index.native.ts index 74a6eef80571..5ea6c7c8d15d 100644 --- a/src/libs/ExportOnyxState/index.native.ts +++ b/src/libs/ExportOnyxState/index.native.ts @@ -40,14 +40,19 @@ const shareAsFile: ShareAsFile = (fileContent) => { try { // Define new filename and path for the app info file const infoFileName = CONST.DEFAULT_ONYX_DUMP_FILE_NAME; - const infoFilePath = `${RNFS.DocumentDirectoryPath}/${infoFileName}`; + // The dump only needs to live long enough to be shared, so it goes in Caches, which + // is never exposed to the user (unlike Documents, which the iOS Files app shows when + // file sharing is enabled) and which the OS can reclaim afterwards + const infoFilePath = `${RNFS.CachesDirectoryPath}/${infoFileName}`; const actualInfoFile = `file://${infoFilePath}`; RNFS.writeFile(infoFilePath, fileContent, 'utf8').then(() => { + // Share targets copy the file while the share sheet is open, so once the promise + // settles (including cancel, since failOnCancel is false) the dump can be deleted Share.open({ url: actualInfoFile, failOnCancel: false, - }); + }).finally(() => RNFS.unlink(infoFilePath).catch(() => {})); }); } catch (error) { console.error('Error renaming and sharing file:', error); diff --git a/src/libs/actions/Attachment/index.native.ts b/src/libs/actions/Attachment/index.native.ts index dd6cb92c7559..e3fd4c0ac055 100644 --- a/src/libs/actions/Attachment/index.native.ts +++ b/src/libs/actions/Attachment/index.native.ts @@ -10,7 +10,9 @@ import Onyx from 'react-native-onyx'; import type {CacheAttachmentProps, GetCachedAttachmentProps, RemoveCachedAttachmentProps} from './types'; -const ATTACHMENT_DIR = `${RNFS.DocumentDirectoryPath}/attachments`; +// Cached attachments are re-downloadable, so they live in Caches, which the OS may purge +// and which is never exposed to the user via the iOS Files app (unlike Documents) +const ATTACHMENT_DIR = `${RNFS.CachesDirectoryPath}/attachments`; async function cacheAttachment({attachmentID, uri, mimeType}: CacheAttachmentProps) { const isLocalFile = uri.startsWith('file://'); @@ -22,6 +24,8 @@ async function cacheAttachment({attachmentID, uri, mimeType}: CacheAttachmentPro const destPath = `${ATTACHMENT_DIR}/${fileName}`; try { + // The OS can purge Caches wholesale, so the directory may need recreating + await RNFS.mkdir(ATTACHMENT_DIR); await RNFS.copyFile(uri, destPath); await Onyx.set(`${ONYXKEYS.COLLECTION.ATTACHMENT}${attachmentID}`, { attachmentID, @@ -56,6 +60,8 @@ async function cacheAttachment({attachmentID, uri, mimeType}: CacheAttachmentPro const fileName = `${attachmentID}.${attachmentFileExtension}`; const filePath = `${ATTACHMENT_DIR}/${fileName}`; + // The OS can purge Caches wholesale, so the directory may need recreating + await RNFS.mkdir(ATTACHMENT_DIR); await RNFetchBlob.config({path: filePath}).fetch('GET', uri); await Onyx.set(`${ONYXKEYS.COLLECTION.ATTACHMENT}${attachmentID}`, { @@ -78,7 +84,13 @@ async function getCachedAttachment({attachmentID, attachment, currentSource}: Ge const localSource = attachment?.source; if (localSource) { - return localSource; + // The OS can purge Caches while the Onyx record survives, so verify the file still + // exists. If it was purged, fall back to the current source and re-cache it. + const localFileExists = await RNFS.exists(localSource); + if (localFileExists) { + return localSource; + } + cacheAttachment({attachmentID, uri: currentSource}); } return currentSource; diff --git a/src/libs/fileDownload/index.ios.ts b/src/libs/fileDownload/index.ios.ts index 77985ca00d2a..418f239424bf 100644 --- a/src/libs/fileDownload/index.ios.ts +++ b/src/libs/fileDownload/index.ios.ts @@ -31,23 +31,15 @@ const isUserCancelled = (err: unknown) => { }; /** - * Downloads the file to Documents section in iOS + * Downloads the file to the given directory. Files the user asked to download go to Documents, + * which the iOS Files app shows to the user as the app's folder because file sharing is enabled. + * Files that only need a temporary local copy (e.g. for saving to Photos) go to the cache + * directory, which the Files app never exposes. */ -function downloadFile(fileUrl: string, fileName: string) { - const dirs = RNFetchBlob.fs.dirs; - - // The iOS files will download to documents directory - const path = dirs.DocumentDir; - - // Fetching the attachment +function downloadFile(fileUrl: string, fileName: string, directory: string) { return RNFetchBlob.config({ fileCache: true, - path: `${path}/${fileName}`, - addAndroidDownloads: { - useDownloadManager: true, - notification: true, - path: `${path}/Expensify/${fileName}`, - }, + path: `${directory}/${fileName}`, }).fetch('GET', fileUrl); } @@ -71,12 +63,17 @@ const postDownloadFile = (translate: LocalizedTranslate, url: string, fileName?: .then((fileData) => { const resolvedFileName = fileName ?? 'Expensify'; const finalFileName = appendTimestamp ? appendTimeToFileName(resolvedFileName) : resolvedFileName; - const expensifyDir = `${RNFS.DocumentDirectoryPath}/Expensify`; + // The file only exists to be handed to the share sheet, so it is written to the + // cache directory, which the iOS Files app never shows to the user + const expensifyDir = `${RNFS.CachesDirectoryPath}/Expensify`; const localPath = `${expensifyDir}/${finalFileName}`; return RNFS.mkdir(expensifyDir).then(() => { - return RNFS.writeFile(localPath, fileData, 'utf8') - .then(() => Share.open({url: localPath, failOnCancel: false, saveToFiles: true})) - .then(() => RNFS.unlink(localPath)); + return ( + RNFS.writeFile(localPath, fileData, 'utf8') + // The share sheet lets the user save the file to the Files app. The staged copy is removed afterwards. + .then(() => Share.open({url: localPath, failOnCancel: false, saveToFiles: true})) + .then(() => RNFS.unlink(localPath)) + ); }); }) .catch((error) => { @@ -104,24 +101,24 @@ function downloadImage(fileUrl: string) { */ function downloadVideo(fileUrl: string, fileName: string): Promise { return new Promise((resolve, reject) => { - let documentPathUri: string | null = null; + let tempPathUri: string | null = null; let cameraRollAsset: PhotoIdentifier; - // Because CameraRoll doesn't allow direct downloads of video with remote URIs, we first download as documents, then copy to photo lib and unlink the original file. - downloadFile(fileUrl, fileName) + // Because CameraRoll doesn't allow direct downloads of video with remote URIs, we first download to the cache, then copy to photo lib and unlink the temporary file. + downloadFile(fileUrl, fileName, RNFetchBlob.fs.dirs.CacheDir) .then((attachment) => { - documentPathUri = attachment.data as string | null; - if (!documentPathUri) { + tempPathUri = attachment.data as string | null; + if (!tempPathUri) { throw new Error('Error downloading video'); } - return CameraRoll.saveAsset(documentPathUri); + return CameraRoll.saveAsset(tempPathUri); }) .then((attachment) => { cameraRollAsset = attachment; - if (!documentPathUri) { + if (!tempPathUri) { throw new Error('Error downloading video'); } - return RNFetchBlob.fs.unlink(documentPathUri); + return RNFetchBlob.fs.unlink(tempPathUri); }) .then(() => { resolve(cameraRollAsset); @@ -154,7 +151,7 @@ const fileDownload: FileDownload = (translate, fileUrl, fileName, successMessage break; } - fileDownloadPromise = downloadFile(fileUrl, attachmentName); + fileDownloadPromise = downloadFile(fileUrl, attachmentName, RNFetchBlob.fs.dirs.DocumentDir); break; } diff --git a/src/libs/localFileCreate/index.native.ts b/src/libs/localFileCreate/index.native.ts index b1a8c893e500..a568874b3c2f 100644 --- a/src/libs/localFileCreate/index.native.ts +++ b/src/libs/localFileCreate/index.native.ts @@ -14,7 +14,10 @@ const localFileCreate: LocalFileCreate = (fileName, textContent, appendTimestamp const {fileExtension} = splitExtensionFromFileName(fileName); const fileNameWithExtension = fileExtension ? fileName : `${fileName}.txt`; const newFileName = appendTimestamp ? appendTimeToFileName(fileNameWithExtension) : fileNameWithExtension; - const dir = RNFetchBlob.fs.dirs.DocumentDir; + // These files are temporary hand-offs to a share/copy flow that deletes them afterwards, + // so they belong in the cache directory, which is never exposed to the user (unlike + // Documents, which the iOS Files app shows when file sharing is enabled) + const dir = RNFetchBlob.fs.dirs.CacheDir; const path = `${dir}/${newFileName}`; return RNFetchBlob.fs.writeFile(path, textContent, 'utf8').then(() => RNFetchBlob.fs.stat(path).then(({size}) => ({path, newFileName, size}))); diff --git a/src/libs/migrateOnyx.ts b/src/libs/migrateOnyx.ts index 21c828349228..c19112f0c855 100644 --- a/src/libs/migrateOnyx.ts +++ b/src/libs/migrateOnyx.ts @@ -2,6 +2,7 @@ import CONST from '@src/CONST'; import Log from './Log'; import ConvertGpsPointsTo2DArray from './migrations/ConvertGpsPointsTo2DArray'; +import MoveFilesOutOfDocuments from './migrations/MoveFilesOutOfDocuments'; import {endSpan, getSpan, startSpan} from './telemetry/activeSpans'; export default function () { @@ -16,7 +17,7 @@ export default function () { }); // Add all migrations to an array so they are executed in order - const migrationPromises: Array<() => Promise> = [ConvertGpsPointsTo2DArray]; + const migrationPromises: Array<() => Promise> = [ConvertGpsPointsTo2DArray, MoveFilesOutOfDocuments]; // Reduce all promises down to a single promise. All promises run in a linear fashion, waiting for the // previous promise to finish before moving onto the next one. diff --git a/src/libs/migrations/MoveFilesOutOfDocuments/index.native.ts b/src/libs/migrations/MoveFilesOutOfDocuments/index.native.ts new file mode 100644 index 000000000000..6fd3dc60ff21 --- /dev/null +++ b/src/libs/migrations/MoveFilesOutOfDocuments/index.native.ts @@ -0,0 +1,156 @@ +import Log from '@libs/Log'; + +import CONST from '@src/CONST'; +import ONYXKEYS from '@src/ONYXKEYS'; +import type {Attachment} from '@src/types/onyx'; +import {isEmptyObject} from '@src/types/utils/EmptyObject'; + +import type {OnyxCollection} from 'react-native-onyx'; + +import RNFS from 'react-native-fs'; +import Onyx from 'react-native-onyx'; +import OnyxUtils from 'react-native-onyx/dist/OnyxUtils'; + +const OLD_ATTACHMENT_DIR = `${RNFS.DocumentDirectoryPath}/attachments`; +const NEW_ATTACHMENT_DIR = `${RNFS.CachesDirectoryPath}/attachments`; + +/** + * The attachment cache now lives in Caches. The old files are moved rather than deleted because + * attachments composed offline have no remote source to re-download from yet, so their cached + * copy is the only one that survives a relaunch. + */ +function moveAttachmentCache(): Promise { + return RNFS.exists(OLD_ATTACHMENT_DIR).then((oldDirExists) => { + if (!oldDirExists) { + return; + } + return RNFS.exists(NEW_ATTACHMENT_DIR) + .then((newDirExists) => { + if (!newDirExists) { + return RNFS.moveFile(OLD_ATTACHMENT_DIR, NEW_ATTACHMENT_DIR); + } + // The new directory already holds files cached since this app version started, so + // the old files are moved one by one, keeping the newer copy on a name collision + return RNFS.readDir(OLD_ATTACHMENT_DIR) + .then((oldFiles) => + Promise.all( + oldFiles.map((oldFile) => + RNFS.exists(`${NEW_ATTACHMENT_DIR}/${oldFile.name}`).then((alreadyCached) => { + if (alreadyCached) { + return; + } + return RNFS.moveFile(oldFile.path, `${NEW_ATTACHMENT_DIR}/${oldFile.name}`); + }), + ), + ), + ) + .then(() => RNFS.unlink(OLD_ATTACHMENT_DIR)); + }) + .then(() => { + Log.info('[Migrate Onyx] MoveFilesOutOfDocuments moved the attachment cache out of the document directory'); + }); + }); +} + +/** + * Attachment records written by older app versions point into the old attachment directory, so + * their paths are rewritten to the new location. Records are rewritten even when the files + * themselves are gone (on iOS the user can delete them via the Files app): reads verify the file + * exists and re-cache on a miss, so a rewritten record never renders a dead path. + * + * The scan reads the whole attachment collection, so once it completes a flag is stored and + * later launches skip it entirely. + */ +function updateAttachmentRecordPaths(): Promise { + return OnyxUtils.get(ONYXKEYS.ATTACHMENT_RECORD_PATHS_MIGRATED).then((hasMigrated) => { + if (hasMigrated) { + return; + } + return new Promise((resolve, reject) => { + // connectWithoutView is appropriate here because migrations run once at startup, before anything renders + const connection = Onyx.connectWithoutView({ + key: ONYXKEYS.COLLECTION.ATTACHMENT, + callback: (attachments: OnyxCollection) => { + Onyx.disconnect(connection); + + const updates: Record = {}; + for (const [key, attachment] of Object.entries(attachments ?? {})) { + if (!attachment?.source?.startsWith(`${OLD_ATTACHMENT_DIR}/`)) { + continue; + } + updates[key] = {...attachment, source: `${NEW_ATTACHMENT_DIR}/${attachment.source.slice(OLD_ATTACHMENT_DIR.length + 1)}`}; + } + + // No need to add a new action just for this migration + const rewrite = isEmptyObject(updates) + ? Promise.resolve() + : // eslint-disable-next-line rulesdir/prefer-actions-set-data + Onyx.mergeCollection(ONYXKEYS.COLLECTION.ATTACHMENT, updates).then(() => { + Log.info('[Migrate Onyx] MoveFilesOutOfDocuments updated attachment records to the new cache directory'); + }); + + rewrite + // The flag is only set after a successful rewrite so a failed run retries on the next launch + // eslint-disable-next-line rulesdir/prefer-actions-set-data + .then(() => Onyx.set(ONYXKEYS.ATTACHMENT_RECORD_PATHS_MIGRATED, true)) + .then(() => resolve()) + // The rejection must propagate so the startup fallback in the migration's + // catch handler runs instead of leaving migrateOnyx() pending forever + .catch(reject); + }, + }); + }); + }); +} + +/** + * Onyx state dumps were previously written to the document directory and never deleted after + * sharing, so a stale dump may still sit there. It is an internal debug file, so it is removed. + */ +function removeStaleOnyxDump(): Promise { + const dumpPath = `${RNFS.DocumentDirectoryPath}/${CONST.DEFAULT_ONYX_DUMP_FILE_NAME}`; + return RNFS.exists(dumpPath).then((exists) => { + if (!exists) { + return; + } + return RNFS.unlink(dumpPath).then(() => { + Log.info('[Migrate Onyx] MoveFilesOutOfDocuments removed a stale Onyx state dump'); + }); + }); +} + +/** + * Older app versions staged CSV/report exports in Documents/Expensify before handing them to the + * iOS share sheet, and cancelling the sheet leaked the staged file. Exports are internal files + * (current versions stage them in Caches), so the whole staging directory is removed. + */ +function removeStaleExportStagingDir(): Promise { + const stagingDir = `${RNFS.DocumentDirectoryPath}/Expensify`; + return RNFS.exists(stagingDir).then((exists) => { + if (!exists) { + return; + } + return RNFS.unlink(stagingDir).then(() => { + Log.info('[Migrate Onyx] MoveFilesOutOfDocuments removed the old export staging directory'); + }); + }); +} + +/** + * Internal app files used to live in the app's document directory, which iOS shows to the user + * (and other apps) through the Files app because file sharing is enabled. This moves or removes + * the ones older app versions left behind, so on iOS the directory only holds files the user + * expects to see there: their downloads and queued receipt uploads. Android keeps the same + * internal files under the same relative paths, so it runs the same cleanup. + */ +export default function (): Promise { + return ( + Promise.all([moveAttachmentCache().then(() => updateAttachmentRecordPaths()), removeStaleOnyxDump(), removeStaleExportStagingDir()]) + .then(() => undefined) + // A failed cleanup must never block app startup. New files already go to the new + // locations, and the cleanup runs again on the next launch. + .catch((error) => { + Log.warn('[Migrate Onyx] MoveFilesOutOfDocuments failed', {error: error instanceof Error ? error.message : String(error)}); + }) + ); +} diff --git a/src/libs/migrations/MoveFilesOutOfDocuments/index.ts b/src/libs/migrations/MoveFilesOutOfDocuments/index.ts new file mode 100644 index 000000000000..1e956c07d6da --- /dev/null +++ b/src/libs/migrations/MoveFilesOutOfDocuments/index.ts @@ -0,0 +1,5 @@ +// This migration only applies to the native platforms, where internal files previously +// lived in the app's document directory. On web it is a no-op. +export default function (): Promise { + return Promise.resolve(); +} diff --git a/tests/actions/AttachmentTest.ts b/tests/actions/AttachmentTest.ts index 2a7074499c99..ebb59e6b350d 100644 --- a/tests/actions/AttachmentTest.ts +++ b/tests/actions/AttachmentTest.ts @@ -16,11 +16,15 @@ import waitForBatchedUpdates from '../utils/waitForBatchedUpdates'; jest.mock('react-native-fs', () => ({ DocumentDirectoryPath: '/mock/documents', + CachesDirectoryPath: '/mock/caches', copyFile: jest.fn(() => Promise.resolve()), exists: jest.fn(() => Promise.resolve(true)), unlink: jest.fn(() => Promise.resolve()), + mkdir: jest.fn(() => Promise.resolve()), })); +const mockRNFS: {exists: jest.Mock} = jest.requireMock('react-native-fs'); + jest.mock('react-native-blob-util', () => ({ config: jest.fn((data: {path?: string} | undefined) => { const filePath: string = data?.path ?? '/mock/documents/file'; @@ -104,9 +108,22 @@ describe('AttachmentStorage', () => { expect(attachmentID).toBeDefined(); expect(attachment).toEqual({ attachmentID, - source: `/mock/documents/attachments/${attachmentID}.jpg`, + source: `/mock/caches/attachments/${attachmentID}.jpg`, }); }); + it('should fall back to the current source when the cached file was purged from disk', async () => { + // Given a cached attachment whose file no longer exists (the OS can purge Caches) + const attachmentID = 'purged-attachment'; + const sourceURL = 'https://images.unsplash.com/photo-1726066012751-2adfb5485977?w=500'; + const attachment = {attachmentID, source: `/mock/caches/attachments/${attachmentID}.jpg`, remoteSource: sourceURL}; + mockRNFS.exists.mockResolvedValueOnce(false); + + // When reading it from the cache + const resolvedSource = await getCachedAttachment({attachmentID, attachment, currentSource: sourceURL}); + + // Then the dead local path is not returned + expect(resolvedSource).toBe(sourceURL); + }); it('should cache markdown attachment', async () => { // Given the attachment data consisting of sourceURL and markdown comment text const sourceURL = 'https://images.unsplash.com/photo-1726066012751-2adfb5485977?w=500'; @@ -144,7 +161,7 @@ describe('AttachmentStorage', () => { expect(attachmentID).toBeDefined(); expect(attachment).toEqual({ attachmentID, - source: `/mock/documents/attachments/${attachmentID}.jpg`, + source: `/mock/caches/attachments/${attachmentID}.jpg`, remoteSource: sourceURL, }); }); @@ -188,7 +205,7 @@ describe('AttachmentStorage', () => { expect(attachmentID).toBeDefined(); expect(attachment).toEqual({ attachmentID, - source: `/mock/documents/attachments/${attachmentID}.jpg`, + source: `/mock/caches/attachments/${attachmentID}.jpg`, remoteSource: sourceURL, }); @@ -207,7 +224,7 @@ describe('AttachmentStorage', () => { // Then the attachment should be updated with new attachment link expect(newAttachment).toEqual({ attachmentID, - source: `/mock/documents/attachments/${attachmentID}.jpg`, + source: `/mock/caches/attachments/${attachmentID}.jpg`, remoteSource: newSourceURL, }); }); @@ -260,7 +277,7 @@ describe('AttachmentStorage', () => { expect(attachmentID).toBeDefined(); expect(newAttachment).toEqual({ attachmentID, - source: `/mock/documents/attachments/${attachmentID}.jpg`, + source: `/mock/caches/attachments/${attachmentID}.jpg`, remoteSource: fileData.uri, }); @@ -323,7 +340,7 @@ describe('AttachmentStorage', () => { expect(attachmentID).toBeDefined(); expect(newAttachment).toEqual({ attachmentID, - source: `/mock/documents/attachments/${attachmentID}.jpg`, + source: `/mock/caches/attachments/${attachmentID}.jpg`, remoteSource: sourceURL, }); @@ -409,14 +426,14 @@ describe('AttachmentStorage', () => { expect(remoteSourceIndex).toBeDefined(); expect(attachment).toEqual({ attachmentID, - source: `/mock/documents/attachments/${attachmentID}.jpg`, + source: `/mock/caches/attachments/${attachmentID}.jpg`, remoteSource: markdownAttachments.at(remoteSourceIndex), }); continue; } expect(attachment).toEqual({ attachmentID, - source: `/mock/documents/attachments/${attachmentID}.jpg`, + source: `/mock/caches/attachments/${attachmentID}.jpg`, }); } diff --git a/tests/unit/MoveFilesOutOfDocumentsTest.ts b/tests/unit/MoveFilesOutOfDocumentsTest.ts new file mode 100644 index 000000000000..0f92417f8531 --- /dev/null +++ b/tests/unit/MoveFilesOutOfDocumentsTest.ts @@ -0,0 +1,182 @@ +import MoveFilesOutOfDocuments from '@libs/migrations/MoveFilesOutOfDocuments/index.native'; + +import CONST from '@src/CONST'; +import ONYXKEYS from '@src/ONYXKEYS'; + +import fs from 'fs'; +import path from 'path'; +import Onyx from 'react-native-onyx'; + +import getOnyxValue from '../utils/getOnyxValue'; +import waitForBatchedUpdates from '../utils/waitForBatchedUpdates'; + +// Importing the real Log module pulls in the network stack, whose persisted-request +// bookkeeping reacts to the Onyx writes made in these tests. +jest.mock('@libs/Log', () => ({ + __esModule: true, + default: {info: jest.fn(), warn: jest.fn(), alert: jest.fn()}, +})); + +jest.mock('react-native-fs', () => ({ + DocumentDirectoryPath: '/mock/documents', + CachesDirectoryPath: '/mock/caches', + exists: jest.fn(() => Promise.resolve(false)), + unlink: jest.fn(() => Promise.resolve()), + moveFile: jest.fn(() => Promise.resolve()), + readDir: jest.fn(() => Promise.resolve([])), +})); + +const mockRNFS: { + exists: jest.Mock; + unlink: jest.Mock; + moveFile: jest.Mock; + readDir: jest.Mock; +} = jest.requireMock('react-native-fs'); + +const OLD_ATTACHMENT_DIR = '/mock/documents/attachments'; +const NEW_ATTACHMENT_DIR = '/mock/caches/attachments'; +const STALE_ONYX_DUMP = `/mock/documents/${CONST.DEFAULT_ONYX_DUMP_FILE_NAME}`; +const OLD_EXPORT_STAGING_DIR = '/mock/documents/Expensify'; + +describe('MoveFilesOutOfDocuments migration (native)', () => { + beforeAll(() => { + Onyx.init({keys: ONYXKEYS}); + }); + + beforeEach(async () => { + jest.clearAllMocks(); + mockRNFS.exists.mockImplementation(() => Promise.resolve(false)); + mockRNFS.unlink.mockImplementation(() => Promise.resolve()); + mockRNFS.moveFile.mockImplementation(() => Promise.resolve()); + mockRNFS.readDir.mockImplementation(() => Promise.resolve([])); + await Onyx.clear(); + await waitForBatchedUpdates(); + }); + + it('does nothing when no internal files are left in the document directory', async () => { + await MoveFilesOutOfDocuments(); + + expect(mockRNFS.unlink).not.toHaveBeenCalled(); + expect(mockRNFS.moveFile).not.toHaveBeenCalled(); + }); + + it('moves the old attachment cache directory when the new one does not exist yet', async () => { + mockRNFS.exists.mockImplementation((existsPath: string) => Promise.resolve(existsPath === OLD_ATTACHMENT_DIR)); + + await MoveFilesOutOfDocuments(); + + expect(mockRNFS.moveFile).toHaveBeenCalledWith(OLD_ATTACHMENT_DIR, NEW_ATTACHMENT_DIR); + expect(mockRNFS.unlink).not.toHaveBeenCalledWith(OLD_ATTACHMENT_DIR); + }); + + it('merges old attachment files into an existing new directory, keeping newer copies on collision', async () => { + mockRNFS.exists.mockImplementation((existsPath: string) => + Promise.resolve(existsPath === OLD_ATTACHMENT_DIR || existsPath === NEW_ATTACHMENT_DIR || existsPath === `${NEW_ATTACHMENT_DIR}/2.jpg`), + ); + mockRNFS.readDir.mockImplementation(() => + Promise.resolve([ + {name: '1.jpg', path: `${OLD_ATTACHMENT_DIR}/1.jpg`}, + {name: '2.jpg', path: `${OLD_ATTACHMENT_DIR}/2.jpg`}, + ]), + ); + + await MoveFilesOutOfDocuments(); + + expect(mockRNFS.moveFile).toHaveBeenCalledWith(`${OLD_ATTACHMENT_DIR}/1.jpg`, `${NEW_ATTACHMENT_DIR}/1.jpg`); + expect(mockRNFS.moveFile).not.toHaveBeenCalledWith(`${OLD_ATTACHMENT_DIR}/2.jpg`, `${NEW_ATTACHMENT_DIR}/2.jpg`); + expect(mockRNFS.unlink).toHaveBeenCalledWith(OLD_ATTACHMENT_DIR); + }); + + it('rewrites attachment record paths to the new cache directory and preserves their other fields', async () => { + mockRNFS.exists.mockImplementation((existsPath: string) => Promise.resolve(existsPath === OLD_ATTACHMENT_DIR)); + await Onyx.set(`${ONYXKEYS.COLLECTION.ATTACHMENT}source1`, {attachmentID: 'source1', source: `${OLD_ATTACHMENT_DIR}/file.jpg`, remoteSource: 'https://example.com/file.jpg'}); + await waitForBatchedUpdates(); + + await MoveFilesOutOfDocuments(); + await waitForBatchedUpdates(); + + const attachment = await getOnyxValue(`${ONYXKEYS.COLLECTION.ATTACHMENT}source1`); + expect(attachment).toEqual({attachmentID: 'source1', source: `${NEW_ATTACHMENT_DIR}/file.jpg`, remoteSource: 'https://example.com/file.jpg'}); + }); + + it('rewrites attachment record paths even when the old directory is already gone', async () => { + await Onyx.set(`${ONYXKEYS.COLLECTION.ATTACHMENT}source1`, {attachmentID: 'source1', source: `${OLD_ATTACHMENT_DIR}/file.jpg`}); + await waitForBatchedUpdates(); + + await MoveFilesOutOfDocuments(); + await waitForBatchedUpdates(); + + const attachment = await getOnyxValue(`${ONYXKEYS.COLLECTION.ATTACHMENT}source1`); + expect(attachment).toEqual({attachmentID: 'source1', source: `${NEW_ATTACHMENT_DIR}/file.jpg`}); + }); + + it('leaves attachment records already pointing at the new directory untouched on a repeat run', async () => { + await Onyx.set(`${ONYXKEYS.COLLECTION.ATTACHMENT}source1`, {attachmentID: 'source1', source: `${NEW_ATTACHMENT_DIR}/file.jpg`}); + await waitForBatchedUpdates(); + const mergeCollectionSpy = jest.spyOn(Onyx, 'mergeCollection'); + + await MoveFilesOutOfDocuments(); + await waitForBatchedUpdates(); + + expect(mergeCollectionSpy).not.toHaveBeenCalled(); + const attachment = await getOnyxValue(`${ONYXKEYS.COLLECTION.ATTACHMENT}source1`); + expect(attachment).toEqual({attachmentID: 'source1', source: `${NEW_ATTACHMENT_DIR}/file.jpg`}); + mergeCollectionSpy.mockRestore(); + }); + + it('skips the attachment record scan on later launches once it has completed', async () => { + await MoveFilesOutOfDocuments(); + await waitForBatchedUpdates(); + expect(await getOnyxValue(ONYXKEYS.ATTACHMENT_RECORD_PATHS_MIGRATED)).toBe(true); + + // A record with an old-directory path written after the first run stays untouched, proving the scan no longer runs + await Onyx.set(`${ONYXKEYS.COLLECTION.ATTACHMENT}source1`, {attachmentID: 'source1', source: `${OLD_ATTACHMENT_DIR}/file.jpg`}); + await waitForBatchedUpdates(); + + await MoveFilesOutOfDocuments(); + await waitForBatchedUpdates(); + + const attachment = await getOnyxValue(`${ONYXKEYS.COLLECTION.ATTACHMENT}source1`); + expect(attachment).toEqual({attachmentID: 'source1', source: `${OLD_ATTACHMENT_DIR}/file.jpg`}); + }); + + it('removes a stale Onyx state dump left by older app versions', async () => { + mockRNFS.exists.mockImplementation((existsPath: string) => Promise.resolve(existsPath === STALE_ONYX_DUMP)); + + await MoveFilesOutOfDocuments(); + + expect(mockRNFS.unlink).toHaveBeenCalledWith(STALE_ONYX_DUMP); + expect(mockRNFS.unlink).not.toHaveBeenCalledWith(OLD_ATTACHMENT_DIR); + }); + + it('removes the old export staging directory left by older app versions', async () => { + mockRNFS.exists.mockImplementation((existsPath: string) => Promise.resolve(existsPath === OLD_EXPORT_STAGING_DIR)); + + await MoveFilesOutOfDocuments(); + + expect(mockRNFS.unlink).toHaveBeenCalledWith(OLD_EXPORT_STAGING_DIR); + }); + + it('does not block startup when the cleanup fails', async () => { + mockRNFS.exists.mockImplementation(() => Promise.resolve(true)); + mockRNFS.moveFile.mockImplementation(() => Promise.reject(new Error('move failed'))); + mockRNFS.unlink.mockImplementation(() => Promise.reject(new Error('unlink failed'))); + + await expect(MoveFilesOutOfDocuments()).resolves.toBeUndefined(); + }); + + it('keeps the native database migration patch in sync with the database name', () => { + const patchDir = path.resolve(__dirname, '../../patches/react-native-nitro-sqlite'); + const patchFileName = fs.readdirSync(patchDir).find((fileName) => fileName.includes('store-database-outside-documents')); + expect(patchFileName).toBeDefined(); + const patchContent = fs.readFileSync(path.join(patchDir, String(patchFileName)), 'utf8'); + + // The patch migrates database files by name at app startup, before any JS runs, so the + // name is hardcoded there. If the database the app opens is ever renamed, the patch (and + // this test) must be updated with it, or the migration would silently strand user data. + expect(patchContent).toContain(`@"${CONST.DEFAULT_DB_NAME}"`); + + const sqliteProviderContent = fs.readFileSync(path.resolve(__dirname, '../../node_modules/react-native-onyx/dist/storage/providers/SQLiteProvider.js'), 'utf8'); + expect(sqliteProviderContent).toContain(`'${CONST.DEFAULT_DB_NAME}'`); + }); +});