Skip to content

Commit cc0fc21

Browse files
synap5esynap5e
authored
Keep temp-directory assets visible while their files exist (#15510)
* Keep temp-directory assets visible while their files exist Assets written to the temp directory were flagged as missing and dropped from GET /api/assets, even with the file sitting on disk. One list of directories was answering two different questions -- where the scanner looks for new files, and which files ComfyUI considers its own -- and temp belongs only in the second, so every temp reference was disowned by the prune that runs at startup and on POST /api/assets/prune. Ownership now covers temp. Discovery still does not: the temp directory is wiped before the scan runs, and assets written there are already registered with a hash, mime type and dimensions, so walking it would find nothing. Temp references are instead reconciled against the filesystem directly, so a temp file that really is gone is still retired rather than lingering as a broken entry. get_prefixes_for_root becomes get_scan_prefixes_for_root so the two questions are told apart by name rather than by comment. * Cover the unhashed temp asset in the reconciliation tests The existing temp tests all registered hashed assets, so they never exercised the path an unhashed asset takes when its file is gone: the orphaned rows are removed rather than kept as missing, exactly as under any other root.
1 parent e5a38e3 commit cc0fc21

6 files changed

Lines changed: 268 additions & 33 deletions

File tree

‎app/assets/scanner.py‎

Lines changed: 38 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,11 @@ class _AssetAccumulator(TypedDict):
5757
refs: list[_RefInfo]
5858

5959

60+
# Temp is deliberately absent: it is wiped before every scan, so walking it finds nothing.
6061
RootType = Literal["models", "input", "output"]
6162

6263

63-
def get_prefixes_for_root(root: RootType) -> list[str]:
64+
def get_scan_prefixes_for_root(root: RootType) -> list[str]:
6465
if root == "models":
6566
bases: list[str] = []
6667
for _bucket, paths, _exts in get_comfy_models_folders():
@@ -73,10 +74,15 @@ def get_prefixes_for_root(root: RootType) -> list[str]:
7374
return []
7475

7576

76-
def get_all_known_prefixes() -> list[str]:
77-
"""Get all known asset prefixes across all root types."""
78-
all_roots: tuple[RootType, ...] = ("models", "input", "output")
79-
return [p for root in all_roots for p in get_prefixes_for_root(root)]
77+
def get_owned_prefixes() -> list[str]:
78+
"""Every directory an asset may live in; references outside these are marked missing."""
79+
scan_roots: tuple[RootType, ...] = ("models", "input", "output")
80+
prefixes = [p for root in scan_roots for p in get_scan_prefixes_for_root(root)]
81+
return prefixes + get_temp_prefixes()
82+
83+
84+
def get_temp_prefixes() -> list[str]:
85+
return [os.path.abspath(folder_paths.get_temp_directory())]
8086

8187

8288
def collect_models_files() -> list[str]:
@@ -107,7 +113,21 @@ def sync_references_with_filesystem(
107113
collect_existing_paths: bool = False,
108114
update_missing_tags: bool = False,
109115
) -> set[str] | None:
110-
"""Reconcile asset references with filesystem for a root.
116+
return sync_prefixes_with_filesystem(
117+
session,
118+
get_scan_prefixes_for_root(root),
119+
collect_existing_paths=collect_existing_paths,
120+
update_missing_tags=update_missing_tags,
121+
)
122+
123+
124+
def sync_prefixes_with_filesystem(
125+
session,
126+
prefixes: list[str],
127+
collect_existing_paths: bool = False,
128+
update_missing_tags: bool = False,
129+
) -> set[str] | None:
130+
"""Reconcile asset references with filesystem under the given prefixes.
111131
112132
- Toggle needs_verify per reference using mtime/size stat check
113133
- For hashed assets with at least one stat-unchanged ref: delete stale missing refs
@@ -117,14 +137,13 @@ def sync_references_with_filesystem(
117137
118138
Args:
119139
session: Database session
120-
root: Root type to scan
140+
prefixes: Absolute directory prefixes whose references to reconcile
121141
collect_existing_paths: If True, return set of surviving file paths
122142
update_missing_tags: If True, update 'missing' tags based on file status
123143
124144
Returns:
125145
Set of surviving absolute paths if collect_existing_paths=True, else None
126146
"""
127-
prefixes = get_prefixes_for_root(root)
128147
if not prefixes:
129148
return set() if collect_existing_paths else None
130149

@@ -251,6 +270,16 @@ def sync_root_safely(root: RootType) -> set[str]:
251270
return set()
252271

253272

273+
def sync_temp_references_safely() -> None:
274+
"""Retire temp references whose file is gone; temp is never scanned, so nothing else stats them."""
275+
try:
276+
with create_session() as sess:
277+
sync_prefixes_with_filesystem(sess, get_temp_prefixes())
278+
sess.commit()
279+
except Exception as e:
280+
logging.exception("temp reference sync failed: %s", e)
281+
282+
254283
def mark_missing_outside_prefixes_safely(prefixes: list[str]) -> int:
255284
"""Mark references as missing when outside the given prefixes.
256285
@@ -384,7 +413,7 @@ def get_unenriched_assets_for_roots(
384413
"""
385414
prefixes: list[str] = []
386415
for root in roots:
387-
prefixes.extend(get_prefixes_for_root(root))
416+
prefixes.extend(get_scan_prefixes_for_root(root))
388417

389418
if not prefixes:
390419
return []

‎app/assets/seeder.py‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,13 @@
1515
build_asset_specs,
1616
collect_paths_for_roots,
1717
enrich_assets_batch,
18-
get_all_known_prefixes,
19-
get_prefixes_for_root,
18+
get_owned_prefixes,
19+
get_scan_prefixes_for_root,
2020
get_unenriched_assets_for_roots,
2121
insert_asset_specs,
2222
mark_missing_outside_prefixes_safely,
2323
sync_root_safely,
24+
sync_temp_references_safely,
2425
)
2526
from app.database.db import dependencies_available
2627

@@ -413,7 +414,7 @@ def mark_missing_outside_prefixes(self) -> int:
413414
)
414415
return 0
415416

416-
all_prefixes = get_all_known_prefixes()
417+
all_prefixes = get_owned_prefixes()
417418
marked = mark_missing_outside_prefixes_safely(all_prefixes)
418419
if marked > 0:
419420
logging.info("Marked %d references as missing", marked)
@@ -523,7 +524,7 @@ def _log_scan_config(self, roots: tuple[RootType, ...]) -> None:
523524
os.path.abspath(folder_paths.models_dir),
524525
)
525526
else:
526-
prefixes = get_prefixes_for_root(root)
527+
prefixes = get_scan_prefixes_for_root(root)
527528
if prefixes:
528529
logging.info("Asset scan [%s] directories: %s", root, prefixes)
529530

@@ -548,10 +549,11 @@ def _run_scan(self) -> None:
548549
return
549550

550551
if self._prune_first:
551-
all_prefixes = get_all_known_prefixes()
552+
all_prefixes = get_owned_prefixes()
552553
marked = mark_missing_outside_prefixes_safely(all_prefixes)
553554
if marked > 0:
554555
logging.info("Marked %d refs as missing before scan", marked)
556+
sync_temp_references_safely()
555557

556558
if self._check_pause_and_cancel():
557559
logging.info("Asset scan cancelled after pruning phase")

‎tests-unit/assets_test/test_sync_references.py‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ def test_needs_verify_toggling(session, temp_dir, case):
150150
)
151151
session.commit()
152152

153-
with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):
153+
with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):
154154
sync_references_with_filesystem(session, "models")
155155
session.commit()
156156

@@ -185,7 +185,7 @@ def test_is_missing_flag(session, temp_dir, case):
185185
_make_asset(session, "a1", fp, "r1", asset_hash="blake3:abc", mtime_ns=mtime)
186186
session.commit()
187187

188-
with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):
188+
with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):
189189
sync_references_with_filesystem(session, "models")
190190
session.commit()
191191

@@ -200,7 +200,7 @@ def test_seed_asset_all_missing_deletes_asset(session, temp_dir):
200200
_make_asset(session, "seed1", fp, "r1", asset_hash=None, mtime_ns=999)
201201
session.commit()
202202

203-
with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):
203+
with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):
204204
sync_references_with_filesystem(session, "models")
205205
session.commit()
206206

@@ -215,7 +215,7 @@ def test_seed_asset_some_exist_returns_survivors(session, temp_dir):
215215
_make_asset(session, "seed1", fp, "r1", asset_hash=None, mtime_ns=mtime)
216216
session.commit()
217217

218-
with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):
218+
with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):
219219
survivors = sync_references_with_filesystem(
220220
session, "models", collect_existing_paths=True,
221221
)
@@ -240,7 +240,7 @@ def test_hashed_asset_prunes_missing_refs_when_one_is_ok(session, temp_dir):
240240
session.add(ref_gone)
241241
session.commit()
242242

243-
with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):
243+
with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):
244244
sync_references_with_filesystem(session, "models")
245245
session.commit()
246246

@@ -255,7 +255,7 @@ def test_hashed_asset_all_missing_keeps_refs(session, temp_dir):
255255
_make_asset(session, "h1", fp, "r1", asset_hash="blake3:aaa", mtime_ns=999)
256256
session.commit()
257257

258-
with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):
258+
with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):
259259
sync_references_with_filesystem(session, "models")
260260
session.commit()
261261

@@ -272,7 +272,7 @@ def test_missing_tag_added_when_all_refs_gone(session, temp_dir):
272272
_make_asset(session, "h1", fp, "r1", asset_hash="blake3:aaa", mtime_ns=999)
273273
session.commit()
274274

275-
with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):
275+
with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):
276276
sync_references_with_filesystem(
277277
session, "models", update_missing_tags=True,
278278
)
@@ -295,7 +295,7 @@ def test_missing_tag_removed_when_ref_ok(session, temp_dir):
295295
))
296296
session.commit()
297297

298-
with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):
298+
with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):
299299
sync_references_with_filesystem(
300300
session, "models", update_missing_tags=True,
301301
)
@@ -313,7 +313,7 @@ def test_missing_tags_not_touched_when_flag_false(session, temp_dir):
313313
_make_asset(session, "h1", fp, "r1", asset_hash="blake3:aaa", mtime_ns=999)
314314
session.commit()
315315

316-
with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):
316+
with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):
317317
sync_references_with_filesystem(
318318
session, "models", update_missing_tags=False,
319319
)
@@ -329,7 +329,7 @@ def test_returns_none_when_collect_false(session, temp_dir):
329329
_make_asset(session, "a1", fp, "r1", asset_hash="blake3:abc", mtime_ns=mtime)
330330
session.commit()
331331

332-
with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):
332+
with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):
333333
result = sync_references_with_filesystem(
334334
session, "models", collect_existing_paths=False,
335335
)
@@ -338,7 +338,7 @@ def test_returns_none_when_collect_false(session, temp_dir):
338338

339339

340340
def test_returns_empty_set_for_no_prefixes(session):
341-
with patch("app.assets.scanner.get_prefixes_for_root", return_value=[]):
341+
with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[]):
342342
result = sync_references_with_filesystem(
343343
session, "models", collect_existing_paths=True,
344344
)
@@ -348,7 +348,7 @@ def test_returns_empty_set_for_no_prefixes(session):
348348

349349
def test_no_references_is_noop(session, temp_dir):
350350
"""No crash and no side effects when there are no references."""
351-
with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):
351+
with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):
352352
survivors = sync_references_with_filesystem(
353353
session, "models", collect_existing_paths=True,
354354
)
@@ -388,7 +388,7 @@ def test_sync_does_not_resurrect_soft_deleted_ref(session, temp_dir):
388388
_soft_delete_ref(session, "r1")
389389
session.commit()
390390

391-
with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):
391+
with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):
392392
sync_references_with_filesystem(session, "models")
393393
session.commit()
394394

@@ -472,7 +472,7 @@ def test_sync_ignores_soft_deleted_seed_asset(session, temp_dir):
472472
_soft_delete_ref(session, "r1")
473473
session.commit()
474474

475-
with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):
475+
with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):
476476
sync_references_with_filesystem(session, "models")
477477
session.commit()
478478

0 commit comments

Comments
 (0)