diff --git a/auxdir/CFIS/im_sims_202606/tile_numbers_1.txt b/auxdir/CFIS/im_sims_202606/tile_numbers_1.txt new file mode 100644 index 000000000..30673d233 --- /dev/null +++ b/auxdir/CFIS/im_sims_202606/tile_numbers_1.txt @@ -0,0 +1,40 @@ +233.293 +237.292 +238.292 +239.291 +239.292 +239.293 +240.291 +240.292 +240.293 +241.293 +242.290 +242.291 +242.292 +242.293 +243.290 +243.291 +243.292 +243.293 +244.293 +245.290 +245.291 +245.292 +246.290 +246.291 +247.289 +247.291 +247.292 +247.293 +248.288 +248.289 +248.290 +248.291 +249.289 +249.290 +249.291 +249.292 +250.289 +250.290 +250.292 +251.287 diff --git a/auxdir/CFIS/im_sims_202606/tile_numbers_2.txt b/auxdir/CFIS/im_sims_202606/tile_numbers_2.txt new file mode 100644 index 000000000..449bd74d6 --- /dev/null +++ b/auxdir/CFIS/im_sims_202606/tile_numbers_2.txt @@ -0,0 +1,43 @@ +251.288 +251.289 +251.291 +252.287 +252.288 +252.289 +252.291 +253.287 +253.288 +253.291 +254.286 +254.287 +254.288 +255.286 +255.288 +255.290 +256.286 +256.287 +257.285 +257.286 +257.287 +257.288 +258.287 +258.289 +259.285 +259.286 +259.287 +259.288 +259.289 +260.284 +260.285 +260.286 +260.287 +260.288 +260.289 +261.284 +261.285 +261.286 +261.287 +261.288 +262.285 +262.286 +262.287 diff --git a/auxdir/CFIS/im_sims_202606/tile_numbers_3.txt b/auxdir/CFIS/im_sims_202606/tile_numbers_3.txt new file mode 100644 index 000000000..d51361b45 --- /dev/null +++ b/auxdir/CFIS/im_sims_202606/tile_numbers_3.txt @@ -0,0 +1,46 @@ +262.288 +263.283 +263.284 +263.285 +263.286 +264.283 +264.284 +264.286 +264.287 +265.284 +265.285 +265.286 +265.287 +266.282 +266.283 +266.285 +266.286 +267.283 +267.284 +267.285 +267.286 +268.282 +268.283 +268.284 +268.285 +269.281 +269.282 +269.283 +269.284 +269.286 +270.281 +270.282 +270.283 +271.281 +271.282 +271.283 +271.284 +271.285 +272.281 +272.282 +272.283 +272.285 +273.280 +273.282 +273.283 +273.284 diff --git a/auxdir/CFIS/im_sims_202606/tile_numbers_4.txt b/auxdir/CFIS/im_sims_202606/tile_numbers_4.txt new file mode 100644 index 000000000..90077aea5 --- /dev/null +++ b/auxdir/CFIS/im_sims_202606/tile_numbers_4.txt @@ -0,0 +1,37 @@ +274.279 +274.280 +274.282 +274.283 +275.279 +275.280 +275.282 +275.283 +276.279 +276.280 +276.281 +276.282 +276.283 +277.279 +277.281 +277.282 +278.279 +278.281 +278.282 +279.281 +279.282 +280.280 +280.281 +280.282 +281.280 +281.281 +282.280 +282.281 +283.280 +284.279 +285.279 +285.280 +286.279 +286.280 +287.279 +288.279 +289.279 diff --git a/auxdir/CFIS/im_sims_202606/tile_numbers_5.txt b/auxdir/CFIS/im_sims_202606/tile_numbers_5.txt new file mode 100644 index 000000000..e69de29bb diff --git a/example/cfis_image_sims/README.md b/example/cfis_image_sims/README.md new file mode 100644 index 000000000..b308cd6a8 --- /dev/null +++ b/example/cfis_image_sims/README.md @@ -0,0 +1,79 @@ +# cfis_image_sims — module configs for ShapePipe on simulated tiles + +This directory is the single `.ini` tree for image-simulation runs. It is +selected by `run_job_sp_canfar_v2.0.bash -t image_sims` (which sets +`config_dir = example/cfis_image_sims`, `retrieve = symlink`, and forces +`tile_det = sx`) and consumed stage-by-stage by `job_sp_canfar_v2.0.bash`. + +The pipeline runs as a bit-coded chain of jobs. `run_job_sp_canfar_v2.0.bash` +loops over the bits set in `-j`, initialises the tile/exposure work directories, +runs each completeness check, and delegates the actual `shapepipe_run` calls to +`job_sp_canfar_v2.0.bash -j `, which is where the `.ini` file for each bit +is selected. A few bits are special-cased for sims *before* that delegation — +those cases are documented in the last column below and, at more length, under +[Sim special-casing](#sim-special-casing). + +## Job-bit dispatch + +Every row is derived from the two bash scripts. "Module(s)" is the ShapePipe +runner(s) the selected `.ini` names; "`.ini` selected" is what +`job_sp_canfar_v2.0.bash` picks for that bit under sim settings +(`retrieve=symlink`, `psf=psfex`, `tile_det=sx`, `star_cat_for_mask=onthefly`). + +| Bit | Stage | Module(s) | `.ini` selected (sim settings) | Sim special-casing | +|----:|-------|-----------|--------------------------------|--------------------| +| 1 | retrieve tile image + weight | `get_images_runner` | `config_tile_Git_symlink.ini` | symlink retrieval (see below); completeness check expects 2 files (image + weight) vs. 4 for data | +| 2 | uncompress tile weight | *(none — faked)* | *(none)* | **Faked weight-uncompress.** run_job skips the `config_tile_Uz.ini` run entirely; sim weights are already uncompressed, so it fabricates a `run_sp_tile_Uz*/uncompress_fits_runner/output` dir and symlinks `input_tiles/CFIS_simu_weight-.fits` into it, satisfying downstream `last:uncompress_fits_runner` lookups | +| 4 | find exposures | `find_exposures_runner` | `config_tile_Fe.ini` | — | +| 8 | retrieve exposure images | `get_images_runner` | `config_exp_Gie_symlink.ini` | symlink retrieval; completeness check expects 3 files vs. 6 for data | +| 16 | split exposures, merge WCS headers | `split_exp_runner` | `config_exp_Sp.ini` | — | +| 32 | mask exposures | `mask_runner` | `config_exp_Ma_onthefly.ini` | — | +| 64 | exposure PSF model | *(none — placeholder)* | *(none)* | **Placeholder.** For data this runs full exposure PSF modelling. For sims run_job writes a placeholder log and does nothing here; the sim PSF (`fake_psf_runner`) actually runs inside bit 512 | +| 128 | merge exposure WCS headers → tile sqlite log | `merge_headers_runner` | `config_tile_Mh_exp.ini` | — | +| 256 | object detection on tiles | `sextractor_runner` | `config_tile_Sx_nomask.ini` | `tile_det` is forced to `sx`, so the SExtractor-no-mask branch is always taken (the `uc` external-catalogue branch is never reached for sims) | +| 512 | fake PSF + postage stamps | `fake_psf_runner`, then `vignetmaker_runner` ×2 | `config_exp_psfex.ini` (fake PSF), then `config_tile_PiViVi_canfar_sx.ini` (vignets) | **Two sub-runs.** run_job first calls the job script with `-j 64` → `config_exp_psfex.ini`, which despite its name runs `fake_psf_runner` (needs the sexcat from bit 256; run dir `run_sp_tile_fpsf`), then `-j 512` → `config_tile_PiViVi_canfar_sx.ini` for the two `vignetmaker_runner` runs. Data instead runs `psfex_interp_runner` + vignets here | +| 1024 | multi-epoch shape measurement | `ngmix_runner` | `config_tile_Ng_batch_psfex_sx.ini` | — | +| 2048 | create final catalogue | `make_cat_runner` | `config_tile_Mc_psfex.ini` | — | + +Notes on cross-cutting conventions used above: + +- **Dashed tile-id convention.** Data tile IDs use dot format (`233.293`); sim + input files use dash format (`233-293`). run_job writes `tile_numbers.txt` in + dash form for `image_sims` (`${ID//./-}`), and the faked bit-2 weight symlink + and sim input patterns all key off the dashed ID. +- **Symlink retrieval.** `-t image_sims` sets `retrieve=symlink`, so bits 1 and 8 + select the `*_symlink.ini` get-images configs (`RETRIEVE = symlink`), which + link sim images out of `$SP_DIR/input_tiles` / `$SP_DIR/input_exp` rather than + downloading from VOSpace. +- **Forced `tile_det=sx`.** The `image_sims` type branch forces `tile_det=sx`, so + bit 256 always runs SExtractor detection and bit 512 selects the `_sx`-suffixed + vignet/ngmix/PiViVi configs. + +## Sim special-casing + +Three bits diverge from the data chain, all handled in +`run_job_sp_canfar_v2.0.bash` before the per-bit delegation: + +- **Bit 2 — faked weight-uncompress.** Sim weights ship uncompressed, so instead + of running `uncompress_fits_runner`, run_job fabricates the expected Uz output + directory and symlinks `CFIS_simu_weight-.fits` in, so downstream + `last:uncompress_fits_runner` references resolve. +- **Bit 64 — placeholder.** run_job writes a placeholder completeness log and + runs nothing; the sim PSF is produced by `fake_psf_runner` inside bit 512. +- **Bit 512 — fake PSF + vignets.** run_job runs `fake_psf_runner` + (`config_exp_psfex.ini`) followed by the two `vignetmaker_runner` runs + (`config_tile_PiViVi_canfar_sx.ini`). `fake_psf_runner` consumes the sexcat + from bit 256, so bit 256 must precede it. + +## Parallelism model and the `PSF_DICT` requirement + +The orchestrator (the sp_validation Snakemake workflow) fans out one job per +tile; within a single job ShapePipe uses its own multiprocessing, controlled by +`-N/--N_SMP` (threaded through to each module's `SMP_BATCH_SIZE`). Every sim +config sets `MODE = SMP` — the sims chain launches no MPI. + +`fake_psf_runner` reads the sims' pickled PSF dictionary from the `$PSF_DICT` +environment variable (`config_exp_psfex.ini` sets `PSF_DICT_PATH = $PSF_DICT`, +expanded from the environment). The orchestrating workflow exports it (from its +`psf_dict` config key); on candide it points at the SKiLLS PSF dictionary. Bit +512 will fail if `$PSF_DICT` is unset. diff --git a/example/cfis_image_sims/config_exp_Gie_symlink.ini b/example/cfis_image_sims/config_exp_Gie_symlink.ini new file mode 100644 index 000000000..a6a3aed92 --- /dev/null +++ b/example/cfis_image_sims/config_exp_Gie_symlink.ini @@ -0,0 +1,90 @@ +# ShapePipe configuration file for: get images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Gie + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get exposures +[GET_IMAGES_RUNNER] + +INPUT_MODULE = last:find_exposures_runner + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + + +# Paths + +# Output path (optional, default is [FILE]:OUTPUT_DIR +# OUTPUT_PATH = input_images + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = $SP_DIR/input_exp/,$SP_DIR/input_exp/,$SP_DIR/input_exp/ + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = simu_image-0000000,simu_weight-0000000,simu_flag-0000000 + +# Input file extensions +INPUT_FILE_EXT = .fits, .fits,.fits + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{7} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = image-,weight-,flag- + +# Method to retrieve images, one in 'vos', 'symlink' +RETRIEVE = symlink diff --git a/example/cfis_image_sims/config_exp_Ma_onthefly.ini b/example/cfis_image_sims/config_exp_Ma_onthefly.ini new file mode 100644 index 000000000..df0307a19 --- /dev/null +++ b/example/cfis_image_sims/config_exp_Ma_onthefly.ini @@ -0,0 +1,76 @@ +# ShapePipe configuration file for masking of exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Ma + +# Add date and time to RUN_NAME, optional, default: False +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = mask_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +### Mask exposures +[MASK_RUNNER] + +# Parent module +INPUT_DIR = last:split_exp_runner + +# Update numbering convention, accounting for HDU number of +# single-exposure single-HDU files +NUMBERING_SCHEME = -0000000-0 + +# Path of mask config file +MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask_simu + +# External mask file flag, use if True, otherwise ignore +USE_EXT_FLAG = True + +# External star catalogue flag, use external cat if True, +# obtain from online catalogue if False +USE_EXT_STAR = False + +# File name suffix for the output flag files (optional) +PREFIX = pipeline diff --git a/example/cfis_image_sims/config_exp_Sp.ini b/example/cfis_image_sims/config_exp_Sp.ini new file mode 100644 index 000000000..ca010b643 --- /dev/null +++ b/example/cfis_image_sims/config_exp_Sp.ini @@ -0,0 +1,74 @@ +# ShapePipe configuration file for single-exposures, +# split images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Sp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = split_exp_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SPLIT_EXP_RUNNER] + +INPUT_DIR = last:get_images_runner + +FILE_PATTERN = image, weight, flag + +# Matches compressed single-exposure files +FILE_EXT = .fits, .fits, .fits + +NUMBERING_SCHEME = -0000000 + +# OUTPUT_SUFFIX, actually file name prefixes. +# Expected keyword "flag" will lead to a behavior where the data are saved as int. +# The code also expects the image data to use the "image" suffix +# (default value in the pipeline). +OUTPUT_SUFFIX = image, weight, flag + +# Number of HDUs/CCDs of mosaic +N_HDU = 40 diff --git a/example/cfis_image_sims/config_exp_psfex.ini b/example/cfis_image_sims/config_exp_psfex.ini new file mode 100644 index 000000000..38b0be0f6 --- /dev/null +++ b/example/cfis_image_sims/config_exp_psfex.ini @@ -0,0 +1,56 @@ +# ShapePipe configuration file for fake PSF postage stamps (image simulations) + + +## Default ShapePipe options +[DEFAULT] + +VERBOSE = True + +RUN_NAME = run_sp_tile_fpsf + +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +MODULE = fake_psf_runner + +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +LOG_NAME = log_sp + +RUN_LOG_NAME = log_run_sp + +INPUT_DIR = $SP_RUN/output + +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +SMP_BATCH_SIZE = 1 + +TIMEOUT = 96:00:00 + + +## Module options +[FAKE_PSF_RUNNER] + +INPUT_MODULE = sextractor_runner + +FILE_PATTERN = sexcat + +FILE_EXT = .fits + +NUMBERING_SCHEME = -000-000 + +# Path to the pickled PSF dictionary. The value is exported by the +# orchestrating workflow (its psf_dict config key), expanded here from the +# environment; on candide it points at the SKiLLS PSF dictionary. +PSF_DICT_PATH = $PSF_DICT diff --git a/example/cfis_image_sims/config_onthefly.mask_simu b/example/cfis_image_sims/config_onthefly.mask_simu new file mode 100644 index 000000000..1a63cc2e5 --- /dev/null +++ b/example/cfis_image_sims/config_onthefly.mask_simu @@ -0,0 +1,86 @@ +# Mask module configuration file for single-exposure images + +## Paths to executables +[PROGRAM_PATH] + +WW_PATH = weightwatcher +WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww + +# Indicate cds client executable if no external star catalogue is available +# (e.g. no internet access on run nodes) +CDSCLIENT_PATH = findgsc2.2 + + +## Border mask +[BORDER_PARAMETERS] + +BORDER_MAKE = True + +BORDER_WIDTH = 50 +BORDER_FLAG_VALUE = 4 + + +## Halo mask +[HALO_PARAMETERS] + +HALO_MAKE = False + +HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg +HALO_MAG_LIM = 13. +HALO_SCALE_FACTOR = 0.05 +HALO_MAG_PIVOT = 13.8 +HALO_FLAG_VALUE = 2 +HALO_REG_FILE = halo.reg + + +## Diffraction spike mask +[SPIKE_PARAMETERS] + +SPIKE_MAKE = False + +SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg +SPIKE_MAG_LIM = 18. +SPIKE_SCALE_FACTOR = 0.3 +SPIKE_MAG_PIVOT = 13.8 +SPIKE_FLAG_VALUE = 128 +SPIKE_REG_FILE = spike.reg + + +## Messier mask +[MESSIER_PARAMETERS] + +MESSIER_MAKE = False + +MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits +MESSIER_SIZE_PLUS = 0. +MESSIER_FLAG_VALUE = 16 + + +## NGC mask +[NGC_PARAMETERS] + +NGC_MAKE = False + +NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits +NGC_SIZE_PLUS = 0. +NGC_FLAG_VALUE = 32 + + + +## Missing data parameters +[MD_PARAMETERS] + +MD_MAKE = False + +MD_THRESH_FLAG = 0.3 +MD_THRESH_REMOVE = 0.75 +MD_REMOVE = False + + +## Other parameters +[OTHER] + +TEMP_DIRECTORY = .temp + +KEEP_REG_FILE = False +KEEP_INDIVIDUAL_MASK = False diff --git a/example/cfis_image_sims/config_tile_Fe.ini b/example/cfis_image_sims/config_tile_Fe.ini new file mode 100644 index 000000000..9c3cb8cd5 --- /dev/null +++ b/example/cfis_image_sims/config_tile_Fe.ini @@ -0,0 +1,68 @@ +# ShapePipe configuration file for: find exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Fe + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = find_exposures_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[FIND_EXPOSURES_RUNNER] + +INPUT_MODULE = last:get_images_runner + +FILE_PATTERN = CFIS_simu_image + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +COLNUM = 2 + +EXP_PREFIX = simu_image- diff --git a/example/cfis_image_sims/config_tile_Git_symlink.ini b/example/cfis_image_sims/config_tile_Git_symlink.ini new file mode 100644 index 000000000..87c4a8816 --- /dev/null +++ b/example/cfis_image_sims/config_tile_Git_symlink.ini @@ -0,0 +1,87 @@ +# ShapePipe configuration file for: get tile images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Git + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[GET_IMAGES_RUNNER] + +FILE_PATTERN = tile_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = + +# Paths + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = $SP_DIR/input_tiles, $SP_DIR/input_tiles + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = CFIS_simu_image-000-000, CFIS_simu_weight-000-000 + +# Input file extensions +INPUT_FILE_EXT = .fits, .fits + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{3}-\d{3} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = CFIS_simu_image-, CFIS_simu_weight- + +# Copy/download method, one in 'vos', 'symlink' +RETRIEVE = symlink + +# Copy command options, optional +RETRIEVE_OPTIONS = -L diff --git a/example/cfis_image_sims/config_tile_Ma_onthefly.ini b/example/cfis_image_sims/config_tile_Ma_onthefly.ini new file mode 100644 index 000000000..0f49eccea --- /dev/null +++ b/example/cfis_image_sims/config_tile_Ma_onthefly.ini @@ -0,0 +1,82 @@ +# ShapePipe configuration file for masking of tiles + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Ma + +# Add date and time to RUN_NAME, optional, default: False +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = mask_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +### Mask tiles +[MASK_RUNNER] + +# Input directory, containing input files, single string or list of names +INPUT_DIR = run_sp_Git:get_images_runner, last:uncompress_fits_runner + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = CFIS_simu_image, CFIS_simu_weight + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits, .fits + +# Path of mask config file +MASK_CONFIG_PATH = $SP_CONFIG/config_tile_onthefly.mask_simu + +# External mask file flag, use if True, otherwise ignore +USE_EXT_FLAG = False + +# External star catalogue flag, use external cat if True, +# obtain from online catalogue if False +USE_EXT_STAR = False + +# File name suffix for the output flag files (optional) +PREFIX = pipeline diff --git a/example/cfis_image_sims/config_tile_Mc_psfex.ini b/example/cfis_image_sims/config_tile_Mc_psfex.ini new file mode 100644 index 000000000..3b186484e --- /dev/null +++ b/example/cfis_image_sims/config_tile_Mc_psfex.ini @@ -0,0 +1,79 @@ +# ShapePipe post-run configuration file: create final catalogs for psfex + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Mc_psfex + +# Add date and time to RUN_NAME, optional, default: True +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = make_cat_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = ./output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MAKE_CAT_RUNNER] + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:fake_psf_runner, last:ngmix_runner + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = sexcat, galaxy_psf, ngmix + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits, .sqlite, .fits + +# Numbering convention, string that exemplifies a numbering pattern. +# Matches input single exposures (with 'p' removed) +# Needs to be given in this section, will be updated in module +# sections below +NUMBERING_SCHEME = -000-000 + +# Star/galaxy spread-model classification. The image-sims pipeline has no +# spread_model_runner stage (make_cat inputs are sextractor + fake_psf + +# ngmix only), so there is nothing to classify — disable it. The newer +# make_cat_runner reads this key unconditionally (NoOptionError if absent). +SM_DO_CLASSIFICATION = False + +SHAPE_MEASUREMENT_TYPE = ngmix diff --git a/example/cfis_image_sims/config_tile_Mh_exp.ini b/example/cfis_image_sims/config_tile_Mh_exp.ini new file mode 100644 index 000000000..94270a7e6 --- /dev/null +++ b/example/cfis_image_sims/config_tile_Mh_exp.ini @@ -0,0 +1,77 @@ +# ShapePipe configuration file for merging per-exposure WCS headers +# at the tile level. Input is the exp_numbers file produced by +# find_exposures_runner; EXP_BASE_DIR tells the runner where to find +# the per-exposure split_exp_runner header .npy files. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Mh_exp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = merge_headers_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MERGE_HEADERS_RUNNER] + +# Input: exp_numbers txt file from find_exposures_runner +INPUT_DIR = last:find_exposures_runner + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# Tile numbering scheme (RA-Dec, e.g. -301-279) +NUMBERING_SCHEME = -000-000 + +# Root directory containing all per-exposure work directories. +# The runner will walk this tree to collect headers-.npy files. +EXP_BASE_DIR = $SP_EXP + +# If True, log a warning and continue when split_exp_runner output is missing +# for some exposures (e.g. those that failed job 16). If False (default), +# raise an error and stop. +WARN_MISSING_EXP = False diff --git a/example/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini b/example/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini new file mode 100644 index 000000000..11f11cea6 --- /dev/null +++ b/example/cfis_image_sims/config_tile_Ng_batch_psfex_sx.ini @@ -0,0 +1,85 @@ +# ShapePipe configuration file for tiles: ngmix + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Ng + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = ngmix_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Model-fitting shapes with ngmix +[NGMIX_RUNNER] + +INPUT_DIR = run_sp_tile_Sx:sextractor_runner,run_sp_tile_fpsf:fake_psf_runner,last:vignetmaker_runner_run_2,run_sp_tile_Mh_exp:merge_headers_runner + +FILE_PATTERN = sexcat, image_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers + +FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Magnitude zero-point +MAG_ZP = 30.0 + +# No background subtraction for image sims (background not simulated) +BKG_SUB = False + +# Pixel scale in arcsec +PIXEL_SCALE = 0.186 + +SAVE_BATCH = 1000 + +# Position-seeded per-object fixnoise RNG (#796/#803): the SKiLLS shear +# branches share their sky-noise realization, so seeding the metacal fixnoise +# from sky position makes the added noise identical across branches too and +# the Pujol estimator (Pujol, Kilbinger, Sureau & Bobin 2018, 621, A2) cancels it. +SEED_FROM_POSITION = True + +ID_OBJ_MIN = -1 +ID_OBJ_MAX = -1 diff --git a/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini b/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini new file mode 100644 index 000000000..025097f81 --- /dev/null +++ b/example/cfis_image_sims/config_tile_PiViVi_canfar_sx.ini @@ -0,0 +1,127 @@ +# ShapePipe configuration file for tile, from detection up to shape measurement. +# Fake PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_ViVi + +# Add date and time to RUN_NAME, optional, default: False +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names + +MODULE = vignetmaker_runner, vignetmaker_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Create vignets for tiles weights +[VIGNETMAKER_RUNNER_RUN_1] + +INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:uncompress_fits_runner + +FILE_PATTERN = sexcat, CFIS_simu_weight + +FILE_EXT = .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = CLASSIC + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = PIX +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is _vignet.fits +PREFIX = weight + + +[VIGNETMAKER_RUNNER_RUN_2] + +# Create multi-epoch vignets for tiles corresponding to +# positions on single-exposures + +INPUT_DIR = run_sp_tile_Sx:sextractor_runner, run_sp_tile_Mh_exp:merge_headers_runner, last:find_exposures_runner + +FILE_PATTERN = sexcat, log_exp_headers, exp_numbers + +FILE_EXT = .fits, .sqlite, .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = SPHE +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is vignet.fits +PREFIX = + +# Additional parameters for path and file pattern corresponding to single-exposure +# run outputs. ME_IMAGE_EXP_DIR/ME_IMAGE_EXP_RUNNERS replace ME_IMAGE_DIR for +# the v2.0 per-exposure pipeline; output dirs are discovered by scanning $SP_EXP. +ME_IMAGE_EXP_DIR = $SP_EXP +ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner +ME_IMAGE_PATTERN = flag, image, weight diff --git a/example/cfis_image_sims/config_tile_Sx_nomask.ini b/example/cfis_image_sims/config_tile_Sx_nomask.ini new file mode 100644 index 000000000..810614c5d --- /dev/null +++ b/example/cfis_image_sims/config_tile_Sx_nomask.ini @@ -0,0 +1,114 @@ +# ShapePipe configuration file for tile detection + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Sx + +# Add date and time to RUN_NAME, optional, default: True +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER] + +INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner, run_sp_tile_Mh_exp:merge_headers_runner + +FILE_PATTERN = CFIS_simu_image, CFIS_simu_weight, log_exp_headers + +FILE_EXT = .fits, .fits, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# SExtractor executable path +EXEC_PATH = source-extractor + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_tile.sex +DOT_PARAM_FILE = $SP_CONFIG/default_noimaflags.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = False + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +ZP_FROM_HEADER = False + +BKG_FROM_HEADER = False + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, +# MINIBACK_RMS, -BACKGROUND, #FILTERED, +# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND + +# File name suffix for the output sextractor files (optional) +SUFFIX = sexcat + +## Post-processing + +# Necessary for tiles, to enable multi-exposure processing +MAKE_POST_PROCESS = True + +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = XWIN_WORLD,YWIN_WORLD + +# Number of pixels in x,y of a CCD. Format: Nx,Ny +CCD_SIZE = 33,2080,1,4612 diff --git a/example/cfis_image_sims/config_tile_onthefly.mask_simu b/example/cfis_image_sims/config_tile_onthefly.mask_simu new file mode 100644 index 000000000..f42d05d39 --- /dev/null +++ b/example/cfis_image_sims/config_tile_onthefly.mask_simu @@ -0,0 +1,90 @@ +# Mask module config file for tiles + +## Paths to executables +[PROGRAM_PATH] + +WW_PATH = weightwatcher +WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww + +# Indicate cds client executable if no external star catalogue is available +# (e.g. no internet access on run nodes) +CDSCLIENT_PATH = findgsc2.2 + +## Border parameters +[BORDER_PARAMETERS] + +BORDER_MAKE = False + +BORDER_WIDTH = 1 +BORDER_FLAG_VALUE = 4 + + +## Halo parameters +[HALO_PARAMETERS] + +HALO_MAKE = False + +HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg +HALO_MAG_LIM = 13. +HALO_SCALE_FACTOR = 0.05 +HALO_MAG_PIVOT = 13.8 +HALO_FLAG_VALUE = 2 +HALO_REG_FILE = halo.reg + + +## Diffraction pike parameters +[SPIKE_PARAMETERS] + +SPIKE_MAKE = False + +SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg +SPIKE_MAG_LIM = 18. +SPIKE_SCALE_FACTOR = 0.3 +SPIKE_MAG_PIVOT = 13.8 +SPIKE_FLAG_VALUE = 128 +SPIKE_REG_FILE = spike.reg + + +## Messier parameters +[MESSIER_PARAMETERS] + +MESSIER_MAKE = False + +MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits +MESSIER_PIXEL_SCALE = 0.187 +MESSIER_SIZE_PLUS = 0. +MESSIER_FLAG_VALUE = 16 + +## NGC mask +[NGC_PARAMETERS] + +NGC_MAKE = False + +NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits +NGC_SIZE_PLUS = 0. +NGC_FLAG_VALUE = 32 + + +## External flag +[EXTERNAL_FLAG] + +EF_MAKE = False + + +## Missing data parameters +[MD_PARAMETERS] + +MD_MAKE = False + +MD_THRESH_FLAG = 0.3 +MD_THRESH_REMOVE = 0.75 +MD_REMOVE = False + + +## Other parameters +[OTHER] + +KEEP_REG_FILE = False +KEEP_INDIVIDUAL_MASK = False + +TEMP_DIRECTORY = .temp_tiles diff --git a/example/cfis_image_sims/default.conv b/example/cfis_image_sims/default.conv new file mode 120000 index 000000000..bd71df850 --- /dev/null +++ b/example/cfis_image_sims/default.conv @@ -0,0 +1 @@ +../cfis/default.conv \ No newline at end of file diff --git a/example/cfis_image_sims/default.param b/example/cfis_image_sims/default.param new file mode 120000 index 000000000..49e000314 --- /dev/null +++ b/example/cfis_image_sims/default.param @@ -0,0 +1 @@ +../cfis/default.param \ No newline at end of file diff --git a/example/cfis_image_sims/default_noimaflags.param b/example/cfis_image_sims/default_noimaflags.param new file mode 120000 index 000000000..75451801e --- /dev/null +++ b/example/cfis_image_sims/default_noimaflags.param @@ -0,0 +1 @@ +../cfis/default_noimaflags.param \ No newline at end of file diff --git a/example/cfis_image_sims/default_tile.sex b/example/cfis_image_sims/default_tile.sex new file mode 120000 index 000000000..8770da87e --- /dev/null +++ b/example/cfis_image_sims/default_tile.sex @@ -0,0 +1 @@ +../cfis/default_tile.sex \ No newline at end of file diff --git a/example/cfis_image_sims/final_cat.param b/example/cfis_image_sims/final_cat.param new file mode 100644 index 000000000..5f96d959e --- /dev/null +++ b/example/cfis_image_sims/final_cat.param @@ -0,0 +1,111 @@ +# Final-catalogue column selection for the image-simulation pipeline. +# +# ShapePipe-v2 (post-#761) ngmix column grammar: ellipticity in named scalar +# components NGMIX_G{1,2}_*, PSF size split into original (T_PSF_ORIG) and +# reconvolved (T_PSF_RECONV) measurements. create_final_cat.py selects exactly +# these columns from the per-tile make_cat output into final_cat_{sim}.hdf5; +# every entry must exist in that catalogue. +# +# This is the image-sims sibling of example/cfis/final_cat.param. The two +# differ only in columns the simulation pipeline does not produce: +# - IMAFLAGS_ISO : the sims skip the imaging-flag masking stage +# - NGMIX_MOM_FAIL : the v2 sims path emits NGMIX_MCAL_TYPES_FAIL instead, +# which the downstream calibration does not read + +# coordinates +XWIN_WORLD +YWIN_WORLD + +# tile ID, for plot of tile-dependent additive bias. +TILE_ID + +# SExtractor number +NUMBER + +# flags +FLAGS +NGMIX_MCAL_FLAGS +NGMIX_MCAL_TYPES_FAIL + +# PSF ellipticity (original image PSF) +NGMIX_G1_PSF_ORIG_NOSHEAR +NGMIX_G2_PSF_ORIG_NOSHEAR + +# Number of epochs (exposures) +N_EPOCH +NGMIX_N_EPOCH + +## Shape measurement outputs +## Ngmix: model fitting + +# galaxy ellipticity +NGMIX_G1_1M +NGMIX_G2_1M +NGMIX_G1_1P +NGMIX_G2_1P +NGMIX_G1_2M +NGMIX_G2_2M +NGMIX_G1_2P +NGMIX_G2_2P +NGMIX_G1_NOSHEAR +NGMIX_G2_NOSHEAR +NGMIX_G1_ERR_NOSHEAR +NGMIX_G2_ERR_NOSHEAR + +# flags +NGMIX_FLAGS_1M +NGMIX_FLAGS_1P +NGMIX_FLAGS_2M +NGMIX_FLAGS_2P +NGMIX_FLAGS_NOSHEAR + +# size and error +NGMIX_T_1M +NGMIX_T_1P +NGMIX_T_2M +NGMIX_T_2P +NGMIX_T_NOSHEAR +NGMIX_T_ERR_1M +NGMIX_T_ERR_1P +NGMIX_T_ERR_2M +NGMIX_T_ERR_2P +NGMIX_T_ERR_NOSHEAR + +# reconvolved PSF size (used by the metacal size cut / Tpsf) +NGMIX_T_PSF_RECONV_1M +NGMIX_T_PSF_RECONV_1P +NGMIX_T_PSF_RECONV_2M +NGMIX_T_PSF_RECONV_2P +NGMIX_T_PSF_RECONV_NOSHEAR + +# flux and error +NGMIX_FLUX_1M +NGMIX_FLUX_1P +NGMIX_FLUX_2M +NGMIX_FLUX_2P +NGMIX_FLUX_NOSHEAR +NGMIX_FLUX_ERR_1M +NGMIX_FLUX_ERR_1P +NGMIX_FLUX_ERR_2M +NGMIX_FLUX_ERR_2P +NGMIX_FLUX_ERR_NOSHEAR + +# magnitudes +MAG_AUTO +MAGERR_AUTO +MAG_WIN +MAGERR_WIN +FLUX_AUTO +FLUXERR_AUTO +FLUX_APER +FLUXERR_APER +FLUX_RADIUS + +# SNR from SExtractor +SNR_WIN + +FWHM_IMAGE +FWHM_WORLD + +# PSF size measured on original image +NGMIX_T_PSF_ORIG_NOSHEAR diff --git a/example/cfis_image_sims/input_dirs.conf b/example/cfis_image_sims/input_dirs.conf new file mode 100644 index 000000000..cd3661983 --- /dev/null +++ b/example/cfis_image_sims/input_dirs.conf @@ -0,0 +1,12 @@ +# Configuration template for init_run_v2.0.sh +# Specifies input directories for image simulations +# Copy and edit this file, then pass with: -c /path/to/config + +# Base directory for input data +input_dir_base="/n09data/hervas/skills_out" + +# Tile images subdirectory +input_dir_tiles="${input_dir_base}/${subdir}/images/SP_tiles" + +# Exposure images subdirectory +input_dir_exp="${input_dir_base}/${subdir}/images/SP_exp" diff --git a/pyproject.toml b/pyproject.toml index dd06ccc42..8b259deec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,7 +104,6 @@ build-backend = "setuptools.build_meta" [tool.setuptools] script-files = [ "scripts/python/update_runs_log_file.py", - "scripts/sh/init_run_v2.0.sh", "scripts/sh/run_job_sp_canfar_v2.0.bash", "scripts/sh/job_sp_canfar_v2.0.bash", ] diff --git a/scripts/python/check_tile_coverage.py b/scripts/python/check_tile_coverage.py new file mode 100755 index 000000000..17f5f9030 --- /dev/null +++ b/scripts/python/check_tile_coverage.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Check input-tile coverage for image simulations. + +Scans the weight maps of the input simulation tiles and computes the +fraction of nonzero pixels (subsampled). Tiles below the coverage +threshold are written to an exclude list (YAML), which the orchestrating +workflow reads at DAG-build time to drop them from the run. Orchestration +now lives in sp_validation; this script is the standalone coverage check +it calls. + +The weight-map location is read from the ShapePipe get-images config +{base}/{sim}/cfis/config_tile_Git_symlink.ini (created by init), for the +first sheared variant (1p2z); coverage is identical across variants. + +(Near-)empty tiles occur at the edge of the simulated footprint; they +crash the pipeline downstream (vignetmaker on 0 objects, ngmix on empty +PSF vignettes) and their edge objects bias the m-bias estimate. + +Usage: + check_tile_coverage.py -c config.yaml [-o output.yaml] [-t THRESH] + [-s SUBSAMPLE] [-v] +""" + +import argparse +import configparser +import os +import re +import sys + +import numpy as np +import yaml +from astropy.io import fits + + +def parse_args(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("-c", "--config", required=True, help="run config YAML file") + p.add_argument( + "-o", "--output", + help="output YAML path (default {base}/tile_coverage_grid_{num}.yaml)", + ) + p.add_argument( + "-t", "--threshold", type=float, + help="minimum coverage fraction (default: min_tile_coverage from" + " config, or 0.5)", + ) + p.add_argument( + "-s", "--subsample", type=int, default=20, + help="read every Nth pixel per axis (default: %(default)s)", + ) + p.add_argument("-v", "--verbose", action="store_true", help="verbose output") + return p.parse_args() + + +def get_tile_ids(config): + """Read the tile ID list the same way the orchestrating workflow does.""" + num = config["num"] + path = config["tile_IDs"].replace("{num}", str(num)) + with open(path) as f: + tile_ids = [line.strip() for line in f if line.strip()] + + exclusive = config.get("tile_IDs_exclusive") + if exclusive and isinstance(exclusive, list): + tile_ids = exclusive + + return tile_ids + + +def get_weight_template(config): + """Weight-map path template from the get-images ShapePipe config. + + Reads {base}/{sim}/cfis/config_tile_Git_symlink.ini of the first + sheared variant (1p2z) and returns (directory, pattern, extension, + numbering regex) of the weight entry. + """ + num = config["num"] + sims_type = config.get("sims_type", "grid") + str_type = f"_{sims_type}" if sims_type == "grid" else "" + sim_dir = os.path.join(config["base"], f"1p2z{str_type}_{num}") + ini_path = os.path.join(sim_dir, "cfis", "config_tile_Git_symlink.ini") + + cp = configparser.ConfigParser(interpolation=None) + if not cp.read(ini_path): + raise FileNotFoundError( + f"{ini_path} not found; run 'snakemake init_all' first" + ) + sec = cp["GET_IMAGES_RUNNER"] + paths = [p.strip() for p in sec["INPUT_PATH"].split(",")] + patterns = [p.strip() for p in sec["INPUT_FILE_PATTERN"].split(",")] + exts = [e.strip() for e in sec["INPUT_FILE_EXT"].split(",")] + numbering = sec["INPUT_NUMBERING"].split(",")[0].strip() + + idx = next( + (i for i, p in enumerate(patterns) if "weight" in p.lower()), None + ) + if idx is None: + raise ValueError(f"no weight entry in INPUT_FILE_PATTERN of {ini_path}") + + weight_dir = paths[idx].replace("$SP_DIR", sim_dir) + return weight_dir, patterns[idx], exts[idx], numbering + + +def coverage_fraction(path, subsample): + """Fraction of nonzero pixels in a weight map, subsampled.""" + with fits.open(path) as hdul: + data = hdul[0].data[::subsample, ::subsample] + return float(np.count_nonzero(data) / data.size) + + +def main(): + args = parse_args() + + with open(args.config) as f: + config = yaml.safe_load(f) + + threshold = args.threshold + if threshold is None: + threshold = config.get("min_tile_coverage", 0.5) + + num = config["num"] + tile_ids = get_tile_ids(config) + weight_dir, pattern, ext, numbering = get_weight_template(config) + out_path = args.output or os.path.join( + config["base"], f"tile_coverage_grid_{num}.yaml" + ) + + print(f"Scanning {len(tile_ids)} tiles in {weight_dir}") + print(f"Coverage threshold: {threshold}") + + coverage = {} + exclude = [] + missing = [] + for tile in tile_ids: + fname = re.sub(numbering, tile.replace(".", "-"), pattern) + ext + path = os.path.join(weight_dir, fname) + if not os.path.isfile(path): + missing.append(tile) + exclude.append(tile) + print(f" {tile}: WEIGHT FILE MISSING -> exclude") + continue + frac = coverage_fraction(path, args.subsample) + coverage[tile] = round(frac, 4) + if frac < threshold: + exclude.append(tile) + if args.verbose or frac < threshold: + flag = " <-- exclude" if frac < threshold else "" + print(f" {tile}: coverage {frac:.3f}{flag}") + + result = { + "num": num, + "min_coverage": threshold, + "coverage": coverage, + "missing": missing, + "exclude": sorted(exclude), + } + with open(out_path, "w") as f: + yaml.dump(result, f, default_flow_style=False, sort_keys=False) + + n_keep = len(tile_ids) - len(exclude) + print(f"\n{len(exclude)} of {len(tile_ids)} tiles below threshold" + f" ({n_keep} remain)") + print(f"Written to {out_path}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/python/create_final_cat.py b/scripts/python/create_final_cat.py index a0b10e8c9..75e3a67c0 100755 --- a/scripts/python/create_final_cat.py +++ b/scripts/python/create_final_cat.py @@ -61,7 +61,7 @@ def params_default(): Set default parameter values. """ - _params = { + _params = { "input_root_dir": ".", "merged_cat_path": "final_cat.hdf5", "param_path": None, @@ -71,29 +71,33 @@ def params_default(): "output_summary": "n_tiles_final.txt", "ID": None, "single_op": None, + "image_sims": False, } - _short_options = { - "input_root_dir": "-i", + _short_options = { + "input_root_dir": "-i", "merged_cat_path": "-m", "param_path": "-p", "patch": "-P", "list_only": "-l", "output_summary": "-o", "single_op": "-s", - } - _types = { - "hdu_num": "int", + "image_sims": "-I", + } + _types = { + "hdu_num": "int", "list_only": "bool", - } - _help_strings = { + "image_sims": "bool", + } + _help_strings = { "input_root_dir": "input root_dir for tile catalogues, default={}", "merged_cat_path": "merged catalogue path (hdf5 file), default={}", "param_path": "parameter file path, if not given use all columns, default={}", - "patch": "patch number, default={}", + "patch": "patch number (data) or grid subdir (image_sims), default={}", "list_only": "print list of patches and IDs only, default={}", "output_summary": "output file for numbre of tiles, default={}", "ID": "ID for single-ID operation, default={}", "single_op": "single ID operation, allowed are 'check', 'add', 'remove'; default={}", + "image_sims": "image simulations mode (different dir layout and run prefix), default={}", } return _params, _short_options, _types, _help_strings @@ -231,17 +235,34 @@ def check_ID(merged_cat_path, ID, verbose=False): def print_list(params): + verbose = params.get("verbose", False) n_tiles = 0 + + if not os.path.exists(params["merged_cat_path"]): + print(f"File {params['merged_cat_path']} not found") + return + with h5py.File(params["merged_cat_path"], "r") as hdf5_file: - for patch in hdf5_file["patches"]: - #print(patch) - for id in hdf5_file[f"patches/{patch}"]: - n_tiles += 1 + if "patches" not in hdf5_file: + print("Warning: no 'patches' group in output file (0 tiles added?)") + else: + for patch in hdf5_file["patches"]: + for id in hdf5_file[f"patches/{patch}"]: + n_tiles += 1 + if verbose: + print(f" {patch}/{id}") + + if verbose: + print(f"Total: {n_tiles} tiles") with open(params["output_summary"], "w") as f_out: print(n_tiles, file=f_out) + # Write n_tiles to HDF5 file header + with h5py.File(params["merged_cat_path"], "a") as hdf5_file: + hdf5_file.attrs["n_tiles"] = n_tiles + def get_patch_group(hdf5_file, patch, verbose=False): """Get Patch group. @@ -339,20 +360,50 @@ def copy_data(param_list, extracted_data, dtype): return structured_data +def collect_tile_ids_image_sims(patch_path): + """Collect tile IDs from image-sims layout: tiles/// + + Parameters + ---------- + patch_path : str + path to the grid subdir (e.g. .../1p2z_grid_1) + + Returns + ------- + list of (tile_id, tile_path) tuples + """ + id_pattern = re.compile(r"^\d+\.\d+$") + tiles_root = os.path.join(patch_path, "tiles") + result = [] + if not os.path.isdir(tiles_root): + return result + for prefix in os.listdir(tiles_root): + prefix_path = os.path.join(tiles_root, prefix) + if not os.path.isdir(prefix_path): + continue + for tile_id in os.listdir(prefix_path): + if id_pattern.match(tile_id): + result.append((tile_id, os.path.join(prefix_path, tile_id))) + return result + + def process(params): - - patch = rf"P{params['patch']}" - patch_pattern = re.compile(patch) - - # Define the nested path pattern for locating `.fits` files - file_pattern = "output/run_sp_Mc_*/make_cat_runner/output/*.fits" + + if params["image_sims"]: + patch_name = params["patch"] + run_prefix = "run_sp_tile_Mc_*" + else: + patch_name = rf"P{params['patch']}" + run_prefix = "run_sp_Mc_*" + + patch_pattern = re.compile(patch_name) # Regex pattern for tile IDs id_pattern = re.compile(r"^\d+\.\d+$") n_added = 0 IDs_added = [] - + # Open the HDF5 file (create it if it doesn't exist) if params["verbose"]: print(f"Initializing file {params['merged_cat_path']}") @@ -364,21 +415,30 @@ def process(params): # Skip non-matching entries if not patch_pattern.fullmatch(patch): continue - + # Full path to patch patch_path = os.path.join(params["input_root_dir"], patch) if not os.path.isdir(patch_path): if params["verbose"]: print(f"Path {patch_path} not found, skipping") continue - + # Get hdf5 group for this patch patch_group = get_patch_group(hdf5_file, patch, params["verbose"]) - # Get paths to all tile IDs and loop - tile_runs_path = os.path.join(patch_path, "tile_runs") - subdirs = os.listdir(tile_runs_path) - for id in tqdm.tqdm(subdirs, total=len(subdirs)): + # Collect (tile_id, tile_path) pairs depending on layout + if params["image_sims"]: + tile_items = collect_tile_ids_image_sims(patch_path) + else: + tile_runs_path = os.path.join(patch_path, "tile_runs") + tile_items = [ + (tid, os.path.join(tile_runs_path, tid)) + for tid in os.listdir(tile_runs_path) + if id_pattern.match(tid) + and os.path.isdir(os.path.join(tile_runs_path, tid)) + ] + + for id, id_path in tqdm.tqdm(tile_items, total=len(tile_items)): # Skip if the patch/ID data already exists if id in patch_group: @@ -386,44 +446,40 @@ def process(params): print(f"Skipping {id} (already processed)") continue - if id_pattern.match(id) and os.path.isdir(os.path.join(tile_runs_path, id)): - id_path = os.path.join(tile_runs_path, id) - - base_pattern = os.path.join(id_path, "output", "run_sp_Mc_*") - all_matches = [d for d in glob.glob(base_pattern) if os.path.isdir(d)] - if not all_matches: - if params["verbose"]: - print(f"Final cat for {id} not found, continuing") - continue - newest_dir = max(all_matches, key=os.path.getmtime) - - id_dash = re.sub("\.", "-", id) - fits_file = f"{newest_dir}/make_cat_runner/output/final_cat-{id_dash}.fits" - - # Exclude unsuccessful run without output FITS file - if not os.path.exists(fits_file): - if params["verbose"]: - print(f"Run without output file found for {id}, skipping") - continue - - if True: - extracted_data, dtype = read_data(fits_file, params) - - structured_data = copy_data(params["param_list"], extracted_data, dtype) - - # Create a new dataset - try: - patch_group.create_dataset( - str(id), - data=structured_data, - dtype=dtype, - ) - except: - print(f"Error for {id}: Could not create dataset in group {patch}") - raise - - n_added += 1 - IDs_added.append(id) + base_pattern = os.path.join(id_path, "output", run_prefix) + all_matches = [d for d in glob.glob(base_pattern) if os.path.isdir(d)] + if not all_matches: + if params["verbose"]: + print(f"Final cat for {id} not found, continuing") + continue + newest_dir = max(all_matches, key=os.path.getmtime) + + id_dash = re.sub(r"\.", "-", id) + fits_file = f"{newest_dir}/make_cat_runner/output/final_cat-{id_dash}.fits" + + # Exclude unsuccessful run without output FITS file + if not os.path.exists(fits_file): + if params["verbose"]: + print(f"Run without output file found for {id}, skipping") + continue + + extracted_data, dtype = read_data(fits_file, params) + + structured_data = copy_data(params["param_list"], extracted_data, dtype) + + # Create a new dataset + try: + patch_group.create_dataset( + str(id), + data=structured_data, + dtype=dtype, + ) + except: + print(f"Error for {id}: Could not create dataset in group {patch}") + raise + + n_added += 1 + IDs_added.append(id) if params["verbose"]: print(f"{n_added} tiles added ({' '.join(IDs_added)})") diff --git a/scripts/python/init_run_v2.0.py b/scripts/python/init_run_v2.0.py new file mode 100755 index 000000000..81b301ba5 --- /dev/null +++ b/scripts/python/init_run_v2.0.py @@ -0,0 +1,449 @@ +#!/usr/bin/env python3 + +""" +Initialise the directory structure for a v2.0 ShapePipe run. +Creates tile and exposure staging directories, config symlinks, +and a tile number list. + +Config file is loaded first (if provided), then overridden by command-line arguments. +""" + +import argparse +import sys +from pathlib import Path +import os + +try: + import yaml +except ImportError: + print("ERROR: PyYAML not installed. Install with: pip install pyyaml", file=sys.stderr) + sys.exit(1) + + +class InitRun: + def __init__(self): + self.sp_root = Path.home() / "shapepipe" + self.quiet = False + self.defaults = { + "type": "data", + "subdir": "", + "base_dir": ".", + "config_dir": "", + "params_src": str(Path.home() / "sp_validation" / "notebooks" / "params.py"), + "tiles_src": "", + "tile_ids_list": None, + "sample": "", + "input_sims_base_dir": "", + "input_tiles": "", + "input_exp": "", + "sims_type": "", + "num": "", + } + self.config = {} + self.args = {} + + def load_config(self, config_file): + """Load YAML config file.""" + config_path = Path(config_file) + if not config_path.exists(): + print(f"ERROR: Config file not found: {config_file}", file=sys.stderr) + sys.exit(2) + + with open(config_path) as f: + self.config = yaml.safe_load(f) or {} + + # Map config keys to script parameters + if "base" in self.config: + self.config["base_dir"] = self.config.pop("base") + if "type" in self.config: + # Keep type from config (data or image_sims) + self.config["type"] = self.config["type"] + if "tile_IDs" in self.config: + tile_ids = self.config.pop("tile_IDs") + # tile_IDs can be a file path (string) or a list of IDs + if isinstance(tile_ids, str): + self.config["tiles_src"] = tile_ids + elif isinstance(tile_ids, list): + # Will be written to file in run directory + self.config["tile_ids_list"] = tile_ids + if "params_py" in self.config: + self.config["params_src"] = self.config.pop("params_py") + # Keep input_tiles and input_exp if present (they may contain placeholders) + # They will be substituted before creating symlinks + + def merge_config(self): + """Merge: defaults <- config <- command-line args.""" + params = dict(self.defaults) + params.update(self.config) + params.update(self.args) + return params + + def substitute_placeholders(self, value, params): + """Substitute {key} placeholders in path strings using params dict.""" + if not isinstance(value, str): + return value + result = value + for key, val in params.items(): + if val is not None and val != "": + # Convert to string and strip trailing slashes + clean_val = str(val).rstrip('/') + result = result.replace(f"{{{key}}}", clean_val) + # Clean up any double slashes that may result + while '//' in result: + result = result.replace('//', '/') + return result + + def construct_default_subdir(self, params): + """Construct default subdir from sims_type and num if not explicitly set.""" + if not params["subdir"] and params.get("sims_type") and params.get("num"): + params["subdir"] = f"1p2z_{params['sims_type']}_{params['num']}" + + def set_type_defaults(self, params): + """Set type-specific defaults for config_dir and tiles_src.""" + if params["type"] == "data": + config_dir = self.sp_root / "example" / "cfis" + if not params["tiles_src"]: + params["tiles_src"] = str( + self.sp_root / "auxdir" / "CFIS" / "tiles_202604" / "tiles_r.txt" + ) + elif params["type"] == "image_sims": + config_dir = self.sp_root / "example" / "cfis_image_sims" + if not params["tiles_src"]: + # Try to use num-specific tile file first + if params.get("num"): + num_tile_file = self.sp_root / "auxdir" / "CFIS" / "im_sims_202606" / f"tile_numbers_{params['num']}.txt" + if num_tile_file.exists(): + params["tiles_src"] = str(num_tile_file) + else: + params["tiles_src"] = str( + self.sp_root / "auxdir" / "CFIS" / "im_sims_202606" / "numbers.txt" + ) + else: + params["tiles_src"] = str( + self.sp_root / "auxdir" / "CFIS" / "im_sims_202606" / "numbers.txt" + ) + else: + print(f"ERROR: Invalid type '{params['type']}'. Must be 'data' or 'image_sims'", file=sys.stderr) + sys.exit(3) + + # Use config_dir from config, or fall back to type-specific default + if not params["config_dir"]: + params["config_dir"] = str(config_dir) + return params + + def create_symlink(self, link_name, target, force=False): + """Create symlink, return True if created.""" + link_path = Path(link_name) + + if link_path.exists() or link_path.is_symlink(): + if force: + link_path.unlink() + else: + if link_path.is_symlink() and not self.quiet: + print(f"{link_name} symlink already exists, skipping") + elif not self.quiet: + print(f"WARNING: {link_name} exists as a regular file, not creating symlink") + return False + + link_path.symlink_to(target) + if not self.quiet: + print(f"Created symlink: {link_name} -> {target}") + return True + + def write_tile_ids(self, tile_ids_list): + """Write tile IDs list to tile_numbers.txt.""" + tile_file = Path("tile_numbers.txt") + with open(tile_file, "w") as f: + for tile_id in tile_ids_list: + f.write(f"{tile_id}\n") + if not self.quiet: + print(f"Created tile_numbers.txt with {len(tile_ids_list)} tiles") + + def copy_params(self, params, force=False): + """Copy and adapt params.py.""" + params_dest = Path("params.py") + + if params_dest.exists() and not force: + if not self.quiet: + print("params.py already exists, skipping") + return + + params_src = Path(params["params_src"]) + if not params_src.exists(): + print(f"ERROR: Source params.py not found: {params_src}", file=sys.stderr) + sys.exit(4) + + content = params_src.read_text() + # Remove IMAFLAGS_ISO + content = content.replace("'IMAFLAGS_ISO', ", "").replace(", 'IMAFLAGS_ISO'", "") + content = content.replace('"IMAFLAGS_ISO", ', "").replace(', "IMAFLAGS_ISO"', "") + + if params["type"] == "image_sims": + # Adapt for image_sims + lines = content.split("\n") + adapted = [] + for line in lines: + if line.startswith("name = "): + line = f"name = '{params['subdir']}'" + elif line.startswith("star_cat_path = "): + line = "star_cat_path = None" + elif line.startswith("output_format = "): + line = "output_format = '.hdf5'" + adapted.append(line) + content = "\n".join(adapted) + if not self.quiet: + print("Copied and adapted: params.py <- " + params["params_src"]) + print(f" name = '{params['subdir']}', star_cat_path = None, output_format = '.hdf5'") + else: + if not self.quiet: + print(f"Copied: params.py <- {params['params_src']}") + + params_dest.write_text(content) + + def run(self, config_file=None, quiet=False, **kwargs): + """Run initialization.""" + self.quiet = quiet + + # Load config first + if config_file: + self.load_config(config_file) + + # Apply command-line overrides (exclude quiet) + self.args = {k: v for k, v in kwargs.items() if v is not None and k != "quiet"} + + # Merge all sources + params = self.merge_config() + + # Construct default subdir from sims_type and num if needed + self.construct_default_subdir(params) + + # Validate that we have a subdir + if not params["subdir"]: + print("ERROR: subdir must be provided (-s) or derivable from sims_type and num in config", file=sys.stderr) + sys.exit(4) + + # Set type-specific defaults (including num-based tile_IDs for image_sims) + params = self.set_type_defaults(params) + + # Expand placeholders in tiles_src (e.g. {num}) + if params.get("tiles_src"): + params["tiles_src"] = self.substitute_placeholders(params["tiles_src"], params) + + # Use pwd if base_dir still empty + if not params["base_dir"]: + params["base_dir"] = str(Path.cwd()) + + base_dir = Path(params["base_dir"]) / params["subdir"] + + if not self.quiet: + print(f"Initialising ShapePipe run directory: {params['base_dir']}") + print() + + # Create base directory + base_dir.mkdir(parents=True, exist_ok=True) + os.chdir(base_dir) + + # Image sims specific: input symlinks + if params["type"] == "image_sims": + # input_tiles and input_exp symlinks (with placeholder substitution) + if params.get("input_tiles"): + input_tiles_path = self.substitute_placeholders(params["input_tiles"], params) + self.create_symlink("input_tiles", input_tiles_path, params["force"]) + if params.get("input_exp"): + input_exp_path = self.substitute_placeholders(params["input_exp"], params) + self.create_symlink("input_exp", input_exp_path, params["force"]) + + # Image sims specific: mask config symlink + if params["type"] == "image_sims" and params.get("sample"): + mask_src = ( + Path.home() + / "sp_validation" + / "config" + / "calibration" + / f"mask_v1.X.{params['sample']}_im_sim.yaml" + ) + self.create_symlink("config_mask.yaml", str(mask_src), params["force"]) + + # Create directories + if not self.quiet: + print("Creating tiles/ directory...") + Path("tiles").mkdir(exist_ok=True) + if not self.quiet: + print(" tiles/ created (subdirs added at download time)") + + if not self.quiet: + print("Creating exp/ directory...") + Path("exp").mkdir(exist_ok=True) + if not self.quiet: + print(" exp/ created (subdirs added at download time)") + + if not self.quiet: + print("Creating log and debug directories...") + Path("logs").mkdir(exist_ok=True) + Path("debug").mkdir(exist_ok=True) + + # Config symlink + self.create_symlink("cfis", params["config_dir"], params["force"]) + + # Tile numbers: either symlink to file or write list + if params.get("tile_ids_list"): + if not self.quiet: + print("Creating tile_numbers.txt from tile_IDs list...") + self.write_tile_ids(params["tile_ids_list"]) + else: + if not self.quiet: + print("Creating tile_numbers.txt symlink...") + self.create_symlink("tile_numbers.txt", params["tiles_src"], params["force"]) + n_tiles = sum(1 for _ in open("tile_numbers.txt")) + if not self.quiet: + print(f" {n_tiles} tiles") + + # params.py + if not self.quiet: + print("Creating params.py...") + self.copy_params(params, params["force"]) + + # Print summary + print(f"Done: {params['subdir']}") + + if not self.quiet: + print() + print("Directory structure:") + print(f" {base_dir}") + print(" ├── tiles/") + print(" ├── exp/") + print(" ├── logs/") + print(" ├── debug/") + print(f" ├── cfis -> {params['config_dir']}") + + # Show tile_numbers.txt info + if params.get("tile_ids_list"): + print(f" ├── tile_numbers.txt ({len(params['tile_ids_list'])} tiles)") + else: + print(f" ├── tile_numbers.txt -> {params['tiles_src']}") + + if params["type"] == "image_sims": + if params.get("sample"): + mask_src = ( + Path.home() + / "sp_validation" + / "config" + / "calibration" + / f"mask_v1.X.{params['sample']}_im_sim.yaml" + ) + print(f" ├── config_mask.yaml -> {mask_src}") + if params.get("input_tiles"): + input_tiles_path = self.substitute_placeholders(params["input_tiles"], params) + print(f" ├── input_tiles -> {input_tiles_path}") + if params.get("input_exp"): + input_exp_path = self.substitute_placeholders(params["input_exp"], params) + print(f" ├── input_exp -> {input_exp_path}") + + print(f" └── params.py <- {params['params_src']}") + + +def main(): + parser = argparse.ArgumentParser( + description="Initialise ShapePipe v2.0 run directory", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Read all from config file (subdir auto-generated from sims_type + num) + init_run_v2.0.py -c config.yaml -t image_sims + + # Override config values with command-line options + init_run_v2.0.py -c config.yaml -t image_sims -d /custom/base -S 6 + + # Explicit subdir (overrides auto-generation) + init_run_v2.0.py -c config.yaml -t image_sims -s my_subdir + + # No config file, use defaults and flags + init_run_v2.0.py -t image_sims -d /tmp/test -s grid_1 + """, + ) + + parser.add_argument( + "-c", "--config", + help="Config file (YAML); entries can be overridden by following flags" + ) + parser.add_argument( + "-t", "--type", + choices=["data", "image_sims"], + help="Input type" + ) + parser.add_argument( + "-d", "--dir", + dest="base_dir", + help="Base run directory" + ) + parser.add_argument( + "-C", "--config-dir", + dest="config_dir", + help="Config directory (overrides type-specific default)" + ) + parser.add_argument( + "-s", "--subdir", + help="Subdirectory for simulations (auto-generated from sims_type+num if not provided)" + ) + parser.add_argument( + "-T", "--tiles", + dest="tiles_src", + help="Tile numbers file" + ) + parser.add_argument( + "-i", "--input-sims", + dest="input_sims_base_dir", + help="Input simulations base directory (image_sims only)" + ) + parser.add_argument( + "--input-tiles", + dest="input_tiles", + help="Input tiles directory path (supports {subdir} and {input_sims_base_dir} placeholders)" + ) + parser.add_argument( + "--input-exp", + dest="input_exp", + help="Input exposures directory path (supports {subdir} and {input_sims_base_dir} placeholders)" + ) + parser.add_argument( + "-S", "--sample", + help="Sample version for mask config (e.g., 6)" + ) + parser.add_argument( + "-P", "--params", + dest="params_src", + help="Source params.py to copy" + ) + parser.add_argument( + "--force", + action="store_true", + help="Recreate existing symlinks and parameter files" + ) + parser.add_argument( + "-q", "--quiet", + action="store_true", + help="Suppress all output except final 'Done' message" + ) + + args = parser.parse_args() + + init = InitRun() + init.run( + config_file=args.config, + type=args.type, + base_dir=args.base_dir, + config_dir=args.config_dir, + subdir=args.subdir, + tiles_src=args.tiles_src, + input_sims_base_dir=args.input_sims_base_dir, + input_tiles=args.input_tiles, + input_exp=args.input_exp, + sample=args.sample, + params_src=args.params_src, + force=args.force, + quiet=args.quiet, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/sh/apptainer_noslurm.sh b/scripts/sh/apptainer_noslurm.sh new file mode 100755 index 000000000..4d28dfb64 --- /dev/null +++ b/scripts/sh/apptainer_noslurm.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Run apptainer exec after stripping all SLURM/PMI/PMIX/OMPI environment +# variables that cause OpenMPI to crash when running inside a SLURM job. +# Usage: apptainer_noslurm.sh [apptainer_args...] -- [cmd_args...] + +for v in $(env | grep -Eo "^(SLURM|PMI|PMIX|OMPI)[^=]*"); do + unset "$v" +done + +exec apptainer exec "$@" diff --git a/scripts/sh/init_run_v2.0.sh b/scripts/sh/init_run_v2.0.sh deleted file mode 100755 index 74ab260c0..000000000 --- a/scripts/sh/init_run_v2.0.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env bash - -# Name: init_run_v2.0.sh -# Description: Initialise the directory structure for a v2.0 ShapePipe run. -# Creates tile and exposure staging directories, config symlinks, -# and a tile number list from the 202604 tile catalogue. -# Author: Martin Kilbinger - -# Version -version="2.0" - -# Default base run directory (permanent storage) -#base_dir="$HOME/cosmostat/v2/v${version}" -base_dir=`pwd` - -# ShapePipe repository root (for config symlink and tile list) -sp_root="$HOME/shapepipe" - -# Tile list source (full filenames, will be stripped to NNN.MMM) -tiles_src="$sp_root/auxdir/CFIS/tiles_202604/tiles_r.txt" - -# Config directory (will be symlinked as $base_dir/cfis) -config_dir="$sp_root/example/cfis" - -## Help string -usage="Usage: $(basename "$0") [OPTIONS] -\nOptions:\n - -h\t\tthis message\n - -d, --dir DIR\tbase run directory, default='$base_dir'\n -" - -## Parse command line -while [ $# -gt 0 ]; do - case "$1" in - -h) - echo -ne $usage - exit 0 - ;; - -d|--dir) - base_dir="$2" - shift - ;; - *) - echo "Unknown option: $1" - echo -ne $usage - exit 1 - ;; - esac - shift -done - -echo "Initialising ShapePipe v${version} run directory: $base_dir" -echo "" - -# --- Base directory --- -mkdir -p "$base_dir" -cd "$base_dir" - -echo "Creating tiles/ directory..." -mkdir -p tiles -echo " exp/ created (subdirs e.g. tiles/301/301.279 added at download time)" - -# --- Exposure staging directory --- -echo "Creating exp/ directory..." -mkdir -p exp -echo " exp/ created (subdirs e.g. exp/28/ added at download time)" - -# --- Output and working directories --- -echo "Creating log and debug directories..." -mkdir -p logs -mkdir -p debug - -# --- Config symlink --- -if [ -L cfis ]; then - echo "cfis symlink already exists, skipping" -elif [ -d cfis ]; then - echo "WARNING: cfis/ exists as a directory, not creating symlink" -else - ln -s "$config_dir" cfis - echo "Created symlink: cfis -> $config_dir" -fi - -# --- Tile number list --- -echo "Creating tile_numbers.txt symlink..." -if [ -L tile_numbers.txt ]; then - echo " tile_numbers.txt symlink already exists, skipping" -elif [ -f tile_numbers.txt ]; then - echo " WARNING: tile_numbers.txt exists as a regular file, not creating symlink" -else - ln -s "$tiles_src" tile_numbers.txt - echo " Created symlink: tile_numbers.txt -> $tiles_src" -fi -n_tiles=$(wc -l < tile_numbers.txt) -echo " $n_tiles tiles" - -echo "" -echo "Done. Directory structure:" -echo " $base_dir/" -echo " ├── tiles/" -echo " ├── exp/" -echo " ├── logs/" -echo " ├── cfis -> ${config_dir}" -echo " └── tile_numbers.txt -> ${tiles_src}" diff --git a/scripts/sh/job_sp_canfar_v2.0.bash b/scripts/sh/job_sp_canfar_v2.0.bash index 1c1ce89ba..592cae70d 100755 --- a/scripts/sh/job_sp_canfar_v2.0.bash +++ b/scripts/sh/job_sp_canfar_v2.0.bash @@ -9,13 +9,17 @@ # machine. # Author: Martin Kilbinger +# Repository root, derived from this script's location (scripts/sh/ -> repo +# root) so the runner is relocatable: no hardcoded ~/shapepipe clone layout. +SP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + # Shared job-list description -source $HOME/shapepipe/scripts/sh/job_list_help.bash +source "$SP_ROOT/scripts/sh/job_list_help.bash" # Command line arguments ## Default values job=255 -config_dir=$HOME/shapepipe/example/cfis +config_dir="$SP_ROOT/example/cfis" psf='mccd' retrieve='vos' star_cat_for_mask='onthefly' @@ -134,9 +138,8 @@ if [ "$star_cat_for_mask" != "onthefly" ] && [ "$star_cat_for_mask" != "save" ]; exit 4 fi -#if [ "$retrieve" != "vos" ] && [ "$retrieve" != "symlink" ]; then -if [ "$retrieve" != "vos" ]; then - echo "method to retrieve images (option -r) needs to be 'vos' for v2.0" +if [ "$retrieve" != "vos" ] && [ "$retrieve" != "symlink" ]; then + echo "method to retrieve images (option -r) needs to be 'vos' (data) or 'symlink' (image sims)" exit 5 fi @@ -291,7 +294,7 @@ function command_cfg_shapepipe() { batch_flag="--batch_size $_n_smp" fi - local cmd="shapepipe_run.py -c $config $batch_flag" + local cmd="shapepipe_run -c $config $batch_flag" command "$cmd" "$str" } @@ -347,12 +350,12 @@ if [[ $do_job != 0 ]]; then fi -## Retrieve exposure images (online, vos) +## Retrieve exposure images (online if retrieve=vos, symlinked if image sims) (( do_job = $job & 8 )) if [[ $do_job != 0 ]]; then command_cfg_shapepipe \ - "config_exp_Gie_vos.ini" \ + "config_exp_Gie_$retrieve.ini" \ "Run shapepipe (get exposure images)" \ $n_smp \ $exclusive @@ -495,11 +498,13 @@ fi (( do_job = $job & 2048 )) if [[ $do_job != 0 ]]; then - suff_sm="_nosm" - - ### Merge all relevant information into final catalogue + ### Merge all relevant information into final catalogue. + ### Config name matches run_job's job-2048 "Mc_${psf}" completeness check and + ### the config present in example/cfis_image_sims. (The data path uses a + ### differently-named make_cat config; harmonizing the two dirs is a separate + ### #766 reconciliation item, not needed for the image-sims rerun.) command_cfg_shapepipe \ - "config_tile_make_cat_$psf${suff_sm}.ini" \ + "config_tile_Mc_$psf.ini" \ "Run shapepipe (tile: create final cat $psf)" \ $n_smp \ $exclusive diff --git a/scripts/sh/run_job_sp_canfar_v2.0.bash b/scripts/sh/run_job_sp_canfar_v2.0.bash index 9da08f598..ba6f5566c 100755 --- a/scripts/sh/run_job_sp_canfar_v2.0.bash +++ b/scripts/sh/run_job_sp_canfar_v2.0.bash @@ -3,8 +3,13 @@ # run_job_sp_canfar_v2.0 # Description: Initialise tile/exposure run directory and launch ShapePipe job. +# Repository root, derived from this script's location (scripts/sh/ -> repo root) +# so the runner is relocatable: no hardcoded ~/shapepipe clone layout. All +# config dirs and sibling scripts are resolved under $SP_ROOT. +SP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + # Shared job-list description -source $HOME/shapepipe/scripts/sh/job_list_help.bash +source "$SP_ROOT/scripts/sh/job_list_help.bash" # ShapePipe version version="2.0" @@ -16,15 +21,28 @@ ID=-1 psf='psfex' tile_det='uc' tile_mask=0 -N_SMP=1 +N_SMP=-1 dry_run=0 dir=`pwd` debug_out="" + +# Input type: data or image_sims +type="data" + +# Config directory override (-c). Empty means "self-compute from --type" (the +# historical behavior); when set, it overrides the computed config_dir for any +# --type, so a caller (e.g. the sp_validation workflow) can point every stage at +# an explicit .ini tree. +config_dir_override="" + #scratch="/scratch/$USER/shapepipe/v${version}" scratch="" test_only=0 -check=0 +check_only=0 +run_and_check=1 force=0 +retry=0 +quiet=0 VERBOSE=1 pat="-- " @@ -40,14 +58,20 @@ ${JOB_LIST_HELP} -e, --exclusive ID\timage ID\n \t\t\tPSF model, one in ['psfex'|'mccd'], default='$psf'\n --tile_det DET\t\ttile detection mode, one in ['sx'|'uc'], default='$tile_det'\n --tile_mask MASK\ttile masking, default='$tile_mask'\n + -t, --type TYPE input type, allowed are 'data', 'image_sims', default='$type'\n + -c, --config_dir DIR\tconfig file directory; overrides the default computed\n + \t\t\tfrom --type (default: computed per --type)\n -N, --N_SMP N_SMP\tnumber of SMP jobs, default from original config files\n -d, --directory DIR\trun directory, default is pwd ($dir)\n -S, --scratch DIR\tprocessing scratch directory, default=none\n -n, --dry_run\t\tDRY RUN, no actual processing; default is $dry_run\n --debug_out PATH\tdebug output file PATH, default=none\n --test\t\t\ttest mode, no processing\n - --check\t\tcheck download completeness only (job 8), no processing\n + --check_only\t\tcheck completeness only, no processing\n --force\t\tremove existing module output dir(s) before running\n + --retry\t\tskip jobs whose existing run is complete; remove and rerun\n + \t\t\tonly those whose existing run is incomplete/failed\n + -q, --quiet\t\tsuppress all output except Complete/Missing/Incomplete/WARNING lines\n " ## Help if no arguments @@ -75,6 +99,14 @@ while [ $# -gt 0 ]; do psf="$2" shift ;; + -t|--type) + type="$2" + shift + ;; + -c|--config_dir) + config_dir_override="$2" + shift + ;; --tile_det) tile_det="$2" shift @@ -106,12 +138,18 @@ while [ $# -gt 0 ]; do --test) test_only=1 ;; - --check) - check=1 + --check_only) + check_only=1 ;; --force) force=1 ;; + --retry) + retry=1 + ;; + -q|--quiet) + quiet=1 + ;; esac shift done @@ -124,10 +162,15 @@ function message() { my_debug_out=$2 my_exit=$3 - echo $msg + if [ "$quiet" == "0" ] || [[ "$msg" =~ (Complete|[Mm]issing|Incomplete|WARNING) ]]; then + echo $msg + fi if [ -n "$my_debug_out" ]; then echo ${pat}$msg >> $my_debug_out fi + if [ -n "$log_file" ]; then + echo ${pat}$msg >> $log_file + fi if [ "$my_exit" != "-1" ]; then if [ -n "$my_debug_out" ]; then @@ -135,12 +178,14 @@ function message() { else echo "${pat}exiting with code $my_exit" fi + if [ -n "$log_file" ]; then + echo "${pat}exiting with code $my_exit" >> $log_file + fi exit $my_exit fi } - # Initialise exposure work directory: create dirs, exp_numbers file, config symlink. # The exp_numbers-000-000.txt file is created only once (skipped if already exists). # Args: $1 = exp_id, $2 = exp_work_dir @@ -158,11 +203,35 @@ function init_exp_work_dir() { fi if [ ! -e "$exp_work_dir/cfis" ]; then - ln -sf ~/shapepipe/example/cfis "$exp_work_dir/cfis" + ln -sf $config_dir "$exp_work_dir/cfis" fi } +# Remove existing run_sp__* output directories. +# Args: $1 = base dir containing the "output" subdir (work_dir or exp_work_dir) +# $2 = "tile" or "exp" +# $3 = space-separated run_prefixes +# $4 = label for the removal message (e.g. "Force-removing") +function remove_run_dirs() { + local base_dir=$1 + local kind=$2 + local run_prefixes=$3 + local label=$4 + local run_prefix + for run_prefix in $run_prefixes; do + local dirs_to_remove + dirs_to_remove=$(ls -d "$base_dir/output/run_sp_${kind}_${run_prefix}"* 2>/dev/null) + if [ -n "$dirs_to_remove" ]; then + for d in $dirs_to_remove; do + message "${label} $d" "$debug_out" -1 + command "rm -rf $d" $dry_run + done + fi + done +} + + # Run a per-exposure job (e.g. job 8, 16). # Args: $1 = job number # $2 = space-separated list of run_sp_exp output dir prefixes (e.g. "Gie") @@ -185,7 +254,7 @@ function run_exp_job() { message "Exposure numbers file exp_numbers-${IDra}-${IDdec}.txt not found in $work_dir/output" "$debug_out" 10 fi - if [ "$check" == "1" ]; then + if [ "$check_only" == "1" ]; then message "Check mode: skipping job $exp_job" "$debug_out" -1 fi @@ -200,30 +269,27 @@ function run_exp_job() { (( n_total++ )) - # exp_id e.g. "2182795p": ab = first 2 chars, abcdefg = all but last char + # exp_id e.g. "2182795p" (data) or "208659" (image_sims) + # Strip trailing letter if present (data format); keep full id if numeric only. local exp_prefix="${exp_id:0:2}" - local exp_base="${exp_id%?}" + local exp_base + if [[ "${exp_id: -1}" =~ [a-zA-Z] ]]; then + exp_base="${exp_id%?}" + else + exp_base="$exp_id" + fi local exp_id_disp="${exp_prefix}/${exp_base}" - local exp_work_dir="$HOME/v${version}/exp/$exp_prefix/$exp_base" + local exp_work_dir="$dir/exp/$exp_prefix/$exp_base" local exp_log_file="$exp_work_dir/job_sp_canfar_v2.0.log" # Create exp_numbers-000-000.txt and cfis link if not existent init_exp_work_dir "$exp_id" "$exp_work_dir" - # force: remove all existing run directories for each prefix before running - if [ "$force" == "1" ]; then - local run_prefix - for run_prefix in $run_prefixes; do - local dirs_to_remove - dirs_to_remove=$(ls -d "$exp_work_dir/output/run_sp_exp_${run_prefix}"* 2>/dev/null) - if [ -n "$dirs_to_remove" ]; then - for d in $dirs_to_remove; do - message "Force-removing $d" "$debug_out" -1 - command "rm -rf $d" $dry_run - done - fi - done - fi + # Acquire a per-exposure lock so parallel tiles that share this exposure + # don't race: one runs, the other waits and then finds it already complete. + local lock_fd + exec {lock_fd}>"${exp_work_dir}/.sp_lock" + flock -x "$lock_fd" # Check completeness of existing run output (main prefix) local run_dir=$(ls -dt "$exp_work_dir/output/run_sp_exp_${main_prefix}"* 2>/dev/null | head -1) @@ -286,20 +352,30 @@ function run_exp_job() { fi done - if [ "$is_complete" == "1" ]; then + # force: always remove and rerun. + # retry: remove and rerun only if the existing run is incomplete/failed. + if [ "$force" == "1" ]; then + remove_run_dirs "$exp_work_dir" "exp" "$run_prefixes" "Force-removing" + elif [ "$retry" == "1" ] && [ "$is_complete" == "0" ]; then + remove_run_dirs "$exp_work_dir" "exp" "$run_prefixes" "Retry: removing incomplete run" + fi + + if [ "$force" != "1" ] && [ "$is_complete" == "1" ]; then message "Complete $exp_id_disp: run_sp_exp_${main_prefix} ( $check_desc)" "$debug_out" -1 (( n_complete++ )) + exec {lock_fd}>&- continue fi # Report incomplete/missing in check mode; in run mode handle and proceed - if [ "$check" == "1" ]; then + if [ "$check_only" == "1" ]; then if [ -n "$run_dir" ]; then message " Benign incomplete: $exp_id_disp ($check_desc)" "$debug_out" -1 else message " missing: $exp_id_disp" "$debug_out" -1 fi (( n_incomplete++ )) + exec {lock_fd}>&- continue fi @@ -311,9 +387,10 @@ function run_exp_job() { [ -n "$debug_out" ] && debug_flag="--debug_out $debug_out" echo "$(basename "$0") -j $exp_job -e $exp_id" > "$exp_log_file" - echo "pwd=`pwd`" - command "job_sp_canfar_v2.0.bash -p $psf --tile_det $tile_det --tile_mask $tile_mask -j $exp_job --n_smp $N_SMP --nsh_jobs $N_SMP $debug_flag" $dry_run 2>&1 | tee -a "$exp_log_file" - echo "Done with job_sp_canfar_v2.0.bash" + [ "$quiet" == "0" ] && echo "pwd=`pwd`" + command "$SP_ROOT/scripts/sh/job_sp_canfar_v2.0.bash -c $config_dir -p $psf -r $retrieve --tile_det $tile_det --tile_mask $tile_mask -j $exp_job --n_smp $N_SMP --nsh_jobs $N_SMP $debug_flag" $dry_run 2>&1 | tee -a "$exp_log_file" + [ "$quiet" == "0" ] && echo "Done with job_sp_canfar_v2.0.bash" + exec {lock_fd}>&- done < "$exp_numbers_file" @@ -343,20 +420,12 @@ function run_tile_job() { local complete_checks=$3 local main_prefix="${run_prefixes%% *}" - # force: remove all existing run directories for each prefix before running - if [ "$force" == "1" ]; then - local run_prefix - for run_prefix in $run_prefixes; do - local dirs_to_remove - dirs_to_remove=$(ls -d "$work_dir/output/run_sp_tile_${run_prefix}"* 2>/dev/null) - if [ -n "$dirs_to_remove" ]; then - for d in $dirs_to_remove; do - message "Force-removing $d" "$debug_out" -1 - command "rm -rf $d" $dry_run - done - fi - done - fi + # Tile jobs must run in the tile work dir: SP_RUN/SP_CONFIG derive from cwd, + # and run_exp_job leaves cwd in the last exposure dir. When run_job is invoked + # once for the whole pipeline (-j 4095, as the sp_validation workflow does), + # the tile jobs after the exposure loop would otherwise inherit that stale + # exposure cwd and write into it. Re-anchor to the tile dir here. + cd "$work_dir" # Locate most recent existing run directory for the main prefix local run_dir @@ -421,16 +490,24 @@ function run_tile_job() { done fi - if [ "$is_complete" == "1" ] && [ -n "$complete_checks" ]; then + # force: always remove and rerun. + # retry: remove and rerun only if the existing run is incomplete/failed. + if [ "$force" == "1" ]; then + remove_run_dirs "$work_dir" "tile" "$run_prefixes" "Force-removing" + elif [ "$retry" == "1" ] && [ "$is_complete" == "0" ]; then + remove_run_dirs "$work_dir" "tile" "$run_prefixes" "Retry: removing incomplete run" + fi + + if [ "$force" != "1" ] && [ "$is_complete" == "1" ] && [ -n "$complete_checks" ]; then message "Complete: ( $check_desc)" "$debug_out" -1 return 0 fi - if [ "$check" == "1" ]; then + if [ "$check_only" == "1" ]; then if [ -n "$run_dir" ]; then - message "Incomplete: ($check_desc)" "$debug_out" -1 + message "Incomplete: ( $check_desc)" "$debug_out" -1 else - message "Missing: ($check_desc)" "$debug_out" -1 + message "Missing: ( $check_desc)" "$debug_out" -1 fi return 0 fi @@ -440,15 +517,93 @@ function run_tile_job() { [ -n "$debug_out" ] && debug_flag="--debug_out $debug_out" if [ ! -e "cfis" ]; then - ln -sf ~/shapepipe/example/cfis "cfis" + ln -sf $config_dir cfis fi command "update_runs_log_file.py" $dry_run # Run job script - command "job_sp_canfar_v2.0.bash -p $psf --tile_det $tile_det --tile_mask $tile_mask -j $tile_job --n_smp $N_SMP --nsh_jobs $N_SMP $debug_flag" $dry_run 2>&1 | tee -a "$log_file" + command "$SP_ROOT/scripts/sh/job_sp_canfar_v2.0.bash -c $config_dir -p $psf -r $retrieve --tile_det $tile_det --tile_mask $tile_mask -j $tile_job --n_smp $N_SMP --nsh_jobs $N_SMP $debug_flag" $dry_run 2>&1 | tee -a "$log_file" +} + + +# Wrap a run_tile_job or run_exp_job call with check-log and force-cleanup. +# Usage: run_job_logged BIT FUNC [FUNC_ARGS...] +# - With --force: removes log_job_BIT.txt before running FUNC +# - With --check: captures output of FUNC; writes log_job_BIT.txt only if no +# Missing/Incomplete lines appear +# - Otherwise: calls FUNC directly +function run_job_logged() { + local bit=$1 + shift + local log="$dir/logs/log_job_${ID}_${bit}.txt" + + [ "$force" == "1" ] && rm -f "$log" + + if [ "$check_only" == "1" ]; then + local out + out=$( "$@" 2>&1 ) + echo "$out" + echo "$out" | grep -qiE "(Missing|Incomplete)" || echo "$out" > "$log" + else + "$@" + local rc=$? + if [ $rc -eq 0 ]; then + if [ "$run_and_check" == "1" ]; then + check_only=1 + local saved_force=$force + force=0 + local out + out=$( "$@" 2>&1 ) + check_only=0 + force=$saved_force + echo "$out" + echo "$out" | grep -qiE "(Missing|Incomplete)" || echo "$out" > "$log" + else + echo "Completed job $bit $(date)" > "$log" + fi + fi + fi } + +if [ "$type" == "data" ]; then + + [ "$quiet" == "0" ] && echo "Running on data" + retrieve="vos" + config_dir="$SP_ROOT/example/cfis" + export SP_DIR=$dir + export SP_CONFIG=$config_dir + +elif [ "$type" == "image_sims" ]; then + + [ "$quiet" == "0" ] && echo "Running on image simulations" + retrieve="symlink" + config_dir="$SP_ROOT/example/cfis_image_sims" + # SP_DIR points to the run directory where input_tiles and input_exp live; + # configs use $SP_DIR/input_* so those dirs stay outside SP_RUN and are + # not found twice by ShapePipe's recursive glob scan. + export SP_DIR=$dir + export SP_CONFIG=$config_dir + tile_det='sx' + +else + + echo "Invalid input type $type" + +fi + +# -c overrides the config_dir computed from --type (uniformly, any type). Kept +# after the type branch so that when unset the computed default is byte-for-byte +# unchanged; SP_CONFIG (exported per-type above) is re-exported to match. +if [ -n "$config_dir_override" ]; then + config_dir="$config_dir_override" + export SP_CONFIG=$config_dir +fi + +[ "$quiet" == "0" ] && echo "config_dir=$config_dir" + + # Init message if [ "$test_only" == "1" ]; then message "$(basename "$0") test mode, exiting." "$debug_out" 0 @@ -469,17 +624,28 @@ if [ "$ID" == "-1" ]; then fi if [ "$psf" != "psfex" ] && [ "$psf" != "mccd" ]; then - message "PSF (option -p) needs to be 'psfex' or 'mccd'" "$debug_out" 4 + message "PSF (option -p) needs to be 'psfex' or 'mccd', not '$psf'" "$debug_out" 4 fi if [ "$dry_run" != "0" ] && [ "$dry_run" != "1" ]; then message "dry_run must be 0 or 1, not $dry_run" "$debug_out" 8 fi +## Check input links +for link in "$dir/input_tiles" "$dir/input_exp" "$dir/cfis"; do + if [ -L "$link" ]; then + if [ ! -e "$link" ]; then + message "Broken symlink: $link" "$debug_out" 6 + fi + elif [ ! -e "$link" ]; then + message "Missing path: $link" "$debug_out" 6 + fi +done + # Start script -source $HOME/shapepipe/scripts/sh/functions.sh +source "$SP_ROOT/scripts/sh/functions.sh" message "Starting $(basename "$0")" "$debug_out" -1 message "`date`" "$debug_out" -1 @@ -495,18 +661,28 @@ Letter=${letter^} cd $dir -# Derive tile path components from ID (e.g. "000.227" -> IDra="000") +# Derive tile path components from ID (e.g. "000.227" -> IDra="000", IDdec="227") IDra=${ID%%.*} +IDdec=${ID##*.} +ID_DASHED="${IDra}-${IDdec}" work_dir="$dir/tiles/$IDra/$ID" log_file="$work_dir/job_sp_canfar_v2.0.log" # Create tile work directory [ ! -d "$work_dir" ] && command "mkdir -p $work_dir" $dry_run cd $work_dir +echo "$0 $@" > "$log_file" # Write ID to first input +# Image sims use dash format (e.g. 233-293); real data uses dot format (233.293) +# which ShapePipe's in2out_pattern converts to dashes for output naming only, +# not for input file lookup — so write the format that matches the actual files. if [ ! -e tile_numbers.txt ]; then - echo $ID > tile_numbers.txt + if [ "$type" == "image_sims" ]; then + echo ${ID//./-} > tile_numbers.txt + else + echo $ID > tile_numbers.txt + fi fi # Output directory @@ -514,7 +690,8 @@ if [ ! -d "output" ]; then command "mkdir output" $dry_run fi -echo -n "pwd: "; pwd + +[ "$quiet" == "0" ] && { echo -n "pwd: "; pwd; } # Avoid Qt error with setools @@ -537,44 +714,99 @@ IDdec=${ID##*.} (( do_job = job & 1 )) if [[ $do_job != 0 ]]; then # Job 1: download tile images and weights - run_tile_job 1 "Git" "get_images_runner:4" + if [ "$type" == "image_sims" ]; then + n_exp=2 + else + n_exp=4 + fi + run_job_logged 1 run_tile_job 1 "Git" "get_images_runner:${n_exp}" fi (( do_job = job & 2 )) if [[ $do_job != 0 ]]; then - # Job 2: uncompress tile weights - run_tile_job 2 "Uz" "uncompress_fits_runner:1" + log_2="$dir/logs/log_job_${ID}_2.txt" + [ "$force" == "1" ] && rm -f "$log_2" + if [ "$type" == "image_sims" ]; then + # Image sims weights are already uncompressed; fake the Uz output directory + # so downstream jobs can find the weight via last:uncompress_fits_runner. + weight_src="$dir/input_tiles/CFIS_simu_weight-${ID//./-}.fits" + if [ "$check_only" == "1" ]; then + uz_run_dir=$(ls -dt "$work_dir/output/run_sp_tile_Uz"* 2>/dev/null | head -1) + if [ -n "$uz_run_dir" ] && [ -e "$uz_run_dir/uncompress_fits_runner/output/$(basename $weight_src)" ]; then + msg="Complete: ( Uz/uncompress_fits_runner[fake] 1/1 )" + message "$msg" "$debug_out" -1 + echo "$msg" > "$log_2" + else + message "Missing: Uz $(basename $weight_src)" "$debug_out" -1 + fi + else + uz_out="$work_dir/output/run_sp_tile_Uz$(date +_%Y-%m-%d_%H-%M-%S)/uncompress_fits_runner/output" + command "mkdir -p $uz_out" $dry_run + if [ -e "$weight_src" ] && [ ! -e "$uz_out/$(basename $weight_src)" ]; then + command "ln -sf $weight_src $uz_out/$(basename $weight_src)" $dry_run + fi + if [ "$run_and_check" == "1" ]; then + uz_run_dir=$(ls -dt "$work_dir/output/run_sp_tile_Uz"* 2>/dev/null | head -1) + if [ -n "$uz_run_dir" ] && [ -e "$uz_run_dir/uncompress_fits_runner/output/$(basename $weight_src)" ]; then + msg="Complete: Uz $(basename $weight_src)" + message "$msg" "$debug_out" -1 + echo "$msg" > "$log_2" + else + message "Missing: Uz $(basename $weight_src)" "$debug_out" -1 + fi + else + echo "Completed job 2 $(date)" > "$log_2" + fi + fi + else + # Job 2: uncompress tile weights + run_job_logged 2 run_tile_job 2 "Uz" "uncompress_fits_runner:1" + fi fi (( do_job = job & 4 )) if [[ $do_job != 0 ]]; then # Job 4: find exposures - run_tile_job 4 "Fe" "find_exposures_runner:1" + run_job_logged 4 run_tile_job 4 "Fe" "find_exposures_runner:1" fi (( do_job = job & 8 )) if [[ $do_job != 0 ]]; then # Job 8: retrieve exposure images - run_exp_job 8 "Gie" "get_images_runner:6" + if [ "$type" == "image_sims" ]; then + n_exp=3 + else + n_exp=6 + fi + run_job_logged 8 run_exp_job 8 "Gie" "get_images_runner:${n_exp}" fi (( do_job = job & 16 )) if [[ $do_job != 0 ]]; then # Job 16: split exposures, get WCS headers - run_exp_job 16 "Sp" "split_exp_runner:121" + run_job_logged 16 run_exp_job 16 "Sp" "split_exp_runner:121" fi (( do_job = job & 32 )) if [[ $do_job != 0 ]]; then # Job 32: mask exposures - run_exp_job 32 "Ma" "mask_runner:40" + run_job_logged 32 run_exp_job 32 "Ma" "mask_runner:40" fi (( do_job = job & 64 )) if [[ $do_job != 0 ]]; then - # Job 64: process stars on exposures, PSF model - if [ "$psf" == "psfex" ]; then - run_exp_job 64 "SxSePsf${Letter}i" "sextractor_runner:80 psfex_runner:80 psfex_interp_runner:40::warn setools_runner:80:rand_split" + # Job 64: PSF model + # For image_sims: fake PSF runs as part of job 512 (requires sexcat from job 256) + # For data: run full exposure-level PSF modelling pipeline + if [ "$type" == "image_sims" ]; then + # Fake PSF is handled inside job 512; write placeholder log so the sequence is complete + log_64="$dir/logs/log_job_${ID}_64.txt" + [ "$force" == "1" ] && rm -f "$log_64" + msg="Complete: job 64 placeholder (fake PSF runs as part of job 512)" + message "$msg" "$debug_out" -1 + echo "$msg" > "$log_64" + elif [ "$psf" == "psfex" ]; then + run_job_logged 64 run_exp_job 64 "SxSePsf${Letter}i" "sextractor_runner:80 psfex_runner:80 psfex_interp_runner:40::warn setools_runner:80:rand_split" else message "MCCD not implemented yet for v2.0" "$debug_out" 10 fi @@ -583,31 +815,81 @@ fi (( do_job = job & 128 )) if [[ $do_job != 0 ]]; then # Job 128: merge exposure WCS headers into tile-level sqlite log - run_tile_job 128 "Mh_exp" "merge_headers_runner:1" + run_job_logged 128 run_tile_job 128 "Mh_exp" "merge_headers_runner:1" fi (( do_job = job & 256 )) if [[ $do_job != 0 ]]; then # Job 256: object selection on tiles if [ "$tile_det" == "uc" ]; then - run_tile_job 256 "Gic Uc" "get_images_runner:2 read_ext_sexcat_runner:1" + run_job_logged 256 run_tile_job 256 "Gic Uc" "get_images_runner:2 read_ext_sexcat_runner:1" else - run_tile_job 256 "Sx" "sextractor_runner:1" + n_exp=2 + run_job_logged 256 run_tile_job 256 "Sx" "sextractor_runner:$n_exp" fi fi (( do_job = job & 512 )) if [[ $do_job != 0 ]]; then - # Job 512: process tiles (PSF interp, vignet) - run_tile_job 512 "${Letter}iViVi ${Letter}iViVi ${Letter}iViVi" "psfex_interp_runner:1 vignetmaker_runner_run_1:1 vignetmaker_runner_run_2:4" + # Job 512: process tiles ([PSF interp,] vignets) + # For image_sims: fake PSF runs first (requires sexcat from job 256), then vignets + log_512="$dir/logs/log_job_${ID}_512.txt" + [ "$force" == "1" ] && rm -f "$log_512" + if [ "$type" == "data" ]; then + if [ "$check_only" == "1" ]; then + out=$(run_tile_job 512 "${Letter}iViVi ${Letter}iViVi ${Letter}iViVi" "psfex_interp_runner:1 vignetmaker_runner_run_1:1 vignetmaker_runner_run_2:4" 2>&1) + echo "$out" + echo "$out" | grep -qiE "(Missing|Incomplete)" || echo "$out" > "$log_512" + else + run_tile_job 512 "${Letter}iViVi ${Letter}iViVi ${Letter}iViVi" "psfex_interp_runner:1 vignetmaker_runner_run_1:1 vignetmaker_runner_run_2:4" + if [ "$run_and_check" == "1" ]; then + check_only=1; local saved_force_512=$force; force=0 + out=$(run_tile_job 512 "${Letter}iViVi ${Letter}iViVi ${Letter}iViVi" "psfex_interp_runner:1 vignetmaker_runner_run_1:1 vignetmaker_runner_run_2:4" 2>&1) + check_only=0; force=$saved_force_512 + echo "$out" + echo "$out" | grep -qiE "(Missing|Incomplete)" || echo "$out" > "$log_512" + else + echo "Completed job 512 $(date)" > "$log_512" + fi + fi + else + if [ "$check_only" == "1" ]; then + out1=$(run_tile_job 64 "fpsf" "fake_psf_runner:1" 2>&1) + out2=$(run_tile_job 512 "ViVi ViVi" "vignetmaker_runner_run_1:1 vignetmaker_runner_run_2:3" 2>&1) + echo "$out1"; echo "$out2" + { echo "$out1"; echo "$out2"; } | grep -qiE "(Missing|Incomplete)" || \ + { echo "$out1"; echo "$out2"; } > "$log_512" + else + run_tile_job 64 "fpsf" "fake_psf_runner:1" + run_tile_job 512 "ViVi ViVi" "vignetmaker_runner_run_1:1 vignetmaker_runner_run_2:3" + if [ "$run_and_check" == "1" ]; then + check_only=1; saved_force_512im=$force; force=0 + out1=$(run_tile_job 64 "fpsf" "fake_psf_runner:1" 2>&1) + out2=$(run_tile_job 512 "ViVi ViVi" "vignetmaker_runner_run_1:1 vignetmaker_runner_run_2:3" 2>&1) + check_only=0; force=$saved_force_512im + echo "$out1"; echo "$out2" + { echo "$out1"; echo "$out2"; } | grep -qiE "(Missing|Incomplete)" || \ + { echo "$out1"; echo "$out2"; } > "$log_512" + else + echo "Completed job 512 $(date)" > "$log_512" + fi + fi + fi fi (( do_job = job & 1024 )) if [[ $do_job != 0 ]]; then # Job 1024: shape measurement - run_tile_job 1024 "Ng" "ngmix_interp_runner:1" + run_job_logged 1024 run_tile_job 1024 "Ng" "ngmix_runner:1" +fi + +(( do_job = job & 2048 )) +if [[ $do_job != 0 ]]; then + # Job 2048: merge catalogues + run_job_logged 2048 run_tile_job 2048 "Mc_${psf}" "make_cat_runner:1" fi + if [ -n "$scratch" ]; then message "Syncing output from scratch back to permanent dir" "$debug_out" -1 command "rsync -a output/ $work_dir/output/" $dry_run diff --git a/src/shapepipe/modules/fake_psf_package/__init__.py b/src/shapepipe/modules/fake_psf_package/__init__.py new file mode 100644 index 000000000..c77dd5658 --- /dev/null +++ b/src/shapepipe/modules/fake_psf_package/__init__.py @@ -0,0 +1,28 @@ +"""FAKE PSF PACKAGE. + +This package creates fake PSF postage-stamp dictionaries for image simulations. + +:Author: Martin Kilbinger + +:Parent module: ``sextractor_runner`` + +:Input: Tile SExtractor catalogue (multi-epoch FITS format) + +:Output: Per-galaxy PSF sqlite dictionary (``galaxy_psf-XXX-XXX.sqlite``) + +Description +=========== + +For each galaxy in the tile catalogue the module looks up the pre-computed +PSF stamp for every contributing exposure-CCD combination from a pickled +PSF dictionary and writes the result to a SqliteDict file in the same format +produced by ``psfex_interp_runner``. + +Module-specific config file entries +==================================== + +PSF_DICT_PATH : str + Path to the pickled PSF dictionary (``Full_psf_dict.pickle``). +""" + +__all__ = ["fake_psf.py"] diff --git a/src/shapepipe/modules/fake_psf_package/fake_psf.py b/src/shapepipe/modules/fake_psf_package/fake_psf.py new file mode 100644 index 000000000..1313b0917 --- /dev/null +++ b/src/shapepipe/modules/fake_psf_package/fake_psf.py @@ -0,0 +1,112 @@ +"""FAKE PSF. + +Create fake PSF postage-stamp dictionaries for image simulations. + +:Author: Martin Kilbinger + +""" + +import pickle + +import numpy as np +from astropy.io import fits +from sqlitedict import SqliteDict + + +class FakePsf: + """Fake PSF. + + Parameters + ---------- + sexcat_path : str + Path to the tile SExtractor catalogue (multi-epoch FITS format). + psf_dict_path : str + Path to the pickled PSF dictionary. + output_path : str + Path for the output SqliteDict file. + w_log : logging.Logger + Pipeline logger. + """ + + def __init__(self, sexcat_path, psf_dict_path, output_path, w_log): + self._sexcat_path = sexcat_path + self._psf_dict_path = psf_dict_path + self._output_path = output_path + self._w_log = w_log + + def process(self): + """Run fake PSF creation.""" + self._w_log.info(f"Reading sexcat: {self._sexcat_path}") + try: + sex = fits.open(self._sexcat_path, ignore_missing_simple=True) + except Exception as e: + self._w_log.error(f"Error opening FITS file {self._sexcat_path}: {e}") + raise + + self._w_log.info(f"Catalogue has {len(sex)} HDUs") + + if len(sex) < 4: + self._w_log.error( + f"Catalogue file has insufficient HDUs: expected ≥4, got {len(sex)} " + f"(file: {self._sexcat_path})" + ) + raise ValueError(f"Invalid catalogue: only {len(sex)} HDUs (need ≥4)") + + self._w_log.info(f"Loading PSF dictionary: {self._psf_dict_path}") + try: + with open(self._psf_dict_path, "rb") as f: + psf_dict = pickle.load(f) + except Exception as e: + self._w_log.error(f"Error loading PSF dictionary {self._psf_dict_path}: {e}") + raise + + try: + n_gal = len(sex[3].data.field("NUMBER")) + except Exception as e: + self._w_log.error( + f"Error reading catalogue data from HDU 3 in {self._sexcat_path}: {e}" + ) + raise + n_exp = len(sex) - 3 + + self._w_log.info(f"Processing {n_gal} galaxies over {n_exp} epochs") + + # Build (n_gal, n_exp) array of "expname-ccdnum" strings + string_array = np.empty((n_gal, n_exp), dtype="object") + for exp_ind in range(3, len(sex)): + ccd_n = sex[exp_ind].data.field("CCD_N") + exp_name = sex[exp_ind].data.field("EXP_NAME") + string_array[:, exp_ind - 3] = [ + f"{int(exp)}-{int(ccd)}" + for exp, ccd in zip(exp_name, ccd_n) + ] + + # Mask invalid entries (exp or ccd == -99 produces strings with "--") + mask = np.array( + [["--" in item for item in row] for row in string_array] + ) + masked = np.ma.masked_array(string_array, mask) + + # Build per-galaxy PSF dictionaries + output_file = SqliteDict(self._output_path) + missing = 0 + for idx, gal_row in enumerate(masked): + galaxy_number = idx + 1 # 1-based, matches NUMBER field + gal_dict = {} + for exp_ccd in gal_row.compressed(): + if exp_ccd not in psf_dict: + missing += 1 + self._w_log.warning( + f"Galaxy {galaxy_number}: key '{exp_ccd}' not in PSF dict" + ) + continue + gal_dict[exp_ccd] = psf_dict[exp_ccd] + output_file[str(galaxy_number)] = gal_dict + + output_file.commit() + output_file.close() + sex.close() + + if missing: + self._w_log.warning(f"{missing} missing PSF entries across all galaxies") + self._w_log.info(f"Written: {self._output_path}") diff --git a/src/shapepipe/modules/fake_psf_runner.py b/src/shapepipe/modules/fake_psf_runner.py new file mode 100644 index 000000000..d3dcbbc0e --- /dev/null +++ b/src/shapepipe/modules/fake_psf_runner.py @@ -0,0 +1,36 @@ +"""FAKE PSF RUNNER. + +Module runner for ``fake_psf``. + +:Author: Martin Kilbinger + +""" + +from shapepipe.modules.fake_psf_package import fake_psf +from shapepipe.modules.module_decorator import module_runner + + +@module_runner( + version="1.0", + file_pattern=["sexcat"], + file_ext=".fits", + depends=["numpy", "astropy", "sqlitedict"], + numbering_scheme="-000-000", +) +def fake_psf_runner( + input_file_list, + run_dirs, + file_number_string, + config, + module_config_sec, + w_log, +): + """Define The Fake PSF Runner.""" + sexcat_path = input_file_list[0] + psf_dict_path = config.getexpanded(module_config_sec, "PSF_DICT_PATH") + output_path = f'{run_dirs["output"]}/galaxy_psf{file_number_string}.sqlite' + + inst = fake_psf.FakePsf(sexcat_path, psf_dict_path, output_path, w_log) + inst.process() + + return None, None diff --git a/src/shapepipe/modules/mask_package/mask.py b/src/shapepipe/modules/mask_package/mask.py index 47354c855..ba6d586ad 100644 --- a/src/shapepipe/modules/mask_package/mask.py +++ b/src/shapepipe/modules/mask_package/mask.py @@ -429,10 +429,14 @@ def make_mask(self): ) # Handle stdout / stderr - general_stdout = f"\nCDSClient\n{self._CDS_stdout}" + # _CDS_stdout/_CDS_stderr are only set when find_stars ran, i.e. + # when HALO_MAKE or SPIKE_MAKE is True (False for image sims) + general_stdout = "" general_stderr = "" - if self._CDS_stderr != "": - general_stderr += f"\nCDSClient\n{self._CDS_stderr}" + if hasattr(self, "_CDS_stdout"): + general_stdout += f"\nCDSClient\n{self._CDS_stdout}" + if self._CDS_stderr != "": + general_stderr += f"\nCDSClient\n{self._CDS_stderr}" if hasattr(self, "_WW_stdout") or hasattr(self, "_WW_stdout"): general_stdout += f"\n\nWeightWatcher\n{self._WW_stdout}" if self._WW_stderr != "": diff --git a/src/shapepipe/modules/merge_headers_runner.py b/src/shapepipe/modules/merge_headers_runner.py index ea9f0d2dd..d7b0bd035 100644 --- a/src/shapepipe/modules/merge_headers_runner.py +++ b/src/shapepipe/modules/merge_headers_runner.py @@ -38,6 +38,7 @@ def merge_headers_runner( # Tile-level mode: input is an exp_numbers txt file; collect header # files from each per-exposure work directory via get_exp_output_files. exp_base_dir = config.getexpanded(module_config_sec, "EXP_BASE_DIR") + # In serial mode several tiles' exp_numbers files can arrive in a # single call; merge each tile into its own per-tile sqlite file. for exp_numbers_file in (item[0] for item in input_file_list): diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 75b5a425b..1c8c2c8a2 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -275,11 +275,10 @@ def __init__( flag_vignet_path, f_wcs_path, bkg_rms_vignet_path=None, - ): self.f_wcs_file = SqliteDict(f_wcs_path) self.gal_vign_cat = SqliteDict(gal_vignet_path) - self.bkg_vign_cat = SqliteDict(bkg_vignet_path) + self.bkg_vign_cat = SqliteDict(bkg_vignet_path) if bkg_vignet_path is not None else None self.psf_vign_cat = SqliteDict(psf_vignet_path) self.weight_vign_cat = SqliteDict(weight_vignet_path) self.flag_vign_cat = SqliteDict(flag_vignet_path) @@ -292,7 +291,8 @@ def __init__( def close(self): self.f_wcs_file.close() self.gal_vign_cat.close() - self.bkg_vign_cat.close() + if self.bkg_vign_cat is not None: + self.bkg_vign_cat.close() self.flag_vign_cat.close() self.weight_vign_cat.close() self.psf_vign_cat.close() @@ -362,26 +362,47 @@ def __init__( save_batch=-1, id_obj_min=-1, id_obj_max=-1, + bkg_sub=True, centroid_source="hsm", seed_from_position=False, metacal_psf="fitgauss", ): - if len(input_file_list) not in {6, 7}: + # Base count = catalogue + vignets, excluding the f_wcs headers (passed + # separately). One fewer when the background vignet is absent + # (``bkg_sub=False``, image sims). An extra trailing slot may carry the + # optional background-rms vignet (#779), so both counts are valid. + n_base = 6 if bkg_sub else 5 + if len(input_file_list) not in {n_base, n_base + 1}: raise IndexError( f"Input file list has length {len(input_file_list)}," - + " required is 6 or 7" + + f" required is {n_base} or {n_base + 1}" ) self._tile_cat_path = input_file_list[0] + if bkg_sub: + bkg_path, psf_path, weight_path, flag_path = ( + input_file_list[2], input_file_list[3], + input_file_list[4], input_file_list[5], + ) + else: + bkg_path, psf_path, weight_path, flag_path = ( + None, input_file_list[2], + input_file_list[3], input_file_list[4], + ) + bkg_rms_vignet_path = ( + input_file_list[n_base] + if len(input_file_list) == n_base + 1 + else None + ) self._vignet_cat = Vignet( input_file_list[1], - input_file_list[2], - input_file_list[3], - input_file_list[4], - input_file_list[5], + bkg_path, + psf_path, + weight_path, + flag_path, f_wcs_path, - input_file_list[6] if len(input_file_list) == 7 else None, + bkg_rms_vignet_path, ) self._output_dir = output_dir @@ -395,6 +416,7 @@ def __init__( self._save_batch = save_batch self._id_obj_min = id_obj_min self._id_obj_max = id_obj_max + self._bkg_sub = bkg_sub self._centroid_source = centroid_source self._seed_from_position = seed_from_position self._metacal_psf = metacal_psf @@ -711,6 +733,61 @@ def check_key(self, expccd_name_tmp, vign_cat, vignet_path): + f" file '{vignet_path}'" ) + def log_mean_ellipticity(self): + """Log mean ellipticity from NOSHEAR HDU to the run log. + + Reports , with standard errors for all objects and for + objects passing the default metacal cuts (flags==0, mcal_flags==0, + 10 < SNR < 500, T/Tpsf > 0.5). + """ + output_path = self.get_output_path(self._output_dir) + try: + with fits.open(output_path) as hdul: + d = hdul['NOSHEAR'].data + g1 = d['g1'].astype(float) + g2 = d['g2'].astype(float) + flags = d['flags'] + mcal_flags = d['mcal_flags'] + s2n = d['s2n'].astype(float) + T = d['T'].astype(float) + Tpsf = d['Tpsf'].astype(float) + except Exception as e: + self._w_log.warning(f"Could not compute mean ellipticity: {e}") + return + + n_total = len(g1) + if n_total == 0: + self._w_log.info("Mean ellipticity: no objects in output catalogue") + return + + def _log_stats(g1_sel, g2_sel, label): + n = len(g1_sel) + if n == 0: + self._w_log.info(f" {label}: 0 objects") + return + mean_g1 = g1_sel.mean() + mean_g2 = g2_sel.mean() + err_g1 = g1_sel.std() / np.sqrt(n) + err_g2 = g2_sel.std() / np.sqrt(n) + self._w_log.info( + f" {label} (N={n}):" + f" = {mean_g1:+.4e} +/- {err_g1:.4e}," + f" = {mean_g2:+.4e} +/- {err_g2:.4e}" + ) + + self._w_log.info(f"Mean ellipticity (NOSHEAR, N_total={n_total}):") + _log_stats(g1, g2, "no cuts") + + with np.errstate(invalid='ignore'): + mask = ( + (flags == 0) + & (mcal_flags == 0) + & (s2n >= 10.0) + & (s2n <= 500.0) + & (T / Tpsf >= 0.5) + ) + _log_stats(g1[mask], g2[mask], "SNR in [10, 500], T/Tpsf > 0.5") + def process(self): """Process. @@ -760,7 +837,7 @@ def process(self): n_empty_cat += 1 continue - stamp = prepare_postage_stamps(vignet_cat, obj_id, i_tile, tile_cat) + stamp = prepare_postage_stamps(vignet_cat, obj_id, i_tile, tile_cat, self._bkg_sub) if len(stamp.gals) == 0: n_no_epoch += 1 @@ -871,9 +948,12 @@ def process(self): # Save results self.save_results(res_dict) -def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat): + # Log mean ellipticity statistics + self.log_mean_ellipticity() + +def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat, bkg_sub=True): # define per-object lists of individual exposures to go into ngmix - stamp = Postage_stamp() + stamp = Postage_stamp(bkg_sub=bkg_sub) #identify exposure and ccd number from psf catalog psf_expccd_names = list(vignet.psf_vign_cat[str(obj_id)].keys()) for expccd_name in psf_expccd_names: @@ -897,6 +977,12 @@ def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat): else: gal_vign_sub_bkg = gal_vign + # Skip epochs where sigma_mad=0 (CCD edge: mostly-zero stamp). + # prepare_ngmix_weights divides by sig_noise; zero sigma causes NaN/inf + # weights that corrupt GalSim's C-level FFT allocations. + if not sigma_mad(gal_vign_sub_bkg) > 0: + continue + tile_vign = ( np.copy(tile_cat.vign[i_tile]) if tile_cat.vign is not None @@ -1147,6 +1233,13 @@ def prepare_ngmix_weights(gal, weight, flag, rng, bkg_rms=None): else sigma_mad(gal) ) + # Guard: sig_noise=0 means galaxy is at the CCD edge (mostly-zero stamp). + # Division by zero would make weight_map NaN/inf, crashing GalSim C code. + # np.all keeps the guard valid for the per-pixel bkg_rms path (any zero + # pixel would produce the same NaN weight the scalar case guards against). + if not np.all(sig_noise > 0): + return np.zeros_like(gal), np.zeros_like(weight_map), np.zeros_like(gal) + noise_img = rng.standard_normal(gal.shape) * sig_noise noise_img_gal = rng.standard_normal(gal.shape) * sig_noise diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 177d608ad..3d98356c4 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -8,6 +8,8 @@ import os +from sqlitedict import SqliteDict + from shapepipe.modules.module_decorator import module_runner from shapepipe.modules.ngmix_package.ngmix import Ngmix @@ -49,9 +51,18 @@ def ngmix_runner( # Pixel scale pixel_scale = config.getfloat(module_config_sec, "PIXEL_SCALE") + # Background subtraction: disable for image sims, where there is no + # background image to subtract. The background vignet occupies one input + # slot, so the f_wcs headers -- and the optional background-rms vignet -- + # shift by one when it is absent. + bkg_sub = config.getboolean(module_config_sec, "BKG_SUB", fallback=True) + wcs_idx = 6 if bkg_sub else 5 + # Path to merged single-exposure single-HDU headers - f_wcs_path = input_file_list[6] + f_wcs_path = input_file_list[wcs_idx] + # Optional background-rms vignet -> ngmix inverse-variance weight map (#779); + # all-or-nothing. Appended as the final positional input after the wcs slice. if config.has_option(module_config_sec, "BKG_RMS_VIGNET_PATH"): bkg_rms_vignet_path = config.getexpanded( module_config_sec, @@ -61,9 +72,9 @@ def ngmix_runner( raise FileNotFoundError( f"Background RMS vignet file not found: {bkg_rms_vignet_path}" ) - input_file_list = input_file_list[:6] + [bkg_rms_vignet_path] + input_file_list = input_file_list[:wcs_idx] + [bkg_rms_vignet_path] else: - input_file_list = input_file_list[:6] + input_file_list = input_file_list[:wcs_idx] # Batch save option if config.has_option(module_config_sec, "SAVE_BATCH"): @@ -79,10 +90,10 @@ def ngmix_runner( id_obj_min = config.getint(module_config_sec, "ID_OBJ_MIN") id_obj_max = config.getint(module_config_sec, "ID_OBJ_MAX") - # Centroid source for the galaxy Jacobian origin: "wcs" (default — the + # Centroid source for the galaxy Jacobian origin: "wcs" (default -- the # catalog sky position projected through the WCS, trusting the astrometry) # or "hsm" (legacy HSM adaptive-moment centroid, being phased out: noisy - # for stars and flagged as incorrect by Fabian — see #767). + # for stars and flagged as incorrect by Fabian -- see #767). if config.has_option(module_config_sec, "CENTROID_SOURCE"): centroid_source = config.get(module_config_sec, "CENTROID_SOURCE") else: @@ -99,6 +110,34 @@ def ngmix_runner( else: seed_from_position = False + # Check PSF vignets first: if all are empty dicts {}, the exposures for this + # tile are absent from the PSF dictionary and no shape measurement is possible. + # This check must come before reading image vignets to avoid a C-level malloc + # crash that occurs when large numbers of numpy arrays are allocated then freed. + psf_idx = 3 if bkg_sub else 2 + psf_vignet_path = input_file_list[psf_idx] + with SqliteDict(psf_vignet_path) as db: + psf_keys = list(db.keys()) + n_empty_psf = sum(1 for k in psf_keys if len(db[k]) == 0) + if psf_keys and n_empty_psf == len(psf_keys): + w_log.warning( + f"All {len(psf_keys)} PSF vignet entries are empty in " + f"{psf_vignet_path} — no PSF coverage for this tile. Skipping ngmix." + ) + return None, None + + # Check that image vignets are not all empty before initialising ngmix + image_vignet_path = input_file_list[1] + with SqliteDict(image_vignet_path) as db: + keys = list(db.keys()) + n_empty = sum(1 for k in keys if db[k] == "empty") + if keys and n_empty == len(keys): + w_log.warning( + f"All {len(keys)} image vignets are 'empty' in {image_vignet_path} " + "— no valid CCD coverage for this tile. Skipping ngmix." + ) + return None, None + # Metacal reconvolution-kernel scheme (metacal_pars['psf']): "fitgauss" # (default; fit a Gaussian to the PSF and round it), "gauss" (fixed round # Gaussian sized from the PSF), "dilate" (dilate the original PSF), or @@ -120,6 +159,7 @@ def ngmix_runner( save_batch=save_batch, id_obj_min=id_obj_min, id_obj_max=id_obj_max, + bkg_sub=bkg_sub, centroid_source=centroid_source, seed_from_position=seed_from_position, metacal_psf=metacal_psf, diff --git a/src/shapepipe/modules/sextractor_package/sextractor_script.py b/src/shapepipe/modules/sextractor_package/sextractor_script.py index 5f4fe44a4..ad5a83bcb 100644 --- a/src/shapepipe/modules/sextractor_package/sextractor_script.py +++ b/src/shapepipe/modules/sextractor_package/sextractor_script.py @@ -157,9 +157,13 @@ def make_post_process(cat_path, f_wcs_path, pos_params, ccd_size, w_log=None): history.append(idx) exp_list = [] - pattern = r"([0-9]*)p\.(.*)" + pattern = r"([0-9]+)p?\.(.*)" for hist in history: m = re.search(pattern, hist) + if m is None: + raise ValueError( + f"Could not parse exposure ID from HISTORY entry: '{hist}'" + ) exp_list.append(m.group(1)) obj_id = np.copy(cat.get_data()["NUMBER"]) diff --git a/src/shapepipe/modules/vignetmaker_package/vignetmaker.py b/src/shapepipe/modules/vignetmaker_package/vignetmaker.py index 74e26915b..505c037dd 100644 --- a/src/shapepipe/modules/vignetmaker_package/vignetmaker.py +++ b/src/shapepipe/modules/vignetmaker_package/vignetmaker.py @@ -323,7 +323,8 @@ def _get_stamp_me(self, image_dirs, image_pattern): else: self._w_log.info( f"All CCDs skipped for exposure {exp_name} " - + "(ccd == -1 for all); epoch excluded from output" + + f"[{image_pattern}]" + + " (ccd == -1 for all); epoch excluded from output" ) cat.close() @@ -380,6 +381,7 @@ def process_me(self, image_dirs, image_pattern, f_wcs_path, rad): else: index = idx + self._w_log.info(f"Processing {image_pattern[idx]} vignets") output_dict = self._get_stamp_me( image_dirs[index], image_pattern[idx], diff --git a/src/shapepipe/pipeline/exp_utils.py b/src/shapepipe/pipeline/exp_utils.py index 8f0df784a..b934f0bb3 100644 --- a/src/shapepipe/pipeline/exp_utils.py +++ b/src/shapepipe/pipeline/exp_utils.py @@ -22,6 +22,7 @@ def get_exp_output_files( file_pattern, file_ext, w_log=None, + warn_only=False, ): """Collect output files from a per-exposure runner for all tile exposures. @@ -31,9 +32,9 @@ def get_exp_output_files( ///output/run_sp_*//output/ - where ``exp_prefix = exp_id[:2]`` and ``exp_base = exp_id[:-1]`` (the - trailing letter, typically ``p``, is stripped because directory names - do not carry it). + where ``exp_prefix = exp_id[:2]`` and ``exp_base`` is ``exp_id`` with the + trailing letter stripped if present (e.g. ``2113864p`` → ``2113864``), or + the full ``exp_id`` for numeric-only IDs (image simulations). Parameters ---------- @@ -52,6 +53,10 @@ def get_exp_output_files( File extension including the leading dot, e.g. ``.npy`` w_log : logging.Logger, optional Pipeline logger; ``None`` silences all logging + warn_only : bool, optional + If ``True``, log a warning for missing exposures and continue with + the files that were found instead of raising ``FileNotFoundError``. + Default is ``False``. Returns ------- @@ -88,9 +93,10 @@ def get_exp_output_files( for exp_id in exp_ids: # Directory structure mirrors run_job_canfar_v2.0.sh: # exp_prefix = first 2 chars of exp_id (e.g. "21") - # exp_base = exp_id without trailing letter (e.g. "2113864") + # exp_base = exp_id without trailing letter if present (e.g. "2113864"), + # or full exp_id for numeric-only ids (image sims) exp_prefix = exp_id[:2] - exp_base = exp_id[:-1] + exp_base = exp_id[:-1] if exp_id[-1].isalpha() else exp_id pattern = os.path.join( exp_base_dir, @@ -115,10 +121,12 @@ def get_exp_output_files( w_log.warning(f" {exp_id}: no match for {pattern}") if missing: - raise FileNotFoundError( - f"No {runner_name} output found for " - f"{len(missing)} exposure(s): {missing}" - ) + msg = f"No {runner_name} output found for {len(missing)} exposure(s): {missing}" + if warn_only: + if w_log: + w_log.warning(msg) + else: + raise FileNotFoundError(msg) if w_log: w_log.info(f"Found {len(file_list)} exposure output files") @@ -183,7 +191,7 @@ def get_exp_output_dirs( for exp_id in exp_ids: exp_prefix = exp_id[:2] - exp_base = exp_id[:-1] + exp_base = exp_id[:-1] if exp_id[-1].isalpha() else exp_id pattern = os.path.join( exp_base_dir, diff --git a/src/shapepipe/testing/simulate.py b/src/shapepipe/testing/simulate.py index 4f263508e..bb1c7823b 100644 --- a/src/shapepipe/testing/simulate.py +++ b/src/shapepipe/testing/simulate.py @@ -152,3 +152,98 @@ def draw_shift(): if return_centers: return gals, psfs, psfs_sigmas, weights, flags, jacob_lists, centers return gals, psfs, psfs_sigmas, weights, flags, jacob_lists + + +def make_data_pujol( + rng, + shear_list, + noise=1e-5, + n_epochs=1, + share_shift=False, + gal_hlr=0.3, + gal_flux=1000.0, + psf_fwhm=0.55, + pixel_scale=0.1857, + img_size=201, +): + """Simulate galaxies at multiple shear values sharing the same noise per epoch. + + Implements the Pujol (2018) denoising approach: all shear variants receive + the identical noise and sub-pixel shift each epoch, so noise contributions + to the shear response R and to the m/c estimator cancel exactly. + + Parameters + ---------- + rng : numpy.random.RandomState + shear_list : list of tuple of float + List of (g1, g2) values to simulate simultaneously. + noise : float, optional + Per-pixel noise sigma. + n_epochs : int, optional + share_shift : bool, optional + If True all epochs share the same sub-pixel shift; the shift is + always shared across shear variants regardless of this flag. + gal_hlr, gal_flux, psf_fwhm, pixel_scale, img_size : float / int, optional + Same meaning as in make_data. + + Returns + ------- + list of tuple + One ``(gals, psfs, psfs_sigmas, weights, flags, jacob_lists)`` per + entry in ``shear_list``, all sharing identical per-epoch noise. + """ + psf_noise_sigma = 1.0e-6 + scale = pixel_scale + wcs = galsim.PixelScale(scale) + n_shears = len(shear_list) + + # output accumulators + all_gals = [[] for _ in range(n_shears)] + all_psfs = [[] for _ in range(n_shears)] + all_psfs_sigmas = [[] for _ in range(n_shears)] + all_weights = [[] for _ in range(n_shears)] + all_flags = [[] for _ in range(n_shears)] + all_jacobs = [[] for _ in range(n_shears)] + + if share_shift: + dy, dx = rng.uniform(low=-scale / 2, high=scale / 2, size=2) + + for epoch in range(n_epochs): + if not share_shift: + dy, dx = rng.uniform(low=-scale / 2, high=scale / 2, size=2) + + # One noise draw shared by all shear variants this epoch + noise_img = rng.normal(scale=noise, size=(img_size, img_size)) + psf_noise_img = rng.normal(scale=psf_noise_sigma, size=(img_size, img_size)) + + psf = galsim.Moffat(beta=2.5, fwhm=psf_fwhm) + psf_im_ = psf.drawImage(nx=img_size, ny=img_size, wcs=wcs) + psf_sigma = galsim.hsm.FindAdaptiveMom(psf_im_).moments_sigma + psf_im = psf_im_.array.astype(np.float64) + psf_noise_img + + jacob = wcs.jacobian() + weight = np.full((img_size, img_size), 1.0 / noise ** 2) + flag = np.zeros((img_size, img_size)) + + for j, shear in enumerate(shear_list): + obj = galsim.Convolve( + psf, + galsim.Exponential(half_light_radius=gal_hlr, flux=gal_flux).shear( + g1=shear[0], g2=shear[1] + ), + ).shift(dx, dy) + im = obj.drawImage(nx=img_size, ny=img_size, wcs=wcs).array.astype(np.float64) + im += noise_img # same noise for every shear variant + + all_gals[j].append(im) + all_psfs[j].append(psf_im) + all_psfs_sigmas[j].append(psf_sigma) + all_weights[j].append(weight.copy()) + all_flags[j].append(flag.copy()) + all_jacobs[j].append(jacob) + + return [ + (all_gals[j], all_psfs[j], all_psfs_sigmas[j], + all_weights[j], all_flags[j], all_jacobs[j]) + for j in range(n_shears) + ]