From ee7b6e77a78575a7bc84eaf5f18c9cee07d76fd1 Mon Sep 17 00:00:00 2001 From: Victoria-Samboco <95472901+Victoria-Samboco@users.noreply.github.com> Date: Thu, 2 May 2024 01:23:55 +0200 Subject: [PATCH] Update solarkat.yaml These new updates incorporate cult-cargo into the workflow. --- solarkat-pipeline/solarkat.yaml | 404 +++++++++++++++++--------------- 1 file changed, 217 insertions(+), 187 deletions(-) diff --git a/solarkat-pipeline/solarkat.yaml b/solarkat-pipeline/solarkat.yaml index c2eec6c..82bf747 100644 --- a/solarkat-pipeline/solarkat.yaml +++ b/solarkat-pipeline/solarkat.yaml @@ -1,7 +1,7 @@ #!/usr/bin/env -S stimela run _include: - - solarkat-cabs.yaml + - solarkat-cabs.yaml cabs: @@ -12,22 +12,62 @@ cabs: do_continue: dtype: bool nom_de_guerre: continue - nwlayers_factor: - dtype: int - join_channels: + nwlayers-factor: + dtype: float + join-channels: dtype: bool - fit_spectral_pol: + fit-spectral-pol: dtype: int auto-threshold: dtype: str - - breizorro: - virtual_env: '/home/samboco/Virtual_enviroments/breizorro_env' + scale: + dtype: str + size: + dtype: int + padding: + dtype: float + nchan: + dtype: int + temp_dir: + dtype: Directory + use-wgridder: + dtype: bool + log-time: + dtype: bool + ms: + dtype: MS + prefix: + dtype: str + column: + dtype: str + weight: + dtype: str + niter: + dtype: int + mgain: + dtype: float + fits-mask: + dtype: str + save-source-list: + dtype: bool + multiscale: + dtype: bool + predict: + dtype: bool + threshold: + dtype: float + versionname: + dtype: str + restored-image: + dtype: File + name: + dtype: str casa: command: overwrite +# override default imaging settings from oms-cabs lib: steps: wsclean: @@ -35,18 +75,19 @@ lib: params: ms: '{recipe.ms}' prefix: '{recipe.image-prefix}' - size: [16300, 16300] + size: [1024, 1024] scale: 1.5asec - -#This augments the standard 'opts' config section to tweak logging settings +# This augments the standard 'opts' config section to tweak logging settings opts: log: dir: './{root.dir_out}/logs/log-{run.datetime}' name: log-{info.fqname}.txt nest: 2 symlink: log + backend: + select: native solarkat: @@ -60,7 +101,7 @@ solarkat: assign_based_on: obs: L1: - ms: 'msdir/1645272078_sdp_l0_1024ch_J0333-2741.ms' + ms: msdir/1645272078_sdp_l0_1024ch_J0333-2741.ms dir_out: obs1 perscan_dir_out: "ms_scans" sun_images_dir: "sun_images" @@ -78,7 +119,7 @@ solarkat: band: L L2: - ms: 'msdir/1671435077_sdp_l0_1024ch_GRS1747-312.ms' + ms: /home/samboco/blfast_imaging/parrot-stew-recipes/msdir/1671435077_sdp_l0_1024ch_GRS1747-312.ms dir_out: obs2 perscan_dir_out: "ms_scans" sun_images_dir: "sun_images" @@ -96,7 +137,7 @@ solarkat: band: L UHF: - ms: 'msdir/1583662427_sdp_l0.1024ch_1024ch_J033230-280757.ms' + ms: msdir/1583662427_sdp_l0.1024ch_1024ch_J033230-280757.ms dir_out: obs3 perscan_dir_out: "ms_scans" sun_images_dir: "sun_images" @@ -131,9 +172,82 @@ solarkat: qcal_output_dir: 'peeled.qc' band: L + L4: + ms: '/home/samboco/solarKAT/SUN_IMAGING_STEPS/pipeline/msdir/1630830331_sdp_l0_719ch_SN2021smj.ms' + dir_out: obs1630 + perscan_dir_out: "ms_scans" + sun_images_dir: "sun_images" + regions_dir_out: "region_files" + masks_dir_out: "sun_masks" + clean_image_sun: "clean_image_sun" + selfcal_subtracted: 'self_sun_subtracted' + quality_control: 'quality_control_subt_images' + quality_control1: 'quality_control_subt_images1' + quality_control2: 'quality_control_subt_images2' + quality_control3: 'quality_control_subt_images3' + residual_vis_dir: 'residual_vis' + concat_image: 'concat_image' + qcal_output_dir: 'peeled.qc' + band: L + + L5: + ms: '/home/samboco/solarKAT/SUN_IMAGING_STEPS/pipeline/msdir/1694074581_sdp_l0_719ch_SN2021smj.ms' + dir_out: obs1694 + perscan_dir_out: "ms_scans" + sun_images_dir: "sun_images" + regions_dir_out: "region_files" + masks_dir_out: "sun_masks" + clean_image_sun: "clean_image_sun" + selfcal_subtracted: 'self_sun_subtracted' + quality_control: 'quality_control_subt_images' + quality_control1: 'quality_control_subt_images1' + quality_control2: 'quality_control_subt_images2' + quality_control3: 'quality_control_subt_images3' + residual_vis_dir: 'residual_vis' + concat_image: 'concat_image' + qcal_output_dir: 'peeled.qc' + band: L + + L6: + ms: '/home/samboco/solarKAT/SUN_IMAGING_STEPS/pipeline/msdir/1695970273_sdp_l0_719ch_SN2021smj.ms' + dir_out: obs81695 + perscan_dir_out: "ms_scans" + sun_images_dir: "sun_images" + regions_dir_out: "region_files" + masks_dir_out: "sun_masks" + clean_image_sun: "clean_image_sun" + selfcal_subtracted: 'self_sun_subtracted' + quality_control: 'quality_control_subt_images' + quality_control1: 'quality_control_subt_images1' + quality_control2: 'quality_control_subt_images2' + quality_control3: 'quality_control_subt_images3' + residual_vis_dir: "residual_vis" + concat_image: 'concat_image' + qcal_output_dir: 'peeled.qc' + band: UHF + + L7: + ms: '/home/samboco/solarKAT/SUN_IMAGING_STEPS/pipeline/msdir/1696746197_sdp_l0_719ch_SN2021smj.ms' + dir_out: obs91696 + perscan_dir_out: "ms_scans" + sun_images_dir: "sun_images" + regions_dir_out: "region_files" + masks_dir_out: "sun_masks" + clean_image_sun: "clean_image_sun" + selfcal_subtracted: 'self_sun_subtracted' + quality_control: 'quality_control_subt_images' + quality_control1: 'quality_control_subt_images1' + quality_control2: 'quality_control_subt_images2' + quality_control3: 'quality_control_subt_images3' + residual_vis_dir: "residual_vis" + concat_image: 'concat_image' + qcal_output_dir: 'peeled.qc' + band: L + + inputs: obs: - choices: [L1, L2, UHF, L3] + choices: [L1, L2, UHF, L3, L4, L5, L6, L7] info: "Selects observation, see labels above" suffix: dtype: str @@ -159,28 +273,28 @@ solarkat: column: DATA niter: 50000 auto-threshold: 3 - size: [8000, 8000] + size: 5500 temp_dir: '{root.dir_out}/tmp' mask-1: - cab: breizorro + cab: breizorro info: 'Creating a fits mask with breizorro' params: threshold: 15 mask: '{previous.prefix}-mask.fits' - restored_image: "{previous.restored-mfs}" - + restored-image: "{previous.restored.mfs}" + image-2: info: "Breizorro-masked deep I clean" _use: lib.steps.wsclean.image params: - padding: 2 + padding: 2.0 column: DATA niter: 100000 auto-threshold: 3 - size: [8000, 8000] + size: 5500 fits-mask: "{previous.mask}" temp_dir: '{root.dir_out}/tmp' @@ -190,35 +304,35 @@ solarkat: info: 'Saving the flags after 1GC' params: ms: '{recipe.ms}' - name: "after 1GC" + versionname: "after 1GC" mode: save - selfcal_1: - cab: quartical - info: "Self-calibration" - params: - input_ms.path: '{recipe.ms}' - input_ms.time_chunk: '128' - input_model.recipe: MODEL_DATA - solver.terms: [K] - solver.iter_recipe: [100] - K.type: phase - K.freq_interval: '0' - K.time_interval: '4' - K.initial_estimate: true - K.direction_dependent: false - output.overwrite: true - output.products: [corrected_data] - output.columns: [CORRECTED_DATA] + # selfcal_1: + # info: "Self-calibration" + # cab: quartical + # params: + # input_ms.path: '{recipe.ms}' + # input_ms.time_chunk: '128' + # input_model.recipe: MODEL_DATA + # solver.terms: [K] + # solver.iter_recipe: [100] + # K.type: phase + # K.freq_interval: '0' + # K.time_interval: '4' + # K.initial_estimate: true + # K.direction_dependent: false + # output.overwrite: true + # output.products: [corrected_data] + # output.columns: [CORRECTED_DATA] save-flags-2: - cab: flagman + cab: casa.flagman info: 'Saving flags after 2GC' params: ms: '{recipe.ms}' - name: "after 2GC" + versionname: "after 2GC" mode: save @@ -226,9 +340,9 @@ solarkat: info: "Breizorro-masked deep I clean" _use: lib.steps.wsclean.image params: - padding: 2 + padding: 2.0 niter: 100000 - size: [8000, 8000] + size: 5500 column: CORRECTED_DATA auto-threshold: =UNSET save-source-list: true @@ -242,8 +356,8 @@ solarkat: params: ms: '{recipe.ms}' within: '{root.dir_out}/im3/bsource.reg' - output_column: DIR1_DATA - sky_model: '{root.dir_out}/im3/im3-sources.txt' + output-column: DIR1_DATA + sky-model: '{root.dir_out}/im3/im3-sources.txt' peeling_source: @@ -289,12 +403,12 @@ solarkat: save-flags-3: - cab: flagman + cab: casa.flagman info: 'Restoring the flags after 1GC, when needed.' skip: true params: ms: '{recipe.ms}' - name: "after 3GC" + versionname: "after 3GC" mode: save @@ -302,9 +416,9 @@ solarkat: info: "Breizorro-masked deep I clean" _use: lib.steps.wsclean.image params: - padding: 2 + padding: 2.0 niter: 100000 - size: [8000, 8000] + size: 5500 auto-threshold: =UNSET column: CORRECTED_DATA temp_dir: '{root.dir_out}/tmp' @@ -317,20 +431,22 @@ solarkat: params: threshold: 15 mask: '{previous.prefix}-mask.fits' - restored_image: '{previous.restored-mfs}' + restored-image: '{previous.restored.mfs}' image-5: info: "Breizorro-masked deep I clean" _use: lib.steps.wsclean.image params: - padding: 2 + padding: 2.0 niter: 100000 auto-threshold: 3 - size: [8000, 8000] + size: 5500 column: CORRECTED_DATA temp_dir: '{root.dir_out}/tmp' - fits-mask: "{steps.image-3.prefix}-mask.fits" + fits-mask: "{previous.mask}" + + backup_model_data: cab: rename_model_data @@ -339,11 +455,11 @@ solarkat: params: ms: ="{recipe.ms}" oldname: 'MODEL_DATA' - newname: 'MODEL_DATA_ORIGINAL' + newname: 'MODEL_DATA_ORIGINAL' scan_numbers_extraction: - info: 'Extract a list of unique scan numbers from a Measurement Set. The scan output parameter is a list of strings representing the unique scan numbers.' + info: 'Extract a list of unique scan numbers from a Seasurement Set. The scans output parameter is a list of strings representing the unique scan numbers.' cab: extract_and_save_scan_numbers params: ms: '{recipe.ms}' @@ -382,7 +498,7 @@ solarkat: get_perscan_old_coords: cab: get_old_coords - info: 'Get the phase centre coordinates of the scan's main field.' + info: 'Get the phase centre coordinates of the scans main field.' params: output_file: '{recipe.dir_out}/old_perscans_coods.txt' ms_list: =GLOB("{recipe.dir_out}/{recipe.perscan_dir_out}/*_scan_*") @@ -403,6 +519,8 @@ solarkat: ms_list: =GLOB("{recipe.dir_out}/{recipe.perscan_dir_out}/*_scan_*") coords: '{recipe.steps.get_sun_coordinates.output_file}' splitted_ms_dir: '{recipe.dir_out}/{recipe.perscan_dir_out}/' + datacolumn: 'all' + image_sun: @@ -421,10 +539,11 @@ solarkat: _use: lib.steps.wsclean.image params: niter: 0 - size: 6000 + size: 5500 ms: '{recipe.mss}' column: CORRECTED_DATA temp_dir: '{root.dir_out}/tmp' + save-source-list: =UNSET prefix: ="{root.dir_out}/{root.perscan_dir_out}/{root.sun_images_dir}/" + STRIPEXT(BASENAME(recipe.mss)) @@ -465,12 +584,13 @@ solarkat: make_mask: cab: breizorro params: - restored_image: ="{root.dir_out}/{root.perscan_dir_out}/{root.sun_images_dir}/" + STRIPEXT(BASENAME(root.ms)) + "_scan_{recipe.scan}-MFS-image.fits" - threshold: 9999999999999999999999999999999 + restored-image: ="{root.dir_out}/{root.perscan_dir_out}/{root.sun_images_dir}/" + STRIPEXT(BASENAME(root.ms)) + "_scan_{recipe.scan}-MFS-image.fits" + threshold: 999999999999 merge: ="{root.dir_out}" + "/" + "{root.regions_dir_out}" + "/" + "sun_region_{recipe.scan}.reg" mask: ="{root.dir_out}" + "/" + "{root.masks_dir_out}" + "/" + STRIPEXT(BASENAME(root.ms)) + "_scan_{recipe.scan}-mask.fits" + deconvolve_sun: info: " clean imaging of the Sun for a better model, using the masks created from the make_mask step." params: @@ -490,12 +610,12 @@ solarkat: _use: lib.steps.wsclean.image params: ms: '{recipe.mss}' - size: 6000 + size: 5500 niter: 100000 multiscale: true threshold: 3.55e-4 - join_channels: true - fit_spectral_pol: 4 + join-channels: true + fit-spectral-pol: 4 auto-threshold: =UNSET save-source-list: true column: CORRECTED_DATA @@ -504,6 +624,7 @@ solarkat: fits-mask: ="{root.dir_out}" + "/" + "{root.masks_dir_out}" + "/" + STRIPEXT(BASENAME(recipe.mss)) + "-mask.fits" + predict_sun_model: info: " Predict Sun Model to the MODEL_DATA column." params: @@ -523,35 +644,12 @@ solarkat: predict: _use: lib.steps.wsclean.predict params: - size: 6000 + size: 5500 predict: true ms: '{recipe.ms}' temp_dir: '{root.dir_out}/tmp' - prefix: ="{root.dir_out}/{root.perscan_dir_out}/{root.clean_image_sun}/" + STRIPEXT(BASENAME(recipe.ms)) #+ "-MFS" - - - quality_control_imaging1: - info: "Image Sun field after subtraction. This step serves to inspect the subtration process" - params: - ms_list: =GLOB('{recipe.dir_out}/{recipe.perscan_dir_out}/*_scan_*.ms') - recipe: - inputs: - ms_list: - dtype: List[MS] - for_loop: - var: mss - over: ms_list - steps: - image: - _use: lib.steps.wsclean.image - params: - ms: '{recipe.mss}' - niter: 0 - column: MODEL_DATA - size: [8000, 8000] - temp_dir: '{root.dir_out}/tmp' - prefix: ="{root.dir_out}/{root.perscan_dir_out}/{root.quality_control}/" + STRIPEXT(BASENAME(recipe.mss)) - + prefix: ="{root.dir_out}/{root.perscan_dir_out}/{root.clean_image_sun}/" + STRIPEXT(BASENAME(recipe.ms)) + rephase: cab: shift_coords @@ -560,53 +658,9 @@ solarkat: ms_list: =GLOB("{recipe.dir_out}/{recipe.perscan_dir_out}/*_scan_*") coords: '{recipe.steps.get_perscan_old_coords.output_file}' splitted_ms_dir: '{recipe.dir_out}/{recipe.perscan_dir_out}/' + datacolumn: 'all' - quality_control_imaging2: - info: "Image Sun field after subtraction. This step serves to inspect the subtration process" - params: - ms_list: =GLOB('{recipe.dir_out}/{recipe.perscan_dir_out}/*_scan_*.ms') - recipe: - inputs: - ms_list: - dtype: List[MS] - for_loop: - var: mss - over: ms_list - steps: - image: - _use: lib.steps.wsclean.image - params: - ms: '{recipe.mss}' - niter: 0 - size: [8000, 8000] - column: CORRECTED_DATA - temp_dir: '{root.dir_out}/tmp' - prefix: ="{root.dir_out}/{root.perscan_dir_out}/{root.quality_control2}/" + STRIPEXT(BASENAME(recipe.mss)) - - - quality_control_imaging3: - info: "Image Sun field after subtraction. This step serves to inspect the subtration process" - params: - ms_list: =GLOB('{recipe.dir_out}/{recipe.perscan_dir_out}/*_scan_*.ms') - recipe: - inputs: - ms_list: - dtype: List[MS] - for_loop: - var: mss - over: ms_list - steps: - image: - _use: lib.steps.wsclean.image - params: - ms: '{recipe.mss}' - niter: 0 - size: [8000, 8000] - column: MODEL_DATA - temp_dir: '{root.dir_out}/tmp' - prefix: ="{root.dir_out}/{root.perscan_dir_out}/{root.quality_control3}/" + STRIPEXT(BASENAME(recipe.mss)) - add_model_data_columnn: cab: add_model_data_column @@ -629,7 +683,7 @@ solarkat: subtract_sun: info: 'Subtracting the SOLAR_MODEL from the visibilities (CORRECTED_DATA) in the original MS' - cab: taql_update + cab: taql.update params: ms: '{recipe.ms}' commands: =LIST("set", "CORRECTED_DATA_SUN=CORRECTED_DATA-MODEL_DATA_SUN") @@ -640,47 +694,24 @@ solarkat: _use: lib.steps.wsclean.image params: ms: '{recipe.ms}' - padding: 2 + padding: 2.0 niter: 100000 auto-threshold: 3 - size: [8000, 8000] + size: 5500 column: CORRECTED_DATA_SUN temp_dir: '{root.dir_out}/tmp' - fits-mask: '{recipe.dir_out}/im3/im3-mask.fits' - - - image-6a: - info: "Quality_control: Image the improved model" - _use: lib.steps.wsclean.image - params: - ms: '{recipe.ms}' - niter: 0 - column: MODEL_DATA - size: [8000, 8000] - temp_dir: '{root.dir_out}/tmp' - -#From this point we have the improved MODEL_DATA of the main field without the Sun (which will be used in the peeling step with the MODEL_DATA_SUN column) - - image-6b: - info: "quality_control: Image the model of the Sun" - _use: lib.steps.wsclean.image - params: - ms: '{recipe.ms}' - niter: 0 - size: [8000, 8000] - column: MODEL_DATA_SUN - temp_dir: '{root.dir_out}/tmp' + fits-mask: '{recipe.dir_out}/im4/im4-mask.fits' save-flags-4: - cab: flagman + cab: casa.flagman params: ms: '{recipe.ms}' - name: "Before 3GC" - mode: save #/restore - - - cal_and_peel_sol: + versionname: "Before 3GC" + mode: save + + + cal_and_peel_sun: cab: quartical info: 'Selfcal + peel sun' params: @@ -690,7 +721,7 @@ solarkat: input_ms.freq_chunk: '0' input_ms.select_uv_range: [0,0] input_ms.group_by: [FIELD_ID,DATA_DESC_ID,SCAN_NUMBER] - input_model.recipe: MODEL_DATA:MODEL_DATA_SUN + input_model.recipe: MODEL_DATA:DIR1_DATA:MODEL_DATA_SUN input_model.apply_p_jones: false input_ms.is_bda: False solver.terms: [K,dE] @@ -704,7 +735,7 @@ solarkat: output.overwrite: true output.products: [corrected_residual] output.columns: [CORRECTED_RESIDUAL] - output.subtract_directions: [1] + output.subtract_directions: [1,2] output.flags: true output.apply_p_jones_inv: false mad_flags.enable: false @@ -723,10 +754,10 @@ solarkat: save-flags-5: - cab: flagman + cab: casa.flagman params: ms: '{recipe.ms}' - name: "After 3GC" + versionname: "After 3GC" mode: save @@ -734,35 +765,34 @@ solarkat: info: "Deconvolve the improved image" _use: lib.steps.wsclean.image params: - padding: 2 + padding: 2.0 niter: 100000 - auto-threshold: 3 - size: [8000, 8000] + auto-threshold: =UNSET + size: 5500 column: CORRECTED_RESIDUAL temp_dir: '{root.dir_out}/tmp' - fits-mask: '{recipe.dir_out}/im3/im3-mask.fits' - - - mask-4: + fits-mask: '{recipe.dir_out}/im4/im4-mask.fits' + + + mask-3: cab: breizorro - info: 'Creating a fits mask with breizorro' + info: 'Creating a fits mask with breizorro.' params: - restored_image: "{previous.restored-mfs}" + threshold: 20 mask: '{previous.prefix}-mask.fits' - threshold: 30 + restored-image: "{previous.restored.mfs}" - image-8: + image-8: info: "Deconvolve the improved image" _use: lib.steps.wsclean.image params: - padding: 2 + padding: 2.0 niter: 100000 - auto-threshold: 3 - size: [8000, 8000] + auto-threshold: =UNSET + size: 5500 column: CORRECTED_RESIDUAL temp_dir: '{root.dir_out}/tmp' - fits-mask: '{previous.mask}' - - + fits-mask: "{previous.mask}" +