Skip to content

Ensure that the fallback to pass through long baseline antennas with too much flagging works with the updates to have per-SPW caltables #11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
psheehan wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Ensure that the fallback to pass through long baseline antennas with too much flagging works with the updates to have per-SPW caltables #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2ff8008
Ensure the unflagging fallback always uses combinespw and removes non…
psheehan Jan 30, 2024
9c9d857
Ensure that the option for antennas to fall back to the last successf…
psheehan Feb 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions gaincal_wrapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -484,34 +484,36 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, ap
# reference antenna.
if unflag_fb_to_prev_solint:
for it, sint in enumerate(selfcal_plan['solints'][0:iteration+1]):
if not os.path.exists(sani_target+'_'+vis+'_'+band+'_'+sint+'_'+str(it)+'_'+selfcal_plan['solmode'][it]+'_'+selfcal_library[vis][solint]['final_mode']+'.g'):
if not os.path.exists(sani_target+'_'+vis+'_'+band+'_'+sint+'_'+str(it)+'_'+selfcal_plan['solmode'][it]+'_'+selfcal_library[vis][sint]['final_mode']+'.g') or 'combinespw' not in selfcal_library[vis][sint]['final_mode']:
continue

# If a previous iteration went through the unflagging routine, it is possible that some antennas fell back to
# a previous solint. In that case, rerefant will flag those antennas because they can't be re-referenced with
# a different time interval. So to be safe, we go back to the pre-pass solutions and then re-run the passing.
# We could probably check more carefully whether this is the case to avoid having to do this... but the
# computing time isn't significant so it's easy just to run through again.
if os.path.exists(sani_target+'_'+vis+'_'+band+'_'+sint+'_'+str(it)+'_'+selfcal_plan['solmode'][it]+'_'+selfcal_library[vis][solint]['final_mode']+'.pre-pass.g'):
rerefant(vis, sani_target+'_'+vis+'_'+band+'_'+sint+'_'+str(it)+'_'+selfcal_plan['solmode'][it]+'_'+selfcal_library[vis][solint]['final_mode']+'.pre-pass.g', \
if os.path.exists(sani_target+'_'+vis+'_'+band+'_'+sint+'_'+str(it)+'_'+selfcal_plan['solmode'][it]+'_'+selfcal_library[vis][sint]['final_mode']+'.pre-pass.g'):
rerefant(vis, sani_target+'_'+vis+'_'+band+'_'+sint+'_'+str(it)+'_'+selfcal_plan['solmode'][it]+'_'+selfcal_library[vis][sint]['final_mode']+'.pre-pass.g', \
refant=selfcal_library[vis]["refant"], refantmode=refantmode if 'inf_EB' not in sint else 'flex')

os.system("rm -rf "+sani_target+'_'+vis+'_'+band+'_'+sint+'_'+str(it)+'_'+selfcal_plan['solmode'][it]+'_'+selfcal_library[vis][solint]['final_mode']+'.g')
os.system("cp -r "+sani_target+'_'+vis+'_'+band+'_'+sint+'_'+str(it)+'_'+selfcal_plan['solmode'][it]+'_'+selfcal_library[vis][solint]['final_mode']+'.pre-pass.g '+\
sani_target+'_'+vis+'_'+band+'_'+sint+'_'+str(it)+'_'+selfcal_plan['solmode'][it]+'_'+selfcal_library[vis][solint]['final_mode']+'.g')
os.system("rm -rf "+sani_target+'_'+vis+'_'+band+'_'+sint+'_'+str(it)+'_'+selfcal_plan['solmode'][it]+'_'+selfcal_library[vis][sint]['final_mode']+'.g')
os.system("cp -r "+sani_target+'_'+vis+'_'+band+'_'+sint+'_'+str(it)+'_'+selfcal_plan['solmode'][it]+'_'+selfcal_library[vis][sint]['final_mode']+'.pre-pass.g '+\
sani_target+'_'+vis+'_'+band+'_'+sint+'_'+str(it)+'_'+selfcal_plan['solmode'][it]+'_'+selfcal_library[vis][sint]['final_mode']+'.g')

if sint == "inf_EB" and len(selfcal_library[vis][sint]["spwmap"][0]) > 0:
unflag_spwmap = selfcal_library[vis][sint]["spwmap"][0]
else:
unflag_spwmap = []

unflag_failed_antennas(vis, sani_target+'_'+vis+'_'+band+'_'+sint+'_'+str(it)+'_'+\
selfcal_plan['solmode'][it]+'_'+selfcal_library[vis][solint]['final_mode']+'.g', flagged_fraction=0.25, solnorm=solnorm, \
selfcal_plan['solmode'][it]+'_'+selfcal_library[vis][sint]['final_mode']+'.g', flagged_fraction=0.25, solnorm=solnorm, \
only_long_baselines=selfcal_plan['solmode'][it]=="ap" if unflag_only_lbants and \
unflag_only_lbants_onlyap else unflag_only_lbants, calonly_max_flagged=calonly_max_flagged, \
spwmap=unflag_spwmap, fb_to_prev_solint=unflag_fb_to_prev_solint, solints=selfcal_plan['solints'], iteration=it)
spwmap=unflag_spwmap, fb_to_prev_solint=unflag_fb_to_prev_solint, solints=selfcal_plan['solints'], \
final_modes=[selfcal_library[vis][s]['final_mode'] if s in selfcal_library[vis] else '' for s in \
selfcal_plan['solints'][0:iteration]], iteration=it)
else:
rerefant(vis, sani_target+'_'+vis+'_'+band+'_'+sint+'_'+str(it)+'_'+selfcal_plan['solmode'][it]+'_'+selfcal_library[vis][solint]['final_mode']+'.g', \
rerefant(vis, sani_target+'_'+vis+'_'+band+'_'+sint+'_'+str(it)+'_'+selfcal_plan['solmode'][it]+'_'+selfcal_library[vis][sint]['final_mode']+'.g', \
refant=selfcal_library[vis]["refant"], refantmode=refantmode if 'inf_EB' not in sint else 'flex')
else:
os.system("cp -r "+sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+selfcal_library[vis][solint]['final_mode']+'.g '+\
Expand Down Expand Up @@ -545,7 +547,9 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, ap
selfcal_plan['solmode'][iteration]+'_'+selfcal_library[vis][solint]['final_mode']+'.g', flagged_fraction=0.25, solnorm=solnorm, \
only_long_baselines=selfcal_plan['solmode'][iteration]=="ap" if unflag_only_lbants and unflag_only_lbants_onlyap else \
unflag_only_lbants, calonly_max_flagged=calonly_max_flagged, spwmap=unflag_spwmap, \
fb_to_prev_solint=unflag_fb_to_prev_solint, solints=selfcal_plan['solints'], iteration=iteration)
fb_to_prev_solint=unflag_fb_to_prev_solint, solints=selfcal_plan['solints'], \
final_modes=[selfcal_library[vis][s]['final_mode'] if s in selfcal_library[vis] else '' for s in \
selfcal_plan['solints'][0:iteration]], iteration=iteration)

# Do some post-gaincal cleanup for mosaics.
if selfcal_library['obstype'] == 'mosaic' or mode == "cocal":
Expand Down
6 changes: 6 additions & 0 deletions run_selfcal.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -479,6 +479,12 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants,
continue
elif delta_beamarea > delta_beam_thresh:
do_fallback_combinespw=False # turn the switch off since this is another repeat

# Remove all but combinespw modes for this iteration and subsequent ones.
for vis in vislist:
selfcal_plan[vis]['solint_settings'][solint]['final_mode']='combinespw'
remove_modes(selfcal_plan,vis,iteration,exclude_start=False)

print('****************Attempting applymode="calonly" fallback*************')
# Loop through up to two times. On the first attempt, try applymode = 'calflag' (assuming this is requested by the user). On the
# second attempt, use applymode = 'calonly'.
Expand Down
21 changes: 11 additions & 10 deletions selfcal_helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3492,7 +3492,7 @@ def get_gaintable_flagging_stats(gc_dict_list,spwlist):


def unflag_failed_antennas(vis, caltable, flagged_fraction=0.25, only_long_baselines=False, solnorm=True, calonly_max_flagged=0., spwmap=[],
fb_to_prev_solint=False, solints=[], iteration=0, plot=False, plot_directory="./"):
fb_to_prev_solint=False, solints=[], final_modes=[], iteration=0, plot=False, plot_directory="./"):
tb.open(caltable, nomodify=plot) # Because we only modify if we aren't plotting, i.e. in the selfcal loop itself plot=False
antennas = tb.getcol("ANTENNA1")
flags = tb.getcol("FLAG")
Expand Down Expand Up @@ -3710,13 +3710,14 @@ def unflag_failed_antennas(vis, caltable, flagged_fraction=0.25, only_long_basel
# Check whether earlier solints have acceptable solutions, and if so use, those instead.

if fb_to_prev_solint:
if "ap" in solints[iteration]:
for i in range(len(solints)):
if "ap" in solints[i]:
min_iter = iteration
for i in range(1,iteration)[::-1]:
if "combinespw" in final_modes[i]:
if "ap" in solints[iteration]:
if "ap" in solints[i]:
min_iter = i
else:
min_iter = i
break
else:
min_iter = 1

for i, solint in enumerate(solints[min_iter:iteration][::-1]):
print("Testing solint ", solint)
Expand Down Expand Up @@ -4091,11 +4092,11 @@ def get_min_SNR_spw(snr_per_spw):
if snr_per_spw[spw] < minsnr: minsnr=snr_per_spw[spw]
return minsnr

def remove_modes(selfcal_plan,vis,start_index):
for j in range(start_index+1,len(selfcal_plan['solints'])):
def remove_modes(selfcal_plan,vis,start_index,exclude_start=True):
preferred_mode=selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][start_index]]['final_mode']
for j in range(start_index+int(exclude_start),len(selfcal_plan['solints'])):
if 'ap' in selfcal_plan['solints'][j] and 'ap' not in selfcal_plan['solints'][start_index]: # exempt over ap solints since they go back to a longer solint
continue
preferred_mode=selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['final_mode']
if preferred_mode == 'per_bb' or preferred_mode == 'combinespw':
if 'per_spw' in selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['modes_to_attempt']:
selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['modes_to_attempt'].remove('per_spw')
Expand Down