From e107219b37af02d369ca874eb1a147a135315b42 Mon Sep 17 00:00:00 2001 From: Clancy James Date: Tue, 3 Jun 2025 14:56:21 +0800 Subject: [PATCH 01/20] Altered handling of scattering functions within survey.py --- zdm/loading.py | 5 +- zdm/parameters.py | 28 +- .../Scattering/test_scattering_properties.py | 38 +- zdm/survey.py | 535 +++++++++++++----- 4 files changed, 438 insertions(+), 168 deletions(-) diff --git a/zdm/loading.py b/zdm/loading.py index dd27004..ae6a018 100644 --- a/zdm/loading.py +++ b/zdm/loading.py @@ -214,7 +214,10 @@ def surveys_and_grids(init_state=None, alpha_method=1, """ # Init state if init_state is None: - state = set_state(alpha_method=alpha_method) + # we should be using defaults... by default! + # if a user wants something else, they can pass it here + #state = set_state(alpha_method=alpha_method) + state = parameters.State() else: state = init_state if state_dict is not None: diff --git a/zdm/parameters.py b/zdm/parameters.py index 410979f..c2ee7cd 100644 --- a/zdm/parameters.py +++ b/zdm/parameters.py @@ -228,6 +228,14 @@ class IGMParams(data_class.myDataClass): # FRB intrinsic width parameters @dataclass class WidthParams(data_class.myDataClass): + WidthFunction: int = field( + default=1, + metadata={ + "help": "ID of function to describe width distribution. 0: log-constant, 1:log-normal", + "unit": "", + "Notation": "", + }, + ) Wlogmean: float = field( default=1.70267, metadata={ @@ -254,15 +262,23 @@ class WidthParams(data_class.myDataClass): ) WNbins: int = field( - default=5, + default=8, metadata={"help": "Number of bins for FRB width distribution", "unit": ""}, ) + WNInternalBins: int = field( + default=1000, + metadata={ + "help": "Number of internal bins to use for calculation purposes in numerical estimates of the width distribution", + "unit": "", + "Notation": "", + }, + ) WMin: int = field( default=0.1, metadata={"help": "Minimum scattering value to model", "unit": "ms"}, ) WMax: int = field( - default=100, + default=1000, metadata={"help": "Maximum scattering value to model", "unit": "ms"}, ) @@ -270,6 +286,14 @@ class WidthParams(data_class.myDataClass): # FRB intrinsic scattering parameters @dataclass class ScatParams(data_class.myDataClass): + ScatFunction: int = field( + default=1, + metadata={ + "help": "Which scattering function to use. 0: log-constant. 1: lognormal. 2: half log-normal", + "unit": "", + "Notation": "", + }, + ) Slogmean: float = field( default=0.7, metadata={ diff --git a/zdm/scripts/Scattering/test_scattering_properties.py b/zdm/scripts/Scattering/test_scattering_properties.py index d189186..2760356 100644 --- a/zdm/scripts/Scattering/test_scattering_properties.py +++ b/zdm/scripts/Scattering/test_scattering_properties.py @@ -43,7 +43,7 @@ def main(): # make this into a list to initialise multiple surveys art once names = [survey_name] - repeaters=True + repeaters=False # sets plotting limits zmax = 2. dmmax = 2000 @@ -78,24 +78,35 @@ def main(): llsum = it.calc_likelihoods_2D(g,s,pNreps=False) print("For scattering method ",methnames[i],", 2D likelihoods are ",llsum) + if i==2: + # gets constant weights from plot + const_weights = s.wplist + # extracts weights from survey and plots as function of z if i==3: - weights = s.wplist - widths = s.wlist[:,0] + weights = s.wplist # 2D + widths = s.wlist # widths are now 1D, they don't vary nw,nz = weights.shape plt.figure() plt.xlabel('z') plt.ylabel('weight') for iw in np.arange(nw): plt.plot(g.zvals,weights[iw,:],label="width = "+str(widths[iw])[0:5]) + plt.plot([g.zvals[0],g.zvals[-1]],[const_weights[iw],const_weights[iw]], + color=plt.gca().lines[-1].get_color(),linestyle=":") total = np.sum(weights,axis=0) plt.plot(g.zvals,total,label="total",color="black") + plt.legend(fontsize=6) + plt.tight_layout() + plt.savefig(opdir+"z_dependent_weights_lin.png") + plt.yscale("log") - plt.legend() plt.tight_layout() - plt.savefig(opdir+"z_dependent_weights.png") + plt.savefig(opdir+"z_dependent_weights_log.png") plt.close() + + figures.plot_grid( g.rates, g.zvals, @@ -128,24 +139,33 @@ def main(): plt.xlabel('z') plt.ylabel('p(z) [arb units]') plt.xlim(0,zmax) - plt.ylim(0,None) + ymax=0. for i,Method in enumerate(imethods): + themax = np.max(zdists[i]) + if ymax < themax: + ymax = themax plt.plot(g.zvals,zdists[i],label=methnames[i]) + plt.ylim(0,ymax) plt.legend() plt.tight_layout() - plt.savefig(opdir+"pdm_comparison.png",) + plt.savefig(opdir+"pz_comparison.png",) plt.close() plt.figure() plt.xlabel('DM') plt.xlim(0,dmmax) - plt.ylim(0,None) plt.ylabel('p(DM) [arb units]') + ymax = 0. for i,Method in enumerate(imethods): + themax = np.max(dmdists[i]) + if ymax < themax: + ymax = themax plt.plot(g.dmvals,dmdists[i],label=methnames[i]) + + plt.ylim(0,ymax) plt.legend() plt.tight_layout() - plt.savefig(opdir+"pz_comparison.png",) + plt.savefig(opdir+"pzdm_comparison.png",) plt.close() if not repeaters: diff --git a/zdm/survey.py b/zdm/survey.py index 5cd946b..e667d89 100644 --- a/zdm/survey.py +++ b/zdm/survey.py @@ -59,12 +59,13 @@ def __init__(self, state, survey_name:str, values in the survey file """ # Proceed + self.state = state self.name = survey_name self.dmvals = dmvals self.zvals = zvals self.NDM = dmvals.size self.NZ = zvals.size - + self.edir = edir # Load up self.process_survey_file(filename, NFRB, iFRB, min_lat=state.analysis.min_lat, dmg_cut=state.analysis.DMG_cut,survey_dict = survey_dict) @@ -84,7 +85,7 @@ def __init__(self, state, survey_name:str, # This is required when mixing CHIME and non-CHIME FRBs beam_method = self.meta['BMETHOD'] beam_thresh = self.meta['BTHRESH'] - width_bias = self.meta['WBIAS'] + #width_bias = self.meta['WBIAS'] self.init_beam( method=beam_method, @@ -94,26 +95,166 @@ def __init__(self, state, survey_name:str, # Efficiency: width_method passed through "self" here # Determines if the model is redshift dependent - if self.meta['WMETHOD'] != 3: - pwidths,pprobs=make_widths(self, state) - _ = self.get_efficiency_from_wlist(pwidths,pprobs, - model=width_bias, edir=edir, iz=None) + ##### need to fix this up by re-organising higher-level routines!!! ##### + + self.init_widths() + + self.calc_max_dm() + + def init_widths(self): + """ + Performs initialisation of width and scattering distributions + + """ + # copies over Width bin information + self.NWbins = self.state.width.WNbins + self.WMin = self.state.width.WMin + self.WMax = self.state.width.WMax + self.thresh = self.state.width.Wthresh + self.wlogmean = self.state.width.Wlogmean + self.wlogsigma = self.state.width.Wlogsigma + self.width_method = self.meta["WMETHOD"] + self.NInternalBins=self.state.width.WNInternalBins + + # records scattering information, scaling + # according to frequency + self.slogmean=self.state.scat.Slogmean \ + + self.state.scat.Sfpower*np.log( + self.meta['FBAR']/self.state.scat.Sfnorm + ) + self.slogsigma=self.state.scat.Slogsigma + self.maxsigma=self.state.scat.Smaxsigma + self.scatdist=self.state.scat.ScatDist + + # sets internal functions + WF = self.state.width.WidthFunction + if WF ==0: + self.WidthFunction = constant + elif WF == 1: + self.WidthFunction = lognormal + elif WF == 2: + self.WidthFunction = halflognormal else: + raise ValueError("state parameter scat.WidthFunction ",WF," not implemented, use 0-2 only") + + SF = self.state.scat.ScatFunction + if SF ==0: + self.ScatFunction = constant + elif SF == 1: + self.ScatFunction = lognormal + elif SF == 2: + self.ScatFunction = halflognormal + else: + raise ValueError("state parameter scat.ScatFunction ",SF," not implemented, use 0-2 only") + + # sets n width bins equal to zero for this survey + if self.meta['WMETHOD'] == 0 or self.meta['WMETHOD'] == 4: + self.NWbins = 1 + + ###### calculate width bins. We fix these here ###### + # Unless w and tau are explicitly being fit, it is not actually necessary + # to have constant bin values over z a d DM + # ensures the first bin begins at 0 + wbins = np.zeros([self.NWbins+1]) + if self.NWbins > 1: + wbins[1:] = np.logspace(np.log10(self.WMin),np.log10(self.WMax),self.NWbins) + dw = np.log10(wbins[2]/wbins[1]) + wlist = np.logspace(np.log10(self.WMin)-dw,np.log10(self.WMax)-dw,self.NWbins) + else: + wbins[0] = np.log10(self.WMin) + wbins[1] = np.log10(self.WMax) + wlist = np.array([(self.WMax*self.WMin)**0.5]) + self.wbins=wbins + self.wlist=wlist + + ####### generates internal width values of numerical calculation purposes ##### + minval = np.min([self.wlogmean - self.maxsigma*self.wlogsigma, + self.slogmean - self.maxsigma*self.slogsigma, + np.log(self.WMin)]) + maxval = np.log(self.WMax) + # I haven't decided yet where to put the value of 1000 internal bins + # in terms of parameters. + self.internal_logwvals = np.linspace(minval,maxval,self.NInternalBins) + + # initialise probability bins + if self.meta['WMETHOD'] == 3 or self.meta['WMETHOD'] == 5: # evaluate efficiencies at each redshift - self.NWbins = state.width.WNbins self.efficiencies = np.zeros([self.NWbins,self.NZ,self.NDM]) self.wplist = np.zeros([self.NWbins,self.NZ]) - self.wlist = np.zeros([self.NWbins,self.NZ]) self.DMlist = np.zeros([self.NZ,self.NDM]) self.mean_efficiencies = np.zeros([self.NZ,self.NDM]) # we have a z-dependent scattering and width model for iz,z in enumerate(self.zvals): - pwidths,pprobs=make_widths(self, state, z=z) - _ = self.get_efficiency_from_wlist(pwidths,pprobs, - model=width_bias, edir=edir,iz=iz) - self.calc_max_dm() - + self.make_widths(iz) + _ = self.get_efficiency_from_wlist(self.wlist,self.wplist[:,iz], + model=self.meta['WBIAS'], edir=self.edir, iz=iz) + else: + self.wplist = np.zeros([self.NWbins]) + self.make_widths() + _ = self.get_efficiency_from_wlist(self.wlist,self.wplist, + model=self.meta['WBIAS'], edir=self.edir, iz=None) + + def make_widths(self,iz=None): + """ + Used to be an exterior method, now interior + """ + + if self.meta['WMETHOD'] == 0: + # do not take a distribution, just use 1ms for everything + # this is done for tests, for complex surveys such as CHIME, + # or for estimating the properties of a single FRB + self.wlist[0] = 1. + self.wplist[0] = 1. + elif self.meta['WMETHOD'] == 1: + # take intrinsic width function only + for i in np.arange(self.NWbins): + norm=(2.*np.pi)**-0.5/self.wlogsigma + args=(self.wlogmean,self.wlogsigma,norm) + weight,err=quad(self.WidthFunction, + np.log(self.wbins[i]),np.log(self.wbins[i+1]),args=args) + #weight,err = quad(self.width_function,np.log(bins[i]),np.log(bins[i+1]),args=args) + self.wplist[i]=weight + elif self.meta['WMETHOD'] == 2 or self.meta['WMETHOD'] == 3: + # include scattering distribution. 3 means include z-dependence + #gets cumulative hist and bin edges + + if iz is not None: + z = self.zvals[iz] + else: + z=0. + + # performs z-scaling (does nothing when z=0.) + wlogmean = self.wlogmean + np.log(1+z) + wlogsigma = self.wlogsigma + slogmean = self.slogmean - 3.*np.log(1+z) + slogsigma = self.slogsigma + #dist,cdist,cbins=geometric_lognormals2(wlogmean, + # self.wlogsigma,slogmean,self.slogsigma,Nsigma=self.maxsigma, + # ScatDist=self.scatdist,bins=self.wbins) + + WidthArgs = (wlogmean,wlogsigma) + ScatArgs = (slogmean,slogsigma) + + dist = geometric_lognormals3(self.WidthFunction, WidthArgs, self.ScatFunction, ScatArgs, + self.internal_logwvals, self.wbins) + + if iz is not None: + self.wplist[:,iz] = dist + else: + self.wplist[:] = dist + + elif self.meta['WMETHOD'] == 4: + # use specific width of FRB. This requires there to be only a single FRB in the survey + if s.meta['NFRB'] != 1: + raise ValueError("If width method in make_widths is 3 only one FRB should be specified in the survey but ", str(s.meta['NFRB']), " FRBs were specified") + else: + self.wplist[0] = 1. + self.wlist[0] = s.frbs['WIDTH'][0] + else: + raise ValueError("Width method in make_widths must be 0, 1 or 2, not ",width_method) + + def init_repeaters(self): """ Checks to see if this is a repeater survey and if so ensures all the @@ -749,15 +890,11 @@ def get_efficiency_from_wlist(self,wlist,plist, if iz is None: self.efficiencies=efficiencies - self.wplist=plist - self.wlist=wlist self.DMlist=DMlist mean_efficiencies=np.mean(efficiencies,axis=0) self.mean_efficiencies=mean_efficiencies #be careful here!!! This may not be what we want! else: self.efficiencies[:,iz,:]=efficiencies - self.wplist[:,iz]=plist - self.wlist[:,iz]=wlist self.DMlist[iz,:]=DMlist mean_efficiencies=np.mean(efficiencies,axis=0) self.mean_efficiencies[iz,:]=mean_efficiencies #be careful here!!! This may not be what we want! @@ -882,6 +1019,168 @@ def calc_relative_sensitivity(DM_frb,DM,w,fbar,t_res,nu_res,Nchan=336,max_idt=No return sensitivity + + +def load_survey(survey_name:str, state:parameters.State, + dmvals:np.ndarray, + zvals:np.ndarray, + sdir:str=None, NFRB:int=None, + nbins=None, iFRB:int=0, + dummy=False, + edir=None, + rand_DMG=False, + survey_dict = None, + verbose=False): + """Load a survey + + Args: + survey_name (str): Name of the survey + e.g. CRAFT/FE + state (parameters.State): Parameters for the state + dmvals (np.ndarray): DM values + zvals (np.ndarray): z values + sdir (str, optional): Path to survey files. Defaults to None. + nbins (int, optional): Sets number of bins for Beam analysis + [was NBeams] + NFRB (int, optional): Cut the total survey down to a random + subset [useful for testing] + iFRB (int, optional): Start grabbing FRBs at this index + Mainly used for Monte Carlo analysis + Requires that NFRB be set + dummy (bool,optional) + Skip many initialisation steps: used only when loading + survey parameters for conversion to the new survey format + survey_dict (dict, optional): dictionary of survey metadata to + over-ride values in file + verbose (bool): print output + Raises: + IOError: [description] + + Returns: + Survey: instance of the class + """ + + if verbose: + print(f"Loading survey: {survey_name}") + + if sdir is None: + sdir = os.path.join( + resource_filename('zdm', 'data'), 'Surveys') + + # Hard code real surveys + if survey_name == 'CRAFT/FE': + dfile = 'CRAFT_class_I_and_II' + elif survey_name == 'CRAFT/ICS': + dfile = 'CRAFT_ICS' + elif survey_name == 'CRAFT/ICS892': + dfile = 'CRAFT_ICS_892' + elif survey_name == 'CRAFT/ICS1632': + dfile = 'CRAFT_ICS_1632' + + elif survey_name == 'PKS/Mb': + dfile = 'parkes_mb_class_I_and_II' + elif 'private' in survey_name: + dfile = survey_name + else: + dfile = survey_name + + # allows a user to input the .ecsv themselves + if dfile[-6:] != ".ecsv": + dfile += '.ecsv' + + print(f"Loading survey: {survey_name} from {dfile}") + + # Do it + srvy = Survey(state, + survey_name, + os.path.join(sdir, dfile), + dmvals, + zvals, + NFRB=NFRB, iFRB=iFRB, + edir=edir, rand_DMG=rand_DMG, + survey_dict = survey_dict) + return srvy + +def vet_frb_table(frb_tbl:pandas.DataFrame, + mandatory:bool=False, + fill:bool=False): + """ + This should not be necessary anymore, since + all required FRB data should be populated with + default values. However, it's great as a check. If + this complains, it means we have a bug in the + replacement with default value procedure. + """ + frb_data = survey_data.FRB() + # Loop on the standard fields + for field in frb_data.__dataclass_fields__.keys(): + if field in frb_tbl.keys(): + not_none = frb_tbl[field].values != None + if np.any(not_none): + idx0 = np.where(not_none)[0][0] + assert isinstance( + frb_tbl.iloc[idx0][field], + frb_data.__dataclass_fields__[field].type), \ + f'Bad data type for {field}' + elif mandatory: + raise ValueError(f'{field} is missing in your table!') + elif fill: + frb_tbl[field] = None + +############ We now define some width/scattering functions ############ +# These all return p(w) dlogw, and must take as arguments np.log(widths) + +def geometric_lognormals3(width_function, width_args, scat_function, scat_args, + internal_logvals, bins): + ''' + Numerically evaluates the resulting distribution of y=\sqrt{x1^2+x2^2}, + where x1 is the width distribution, and x2 is the scattering distribution. + + Args: + width_function (float function(float,args)): function to call giving p(logw) dlogw + width_args (*list): arguments to pass to width function + scat_function (float function(float,args)): function to call giving p(logtau) dlogtau + scat_args (*list): arguments to pass to scattering function + internal_vals (np.ndarray): numpy array of length NIbins giving internal + values of log dw to use for internal calculation purposes. + bins (np.ndarray([NBINS+1],dtype='float')): bin edges for final width distribution + + Returns: + hist: histogram of probability within bins + chist: cumulative histogram of probability within bins + bins: bin edges for histogram + + ''' + + #draw from both distributions + np.random.seed(1234) + + # these need to be normalised by the internal bin width + logbinwidth = internal_logvals[-1] - internal_logvals[-2] + + pw = width_function(internal_logvals, *width_args)*logbinwidth + ptau = scat_function(internal_logvals, *scat_args)*logbinwidth + + # adds extra bits onto the lowest bin. Assumes exp(-20) is small enough! + lowest = internal_logvals[0] - logbinwidth/2. + extrapw,err = quad(width_function,lowest-10,lowest,args=width_args) + extraptau,err = quad(scat_function,lowest-10,lowest,args=scat_args) + pw[0] += extrapw + ptau[0] += extraptau + + linvals = np.exp(internal_logvals) + + # calculate widths - all done in linear domain + Nbins = bins.size-1 + hist = np.zeros([Nbins]) + for i,x1 in enumerate(linvals): + totalwidths = (x1**2 + linvals**2)**0.5 + probs = pw[i]*ptau + h,b = np.histogram(totalwidths,bins=bins,weights=probs) + hist += h + + return hist + def geometric_lognormals2(lmu1,ls1,lmu2,ls2,bins=None, Ndivs=100,Nsigma=3.,plot=False,Nbins=101, ScatDist=1): @@ -1023,34 +1322,7 @@ def geometric_lognormals(lmu1,ls1,lmu2,ls2,bins=None, #hist = hist/Nrand return hist,chist,bins -def halflognormal(logmean,logsigma,minw,maxw,nbins): - """ - Generates a parameterised half-lognormal distribution. - This acts as a lognormal in the lower half, but - keeps a constant per-log-bin width in the upper half - """ - - logmin = np.log(minw) - logmax = np.log(maxw) - logbins = np.linspace(logmin,logmax,nbins+1) - dlogbin = (logmax - logmin)/nbins - logbinmean = logbins[:-1] + dlogbin/2. - - probs = np.zeros([nbins]) - - # gets weighting in smaller bins - args=[logmin,logmax,1.] - for i in np.arange(nbins): - weight,err=quad(pcosmic.loglognormal_dlog,logbins[i],logbins[i+1],args=args) - probs[i] = weight - if logbins[i+1] > logmean: - break - # fills up remaining bins with the peak value - probs[i+1:] = probs[i] - # normalisation - probs /= np.sum(pobs) - return probs - + def make_widths(s:Survey,state,z=0.): """ This method takes a distribution of intrinsic FRB widths @@ -1078,8 +1350,8 @@ def make_widths(s:Survey,state,z=0.): wlogmean=state.width.Wlogmean wlogsigma=state.width.Wlogsigma width_method = s.meta["WMETHOD"] - wmin = state.width.WMin - wmax = state.width.WMax + WMin = state.width.WMin + WMax = state.width.WMax slogmean=state.scat.Slogmean slogsigma=state.scat.Slogsigma @@ -1115,12 +1387,12 @@ def make_widths(s:Survey,state,z=0.): if width_method == 1 or width_method==2 or width_method==3: bins = np.zeros([nbins+1]) - logwmin = np.log10(wmin) - logwmax = np.log10(wmax) - dbin = (logwmax - logwmin)/(nbins-1.) - # bins ignore wmax - scale takes precedent - bins[1:] = np.logspace(logwmin,logwmax, nbins) - widths = 10**(dbin * (np.arange(nbins)-0.5) + logwmin) + logWMin = np.log10(WMin) + logWMax = np.log10(WMax) + dbin = (logWMax - logWMin)/(nbins-1.) + # bins ignore WMax - scale takes precedent + bins[1:] = np.logspace(logWMin,logWMax, nbins) + widths = 10**(dbin * (np.arange(nbins)-0.5) + logWMin) bins[0] = 1.e-10 # a very tiny value to avoid bad things in log space if width_method==0: @@ -1132,17 +1404,13 @@ def make_widths(s:Survey,state,z=0.): elif width_method==1: # take intrinsic lognormal width distribution only # normalisation of a log-normal - norm=(2.*np.pi)**-0.5/wlogsigma - args=(wlogmean,wlogsigma,norm) + args=(wlogmean,wlogsigma) weights = np.zeros([nbins]) for i in np.arange(nbins): weight,err=quad(pcosmic.loglognormal_dlog,np.log(bins[i]),np.log(bins[i+1]),args=args) - #width=(wmin*wmax)**0.5 + #width=(WMin*WMax)**0.5 #widths.append(width) - weights[i] = weight #.append(weight) - #wsum += weight - #wmin = wmax - #wmax *= scale + weights[i] = weight elif width_method==2 or width_method==3: # include scattering distribution. 3 means include z-dependence # scale scattering time according to frequency in logspace @@ -1180,108 +1448,63 @@ def make_widths(s:Survey,state,z=0.): return widths,weights -def load_survey(survey_name:str, state:parameters.State, - dmvals:np.ndarray, - zvals:np.ndarray, - sdir:str=None, NFRB:int=None, - nbins=None, iFRB:int=0, - dummy=False, - edir=None, - rand_DMG=False, - survey_dict = None, - verbose=False): - """Load a survey +def lognormal(logw, *args): + """ + Lognormal probability distribution + Args: - survey_name (str): Name of the survey - e.g. CRAFT/FE - state (parameters.State): Parameters for the state - dmvals (np.ndarray): DM values - zvals (np.ndarray): z values - sdir (str, optional): Path to survey files. Defaults to None. - nbins (int, optional): Sets number of bins for Beam analysis - [was NBeams] - NFRB (int, optional): Cut the total survey down to a random - subset [useful for testing] - iFRB (int, optional): Start grabbing FRBs at this index - Mainly used for Monte Carlo analysis - Requires that NFRB be set - dummy (bool,optional) - Skip many initialisation steps: used only when loading - survey parameters for conversion to the new survey format - survey_dict (dict, optional): dictionary of survey metadata to - over-ride values in file - verbose (bool): print output - Raises: - IOError: [description] - + logw: natural log of widths + args: vector of [logmean,logsigma] mean and std dev + Returns: - Survey: instance of the class + result: p(logw) d logw """ + logmean = args[0] + logsigma = args[1] + norm = (2.*np.pi)**-0.5/logsigma + result = norm * np.exp(-0.5 * ((logw - logmean) / logsigma) ** 2) + return result - if verbose: - print(f"Loading survey: {survey_name}") - - if sdir is None: - sdir = os.path.join( - resource_filename('zdm', 'data'), 'Surveys') - - # Hard code real surveys - if survey_name == 'CRAFT/FE': - dfile = 'CRAFT_class_I_and_II' - elif survey_name == 'CRAFT/ICS': - dfile = 'CRAFT_ICS' - elif survey_name == 'CRAFT/ICS892': - dfile = 'CRAFT_ICS_892' - elif survey_name == 'CRAFT/ICS1632': - dfile = 'CRAFT_ICS_1632' - - elif survey_name == 'PKS/Mb': - dfile = 'parkes_mb_class_I_and_II' - elif 'private' in survey_name: - dfile = survey_name - else: - dfile = survey_name +def halflognormal(logw, *args):#logmean,logsigma,minw,maxw,nbins): + """ + Generates a parameterised half-lognormal distribution. + This acts as a lognormal in the lower half, but + keeps a constant per-log-bin width in the upper half - # allows a user to input the .ecsv themselves - if dfile[-6:] != ".ecsv": - dfile += '.ecsv' - - print(f"Loading survey: {survey_name} from {dfile}") - - # Do it - srvy = Survey(state, - survey_name, - os.path.join(sdir, dfile), - dmvals, - zvals, - NFRB=NFRB, iFRB=iFRB, - edir=edir, rand_DMG=rand_DMG, - survey_dict = survey_dict) - return srvy + Args: + logw: natural log of widths + args: vector of [logmean,logsigma] mean and std dev + + Returns: + result: p(logw) d logw + """ + logmean = args[0] + logsigma = args[1] + norm = (2.*np.pi)**-0.5/logsigma + + large = np.where(logw > logmean) + + modlogw = logw + modlogw[large] = logmean # subs mean value in for values larger than the mean + result = lognormal(modlogw,args) + return result -def vet_frb_table(frb_tbl:pandas.DataFrame, - mandatory:bool=False, - fill:bool=False): +def constant(logw,*args): """ - This should not be necessary anymore, since - all required FRB data should be populated with - default values. However, it's great as a check. If - this complains, it means we have a bug in the - replacement with default value procedure. + Dummy function that returns a constant of unity. + NOTE: to include 1+z scaling here, one will need to + reduce the minimum width argument with z. Feature + to be added. Maybe have args also contain min and max values? + + Args: + logw: natural log of widths + args: vector of [logmean,logsigma] mean and std dev + + Returns: + result: p(logw) d logw """ - frb_data = survey_data.FRB() - # Loop on the standard fields - for field in frb_data.__dataclass_fields__.keys(): - if field in frb_tbl.keys(): - not_none = frb_tbl[field].values != None - if np.any(not_none): - idx0 = np.where(not_none)[0][0] - assert isinstance( - frb_tbl.iloc[idx0][field], - frb_data.__dataclass_fields__[field].type), \ - f'Bad data type for {field}' - elif mandatory: - raise ValueError(f'{field} is missing in your table!') - elif fill: - frb_tbl[field] = None + nvals = logw.size + result = np.fill([nvals],1.) + return result + From 7de457e4b1187869ea9029e669d8914a605e1f9c Mon Sep 17 00:00:00 2001 From: Clancy James Date: Tue, 3 Jun 2025 14:57:54 +0800 Subject: [PATCH 02/20] deleted unused routines in survey.py --- zdm/survey.py | 272 +------------------------------------------------- 1 file changed, 2 insertions(+), 270 deletions(-) diff --git a/zdm/survey.py b/zdm/survey.py index e667d89..2a4dda8 100644 --- a/zdm/survey.py +++ b/zdm/survey.py @@ -236,7 +236,7 @@ def make_widths(self,iz=None): WidthArgs = (wlogmean,wlogsigma) ScatArgs = (slogmean,slogsigma) - dist = geometric_lognormals3(self.WidthFunction, WidthArgs, self.ScatFunction, ScatArgs, + dist = geometric_lognormals(self.WidthFunction, WidthArgs, self.ScatFunction, ScatArgs, self.internal_logwvals, self.wbins) if iz is not None: @@ -1130,7 +1130,7 @@ def vet_frb_table(frb_tbl:pandas.DataFrame, ############ We now define some width/scattering functions ############ # These all return p(w) dlogw, and must take as arguments np.log(widths) -def geometric_lognormals3(width_function, width_args, scat_function, scat_args, +def geometric_lognormals(width_function, width_args, scat_function, scat_args, internal_logvals, bins): ''' Numerically evaluates the resulting distribution of y=\sqrt{x1^2+x2^2}, @@ -1147,8 +1147,6 @@ def geometric_lognormals3(width_function, width_args, scat_function, scat_args, Returns: hist: histogram of probability within bins - chist: cumulative histogram of probability within bins - bins: bin edges for histogram ''' @@ -1181,272 +1179,6 @@ def geometric_lognormals3(width_function, width_args, scat_function, scat_args, return hist -def geometric_lognormals2(lmu1,ls1,lmu2,ls2,bins=None, - Ndivs=100,Nsigma=3.,plot=False,Nbins=101, - ScatDist=1): - ''' - Numerically evaluates the resulting distribution of y=\sqrt{x1^2+x2^2}, - where logx1~normal and logx2~normal with log-mean lmu and - log-sigma ls. - This is typically used for two log-normals of intrinsic - FRB width and scattering time - - lmu1, ls1 (float, float): log mean and log-sigma of the first distribution - - lmu2, ls2 (float, float): log-mean and log-sigma of the second distribution - - bins (np.ndarray([NBINS+1],dtype='float')): bin edges for resulting plot. - - Returns: - hist: histogram of probability within bins - chist: cumulative histogram of probability within bins - bins: bin edges for histogram - - ''' - - #draw from both distributions - np.random.seed(1234) - - xvals1 = np.linspace(lmu1-Nsigma*ls1,lmu1+Nsigma*ls1,Ndivs) - yvals1 = pcosmic.loglognormal_dlog(xvals1,lmu1,ls1,1.) - yvals1 /= np.sum(yvals1) - - # xvals in ln space - lnlog = np.log10(np.exp(1)) - xvals2 = np.logspace(lnlog*(lmu2-Nsigma*ls2),lnlog*(lmu2+Nsigma*ls2),Ndivs) - if ScatDist == 0: - # log uniform - yvals2 = np.full([Ndivs],2./Ndivs) - elif ScatDist == 1: - # lognormal - yvals2 = pcosmic.loglognormal_dlog(xvals2,lmu2,ls2,2.) - yvals2 /= np.sum(yvals2) - elif ScatDist == 2: - # upper lognormal is flat - yvals2 = pcosmic.loglognormal_dlog(xvals2,lmu2,ls2,2.) - upper = np.where(xvals2 > lmu2)[0] - ymax = np.max(yvals2) - yvals2[upper] = ymax - yvals2 /= np.sum(yvals2) - - xvals1 = np.exp(xvals1) - xvals2 = np.exp(xvals2) - themin = np.min([np.min(xvals1),np.min(xvals2)]) - themax = 2**0.5 * np.max([np.max(xvals1),np.max(xvals2)]) - - if bins is None: - #bins=np.linspace(0,np.max(ys)/4.,Nbins) - delta=1e-3 - # ensures the first bin begins at 0 - bins=np.zeros([Nbins+1]) - bins[1:]=np.logspace(np.log10(themin)-delta,np.log10(themax)+delta,Nbins) - else: - Nbins = len(bins)-1 - - # calculate widths - hist = np.zeros([Nbins]) - for i,x1 in enumerate(xvals1): - widths = (x1**2 + xvals2**2)**0.5 - probs = yvals1[i]*yvals2 - h,b = np.histogram(widths,bins=bins,weights=probs) - hist += h - - chist=np.zeros([Nbins+1]) - chist[1:]=np.cumsum(hist) - # we do not want to renormalise, since the normalisation reflects the values - # which are too large - #hist /= chist[-1] - chist /= chist[-1] - - return hist,chist,bins - -def geometric_lognormals(lmu1,ls1,lmu2,ls2,bins=None, - Nrand=10000,plot=False,Nbins=101): - ''' - Numerically evaluates the resulting distribution of y=\sqrt{x1^2+x2^2}, - where logx1~normal and logx2~normal with log-mean lmu and - log-sigma ls. - This is typically used for two log-normals of intrinsic - FRB width and scattering time - - lmu1, ls1 (float, float): log mean and log-sigma of the first distribution - - lmu2, ls2 (float, float): log-mean and log-sigma of the second distribution - - bins (np.ndarray([NBINS+1],dtype='float')): bin edges for resulting plot. - - Returns: - hist: histogram of probability within bins - chist: cumulative histogram of probability within bins - bins: bin edges for histogram - - ''' - - #draw from both distributions - np.random.seed(1234) - x1s=np.random.normal(lmu1,ls1,Nrand) - x2s=np.random.normal(lmu2,ls2,Nrand) - - ys=(np.exp(x1s*2)+np.exp(x2s*2))**0.5 - - if bins is None: - #bins=np.linspace(0,np.max(ys)/4.,Nbins) - delta=1e-3 - # ensures the first bin begins at 0 - bins=np.zeros([Nbins+1]) - bins[1:]=np.logspace(np.log10(np.min(ys))-delta,np.log10(np.max(ys))+delta,Nbins) - hist,bins=np.histogram(ys,bins=bins) - chist=np.zeros([Nbins+1]) - chist[1:]=np.cumsum(hist) - chist /= chist[-1] - - if plot: - plt.figure() - plt.hist(ys,bins=bins) - plt.xlabel('$y, Y=\\sqrt{X_1^2+X_2^2}$') - plt.ylabel('$P(Y=y)$') - plt.tight_layout() - plt.savefig('adding_lognormals.pdf') - plt.close() - - lbins=np.linspace(-3.,5.,81) - plt.figure() - plt.xlabel('$log y, Y=\\sqrt{X_1^2+X_2^2}$') - plt.ylabel('$P(logY=logy)$') - plt.hist(np.log(ys),bins=lbins) - plt.savefig('log_adding_lognormals.pdf') - plt.close() - - # renomalises - total will be less than unity, assuming some large - # values fall off the largest bin - #hist = hist/Nrand - return hist,chist,bins - - -def make_widths(s:Survey,state,z=0.): - """ - This method takes a distribution of intrinsic FRB widths - (lognormal, defined by wlogmean and wlogsigma), and returns - a list of w_i, p(w_i), where the w_i are i=1...N values of - width, and p(w_i) are statistical weights associated with each. - - The \sum_i p(w_i) should sum to unity always. Each w_i is used - to calculate a separate efficiency table. - - Args: - s (Survey,required): instance of survey class - state (state class,required): instance of the state class - z (float): redshift at which this is being calculated - - Returns: - list: list of widths - """ - # variables which can be over-ridden by a survey, but which - # appear by default in the parameter set - - # just extracting for now to get things straight - nbins=state.width.WNbins - thresh=state.width.Wthresh - wlogmean=state.width.Wlogmean - wlogsigma=state.width.Wlogsigma - width_method = s.meta["WMETHOD"] - WMin = state.width.WMin - WMax = state.width.WMax - - slogmean=state.scat.Slogmean - slogsigma=state.scat.Slogsigma - sfnorm=state.scat.Sfnorm - sfpower=state.scat.Sfpower - maxsigma=state.scat.Smaxsigma - scatdist=state.scat.ScatDist - - # adjusts these model values according to redshift - wlogmean += np.log(1.+z) # scales with (1+z) - slogmean -= 3.*np.log(1.+z) # scales with (1+z)^-3 - - # constant of DM - k_DM=4.149 #ms GHz^2 pc^-1 cm^3 - - tres=s.meta['TRES'] - nu_res=s.meta['FRES'] - fbar=s.meta['FBAR'] - - ###### calculate a characteristic scaling pulse width ######## - - # estimates this for a DM of 100 - DM=100 - - # total smearing factor within a channel - dm_smearing=2*(nu_res/1.e3)*k_DM*DM/(fbar/1e3)**3 #smearing factor of FRB in the band - wsum=0. - - ######## generate width distribution ###### - # arrays to hold widths and weights - weights=[] - widths=[] - - if width_method == 1 or width_method==2 or width_method==3: - bins = np.zeros([nbins+1]) - logWMin = np.log10(WMin) - logWMax = np.log10(WMax) - dbin = (logWMax - logWMin)/(nbins-1.) - # bins ignore WMax - scale takes precedent - bins[1:] = np.logspace(logWMin,logWMax, nbins) - widths = 10**(dbin * (np.arange(nbins)-0.5) + logWMin) - bins[0] = 1.e-10 # a very tiny value to avoid bad things in log space - - if width_method==0: - # do not take a distribution, just use 1ms for everything - # this is done for tests, for complex surveys such as CHIME, - # or for estimating the properties of a single FRB - weights.append(1.) - widths.append(np.exp(wlogmean)) - elif width_method==1: - # take intrinsic lognormal width distribution only - # normalisation of a log-normal - args=(wlogmean,wlogsigma) - weights = np.zeros([nbins]) - for i in np.arange(nbins): - weight,err=quad(pcosmic.loglognormal_dlog,np.log(bins[i]),np.log(bins[i+1]),args=args) - #width=(WMin*WMax)**0.5 - #widths.append(width) - weights[i] = weight - elif width_method==2 or width_method==3: - # include scattering distribution. 3 means include z-dependence - # scale scattering time according to frequency in logspace - slogmean = slogmean + sfpower*np.log(fbar/sfnorm) - - # generates bins - - - #gets cumulative hist and bin edges - dist,cdist,cbins=geometric_lognormals2(wlogmean, - wlogsigma,slogmean,slogsigma,Nsigma=maxsigma, - ScatDist=scatdist,bins=bins) - weights = dist - - elif width_method==4: - # use specific width of FRB. This requires there to be only a single FRB in the survey - if s.meta['NFRB'] != 1: - raise ValueError("If width method in make_widths is 3 only one FRB should be specified in the survey but ", str(s.meta['NFRB']), " FRBs were specified") - else: - weights.append(1.) - widths.append(s.frbs['WIDTH'][0]) - else: - raise ValueError("Width method in make_widths must be 0, 1 or 2, not ",width_method) - # check this is correct - we may wish to lose extra probability - # off the top, though never off the bottom - #weights[-1] += 1.-wsum #adds defecit here - weights=np.array(weights) - widths=np.array(widths) - # removes unneccesary bins - # cannot do this when considering z-dependent bins for consistency - #keep=np.where(weights>1e-4)[0] - #weights=weights[keep] - #widths=widths[keep] - - return widths,weights - def lognormal(logw, *args): From dfe91660274f4cd09ea7648a929c6f653d0cdf7e Mon Sep 17 00:00:00 2001 From: Clancy James Date: Wed, 4 Jun 2025 14:17:30 +0800 Subject: [PATCH 03/20] Added calculation of pw and ptau from observed FRB properties --- zdm/parameters.py | 8 +++ .../Scattering/test_scattering_properties.py | 49 +++++++++++++- zdm/survey.py | 66 +++++++++++++++---- 3 files changed, 111 insertions(+), 12 deletions(-) diff --git a/zdm/parameters.py b/zdm/parameters.py index c2ee7cd..e22e6a5 100644 --- a/zdm/parameters.py +++ b/zdm/parameters.py @@ -342,6 +342,14 @@ class ScatParams(data_class.myDataClass): "Notation": "", }, ) + Sbackproject: bool = field( + default=False, + metadata={ + "help": "If TRUE, calculate internal arrays to estimate p(tau|w,DM,z)", + "unit": "", + "Notation": "", + }, + ) # FRB Energetics -- energy diff --git a/zdm/scripts/Scattering/test_scattering_properties.py b/zdm/scripts/Scattering/test_scattering_properties.py index 2760356..c4edf06 100644 --- a/zdm/scripts/Scattering/test_scattering_properties.py +++ b/zdm/scripts/Scattering/test_scattering_properties.py @@ -65,10 +65,14 @@ def main(): for i,Method in enumerate(imethods): survey_dict = {"WMETHOD": Method} + state_dict = {} + state_dict["scat"] = {} + state_dict["scat"]["Sbackproject"] = True # turns on backprojection of tau and width for our model + # Write True if you want to do repeater grids - see "plot_repeaters.py" to make repeater plots surveys, grids = loading.surveys_and_grids(survey_names = names,\ repeaters=repeaters, sdir=sdir,nz=70,ndm=140, - survey_dict = survey_dict) + survey_dict = survey_dict, state_dict = state_dict) g=grids[0] s=surveys[0] @@ -106,6 +110,49 @@ def main(): plt.close() + # plots back-projected probabilities + plt.figure() + ax1 = plt.gca() + plt.figure() + ax2 = plt.gca() + + # gets zvalues correponding to 0.1,0.5,1,2 + izs = [] + for z in [ 0.1,0.5,1,2]: + # gets index of the above redshift values + iz = np.where(g.zvals>z)[0][0] + izs.append(iz) + styles=["-","--","-.",":"] + for iw in np.arange(s.NWbins): + label=str(s.wlist[iw])[0:5] + for j,iz in enumerate(izs): + if j==0: + ax1.plot(s.internal_logwvals,s.ptaus[iz,:,iw],label=label, + linestyle = styles[j]) + ax2.plot(s.internal_logwvals,s.pws[iz,:,iw],label=label, + linestyle = styles[j]) + else: + ax1.plot(s.internal_logwvals,s.ptaus[iz,:,iw],label=label, + linestyle = styles[j],color=plt.gca().lines[-1].get_color()) + ax2.plot(s.internal_logwvals,s.pws[iz,:,iw],label=label, + linestyle = styles[j],color=plt.gca().lines[-1].get_color()) + label=None + plt.sca(ax1) + plt.xlabel("Natural log of scattering width (observed)") + plt.ylabel("Probability given total width and redshift") + plt.legend(fontsize=6) + plt.tight_layout() + plt.savefig(opdir+"z_dependent_ptau.png") + plt.close() + + + plt.sca(ax2) + plt.xlabel("Natural log of intrinsic width (observed)") + plt.ylabel("Probability given total width and redshift") + plt.legend(fontsize=6) + plt.tight_layout() + plt.savefig(opdir+"z_dependent_pw.png") + plt.close() figures.plot_grid( g.rates, diff --git a/zdm/survey.py b/zdm/survey.py index 2a4dda8..fd58213 100644 --- a/zdm/survey.py +++ b/zdm/survey.py @@ -125,6 +125,7 @@ def init_widths(self): self.slogsigma=self.state.scat.Slogsigma self.maxsigma=self.state.scat.Smaxsigma self.scatdist=self.state.scat.ScatDist + self.backproject=self.state.scat.Sbackproject # sets internal functions WF = self.state.width.WidthFunction @@ -177,13 +178,17 @@ def init_widths(self): self.internal_logwvals = np.linspace(minval,maxval,self.NInternalBins) # initialise probability bins - if self.meta['WMETHOD'] == 3 or self.meta['WMETHOD'] == 5: + if self.meta['WMETHOD'] == 3: # evaluate efficiencies at each redshift self.efficiencies = np.zeros([self.NWbins,self.NZ,self.NDM]) self.wplist = np.zeros([self.NWbins,self.NZ]) self.DMlist = np.zeros([self.NZ,self.NDM]) self.mean_efficiencies = np.zeros([self.NZ,self.NDM]) + if self.backproject: + self.pws = np.zeros([self.NZ,self.internal_logwvals.size,self.NWbins]) #[iz,:,:] = pw + self.ptaus = np.zeros([self.NZ,self.internal_logwvals.size,self.NWbins]) #[iz,:,:] = ptau + # we have a z-dependent scattering and width model for iz,z in enumerate(self.zvals): self.make_widths(iz) @@ -192,6 +197,9 @@ def init_widths(self): else: self.wplist = np.zeros([self.NWbins]) self.make_widths() + if self.backproject: + self.pws = np.zeros([self.internal_logwvals.size,self.NWbins]) #[iz,:,:] = pw + self.ptaus = np.zeros([self.internal_logwvals.size,self.NWbins]) #[iz,:,:] = ptau _ = self.get_efficiency_from_wlist(self.wlist,self.wplist, model=self.meta['WBIAS'], edir=self.edir, iz=None) @@ -229,20 +237,27 @@ def make_widths(self,iz=None): wlogsigma = self.wlogsigma slogmean = self.slogmean - 3.*np.log(1+z) slogsigma = self.slogsigma - #dist,cdist,cbins=geometric_lognormals2(wlogmean, - # self.wlogsigma,slogmean,self.slogsigma,Nsigma=self.maxsigma, - # ScatDist=self.scatdist,bins=self.wbins) WidthArgs = (wlogmean,wlogsigma) ScatArgs = (slogmean,slogsigma) - dist = geometric_lognormals(self.WidthFunction, WidthArgs, self.ScatFunction, ScatArgs, - self.internal_logwvals, self.wbins) + if self.backproject: + dist,pw,ptau = quadrature_convolution(self.WidthFunction, WidthArgs, self.ScatFunction, ScatArgs, + self.internal_logwvals, self.wbins,backproject=self.backproject) + else: + dist = quadrature_convolution(self.WidthFunction, WidthArgs, self.ScatFunction, ScatArgs, + self.internal_logwvals, self.wbins,backproject=self.backproject) if iz is not None: self.wplist[:,iz] = dist + if self.backproject: + self.pws[iz,:,:] = pw + self.ptaus[iz,:,:] = ptau else: self.wplist[:] = dist + if self.backproject: + self.pws=pw + self.ptaus=ptau elif self.meta['WMETHOD'] == 4: # use specific width of FRB. This requires there to be only a single FRB in the survey @@ -1130,8 +1145,8 @@ def vet_frb_table(frb_tbl:pandas.DataFrame, ############ We now define some width/scattering functions ############ # These all return p(w) dlogw, and must take as arguments np.log(widths) -def geometric_lognormals(width_function, width_args, scat_function, scat_args, - internal_logvals, bins): +def quadrature_convolution(width_function, width_args, scat_function, scat_args, + internal_logvals, bins, backproject = False): ''' Numerically evaluates the resulting distribution of y=\sqrt{x1^2+x2^2}, where x1 is the width distribution, and x2 is the scattering distribution. @@ -1144,9 +1159,12 @@ def geometric_lognormals(width_function, width_args, scat_function, scat_args, internal_vals (np.ndarray): numpy array of length NIbins giving internal values of log dw to use for internal calculation purposes. bins (np.ndarray([NBINS+1],dtype='float')): bin edges for final width distribution - + backproject (bool, optional): if True, calculates p(tau|totalw) and p(w|totalw) + for this redshift, and returns additional values. Returns: - hist: histogram of probability within bins + hist (np.ndarray): histogram of probability within bins + wfracs (np.ndarray,only if backproject): p(tau|tw) + taufracs (np.ndarray,only if backproject): p(w|tw) ''' @@ -1177,7 +1195,33 @@ def geometric_lognormals(width_function, width_args, scat_function, scat_args, h,b = np.histogram(totalwidths,bins=bins,weights=probs) hist += h - return hist + # calculate p(w) and p(tau) for each w fior this z + if backproject: + # generate arrays to hold probabilities, so values of tau and + # w can be fit + wfracs = np.zeros([internal_logvals.size,Nbins]) + taufracs = np.zeros([internal_logvals.size,Nbins]) + + # maps the probabilities as a function of intrinsic + # width to generate a p(w|total_width) and p(tau|total_width) + # note that these are p(observed) values, i.e. after z-correction + for i,x1 in enumerate(linvals): + totalwidths = (x1**2 + linvals**2)**0.5 + + probs = pw[i]*ptau + h,b = np.histogram(totalwidths,bins=bins,weights=probs) + wfracs[i,:] = h + + probs = ptau[i]*pw + h,b = np.histogram(totalwidths,bins=bins,weights=probs) + taufracs[i,:] = h + + # normalise probabilities for each intrinsic w + wfracs = (wfracs.T/np.sum(wfracs,axis=1)).T + taufracs = (taufracs.T/np.sum(taufracs,axis=1)).T + return hist,wfracs,taufracs + else: + return hist From 9ccf667b3c7febb2cd6aa89a01882fae7cd0427f Mon Sep 17 00:00:00 2001 From: Clancy James Date: Thu, 5 Jun 2025 11:31:56 +0800 Subject: [PATCH 04/20] Implemented p(w,tau) into iteration.py for 2D only --- zdm/data/Surveys/CRAFT_ICS_1300.ecsv | 44 ++--- zdm/data/Surveys/CRAFT_ICS_1632.ecsv | 14 +- zdm/data/Surveys/CRAFT_ICS_892.ecsv | 38 ++-- zdm/iteration.py | 50 ++++- zdm/misc_functions.py | 25 +++ .../Scattering/test_scattering_properties.py | 3 +- zdm/survey.py | 171 ++++++++++++++---- 7 files changed, 263 insertions(+), 82 deletions(-) diff --git a/zdm/data/Surveys/CRAFT_ICS_1300.ecsv b/zdm/data/Surveys/CRAFT_ICS_1300.ecsv index 27dace6..82a17d6 100644 --- a/zdm/data/Surveys/CRAFT_ICS_1300.ecsv +++ b/zdm/data/Surveys/CRAFT_ICS_1300.ecsv @@ -15,30 +15,30 @@ # - {name: TRES, datatype: float64} # - {name: WIDTH, datatype: float64} # - {name: TAU, datatype: float64} -# - {name: XDec, datatype: string} -# - {name: XRA, datatype: string} +# - {name: DEC, datatype: string} +# - {name: RA, datatype: string} # - {name: Z, datatype: float64} # meta: !!omap # - {survey_data: '{"observing": {"NORM_FRB": 19, "TOBS": 165.506756761, "MAX_IDT": 4096, "MAX_IW": 12, "MAXWMETH": 1}, # "telescope": {"BEAM": "ASKAP_1300", "DIAM": 12.0, "NBEAMS": 36, "NBINS": 5, "WDATA": 1}}'} # schema: astropy-2.0 -TNS BW DM DMG FBAR FRES Gb Gl SNR SNRTHRESH THRESH TRES WIDTH TAU XDec XRA Z -20180924B 336.0 362.4 40.5 1297.5 1.0 -74.40520121494983 277.20651893408893 21.1 9.0 4.4 0.864 2 0.59 -40:54:00.1 21:44:25.255 0.3214 -20181112A 336.0 589.0 40.2 1297.5 1.0 -63.30983709852826 290.87390392674445 19.3 9.0 4.4 0.864 0.8 0.023 -52:58:15.39 21:49:23.630 0.4755 -20190102C 336.0 364.5 57.3 1271.5 1.0 -37.519392943553534 300.95052401722796 14.0 9.0 4.4 0.864 1.25 0.027 -79:28:32.2845 21:29:39.70836 0.291 -20190608B 336.0 339.5 37.2 1271.5 1.0 -67.23724646562972 88.21721604792883 16.1 9.0 4.4 1.728 10.8 3.8 -07:53:53.6 22:16:04.7472s 0.1178 -20190611B 336.0 322.2 57.6 1271.5 1.0 -37.59976893568559 300.9594501909617 9.3 9.0 4.4 1.728 1.59 0.03 -79:23:51.284 21:22:58.372 0.378 -20190711A 336.0 594.6 56.6 1271.5 1.0 -36.63629532677801 301.03976293370494 23.8 9.0 4.4 1.728 11.0 0.0076 -80:21:28.18 21:57:40.012 0.522 -20190714A 336.0 504.7 38.5 1271.5 1.0 -75.88144720209479 120.55805492153455 10.7 9.0 4.4 1.728 3.0 0.422 -13:01:14.36 12:15:55.081 0.2365 -20191228A 336.0 297.5 32.9 1271.5 1.0 -80.77822140033614 230.79855541687724 22.9 9.0 4.4 1.728 13.6 5.85 -29:35:37.85 22:57:43.269 0.243 -20210117A 336.0 730.0 34.4 1271.5 1.0 -75.7801432700954 164.65014968696988 27.1 9.0 4.4 1.182 3.6 0.25 -16:11:25.2 22:39:36.0 0.214 -20210214A 336.0 398.3 31.9 1271.5 1.0 -65.65372930742399 91.72782990931984 11.6 9.0 4.4 1.182 3.5 -1 -05:49:56 00:27:43 -1.0 -20210407 336.0 1785.3 154.0 1271.5 1.0 -35.320866474063905 114.6146256941771 19.1 9.0 4.4 1.182 1.62 0.09 27:03:30.24 05:14:36.202 -1.0 -20210912 336.0 1234.5 30.9 1271.5 1.0 -80.16655338584705 235.42165843951094 31.7 9.0 4.4 1.182 1.61 0.048 -30:29:33.1 23:24:40.3 -1.0 -20211127 336.0 234.83 42.5 1271.5 1.0 -81.68554744714709 125.94306109583985 37.9 9.0 4.4 1.182 0.48 0.025 -18:49:28.4 13:19:09.5 0.046946 -20220531A 336.0 727.0 70.0 1271.5 1.0 -56.509199987039224 296.83938685003784 9.7 9.0 4.4 1.182 11.0 -1 -60:17:48.2 19:38:50.2 -1.0 -20220610A 336.0 1458.1 31.0 1271.5 1.0 -78.89122591551634 250.56280818953536 29.8 9.0 4.4 1.182 2.0 0.521 -33:30:49.87 23:24:17.559 1.016 -20220918A 336.0 656.8 40.7 1271.5 1.0 -45.81623556809721 308.4134807482381 26.4 9.0 4.4 1.182 13.9 7.66 -70:48:40.5 01:10:22.1 0.45 -20230526A 336.0 316.4 50.0 1271.5 1.0 -62.994316139408156 318.1591960546148 22.1 9.0 4.4 1.182 2.7 1.16 -52:46:07.7 01:29:27.5 0.157 -20230718A 336.0 477.0 396.0 1271.5 1.0 -75.66933767869608 316.30925962515585 10.9 9.0 4.4 1.182 0.7 0.117 -41:00:12.8 08:30:27.1 -1.0 -20230731A 336.0 701.1 547.0 1271.5 1.0 -60.14372530213189 304.262204790738 16.6 9.0 4.4 1.182 2.7 0.45 -56:58:19.1 11:38:40.1 -1.0 +TNS BW DM DMG FBAR FRES Gb Gl SNR SNRTHRESH THRESH TRES WIDTH TAU DEC RA Z +20180924B 336.0 362.4 40.5 1297.5 1.0 -74.40520121494983 277.20651893408893 21.1 9.0 4.4 0.864 2 0.59 -40:54:00.1 21:44:25.255 0.3214 +20181112A 336.0 589.0 40.2 1297.5 1.0 -63.30983709852826 290.87390392674445 19.3 9.0 4.4 0.864 0.8 0.023 -52:58:15.39 21:49:23.630 0.4755 +20190102C 336.0 364.5 57.3 1271.5 1.0 -37.519392943553534 300.95052401722796 14.0 9.0 4.4 0.864 1.25 0.027 -79:28:32.2845 21:29:39.70836 0.291 +20190608B 336.0 339.5 37.2 1271.5 1.0 -67.23724646562972 88.21721604792883 16.1 9.0 4.4 1.728 10.8 3.8 -07:53:53.6 22:16:04.7472 0.1178 +20190611B 336.0 322.2 57.6 1271.5 1.0 -37.59976893568559 300.9594501909617 9.3 9.0 4.4 1.728 1.59 0.03 -79:23:51.284 21:22:58.372 0.378 +20190711A 336.0 594.6 56.6 1271.5 1.0 -36.63629532677801 301.03976293370494 23.8 9.0 4.4 1.728 11.0 0.0076 -80:21:28.18 21:57:40.012 0.522 +20190714A 336.0 504.7 38.5 1271.5 1.0 -75.88144720209479 120.55805492153455 10.7 9.0 4.4 1.728 3.0 0.422 -13:01:14.36 12:15:55.081 0.2365 +20191228A 336.0 297.5 32.9 1271.5 1.0 -80.77822140033614 230.79855541687724 22.9 9.0 4.4 1.728 13.6 5.85 -29:35:37.85 22:57:43.269 0.243 +20210117A 336.0 730.0 34.4 1271.5 1.0 -75.7801432700954 164.65014968696988 27.1 9.0 4.4 1.182 3.6 0.25 -16:11:25.2 22:39:36.0 0.214 +20210214A 336.0 398.3 31.9 1271.5 1.0 -65.65372930742399 91.72782990931984 11.6 9.0 4.4 1.182 3.5 -1 -05:49:56 00:27:43 -1.0 +20210407E 336.0 1785.3 154.0 1271.5 1.0 -35.320866474063905 114.6146256941771 19.1 9.0 4.4 1.182 1.62 0.09 27:03:30.24 05:14:36.202 -1.0 +20210912A 336.0 1234.5 30.9 1271.5 1.0 -80.16655338584705 235.42165843951094 31.7 9.0 4.4 1.182 1.61 0.048 -30:29:33.1 23:24:40.3 -1.0 +20211127I 336.0 234.83 42.5 1271.5 1.0 -81.68554744714709 125.94306109583985 37.9 9.0 4.4 1.182 0.48 0.025 -18:49:28.4 13:19:09.5 0.046946 +20220531A 336.0 727.0 70.0 1271.5 1.0 -56.509199987039224 296.83938685003784 9.7 9.0 4.4 1.182 11.0 -1 -60:17:48.2 19:38:50.2 -1.0 +20220610A 336.0 1458.1 31.0 1271.5 1.0 -78.89122591551634 250.56280818953536 29.8 9.0 4.4 1.182 2.0 0.521 -33:30:49.87 23:24:17.559 1.016 +20220918A 336.0 656.8 40.7 1271.5 1.0 -45.81623556809721 308.4134807482381 26.4 9.0 4.4 1.182 13.9 7.66 -70:48:40.5 01:10:22.1 0.45 +20230526A 336.0 316.4 50.0 1271.5 1.0 -62.994316139408156 318.1591960546148 22.1 9.0 4.4 1.182 2.7 1.16 -52:46:07.7 01:29:27.5 0.157 +20230718A 336.0 477.0 396.0 1271.5 1.0 -75.66933767869608 316.30925962515585 10.9 9.0 4.4 1.182 0.7 0.117 -41:00:12.8 08:30:27.1 -1.0 +20230731A 336.0 701.1 547.0 1271.5 1.0 -60.14372530213189 304.262204790738 16.6 9.0 4.4 1.182 2.7 0.45 -56:58:19.1 11:38:40.1 -1.0 diff --git a/zdm/data/Surveys/CRAFT_ICS_1632.ecsv b/zdm/data/Surveys/CRAFT_ICS_1632.ecsv index 71c6ce6..9780636 100644 --- a/zdm/data/Surveys/CRAFT_ICS_1632.ecsv +++ b/zdm/data/Surveys/CRAFT_ICS_1632.ecsv @@ -14,15 +14,15 @@ # - {name: THRESH, datatype: float64} # - {name: TRES, datatype: float64} # - {name: WIDTH, datatype: float64} -# - {name: XC, datatype: string} -# - {name: XDec, datatype: string} -# - {name: XRA, datatype: string} +# - {name: TAU, datatype: float64} +# - {name: DEC, datatype: string} +# - {name: RA, datatype: string} # - {name: Z, datatype: float64} # meta: !!omap # - {survey_data: '{"observing": {"NORM_FRB": 3, "TOBS": 50.866971336, "MAX_IDT": 4096, "MAX_IW": 12, "MAXWMETH": 1}, # "telescope": {"BEAM": "ASKAP_1632", "DIAM": 12.0, "NBEAMS": 1, "NBINS": 5}}'} # schema: astropy-2.0 -TNS BW DM DMG FBAR FRES Gb Gl SNR SNRTHRESH THRESH TRES WIDTH XC XDec XRA Z -20211212 336.0 206.0 27.1 1631.5 1.0 -61.10615576177682 118.0697870677333 12.8 9.0 4.4 1.182 2.7 closepack36/45/0.9 01:40:36.8 10:30:40.7 0.0715 -20220105 336.0 583.0 22.0 1631.5 1.0 -40.39508677874226 124.21912132806385 9.8 9.0 4.4 1.182 2.0 closepack36/45/0.9 22:27:57.8 13:55:13.01 0.2785 -20221106 336.0 343.8 34.8 1631.5 1.0 -81.71820282415577 41.73623647983608 19.5 9.0 4.4 1.182 5.7 closepack36/45/0.9 -25:34:10.5166 03:46:49.0982 0.204 +TNS BW DM DMG FBAR FRES Gb Gl SNR SNRTHRESH THRESH TRES WIDTH TAU DEC RA Z +20211212A 336.0 206.0 27.1 1631.5 1.0 -61.10615576177682 118.0697870677333 12.8 9.0 4.4 1.182 5.628 1.8 01:40:36.8 10:30:40.7 0.0715 +20220105A 336.0 583.0 22.0 1631.5 1.0 -40.39508677874226 124.21912132806385 9.8 9.0 4.4 1.182 2.25 0.43 22:27:57.8 13:55:13.01 0.2785 +20221106A 336.0 343.8 34.8 1631.5 1.0 -81.71820282415577 41.73623647983608 19.5 9.0 4.4 1.182 6.895 0.182 -25:34:10.5166 03:46:49.0982 0.204 diff --git a/zdm/data/Surveys/CRAFT_ICS_892.ecsv b/zdm/data/Surveys/CRAFT_ICS_892.ecsv index 25976d6..3b6d18c 100644 --- a/zdm/data/Surveys/CRAFT_ICS_892.ecsv +++ b/zdm/data/Surveys/CRAFT_ICS_892.ecsv @@ -14,27 +14,27 @@ # - {name: THRESH, datatype: float64} # - {name: TRES, datatype: float64} # - {name: WIDTH, datatype: float64} -# - {name: XC, datatype: string} -# - {name: XDec, datatype: string} -# - {name: XRA, datatype: string} +# - {name: TAU, datatype: float64} +# - {name: DEC, datatype: string} +# - {name: RA, datatype: string} # - {name: Z, datatype: float64} # meta: !!omap # - {survey_data: '{"observing": {"NORM_FRB": 15, "TOBS": 317.293429793, "MAX_IDT": 4096, "MAX_IW": 12, "MAXWMETH": 1}, # "telescope": {"BEAM": "ASKAP_892", "DIAM": 12.0, "NBEAMS": 1, "NBINS": 5}}'} # schema: astropy-2.0 -TNS BW DM DMG FBAR FRES Gb Gl SNR SNRTHRESH THRESH TRES WIDTH XC XDec XRA Z -20191001A 336.0 506.92 44.2 919.5 1.0 -61.65860153281646 292.33820148500735 62.0 9.0 4.4 1.728 4.2 closepack36/45 -54:44:54 21:33:24 0.23 -20200430A 336.0 380.1 27.0 864.5 1.0 -50.423867191684614 126.69623499918922 16.0 9.0 4.4 1.728 6.5 square_6x6 12:22:34.007 15:18:49.581 0.161 -20200627 336.0 294.0 40.0 920.5 1.0 -75.58714298569731 274.1944222920371 11.0 9.0 4.4 1.728 2.9 closepack36 -39:29:05.0 21:46:47.0 -1.0 -20200906A 336.0 577.8 35.9 864.5 1.0 -74.33639680729853 87.47153687885996 19.2 9.0 4.4 1.728 6.0 closepack36 -14:04:59.9136 03:33:58.9364 0.36879 -20210320 336.0 384.8 42.2 864.5 1.0 -78.99989352050102 126.81646106581941 15.3 9.0 4.4 1.728 5.4 square_6x6/45/1.05 -16:09:05.1 13:37:50.08605 0.28 -20210807 336.0 251.9 121.2 920.5 1.0 -62.769666370514145 138.57666701461682 47.1 9.0 4.4 1.182 10.0 square6x6/45/0.9 -00:45:44.5 19:56:53.144 0.12969 -20210809 336.0 651.5 190.1 920.5 1.0 -61.10280652957891 133.77599069157813 16.8 9.0 4.4 1.182 14.2 square6x6/45/0.9 01:19:43.5 18:04:37.7 -1.0 -20211203 336.0 636.2 63.4 920.5 1.0 -85.70736537183255 294.09236840807733 14.2 9.0 4.4 1.182 9.6 closepack36/45/0.9 -31:22:04.0 13:37:52.8 0.34386 -20220501C 336.0 449.5 30.6 863.5 1.0 -79.3478690413585 245.52087739106628 16.1 9.0 4.4 1.182 6.5 square6x6/45/1.05 -32:27:41.4 23:29:46.8 0.381 -20220725A 336.0 290.4 30.7 920.5 1.0 -77.2090157833712 260.2992248635945 12.7 9.0 4.4 1.182 4.1 closepack36/45/0.9 -36:07:51.2 23:33:32.1 0.1926 -20230521A 336.0 640.2 41.8 831.5 1.0 -63.802137907452966 143.6440869115044 15.2 9.0 4.4 1.182 11.0 square6x6/45/1.05 -02:23:09.6 21:51:00.3 -1.0 -20230708A 336.0 411.5 50.2 920.5 1.0 -61.245461788949015 294.23527074045234 31.5 9.0 4.4 1.182 2.7 closepack36/45/0.9 -55:22:59.4 20:12:56.9 0.105 -20230902A 336.0 440.1 34.3 831.5 1.0 -68.28916777429704 320.20535326767765 11.8 9.0 4.4 1.182 6.0 square6x6/45/1.05 -47:33:45.5 03:29:28.1 -1.0 -20231006 336.0 509.7 67.5 863.5 1.0 -52.222367446784375 298.13333823887274 15.2 9.0 4.4 1.182 6.5 square6x6/45/1.05 -64:38:56.2 19:44:00.8 -1.0 -20231226 336.0 329.9 38.0 863.5 1.0 -56.65392198775274 118.39332419712602 17.8 9.0 4.4 1.182 7.1 square6x6/45/1.05 06:07:45.9 10:21:07.6 -1.0 +TNS BW DM DMG FBAR FRES Gb Gl SNR SNRTHRESH THRESH TRES WIDTH TAU DEC RA Z +20191001A 336.0 506.92 44.2 919.5 1.0 -61.65860153281646 292.33820148500735 62.0 9.0 4.4 1.728 13.468 4.52 -54:44:54 21:33:24 0.23 +20200430A 336.0 380.1 27.0 864.5 1.0 -50.423867191684614 126.69623499918922 16.0 9.0 4.4 1.728 23.68 6.5 12:22:34.007 15:18:49.581 0.161 +20200627 336.0 294.0 40.0 920.5 1.0 -75.58714298569731 274.1944222920371 11.0 9.0 4.4 1.728 2.9 -1 -39:29:05.0 21:46:47.0 -1.0 +20200906A 336.0 577.8 35.9 864.5 1.0 -74.33639680729853 87.47153687885996 19.2 9.0 4.4 1.728 0.128 0.0315 -14:04:59.9136 03:33:58.9364 0.36879 +20210320C 336.0 384.8 42.2 864.5 1.0 -78.99989352050102 126.81646106581941 15.3 9.0 4.4 1.728 0.884 0.193 -16:09:05.1 13:37:50.08605 0.28 +20210807 336.0 251.9 121.2 920.5 1.0 -62.769666370514145 138.57666701461682 47.1 9.0 4.4 1.182 10.0 -1 -00:45:44.5 19:56:53.144 0.12969 +20210809 336.0 651.5 190.1 920.5 1.0 -61.10280652957891 133.77599069157813 16.8 9.0 4.4 1.182 14.2 -1 01:19:43.5 18:04:37.7 -1.0 +20211203C 336.0 636.2 63.4 920.5 1.0 -85.70736537183255 294.09236840807733 14.2 9.0 4.4 1.182 25.449 1.66 -31:22:04.0 13:37:52.8 0.34386 +20220501C 336.0 449.5 30.6 863.5 1.0 -79.3478690413585 245.52087739106628 16.1 9.0 4.4 1.182 6.9 0.35 -32:27:41.4 23:29:46.8 0.381 +20220725A 336.0 290.4 30.7 920.5 1.0 -77.2090157833712 260.2992248635945 12.7 9.0 4.4 1.182 8.016 2.29 -36:07:51.2 23:33:32.1 0.1926 +20230521A 336.0 640.2 41.8 831.5 1.0 -63.802137907452966 143.6440869115044 15.2 9.0 4.4 1.182 11.0 -1 -02:23:09.6 21:51:00.3 -1.0 +20230708A 336.0 411.5 50.2 920.5 1.0 -61.245461788949015 294.23527074045234 31.5 9.0 4.4 1.182 23.578 0.24 -55:22:59.4 20:12:56.9 0.105 +20230902A 336.0 440.1 34.3 831.5 1.0 -68.28916777429704 320.20535326767765 11.8 9.0 4.4 1.182 0.678 0.123 -47:33:45.5 03:29:28.1 -1.0 +20231006 336.0 509.7 67.5 863.5 1.0 -52.222367446784375 298.13333823887274 15.2 9.0 4.4 1.182 6.5 -1 -64:38:56.2 19:44:00.8 -1.0 +20231226A 336.0 329.9 38.0 863.5 1.0 -56.65392198775274 118.39332419712602 17.8 9.0 4.4 1.182 9.72 0.1 06:07:45.9 10:21:07.6 -1.0 diff --git a/zdm/iteration.py b/zdm/iteration.py index 9621c30..91efc36 100644 --- a/zdm/iteration.py +++ b/zdm/iteration.py @@ -534,7 +534,8 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True,Pn=False,pN return llsum,lllist,expected,[0.,0.,0.,0.] -def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=False,Pn=False,pNreps=True,dolist=0,verbose=False,grid_type=0): +def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=False, + Pn=False,pNreps=True,ptauw=False,dolist=0,verbose=False,grid_type=0): """ Calculates 2D likelihoods using observed DM,z values grid: the grid object calculated from survey @@ -608,6 +609,19 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal DMobs=survey.DMEGs[survey.zlist] Zobs=survey.Zs[survey.zlist] zlist=survey.zlist + if ptauw: + + # checks which have OK tau values + ztaulist = [] + for i,iz in enumerate(survey.zlist): + if iz in survey.OKTAU: + ztaulist.append(iz) + Wobs = survey.WIDTHs[ztaulist] + Tauobs = survey.TAUs[ztaulist] + Iwobs = survey.IWIDTHs[ztaulist] + ztDMobs=survey.DMEGs[ztaulist] + ztZobs=survey.Zs[ztaulist] + else: raise ValueError("No nlocalised FRBs in this survey, cannot calculate 1D likelihoods") @@ -628,6 +642,16 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal idms1,idms2,dkdms1,dkdms2 = grid.get_dm_coeffs(DMobs) izs1,izs2,dkzs1,dkzs2 = grid.get_z_coeffs(Zobs) + if ptauw: + # This could all be precalculated within the survey. + iws1,iws2,dkws1,dkws2 = survey.get_w_coeffs(Wobs) # total width in survey width bins + itaus1,itaus2,dktaus1,dktaus2 = survey.get_internal_coeffs(Tauobs) # scattering time tau + iis1,iis2,dkis1,dkis2 = survey.get_internal_coeffs(Iwobs) # intrinsic width + + ztidms1,ztidms2,ztdkdms1,ztdkdms2 = grid.get_dm_coeffs(ztDMobs) + ztizs1,ztizs2,ztdkzs1,ztdkzs2 = grid.get_z_coeffs(ztZobs) + + # Calculate probability if grid.state.MW.sigmaDMG == 0.0 and grid.state.MW.sigmaHalo == 0.0: if np.any(DMobs < 0): @@ -737,6 +761,30 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal plt.savefig('1d_dm_fit.pdf') plt.close() + ####### Calculates p(tau,w| total width) #### + if ptauw: + + ptaus = survey.ptaus[ztizs1,itaus1,iws1]*ztdkzs1*dktaus1*dkws1 \ + + survey.ptaus[ztizs1,itaus1,iws2]*ztdkzs1*dktaus1*dkws2 \ + + survey.ptaus[ztizs1,itaus2,iws1]*ztdkzs1*dktaus1*dkws1 \ + + survey.ptaus[ztizs1,itaus2,iws2]*ztdkzs1*dktaus1*dkws2 \ + + survey.ptaus[ztizs2,itaus1,iws1]*ztdkzs2*dktaus1*dkws1 \ + + survey.ptaus[ztizs2,itaus1,iws2]*ztdkzs2*dktaus1*dkws2 \ + + survey.ptaus[ztizs2,itaus2,iws1]*ztdkzs2*dktaus2*dkws1 \ + + survey.ptaus[ztizs2,itaus2,iws2]*ztdkzs2*dktaus2*dkws2 + pws = survey.pws[ztizs1,iis1,iws1]*ztdkzs1*dkis1*dkws1 \ + + survey.pws[ztizs1,iis1,iws2]*ztdkzs1*dkis1*dkws2 \ + + survey.pws[ztizs1,iis2,iws1]*ztdkzs1*dkis1*dkws1 \ + + survey.pws[ztizs1,iis2,iws2]*ztdkzs1*dkis1*dkws2 \ + + survey.pws[ztizs2,iis1,iws1]*ztdkzs2*dkis1*dkws1 \ + + survey.pws[ztizs2,iis1,iws2]*ztdkzs2*dkis1*dkws2 \ + + survey.pws[ztizs2,iis2,iws1]*ztdkzs2*dkis2*dkws1 \ + + survey.pws[ztizs2,iis2,iws2]*ztdkzs2*dkis2*dkws2 + + ptwll = np.sum(np.log10(ptaus) + np.log10(pws)) + llsum += ptwll + lllist.append(ptwll) + ###### Calculates p(E | z,DM) ######## # i.e. the probability of observing an FRB # with energy E given redshift and DM diff --git a/zdm/misc_functions.py b/zdm/misc_functions.py index 47d18e6..9651885 100644 --- a/zdm/misc_functions.py +++ b/zdm/misc_functions.py @@ -82,6 +82,31 @@ def galactic_to_j2000(l_deg, b_deg): return equatorial_coord.ra.degree, equatorial_coord.dec.degree +def coord_string_to_deg(cstring,hr=False): + """ + Converts a coordinate string in form of deg:min:sec to deg + + Args: + cstring (string): string of deg:min:sec + hr (optional): if True, assumes its hr:min:sec + + Returns: + deg (float): coordinate in degrees + """ + parts = cstring.split(":") + if len(parts) == 3: + deg = float(parts[0]) + float(parts[1])/60. + float(parts[2])/3600. + elif len(parts) == 1: + deg = float(parts) + else: + raise ValueError("Do not know how to convert string ",cstring," to degrees") + + if hr: + deg *= 15. # accounts for hour to degree conversion + return deg + + + def get_source_counts(grid, plot=None, Slabel=None): """ Calculates the source-counts function for a given grid diff --git a/zdm/scripts/Scattering/test_scattering_properties.py b/zdm/scripts/Scattering/test_scattering_properties.py index c4edf06..7c6f876 100644 --- a/zdm/scripts/Scattering/test_scattering_properties.py +++ b/zdm/scripts/Scattering/test_scattering_properties.py @@ -15,7 +15,6 @@ from zdm import io from pkg_resources import resource_filename import numpy as np -from zdm import survey from matplotlib import pyplot as plt import matplotlib @@ -28,7 +27,7 @@ matplotlib.rc('font', **font) def main(): - + # in case you wish to switch to another output directory opdir = "Plots/" if not os.path.exists(opdir): diff --git a/zdm/survey.py b/zdm/survey.py index fd58213..75d7ac3 100644 --- a/zdm/survey.py +++ b/zdm/survey.py @@ -119,7 +119,7 @@ def init_widths(self): # records scattering information, scaling # according to frequency self.slogmean=self.state.scat.Slogmean \ - + self.state.scat.Sfpower*np.log( + + self.state.scat.Sfpower*np.log10( self.meta['FBAR']/self.state.scat.Sfnorm ) self.slogsigma=self.state.scat.Slogsigma @@ -158,21 +158,25 @@ def init_widths(self): # ensures the first bin begins at 0 wbins = np.zeros([self.NWbins+1]) if self.NWbins > 1: + wbins[0] = 1.e-10 # set to a tiny value, to ensure we capture all small widths wbins[1:] = np.logspace(np.log10(self.WMin),np.log10(self.WMax),self.NWbins) - dw = np.log10(wbins[2]/wbins[1]) - wlist = np.logspace(np.log10(self.WMin)-dw,np.log10(self.WMax)-dw,self.NWbins) + dlogw = np.log10(wbins[2]/wbins[1]) + # offsets the mean by half the log-spacing for each + wlist = np.logspace(np.log10(self.WMin)-dlogw/2.,np.log10(self.WMax)-dlogw/2.,self.NWbins) else: wbins[0] = np.log10(self.WMin) wbins[1] = np.log10(self.WMax) + dlogw = np.log10(wbins[1]/wbins[0]) wlist = np.array([(self.WMax*self.WMin)**0.5]) - self.wbins=wbins - self.wlist=wlist + self.wbins = wbins + self.wlist = wlist + self.dlogw = dlogw ####### generates internal width values of numerical calculation purposes ##### minval = np.min([self.wlogmean - self.maxsigma*self.wlogsigma, self.slogmean - self.maxsigma*self.slogsigma, - np.log(self.WMin)]) - maxval = np.log(self.WMax) + np.log10(self.WMin)]) + maxval = np.log10(self.WMax) # I haven't decided yet where to put the value of 1000 internal bins # in terms of parameters. self.internal_logwvals = np.linspace(minval,maxval,self.NInternalBins) @@ -220,8 +224,7 @@ def make_widths(self,iz=None): norm=(2.*np.pi)**-0.5/self.wlogsigma args=(self.wlogmean,self.wlogsigma,norm) weight,err=quad(self.WidthFunction, - np.log(self.wbins[i]),np.log(self.wbins[i+1]),args=args) - #weight,err = quad(self.width_function,np.log(bins[i]),np.log(bins[i+1]),args=args) + np.log10(self.wbins[i]),np.log10(self.wbins[i+1]),args=args) self.wplist[i]=weight elif self.meta['WMETHOD'] == 2 or self.meta['WMETHOD'] == 3: # include scattering distribution. 3 means include z-dependence @@ -233,9 +236,9 @@ def make_widths(self,iz=None): z=0. # performs z-scaling (does nothing when z=0.) - wlogmean = self.wlogmean + np.log(1+z) + wlogmean = self.wlogmean + np.log10(1+z) wlogsigma = self.wlogsigma - slogmean = self.slogmean - 3.*np.log(1+z) + slogmean = self.slogmean - 3.*np.log10(1+z) slogsigma = self.slogsigma WidthArgs = (wlogmean,wlogsigma) @@ -362,7 +365,76 @@ def init_repeaters(self): # Initialise repeater zs self.init_zs_reps() - + + + def get_internal_coeffs(self,wlist): + """ + Returns indices and coefficients for linear interpolation between + intrinsic width values + + wlist: np.ndarray of observed intrinsic widths or taus + + Returns: + iws1 (int): index of lower value + iws2 (int): index of upper value + dkws1 (float): coefficient for iws1 + dkws2 (float): coefficient for iws2 + """ + # convert to log-widths - the bins are in log10 space + logwlist = np.log10(wlist) + dinternal = self.internal_logwvals[1]-self.internal_logwvals[0] + kws=(logwlist-self.internal_logwvals[0])/dinternal + Bin0 = np.where(kws < 0.)[0] + kws[Bin0] = 0. + + iws1=kws.astype('int') + iws2=iws1+1 + dkws2=kws-iws1 # applies to izs2 + dkws1 = 1. - dkws2 + + # checks for values which are too large + toobigw = np.where(logwlist > self.internal_logwvals[-1])[0] + if len(toobigw) > 0: + raise ValueError("Width value ",wlist[toobigw], + " too large for max internal log value of ",10**self.internal_logwvals[-1]) + + return iws1, iws2, dkws1, dkws2 + + def get_w_coeffs(self,wlist): + """ + Returns indices and coefficients for linear interpolation between width values + + wlist: np.ndarray of observed widths + + Returns: + iws1 (int): index of lower value + iws2 (int): index of upper value + dkws1 (float): coefficient for iws1 + dkws2 (float): coefficient for iws2 + """ + + # convert to log-widths - the bins are in log10 space + logwlist = np.log10(wlist) + kws=(logwlist-np.log10(self.WMin))/self.dlogw +1. + print(wlist) + print(logwlist) + print(self.wbins) + Bin0 = np.where(kws < 0.)[0] + kws[Bin0] = 0. + + iws1=kws.astype('int') + iws2=iws1+1 + dkws2=kws-iws1 # applies to izs2 + dkws1 = 1. - dkws2 + + # checks for values which are too large + toobigw = np.where(wlist > self.WMax)[0] + if len(toobigw) > 0: + raise ValueError("Width value ",wlist[toobigw], + " too large for Wmax of ",self.WMax) + + return iws1, iws2, dkws1, dkws2 + def randomise_DMG(self, uDMG=0.5): """ Change the DMG_ISM values to a random value within uDMG Gaussian uncertainty """ @@ -396,12 +468,12 @@ def process_dmhalo(self, halo_method): # Yamasaki and Totani 2020 elif halo_method == 1: no_coords = np.where(self.Gls == 1.0)[0] - # if np.any(np.isnan(self.XRA[no_coords])) or np.any(np.isnan(self.XDec[no_coords])): + # if np.any(np.isnan(self.RA[no_coords])) or np.any(np.isnan(self.Dec[no_coords])): # raise ValueError('Galactic coordinates must be set if using directional dependence') if len(no_coords) != 0: for i in no_coords: - coords = SkyCoord(ra=self.XRA[i], dec=self.XDec[i], frame='icrs', unit="deg") + coords = SkyCoord(ra=self.RA[i], dec=self.Dec[i], frame='icrs', unit="deg") self.Gls[i] = coords.galactic.l.value self.Gbs[i] = coords.galactic.b.value @@ -429,12 +501,12 @@ def process_dmhalo(self, halo_method): # Sanskriti et al. 2020 elif halo_method == 2: no_coords = np.where(self.Gls == 1.0)[0] - # if np.any(np.isnan(self.XRA[no_coords])) or np.any(np.isnan(self.XDec[no_coords])): + # if np.any(np.isnan(self.XRA[no_coords])) or np.any(np.isnan(self.Dec[no_coords])): # raise ValueError('Galactic coordinates must be set if using directional dependence') if len(no_coords) != 0: for i in no_coords: - coords = SkyCoord(ra=self.XRA[i], dec=self.XDec[i], frame='icrs', unit="deg") + coords = SkyCoord(ra=self.RA[i], dec=self.Dec[i], frame='icrs', unit="deg") self.Gls[i] = coords.galactic.l.value self.Gbs[i] = coords.galactic.b.value @@ -662,6 +734,7 @@ def process_survey_file(self,filename:str, self.frbs=self.frbs[iFRB:themax] # fills in missing coordinates if possible + # also converts RA and Dec strings to floats self.fix_coordinates(verbose=False) # Min latitude @@ -733,6 +806,17 @@ def process_survey_file(self,filename:str, self.Ss=self.SNRs/self.SNRTHRESHs self.TOBS=self.meta['TOBS'] self.NORM_FRB=self.meta['NORM_FRB'] + + # calculates intrinsic widths + # likely needs to modify this because TAU is is the 1/e timescale, + # w is the 90% width timescale. Hence, should subtract 2.3 times + # Perhaps include a code within each survey to govern this? + TEMP = self.frbs['WIDTH'].values**2 - self.frbs['TAU'].values**2 + self.OKTAU = np.where(self.frbs['TAU'].values != -1.)[0] # code for non-existent + toolow = np.where(TEMP <= 0.) + TEMP[toolow] = 0.01**2 # 10 microsecond width + self.IWIDTHs = TEMP**0.5 + self.TAUs = self.frbs['TAU'].values # sets the 'beam' values to unity by default self.beam_b=np.array([1]) @@ -747,12 +831,37 @@ def process_survey_file(self,filename:str, print("FRB survey sucessfully initialised with ",self.NFRB," FRBs starting from", self.iFRB) + def fix_coordinates(self,verbose=False): """ Takes and FRB, and fills out missing coordinate values Note that now, RA, DEC, Gl, and Gb will be present But their default values are None """ + # converts to float if in string. Will do nothing if None + if isinstance(self.frbs['RA'][0],str): + RAs = np.zeros([len(self.frbs['RA'])]) + for i,RA in enumerate(self.frbs['RA']): + RAs[i] = misc_functions.coord_string_to_deg(self.frbs['RA'][i],hr=True) + self.frbs['RA'] = RAs + + if isinstance(self.frbs['DEC'][0],str): + DECs = np.zeros([len(self.frbs['DEC'])]) + for i,DEC in enumerate(self.frbs['DEC']): + DECs[i] = misc_functions.coord_string_to_deg(self.frbs['DEC'][i],hr=False) + self.frbs['DEC'] = DECs + + if isinstance(self.frbs['Gb'][0],str): + Gbs = np.zeros([len(self.frbs['Gb'])]) + for i,Gb in enumerate(self.frbs['Gb']): + Gbs[i] = misc_functions.coord_string_to_deg(self.frbs['Gb'][i],hr=True) + self.frbs['Gb'] = Gbs + + if isinstance(self.frbs['Gl'][0],str): + Gls = np.zeros([len(self.frbs['Gl'])]) + for i,Gl in enumerate(self.frbs['Gl']): + Gls[i] = misc_functions.coord_string_to_deg(self.frbs['Gl'][i],hr=False) + self.frbs['Gl'] = Gls for i,gl in enumerate(self.frbs['Gl']): @@ -761,15 +870,15 @@ def fix_coordinates(self,verbose=False): if self.frbs['RA'][i] is None or self.frbs['DEC'][i] is None: if verbose: print("WARNING: no coordinates calculable for FRB ",i) - else: + else: Gb,Gl = misc_functions.j2000_to_galactic(self.frbs['RA'][i], self.frbs['DEC'][i]) self.frbs[i,'Gb'] = Gb self.frbs[i,'Gl'] = Gl elif self.frbs['RA'][i] is None or self.frbs['DEC'][i] is None: - RA,Dec = misc_functions.galactic_to_j2000(gl, self.frbs['Gb'][i]) + RA,Dec = misc_functions.galactic_to_j2000(self.frbs['Gl'][i], self.frbs['Gb'][i]) self.frbs[i,'RA'] = RA self.frbs[i,'DEC'] = Dec - + def process_dmg(self): """ Estimates galactic DM according to Galactic lat and lon only if not otherwise provided @@ -1143,7 +1252,7 @@ def vet_frb_table(frb_tbl:pandas.DataFrame, frb_tbl[field] = None ############ We now define some width/scattering functions ############ -# These all return p(w) dlogw, and must take as arguments np.log(widths) +# These all return p(w) dlogw, and must take as arguments np.log10(widths) def quadrature_convolution(width_function, width_args, scat_function, scat_args, internal_logvals, bins, backproject = False): @@ -1225,12 +1334,12 @@ def quadrature_convolution(width_function, width_args, scat_function, scat_args, -def lognormal(logw, *args): +def lognormal(log10w, *args): """ Lognormal probability distribution Args: - logw: natural log of widths + log10w: log base 10 of widths args: vector of [logmean,logsigma] mean and std dev Returns: @@ -1239,34 +1348,34 @@ def lognormal(logw, *args): logmean = args[0] logsigma = args[1] norm = (2.*np.pi)**-0.5/logsigma - result = norm * np.exp(-0.5 * ((logw - logmean) / logsigma) ** 2) + result = norm * np.exp(-0.5 * ((log10w - logmean) / logsigma) ** 2) return result -def halflognormal(logw, *args):#logmean,logsigma,minw,maxw,nbins): +def halflognormal(log10w, *args):#logmean,logsigma,minw,maxw,nbins): """ Generates a parameterised half-lognormal distribution. This acts as a lognormal in the lower half, but keeps a constant per-log-bin width in the upper half Args: - logw: natural log of widths + log10w: log base 10 of widths args: vector of [logmean,logsigma] mean and std dev Returns: - result: p(logw) d logw + result: p(log10w) d log10w """ logmean = args[0] logsigma = args[1] norm = (2.*np.pi)**-0.5/logsigma - large = np.where(logw > logmean) + large = np.where(log10w > logmean) - modlogw = logw + modlogw = log10w modlogw[large] = logmean # subs mean value in for values larger than the mean result = lognormal(modlogw,args) return result -def constant(logw,*args): +def constant(log10w,*args): """ Dummy function that returns a constant of unity. NOTE: to include 1+z scaling here, one will need to @@ -1274,13 +1383,13 @@ def constant(logw,*args): to be added. Maybe have args also contain min and max values? Args: - logw: natural log of widths + log10w: log base 10 of widths args: vector of [logmean,logsigma] mean and std dev Returns: result: p(logw) d logw """ - nvals = logw.size + nvals = log10w.size result = np.fill([nvals],1.) return result From 0f660067a7dd9b98a7ff1ef3dbaaa6764aac3b03 Mon Sep 17 00:00:00 2001 From: Clancy James Date: Thu, 5 Jun 2025 12:28:02 +0800 Subject: [PATCH 05/20] Implemented 1D pz,tau into likelihood calculation --- zdm/iteration.py | 148 +++++++++++++++++++++------ zdm/scripts/Scattering/fit_w_scat.py | 83 +++++++++++++++ zdm/survey.py | 3 - 3 files changed, 199 insertions(+), 35 deletions(-) create mode 100644 zdm/scripts/Scattering/fit_w_scat.py diff --git a/zdm/iteration.py b/zdm/iteration.py index 91efc36..0a88190 100644 --- a/zdm/iteration.py +++ b/zdm/iteration.py @@ -177,7 +177,8 @@ def get_log_likelihood(grid, s, norm=True, psnr=True, Pn=False, pNreps=True): return llsum -def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True,Pn=False,pNreps=True,dolist=0,grid_type=0): +def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True, + Pn=False,pNreps=True,ptauw=False,dolist=0,grid_type=0): """ Calculates 1D likelihoods using only observedDM values Here, Zfrbs is a dummy variable allowing it to be treated like a 2D function for purposes of calling. @@ -199,7 +200,11 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True,Pn=False,pN pNreps: True: calculate probability of the number of repetitions for each repeater False: do not calculate this - + + ptauw: + True: calculate probability of intrinsic width and scattering *given* total width + False: do not calculate this + dolist 2: llsum,lllist [Pzdm,Pn,Ps],expected,longlist longlist holds the LL for each FRB @@ -212,6 +217,10 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True,Pn=False,pN """ + if ptauw: + if not survey.backproject: + print("WARNING: cannot calculate ptauw for this survey, please initialised backproject") + # Determine which array to perform operations on and initialise if grid_type == 1: rates = grid.exact_reps @@ -234,7 +243,20 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True,Pn=False,pN nozlist=survey.nozlist else: raise ValueError("No non-localised FRBs in this survey, cannot calculate 1D likelihoods") - + + if ptauw: + # checks which have OK tau values - in general, this is a subset + # ALSO: note that this only checks p(tau,iw | w)! It does NOT + # evaluate p(w)!!! Which is a pretty key thing... + noztaulist = [] + for i,iz in enumerate(nozlist): + if iz in survey.OKTAU: + noztaulist.append(iz) + Wobs = survey.WIDTHs[noztaulist] + Tauobs = survey.TAUs[noztaulist] + Iwobs = survey.IWIDTHs[noztaulist] + ztDMobs=survey.DMEGs[noztaulist] + dmvals=grid.dmvals zvals=grid.zvals @@ -260,12 +282,13 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True,Pn=False,pN log_global_norm=0 idms1,idms2,dkdms1,dkdms2 = grid.get_dm_coeffs(DMobs) - + + ################ Calculation of p(DM) ################# if grid.state.MW.sigmaDMG == 0.0 and grid.state.MW.sigmaHalo == 0.0: if np.any(DMobs < 0): raise ValueError("Negative DMobs with no uncertainty") - # Linear interpolation + # Linear interpolation between DMs pvals=pdm[idms1]*dkdms1 + pdm[idms2]*dkdms2 else: dm_weights, iweights = calc_DMG_weights(DMobs, survey.DMhalos[nozlist], survey.DMGs[nozlist], dmvals, grid.state.MW.sigmaDMG, @@ -283,7 +306,58 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True,Pn=False,pN llsum=np.sum(np.log10(pvals))-log_global_norm*DMobs.size lllist=[llsum] - ### Assesses total number of FRBs ### + ########### Calculation of p((Tau,w)) ############## + + if ptauw:# Gets linear interpolation coefficients. + # This could all be precalculated within the survey. + iws1,iws2,dkws1,dkws2 = survey.get_w_coeffs(Wobs) # total width in survey width bins + itaus1,itaus2,dktaus1,dktaus2 = survey.get_internal_coeffs(Tauobs) # scattering time tau + iis1,iis2,dkis1,dkis2 = survey.get_internal_coeffs(Iwobs) # intrinsic width + + # ensures a normalised p(z) distribution for each FRB (shape: nz,nDM) + if grid.state.MW.sigmaDMG == 0.0 and grid.state.MW.sigmaHalo == 0.0: + # here, each FRB only has two DM weightings (linear interolation) + ztidms1,ztidms2,ztdkdms1,ztdkdms2 = grid.get_dm_coeffs(ztDMobs) + tomult = rates[:,ztidms1]*ztdkdms1 + rates[:,ztidms2]*ztdkdms2 + # normalise to a p(z) distribution for each FRB + tomult = (tomult.T/np.sum(tomult,axis=0)).T + else: + dm_weights, iweights = calc_DMG_weights(DMobs, survey.DMhalos[noztaulist], + survey.DMGs[noztaulist], dmvals, grid.state.MW.sigmaDMG, + grid.state.MW.sigmaHalo, grid.state.MW.logu) + # here, each FRB has many DM weightings + tomult = np.zeros([grid.zvals.size,len(iweights)]) + # construct a p(z) distribution. + for iFRB,indices in enumerate(iweights): + # we construct a p(z) vector for each FRB + indices = indices[0] + tomult[:,iFRB] = np.sum(rates[:,indices] * dm_weights[iFRB],axis=1) + # normalise to a p(z) distribution for each FRB + tomult /= np.sum(tomult,axis=0) + + # vectors below are [nz,NFRB] in length + ptaus = survey.ptaus[:,itaus1,iws1]*dktaus1*dkws1\ + + survey.ptaus[:,itaus1,iws2]*dktaus1*dkws2 \ + + survey.ptaus[:,itaus2,iws1]*dktaus1*dkws1 \ + + survey.ptaus[:,itaus2,iws2]*dktaus1*dkws2 + + pws = survey.pws[:,iis1,iws1]*dkis1*dkws1 \ + + survey.pws[:,iis1,iws2]*dkis1*dkws2 \ + + survey.pws[:,iis2,iws1]*dkis1*dkws1 \ + + survey.pws[:,iis2,iws2]*dkis1*dkws2 + + # we now multiply by the z-dependencies + ptaus *= tomult + pws *= tomult + + # sum down the redshift axis to get sum p(tau,w|z)*p(z) + ptaus = np.sum(ptaus,axis=0) + pws = np.sum(pws,axis=0) + ptwll = np.sum(np.log10(ptaus) + np.log10(pws)) + llsum += ptwll + lllist.append(ptwll) + + ############# Assesses total number of FRBs, P(N) ######### # TODO: make the grid tell you the correct nromalisation if Pn and (survey.TOBS is not None): if grid_type==1: @@ -556,6 +630,10 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal True: calculate probability that each repeater detects the given number of bursts False: do not calculate this + ptauw: + True: calculate probability of intrinsic width and scattering *given* total width + False: do not calculate this + dolist: 0: returns total log10 likelihood llsum only [float] 1: returns llsum, log10([Pzdm,Pn,Ps]), @@ -586,6 +664,10 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal # an FRB has been observed. The normalisation # below is proportional to the total rate (ish) + if ptauw: + if not survey.backproject: + print("WARNING: cannot calculate ptauw for this survey, please initialised backproject") + # Determine which array to perform operations on and initialise if grid_type == 1: rates = grid.exact_reps @@ -609,22 +691,23 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal DMobs=survey.DMEGs[survey.zlist] Zobs=survey.Zs[survey.zlist] zlist=survey.zlist - if ptauw: - - # checks which have OK tau values - ztaulist = [] - for i,iz in enumerate(survey.zlist): - if iz in survey.OKTAU: - ztaulist.append(iz) - Wobs = survey.WIDTHs[ztaulist] - Tauobs = survey.TAUs[ztaulist] - Iwobs = survey.IWIDTHs[ztaulist] - ztDMobs=survey.DMEGs[ztaulist] - ztZobs=survey.Zs[ztaulist] - else: raise ValueError("No nlocalised FRBs in this survey, cannot calculate 1D likelihoods") - + + if ptauw: + # checks which have OK tau values - in general, this is a subset + # ALSO: note that this only checks p(tau,iw | w)! It does NOT + # evaluate p(w)!!! Which is a pretty key thing... + ztaulist = [] + for i,iz in enumerate(zlist): + if iz in survey.OKTAU: + ztaulist.append(iz) + Wobs = survey.WIDTHs[ztaulist] + Tauobs = survey.TAUs[ztaulist] + Iwobs = survey.IWIDTHs[ztaulist] + ztDMobs=survey.DMEGs[ztaulist] + ztZobs=survey.Zs[ztaulist] + zvals=grid.zvals dmvals=grid.dmvals @@ -642,17 +725,7 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal idms1,idms2,dkdms1,dkdms2 = grid.get_dm_coeffs(DMobs) izs1,izs2,dkzs1,dkzs2 = grid.get_z_coeffs(Zobs) - if ptauw: - # This could all be precalculated within the survey. - iws1,iws2,dkws1,dkws2 = survey.get_w_coeffs(Wobs) # total width in survey width bins - itaus1,itaus2,dktaus1,dktaus2 = survey.get_internal_coeffs(Tauobs) # scattering time tau - iis1,iis2,dkis1,dkis2 = survey.get_internal_coeffs(Iwobs) # intrinsic width - - ztidms1,ztidms2,ztdkdms1,ztdkdms2 = grid.get_dm_coeffs(ztDMobs) - ztizs1,ztizs2,ztdkzs1,ztdkzs2 = grid.get_z_coeffs(ztZobs) - - - # Calculate probability + ############## Calculate probability p(z,DM) ################ if grid.state.MW.sigmaDMG == 0.0 and grid.state.MW.sigmaHalo == 0.0: if np.any(DMobs < 0): raise ValueError("Negative DMobs with no uncertainty") @@ -719,6 +792,8 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal llpz = np.sum(np.log10(pzvals)) - np.log10(norm)*Zobs.size dolist5_return = [llpzgdm,llpdm,llpdmgz,llpz] + + ############### Calculate p(N) ###############3 if Pn and (survey.TOBS is not None): if grid_type == 1: observed=survey.NORM_REPS @@ -761,9 +836,17 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal plt.savefig('1d_dm_fit.pdf') plt.close() - ####### Calculates p(tau,w| total width) #### + ################ Calculates p(tau,w| total width) ############### if ptauw: + # This could all be precalculated within the survey. + iws1,iws2,dkws1,dkws2 = survey.get_w_coeffs(Wobs) # total width in survey width bins + itaus1,itaus2,dktaus1,dktaus2 = survey.get_internal_coeffs(Tauobs) # scattering time tau + iis1,iis2,dkis1,dkis2 = survey.get_internal_coeffs(Iwobs) # intrinsic width + #ztidms1,ztidms2,ztdkdms1,ztdkdms2 = grid.get_dm_coeffs(ztDMobs) + ztizs1,ztizs2,ztdkzs1,ztdkzs2 = grid.get_z_coeffs(ztZobs) + + ptaus = survey.ptaus[ztizs1,itaus1,iws1]*ztdkzs1*dktaus1*dkws1 \ + survey.ptaus[ztizs1,itaus1,iws2]*ztdkzs1*dktaus1*dkws2 \ + survey.ptaus[ztizs1,itaus2,iws1]*ztdkzs1*dktaus1*dkws1 \ @@ -772,6 +855,7 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal + survey.ptaus[ztizs2,itaus1,iws2]*ztdkzs2*dktaus1*dkws2 \ + survey.ptaus[ztizs2,itaus2,iws1]*ztdkzs2*dktaus2*dkws1 \ + survey.ptaus[ztizs2,itaus2,iws2]*ztdkzs2*dktaus2*dkws2 + pws = survey.pws[ztizs1,iis1,iws1]*ztdkzs1*dkis1*dkws1 \ + survey.pws[ztizs1,iis1,iws2]*ztdkzs1*dkis1*dkws2 \ + survey.pws[ztizs1,iis2,iws1]*ztdkzs1*dkis1*dkws1 \ diff --git a/zdm/scripts/Scattering/fit_w_scat.py b/zdm/scripts/Scattering/fit_w_scat.py new file mode 100644 index 0000000..93cbafa --- /dev/null +++ b/zdm/scripts/Scattering/fit_w_scat.py @@ -0,0 +1,83 @@ +""" +This script creates a 3D distribution. For each values +of width w, it determines the relative contribution of +each scattering and width value to that w. + +This then allows FRB w and tau values to be fit directly, +rather than indirectly via a total effective width. + +This is not a 5D problem (z,DM,w,scat,tau) because +p(w|z,DM) is independent of the tau and w that +contributed to it. + +Nonetheless, p(tau,w) is z-dependent, hence we need a p + +""" + +import os + +from zdm import cosmology as cos +from zdm import figures +from zdm import parameters +from zdm import survey +from zdm import pcosmic +from zdm import iteration as it +from zdm import loading +from zdm import io +from pkg_resources import resource_filename +import numpy as np +from matplotlib import pyplot as plt + +import matplotlib + +defaultsize=14 +ds=4 +font = {'family' : 'Helvetica', + 'weight' : 'normal', + 'size' : defaultsize} +matplotlib.rc('font', **font) + + +def main(): + """ + + """ + + # in case you wish to switch to another output directory + opdir = "Plots/" + if not os.path.exists(opdir): + os.mkdir(opdir) + + # directory where the survey files are located. The below is the default - + # you can leave this out, or change it for a different survey file location. + sdir = os.path.join(resource_filename('zdm', 'data'), 'Surveys') + + survey_name = "CRAFT_average_ICS" + + # make this into a list to initialise multiple surveys art once + names = ["CRAFT_ICS_892","CRAFT_ICS_1300","CRAFT_ICS_1632"] + + repeaters=False + # sets plotting limits + zmax = 2. + dmmax = 2000 + + survey_dict = {"WMETHOD": 3} + state_dict = {} + state_dict["scat"] = {} + state_dict["scat"]["Sbackproject"] = True # turns on backprojection of tau and width for our model + state_dict["scat"]["Sbackproject"] = True + state_dict["width"] = {} + state_dict["width"]["WNInternalBins"] = 100 # sets it to a small quantity + + surveys, grids = loading.surveys_and_grids(survey_names = names,\ + repeaters=repeaters, sdir=sdir,nz=70,ndm=140, + survey_dict = survey_dict, state_dict = state_dict) + + # gets log-likelihoods including tau,w + s=surveys[0] + g=grids[0] + ll1 = it.calc_likelihoods_1D(g,s,Pn=True,pNreps=True,ptauw=False,dolist=0) + ll2 = it.calc_likelihoods_2D(g,s,Pn=True,pNreps=True,ptauw=False,dolist=0) + print(ll1,ll2) +main() diff --git a/zdm/survey.py b/zdm/survey.py index 75d7ac3..78edef9 100644 --- a/zdm/survey.py +++ b/zdm/survey.py @@ -416,9 +416,6 @@ def get_w_coeffs(self,wlist): # convert to log-widths - the bins are in log10 space logwlist = np.log10(wlist) kws=(logwlist-np.log10(self.WMin))/self.dlogw +1. - print(wlist) - print(logwlist) - print(self.wbins) Bin0 = np.where(kws < 0.)[0] kws[Bin0] = 0. From c9dda932d7e0e2e57d701f733b5edb2fb39e7dd1 Mon Sep 17 00:00:00 2001 From: Clancy James Date: Thu, 5 Jun 2025 13:33:58 +0800 Subject: [PATCH 06/20] Implemented p_total_width for ptauw in 1D --- zdm/iteration.py | 139 +++++++++++++++++++-------- zdm/scripts/Scattering/fit_w_scat.py | 4 +- 2 files changed, 99 insertions(+), 44 deletions(-) diff --git a/zdm/iteration.py b/zdm/iteration.py index 0a88190..85f704b 100644 --- a/zdm/iteration.py +++ b/zdm/iteration.py @@ -244,19 +244,6 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True, else: raise ValueError("No non-localised FRBs in this survey, cannot calculate 1D likelihoods") - if ptauw: - # checks which have OK tau values - in general, this is a subset - # ALSO: note that this only checks p(tau,iw | w)! It does NOT - # evaluate p(w)!!! Which is a pretty key thing... - noztaulist = [] - for i,iz in enumerate(nozlist): - if iz in survey.OKTAU: - noztaulist.append(iz) - Wobs = survey.WIDTHs[noztaulist] - Tauobs = survey.TAUs[noztaulist] - Iwobs = survey.IWIDTHs[noztaulist] - ztDMobs=survey.DMEGs[noztaulist] - dmvals=grid.dmvals zvals=grid.zvals @@ -307,8 +294,21 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True, lllist=[llsum] ########### Calculation of p((Tau,w)) ############## + if ptauw: + # checks which have OK tau values - in general, this is a subset + # ALSO: note that this only checks p(tau,iw | w)! It does NOT + # evaluate p(w)!!! Which is a pretty key thing... + noztaulist = [] + inoztaulist = [] + for i,iz in enumerate(nozlist): + if iz in survey.OKTAU: + noztaulist.append(iz) # for direct indexing of survey + inoztaulist.append(i) # for getting a subset of zlist + Wobs = survey.WIDTHs[noztaulist] + Tauobs = survey.TAUs[noztaulist] + Iwobs = survey.IWIDTHs[noztaulist] + ztDMobs=survey.DMEGs[noztaulist] - if ptauw:# Gets linear interpolation coefficients. # This could all be precalculated within the survey. iws1,iws2,dkws1,dkws2 = survey.get_w_coeffs(Wobs) # total width in survey width bins itaus1,itaus2,dktaus1,dktaus2 = survey.get_internal_coeffs(Tauobs) # scattering time tau @@ -341,21 +341,26 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True, + survey.ptaus[:,itaus2,iws1]*dktaus1*dkws1 \ + survey.ptaus[:,itaus2,iws2]*dktaus1*dkws2 - pws = survey.pws[:,iis1,iws1]*dkis1*dkws1 \ + piws = survey.pws[:,iis1,iws1]*dkis1*dkws1 \ + survey.pws[:,iis1,iws2]*dkis1*dkws2 \ + survey.pws[:,iis2,iws1]*dkis1*dkws1 \ + survey.pws[:,iis2,iws2]*dkis1*dkws2 + # we now multiply by the z-dependencies ptaus *= tomult - pws *= tomult + piws *= tomult # sum down the redshift axis to get sum p(tau,w|z)*p(z) ptaus = np.sum(ptaus,axis=0) - pws = np.sum(pws,axis=0) - ptwll = np.sum(np.log10(ptaus) + np.log10(pws)) - llsum += ptwll - lllist.append(ptwll) + piws = np.sum(piws,axis=0) + + llptw = np.sum(np.log10(ptaus)) + llpiw = np.sum(np.log10(piws)) + llsum += llptw + llsum += llpiw + lllist.append(llptw) + lllist.append(llpiw) ############# Assesses total number of FRBs, P(N) ######### # TODO: make the grid tell you the correct nromalisation @@ -449,7 +454,10 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True, ax2 = plt.gca() plt.xlabel("$z$") plt.ylabel("p(b,w|z)") - + if ptauw: + # hold array representing p(w) + dpbws = np.zeros([nw,nz,nfrb]) + for i,b in enumerate(survey.beam_b): #iterate over the grid of weights bEths=Eths/b #this is the only bit that depends on j, but OK also! @@ -473,6 +481,10 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True, pbw_norm += dpbw zpsnr += differential*survey.beam_o[i]*usew + if ptauw: + # record probability of this w summed over all beams for each FRB + dpbws[j,:,:] += dpbw + if doplot and j==5: #arbitrrily plots for FRB iFRB iFRB=0 @@ -493,7 +505,23 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True, plt.legend(fontsize=6) plt.savefig("FRB1_pbw_given_z.png") plt.close() - + + # calculate p(w) + if ptauw: + # we would like to calculate \int p(w|z) p(z) dz + # we begin by calculating p(w|z), below, by normalising for each z + # normalise over all w values for each z + dpbws /= np.sum(dpbws,axis=0) + temp = dpbws[iws1,:,inoztaulist] + temp *= tomult.T + pws = np.sum(temp,axis=1) + bad = np.where(pws == 0.)[0] + pws[bad] = 1.e-10 # prevents nans, but + llpws = np.sum(np.log10(pws)) + llsum += llpws + lllist.append(llpws) + + # normalise by the beam and FRB width values #This ensures that regions with zero probability don't produce nans due to 0/0 OK = np.where(pbw_norm.flatten() > 0.) @@ -693,21 +721,6 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal zlist=survey.zlist else: raise ValueError("No nlocalised FRBs in this survey, cannot calculate 1D likelihoods") - - if ptauw: - # checks which have OK tau values - in general, this is a subset - # ALSO: note that this only checks p(tau,iw | w)! It does NOT - # evaluate p(w)!!! Which is a pretty key thing... - ztaulist = [] - for i,iz in enumerate(zlist): - if iz in survey.OKTAU: - ztaulist.append(iz) - Wobs = survey.WIDTHs[ztaulist] - Tauobs = survey.TAUs[ztaulist] - Iwobs = survey.IWIDTHs[ztaulist] - ztDMobs=survey.DMEGs[ztaulist] - ztZobs=survey.Zs[ztaulist] - zvals=grid.zvals dmvals=grid.dmvals @@ -838,6 +851,21 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal ################ Calculates p(tau,w| total width) ############### if ptauw: + # checks which have OK tau values - in general, this is a subset + # ALSO: note that this only checks p(tau,iw | w)! It does NOT + # evaluate p(w)!!! Which is a pretty key thing... + ztaulist = [] + iztaulist = [] + for i,iz in enumerate(zlist): + if iz in survey.OKTAU: + ztaulist.append(iz) # for direct indexing of survey + iztaulist.append(i) # for getting a subset of zlist + Wobs = survey.WIDTHs[ztaulist] + Tauobs = survey.TAUs[ztaulist] + Iwobs = survey.IWIDTHs[ztaulist] + ztDMobs=survey.DMEGs[ztaulist] + ztZobs=survey.Zs[ztaulist] + # This could all be precalculated within the survey. iws1,iws2,dkws1,dkws2 = survey.get_w_coeffs(Wobs) # total width in survey width bins itaus1,itaus2,dktaus1,dktaus2 = survey.get_internal_coeffs(Tauobs) # scattering time tau @@ -856,7 +884,7 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal + survey.ptaus[ztizs2,itaus2,iws1]*ztdkzs2*dktaus2*dkws1 \ + survey.ptaus[ztizs2,itaus2,iws2]*ztdkzs2*dktaus2*dkws2 - pws = survey.pws[ztizs1,iis1,iws1]*ztdkzs1*dkis1*dkws1 \ + piws = survey.pws[ztizs1,iis1,iws1]*ztdkzs1*dkis1*dkws1 \ + survey.pws[ztizs1,iis1,iws2]*ztdkzs1*dkis1*dkws2 \ + survey.pws[ztizs1,iis2,iws1]*ztdkzs1*dkis1*dkws1 \ + survey.pws[ztizs1,iis2,iws2]*ztdkzs1*dkis1*dkws2 \ @@ -865,11 +893,15 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal + survey.pws[ztizs2,iis2,iws1]*ztdkzs2*dkis2*dkws1 \ + survey.pws[ztizs2,iis2,iws2]*ztdkzs2*dkis2*dkws2 - ptwll = np.sum(np.log10(ptaus) + np.log10(pws)) - llsum += ptwll - lllist.append(ptwll) + llptw = np.sum(np.log10(ptaus)) + llpiw = np.sum(np.log10(piws)) + llsum += llptw + llsum += llpiw + lllist.append(llptw) + lllist.append(llpiw) + - ###### Calculates p(E | z,DM) ######## + ############ Calculates p(s | z,DM) ############# # i.e. the probability of observing an FRB # with energy E given redshift and DM # this calculation ignores beam values @@ -933,6 +965,11 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal # initialised to hold w-b normalisations pbw_norm = 0. + + if ptauw: + # hold array representing p(w) + dpbws = np.zeros([nw,nfrb]) + for i,b in enumerate(survey.beam_b): bEths=Eths/b # array of shape NFRB, 1/b bEobs=bEths*survey.Ss[zlist] @@ -944,6 +981,7 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal temp2=grid.array_cum_lf(bEths[j,:],Emin,Emax,gamma) # * FtoE #one dim in beamshape, one dim in FRB cumulative = temp2.T #*bEths[j,:] #multiplies by beam factors and weight + if zwidths: # a function of redshift usew = w[izs1]*dkzs1 + w[izs2]*dkzs2 @@ -958,10 +996,27 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal # hence, the "cumulative" part cancels dpbw = survey.beam_o[i]*usew*cumulative + + if ptauw: + # record probability of this w summed over all beams for each FRB + dpbws[j,:] += dpbw + pbw_norm += dpbw psnr += differential*survey.beam_o[i]*usew + # calculate p(w) + if ptauw: + # normalise over all w values + dpbws /= np.sum(dpbws,axis=0) + # calculate pws + pws = dpbws[iws1,iztaulist]*dkws1 + dpbws[iws2,iztaulist]*dkws2 + bad = np.where(pws == 0.)[0] + pws[bad] = 1.e-10 # prevents nans, but + llpws = np.sum(np.log10(pws)) + llsum += llpws + lllist.append(llpws) + OK = np.where(pbw_norm > 0.)[0] psnr[OK] /= pbw_norm[OK] diff --git a/zdm/scripts/Scattering/fit_w_scat.py b/zdm/scripts/Scattering/fit_w_scat.py index 93cbafa..480d336 100644 --- a/zdm/scripts/Scattering/fit_w_scat.py +++ b/zdm/scripts/Scattering/fit_w_scat.py @@ -77,7 +77,7 @@ def main(): # gets log-likelihoods including tau,w s=surveys[0] g=grids[0] - ll1 = it.calc_likelihoods_1D(g,s,Pn=True,pNreps=True,ptauw=False,dolist=0) - ll2 = it.calc_likelihoods_2D(g,s,Pn=True,pNreps=True,ptauw=False,dolist=0) + ll1 = it.calc_likelihoods_1D(g,s,Pn=True,pNreps=True,ptauw=True,dolist=0) + ll2 = it.calc_likelihoods_2D(g,s,Pn=True,pNreps=True,psnr=True,ptauw=True,dolist=0) print(ll1,ll2) main() From 203dd4bbcb0e84fde8b29017ef57ef32ff7a528d Mon Sep 17 00:00:00 2001 From: Clancy James Date: Thu, 5 Jun 2025 14:04:15 +0800 Subject: [PATCH 07/20] fixed handling of zvals and dmvals in MCMC --- zdm/scripts/MCMC/MCMC_wrap.py | 10 ++++++---- zdm/survey.py | 11 +++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/zdm/scripts/MCMC/MCMC_wrap.py b/zdm/scripts/MCMC/MCMC_wrap.py index ac98d19..185ed70 100644 --- a/zdm/scripts/MCMC/MCMC_wrap.py +++ b/zdm/scripts/MCMC/MCMC_wrap.py @@ -92,17 +92,19 @@ def main(): grid_params['ndm'] = 1400 grid_params['nz'] = 500 ddm = grid_params['dmmax'] / grid_params['ndm'] - dmvals = (np.arange(grid_params['ndm']) + 1) * ddm + # it would be best to trial initialising a "get_zdm_grid" here to extract z and dm values + dmvals = (np.arange(grid_params['ndm']) + 0.5) * ddm + zvals = (np.arange(nz) + 0.5) * dz if args.files is not None: for survey_name in args.files: - s = survey.load_survey(survey_name, state, dmvals, + s = survey.load_survey(survey_name, state, dmvals, zvals=zvals, sdir=args.sdir, edir=args.edir, rand_DMG=args.rand) surveys[0].append(s) if args.rep_surveys is not None: for survey_name in args.rep_surveys: - s = survey.load_survey(survey_name, state, dmvals, + s = survey.load_survey(survey_name, state, dmvals, zvals=zvals, sdir=args.sdir, edir=args.edir, rand_DMG=args.rand) surveys[1].append(s) @@ -116,4 +118,4 @@ def main(): #============================================================================== -main() \ No newline at end of file +main() diff --git a/zdm/survey.py b/zdm/survey.py index 78edef9..fb20e37 100644 --- a/zdm/survey.py +++ b/zdm/survey.py @@ -64,7 +64,8 @@ def __init__(self, state, survey_name:str, self.dmvals = dmvals self.zvals = zvals self.NDM = dmvals.size - self.NZ = zvals.size + if zvals is not None: + self.NZ = zvals.size self.edir = edir # Load up self.process_survey_file(filename, NFRB, iFRB, min_lat=state.analysis.min_lat, @@ -114,6 +115,8 @@ def init_widths(self): self.wlogmean = self.state.width.Wlogmean self.wlogsigma = self.state.width.Wlogsigma self.width_method = self.meta["WMETHOD"] + if self.width_method == 3 and self.zvals is None: + raise ValueError("Width method 3 requires z-values to be set") self.NInternalBins=self.state.width.WNInternalBins # records scattering information, scaling @@ -1144,7 +1147,7 @@ def calc_relative_sensitivity(DM_frb,DM,w,fbar,t_res,nu_res,Nchan=336,max_idt=No def load_survey(survey_name:str, state:parameters.State, dmvals:np.ndarray, - zvals:np.ndarray, + zvals:np.ndarray=None, sdir:str=None, NFRB:int=None, nbins=None, iFRB:int=0, dummy=False, @@ -1159,7 +1162,7 @@ def load_survey(survey_name:str, state:parameters.State, e.g. CRAFT/FE state (parameters.State): Parameters for the state dmvals (np.ndarray): DM values - zvals (np.ndarray): z values + zvals (np.ndarray,optional): z values sdir (str, optional): Path to survey files. Defaults to None. nbins (int, optional): Sets number of bins for Beam analysis [was NBeams] @@ -1216,7 +1219,7 @@ def load_survey(survey_name:str, state:parameters.State, survey_name, os.path.join(sdir, dfile), dmvals, - zvals, + zvals=zvals, NFRB=NFRB, iFRB=iFRB, edir=edir, rand_DMG=rand_DMG, survey_dict = survey_dict) From 7c2059722e6823a681684d9fb4821768b311695d Mon Sep 17 00:00:00 2001 From: Clancy James Date: Mon, 16 Jun 2025 13:52:23 +0800 Subject: [PATCH 08/20] Updated with SKA science plots, scattering work, and various mods to MCMC process --- .../inputs/Band1AA4_ID_radius_AonT_FoVdeg2 | 197 +++++++ .../inputs/Band1AAstar_ID_radius_AonT_FoVdeg2 | 144 +++++ .../inputs/Band2AA4_ID_radius_AonT_FoVdeg2 | 197 +++++++ .../inputs/Band2AAstar_ID_radius_AonT_FoVdeg2 | 144 +++++ .../inputs/LowAA4_ID_radius_AonT_FoVdeg2 | 512 ++++++++++++++++++ .../inputs/LowAAstar_ID_radius_AonT_FoVdeg2 | 307 +++++++++++ papers/SKA_science/make_zdists.py | 272 ++++++++++ papers/SKA_science/sim_SKA_configs.py | 207 +++++++ .../Scattering/MCMC_inputs/scat_w_only.json | 110 ++++ .../MCMC_inputs/scat_w_only_halflog.json | 110 ++++ papers/Scattering/run_MCMC.sh | 54 ++ papers/Scattering/visualise_mcmc.py | 404 ++++++++++++++ zdm/MCMC.py | 86 ++- zdm/MCMC_analysis.py | 218 ++++++++ zdm/data/Surveys/SKA_low.ecsv | 26 + zdm/data/Surveys/SKA_mid.ecsv | 2 +- zdm/grid.py | 91 +++- zdm/iteration.py | 109 ++-- zdm/parameters.py | 8 +- zdm/scripts/MCMC/MCMC_wrap.py | 47 +- zdm/scripts/MCMC/average_grids.py | 4 +- zdm/scripts/MCMC/check_emcee.py | 49 ++ zdm/scripts/Scattering/fit_w_scat.py | 83 --- zdm/scripts/Scattering/width_scat_dists.py | 208 +++++++ zdm/survey.py | 143 +++-- zdm/survey_data.py | 4 +- 26 files changed, 3528 insertions(+), 208 deletions(-) create mode 100644 papers/SKA_science/inputs/Band1AA4_ID_radius_AonT_FoVdeg2 create mode 100644 papers/SKA_science/inputs/Band1AAstar_ID_radius_AonT_FoVdeg2 create mode 100644 papers/SKA_science/inputs/Band2AA4_ID_radius_AonT_FoVdeg2 create mode 100644 papers/SKA_science/inputs/Band2AAstar_ID_radius_AonT_FoVdeg2 create mode 100644 papers/SKA_science/inputs/LowAA4_ID_radius_AonT_FoVdeg2 create mode 100644 papers/SKA_science/inputs/LowAAstar_ID_radius_AonT_FoVdeg2 create mode 100644 papers/SKA_science/make_zdists.py create mode 100644 papers/SKA_science/sim_SKA_configs.py create mode 100644 papers/Scattering/MCMC_inputs/scat_w_only.json create mode 100644 papers/Scattering/MCMC_inputs/scat_w_only_halflog.json create mode 100755 papers/Scattering/run_MCMC.sh create mode 100644 papers/Scattering/visualise_mcmc.py create mode 100644 zdm/MCMC_analysis.py create mode 100644 zdm/data/Surveys/SKA_low.ecsv create mode 100644 zdm/scripts/MCMC/check_emcee.py delete mode 100644 zdm/scripts/Scattering/fit_w_scat.py create mode 100644 zdm/scripts/Scattering/width_scat_dists.py diff --git a/papers/SKA_science/inputs/Band1AA4_ID_radius_AonT_FoVdeg2 b/papers/SKA_science/inputs/Band1AA4_ID_radius_AonT_FoVdeg2 new file mode 100644 index 0000000..1eed7a5 --- /dev/null +++ b/papers/SKA_science/inputs/Band1AA4_ID_radius_AonT_FoVdeg2 @@ -0,0 +1,197 @@ +M003 0.0229337 5.2 1.78731 +SKA073 0.0287576 11.3 1.78731 +SKA069 0.0301066 17.4 1.78731 +SKA065 0.0314908 23.5 1.78731 +SKA074 0.0322734 29.6 1.78731 +SKA064 0.0493962 35.7 1.78731 +SKA062 0.0522361 41.8 1.78731 +M004 0.0536792 47 1.78731 +M002 0.0536953 52.2 1.78731 +SKA080 0.0588359 58.3 1.78731 +SKA057 0.0612969 64.4 1.78731 +SKA072 0.0641173 70.5 1.78731 +M005 0.0698553 75.7 1.78731 +SKA079 0.0746996 81.8 1.78731 +M000 0.0772682 87 1.78731 +SKA053 0.0820785 93.1 1.78731 +SKA078 0.084211 99.2 1.78731 +SKA056 0.090197 105.3 1.78731 +SKA058 0.0917585 111.4 1.78731 +M029 0.0919264 116.6 1.78731 +SKA052 0.0948587 122.7 1.78731 +SKA075 0.0967177 128.8 1.78731 +M001 0.0967796 134 1.78731 +SKA085 0.102865 140.1 1.78731 +SKA086 0.103027 146.2 1.78731 +SKA068 0.104161 152.3 1.78731 +M006 0.104363 157.5 1.78731 +SKA071 0.105845 163.6 1.78731 +SKA049 0.113596 169.7 1.78731 +SKA087 0.116481 175.8 1.78731 +SKA051 0.124464 181.9 1.78731 +SKA082 0.124806 188 1.78731 +SKA048 0.124825 194.1 1.78731 +SKA059 0.128002 200.2 1.78731 +SKA076 0.129785 206.3 1.78731 +M028 0.133336 211.5 1.78731 +SKA084 0.138523 217.6 1.78731 +SKA055 0.144263 223.7 1.78731 +SKA088 0.144646 229.8 1.78731 +SKA066 0.145543 235.9 1.78731 +SKA002 0.163369 242 1.78731 +SKA050 0.164309 248.1 1.78731 +SKA081 0.168282 254.2 1.78731 +SKA046 0.184472 260.3 1.78731 +M007 0.187911 265.5 1.78731 +M018 0.193092 270.7 1.78731 +M009 0.195283 275.9 1.78731 +M020 0.200588 281.1 1.78731 +SKA067 0.202723 287.2 1.78731 +SKA044 0.204847 293.3 1.78731 +SKA091 0.204847 299.4 1.78731 +M011 0.217555 304.6 1.78731 +SKA093 0.224876 310.7 1.78731 +M027 0.230428 315.9 1.78731 +M026 0.233975 321.1 1.78731 +SKA003 0.235017 327.2 1.78731 +M021 0.23855 332.4 1.78731 +M023 0.248099 337.6 1.78731 +SKA061 0.260541 343.7 1.78731 +SKA047 0.264243 349.8 1.78731 +SKA090 0.264533 355.9 1.78731 +SKA094 0.264957 362 1.78731 +M019 0.26506 367.2 1.78731 +M012 0.272192 372.4 1.78731 +SKA077 0.287022 378.5 1.78731 +SKA042 0.287687 384.6 1.78731 +SKA041 0.288367 390.7 1.78731 +M015 0.295515 395.9 1.72685 +SKA089 0.297098 402 1.7085 +M017 0.302812 407.2 1.64463 +M008 0.320781 412.4 1.46553 +M010 0.34412 417.6 1.27348 +SKA097 0.348738 423.7 1.23998 +M022 0.36023 428.9 1.16213 +M013 0.367855 434.1 1.11445 +M042 0.369698 439.3 1.10336 +M014 0.372178 444.5 1.08871 +M016 0.37412 449.7 1.07744 +M030 0.41843 454.9 0.861326 +SKA054 0.42007 461 0.854614 +SKA043 0.420413 467.1 0.85322 +SKA095 0.421013 473.2 0.85079 +SKA099 0.421498 479.3 0.848833 +SKA039 0.422144 485.4 0.846237 +M024 0.453134 490.6 0.734446 +M025 0.453392 495.8 0.733611 +M031 0.454167 501 0.731109 +M038 0.463492 506.2 0.701987 +M035 0.472106 511.4 0.676604 +SKA092 0.477359 517.5 0.661794 +M036 0.478902 522.7 0.657537 +M034 0.480797 527.9 0.652364 +M041 0.491346 533.1 0.624653 +M040 0.501951 538.3 0.598537 +M039 0.507463 543.5 0.585605 +SKA083 0.50808 549.6 0.584183 +SKA038 0.510061 555.7 0.579654 +M037 0.525402 560.9 0.546298 +M043 0.551129 566.1 0.496486 +M047 0.578283 571.3 0.450954 +SKA001 0.61451 577.4 0.399352 +SKA101 0.617292 583.5 0.39576 +SKA036 0.617347 589.6 0.39569 +SKA098 0.622736 595.7 0.388871 +M032 0.672404 600.9 0.333544 +SKA063 0.743379 607 0.272893 +SKA100 0.744814 613.1 0.271843 +SKA103 0.746557 619.2 0.270575 +SKA035 0.746894 625.3 0.270331 +SKA040 0.879568 631.4 0.194928 +SKA045 0.899304 637.5 0.186466 +SKA102 0.901382 643.6 0.185608 +SKA104 0.902763 649.7 0.18504 +SKA033 0.903571 655.8 0.184709 +M051 0.945107 661 0.168831 +M054 0.99781 666.2 0.151467 +M052 1.06536 671.4 0.132868 +M053 1.0972 676.6 0.125268 +M044 1.1522 681.8 0.113595 +SKA096 1.23822 687.9 0.0983598 +SKA030 1.24375 694 0.0974871 +M033 1.27161 699.2 0.0932622 +SKA106 1.31132 705.3 0.0876993 +M055 1.3232 710.5 0.0861316 +SKA070 1.47322 716.6 0.0694829 +SKA029 1.53943 722.7 0.0636346 +SKA108 1.6114 728.8 0.0580773 +M045 1.81175 734 0.0459427 +M056 1.81589 739.2 0.0457335 +SKA028 1.83355 745.3 0.0448568 +SKA037 1.90031 751.4 0.0417604 +SKA109 1.90851 757.5 0.0414023 +M050 2.06386 762.7 0.035404 +M061 2.11755 767.9 0.0336315 +SKA031 2.35708 774 0.0271434 +SKA032 2.35717 780.1 0.0271414 +SKA111 2.3663 786.2 0.0269323 +M046 2.40993 791.4 0.025966 +M062 2.66658 796.6 0.0212082 +SKA034 2.91776 802.7 0.0177139 +SKA026 3.05526 808.8 0.0161554 +SKA114 3.11259 814.9 0.0155657 +SKA024 3.45592 821 0.0126266 +M057 3.50975 826.2 0.0122422 +M049 3.57741 831.4 0.0117835 +SKA060 3.61416 837.5 0.0115451 +SKA116 3.62845 843.6 0.0114544 +M063 3.70913 848.8 0.0109615 +M060 3.86275 854 0.010107 +SKA117 3.88966 860.1 0.00996759 +M059 3.89241 865.3 0.00995351 +M048 3.98426 870.5 0.00949988 +M058 4.10225 875.7 0.00896126 +SKA107 4.47925 881.8 0.00751628 +SKA023 4.61822 887.9 0.00707073 +SKA115 5.54665 894 0.00490176 +SKA113 5.5537 900.1 0.00488932 +SKA020 6.03462 906.2 0.00414108 +SKA118 6.36567 912.3 0.00372156 +SKA018 6.78374 918.4 0.00327699 +SKA110 6.95645 924.5 0.00311629 +SKA119 8.51436 930.6 0.00208022 +SKA017 8.54988 936.7 0.00206297 +SKA105 8.62976 942.8 0.00202496 +SKA121 10.3204 948.9 0.00141586 +SKA027 10.5415 955 0.00135709 +SKA015 10.5812 961.1 0.00134693 +SKA022 13.4226 967.2 0.000837028 +SKA016 13.6083 973.3 0.00081434 +SKA123 14.4327 979.4 0.000723966 +SKA014 15.6674 985.5 0.000614355 +SKA019 17.0023 991.6 0.000521673 +SKA125 17.0215 997.7 0.000520496 +SKA126 19.5981 1003.8 0.000392632 +SKA025 20.5117 1009.9 0.000358435 +SKA013 21.2236 1016 0.000334792 +SKA112 24.872 1022.1 0.000243777 +SKA128 25.339 1028.2 0.000234874 +SKA012 25.855 1034.3 0.000225592 +SKA122 30.5057 1040.4 0.000162051 +SKA127 30.8785 1046.5 0.000158162 +SKA010 30.9017 1052.6 0.000157924 +SKA124 37.5657 1058.7 0.000106864 +SKA129 38.8109 1064.8 0.000100117 +SKA009 39.1384 1070.9 9.84481e-05 +SKA120 42.851 1077 8.2128e-05 +SKA007 48.1337 1083.1 6.50901e-05 +SKA130 50.6575 1089.2 5.8766e-05 +SKA021 58.091 1095.3 4.46885e-05 +SKA006 59.4713 1101.4 4.26382e-05 +SKA131 60.3257 1107.5 4.14389e-05 +SKA132 71.5725 1113.6 2.94389e-05 +SKA005 72.6958 1119.7 2.85361e-05 +SKA011 72.7501 1125.8 2.84935e-05 +SKA004 88.7118 1131.9 1.91624e-05 +SKA133 89.9489 1138 1.8639e-05 +SKA008 92.6948 1144.1 1.7551e-05 diff --git a/papers/SKA_science/inputs/Band1AAstar_ID_radius_AonT_FoVdeg2 b/papers/SKA_science/inputs/Band1AAstar_ID_radius_AonT_FoVdeg2 new file mode 100644 index 0000000..78f4878 --- /dev/null +++ b/papers/SKA_science/inputs/Band1AAstar_ID_radius_AonT_FoVdeg2 @@ -0,0 +1,144 @@ +M003 0.0229337 5.2 1.78731 +M004 0.0536792 10.4 1.78731 +M002 0.0536953 15.6 1.78731 +M005 0.0698553 20.8 1.78731 +SKA079 0.0746996 26.9 1.78731 +M000 0.0772682 32.1 1.78731 +M029 0.0919264 37.3 1.78731 +SKA075 0.0967177 43.4 1.78731 +M001 0.0967796 48.6 1.78731 +SKA068 0.104161 54.7 1.78731 +M006 0.104363 59.9 1.78731 +SKA049 0.113596 66 1.78731 +SKA082 0.124806 72.1 1.78731 +SKA048 0.124825 78.2 1.78731 +M028 0.133336 83.4 1.78731 +SKA055 0.144263 89.5 1.78731 +SKA050 0.164309 95.6 1.78731 +SKA081 0.168282 101.7 1.78731 +SKA046 0.184472 107.8 1.78731 +M007 0.187911 113 1.78731 +M018 0.193092 118.2 1.78731 +M009 0.195283 123.4 1.78731 +M020 0.200588 128.6 1.78731 +SKA067 0.202723 134.7 1.78731 +SKA091 0.204847 140.8 1.78731 +M011 0.217555 146 1.78731 +M027 0.230428 151.2 1.78731 +M026 0.233975 156.4 1.78731 +M021 0.23855 161.6 1.78731 +M023 0.248099 166.8 1.78731 +SKA061 0.260541 172.9 1.78731 +M019 0.26506 178.1 1.78731 +M012 0.272192 183.3 1.78731 +SKA077 0.287022 189.4 1.78731 +SKA042 0.287687 195.5 1.78731 +SKA041 0.288367 201.6 1.78731 +M015 0.295515 206.8 1.72685 +SKA089 0.297098 212.9 1.7085 +M017 0.302812 218.1 1.64463 +M008 0.320781 223.3 1.46553 +M010 0.34412 228.5 1.27348 +M022 0.36023 233.7 1.16213 +M013 0.367855 238.9 1.11445 +M042 0.369698 244.1 1.10336 +M014 0.372178 249.3 1.08871 +M016 0.37412 254.5 1.07744 +M030 0.41843 259.7 0.861326 +SKA043 0.420413 265.8 0.85322 +SKA095 0.421013 271.9 0.85079 +SKA099 0.421498 278 0.848833 +SKA039 0.422144 284.1 0.846237 +M024 0.453134 289.3 0.734446 +M025 0.453392 294.5 0.733611 +M031 0.454167 299.7 0.731109 +M038 0.463492 304.9 0.701987 +M035 0.472106 310.1 0.676604 +SKA092 0.477359 316.2 0.661794 +M036 0.478902 321.4 0.657537 +M034 0.480797 326.6 0.652364 +M041 0.491346 331.8 0.624653 +M040 0.501951 337 0.598537 +M039 0.507463 342.2 0.585605 +SKA083 0.50808 348.3 0.584183 +SKA038 0.510061 354.4 0.579654 +M037 0.525402 359.6 0.546298 +M043 0.551129 364.8 0.496486 +M047 0.578283 370 0.450954 +SKA001 0.61451 376.1 0.399352 +SKA101 0.617292 382.2 0.39576 +SKA036 0.617347 388.3 0.39569 +SKA098 0.622736 394.4 0.388871 +M032 0.672404 399.6 0.333544 +SKA063 0.743379 405.7 0.272893 +SKA100 0.744814 411.8 0.271843 +SKA103 0.746557 417.9 0.270575 +SKA035 0.746894 424 0.270331 +SKA040 0.879568 430.1 0.194928 +SKA045 0.899304 436.2 0.186466 +SKA102 0.901382 442.3 0.185608 +SKA104 0.902763 448.4 0.18504 +SKA033 0.903571 454.5 0.184709 +M051 0.945107 459.7 0.168831 +M054 0.99781 464.9 0.151467 +M052 1.06536 470.1 0.132868 +M053 1.0972 475.3 0.125268 +M044 1.1522 480.5 0.113595 +SKA096 1.23822 486.6 0.0983598 +SKA030 1.24375 492.7 0.0974871 +M033 1.27161 497.9 0.0932622 +SKA106 1.31132 504 0.0876993 +M055 1.3232 509.2 0.0861316 +SKA070 1.47322 515.3 0.0694829 +SKA029 1.53943 521.4 0.0636346 +SKA108 1.6114 527.5 0.0580773 +M045 1.81175 532.7 0.0459427 +M056 1.81589 537.9 0.0457335 +SKA028 1.83355 544 0.0448568 +SKA037 1.90031 550.1 0.0417604 +SKA109 1.90851 556.2 0.0414023 +M050 2.06386 561.4 0.035404 +M061 2.11755 566.6 0.0336315 +SKA031 2.35708 572.7 0.0271434 +SKA032 2.35717 578.8 0.0271414 +SKA111 2.3663 584.9 0.0269323 +M046 2.40993 590.1 0.025966 +M062 2.66658 595.3 0.0212082 +SKA034 2.91776 601.4 0.0177139 +SKA026 3.05526 607.5 0.0161554 +SKA114 3.11259 613.6 0.0155657 +SKA024 3.45592 619.7 0.0126266 +M057 3.50975 624.9 0.0122422 +M049 3.57741 630.1 0.0117835 +SKA060 3.61416 636.2 0.0115451 +SKA116 3.62845 642.3 0.0114544 +M063 3.70913 647.5 0.0109615 +M060 3.86275 652.7 0.010107 +SKA117 3.88966 658.8 0.00996759 +M059 3.89241 664 0.00995351 +M048 3.98426 669.2 0.00949988 +M058 4.10225 674.4 0.00896126 +SKA107 4.47925 680.5 0.00751628 +SKA023 4.61822 686.6 0.00707073 +SKA115 5.54665 692.7 0.00490176 +SKA113 5.5537 698.8 0.00488932 +SKA020 6.03462 704.9 0.00414108 +SKA118 6.36567 711 0.00372156 +SKA018 6.78374 717.1 0.00327699 +SKA110 6.95645 723.2 0.00311629 +SKA119 8.51436 729.3 0.00208022 +SKA017 8.54988 735.4 0.00206297 +SKA105 8.62976 741.5 0.00202496 +SKA121 10.3204 747.6 0.00141586 +SKA027 10.5415 753.7 0.00135709 +SKA015 10.5812 759.8 0.00134693 +SKA022 13.4226 765.9 0.000837028 +SKA016 13.6083 772 0.00081434 +SKA123 14.4327 778.1 0.000723966 +SKA014 15.6674 784.2 0.000614355 +SKA019 17.0023 790.3 0.000521673 +SKA125 17.0215 796.4 0.000520496 +SKA126 19.5981 802.5 0.000392632 +SKA025 20.5117 808.6 0.000358435 +SKA013 21.2236 814.7 0.000334792 +SKA008 92.6948 820.8 1.7551e-05 diff --git a/papers/SKA_science/inputs/Band2AA4_ID_radius_AonT_FoVdeg2 b/papers/SKA_science/inputs/Band2AA4_ID_radius_AonT_FoVdeg2 new file mode 100644 index 0000000..bb705b0 --- /dev/null +++ b/papers/SKA_science/inputs/Band2AA4_ID_radius_AonT_FoVdeg2 @@ -0,0 +1,197 @@ +M003 0.0229337 6.34 0.583611 +SKA073 0.0287576 16.5 0.583611 +SKA069 0.0301066 26.66 0.583611 +SKA065 0.0314908 36.82 0.583611 +SKA074 0.0322734 46.98 0.583611 +SKA064 0.0493962 57.14 0.583611 +SKA062 0.0522361 67.3 0.583611 +M004 0.0536792 73.64 0.583611 +M002 0.0536953 79.98 0.583611 +SKA080 0.0588359 90.14 0.583611 +SKA057 0.0612969 100.3 0.583611 +SKA072 0.0641173 110.46 0.583611 +M005 0.0698553 116.8 0.583611 +SKA079 0.0746996 126.96 0.583611 +M000 0.0772682 133.3 0.583611 +SKA053 0.0820785 143.46 0.583611 +SKA078 0.084211 153.62 0.583611 +SKA056 0.090197 163.78 0.583611 +SKA058 0.0917585 173.94 0.583611 +M029 0.0919264 180.28 0.583611 +SKA052 0.0948587 190.44 0.583611 +SKA075 0.0967177 200.6 0.583611 +M001 0.0967796 206.94 0.583611 +SKA085 0.102865 217.1 0.583611 +SKA086 0.103027 227.26 0.583611 +SKA068 0.104161 237.42 0.583611 +M006 0.104363 243.76 0.583611 +SKA071 0.105845 253.92 0.583611 +SKA049 0.113596 264.08 0.583611 +SKA087 0.116481 274.24 0.583611 +SKA051 0.124464 284.4 0.583611 +SKA082 0.124806 294.56 0.583611 +SKA048 0.124825 304.72 0.583611 +SKA059 0.128002 314.88 0.583611 +SKA076 0.129785 325.04 0.583611 +M028 0.133336 331.38 0.583611 +SKA084 0.138523 341.54 0.583611 +SKA055 0.144263 351.7 0.583611 +SKA088 0.144646 361.86 0.583611 +SKA066 0.145543 372.02 0.583611 +SKA002 0.163369 382.18 0.583611 +SKA050 0.164309 392.34 0.583611 +SKA081 0.168282 402.5 0.583611 +SKA046 0.184472 412.66 0.583611 +M007 0.187911 419 0.583611 +M018 0.193092 425.34 0.583611 +M009 0.195283 431.68 0.583611 +M020 0.200588 438.02 0.583611 +SKA067 0.202723 448.18 0.583611 +SKA044 0.204847 458.34 0.583611 +SKA091 0.204847 468.5 0.583611 +M011 0.217555 474.84 0.583611 +SKA093 0.224876 485 0.583611 +M027 0.230428 491.34 0.583611 +M026 0.233975 497.68 0.583611 +SKA003 0.235017 507.84 0.583611 +M021 0.23855 514.18 0.583611 +M023 0.248099 520.52 0.583611 +SKA061 0.260541 530.68 0.583611 +SKA047 0.264243 540.84 0.583611 +SKA090 0.264533 551 0.583611 +SKA094 0.264957 561.16 0.583611 +M019 0.26506 567.5 0.583611 +M012 0.272192 573.84 0.583611 +SKA077 0.287022 584 0.583611 +SKA042 0.287687 594.16 0.583611 +SKA041 0.288367 604.32 0.583611 +M015 0.295515 610.66 0.563869 +SKA089 0.297098 620.82 0.557876 +M017 0.302812 627.16 0.537021 +M008 0.320781 633.5 0.478542 +M010 0.34412 639.84 0.415832 +SKA097 0.348738 650 0.404892 +M022 0.36023 656.34 0.37947 +M013 0.367855 662.68 0.363902 +M042 0.369698 669.02 0.360282 +M014 0.372178 675.36 0.355497 +M016 0.37412 681.7 0.351816 +M030 0.41843 688.04 0.281249 +SKA054 0.42007 698.2 0.279058 +SKA043 0.420413 708.36 0.278602 +SKA095 0.421013 718.52 0.277809 +SKA099 0.421498 728.68 0.27717 +SKA039 0.422144 738.84 0.276322 +M024 0.453134 745.18 0.239819 +M025 0.453392 751.52 0.239546 +M031 0.454167 757.86 0.23873 +M038 0.463492 764.2 0.22922 +M035 0.472106 770.54 0.220932 +SKA092 0.477359 780.7 0.216096 +M036 0.478902 787.04 0.214706 +M034 0.480797 793.38 0.213017 +M041 0.491346 799.72 0.203968 +M040 0.501951 806.06 0.195441 +M039 0.507463 812.4 0.191218 +SKA083 0.50808 822.56 0.190754 +SKA038 0.510061 832.72 0.189275 +M037 0.525402 839.06 0.178383 +M043 0.551129 845.4 0.162118 +M047 0.578283 851.74 0.14725 +SKA001 0.61451 861.9 0.130401 +SKA101 0.617292 872.06 0.129228 +SKA036 0.617347 882.22 0.129205 +SKA098 0.622736 892.38 0.126978 +M032 0.672404 898.72 0.108912 +SKA063 0.743379 908.88 0.089108 +SKA100 0.744814 919.04 0.088765 +SKA103 0.746557 929.2 0.088351 +SKA035 0.746894 939.36 0.0882713 +SKA040 0.879568 949.52 0.06365 +SKA045 0.899304 959.68 0.0608869 +SKA102 0.901382 969.84 0.0606065 +SKA104 0.902763 980 0.0604213 +SKA033 0.903571 990.16 0.0603132 +M051 0.945107 996.5 0.0551284 +M054 0.99781 1002.84 0.0494586 +M052 1.06536 1009.18 0.0433855 +M053 1.0972 1015.52 0.040904 +M044 1.1522 1021.86 0.0370921 +SKA096 1.23822 1032.02 0.0321175 +SKA030 1.24375 1042.18 0.0318325 +M033 1.27161 1048.52 0.030453 +SKA106 1.31132 1058.68 0.0286365 +M055 1.3232 1065.02 0.0281246 +SKA070 1.47322 1075.18 0.0226883 +SKA029 1.53943 1085.34 0.0207787 +SKA108 1.6114 1095.5 0.018964 +M045 1.81175 1101.84 0.0150017 +M056 1.81589 1108.18 0.0149334 +SKA028 1.83355 1118.34 0.0146471 +SKA037 1.90031 1128.5 0.013636 +SKA109 1.90851 1138.66 0.0135191 +M050 2.06386 1145 0.0115605 +M061 2.11755 1151.34 0.0109817 +SKA031 2.35708 1161.5 0.00886316 +SKA032 2.35717 1171.66 0.00886248 +SKA111 2.3663 1181.82 0.00879423 +M046 2.40993 1188.16 0.00847868 +M062 2.66658 1194.5 0.00692513 +SKA034 2.91776 1204.66 0.00578413 +SKA026 3.05526 1214.82 0.00527522 +SKA114 3.11259 1224.98 0.00508269 +SKA024 3.45592 1235.14 0.00412297 +M057 3.50975 1241.48 0.00399747 +M049 3.57741 1247.82 0.00384769 +SKA060 3.61416 1257.98 0.00376984 +SKA116 3.62845 1268.14 0.0037402 +M063 3.70913 1274.48 0.00357926 +M060 3.86275 1280.82 0.00330023 +SKA117 3.88966 1290.98 0.00325472 +M059 3.89241 1297.32 0.00325013 +M048 3.98426 1303.66 0.003102 +M058 4.10225 1310 0.00292613 +SKA107 4.47925 1320.16 0.00245429 +SKA023 4.61822 1330.32 0.00230881 +SKA115 5.54665 1340.48 0.00160057 +SKA113 5.5537 1350.64 0.00159651 +SKA020 6.03462 1360.8 0.00135219 +SKA118 6.36567 1370.96 0.0012152 +SKA018 6.78374 1381.12 0.00107004 +SKA110 6.95645 1391.28 0.00101756 +SKA119 8.51436 1401.44 0.000679256 +SKA017 8.54988 1411.6 0.000673623 +SKA105 8.62976 1421.76 0.000661211 +SKA121 10.3204 1431.92 0.000462322 +SKA027 10.5415 1442.08 0.000443131 +SKA015 10.5812 1452.24 0.000439812 +SKA022 13.4226 1462.4 0.000273315 +SKA016 13.6083 1472.56 0.000265907 +SKA123 14.4327 1482.72 0.000236397 +SKA014 15.6674 1492.88 0.000200606 +SKA019 17.0023 1503.04 0.000170342 +SKA125 17.0215 1513.2 0.000169958 +SKA126 19.5981 1523.36 0.000128206 +SKA025 20.5117 1533.52 0.00011704 +SKA013 21.2236 1543.68 0.00010932 +SKA112 24.872 1553.84 7.96005e-05 +SKA128 25.339 1564 7.66935e-05 +SKA012 25.855 1574.16 7.36628e-05 +SKA122 30.5057 1584.32 5.29146e-05 +SKA127 30.8785 1594.48 5.16446e-05 +SKA010 30.9017 1604.64 5.15671e-05 +SKA124 37.5657 1614.8 3.48943e-05 +SKA129 38.8109 1624.96 3.26911e-05 +SKA009 39.1384 1635.12 3.21463e-05 +SKA120 42.851 1645.28 2.68173e-05 +SKA007 48.1337 1655.44 2.12539e-05 +SKA130 50.6575 1665.6 1.91889e-05 +SKA021 58.091 1675.76 1.45922e-05 +SKA006 59.4713 1685.92 1.39227e-05 +SKA131 60.3257 1696.08 1.35311e-05 +SKA132 71.5725 1706.24 9.61269e-06 +SKA005 72.6958 1716.4 9.31791e-06 +SKA011 72.7501 1726.56 9.30401e-06 +SKA004 88.7118 1736.72 6.25712e-06 +SKA133 89.9489 1746.88 6.08619e-06 +SKA008 92.6948 1757.04 5.73095e-06 diff --git a/papers/SKA_science/inputs/Band2AAstar_ID_radius_AonT_FoVdeg2 b/papers/SKA_science/inputs/Band2AAstar_ID_radius_AonT_FoVdeg2 new file mode 100644 index 0000000..3d82598 --- /dev/null +++ b/papers/SKA_science/inputs/Band2AAstar_ID_radius_AonT_FoVdeg2 @@ -0,0 +1,144 @@ +M003 0.0229337 6.34 0.583611 +M004 0.0536792 12.68 0.583611 +M002 0.0536953 19.02 0.583611 +M005 0.0698553 25.36 0.583611 +SKA079 0.0746996 35.52 0.583611 +M000 0.0772682 41.86 0.583611 +M029 0.0919264 48.2 0.583611 +SKA075 0.0967177 58.36 0.583611 +M001 0.0967796 64.7 0.583611 +SKA068 0.104161 74.86 0.583611 +M006 0.104363 81.2 0.583611 +SKA049 0.113596 91.36 0.583611 +SKA082 0.124806 101.52 0.583611 +SKA048 0.124825 111.68 0.583611 +M028 0.133336 118.02 0.583611 +SKA055 0.144263 128.18 0.583611 +SKA050 0.164309 138.34 0.583611 +SKA081 0.168282 148.5 0.583611 +SKA046 0.184472 158.66 0.583611 +M007 0.187911 165 0.583611 +M018 0.193092 171.34 0.583611 +M009 0.195283 177.68 0.583611 +M020 0.200588 184.02 0.583611 +SKA067 0.202723 194.18 0.583611 +SKA091 0.204847 204.34 0.583611 +M011 0.217555 210.68 0.583611 +M027 0.230428 217.02 0.583611 +M026 0.233975 223.36 0.583611 +M021 0.23855 229.7 0.583611 +M023 0.248099 236.04 0.583611 +SKA061 0.260541 246.2 0.583611 +M019 0.26506 252.54 0.583611 +M012 0.272192 258.88 0.583611 +SKA077 0.287022 269.04 0.583611 +SKA042 0.287687 279.2 0.583611 +SKA041 0.288367 289.36 0.583611 +M015 0.295515 295.7 0.563869 +SKA089 0.297098 305.86 0.557876 +M017 0.302812 312.2 0.537021 +M008 0.320781 318.54 0.478542 +M010 0.34412 324.88 0.415832 +M022 0.36023 331.22 0.37947 +M013 0.367855 337.56 0.363902 +M042 0.369698 343.9 0.360282 +M014 0.372178 350.24 0.355497 +M016 0.37412 356.58 0.351816 +M030 0.41843 362.92 0.281249 +SKA043 0.420413 373.08 0.278602 +SKA095 0.421013 383.24 0.277809 +SKA099 0.421498 393.4 0.27717 +SKA039 0.422144 403.56 0.276322 +M024 0.453134 409.9 0.239819 +M025 0.453392 416.24 0.239546 +M031 0.454167 422.58 0.23873 +M038 0.463492 428.92 0.22922 +M035 0.472106 435.26 0.220932 +SKA092 0.477359 445.42 0.216096 +M036 0.478902 451.76 0.214706 +M034 0.480797 458.1 0.213017 +M041 0.491346 464.44 0.203968 +M040 0.501951 470.78 0.195441 +M039 0.507463 477.12 0.191218 +SKA083 0.50808 487.28 0.190754 +SKA038 0.510061 497.44 0.189275 +M037 0.525402 503.78 0.178383 +M043 0.551129 510.12 0.162118 +M047 0.578283 516.46 0.14725 +SKA001 0.61451 526.62 0.130401 +SKA101 0.617292 536.78 0.129228 +SKA036 0.617347 546.94 0.129205 +SKA098 0.622736 557.1 0.126978 +M032 0.672404 563.44 0.108912 +SKA063 0.743379 573.6 0.089108 +SKA100 0.744814 583.76 0.088765 +SKA103 0.746557 593.92 0.088351 +SKA035 0.746894 604.08 0.0882713 +SKA040 0.879568 614.24 0.06365 +SKA045 0.899304 624.4 0.0608869 +SKA102 0.901382 634.56 0.0606065 +SKA104 0.902763 644.72 0.0604213 +SKA033 0.903571 654.88 0.0603132 +M051 0.945107 661.22 0.0551284 +M054 0.99781 667.56 0.0494586 +M052 1.06536 673.9 0.0433855 +M053 1.0972 680.24 0.040904 +M044 1.1522 686.58 0.0370921 +SKA096 1.23822 696.74 0.0321175 +SKA030 1.24375 706.9 0.0318325 +M033 1.27161 713.24 0.030453 +SKA106 1.31132 723.4 0.0286365 +M055 1.3232 729.74 0.0281246 +SKA070 1.47322 739.9 0.0226883 +SKA029 1.53943 750.06 0.0207787 +SKA108 1.6114 760.22 0.018964 +M045 1.81175 766.56 0.0150017 +M056 1.81589 772.9 0.0149334 +SKA028 1.83355 783.06 0.0146471 +SKA037 1.90031 793.22 0.013636 +SKA109 1.90851 803.38 0.0135191 +M050 2.06386 809.72 0.0115605 +M061 2.11755 816.06 0.0109817 +SKA031 2.35708 826.22 0.00886316 +SKA032 2.35717 836.38 0.00886248 +SKA111 2.3663 846.54 0.00879423 +M046 2.40993 852.88 0.00847868 +M062 2.66658 859.22 0.00692513 +SKA034 2.91776 869.38 0.00578413 +SKA026 3.05526 879.54 0.00527522 +SKA114 3.11259 889.7 0.00508269 +SKA024 3.45592 899.86 0.00412297 +M057 3.50975 906.2 0.00399747 +M049 3.57741 912.54 0.00384769 +SKA060 3.61416 922.7 0.00376984 +SKA116 3.62845 932.86 0.0037402 +M063 3.70913 939.2 0.00357926 +M060 3.86275 945.54 0.00330023 +SKA117 3.88966 955.7 0.00325472 +M059 3.89241 962.04 0.00325013 +M048 3.98426 968.38 0.003102 +M058 4.10225 974.72 0.00292613 +SKA107 4.47925 984.88 0.00245429 +SKA023 4.61822 995.04 0.00230881 +SKA115 5.54665 1005.2 0.00160057 +SKA113 5.5537 1015.36 0.00159651 +SKA020 6.03462 1025.52 0.00135219 +SKA118 6.36567 1035.68 0.0012152 +SKA018 6.78374 1045.84 0.00107004 +SKA110 6.95645 1056 0.00101756 +SKA119 8.51436 1066.16 0.000679256 +SKA017 8.54988 1076.32 0.000673623 +SKA105 8.62976 1086.48 0.000661211 +SKA121 10.3204 1096.64 0.000462322 +SKA027 10.5415 1106.8 0.000443131 +SKA015 10.5812 1116.96 0.000439812 +SKA022 13.4226 1127.12 0.000273315 +SKA016 13.6083 1137.28 0.000265907 +SKA123 14.4327 1147.44 0.000236397 +SKA014 15.6674 1157.6 0.000200606 +SKA019 17.0023 1167.76 0.000170342 +SKA125 17.0215 1177.92 0.000169958 +SKA126 19.5981 1188.08 0.000128206 +SKA025 20.5117 1198.24 0.00011704 +SKA013 21.2236 1208.4 0.00010932 +SKA008 92.6948 1218.56 5.73095e-06 diff --git a/papers/SKA_science/inputs/LowAA4_ID_radius_AonT_FoVdeg2 b/papers/SKA_science/inputs/LowAA4_ID_radius_AonT_FoVdeg2 new file mode 100644 index 0000000..b410c3a --- /dev/null +++ b/papers/SKA_science/inputs/LowAA4_ID_radius_AonT_FoVdeg2 @@ -0,0 +1,512 @@ +C5 0.00576224 1.97763 5.12201 +C59 0.0369839 3.95525 5.12201 +C20 0.0415433 5.93287 5.12201 +C8 0.0523278 7.91051 5.12201 +C15 0.0549779 9.8881 5.12201 +C37 0.0742842 11.8657 5.12201 +C19 0.0817277 13.8434 5.12201 +C3 0.0850761 15.821 5.12201 +C44 0.0862676 17.7986 5.12201 +C63 0.0918574 19.7763 5.12201 +C14 0.0942051 21.7539 5.12201 +C41 0.103418 23.7315 5.12201 +C13 0.105762 25.7091 5.12201 +C35 0.114899 27.6868 5.12201 +C75 0.115194 29.6644 5.12201 +C54 0.129174 31.642 5.12201 +C17 0.12994 33.6196 5.12201 +C9 0.130395 35.5973 5.12201 +C29 0.132841 37.5749 5.12201 +C38 0.136805 39.5525 5.12201 +C7 0.137425 41.5302 5.12201 +C70 0.138274 43.5077 5.12201 +C31 0.141194 45.4854 5.12201 +C57 0.141603 47.4628 5.12201 +C4 0.144913 49.4408 5.12201 +C2 0.145246 51.4183 5.12201 +C11 0.14526 53.3958 5.12201 +C53 0.152397 55.3737 5.12201 +C23 0.152982 57.3512 5.12201 +C60 0.15665 59.3287 5.12201 +C52 0.160377 61.3062 5.12201 +C30 0.167195 63.2842 5.12201 +C21 0.167956 65.2617 5.12201 +C24 0.169527 67.2392 5.12201 +C67 0.170527 69.2167 5.12201 +C69 0.170992 71.1946 5.12201 +C18 0.173752 73.1721 5.12201 +C58 0.174266 75.1496 5.12201 +C27 0.174354 77.1276 5.12201 +C71 0.183848 79.1051 5.12201 +C33 0.184147 81.0826 5.12201 +C48 0.184878 83.0601 5.12201 +C28 0.187098 85.0381 5.12201 +C36 0.187165 87.0156 5.12201 +C10 0.191743 88.9931 5.12201 +C56 0.191904 90.9706 5.12201 +C51 0.196303 92.9485 5.12201 +C1 0.202648 94.926 5.12201 +C6 0.202739 96.9035 5.12201 +C22 0.202784 98.881 5.12201 +C39 0.202795 100.859 5.12201 +C25 0.206082 102.837 5.12201 +C26 0.207547 104.814 5.12201 +C46 0.210027 106.792 5.12201 +C47 0.210896 108.769 5.12201 +C16 0.212845 110.747 5.12201 +C34 0.214628 112.724 5.12201 +C12 0.214654 114.702 5.12201 +C42 0.215397 116.68 5.12201 +C64 0.216443 118.657 5.12201 +C66 0.220276 120.635 5.12201 +C43 0.220871 122.613 5.12201 +C49 0.221708 124.59 5.12201 +C62 0.222152 126.568 5.12201 +C61 0.223793 128.546 5.12201 +C45 0.224835 130.523 5.12201 +C72 0.225332 132.501 5.12201 +C55 0.227457 134.478 5.12201 +C50 0.22826 136.456 5.12201 +C73 0.229041 138.434 5.12201 +C74 0.232021 140.411 5.12201 +C40 0.23317 142.389 5.12201 +C65 0.235341 144.367 5.12201 +C32 0.238894 146.344 5.12201 +C100 0.238903 148.322 5.12201 +C68 0.240166 150.3 5.12201 +C116 0.240189 152.277 5.12201 +C108 0.251241 154.255 5.12201 +C120 0.255888 156.232 5.12201 +C117 0.259439 158.21 5.12201 +C123 0.260943 160.188 5.12201 +C148 0.262085 162.165 5.12201 +C79 0.262128 164.143 5.12201 +C111 0.262686 166.121 5.12201 +C131 0.264327 168.098 5.12201 +C142 0.264605 170.076 5.12201 +C150 0.265973 172.053 5.12201 +C105 0.269939 174.031 5.12201 +C95 0.273682 176.009 5.12201 +C102 0.275264 177.986 5.12201 +C97 0.276116 179.964 5.12201 +C87 0.278431 181.942 5.12201 +C112 0.28345 183.919 5.12201 +C147 0.285824 185.897 5.12201 +C76 0.288097 187.875 5.12201 +C140 0.288119 189.852 5.12201 +C84 0.288389 191.83 5.12201 +C82 0.289496 193.807 5.12201 +C88 0.289685 195.785 5.12201 +C89 0.290275 197.763 5.12201 +C86 0.291663 199.74 5.12201 +C145 0.292323 201.718 5.12201 +C121 0.292419 203.696 5.12201 +C200 0.294231 205.673 5.12201 +C107 0.297548 207.651 5.12201 +C137 0.299766 209.628 5.12201 +C114 0.300545 211.606 5.12201 +C103 0.301607 213.583 5.12201 +C122 0.302478 215.561 5.12201 +C85 0.303895 217.539 5.12201 +C119 0.305865 219.516 5.12201 +C146 0.306192 221.494 5.12201 +C92 0.306577 223.472 5.12201 +C110 0.306609 225.449 5.12201 +C98 0.307402 227.427 5.12201 +C99 0.308294 229.404 5.12201 +C128 0.311374 231.382 5.12201 +C139 0.314429 233.36 5.12201 +C130 0.317243 235.337 5.12201 +C132 0.319367 237.315 5.12201 +C77 0.321173 239.293 5.12201 +C124 0.321206 241.27 5.12201 +C135 0.321546 243.248 5.12201 +C129 0.321763 245.225 5.12201 +C125 0.326198 247.203 5.12201 +C101 0.326702 249.181 5.12201 +C91 0.331314 251.158 5.12201 +C109 0.332076 253.136 5.12201 +C134 0.332377 255.114 5.12201 +C104 0.33599 257.091 5.12201 +C126 0.336287 259.069 5.12201 +C90 0.336931 261.047 5.12201 +C113 0.337077 263.024 5.12201 +C80 0.338807 265.002 5.12201 +C144 0.339712 266.979 5.12201 +C115 0.342923 268.957 5.12201 +C78 0.344542 270.935 5.12201 +C96 0.345244 272.912 5.12201 +C118 0.345323 274.89 5.12201 +C81 0.345599 276.868 5.12201 +C106 0.345887 278.845 5.12201 +C149 0.348595 280.823 5.12201 +C94 0.34968 282.801 5.12201 +C133 0.351003 284.778 5.12201 +C93 0.351674 286.756 5.12201 +C151 0.354576 288.733 5.12201 +C141 0.354655 290.711 5.12201 +C136 0.357334 292.689 5.12201 +C127 0.357673 294.666 5.12201 +C83 0.359185 296.644 5.12201 +C138 0.35924 298.622 5.12201 +C197 0.363273 300.599 5.12201 +C178 0.369376 302.577 5.12201 +C143 0.371144 304.555 5.12201 +C167 0.374005 306.532 5.12201 +C202 0.377233 308.51 5.12201 +C152 0.378404 310.487 5.12201 +C186 0.381145 312.465 5.12201 +C181 0.382885 314.442 5.12201 +C182 0.384482 316.42 5.12201 +C191 0.386202 318.397 5.12201 +C162 0.388627 320.375 5.12201 +C206 0.391895 322.353 5.12201 +C185 0.397118 324.33 5.12201 +C180 0.399151 326.308 5.12201 +C168 0.402481 328.286 5.12201 +C187 0.404253 330.263 5.12201 +C205 0.40558 332.241 5.12201 +C174 0.407227 334.219 5.12201 +C169 0.407595 336.196 5.12201 +C158 0.409651 338.174 5.12201 +C171 0.414559 340.151 5.12201 +C194 0.415997 342.129 5.12201 +C172 0.417054 344.107 5.12201 +C210 0.420965 346.084 5.12201 +C176 0.422348 348.062 5.12201 +C214 0.424527 350.04 5.12201 +C196 0.424649 352.017 5.12201 +C179 0.428255 353.995 5.12201 +C188 0.430149 355.973 5.12201 +C170 0.431131 357.95 5.12201 +C177 0.431906 359.928 5.12201 +C156 0.433198 361.905 5.12201 +C166 0.43327 363.883 5.12201 +C198 0.437027 365.861 5.12201 +C195 0.43915 367.838 5.12201 +C157 0.441369 369.816 5.12201 +C189 0.441614 371.794 5.12201 +C163 0.444326 373.771 5.12201 +C160 0.446469 375.749 5.12201 +C224 0.447065 377.727 5.12201 +C164 0.448648 379.704 5.08931 +C218 0.451107 381.682 5.03397 +C175 0.456321 383.659 4.91959 +C221 0.456758 385.637 4.91019 +C209 0.456879 387.615 4.90758 +C154 0.458576 389.592 4.87133 +C155 0.459584 391.57 4.84998 +C192 0.460215 393.548 4.83669 +C193 0.461054 395.525 4.81911 +C190 0.466042 397.503 4.7165 +C203 0.466715 399.481 4.70291 +C199 0.471507 401.458 4.6078 +C211 0.472675 403.436 4.58506 +C165 0.473166 405.413 4.57555 +C159 0.473566 407.391 4.56782 +C184 0.473864 409.369 4.56208 +C204 0.475849 411.346 4.5241 +C161 0.480174 413.324 4.44297 +C153 0.481096 415.302 4.42595 +C183 0.481755 417.279 4.41385 +C173 0.482394 419.257 4.40217 +C215 0.482991 421.234 4.39129 +C201 0.485276 423.212 4.35003 +C222 0.487366 425.189 4.3128 +C220 0.490244 427.167 4.26232 +C223 0.498014 429.145 4.13035 +C213 0.503017 431.122 4.0486 +C207 0.511351 433.1 3.91771 +C208 0.513384 435.077 3.88674 +C212 0.51713 437.055 3.83063 +C216 0.521526 439.033 3.76633 +C219 0.53608 441.01 3.5646 +C217 0.544352 442.988 3.45709 +S1-3 0.601116 444.966 2.835 +E1-1 0.6089 446.943 2.76298 +N1-5 0.641704 448.921 2.48772 +S1-2 0.644623 450.899 2.46524 +S1-5 0.646693 452.876 2.44948 +E1-2 0.647829 454.854 2.4409 +N1-6 0.652857 456.831 2.40344 +N1-3 0.655539 458.809 2.38382 +E1-3 0.660003 460.787 2.35168 +N1-4 0.680401 462.764 2.21279 +S1-4 0.68313 464.742 2.19515 +E1-5 0.684296 466.722 2.18767 +E1-4 0.687799 468.696 2.16544 +S1-6 0.689039 470.675 2.15766 +S1-1 0.689533 472.654 2.15457 +N1-2 0.690869 474.628 2.14624 +N1-1 0.73232 476.607 1.91015 +E1-6 0.735933 478.586 1.89144 +S2-3 0.8648 480.565 1.36974 +E2-3 0.873016 482.539 1.34408 +S2-4 0.877364 484.518 1.33079 +S2-2 0.893395 486.497 1.28346 +N2-6 0.902607 488.476 1.2574 +E2-1 0.908209 490.45 1.24193 +N2-5 0.913589 492.429 1.22735 +E2-6 0.926287 494.408 1.19393 +E2-4 0.9283 496.382 1.18876 +N2-4 0.937329 498.361 1.16597 +N2-3 0.955956 500.34 1.12097 +E2-2 0.971062 502.318 1.08637 +N2-1 0.971652 504.293 1.08505 +S2-5 0.976536 506.272 1.07422 +N2-2 0.981157 508.251 1.06413 +S2-6 0.988134 510.229 1.04915 +E2-5 0.989237 512.204 1.04681 +S2-1 0.99669 514.183 1.03122 +N3-5 1.24813 516.161 0.657583 +E3-2 1.25937 518.136 0.645897 +N3-6 1.26313 520.115 0.642058 +E3-4 1.26416 522.093 0.641012 +S3-1 1.2725 524.072 0.632637 +S3-3 1.27962 526.047 0.625616 +N3-3 1.2846 528.026 0.620775 +E3-6 1.29312 530.004 0.612622 +S3-2 1.29788 531.983 0.608137 +E3-1 1.30427 533.958 0.602192 +E3-5 1.30687 535.936 0.599799 +N3-4 1.30871 537.915 0.598113 +S3-5 1.32547 539.89 0.583083 +N3-2 1.33132 541.868 0.57797 +S3-4 1.33756 543.847 0.57259 +E3-3 1.34962 545.826 0.562402 +N3-1 1.35458 547.801 0.558291 +S3-6 1.35805 549.779 0.555442 +N4-4 1.73135 551.758 0.341744 +N4-6 1.7453 553.737 0.336302 +E4-4 1.74702 555.711 0.335641 +S4-1 1.74865 557.69 0.335015 +N4-2 1.7657 559.669 0.328576 +E4-6 1.7671 561.643 0.328056 +S4-2 1.76837 563.622 0.327585 +S4-3 1.7742 565.601 0.325436 +E4-3 1.78722 567.58 0.320711 +N4-5 1.79851 569.554 0.316697 +S4-4 1.80542 571.533 0.314278 +N4-1 1.80918 573.512 0.312973 +N4-3 1.81474 575.491 0.311058 +E4-2 1.82214 577.465 0.308537 +E4-5 1.82477 579.444 0.307648 +S4-5 1.83854 581.423 0.303057 +S4-6 1.85149 583.397 0.298832 +E4-1 1.86332 585.376 0.29505 +N5-3 2.08337 587.355 0.236014 +N5-1 2.09855 589.334 0.232612 +E5-6 2.10097 591.308 0.232076 +S5-1 2.1102 593.287 0.23005 +S5-3 2.12056 595.266 0.227808 +E5-5 2.13008 597.245 0.225776 +N5-4 2.13175 599.219 0.225423 +N5-6 2.13672 601.198 0.224375 +N5-2 2.14829 603.177 0.221965 +S5-2 2.1541 605.151 0.220769 +S5-5 2.16001 607.13 0.219563 +E5-2 2.17133 609.109 0.217279 +E5-4 2.17944 611.088 0.215665 +E5-3 2.1809 613.062 0.215377 +N5-5 2.19425 615.041 0.212764 +S5-4 2.2054 617.02 0.210618 +S5-6 2.21387 618.999 0.209009 +E5-1 2.22188 620.973 0.207505 +S6-4 2.75989 622.952 0.134489 +S6-2 2.76144 624.931 0.134338 +E6-6 2.77145 626.905 0.133369 +E6-5 2.77441 628.884 0.133085 +S6-3 2.7982 630.863 0.130832 +N6-1 2.79897 632.842 0.13076 +E6-3 2.80659 634.816 0.130051 +S6-6 2.80881 636.795 0.129845 +E6-4 2.81448 638.774 0.129322 +N6-4 2.81767 640.753 0.12903 +S6-1 2.81965 642.727 0.128849 +N6-3 2.84601 644.706 0.126473 +E6-2 2.8469 646.685 0.126394 +N6-2 2.84822 648.659 0.126277 +E6-1 2.852 650.638 0.125942 +S6-5 2.85447 652.617 0.125724 +N6-5 2.88382 654.596 0.123178 +N6-6 2.88981 656.57 0.122668 +S7-5 3.64552 658.549 0.0770817 +S7-3 3.64904 660.528 0.076933 +E7-5 3.66048 662.507 0.0764529 +N7-1 3.66475 664.481 0.0762749 +N7-2 3.67335 666.46 0.0759181 +S7-4 3.6811 668.439 0.0755988 +N7-3 3.68841 670.413 0.0752994 +E7-3 3.69767 672.392 0.0749228 +S7-1 3.69941 674.371 0.0748523 +E7-6 3.69994 676.35 0.0748309 +S7-6 3.71635 678.324 0.0741715 +N7-4 3.7308 680.303 0.073598 +N7-5 3.73502 682.282 0.0734318 +N7-6 3.73867 684.261 0.0732885 +S7-2 3.74205 686.235 0.0731562 +E7-1 3.74739 688.214 0.0729478 +E7-2 3.75156 690.193 0.0727858 +E7-4 3.78879 692.167 0.0713623 +E8-6 4.76047 694.146 0.0452033 +E8-3 4.77877 696.125 0.0448578 +S8-6 4.81004 698.104 0.0442765 +E8-5 4.81612 700.078 0.0441647 +N8-1 4.8215 702.057 0.0440662 +S8-3 4.83039 704.036 0.0439042 +N8-2 4.83306 706.014 0.0438557 +E8-1 4.83554 707.989 0.0438107 +S8-4 4.84056 709.968 0.0437199 +E8-4 4.86131 711.946 0.0433474 +S8-5 4.86136 713.921 0.0433466 +N8-3 4.86685 715.9 0.0432488 +N8-4 4.87105 717.879 0.0431743 +S8-2 4.88682 719.857 0.0428961 +E8-2 4.88693 721.832 0.0428941 +N8-6 4.90031 723.811 0.0426602 +N8-5 4.90452 725.789 0.042587 +S8-1 4.91929 727.768 0.0423317 +E9-1 6.32647 729.743 0.0255945 +N9-1 6.32659 731.721 0.0255936 +N9-2 6.3293 733.7 0.0255717 +E9-3 6.33752 735.675 0.0255054 +E9-5 6.3681 737.654 0.025261 +N9-4 6.37328 739.632 0.0252199 +E9-2 6.37629 741.611 0.0251961 +S9-6 6.38244 743.586 0.0251476 +N9-5 6.38382 745.564 0.0251367 +S9-5 6.39757 747.543 0.0250288 +N9-3 6.39778 749.522 0.0250272 +E9-6 6.4165 751.496 0.0248813 +S9-3 6.42287 753.475 0.024832 +S9-4 6.42493 755.454 0.0248161 +E9-4 6.42719 757.429 0.0247986 +N9-6 6.43006 759.407 0.0247765 +S9-1 6.45565 761.386 0.0245805 +S9-2 6.46591 763.365 0.0245025 +E10-5 8.03339 765.339 0.0158735 +E10-2 8.06329 767.318 0.015756 +E10-4 8.082 769.297 0.0156831 +E10-6 8.10239 771.276 0.0156043 +E10-1 8.11211 773.25 0.0155669 +E10-3 8.15486 775.229 0.0154041 +N10-1 8.19398 777.208 0.0152574 +S10-3 8.20026 779.182 0.015234 +N10-2 8.20121 781.161 0.0152305 +S10-5 8.20484 783.14 0.015217 +N10-3 8.22078 785.119 0.0151581 +S10-2 8.24034 787.093 0.0150862 +N10-4 8.2414 789.072 0.0150823 +S10-6 8.24824 791.051 0.0150573 +N10-5 8.25734 793.03 0.0150241 +S10-4 8.26008 795.004 0.0150142 +S10-1 8.2796 796.983 0.0149435 +N10-6 8.29016 798.962 0.0149054 +E11-5 10.5216 800.936 0.00925351 +E11-2 10.5487 802.915 0.00920603 +E11-3 10.5567 804.894 0.00919208 +E11-6 10.5821 806.873 0.00914801 +E11-1 10.6103 808.847 0.00909944 +E11-4 10.6289 810.826 0.00906762 +N11-1 12.1283 812.805 0.00696419 +N11-2 12.1297 814.784 0.00696258 +N11-3 12.1732 816.758 0.00691291 +N11-4 12.1737 818.737 0.00691234 +N11-5 12.2066 820.716 0.00687513 +N11-6 12.2331 822.69 0.00684537 +S11-6 12.7558 824.669 0.00629586 +S11-3 12.7632 826.648 0.00628856 +S11-2 12.8055 828.627 0.00624708 +S11-5 12.8121 830.601 0.00624065 +S11-1 12.84 832.58 0.00621355 +S11-4 12.866 834.559 0.00618847 +N12-2 13.0269 836.538 0.00603654 +N12-1 13.0461 838.512 0.00601878 +N12-3 13.0636 840.491 0.00600267 +N12-4 13.0863 842.47 0.00598186 +N12-6 13.1075 844.444 0.00596253 +N12-5 13.1212 846.423 0.00595008 +E12-6 13.4601 848.402 0.00565423 +E12-3 13.4813 850.381 0.00563646 +E12-5 13.4929 852.355 0.00562678 +E12-2 13.531 854.334 0.00559513 +E12-4 13.538 856.313 0.00558935 +E12-1 13.591 858.292 0.00554584 +S12-6 14.8563 860.266 0.0046414 +S12-1 14.8761 862.245 0.00462905 +S12-4 14.889 864.224 0.00462103 +S12-5 14.9028 866.198 0.00461248 +S12-2 14.9272 868.177 0.00459741 +S12-3 14.9633 870.156 0.00457526 +N13-2 17.2187 872.135 0.00345517 +N13-1 17.2327 874.109 0.00344956 +N13-4 17.2714 876.088 0.00343412 +N13-3 17.275 878.067 0.00343269 +E13-2 17.2979 880.046 0.0034236 +E13-4 17.3037 882.02 0.00342131 +N13-6 17.3056 883.999 0.00342056 +N13-5 17.3169 885.978 0.00341609 +E13-6 17.347 887.952 0.00340425 +E13-3 17.3487 889.931 0.00340358 +E13-5 17.3788 891.91 0.0033918 +E13-1 17.3903 893.889 0.00338732 +S13-5 17.6056 895.863 0.00330498 +S13-6 17.6428 897.842 0.00329106 +S13-3 17.6568 899.821 0.00328584 +S13-4 17.6826 901.8 0.00327626 +S13-1 17.6929 903.774 0.00327244 +S13-2 17.7117 905.753 0.0032655 +S14-6 21.7962 907.732 0.0021563 +S14-4 21.8349 909.706 0.00214866 +S14-5 21.8503 911.685 0.00214563 +S14-3 21.8605 913.664 0.00214363 +S14-2 21.891 915.642 0.00213766 +S14-1 21.914 917.617 0.00213318 +N14-2 22.7731 919.596 0.00197527 +N14-1 22.7807 921.575 0.00197395 +N14-3 22.8217 923.553 0.00196686 +N14-5 22.8375 925.528 0.00196414 +N14-4 22.8508 927.507 0.00196186 +N14-6 22.8858 929.485 0.00195586 +E14-2 23.0808 931.46 0.00192295 +E14-5 23.1192 933.439 0.00191657 +E14-3 23.1227 935.417 0.00191599 +E14-1 23.1332 937.396 0.00191425 +E14-6 23.185 939.371 0.00190571 +E14-4 23.1907 941.35 0.00190477 +N15-1 27.7573 943.328 0.00132958 +N15-2 27.7583 945.307 0.00132949 +N15-4 27.8059 947.282 0.00132494 +N15-3 27.8105 949.26 0.0013245 +N15-5 27.8138 951.239 0.00132419 +N15-6 27.8598 953.214 0.00131982 +S15-4 28.7941 955.192 0.00123556 +S15-3 28.8448 957.171 0.00123122 +S15-1 28.8647 959.15 0.00122952 +S15-5 28.8863 961.125 0.00122768 +S15-2 28.8979 963.103 0.0012267 +S15-6 28.9489 965.082 0.00122238 +E15-6 29.0732 967.061 0.00121195 +E15-2 29.0847 969.035 0.00121099 +E15-4 29.1229 971.014 0.00120782 +E15-5 29.1636 972.993 0.00120445 +E15-1 29.1699 974.967 0.00120393 +E15-3 29.2 976.946 0.00120145 +N16-1 37.809 978.925 0.000716605 +N16-2 37.8146 980.904 0.000716393 +N16-3 37.8536 982.878 0.000714917 +N16-4 37.8591 984.857 0.00071471 +N16-5 37.8911 986.836 0.000713503 +N16-6 37.8971 988.815 0.000713277 +S16-3 38.4957 990.789 0.000691267 +S16-6 38.5041 992.768 0.000690965 +S16-4 38.5104 994.747 0.000690739 +S16-1 38.5475 996.721 0.00068941 +S16-5 38.5604 998.7 0.000688949 +S16-2 38.5765 1000.68 0.000688374 +E16-6 45.0375 1002.66 0.000505035 +E16-2 45.04 1004.63 0.000504979 +E16-4 45.0497 1006.61 0.000504762 +E16-5 45.0877 1008.59 0.000503911 +E16-1 45.0937 1010.56 0.000503777 +E16-3 45.1499 1012.54 0.000502524 diff --git a/papers/SKA_science/inputs/LowAAstar_ID_radius_AonT_FoVdeg2 b/papers/SKA_science/inputs/LowAAstar_ID_radius_AonT_FoVdeg2 new file mode 100644 index 0000000..1757820 --- /dev/null +++ b/papers/SKA_science/inputs/LowAAstar_ID_radius_AonT_FoVdeg2 @@ -0,0 +1,307 @@ +C5 0.00576224 1.97763 5.12201 +C59 0.0369839 3.95525 5.12201 +C20 0.0415433 5.93287 5.12201 +C8 0.0523278 7.91051 5.12201 +C15 0.0549779 9.8881 5.12201 +C37 0.0742842 11.8657 5.12201 +C19 0.0817277 13.8434 5.12201 +C3 0.0850761 15.821 5.12201 +C44 0.0862676 17.7986 5.12201 +C63 0.0918574 19.7763 5.12201 +C14 0.0942051 21.7539 5.12201 +C41 0.103418 23.7315 5.12201 +C13 0.105762 25.7091 5.12201 +C35 0.114899 27.6868 5.12201 +C75 0.115194 29.6644 5.12201 +C54 0.129174 31.642 5.12201 +C17 0.12994 33.6196 5.12201 +C9 0.130395 35.5973 5.12201 +C29 0.132841 37.5749 5.12201 +C38 0.136805 39.5525 5.12201 +C7 0.137425 41.5302 5.12201 +C70 0.138274 43.5077 5.12201 +C31 0.141194 45.4854 5.12201 +C57 0.141603 47.4628 5.12201 +C4 0.144913 49.4408 5.12201 +C2 0.145246 51.4183 5.12201 +C11 0.14526 53.3958 5.12201 +C53 0.152397 55.3737 5.12201 +C23 0.152982 57.3512 5.12201 +C60 0.15665 59.3287 5.12201 +C52 0.160377 61.3062 5.12201 +C30 0.167195 63.2842 5.12201 +C21 0.167956 65.2617 5.12201 +C24 0.169527 67.2392 5.12201 +C67 0.170527 69.2167 5.12201 +C69 0.170992 71.1946 5.12201 +C18 0.173752 73.1721 5.12201 +C58 0.174266 75.1496 5.12201 +C27 0.174354 77.1276 5.12201 +C71 0.183848 79.1051 5.12201 +C33 0.184147 81.0826 5.12201 +C48 0.184878 83.0601 5.12201 +C28 0.187098 85.0381 5.12201 +C36 0.187165 87.0156 5.12201 +C10 0.191743 88.9931 5.12201 +C56 0.191904 90.9706 5.12201 +C51 0.196303 92.9485 5.12201 +C1 0.202648 94.926 5.12201 +C6 0.202739 96.9035 5.12201 +C22 0.202784 98.881 5.12201 +C39 0.202795 100.859 5.12201 +C25 0.206082 102.837 5.12201 +C26 0.207547 104.814 5.12201 +C46 0.210027 106.792 5.12201 +C47 0.210896 108.769 5.12201 +C16 0.212845 110.747 5.12201 +C34 0.214628 112.724 5.12201 +C12 0.214654 114.702 5.12201 +C42 0.215397 116.68 5.12201 +C64 0.216443 118.657 5.12201 +C66 0.220276 120.635 5.12201 +C43 0.220871 122.613 5.12201 +C49 0.221708 124.59 5.12201 +C62 0.222152 126.568 5.12201 +C61 0.223793 128.546 5.12201 +C45 0.224835 130.523 5.12201 +C72 0.225332 132.501 5.12201 +C55 0.227457 134.478 5.12201 +C50 0.22826 136.456 5.12201 +C73 0.229041 138.434 5.12201 +C74 0.232021 140.411 5.12201 +C40 0.23317 142.389 5.12201 +C65 0.235341 144.367 5.12201 +C32 0.238894 146.344 5.12201 +C100 0.238903 148.322 5.12201 +C68 0.240166 150.3 5.12201 +C108 0.251241 152.277 5.12201 +C120 0.255888 154.255 5.12201 +C117 0.259439 156.232 5.12201 +C123 0.260943 158.21 5.12201 +C148 0.262085 160.188 5.12201 +C79 0.262128 162.165 5.12201 +C111 0.262686 164.143 5.12201 +C142 0.264605 166.121 5.12201 +C150 0.265973 168.098 5.12201 +C105 0.269939 170.076 5.12201 +C95 0.273682 172.053 5.12201 +C102 0.275264 174.031 5.12201 +C97 0.276116 176.009 5.12201 +C87 0.278431 177.986 5.12201 +C112 0.28345 179.964 5.12201 +C147 0.285824 181.942 5.12201 +C76 0.288097 183.919 5.12201 +C140 0.288119 185.897 5.12201 +C84 0.288389 187.875 5.12201 +C82 0.289496 189.852 5.12201 +C88 0.289685 191.83 5.12201 +C89 0.290275 193.807 5.12201 +C86 0.291663 195.785 5.12201 +C145 0.292323 197.763 5.12201 +C121 0.292419 199.74 5.12201 +C200 0.294231 201.718 5.12201 +C107 0.297548 203.696 5.12201 +C137 0.299766 205.673 5.12201 +C114 0.300545 207.651 5.12201 +C103 0.301607 209.628 5.12201 +C85 0.303895 211.606 5.12201 +C119 0.305865 213.583 5.12201 +C146 0.306192 215.561 5.12201 +C92 0.306577 217.539 5.12201 +C110 0.306609 219.516 5.12201 +C98 0.307402 221.494 5.12201 +C99 0.308294 223.472 5.12201 +C128 0.311374 225.449 5.12201 +C139 0.314429 227.427 5.12201 +C130 0.317243 229.404 5.12201 +C132 0.319367 231.382 5.12201 +C77 0.321173 233.36 5.12201 +C124 0.321206 235.337 5.12201 +C135 0.321546 237.315 5.12201 +C129 0.321763 239.293 5.12201 +C125 0.326198 241.27 5.12201 +C101 0.326702 243.248 5.12201 +C91 0.331314 245.225 5.12201 +C109 0.332076 247.203 5.12201 +C104 0.33599 249.181 5.12201 +C126 0.336287 251.158 5.12201 +C90 0.336931 253.136 5.12201 +C113 0.337077 255.114 5.12201 +C80 0.338807 257.091 5.12201 +C144 0.339712 259.069 5.12201 +C115 0.342923 261.047 5.12201 +C78 0.344542 263.024 5.12201 +C96 0.345244 265.002 5.12201 +C81 0.345599 266.979 5.12201 +C106 0.345887 268.957 5.12201 +C149 0.348595 270.935 5.12201 +C133 0.351003 272.912 5.12201 +C93 0.351674 274.89 5.12201 +C151 0.354576 276.868 5.12201 +C141 0.354655 278.845 5.12201 +C83 0.359185 280.823 5.12201 +C138 0.35924 282.801 5.12201 +C197 0.363273 284.778 5.12201 +C143 0.371144 286.756 5.12201 +C167 0.374005 288.733 5.12201 +C202 0.377233 290.711 5.12201 +C186 0.381145 292.689 5.12201 +C181 0.382885 294.666 5.12201 +C191 0.386202 296.644 5.12201 +C162 0.388627 298.622 5.12201 +C206 0.391895 300.599 5.12201 +C185 0.397118 302.577 5.12201 +C180 0.399151 304.555 5.12201 +C168 0.402481 306.532 5.12201 +C187 0.404253 308.51 5.12201 +C205 0.40558 310.487 5.12201 +C158 0.409651 312.465 5.12201 +C171 0.414559 314.442 5.12201 +C194 0.415997 316.42 5.12201 +C172 0.417054 318.397 5.12201 +C176 0.422348 320.375 5.12201 +C214 0.424527 322.353 5.12201 +C179 0.428255 324.33 5.12201 +C170 0.431131 326.308 5.12201 +C177 0.431906 328.286 5.12201 +C156 0.433198 330.263 5.12201 +C166 0.43327 332.241 5.12201 +C198 0.437027 334.219 5.12201 +C195 0.43915 336.196 5.12201 +C163 0.444326 338.174 5.12201 +C160 0.446469 340.151 5.12201 +C224 0.447065 342.129 5.12201 +C164 0.448648 344.107 5.08931 +C175 0.456321 346.084 4.91959 +C154 0.458576 348.062 4.87133 +C155 0.459584 350.04 4.84998 +C192 0.460215 352.017 4.83669 +C193 0.461054 353.995 4.81911 +C190 0.466042 355.973 4.7165 +C203 0.466715 357.95 4.70291 +C199 0.471507 359.928 4.6078 +C165 0.473166 361.905 4.57555 +C184 0.473864 363.883 4.56208 +C204 0.475849 365.861 4.5241 +C161 0.480174 367.838 4.44297 +C153 0.481096 369.816 4.42595 +C183 0.481755 371.794 4.41385 +C173 0.482394 373.771 4.40217 +C201 0.485276 375.749 4.35003 +C222 0.487366 377.727 4.3128 +C223 0.498014 379.704 4.13035 +C213 0.503017 381.682 4.0486 +C207 0.511351 383.659 3.91771 +C208 0.513384 385.637 3.88674 +C212 0.51713 387.615 3.83063 +C216 0.521526 389.592 3.76633 +C219 0.53608 391.57 3.5646 +C217 0.544352 393.548 3.45709 +E1-1 0.6089 395.525 2.76298 +S1-2 0.644623 397.503 2.46524 +E1-2 0.647829 399.481 2.4409 +S1-1 0.689533 401.458 2.15457 +N1-2 0.690869 403.436 2.14624 +N1-1 0.73232 405.413 1.91015 +S2-3 0.8648 407.391 1.36974 +E2-3 0.873016 409.369 1.34408 +S2-2 0.893395 411.346 1.28346 +E2-1 0.908209 413.324 1.24193 +N2-3 0.955956 415.302 1.12097 +E2-2 0.971062 417.279 1.08637 +N2-1 0.971652 419.257 1.08505 +N2-2 0.981157 421.234 1.06413 +S2-1 0.99669 423.212 1.03122 +E3-2 1.25937 425.189 0.645897 +S3-1 1.2725 427.167 0.632637 +N3-3 1.2846 429.145 0.620775 +S3-2 1.29788 431.122 0.608137 +E3-1 1.30427 433.1 0.602192 +N3-2 1.33132 435.077 0.57797 +E3-3 1.34962 437.055 0.562402 +N3-1 1.35458 439.033 0.558291 +S4-1 1.74865 441.01 0.335015 +N4-2 1.7657 442.988 0.328576 +S4-2 1.76837 444.966 0.327585 +S4-3 1.7742 446.943 0.325436 +E4-3 1.78722 448.921 0.320711 +N4-1 1.80918 450.899 0.312973 +N4-3 1.81474 452.876 0.311058 +E4-2 1.82214 454.854 0.308537 +E4-1 1.86332 456.831 0.29505 +E8-3 4.77877 458.809 0.0448578 +S8-6 4.81004 460.787 0.0442765 +N8-1 4.8215 462.764 0.0440662 +S8-3 4.83039 464.742 0.0439042 +N8-2 4.83306 466.722 0.0438557 +E8-1 4.83554 468.696 0.0438107 +S8-4 4.84056 470.675 0.0437199 +E8-4 4.86131 472.654 0.0433474 +S8-5 4.86136 474.628 0.0433466 +N8-3 4.86685 476.607 0.0432488 +N8-4 4.87105 478.586 0.0431743 +S8-2 4.88682 480.565 0.0428961 +E8-2 4.88693 482.539 0.0428941 +S8-1 4.91929 484.518 0.0423317 +E9-1 6.32647 486.497 0.0255945 +N9-1 6.32659 488.476 0.0255936 +N9-2 6.3293 490.45 0.0255717 +E9-3 6.33752 492.429 0.0255054 +N9-4 6.37328 494.408 0.0252199 +E9-2 6.37629 496.382 0.0251961 +N9-3 6.39778 498.361 0.0250272 +S9-3 6.42287 500.34 0.024832 +S9-4 6.42493 502.318 0.0248161 +E9-4 6.42719 504.293 0.0247986 +S9-1 6.45565 506.272 0.0245805 +S9-2 6.46591 508.251 0.0245025 +E10-2 8.06329 510.229 0.015756 +E10-4 8.082 512.204 0.0156831 +E10-1 8.11211 514.183 0.0155669 +E10-3 8.15486 516.161 0.0154041 +N10-1 8.19398 518.136 0.0152574 +S10-3 8.20026 520.115 0.015234 +N10-2 8.20121 522.093 0.0152305 +S10-5 8.20484 524.072 0.015217 +N10-3 8.22078 526.047 0.0151581 +S10-2 8.24034 528.026 0.0150862 +N10-4 8.2414 530.004 0.0150823 +S10-6 8.24824 531.983 0.0150573 +S10-4 8.26008 533.958 0.0150142 +S10-1 8.2796 535.936 0.0149435 +N13-2 17.2187 537.915 0.00345517 +N13-1 17.2327 539.89 0.00344956 +N13-4 17.2714 541.868 0.00343412 +N13-3 17.275 543.847 0.00343269 +E13-2 17.2979 545.826 0.0034236 +E13-4 17.3037 547.801 0.00342131 +E13-3 17.3487 549.779 0.00340358 +E13-1 17.3903 551.758 0.00338732 +S13-3 17.6568 553.737 0.00328584 +S13-4 17.6826 555.711 0.00327626 +S13-1 17.6929 557.69 0.00327244 +S13-2 17.7117 559.669 0.0032655 +N15-1 27.7573 561.643 0.00132958 +N15-2 27.7583 563.622 0.00132949 +N15-4 27.8059 565.601 0.00132494 +N15-3 27.8105 567.58 0.0013245 +S15-4 28.7941 569.554 0.00123556 +S15-3 28.8448 571.533 0.00123122 +S15-1 28.8647 573.512 0.00122952 +S15-2 28.8979 575.491 0.0012267 +E15-2 29.0847 577.465 0.00121099 +E15-4 29.1229 579.444 0.00120782 +E15-1 29.1699 581.423 0.00120393 +E15-3 29.2 583.397 0.00120145 +N16-1 37.809 585.376 0.000716605 +N16-2 37.8146 587.355 0.000716393 +N16-3 37.8536 589.334 0.000714917 +N16-4 37.8591 591.308 0.00071471 +S16-3 38.4957 593.287 0.000691267 +S16-4 38.5104 595.266 0.000690739 +S16-1 38.5475 597.245 0.00068941 +S16-2 38.5765 599.219 0.000688374 +E16-2 45.04 601.198 0.000504979 +E16-4 45.0497 603.177 0.000504762 +E16-1 45.0937 605.151 0.000503777 +E16-3 45.1499 607.13 0.000502524 diff --git a/papers/SKA_science/make_zdists.py b/papers/SKA_science/make_zdists.py new file mode 100644 index 0000000..79eac2a --- /dev/null +++ b/papers/SKA_science/make_zdists.py @@ -0,0 +1,272 @@ +""" +This script creates plots of p(z) for different SKA configs + +It first loads in the simulation info from the script +"sim_SKA_configs.py", and generates plots for the best cases. + +""" +import os +import emcee +from astropy.cosmology import Planck18 +from zdm import cosmology as cos +from zdm import parameters +from zdm import survey +from zdm import pcosmic +from zdm import io +from zdm import misc_functions as mf +from zdm import grid as zdm_grid +from zdm import survey + +import numpy as np +import copy +from matplotlib import pyplot as plt +from pkg_resources import resource_filename + +def main(): + """ + Main program to loop over SKA frequency ranges and deployment milestones + + """ + + #### Initialises general zDM stuff ##### + state = parameters.State() + # approximate best-fit values from recent analysis + state.set_astropy_cosmo(Planck18) + + zDMgrid, zvals, dmvals = mf.get_zdm_grid( + state, new=True, plot=False, method='analytic', + datdir=resource_filename('zdm', 'GridData')) + + + ####### sample MCMC parameter sets ###### + infile = resource_filename('zdm', 'scripts/MCMC')+"/H0_prior10.h5" + nsets=100 + sample, params, pconfig = get_samples(infile,nsets) + + ####### Loop over input files ######### + # these set the frequencies in MHz and bandwidths in MHz + names = ["SKA_mid","SKA_mid","SKA_low"] + freqs = [865,1400,190] + bws = [300,300,120] + for i,tel in enumerate(["Band1", "Band2", "Low"]): + # sets frequency and bandwidth for each instrument + freq = freqs[i] + bw = bws[i] + survey_name = names[i] + for telconfig in ["AA4","AAstar"]: + infile = "inputs/"+tel+telconfig+"_ID_radius_AonT_FoVdeg2" + label = tel+"_"+telconfig + generate_sensitivity_plot(infile,state,zDMgrid, zvals, dmvals, label, freq, bw, survey_name, + sample, params, pconfig) + + +def generate_sensitivity_plot(infile,state,zDMgrid, zvals, dmvals, label, freq, bw, survey_name, + samples, params, pconfig, + opdir = "outputs/", plotdir = "plotdir/"): + """ + generates a plot of FRB rate vs Nelements used + + Args: + infile: input file from Evan Keane relating sensitivity to FOV + + state: zDM parameters.state class + + zDMgrid (np.ndarray, 2D): intrinsic grid of p(DM|z) + + zvals (np.ndarray): redshift values + + dmvals (np.ndarray): dispersion measure values + + label (string): string label to add to outputs + + freq (float): mean frequency in MHz + + bw (float): bandwidth in MHz + + sample: set of MCMC values + + params: parameter names for ordered values + + pconfig: configuration dict specifying values of other variables + """ + + if pconfig is not None: + state.update_params(pconfig) + + data=np.loadtxt(infile,dtype=str) + radius = data[:,1].astype(float) + sense_m2K = data[:,2].astype(float) + fov = data[:,3].astype(float) + + Ngrids = radius.size + + # defines name of output files + opfile1 = opdir+label+"_N.npy" + opfile2 = opdir+label+"_pdm.npy" + opfile3 = opdir+label+"_pz.npy" + opfile4 = opdir+label+"_Nz.npy" + opfile5 = opdir+label+"_Tobs.npy" + opfile6 = opdir+label+"_thresh.npy" + oldNs = np.load(opfile1) + #pdms = np.load(opfile2) + #pzs = np.load(opfile3) + #Nizs = np.load(opfile4) + TOBS = np.load(opfile5) + thresh_Jyms = np.load(opfile6) + + # finds the best + survey_name='SKA_mid' + ibest = np.argmax(oldNs) + survey_dict = {"THRESH": thresh_Jyms[ibest], "TOBS": TOBS[ibest], "FBAR": freq, "BW": bw} + + + Nsamples = samples.shape[0] + Nz = zvals.size + Ndm = dmvals.size + + pdms = np.zeros([Nsamples,Ndm]) + pzs = np.zeros([Nsamples,Nz]) + Ns = np.zeros([Nsamples]) + + opdir = "sys_outputs/" + opfile7 = opdir+label+"_sys_N.npy" + opfile8 = opdir+label+"_sys_pz.npy" + opfile9 = opdir+label+"_sys_pdm.npy" + + + + np.save("sysplotdir/zvals.npy",zvals) + np.save("sysplotdir/dmvals.npy",dmvals) + exit() + #load=True + load=True + + verbose=True + for i in range(samples.shape[0]): + if load: + continue + + if verbose: + print("Sampling parameter set ",i,". Params: ") + vals = samples[i,:] + + dict = {} + for j in range(len(vals)): + dict[params[j]] = vals[j] + if verbose: + print(" ",params[j],": ",vals[j]) + + state.update_params(dict) + + mask = pcosmic.get_dm_mask(dmvals, (state.host.lmean, state.host.lsigma), zvals, plot=False) + + # normalise number of FRBs to the CRAFT Fly's Eye survey + s = survey.load_survey(survey_name, state, dmvals, zvals=zvals) + g = zdm_grid.Grid(s, copy.deepcopy(state), zDMgrid, zvals, dmvals, mask, wdist=True) + # we expect np.sum(g.rates)*s.TOBS * C = s.NORM_FRB + norm = s.NORM_FRB/(s.TOBS*np.sum(g.rates)) + + s = survey.load_survey(survey_name, state, dmvals, zvals=zvals, survey_dict=survey_dict) + + g = zdm_grid.Grid(s, copy.deepcopy(state), zDMgrid, zvals, dmvals, mask, wdist=True) + scale = TOBS[i] * norm + + if verbose: + print("Finished iteration ",i," norm is ",norm) + + pz = np.sum(g.rates,axis=1)*scale + pdm = np.sum(g.rates,axis=0)*scale + N = np.sum(pdm) + + Ns[i] = N + pzs[i,:] = pz + pdms[i,:] = pdm + + if load: + Ns = np.load(opfile7) + pzs = np.load(opfile8) + pdms = np.load(opfile9) + + else: + np.save(opfile7,Ns) + np.save(opfile8,pzs) + np.save(opfile9,pdms) + + print("######### Finished ",label," #########") + plotdir = "sysplotdir/" + make_pz_plots(zvals,pzs,plotdir+label) + + +def make_pz_plots(zvals,pzs,label): + """ + + """ + + Nparams,NZ = pzs.shape + + scale = 1./(zvals[1]-zvals[0]) + + mean = np.sum(pzs,axis=0)/Nparams + + # make un-normalised plots + plt.figure() + plt.xlabel("z") + plt.ylabel("N(z)") + plt.xlim(0,5) + + for i in np.arange(Nparams): + plt.plot(zvals,pzs[i,:],color="grey",linestyle="-") + + plt.plot(zvals,mean,color="black",linestyle="-",linewidth=2) + + plt.legend() + plt.tight_layout() + plt.savefig(label+"_pz.png") + plt.close() + + + + +def get_samples(infile,nsets): + # Read in the MCMC results without the burnin + #infile = os.path.join(args.directory, args.infile) + reader = emcee.backends.HDFBackend(infile) + sample = reader.get_chain(discard=50, flat=True) + + # Thin the results + step = len(sample) // nsets + sample = sample[::step,:] + + # Get the corresponding parameters + # If there is a corresponding .out file, it will contain all the necessary information that was used during that run, + # otherwise parameters must be specified manually + if os.path.exists(infile + '.out'): + with open(infile + '.out', 'r') as f: + # Get configuration + line = f.readline() + + while not line.startswith('Config') and line: + line = f.readline() + if not line: + raise ValueError("No 'Config' line found in the .out file.") + config = json.loads(line[9:].replace("'", '"')) + + # Read down to parameter lines + while ('priors' not in line) and line: + line = f.readline() + + # Read parameters + params = [] + while ('priors' in line) and line: + vals = line.split() + params.append(vals[0]) + line = f.readline() + + # If there is no .out file, then the parameters must be specified manually + else: + params = ["sfr_n", "alpha", "lmean", "lsigma", "lEmax", "lEmin", "gamma", "H0"] + config = None + + return sample, params, config + +main() diff --git a/papers/SKA_science/sim_SKA_configs.py b/papers/SKA_science/sim_SKA_configs.py new file mode 100644 index 0000000..06e668b --- /dev/null +++ b/papers/SKA_science/sim_SKA_configs.py @@ -0,0 +1,207 @@ +""" +This script creates a zDM plot for SKA_Mid + +It also estimates the raction of SKA bursts that will have +unseen hosts by a VLT-like optical obeservation +""" +import os + +from astropy.cosmology import Planck18 +from zdm import cosmology as cos +from zdm import parameters +from zdm import survey +from zdm import pcosmic +from zdm import io +from zdm import misc_functions as mf +from zdm import grid as zdm_grid +from zdm import survey + +import numpy as np +import copy +from matplotlib import pyplot as plt +from pkg_resources import resource_filename + +def main(): + """ + + + """ + + #### Initialises general zDM stuff ##### + state = parameters.State() + # approximate best-fit values from recent analysis + state.set_astropy_cosmo(Planck18) + + zDMgrid, zvals, dmvals = mf.get_zdm_grid( + state, new=True, plot=False, method='analytic', + datdir=resource_filename('zdm', 'GridData')) + + ####### Loop over input files ######### + + #indir = "inputs/" + #infiles = os.listdir(indir) + # loop over all files + freqs = [865,1400,190] + bws = [300,300,120] + + + for i,tel in enumerate(["Band1", "Band2", "Low"]): + # sets frequency and bandwidth for each instrument + freq = freqs[i] + bw = bws[i] + for config in ["AA4","AAstar"]: + infile = "inputs/"+tel+config+"_ID_radius_AonT_FoVdeg2" + label = tel+"_"+config + generate_sensitivity_plot(infile,state,zDMgrid, zvals, dmvals, label, freq, bw) + +def generate_sensitivity_plot(infile,state,zDMgrid, zvals, dmvals, label, freq, bw, + opdir = "outputs/", plotdir = "plotdir/"): + """ + generates a plot of FRB rate vs Nelements used + """ + + data=np.loadtxt(infile,dtype=str) + radius = data[:,1].astype(float) + sense_m2K = data[:,2].astype(float) + fov = data[:,3].astype(float) + + Ngrids = radius.size + + # now converts sensitivity to Jy ms threshold + Tint = 1e-3 # normalise sensitiivty at 1 ms + Nsamps = 2*Tint*bw*1e6 # number of independent samples. bw in MHz + Nsigma = 10. # need 10 sigma detection + kboltzmann = 1.380649e-23 + + ######## scale inputs to relevant zDM ones ####### + + ## calculates threshold in Jy ms ######### + SEFD = 2*kboltzmann/sense_m2K + thresh_Jyms = 1e26 * Nsigma * SEFD/Nsamps**0.5 + + # calculates effective observation time + # We begin by characterising a "full beam" as being + # the FOV for a single element. This is a Gaussian + # we then set that Tobs to be one year. Then, scale that down + # as FOV decreases + TOBS0 = 365.25 #calendar year + TOBS = fov*TOBS0/fov[0] + + if True: + # produces a plot of sensitivity + plt.figure() + p1,=plt.plot(thresh_Jyms,label="Thresh",color="blue") + plt.xscale('log') + plt.yscale('log') + plt.ylabel("Threshold [Jy ms]") + + ax = plt.gca() + ax2 = ax.twinx() + ax2.set_yscale("log") + p2,=ax2.plot(fov,label="FOV",color="orange") + ax2.set_ylabel("Field of view [deg 2]") + + naive = fov * thresh_Jyms**-1.5 + p3,=ax.plot(naive,label="Euclidean rate",color="red") + + plt.legend(labels = ["Thresh","FOV","Euclidean rate"], handles = [p1,p2,p3]) + + plt.tight_layout() + plt.savefig(label+"_thresh_fov.png") + plt.close() + + + ######### + # determines z-values of interest + + zis = np.array([1.,2.,3.,4.]) + izs = [] + for i,z in enumerate(zis): + iz = np.where(zvals > z)[0][0] + izs.append(iz) + izs = np.array(izs) + ########## speedups ############ + + #set survey path + sdir = os.path.join(resource_filename('zdm', 'data'), 'Surveys') + # we use SKA mid, but actually we will over-ride may attributes here + survey_name='SKA_mid' + + # we can keep this constant - it smears DM due to host DM + mask = pcosmic.get_dm_mask(dmvals, (state.host.lmean, state.host.lsigma), zvals, plot=False) + + prev_grid = None + + Nizs = np.zeros([Ngrids,zis.size]) + Ns = np.zeros([Ngrids]) + pzs = np.zeros([Ngrids,zvals.size]) + pdms = np.zeros([Ngrids,dmvals.size]) + + # defines name of output files + opfile1 = opdir+label+"_N.npy" + opfile2 = opdir+label+"_pdm.npy" + opfile3 = opdir+label+"_pz.npy" + opfile4 = opdir+label+"_Nz.npy" + opfile5 = opdir+label+"_Tobs.npy" + opfile6 = opdir+label+"_thresh.npy" + + if os.path.exists(opfile1): + load=True + else: + np.save(opfile5,TOBS) + np.save(opfile6,thresh_Jyms) + + + ######### Loop over all options ########## + for i in np.arange(Ngrids): + if load: + continue + # ignore if we are loading + survey_dict = {"THRESH": thresh_Jyms[i], "TOBS": TOBS[i], "FBAR": freq, "BW": bw} + s = survey.load_survey(survey_name, state, dmvals, zvals=zvals, survey_dict=survey_dict) + print("iteration ",i,"thresh ",s.meta['THRESH'],"tobs ",s.meta['TOBS'],s.meta['FBAR']) + g = zdm_grid.Grid(s, copy.deepcopy(state), zDMgrid, zvals, dmvals, mask, wdist=True, prev_grid=prev_grid) + scale = TOBS[i] * 10**(state.FRBdemo.lC) + + pz = np.sum(g.rates,axis=1)*scale + pdm = np.sum(g.rates,axis=0)*scale + N = np.sum(pdm) + + Ns[i] = N + pzs[i,:] = pz + pdms[i,:] = pdm + + # calculates number of FRBs beyond a certain redshift + for j,iz in enumerate(izs): + Niz = np.sum(pz[iz:])*scale + Nizs[i,j] = Niz + + print("Did ",i," of ",Ngrids) + + if load: + Ns = np.load(opfile1) + pdms = np.load(opfile2) + pzs = np.load(opfile3) + Nizs = np.load(opfile4) + + else: + #save outputs + np.save(opfile1,Ns) + np.save(opfile2,pdms) + np.save(opfile3,pzs) + np.save(opfile4,Nizs) + + # generates plots + plt.figure() + plt.xlabel("Number of elements used in tied beamforming") + plt.ylabel("Relative number of FRBs detected") + plt.plot(Ns/np.max(Ns),label = "All FRBs") + for iz, zi in enumerate(zis): + plt.plot(Nizs[:,iz]/np.max(Nizs[:,iz]),label="$z_{\\rm FRB} > "+str(zi)[0]+"$") + plt.legend() + plt.tight_layout() + plt.savefig(plotdir+label+"_PN.png") + plt.close() + + +main() diff --git a/papers/Scattering/MCMC_inputs/scat_w_only.json b/papers/Scattering/MCMC_inputs/scat_w_only.json new file mode 100644 index 0000000..c61d78b --- /dev/null +++ b/papers/Scattering/MCMC_inputs/scat_w_only.json @@ -0,0 +1,110 @@ +{ + "mcmc": { + "parameter_order": ["Wlogmean", "Wlogsigma", "Slogmean", "Slogsigma"] + }, + "config": { + "luminosity_function": 2, + "alpha_method": 1, + "halo_method": 0, + "sigmaDMG": 0.0, + "sigmaHalo": 15.0, + "lmean": 2.02, + "lsigma": 0.46, + "sfr_n": 0.95, + "gamma": -1.16, + "alpha": 1.5, + "WNbins": 33, + "WNInternalBins": 1000, + "ScatFunction": 1 + }, + "Wlogmean": { + "DC": "width", + "min": -3, + "max": 3 + }, + "Wlogsigma": { + "DC": "width", + "min": 0.2, + "max": 4 + }, + "Slogmean": { + "DC": "scat", + "min": -3, + "max": 3 + }, + "Slogsigma": { + "DC": "scat", + "min": 0.2, + "max": 4 + }, + "logF": { + "DC": "cosmo", + "min": -2.0, + "max": 0.0 + }, + "lEmax": { + "DC": "energy", + "min": 40.35, + "max": 45.0 + }, + "lEmin": { + "DC": "energy", + "min": 35.0, + "max": 40.35 + }, + "DMhalo": { + "DC": "MW", + "min": 0.0, + "max": 100 + }, + "sigmaDMG": { + "DC": "MW", + "min": 0.1, + "max": 0.5 + }, + "H0": { + "DC": "cosmo", + "min": 35.0, + "max": 110.0 + }, + "alpha": { + "DC": "energy", + "min": 0.5, + "max": 2.5 + }, + "gamma": { + "DC": "energy", + "min": -3.0, + "max": 0.0 + }, + "sfr_n": { + "DC": "FRBdemo", + "min": -2.0, + "max": 4.0 + }, + "lmean": { + "DC": "host", + "min": 1.0, + "max": 3.0 + }, + "lsigma": { + "DC": "host", + "min": 0.1, + "max": 1.5 + }, + "lRmin": { + "DC": "rep", + "min": -2.0, + "max": 1.0 + }, + "lRmax": { + "DC": "rep", + "min": 0.0, + "max": 4.0 + }, + "Rgamma": { + "DC": "rep", + "min": -3.0, + "max": -0.5 + } +} diff --git a/papers/Scattering/MCMC_inputs/scat_w_only_halflog.json b/papers/Scattering/MCMC_inputs/scat_w_only_halflog.json new file mode 100644 index 0000000..a336a6b --- /dev/null +++ b/papers/Scattering/MCMC_inputs/scat_w_only_halflog.json @@ -0,0 +1,110 @@ +{ + "mcmc": { + "parameter_order": ["Wlogmean", "Wlogsigma", "Slogmean", "Slogsigma"] + }, + "config": { + "luminosity_function": 2, + "alpha_method": 1, + "halo_method": 0, + "sigmaDMG": 0.0, + "sigmaHalo": 15.0, + "lmean": 2.02, + "lsigma": 0.46, + "sfr_n": 0.95, + "gamma": -1.16, + "alpha": 1.5, + "WNbins": 33, + "WNInternalBins": 100, + "ScatFunction": 2 + }, + "Wlogmean": { + "DC": "width", + "min": -3, + "max": 3 + }, + "Wlogsigma": { + "DC": "width", + "min": 0.2, + "max": 4 + }, + "Slogmean": { + "DC": "scat", + "min": -3, + "max": 3 + }, + "Slogsigma": { + "DC": "scat", + "min": 0.2, + "max": 4 + }, + "logF": { + "DC": "cosmo", + "min": -2.0, + "max": 0.0 + }, + "lEmax": { + "DC": "energy", + "min": 40.35, + "max": 45.0 + }, + "lEmin": { + "DC": "energy", + "min": 35.0, + "max": 40.35 + }, + "DMhalo": { + "DC": "MW", + "min": 0.0, + "max": 100 + }, + "sigmaDMG": { + "DC": "MW", + "min": 0.1, + "max": 0.5 + }, + "H0": { + "DC": "cosmo", + "min": 35.0, + "max": 110.0 + }, + "alpha": { + "DC": "energy", + "min": 0.5, + "max": 2.5 + }, + "gamma": { + "DC": "energy", + "min": -3.0, + "max": 0.0 + }, + "sfr_n": { + "DC": "FRBdemo", + "min": -2.0, + "max": 4.0 + }, + "lmean": { + "DC": "host", + "min": 1.0, + "max": 3.0 + }, + "lsigma": { + "DC": "host", + "min": 0.1, + "max": 1.5 + }, + "lRmin": { + "DC": "rep", + "min": -2.0, + "max": 1.0 + }, + "lRmax": { + "DC": "rep", + "min": 0.0, + "max": 4.0 + }, + "Rgamma": { + "DC": "rep", + "min": -3.0, + "max": -0.5 + } +} diff --git a/papers/Scattering/run_MCMC.sh b/papers/Scattering/run_MCMC.sh new file mode 100755 index 0000000..a494402 --- /dev/null +++ b/papers/Scattering/run_MCMC.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# script to run MCMC for CRAFT width parameters + +#files="CRAFT_ICS_892 CRAFT_ICS_1300 CRAFT_ICS_1632" + +# use this for a halflognormal distribution +#opfile="v3_mcmc_halflognormal" +#pfile="MCMC_inputs/scat_w_only_halflog.json" +# use this for a lognormal distribution +opfile="v6_mcmc_lognormal" # v4 is done with 300x300 nz ndm bins +pfile="MCMC_inputs/scat_w_only.json" + +# LOG +# files="CRAFT_ICS_892 CRAFT_ICS_1300 CRAFT_ICS_1632" +# v2: 1000 I think +# v3: 5000 +# v4: 300 x 300 zDM values +# v5: try with 100x100, but with 15 beam values instead of 5. +#v6 Unlocalised ones removed ("mod") +# [I still don't know the origin of these weird ridges in scat-sigmascat space] + +# for v6: these are now all localised - z=-1 is removed +#files="modCRAFT_ICS_892 modCRAFT_ICS_1300 modCRAFT_ICS_1632" +#opfile="v6_mcmc_lognormal" + +# for v7 - only one survey. Faster! Turns off the P(w) function +files="modCRAFT_ICS_1300" +opfile="MCMC_outputs/v7_mcmc_lognormal" + +# for v8 - turns off the Pscat | w function. (in p(2D) only) +#files="modCRAFT_ICS_1300" # takes away 1D as well, only 2D +#opfile="v8_mcmc_lognormal" + +# for v8 - has 1000 internal bins in width, and 33 evaluation bins +files="modCRAFT_ICS_1300" # takes away 1D as well, only 2D +opfile="MCMC_outputs/v9_mcmc_lognormal" + +Pn=False +ptauw=True +steps=1000 +walkers=14 + +Nz=100 +Ndm=100 +zmax=2 +dmmax=2000 + +script="../../zdm/scripts/MCMC/MCMC_wrap.py" + +runcommand="python $script -f $files --opfile=$opfile --pfile=$pfile --ptauw -s $steps -w $walkers --Nz=$Nz --Ndm=$Ndm --zmax=$zmax --dmmax=$dmmax" + +echo $runcommand +$runcommand diff --git a/papers/Scattering/visualise_mcmc.py b/papers/Scattering/visualise_mcmc.py new file mode 100644 index 0000000..3f164ca --- /dev/null +++ b/papers/Scattering/visualise_mcmc.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python +# coding: utf-8 + +# # Purpose +# +# - Used to visualise HDF5 files from MCMC analysis +# - Developed to handle output files from MCMC.py and MCMC2.py +# - Produces plots for walkers +# - Produces corner plot +# - Produces more detailed analysis for the best fit parameters + +# In[41]: + + +import numpy as np +from scipy import signal +import matplotlib.pyplot as plt +import corner +import emcee +import json + +from zdm import survey +from zdm import cosmology as cos +from zdm import loading as loading +from zdm import MCMC_analysis as analysis +import zdm.misc_functions as mf +import zdm.iteration as it +from zdm import parameters +from zdm.MCMC import calc_log_posterior +from astropy.cosmology import Planck18 + + +plt.rcParams['font.size'] = 16 + + +# # Load files +# +# - labels = list of parameters (in order) +# - filenames = list of .h5 files to use (without .h5 extension) + +# In[ ]: + + +# labels = ["sfr_n", "alpha", "lmean", "lsigma", "lEmax", "lEmin", "gamma", "H0", "DMhalo"] +# labels = [r"$n$", r"$\alpha$", r"log$\mu$", r"log$\sigma$", r"log$E_{\mathrm{max}}$", r"log$E_{\mathrm{min}}$", r"$\gamma$", r"$H_0$", "DMhalo"] +labels = [r"$\mu_w$", r"$\sigma_w$", r"$\mu_\tau$", r"$\sigma_\tau$"] + +half = False + +if half: + + filenames = ['MCMC_outputs/v2_mcmc_halflognormal','MCMC_outputs/v3_mcmc_halflognormal'] + # this name gets added to all produced plots + prefix="MCMC_Plots/halflognormal_" +else: + filenames = ['MCMC_outputs/v2_mcmc_lognormal','MCMC_outputs/v3_mcmc_lognormal'] + # this name gets added to all produced plots + prefix="MCMC_Plots/lognormal_" # 100x100 zDM points + + filenames = ['MCMC_outputs/v4_mcmc_lognormal'] # 300 x 300 zdm points + prefix = "MCMC_Plots/v4lognormal_" + + filenames = ['MCMC_outputs/v5_mcmc_lognormal'] # 15 beam values + prefix = "MCMC_Plots/v5lognormal_" + + filenames = ['MCMC_outputs/v6_mcmc_lognormal'] # 15 beam values + prefix = "MCMC_Plots/v6lognormal_" + + filenames = ['MCMC_outputs/v7_mcmc_lognormal'] #turn off p(w) + prefix = "MCMC_Plots/v7lognormal_" + + filenames = ['MCMC_outputs/v8_mcmc_lognormal'] # turn off p(scat|w) + prefix = "MCMC_Plots/v8lognormal_" + +samples = [] + +# Q1: why multiple files? Are these independent MCMC runs that can be added together for a larger data-set? +for i, filename in enumerate(filenames): + reader = emcee.backends.HDFBackend(filename + '.h5') + samples.append(reader.get_chain()) + + +# # Negate $\alpha$ +# +# - In our code we assume $\alpha$ is negative and so $\alpha=2$ here corresponds to a negative spectral index. +# - So here, we change that to a negative for clarity + +# Make alpha negative +a=-1 +for i, x in enumerate(labels): + if x == r"$\alpha$": + a = i + break + +if a != -1: + for sample in samples: + sample[:,:,a] = -sample[:,:,a] + +# plot walkers +analysis.plot_walkers(samples,labels,prefix+"raw_walkers.png") + + +######## Define burnin sample ########## +# here are many ways to do this. But visually tends to be +# the best. Here, a hard-coded value of 200 is used. +# Please inspect walker +good_samples = analysis.std_rej(samples, burnin=200) + + +analysis.plot_autocorrelations(good_samples,prefix+"autocorrelation_times.png") + + +# # Implement burnin and change priors + +burnin = 250 # define this by inspecting the raw walkers +# keep burnin the same over all samples - in theory, this could be different though +# (but probably should not be!) +burnin = (np.ones(len(good_samples)) * burnin).astype(int) + +# removes the burnin from each sample + + +analysis.plot_walkers(good_samples,labels,prefix+"final_walkers.png",burnin=burnin) +# NOTE - there is no current way to plot the final walkers with bad points removed + +# Get the final sample without burnin and without bad walkers +final_sample = [[] for i in range(samples[0].shape[2])] + +# we now remove the burnin from each +for j,sample in enumerate(good_samples): + for i in range(sample.shape[2]): + final_sample[i].append(sample[burnin[j]:,:,i].flatten()) +final_sample = np.array([np.hstack(final_sample[i]) for i in range(len(final_sample))]).T + +# - Changes prior to discard samples outside the specified prior range +# - Implements the burnin using either the predefined burnin or a constant specified +# e.g.: +# final_sample = analysis.change_priors(final_sample, 5, min=38) +# final_sample = analysis.change_priors(final_sample, 7, max=110.0) +# final_sample = analysis.change_priors(final_sample, 9, max=80.0) +# final_sample = analysis.change_priors(final_sample, 1, max=1.0, min=-3.5) + + +######## Cornerplot ######### + +# use the below to show other lines on this plot. E.g. to show a standard H0 value. +# Typically used to show "correct", i.e. true, values, against the MCMC estimates +truth = False +if truth: + lmean = 2.27 + DMhalo = np.log10(50.0) + param_dict={'logF': np.log10(0.32), 'sfr_n': 1.13, 'alpha': 1.5, 'lmean': lmean, 'lsigma': 0.55, + 'lEmax': 41.26, 'lEmin': 39.5, 'gamma': -0.95, 'DMhalo': DMhalo, 'H0': 73, + 'min_lat': None} + truths = [param_dict[param] for param in labels] +else: + truths = None + +fig = plt.figure(figsize=(12,12)) + +titles = ['' for i in range(final_sample.shape[1])] +corner.corner(final_sample,labels=labels, show_titles=True, titles=titles, + fig=fig,title_kwargs={"fontsize": 15},label_kwargs={"fontsize": 15}, + quantiles=[0.16,0.5,0.84], truths=truths); + +plt.savefig(prefix+"cornerplot.png") +exit() +#### LEAVE HERE ###### + +# # Point estimates +# +# - Use finer histogram binning than the corner plot +# - Obtain point estimates and confidence intervals using median / mode + +# In[64]: + + +nBins = 20 +win_len = int(nBins/10) +CL = 0.68 + +best_fit = {} + +for i in range(len(labels)): + fig = plt.figure(figsize=(6,4)) + ax = fig.add_subplot(1,1,1) + hist, bin_edges, _ = ax.hist(final_sample[:,i], bins=nBins, density=True) + bin_width = bin_edges[1] - bin_edges[0] + bins = -np.diff(bin_edges)/2.0 + bin_edges[1:] + + ax.set_xlabel(labels[i]) + ax.set_ylabel("P("+labels[i]+")") + + # Use mode ordered + # ordered_idxs = np.argsort(hist) + + # sum = hist[ordered_idxs[0]] * bin_width + # j = 1 + # while(sum < 1-CL): + # sum += hist[ordered_idxs[j]] * bin_width + # j = j+1 + + # best = bins[ordered_idxs[-1]] + # lower = bins[np.min(ordered_idxs[j:])] + # upper = bins[np.max(ordered_idxs[j:])] + + # Use median + best = np.quantile(final_sample[:,i], 0.5) + # best = bins[np.argmax(hist)] + lower = np.quantile(final_sample[:,i], 0.16) + upper = np.quantile(final_sample[:,i], 0.84) + + best_fit[labels[i]] = best + u_lower = best - lower + u_upper = upper - best + ax.axvline(lower, color='r') + ax.axvline(best, color='r') + ax.axvline(upper, color='r') + # print(labels[i] + ": " + str(best) + " (-" + str(u_lower) + "/+" + str(u_upper) + ")") + print(rf'{labels[i]}: {best} (-{u_lower}/+{u_upper})') + +print(best_fit) + + + +# In[65]: + + +import scipy.stats as st + + +# In[66]: + + +# nsamps = np.linspace(3, np.log10(final_sample.shape[0]/10), 30) +# nsamps = [int(10**x) for x in nsamps] +# print("Number of samps: " + str(nsamps)) + +# for i in range(len(labels)): +# # nsamps = [] +# std = [] +# for j in range(len(nsamps)): +# best = [] +# nruns = int(final_sample.shape[0] / nsamps[j]) +# for k in range(nruns): +# # best.append(np.quantile(final_sample[nsamps[j]*k:nsamps[j]*(k+1),i], 0.5)) +# step = int(final_sample.shape[0]/nsamps[j]) +# best.append(np.quantile(final_sample[k::step,i], 0.5)) +# std.append(np.std(best)) + +# # print(labels[i] + ": " + str(std)) + +# line = st.linregress(np.log10(nsamps),np.log10(std)) +# x = np.linspace(nsamps[0], nsamps[-1], 50) +# y = 1/np.sqrt(x) +# y = y / y[0] * std[0] +# y = 10**(line.slope*np.log10(x) + line.intercept) +# # print(line.slope) +# print(labels[i] + ": " + str(10**(line.slope*np.log10(final_sample.shape[0]) + line.intercept))) +# print(str(line.slope)) +# fig = plt.figure(figsize=(6,4)) +# ax = fig.add_subplot(1,1,1) + +# ax.plot(nsamps, std) +# ax.loglog(x,y) +# ax.set_xlabel("Number of samples") +# ax.set_ylabel("Standard deviation") +# ax.set_title(labels[i]) + + +# # Load surveys and grids +# +# - Loads the surveys and grids with the best fit parameters from above. +# - Plots P(DM) and DMEG weights for each FRB + +# In[67]: + + +s_names = [ + # "FAST2", + # "FAST2_old" + # "DSA", + "FAST", + # "CRAFT_class_I_and_II", + # "private_CRAFT_ICS_892_14", + # "private_CRAFT_ICS_1300_14", + # "private_CRAFT_ICS_1632_14", + # "parkes_mb_class_I_and_II" +] +# rs_names = ["CHIME/CHIME_decbin_0_of_6", +# "CHIME/CHIME_decbin_1_of_6", +# "CHIME/CHIME_decbin_2_of_6", +# "CHIME/CHIME_decbin_3_of_6", +# "CHIME/CHIME_decbin_4_of_6", +# "CHIME/CHIME_decbin_5_of_6"] +rs_names = [] + +state = parameters.State() +state.set_astropy_cosmo(Planck18) +# state.update_params(best_fit) +# state.update_param('luminosity_function', 2) +# state.update_param('alpha_method', 0) +# state.update_param('sfr_n', 1.36) +# state.update_param('alpha', 1.5) +# state.update_param('lmean', 1.97) +# state.update_param('lsigma', 0.92) +# state.update_param('lEmax', 41.3) +# state.update_param('gamma', -0.63) +# state.update_param('H0', 70.0) +# state.update_param('DMhalo', 50.0) + +if len(s_names) != 0: + surveys, grids = loading.surveys_and_grids(survey_names = s_names, init_state=state, repeaters=False, nz=500, ndm=1400) +else: + surveys = [] + grids = [] + +if len(rs_names) != 0: + rep_surveys, rep_grids = loading.surveys_and_grids(survey_names = rs_names, init_state=state, repeaters=True, nz=500, ndm=1400) + for s,g in zip(rep_surveys, rep_grids): + surveys.append(s) + grids.append(g) + + +# In[68]: + + +newC, llc = it.minimise_const_only(None, grids, surveys) +llsum = 0 +for s,g in zip(surveys, grids): + + g.state.FRBdemo.lC = newC + + # Calc pdm + rates=g.rates + dmvals=g.dmvals + pdm=np.sum(rates,axis=0) + + # # Calc psnr + # min = s.SNRTHRESHs[0] + # max = np.max(s.SNRs) + # snrs = np.linspace(min,max, 50) + # psnr = get_psnr(snrs, s, g) + + # Plot pdm + snr + fig = plt.figure(figsize=(10,4)) + ax = fig.add_subplot(1,2,1) + ax.set_title(s.name) + ax.set_xlabel("DM") + ax.set_ylabel("P(DM)") + ax.set_xlim(xmax=3000) + ax.plot(dmvals, pdm) + ax.vlines(s.DMEGs, np.zeros(len(s.DMs)), np.max(pdm)*np.ones(len(s.DMs)), ls='--', colors='r') + + # ax = fig.add_subplot(1,2,2) + # ax.set_xlabel("log SNR") + # ax.set_ylabel("log P(SNR)") + # ax.plot(np.log10(snrs), np.log10(psnr)) + # ax.vlines(np.log10(s.SNRs), np.min(np.log10(psnr))*np.ones(len(s.SNRs)), np.max(np.log10(psnr))*np.ones(len(s.SNRs)), ls='--', colors='r') + + # Get expected and observed + expected=it.CalculateIntegral(g.rates,s) + expected *= 10**g.state.FRBdemo.lC + observed=s.NORM_FRB + + print(s.name + " - expected, observed: " + str(expected) + ", " + str(observed)) + + llsum += it.get_log_likelihood(g,s,Pn=True) + + +# In[ ]: + + +uDMGs = 0.5 +# DMhalo = 100.0 + +fig = plt.figure(figsize=(6,4*len(s_names))) + +for j,(s,g) in enumerate(zip(surveys, grids)): + ax = fig.add_subplot(len(surveys),1,j+1) + plt.title(s.name) + ax.set_xlabel('DM') + ax.set_ylabel('Weight') + + # s.DMhalo = DMhalo + # s.init_DMEG(DMhalo) + + dmvals=g.dmvals + DMobs=s.DMEGs + + # calc_DMG_weights(DMEGs, DMhalos, DM_ISMs, dmvals, sigma_ISM=0.5, sigma_halo=15.0, percent_ISM=True) + dm_weights, iweights = it.calc_DMG_weights(DMobs, s.DMhalos, s.DMGs, dmvals, uDMGs) + + pdm = np.sum(g.rates, axis=0) + pdm = pdm / np.max(pdm) * np.max(dm_weights[0]) + + for i in range(len(DMobs)): + ax.plot(dmvals[iweights[i]], dm_weights[i], '.-', label=s.frbs["TNS"][i] + " " + str(s.DMGs[i])) + + # ax.plot(dmvals, pdm) # Upper limit is not correct because grid has not been updated so efficiencies have not been recalc'd + ax.set_xlim(right=3000) + # ax.legend() + + diff --git a/zdm/MCMC.py b/zdm/MCMC.py index 3125c4e..52a82de 100644 --- a/zdm/MCMC.py +++ b/zdm/MCMC.py @@ -28,10 +28,11 @@ from zdm import misc_functions as mf from zdm import repeat_grid - +import os #============================================================================== -def calc_log_posterior(param_vals, state, params, surveys_sep, grid_params, Pn=False, pNreps=True, log_halo=False, lin_host=False, ind_surveys=False): +def calc_log_posterior(param_vals, state, params, surveys_sep, Pn=False, pNreps=True, ptauw=False, + log_halo=False, lin_host=False, ind_surveys=False, g0info=None): """ Calculates the log posterior for a given set of parameters. Assumes uniform priors between the minimum and maximum values provided in 'params'. @@ -44,9 +45,12 @@ def calc_log_posterior(param_vals, state, params, surveys_sep, grid_params, Pn=F surveys_sep[1] : list of repeater surveys grid_params (dictionary) = nz, ndm, dmmax Pn (bool) = Include Pn or not + pNreps (bool) = Include p(N repeaters) or not + ptauw (bool) = Include p(tau,w) or not log_halo (bool) = Use a log uniform prior on DMhalo lin_host (bool) = Use a linear uniform prior on host mean ind_surveys (bool) = Return likelihoods for each survey + g0info (list) = List of [zDMgrid, zvals, DMvals] Passed to use as speedup if needed Outputs: llsum (double) = Total log likelihood for param_vals which is equivalent @@ -58,7 +62,7 @@ def calc_log_posterior(param_vals, state, params, surveys_sep, grid_params, Pn=F # given every value is in the correct range. If any value is not in the correct range, log posterior is -inf in_priors = True param_dict = {} - + for i, (key,val) in enumerate(params.items()): if param_vals[i] < val['min'] or param_vals[i] > val['max']: in_priors = False @@ -73,6 +77,12 @@ def calc_log_posterior(param_vals, state, params, surveys_sep, grid_params, Pn=F if ind_surveys: ll_list = [] + if g0info is not None: + # extract zm grid initial info + zDMgrid = g0info[0] + zvals = g0info[1] + dmvals = g0info[2] + # Check if it is in the priors and do the calculations if in_priors is False: llsum = -np.inf @@ -82,7 +92,8 @@ def calc_log_posterior(param_vals, state, params, surveys_sep, grid_params, Pn=F # it is easy to reach impossible regions of the parameter space. This results in math errors # (log(0), log(negative), sqrt(negative), divide 0 etc.) and hence we assume that these math errors # correspond to an impossible region of the parameter space and so set ll = -inf - try: + #try: + if True: # Set state state.update_params(param_dict) @@ -90,34 +101,43 @@ def calc_log_posterior(param_vals, state, params, surveys_sep, grid_params, Pn=F # Recreate grids every time, but not surveys, so must update survey params for i,s in enumerate(surveys): + + + # updates survey according to DMhalo estimates if 'DMhalo' in param_dict: if log_halo: DMhalo = 10**param_dict['DMhalo'] else: DMhalo = param_dict['DMhalo'] s.init_DMEG(DMhalo) - s.get_efficiency_from_wlist(s.DMlist,s.wlist,s.wplist,model=s.meta['WBIAS']) - + + if ('Wlogmean' in param_dict or 'Wlogsigma' in param_dict or \ + 'Slogmean' in param_dict or 'Slogsigma' in param_dict): + state.scat.Sbackproject = True + s.init_widths(state=state) + elif 'DMhalo' in param_dict: + # this would get re-done within init_widths above, so only do this + # if it has *not* been recalculated + s.do_efficiencies() #get_efficiency_from_wlist(s.wlist,s.wplist,model=s.meta['WBIAS']) + # Initialise grids grids = [] - if len(surveys_sep[0]) != 0: + + # gets new zDM grid if F and H0 in the param_dict + if 'H0' in param_dict or 'logF' in param_dict or g0info is None: zDMgrid, zvals,dmvals = mf.get_zdm_grid( - state, new=True, plot=False, method='analytic', - nz=grid_params['nz'], ndm=grid_params['ndm'], dmmax=grid_params['dmmax'], + state, new=True, plot=False, method='analytic', datdir=resource_filename('zdm', 'GridData')) - + g0info = [zDMgrid, zvals,dmvals] + + if len(surveys_sep[0]) != 0: # generates zdm grid grids += mf.initialise_grids(surveys_sep[0], zDMgrid, zvals, dmvals, state, wdist=True, repeaters=False) if len(surveys_sep[1]) != 0: - zDMgrid, zvals,dmvals = mf.get_zdm_grid( - state, new=True, plot=False, method='analytic', - nz=grid_params['nz'], ndm=grid_params['ndm'], dmmax=grid_params['dmmax'], - datdir=resource_filename('zdm', 'GridData')) - # generates zdm grid grids += mf.initialise_grids(surveys_sep[1], zDMgrid, zvals, dmvals, state, wdist=True, repeaters=True) - + # Minimse the constant accross all surveys if Pn: newC, llC = it.minimise_const_only(None, grids, surveys, update=True) @@ -130,22 +150,22 @@ def calc_log_posterior(param_vals, state, params, surveys_sep, grid_params, Pn=F # calculate all the likelihoods llsum = 0 for s, grid in zip(surveys, grids): - ll = it.get_log_likelihood(grid,s,Pn=Pn,pNreps=pNreps) + ll = it.get_log_likelihood(grid,s,Pn=Pn,pNreps=pNreps,ptauw=ptauw) llsum += ll - if ind_surveys: ll_list.append(ll) - except ValueError as e: - print("Error, setting likelihood to -inf: " + str(e)) - llsum = -np.inf - ll_list = [-np.inf for _ in range(len(surveys))] + #except ValueError as e: + # print("Error, setting likelihood to -inf: " + str(e)) + # llsum = -np.inf + # ll_list = [-np.inf for _ in range(len(surveys))] if np.isnan(llsum): print("llsum was NaN. Setting to -infinity", param_dict) llsum = -np.inf # print("Posterior calc time: " + str(time.time()-t0) + " seconds", flush=True) + if ind_surveys: return llsum, ll_list else: @@ -153,7 +173,8 @@ def calc_log_posterior(param_vals, state, params, surveys_sep, grid_params, Pn=F #============================================================================== -def mcmc_runner(logpf, outfile, state, params, surveys, grid_params, nwalkers=10, nsteps=100, nthreads=1, Pn=False, pNreps=True, log_halo=False, lin_host=False): +def mcmc_runner(logpf, outfile, state, params, surveys, nwalkers=10, nsteps=100, nthreads=1, Pn=False, + pNreps=True, ptauw = False, log_halo=False, lin_host=False, ind_surveys=False, g0info=None): """ Handles the MCMC running. @@ -170,7 +191,10 @@ def mcmc_runner(logpf, outfile, state, params, surveys, grid_params, nwalkers=10 nthreads (int) = Number of threads (currently not implemented - uses default) Pn (bool) = Include Pn or not pNreps (bool) = Include pNreps or not + ptauw (bool) = Include ptauw or not log_halo (bool) = Use a log uniform prior on DMhalo + ind_surveys (bool) = Return individual survey data + g0info (list) = List of [zDMgrid, zvals, DMvals] Passed to use as speedup if needed Outputs: posterior_sample (emcee.EnsembleSampler) = Final sample @@ -191,8 +215,18 @@ def mcmc_runner(logpf, outfile, state, params, surveys, grid_params, nwalkers=10 backend.reset(nwalkers, ndim) start = time.time() - with mp.Pool() as pool: - sampler = emcee.EnsembleSampler(nwalkers, ndim, logpf, args=[state, params, surveys, grid_params, Pn, pNreps, log_halo, lin_host], backend=backend, pool=pool) + + # may or may not be needed + #os.environ["OMP_NUM_THREADS"] = "1" + import multiprocessing as mp + Pool = mp.get_context('fork').Pool + + + + with Pool() as pool: # could add mp.Pool(ntrheads=5) or Pool = None + + sampler = emcee.EnsembleSampler(nwalkers, ndim, logpf, args=[state, params, surveys, Pn, pNreps, + ptauw, log_halo, lin_host, ind_surveys, g0info], backend=backend, pool=pool) sampler.run_mcmc(starting_guesses, nsteps, progress=True) end = time.time() print("Total time taken: " + str(end - start)) @@ -201,4 +235,4 @@ def mcmc_runner(logpf, outfile, state, params, surveys, grid_params, nwalkers=10 return posterior_sample -#============================================================================== \ No newline at end of file +#============================================================================== diff --git a/zdm/MCMC_analysis.py b/zdm/MCMC_analysis.py new file mode 100644 index 0000000..3216efd --- /dev/null +++ b/zdm/MCMC_analysis.py @@ -0,0 +1,218 @@ +""" +This file contains routines for analysing MCMC results + +Routines here were written by Jordan Hoffmann + +""" + + +import numpy as np +from matplotlib import pyplot as plt + +# Here are different plotting functions + +def plot_walkers(samples,labels,outfile,burnin=None): + """ + Puts all walkers from all samples on one plot + If you want different samples per plot, call this function + multiple times + """ + plt.rcParams['font.size'] = 16 + # get the number of parameters + fig, axes = plt.subplots(len(labels), 1, figsize=(20,30), sharex=True) + #plt.title("Sample: " + filenames[j]) + for j,sample in enumerate(samples): + for i,ax in enumerate(axes): + for k in range(sample.shape[1]): + if burnin is None: + ax.plot(sample[:,k,i], '.-', label=str(k)) + else: + ax.plot(sample[burnin[j]:,k,i], '.-', label=str(k)) + + ax.set_ylabel(labels[i]) + + axes[-1].set_xlabel("Step number") + axes[-1].legend() + + plt.tight_layout() + plt.savefig(outfile) + plt.close() + +def plot_autocorrelations(samples,opfile): + """ + Plot the autocorrelation time to estimate the burnin + Do this once bad walkers have been discarded. + To be done: proper explanation of what this routine is doing + """ + burnin = [] + fig, ax = plt.subplots(1,1,figsize=(8,6)) + for isample,sample in enumerate(samples): + # Compute the estimators for a few different chain lengths + N = np.exp(np.linspace(np.log(10), np.log(sample.shape[0]), 10)).astype(int) + new = np.empty(len(N)) + for i, n in enumerate(N): + new[i] = autocorr(sample[:, :n, 0].T) + + # Plot the comparisons + + ax.loglog(N, new, "o-", label="new") + ylim = ax.get_ylim() + ax.plot(N, N / 50.0, "--k", label=r"$\tau = N/50$") + ax.set_ylim(ylim) + ax.legend(fontsize=14); + + burnin.append(int(1.5*new[-1])) + + ax.set_xlabel("number of samples, $N$") + ax.set_ylabel(r"$\tau$ estimates") + plt.tight_layout() + plt.savefig(opfile) + plt.close() + + +# # Implement burnin and change priors +# +# - Changes prior to discard samples outside the specified prior range +# - Implements the burnin using either the predefined burnin or a constant specified + +# Enforce more restrictive priors on a parameter +# get rid of burnin first! +def change_priors(sample, param_num, max=np.inf, min=-np.inf): + + condition = np.logical_and(sample[:,param_num] > min, sample[:,param_num] < max) + good_idxs = np.flatnonzero(condition) + + return sample[good_idxs, :] + +# Here we present different methods to get the burnin from +# https://emcee.readthedocs.io/en/stable/tutorials/autocorr/#a-more-realistic-example +# however we note that in actuality it is generally easier and more useful to specify +# burnin=200 or something similar which is done further below. + +def next_pow_two(n): + i = 1 + while i < n: + i = i << 1 + return i + +def autocorr_func_1d(x, norm=True): + x = np.atleast_1d(x) + if len(x.shape) != 1: + raise ValueError("invalid dimensions for 1D autocorrelation function") + n = next_pow_two(len(x)) + + # Compute the FFT and then (from that) the auto-correlation function + f = np.fft.fft(x - np.mean(x), n=2 * n) + acf = np.fft.ifft(f * np.conjugate(f))[: len(x)].real + acf /= 4 * n + + # Optionally normalize + if norm and acf[0] != 0: + acf /= acf[0] + + return acf + +# Automated windowing procedure following Sokal (1989) +def auto_window(taus, c): + m = np.arange(len(taus)) < c * taus + if np.any(m): + return np.argmin(m) + return len(taus) - 1 + +def autocorr(y, c=5.0): + f = np.zeros(y.shape[1]) + for yy in y: + f += autocorr_func_1d(yy) + f /= len(y) + taus = 2.0 * np.cumsum(f) - 1.0 + window = auto_window(taus, c) + return taus[window] + + + +# - Discards any walkers that do not converge +# Reject walkers with bad autocorrelation values +def auto_corr_rej(samples, burnin=0): + good_samples = [] + + # Loop through each sample and generate a list of good walkers and bad walkers + for j,sample in enumerate(samples): + # burnin=200 + good_walkers = [] + bad_walkers = [] + + + # for i in range(sample.shape[1]): + # # if np.all(sample[burnin:burnin+30,i,0] == sample[burnin,i,0]): + # if ( np.std(sample[burnin:burnin+30,i,0] ) ) + # bad_walkers.append(i) + # else: + # good_walkers.append(i) + + # Loop through each walker in the current sample + for i in range(sample.shape[1]): + bad = False + + # Loop through each parameter for the walker + for k in range(sample.shape[2]): + + # If any of the parameters have a bad autocorrelation function then set as a bad walker + acf = autocorr_func_1d(sample[burnin:,i,k], norm=False) + if np.max(acf) < 1e-10: + bad = True + break + + if bad: + bad_walkers.append(i) + else: + good_walkers.append(i) + + + print("Discarded walkers for sample " + str(j) + ": " + str(bad_walkers)) + + # Add the new sample with the bad walkers discarded to the good_samples list + good_samples.append(sample[burnin:,good_walkers,:]) + + return good_samples + +# Reject walkers with small standard deviations +def std_rej(samples, burnin=0): + good_samples = [] + + if not type(burnin) == list: + burnin = [burnin for i in range(len(samples))] + + # Loop through each sample + for i, sample in enumerate(samples): + bad_walkers = [] + good_walkers = [] + + # For each parameter + for k in range(sample.shape[2]): + sd = [] + + # Loop through every walker and get a list of the standard deviations + for j in range(sample.shape[1]): + sd.append(np.std(sample[burnin[i]:burnin[i]+100,j,k])) + + # Normalise standard deviation + sd = sd / np.max(sd) + + # Flag any walkers with standard deviations less than 1e-2 + bad_walkers = [] # np.flatnonzero(sd < 1e-2) + temp = [] + for m in range(len(sd)): + if sd[m] < 1e-2: + bad_walkers.append(m) + + bad_walkers = np.unique(np.array(bad_walkers)) + + print("Discarded walkers for sample " + str(i) + ": " + str(bad_walkers)) + for l in range(sample.shape[1]): + if l not in bad_walkers: + good_walkers.append(l) + + # Add the new sample with the bad walkers discarded to the good_samples list + good_samples.append(sample[:,good_walkers,:]) + + return good_samples diff --git a/zdm/data/Surveys/SKA_low.ecsv b/zdm/data/Surveys/SKA_low.ecsv new file mode 100644 index 0000000..1bf4761 --- /dev/null +++ b/zdm/data/Surveys/SKA_low.ecsv @@ -0,0 +1,26 @@ +# %ECSV 1.0 +# --- +# datatype: +# - {name: TNS, datatype: string} +# - {name: BW, datatype: float64} +# - {name: DM, datatype: float64} +# - {name: DMG, datatype: float64} +# - {name: FBAR, datatype: float64} +# - {name: FRES, datatype: float64} +# - {name: Gb, datatype: string, subtype: 'float64[null]'} +# - {name: Gl, datatype: string, subtype: 'float64[null]'} +# - {name: NREP, datatype: int64} +# - {name: SNR, datatype: float64} +# - {name: SNRTHRESH, datatype: float64} +# - {name: THRESH, datatype: float64} +# - {name: TRES, datatype: float64} +# - {name: WIDTH, datatype: float64} +# - {name: XDec, datatype: string, subtype: 'float64[null]'} +# - {name: XRA, datatype: string, subtype: 'float64[null]'} +# - {name: Z, datatype: float64} +# meta: !!omap +# - {survey_data: "{\n \"observing\": {\n \"NORM_FRB\": 1,\n \"TOBS\": 1\n },\n \"telescope\": {\n \ +# \ \"DIAM\": 40,\n \"BMETHOD\": 0,\n \"NBEAMS\": 1,\n \"NBINS\": 10\n }\n}"} +# schema: astropy-2.0 +TNS BW DM DMG FBAR FRES Gb Gl NREP SNR SNRTHRESH THRESH TRES WIDTH XDec XRA Z +0 770 200 35.0 1284 0.0145 "" "" 1 10. 10. 0.2 0.069 1.0 "" "" 0.276 diff --git a/zdm/data/Surveys/SKA_mid.ecsv b/zdm/data/Surveys/SKA_mid.ecsv index 05176ae..476400c 100644 --- a/zdm/data/Surveys/SKA_mid.ecsv +++ b/zdm/data/Surveys/SKA_mid.ecsv @@ -23,4 +23,4 @@ # \ \"DIAM\": 15,\n \"BMETHOD\": 0,\n \"NBEAMS\": 1,\n \"NBINS\": 10\n }\n}"} # schema: astropy-2.0 TNS BW DM DMG FBAR FRES Gb Gl NREP SNR SNRTHRESH THRESH TRES WIDTH XDec XRA Z -0 770 200 35.0 1284 0.836 "" "" 1 10. 10. 0.2 0.30624 1.0 "" "" 0.276 +0 770 200 35.0 1284 0.1075 "" "" 1 10. 10. 0.2 0.069 1.0 "" "" 0.276 diff --git a/zdm/grid.py b/zdm/grid.py index 652464b..b2e6bf4 100644 --- a/zdm/grid.py +++ b/zdm/grid.py @@ -338,7 +338,7 @@ def calc_pdv(self, beam_b=None, beam_o=None): Emax = 10 ** self.state.energy.lEmax # this implementation allows us to access the b-fractions later on - if (not (self.b_fractions is not None)) or (beam_b is not None): + if (self.b_fractions is None) or (beam_b is not None): self.b_fractions = np.zeros( [self.zvals.size, self.dmvals.size, self.beam_b.size] ) @@ -352,7 +352,7 @@ def calc_pdv(self, beam_b=None, beam_o=None): self.thresholds ) # use when calling in log10 space conversion main_beam_b = np.log10(main_beam_b) - + for i, b in enumerate(main_beam_b): # if eff_weights is 2D (i.e., z-dependent) then w is a vector of length NZ for j, w in enumerate(self.eff_weights): @@ -376,7 +376,86 @@ def calc_pdv(self, beam_b=None, beam_o=None): self.b_fractions, axis=2 ) # sums over b-axis [ we could ignore this step?] self.pdv = np.multiply(self.fractions.T, self.dV).T - + + def get_pw_dist(self): + """ + Function asking the grid to return the p(w) distribution. + + This will be an "all-burst" distribution in case of a + repeater inherited class. + + Note that a grid does not actually know what a "width" means: + it is simply an abstract category of FRBs corresponding to a + particular efficiency and fraction of the population. + + Args: None + + Returns: + Wtots (np.ndarray): Rate per width bin + Wzs (np.ndarray: Nw x Nz): Rate as a function of w and z + Wdms (np.ndarray: Nw x Ndm): Rate as a function of w and DM + """ + + Wtots = np.zeros([self.nw]) + Wzs = np.zeros([self.nw,self.nz]) + Wdms = np.zeros([self.nw,self.ndm]) + + const = 10**self.state.FRBdemo.lC + + # consider if beamb in log10 space + # call log10 beam + if self.use_log10: + new_thresh = np.log10( + self.thresholds + ) # use when calling in log10 space conversion + main_beam_b = np.log10(self.beam_b) + else: + main_beam_b = self.beam_b + + # For convenience and speed up + Emin = 10 ** self.state.energy.lEmin + Emax = 10 ** self.state.energy.lEmax + + # we record b-fractions, but NOT the width increments in each + for j, w in enumerate(self.eff_weights): + # if eff_weights is 2D (i.e., z-dependent) then w is a vector of length NZ + + # resets p(z,dm) for this w + Warray = np.zeros([self.nz,self.ndm]) # we re-use this array for each w + + # sums over the beam values + for i, b in enumerate(self.beam_b): + + # using log10 space conversion + if self.use_log10: + thresh = new_thresh[j, :, :] - b + else: # original + thresh = self.thresholds[j, :, :] / b + + # the below is to ensure this works when w is a vector of length nz + w = np.array(w) + + Warray[:, :] += ( + self.beam_o[i] + * (self.array_cum_lf( + thresh, Emin, Emax, self.state.energy.gamma, self.use_log10 + ).T * w.T).T + ) + + # accounts for z-dependent volumetric fractions + Warray = np.multiply(Warray.T, self.dV).T + + # multiply by the DM distribution and star-formation-rate scaling + # and constant number of FRBs + Warray *= self.sfr_smear * const + + Wzs[j,:] = np.sum(Warray,axis = 1) + Wdms[j,:] = np.sum(Warray,axis = 0) + Wtots[j] = np.sum(Wzs[j,:]) + + return Wtots,Wzs,Wdms + + def calc_rates(self): """ multiplies the rate per cell with the appropriate pdm plot """ @@ -449,6 +528,8 @@ def calc_thresholds(self, F0:float, self.eff_table = eff_table self.eff_weights = weights + self.nw = self.eff_weights.shape[0] + # now two or three dimensions Eff_thresh = F0 / self.eff_table @@ -580,7 +661,7 @@ def initMC(self): Emax = 10 ** lEmax # grid of beam values, weights - nw = self.eff_weights.size + nw = self.nw nb = self.beam_b.size if self.eff_weights.ndim > 1: @@ -676,7 +757,7 @@ def GenMCFRB(self, Emax_boost): Emax = 10 ** lEmax # grid of beam values, weights - nw = self.eff_weights.size + nw = self.nw if self.eff_weights.ndim > 1: raise ValueError("MC generation from z-dependent widths not currently enabled") nb = self.beam_b.size diff --git a/zdm/iteration.py b/zdm/iteration.py index 85f704b..1bc6d7f 100644 --- a/zdm/iteration.py +++ b/zdm/iteration.py @@ -113,7 +113,7 @@ def maximise_likelihood(grid,survey): return results -def get_log_likelihood(grid, s, norm=True, psnr=True, Pn=False, pNreps=True): +def get_log_likelihood(grid, s, norm=True, psnr=True, Pn=False, pNreps=True, ptauw=False): """ Returns the likelihood for the grid given the survey. @@ -123,6 +123,7 @@ def get_log_likelihood(grid, s, norm=True, psnr=True, Pn=False, pNreps=True): norm = Normalise psnr = Include psnr in likelihood Pn = Include Pn in likelihood + ptauw = Include p(tau,width) Outputs: llsum = Total loglikelihood for the grid @@ -131,15 +132,19 @@ def get_log_likelihood(grid, s, norm=True, psnr=True, Pn=False, pNreps=True): if isinstance(grid, zdm_repeat_grid.repeat_Grid): # Repeaters if s.nDr==1: - llsum1, lllist, expected = calc_likelihoods_1D(grid, s, norm=norm, psnr=psnr, dolist=1, grid_type=1, Pn=Pn, pNreps=pNreps) + llsum1, lllist, expected = calc_likelihoods_1D(grid, s, norm=norm, psnr=psnr, + dolist=1, grid_type=1, Pn=Pn, pNreps=pNreps, ptauw=ptauw) llsum = llsum1 # print(s.name, "repeaters:", lllist) elif s.nDr==2: - llsum1, lllist, expected = calc_likelihoods_2D(grid, s, norm=norm, psnr=psnr, dolist=1, grid_type=1, Pn=Pn, pNreps=pNreps) + llsum1, lllist, expected = calc_likelihoods_2D(grid, s, norm=norm, psnr=psnr, + dolist=1, grid_type=1, Pn=Pn, pNreps=pNreps, ptauw=ptauw) llsum = llsum1 elif s.nDr==3: - llsum1, lllist1, expected1 = calc_likelihoods_1D(grid, s, norm=norm, psnr=psnr, dolist=1, grid_type=1, Pn=Pn, pNreps=pNreps) - llsum2, lllist2, expected2 = calc_likelihoods_2D(grid, s, norm=norm, psnr=psnr, dolist=1, grid_type=1, Pn=False, pNreps=False) + llsum1, lllist1, expected1 = calc_likelihoods_1D(grid, s, norm=norm, psnr=psnr, + dolist=1, grid_type=1, Pn=Pn, pNreps=pNreps, ptauw=ptauw) + llsum2, lllist2, expected2 = calc_likelihoods_2D(grid, s, norm=norm, psnr=psnr, + dolist=1, grid_type=1, Pn=False, pNreps=False, ptauw=ptauw) llsum = llsum1 + llsum2 else: print("Implementation is only completed for nD 1-3.") @@ -147,29 +152,37 @@ def get_log_likelihood(grid, s, norm=True, psnr=True, Pn=False, pNreps=True): # Singles if s.nDs==1: - llsum1, lllist, expected = calc_likelihoods_1D(grid, s, norm=norm, psnr=psnr, dolist=1, grid_type=2, Pn=Pn) + llsum1, lllist, expected = calc_likelihoods_1D(grid, s, norm=norm, psnr=psnr, + dolist=1, grid_type=2, Pn=Pn, ptauw=ptauw) llsum += llsum1 # print(s.name, "singles:", lllist) elif s.nDs==2: - llsum1, lllist, expected = calc_likelihoods_2D(grid, s, norm=norm, psnr=psnr, dolist=1, grid_type=2, Pn=Pn) + llsum1, lllist, expected = calc_likelihoods_2D(grid, s, norm=norm, psnr=psnr, + dolist=1, grid_type=2, Pn=Pn, ptauw=ptauw) llsum += llsum1 elif s.nDs==3: - llsum1, lllist1, expected1 = calc_likelihoods_1D(grid, s, norm=norm, psnr=psnr, dolist=1, grid_type=2, Pn=Pn) - llsum2, lllist2, expected2 = calc_likelihoods_2D(grid, s, norm=norm, psnr=psnr, dolist=1, grid_type=2, Pn=False) + llsum1, lllist1, expected1 = calc_likelihoods_1D(grid, s, norm=norm, psnr=psnr, + dolist=1, grid_type=2, Pn=Pn, ptauw=ptauw) + llsum2, lllist2, expected2 = calc_likelihoods_2D(grid, s, norm=norm, psnr=psnr, + dolist=1, grid_type=2, Pn=False, ptauw=ptauw) llsum = llsum + llsum1 + llsum2 else: print("Implementation is only completed for nD 1-3.") exit() else: if s.nD==1: - llsum1, lllist, expected = calc_likelihoods_1D(grid, s, norm=norm, psnr=psnr, dolist=1, Pn=Pn) + llsum1, lllist, expected = calc_likelihoods_1D(grid, s, norm=norm, psnr=psnr, + dolist=1, Pn=Pn, ptauw=ptauw) llsum = llsum1 elif s.nD==2: - llsum1, lllist, expected = calc_likelihoods_2D(grid, s, norm=norm, psnr=psnr, dolist=1, Pn=Pn) + llsum1, lllist, expected = calc_likelihoods_2D(grid, s, norm=norm, psnr=psnr, + dolist=1, Pn=Pn, ptauw=ptauw) llsum = llsum1 elif s.nD==3: - llsum1, lllist1, expected1 = calc_likelihoods_1D(grid, s, norm=norm, psnr=psnr, dolist=1, Pn=Pn) - llsum2, lllist2, expected2 = calc_likelihoods_2D(grid, s, norm=norm, psnr=psnr, dolist=1, Pn=False) + llsum1, lllist1, expected1 = calc_likelihoods_1D(grid, s, norm=norm, psnr=psnr, + dolist=1, Pn=Pn, ptauw=ptauw) + llsum2, lllist2, expected2 = calc_likelihoods_2D(grid, s, norm=norm, psnr=psnr, + dolist=1, Pn=False, ptauw=ptauw) llsum = llsum1 + llsum2 else: print("Implementation is only completed for nD 1-3.") @@ -336,16 +349,28 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True, tomult /= np.sum(tomult,axis=0) # vectors below are [nz,NFRB] in length - ptaus = survey.ptaus[:,itaus1,iws1]*dktaus1*dkws1\ - + survey.ptaus[:,itaus1,iws2]*dktaus1*dkws2 \ - + survey.ptaus[:,itaus2,iws1]*dktaus1*dkws1 \ - + survey.ptaus[:,itaus2,iws2]*dktaus1*dkws2 - piws = survey.pws[:,iis1,iws1]*dkis1*dkws1 \ + survey.pws[:,iis1,iws2]*dkis1*dkws2 \ + survey.pws[:,iis2,iws1]*dkis1*dkws1 \ + survey.pws[:,iis2,iws2]*dkis1*dkws2 + ptaus = survey.ptaus[:,itaus1,iws1]*dktaus1*dkws1\ + + survey.ptaus[:,itaus1,iws2]*dktaus1*dkws2 \ + + survey.ptaus[:,itaus2,iws1]*dktaus1*dkws1 \ + + survey.ptaus[:,itaus2,iws2]*dktaus1*dkws2 + + if False: + plt.figure() + plt.xlabel("z") + plt.ylabel("ptau") + plt.yscale("log") + NFRB = len(Tauobs) + for i in np.arange(NFRB): + plt.plot(zvals,ptaus[:,i],label=str(Tauobs[i])[0:4]+", "+str(Iwobs[i])[0:4]+", "+str(Wobs[i])[0:4]) + plt.text(1,0.05,str(10**survey.slogmean)[0:5]+" "+str(survey.slogsigma)[0:5]) + plt.legend() + plt.show() + exit() # we now multiply by the z-dependencies ptaus *= tomult @@ -355,10 +380,17 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True, ptaus = np.sum(ptaus,axis=0) piws = np.sum(piws,axis=0) + bad1 = np.where(piws==0) + bad2 = np.where(ptaus==0) + piws[bad1] = 1e-10 + ptaus[bad2] = 1e-10 + llptw = np.sum(np.log10(ptaus)) llpiw = np.sum(np.log10(piws)) llsum += llptw llsum += llpiw + + lllist.append(llptw) lllist.append(llpiw) @@ -511,12 +543,14 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True, # we would like to calculate \int p(w|z) p(z) dz # we begin by calculating p(w|z), below, by normalising for each z # normalise over all w values for each z + # Q: should we calculate p(w|b,z) then multiply by p(b,w)? dpbws /= np.sum(dpbws,axis=0) temp = dpbws[iws1,:,inoztaulist] + # tomult is p(z) temp *= tomult.T pws = np.sum(temp,axis=1) bad = np.where(pws == 0.)[0] - pws[bad] = 1.e-10 # prevents nans, but + pws[bad] = 1.e-10 # prevents nans, but penalty is a bit arbitrary. llpws = np.sum(np.log10(pws)) llsum += llpws lllist.append(llpws) @@ -826,8 +860,9 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal Pn=Poisson_p(observed,expected) if Pn==0: Pll=-1e10 - if dolist==0: - return Pll + # otherwise 1e-10 might be better than the actual total ll! + #if dolist==0: + # return Pll else: Pll=np.log10(Pn) lllist.append(Pll) @@ -874,16 +909,6 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal #ztidms1,ztidms2,ztdkdms1,ztdkdms2 = grid.get_dm_coeffs(ztDMobs) ztizs1,ztizs2,ztdkzs1,ztdkzs2 = grid.get_z_coeffs(ztZobs) - - ptaus = survey.ptaus[ztizs1,itaus1,iws1]*ztdkzs1*dktaus1*dkws1 \ - + survey.ptaus[ztizs1,itaus1,iws2]*ztdkzs1*dktaus1*dkws2 \ - + survey.ptaus[ztizs1,itaus2,iws1]*ztdkzs1*dktaus1*dkws1 \ - + survey.ptaus[ztizs1,itaus2,iws2]*ztdkzs1*dktaus1*dkws2 \ - + survey.ptaus[ztizs2,itaus1,iws1]*ztdkzs2*dktaus1*dkws1 \ - + survey.ptaus[ztizs2,itaus1,iws2]*ztdkzs2*dktaus1*dkws2 \ - + survey.ptaus[ztizs2,itaus2,iws1]*ztdkzs2*dktaus2*dkws1 \ - + survey.ptaus[ztizs2,itaus2,iws2]*ztdkzs2*dktaus2*dkws2 - piws = survey.pws[ztizs1,iis1,iws1]*ztdkzs1*dkis1*dkws1 \ + survey.pws[ztizs1,iis1,iws2]*ztdkzs1*dkis1*dkws2 \ + survey.pws[ztizs1,iis2,iws1]*ztdkzs1*dkis1*dkws1 \ @@ -893,12 +918,26 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal + survey.pws[ztizs2,iis2,iws1]*ztdkzs2*dkis2*dkws1 \ + survey.pws[ztizs2,iis2,iws2]*ztdkzs2*dkis2*dkws2 - llptw = np.sum(np.log10(ptaus)) + ptaus = survey.ptaus[ztizs1,itaus1,iws1]*ztdkzs1*dktaus1*dkws1 \ + + survey.ptaus[ztizs1,itaus1,iws2]*ztdkzs1*dktaus1*dkws2 \ + + survey.ptaus[ztizs1,itaus2,iws1]*ztdkzs1*dktaus1*dkws1 \ + + survey.ptaus[ztizs1,itaus2,iws2]*ztdkzs1*dktaus1*dkws2 \ + + survey.ptaus[ztizs2,itaus1,iws1]*ztdkzs2*dktaus1*dkws1 \ + + survey.ptaus[ztizs2,itaus1,iws2]*ztdkzs2*dktaus1*dkws2 \ + + survey.ptaus[ztizs2,itaus2,iws1]*ztdkzs2*dktaus2*dkws1 \ + + survey.ptaus[ztizs2,itaus2,iws2]*ztdkzs2*dktaus2*dkws2 + + # safegaudr zero probabilities + bad1 = np.where(piws==0)[0] + bad2 = np.where(ptaus==0)[0] + piws[bad1] = 1e-10 + ptaus[bad2] = 1e-10 llpiw = np.sum(np.log10(piws)) - llsum += llptw + llptw = np.sum(np.log10(ptaus)) llsum += llpiw - lllist.append(llptw) + llsum += llptw lllist.append(llpiw) + lllist.append(llptw) ############ Calculates p(s | z,DM) ############# @@ -1013,6 +1052,7 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal pws = dpbws[iws1,iztaulist]*dkws1 + dpbws[iws2,iztaulist]*dkws2 bad = np.where(pws == 0.)[0] pws[bad] = 1.e-10 # prevents nans, but + llpws = np.sum(np.log10(pws)) llsum += llpws lllist.append(llpws) @@ -1069,7 +1109,6 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal f"wzterm={np.sum(np.log10(psnr)):0.2f}," \ f"comb={np.sum(np.log10(psnr*pvals)):0.2f}") - if dolist==0: return llsum elif dolist==1: diff --git a/zdm/parameters.py b/zdm/parameters.py index e22e6a5..f5339a2 100644 --- a/zdm/parameters.py +++ b/zdm/parameters.py @@ -155,7 +155,6 @@ class RepeatParams(data_class.myDataClass): 'Notation': '$E_R$', }) - # Galactic parameters @dataclass class MWParams(data_class.myDataClass): @@ -260,9 +259,8 @@ class WidthParams(data_class.myDataClass): "Notation": "w_{\\rm min}", }, ) - WNbins: int = field( - default=8, + default=11, metadata={"help": "Number of bins for FRB width distribution", "unit": ""}, ) WNInternalBins: int = field( @@ -275,11 +273,11 @@ class WidthParams(data_class.myDataClass): ) WMin: int = field( default=0.1, - metadata={"help": "Minimum scattering value to model", "unit": "ms"}, + metadata={"help": "Minimum width value to model", "unit": "ms"}, ) WMax: int = field( default=1000, - metadata={"help": "Maximum scattering value to model", "unit": "ms"}, + metadata={"help": "Maximum width value to model", "unit": "ms"}, ) diff --git a/zdm/scripts/MCMC/MCMC_wrap.py b/zdm/scripts/MCMC/MCMC_wrap.py index 185ed70..47b4b14 100644 --- a/zdm/scripts/MCMC/MCMC_wrap.py +++ b/zdm/scripts/MCMC/MCMC_wrap.py @@ -16,12 +16,13 @@ import numpy as np from astropy.cosmology import Planck18 - +from pkg_resources import resource_filename from zdm import survey from zdm import cosmology as cos from zdm import loading from zdm import MCMC from zdm import parameters +from zdm import misc_functions as mf import pickle import json @@ -50,11 +51,16 @@ def main(): parser.add_argument('-w', '--walkers', default=20, type=int, help="Number of MCMC walkers") parser.add_argument('-s', '--steps', default=100, type=int, help="Number of MCMC steps") parser.add_argument('-n', '--nthreads', default=1, type=int, help="Number of threads") + parser.add_argument('--Nz', default=500, type=int, help="Number of z values") + parser.add_argument('--Ndm', default=1400, type=int, help="Number of DM values") + parser.add_argument('--zmax', default=5., type=int, help="Maximum z value") + parser.add_argument('--dmmax', default=7000., type=int, help="Maximum DM value") parser.add_argument('--sdir', default=None, type=str, help="Directory containing surveys") parser.add_argument('--edir', default=None, type=str, help="Directory containing efficiency files") parser.add_argument('--outdir', default="", type=str, help="Output directory") parser.add_argument('--Pn', default=False, action='store_true', help="Include Pn") parser.add_argument('--pNreps', default=False, action='store_true', help="Include pNreps") + parser.add_argument('--ptauw', default=False, action='store_true', help="Include p(tau,w)") parser.add_argument('--rand', default=False, action='store_true', help="Randomise DMG within uncertainty") parser.add_argument('--log_halo', default=False, action='store_true', help="Give a log prior on the halo instead of linear") parser.add_argument('--lin_host', default=False, action='store_true', help="Give a linear prior on host mean contribution") @@ -87,24 +93,39 @@ def main(): # Initialise surveys surveys = [[], []] - grid_params = {} - grid_params['dmmax'] = 7000.0 - grid_params['ndm'] = 1400 - grid_params['nz'] = 500 - ddm = grid_params['dmmax'] / grid_params['ndm'] - # it would be best to trial initialising a "get_zdm_grid" here to extract z and dm values - dmvals = (np.arange(grid_params['ndm']) + 0.5) * ddm - zvals = (np.arange(nz) + 0.5) * dz + zDMgrid, zvals, dmvals = mf.get_zdm_grid( + state, new=True, plot=False, method='analytic', + datdir=resource_filename('zdm', 'GridData'), + nz=args.Nz,ndm=args.Ndm,zmax=args.zmax,dmmax=args.dmmax) + + # pass this to starting iteration + g0info = [zDMgrid,zvals,dmvals] + + # set z-dependent weights in surveys + if ('Wlogmean' in params or 'Wlogsigma' in params or \ + 'Slogmean' in params or 'Slogsigma' in params): + survey_dict = {"WMETHOD": 3} + else: + survey_dict = None if args.files is not None: for survey_name in args.files: - s = survey.load_survey(survey_name, state, dmvals, zvals=zvals, + if "CHIME" in survey_name: + use_dict = None + else: + use_dict=survey_dict + s = survey.load_survey(survey_name, state, dmvals, zvals=zvals, survey_dict=use_dict, sdir=args.sdir, edir=args.edir, rand_DMG=args.rand) surveys[0].append(s) if args.rep_surveys is not None: for survey_name in args.rep_surveys: - s = survey.load_survey(survey_name, state, dmvals, zvals=zvals, + if "CHIME" in survey_name: + use_dict = None + else: + use_dict=survey_dict + + s = survey.load_survey(survey_name, state, dmvals, zvals=zvals, survey_dict=use_dict, sdir=args.sdir, edir=args.edir, rand_DMG=args.rand) surveys[1].append(s) @@ -113,8 +134,8 @@ def main(): os.mkdir(args.outdir) MCMC.mcmc_runner(MCMC.calc_log_posterior, os.path.join(args.outdir, args.opfile), state, params, surveys, - grid_params, nwalkers=args.walkers, nsteps=args.steps, nthreads=args.nthreads, Pn=args.Pn, pNreps=args.pNreps, - log_halo=args.log_halo, lin_host=args.lin_host) + nwalkers=args.walkers, nsteps=args.steps, nthreads=args.nthreads, Pn=args.Pn, pNreps=args.pNreps, + ptauw = args.ptauw, log_halo=args.log_halo, lin_host=args.lin_host,g0info=g0info) #============================================================================== diff --git a/zdm/scripts/MCMC/average_grids.py b/zdm/scripts/MCMC/average_grids.py index 313d305..085de24 100644 --- a/zdm/scripts/MCMC/average_grids.py +++ b/zdm/scripts/MCMC/average_grids.py @@ -61,7 +61,8 @@ def main(): # Set up state state = parameters.State() state.set_astropy_cosmo(Planck18) - state.update_params(config) + if config is not None: + state.update_params(config) # We explicitly load this to allow the "average grids" routine # to more speedily calculate many grids @@ -221,6 +222,7 @@ def get_samples(args): # If there is no .out file, then the parameters must be specified manually else: params = ["sfr_n", "alpha", "lmean", "lsigma", "lEmax", "lEmin", "gamma", "H0"] + config = None return sample, params, config diff --git a/zdm/scripts/MCMC/check_emcee.py b/zdm/scripts/MCMC/check_emcee.py new file mode 100644 index 0000000..632b469 --- /dev/null +++ b/zdm/scripts/MCMC/check_emcee.py @@ -0,0 +1,49 @@ +""" +This is a function to test if emcee is running on your computer + +It defines a simple probability (log_prob) and +submits this a a pooled job over your cpus. + +Original test code taken from +https://emcee.readthedocs.io/en/stable/tutorials/parallel/ +Note a modification to allow emcee to work on mac osx +""" + + +import time +import numpy as np + + +def log_prob(theta): + t = time.time() + np.random.uniform(0.005, 0.008) + while True: + if time.time() >= t: + break + return -0.5 * np.sum(theta**2) + +import emcee + + +np.random.seed(42) +initial = np.random.randn(32, 5) +nwalkers, ndim = initial.shape +nsteps = 20 + +import os +os.environ["OMP_NUM_THREADS"] = "1" + +import multiprocessing as mp +# this mod is required for running on mac osx +Pool = mp.get_context('fork').Pool +from multiprocessing import cpu_count + +ncpu = cpu_count() +print("{0} CPUs".format(ncpu)) + +with Pool() as pool: + sampler = emcee.EnsembleSampler(nwalkers, ndim, log_prob,pool=pool) + start = time.time() + sampler.run_mcmc(initial, nsteps, progress=True) + end = time.time() + serial_time = end - start + print("MP took {0:.1f} seconds".format(serial_time)) diff --git a/zdm/scripts/Scattering/fit_w_scat.py b/zdm/scripts/Scattering/fit_w_scat.py deleted file mode 100644 index 480d336..0000000 --- a/zdm/scripts/Scattering/fit_w_scat.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -This script creates a 3D distribution. For each values -of width w, it determines the relative contribution of -each scattering and width value to that w. - -This then allows FRB w and tau values to be fit directly, -rather than indirectly via a total effective width. - -This is not a 5D problem (z,DM,w,scat,tau) because -p(w|z,DM) is independent of the tau and w that -contributed to it. - -Nonetheless, p(tau,w) is z-dependent, hence we need a p - -""" - -import os - -from zdm import cosmology as cos -from zdm import figures -from zdm import parameters -from zdm import survey -from zdm import pcosmic -from zdm import iteration as it -from zdm import loading -from zdm import io -from pkg_resources import resource_filename -import numpy as np -from matplotlib import pyplot as plt - -import matplotlib - -defaultsize=14 -ds=4 -font = {'family' : 'Helvetica', - 'weight' : 'normal', - 'size' : defaultsize} -matplotlib.rc('font', **font) - - -def main(): - """ - - """ - - # in case you wish to switch to another output directory - opdir = "Plots/" - if not os.path.exists(opdir): - os.mkdir(opdir) - - # directory where the survey files are located. The below is the default - - # you can leave this out, or change it for a different survey file location. - sdir = os.path.join(resource_filename('zdm', 'data'), 'Surveys') - - survey_name = "CRAFT_average_ICS" - - # make this into a list to initialise multiple surveys art once - names = ["CRAFT_ICS_892","CRAFT_ICS_1300","CRAFT_ICS_1632"] - - repeaters=False - # sets plotting limits - zmax = 2. - dmmax = 2000 - - survey_dict = {"WMETHOD": 3} - state_dict = {} - state_dict["scat"] = {} - state_dict["scat"]["Sbackproject"] = True # turns on backprojection of tau and width for our model - state_dict["scat"]["Sbackproject"] = True - state_dict["width"] = {} - state_dict["width"]["WNInternalBins"] = 100 # sets it to a small quantity - - surveys, grids = loading.surveys_and_grids(survey_names = names,\ - repeaters=repeaters, sdir=sdir,nz=70,ndm=140, - survey_dict = survey_dict, state_dict = state_dict) - - # gets log-likelihoods including tau,w - s=surveys[0] - g=grids[0] - ll1 = it.calc_likelihoods_1D(g,s,Pn=True,pNreps=True,ptauw=True,dolist=0) - ll2 = it.calc_likelihoods_2D(g,s,Pn=True,pNreps=True,psnr=True,ptauw=True,dolist=0) - print(ll1,ll2) -main() diff --git a/zdm/scripts/Scattering/width_scat_dists.py b/zdm/scripts/Scattering/width_scat_dists.py new file mode 100644 index 0000000..7ba3e8e --- /dev/null +++ b/zdm/scripts/Scattering/width_scat_dists.py @@ -0,0 +1,208 @@ +""" +This script creates a 3D distribution. For each values +of width w, it determines the relative contribution of +each scattering and width value to that w. + +This then allows FRB w and tau values to be fit directly, +rather than indirectly via a total effective width. + +This is not a 5D problem (z,DM,w,scat,tau) because +p(w|z,DM) is independent of the tau and w that +contributed to it. + + +""" + +import os + +from zdm import cosmology as cos +from zdm import figures +from zdm import parameters +from zdm import survey +from zdm import pcosmic +from zdm import iteration as it +from zdm import loading +from zdm import io +from pkg_resources import resource_filename +import numpy as np +from matplotlib import pyplot as plt + +import matplotlib + +defaultsize=14 +ds=4 +font = {'family' : 'Helvetica', + 'weight' : 'normal', + 'size' : defaultsize} +matplotlib.rc('font', **font) + + +def main(): + """ + + """ + + # in case you wish to switch to another output directory + opdir = "Plots/" + if not os.path.exists(opdir): + os.mkdir(opdir) + + # directory where the survey files are located. The below is the default - + # you can leave this out, or change it for a different survey file location. + sdir = os.path.join(resource_filename('zdm', 'data'), 'Surveys') + + # make this into a list to initialise multiple surveys art once + names = ["CRAFT_ICS_1300"] # for example + + repeaters=False + # sets plotting limits + zmax = 2. + dmmax = 2000 + + survey_dict = {"WMETHOD": 3} + state_dict = {} + state_dict["scat"] = {} + state_dict["scat"]["Sbackproject"] = True # turns on backprojection of tau and width for our model + state_dict["width"] = {} + state_dict["width"]["WNInternalBins"] = 1000 # sets it to a small quantity + state_dict["width"]["WNbins"] = 33 # set to large number for this analysis + + surveys, grids = loading.surveys_and_grids(survey_names = names,\ + repeaters=repeaters, sdir=sdir,nz=70,ndm=140, + survey_dict = survey_dict, state_dict = state_dict) + + # gets log-likelihoods including tau,w + s=surveys[0] + g=grids[0] + # this assumes we have both 1D and 2D components + ll1 = it.calc_likelihoods_1D(g,s,Pn=True,pNreps=True,ptauw=True,dolist=0) + ll2 = it.calc_likelihoods_2D(g,s,Pn=True,pNreps=True,psnr=True,ptauw=True,dolist=0) + print("Calculated log-likelihoods including w,scat are ",ll1,ll2) + + + ################ Generates some plots ################ + + # extracts the p(W) distribution + Nw,Nwz,Nwdm = g.get_pw_dist() + + # gets the actual values of width + ws = s.wlist + + if s.wplist.ndim > 1: + # plist is z-dependent + # get expected distribution at z=0 + wplist = s.wplist[:,0] + else: + wplist = s.wplist + + # generates some plots!!! + # these need to be normalised by the internal bin width + logbinwidth = s.internal_logwvals[-1] - s.internal_logwvals[-2] + + # values at z=0 + WidthArgs = (s.wlogmean,s.wlogsigma) + ScatArgs = (s.slogmean,s.slogsigma) + pw = s.WidthFunction(s.internal_logwvals, *WidthArgs)*s.dlogw #/logbinwidth + ptau = s.ScatFunction(s.internal_logwvals, *ScatArgs)*s.dlogw #/logbinwidth + + # these two arrays hold p(tau) and p(iw) values with dimensions: + # z, internal tau values, iwidth + # the normalisation is such that the sum over internal widths is unity + # that is, for a given tau, what is p(w). NOT for a given w, what is ptau! + # this is all a function of z + + Rtau = np.zeros([s.internal_logwvals.size]) + Rw = np.zeros([s.internal_logwvals.size]) + # calculates ptauw + for i,t in enumerate(s.internal_logwvals): + ptz = s.ptaus[:,i,:] # this is p(tau) given z and w + Rtz = ptz * Nwz.T + Rtau[i] = np.sum(Rtz) + + pwz = s.pws[:,i,:] # this is p(tau) given z and w + Rwz = pwz * Nwz.T + Rw[i] = np.sum(Rwz) + + + norm=True + if norm: + # We require that the following functions obey \int function dlogw = 1 + # hence, we need to divide by total sum, then divide by bin + # width in units of dlogw + + # n1,n2 are probability "per bin". hence, to convert to a p(w) dlogw, we diving by the bin width in logw + n1 = np.sum(wplist) * s.dlogw # n1 is probability within the bin. n3-6 are probability dlogp + n2 = np.sum(Nw) * s.dlogw + + n3 = np.sum(ptau) * logbinwidth + n4 = np.sum(Rtau) * logbinwidth + n5 = np.sum(pw) * logbinwidth + n6 = np.sum(Rw) * logbinwidth + else: + n1=1. + n2=1. + n3=1. + n4=1. + n5=1. + n6=1. + + + #### Plot 1: Intrinsic vs detected distributions ##### + plt.figure() + plt.xscale("log") + plt.yscale("log") + plt.ylim(1e-2,1) + plt.xlim(1e-2,1e2) + + plt.plot(ws,wplist/n1,label="Total width (z=0)",linestyle="-") + plt.plot(ws,Nw/n2,label="Detected total",linestyle=":",color=plt.gca().lines[-1].get_color()) + + plt.plot(10**s.internal_logwvals,ptau/n3,label="Scattering (z=0)",linestyle="-") + plt.plot(10**s.internal_logwvals,Rtau/n4,label="Detected scattering",linestyle=":",color=plt.gca().lines[-1].get_color()) + + plt.plot(10**s.internal_logwvals,pw/n5,label="Intrinsic widths (z=0)",linestyle="-") + plt.plot(10**s.internal_logwvals,Rw/n6,label="Detected width",linestyle=":",color=plt.gca().lines[-1].get_color()) + + plt.xlabel("width [ms]") + plt.ylabel("$\\rm p(w) d\\log_{10} w$") + plt.legend() + plt.tight_layout() + plt.savefig(opdir+"pw.png") + plt.close() + + + #### Plot 2: Redsift dependence ##### + + plt.figure() + plt.xscale("log") + plt.yscale("log") + plt.ylim(1e-3,None) + plt.plot(ws,wplist/np.sum(wplist),label="Intrinsic width",color="black") + plt.plot(ws,Nw/np.sum(Nw),label="Detected FRBs",linestyle="--") + plt.plot(ws,Nwz[:,4]/np.sum(Nwz[:,4]),label=" (z=0.25)",linestyle="--") + plt.plot(ws,Nwz[:,18]/np.sum(Nwz[:,18]),label=" (z=1.25)",linestyle=":") + plt.xlabel("FRB effective width [ms]") + plt.ylabel("FRBs/day") + plt.legend() + plt.tight_layout() + plt.savefig(opdir+"pw_zdep.png") + plt.close() + + #### Plot 2: DM dependence ##### + + plt.figure() + plt.xscale("log") + plt.yscale("log") + plt.ylim(1e-3,None) + plt.plot(ws,wplist/np.sum(wplist),label="Intrinsic width",color="black") + plt.plot(ws,Nw/np.sum(Nw),label="Detected FRBs",linestyle="--") + plt.plot(ws,Nwdm[:,3]/np.sum(Nwdm[:,3]),label=" (DM=125)",linestyle="--") + plt.plot(ws,Nwdm[:,21]/np.sum(Nwdm[:,21]),label=" (DM=1025)",linestyle=":") + plt.xlabel("FRB effective width [ms]") + plt.ylabel("FRBs/day") + plt.legend() + plt.tight_layout() + plt.savefig(opdir+"pw_dmdep.png") + plt.close() + +main() diff --git a/zdm/survey.py b/zdm/survey.py index fb20e37..461a618 100644 --- a/zdm/survey.py +++ b/zdm/survey.py @@ -82,31 +82,34 @@ def __init__(self, state, survey_name:str, self.init_DMEG(state.MW.DMhalo, state.MW.halo_method) # Zs self.init_zs() # This should be redone every time DMhalo is changed IF we use a flat cutoff on DMEG + # Allows survey metadata to over-ride parameter defaults if present. # This is required when mixing CHIME and non-CHIME FRBs beam_method = self.meta['BMETHOD'] beam_thresh = self.meta['BTHRESH'] - #width_bias = self.meta['WBIAS'] self.init_beam( method=beam_method, plot=False, thresh=beam_thresh) # tells the survey to use the beam file - # Efficiency: width_method passed through "self" here - # Determines if the model is redshift dependent - - ##### need to fix this up by re-organising higher-level routines!!! ##### - + # initialise scattering/width and resulting efficiences self.init_widths() self.calc_max_dm() - def init_widths(self): + def init_widths(self,state=None): """ Performs initialisation of width and scattering distributions + Args: + state (parameters.state object., optional): if set, assume + this state contains new scattering/width parameters + """ + if state is not None: + self.state = state + # copies over Width bin information self.NWbins = self.state.width.WNbins self.WMin = self.state.width.WMin @@ -199,17 +202,30 @@ def init_widths(self): # we have a z-dependent scattering and width model for iz,z in enumerate(self.zvals): self.make_widths(iz) - _ = self.get_efficiency_from_wlist(self.wlist,self.wplist[:,iz], - model=self.meta['WBIAS'], edir=self.edir, iz=iz) + #_ = self.get_efficiency_from_wlist(self.wlist,self.wplist[:,iz], + # model=self.meta['WBIAS'], edir=self.edir, iz=iz) else: self.wplist = np.zeros([self.NWbins]) self.make_widths() if self.backproject: self.pws = np.zeros([self.internal_logwvals.size,self.NWbins]) #[iz,:,:] = pw self.ptaus = np.zeros([self.internal_logwvals.size,self.NWbins]) #[iz,:,:] = ptau + #_ = self.get_efficiency_from_wlist(self.wlist,self.wplist, + # model=self.meta['WBIAS'], edir=self.edir, iz=None) + self.do_efficiencies() + + def do_efficiencies(self): + """ + Function to handle calculation of efficiencies. + Allows this to be called externally, e.g. if DM halo model changes + """ + if self.meta['WMETHOD'] == 3: + for iz,z in enumerate(self.zvals): + _ = self.get_efficiency_from_wlist(self.wlist,self.wplist[:,iz], + model=self.meta['WBIAS'], edir=self.edir, iz=iz) + else: _ = self.get_efficiency_from_wlist(self.wlist,self.wplist, model=self.meta['WBIAS'], edir=self.edir, iz=None) - def make_widths(self,iz=None): """ Used to be an exterior method, now interior @@ -383,13 +399,13 @@ def get_internal_coeffs(self,wlist): dkws1 (float): coefficient for iws1 dkws2 (float): coefficient for iws2 """ + # convert to log-widths - the bins are in log10 space logwlist = np.log10(wlist) dinternal = self.internal_logwvals[1]-self.internal_logwvals[0] kws=(logwlist-self.internal_logwvals[0])/dinternal Bin0 = np.where(kws < 0.)[0] kws[Bin0] = 0. - iws1=kws.astype('int') iws2=iws1+1 dkws2=kws-iws1 # applies to izs2 @@ -406,6 +422,8 @@ def get_internal_coeffs(self,wlist): def get_w_coeffs(self,wlist): """ Returns indices and coefficients for linear interpolation between width values + Bin edges run from [small~1e-10, self.WMin, self.WMin + self.dlowg, ..., self.Wmax] + We should use bin centres to determine which linear interpolations we sit between wlist: np.ndarray of observed widths @@ -418,7 +436,7 @@ def get_w_coeffs(self,wlist): # convert to log-widths - the bins are in log10 space logwlist = np.log10(wlist) - kws=(logwlist-np.log10(self.WMin))/self.dlogw +1. + kws=(logwlist-np.log10(self.WMin))/self.dlogw + 0.5 Bin0 = np.where(kws < 0.)[0] kws[Bin0] = 0. @@ -692,19 +710,15 @@ def process_survey_file(self,filename:str, DC = self.survey_data.params[key] self.meta[key] = getattr(self.survey_data[DC],key) - # over-rides survey data if applicable - if survey_dict is not None: - for key in survey_dict: - self.meta[key] = survey_dict[key] - - # Get default values from default frb data default_frb = survey_data.FRB() # we now populate missing fields with the default values for field in fields(default_frb): # checks to see if this is a field in metadata: if so, takes priority - if field.name in self.meta.keys(): + if survey_dict is not None and field.name in survey_dict.keys(): + default_vaue = survey_dict[field.name] + elif field.name in self.meta.keys(): default_value = self.meta[field.name] else: default_value = getattr(default_frb, field.name) @@ -716,7 +730,7 @@ def process_survey_file(self,filename:str, if isinstance(val,np.ma.core.MaskedArray): frb_tbl[field.name][i] = default_value else: - default_value = getattr(default_frb, field.name) + #default_value = getattr(default_frb, field.name) frb_tbl[field.name] = default_value print("WARNING: no ",field.name," found in survey", "replcing with default value of ",default_value) @@ -789,6 +803,11 @@ def process_survey_file(self,filename:str, self.meta['WIDTH'] = np.median(self.frbs['WIDTH']) self.meta['DMG'] = np.mean(self.frbs['DMG']) + # over-rides survey data if applicable + if survey_dict is not None: + for key in survey_dict: + self.meta[key] = survey_dict[key] + ### processes galactic contributions self.process_dmg() @@ -809,13 +828,20 @@ def process_survey_file(self,filename:str, # calculates intrinsic widths # likely needs to modify this because TAU is is the 1/e timescale, - # w is the 90% width timescale. Hence, should subtract 2.3 times - # Perhaps include a code within each survey to govern this? - TEMP = self.frbs['WIDTH'].values**2 - self.frbs['TAU'].values**2 + # For a Gaussian burst, SNR max at 1.4 sigma, w95 at 1.9 sigma + # For exponential, SNR max at 1.3 tau, w95 at 3 tau + # Hence, should multiply tau by 3/1.3, calculate new w95, then divide by 1.9/1.4 to get SNR max width + # Note that none of this is satisfactory - it is dependent on pulse shape and DM, neither + # of which we can account for here + tscale = 3./1.3 + wscale = 1.4/1.9 + TEMP = self.frbs['WIDTH'].values**2 - (tscale*self.frbs['TAU'].values)**2 self.OKTAU = np.where(self.frbs['TAU'].values != -1.)[0] # code for non-existent toolow = np.where(TEMP <= 0.) - TEMP[toolow] = 0.01**2 # 10 microsecond width - self.IWIDTHs = TEMP**0.5 + TEMP[toolow] = 0.01**2 # 10 microsecond width. Really could be anything up to tau + iwidths = wscale * TEMP**0.5 # scale to SNR max width assuming Gaussian shape + self.IWIDTHs = iwidths + self.WIDTHs *= wscale # scales to SNR maximising width assuming Gaussian shape self.TAUs = self.frbs['TAU'].values # sets the 'beam' values to unity by default @@ -978,7 +1004,7 @@ def get_efficiency_from_wlist(self,wlist,plist, addGalacticDM: - True: this routine adds in contributions from the MW Halo and ISM, i.e. it acts like DMlist is an extragalactic DM - - False: just used the supplied DMlist + - False: just use the supplied DMlist edir: - Directory where efficiency files are contained. Only relevant if specific FRB responses are used @@ -1286,14 +1312,14 @@ def quadrature_convolution(width_function, width_args, scat_function, scat_args, pw = width_function(internal_logvals, *width_args)*logbinwidth ptau = scat_function(internal_logvals, *scat_args)*logbinwidth - # adds extra bits onto the lowest bin. Assumes exp(-20) is small enough! + # adds extra bits onto the lowest bin. Assumes exp(-10) is small enough! lowest = internal_logvals[0] - logbinwidth/2. extrapw,err = quad(width_function,lowest-10,lowest,args=width_args) extraptau,err = quad(scat_function,lowest-10,lowest,args=scat_args) pw[0] += extrapw ptau[0] += extraptau - linvals = np.exp(internal_logvals) + linvals = 10**internal_logvals # calculate widths - all done in linear domain Nbins = bins.size-1 @@ -1314,6 +1340,8 @@ def quadrature_convolution(width_function, width_args, scat_function, scat_args, # maps the probabilities as a function of intrinsic # width to generate a p(w|total_width) and p(tau|total_width) # note that these are p(observed) values, i.e. after z-correction + # arrays have dimensions(Nw,Ntotal_width) so for each total + # width, we get the probability for i,x1 in enumerate(linvals): totalwidths = (x1**2 + linvals**2)**0.5 @@ -1325,9 +1353,43 @@ def quadrature_convolution(width_function, width_args, scat_function, scat_args, h,b = np.histogram(totalwidths,bins=bins,weights=probs) taufracs[i,:] = h + # we need p(tau|w). This means sum for a given w must be 1! + wnorm = np.sum(wfracs,axis=0) + # where is p(tau=0 for a given w?) + bad = np.where(wnorm == 0.)[0] + wfracs[:,bad] = 1./internal_logvals.size # if a particular width value has no iw, equalise probability over all iw + wnorm[bad] = 1. + + tnorm = np.sum(taufracs,axis=0) + bad = np.where(tnorm == 0.)[0] + taufracs[:,bad] = 1./internal_logvals.size # if a particular width value has no possible tau, equalise probability over all tau + tnorm[bad] = 1. + # normalise probabilities for each intrinsic w - wfracs = (wfracs.T/np.sum(wfracs,axis=1)).T - taufracs = (taufracs.T/np.sum(taufracs,axis=1)).T + #wfracs = (wfracs.T/wnorm).T + #taufracs = (taufracs.T/tnorm).T + wfracs = wfracs/wnorm + taufracs = taufracs/tnorm + + # plot some examples. This code is kept here for internal analysis purposes + if False: + plt.figure() + plt.plot(internal_logvals,ptau,label="Intrinsic p(tau)") + plt.plot(internal_logvals,pw,label="Intrinsic p(w)") + for ib in np.arange(Nbins): + plt.plot(internal_logvals,taufracs[:,ib],label="width "+str(ib)) + plt.plot(np.log10([bins[ib],bins[ib]]),[0,1],linestyle=":",color=plt.gca().lines[-1].get_color()) + plt.plot(np.log10([bins[ib+1],bins[ib+1]]),[0,1],linestyle=":",color=plt.gca().lines[-1].get_color()) + plt.yscale("log") + #plt.xscale("log") + plt.xlabel("log10 Tau [ms]") + plt.ylabel("p(tau |w)") + plt.legend() + plt.tight_layout() + plt.show() + plt.close() + # exit now, to prevent very many such plots being generated + exit() return hist,wfracs,taufracs else: return hist @@ -1336,7 +1398,9 @@ def quadrature_convolution(width_function, width_args, scat_function, scat_args, def lognormal(log10w, *args): """ - Lognormal probability distribution + Lognormal probability distribution. Note that the x values and args + could theoretically be in natural log (or other) space, + including linear. Args: log10w: log base 10 of widths @@ -1367,12 +1431,17 @@ def halflognormal(log10w, *args):#logmean,logsigma,minw,maxw,nbins): logmean = args[0] logsigma = args[1] norm = (2.*np.pi)**-0.5/logsigma - - large = np.where(log10w > logmean) - - modlogw = log10w - modlogw[large] = logmean # subs mean value in for values larger than the mean - result = lognormal(modlogw,args) + if hasattr(log10w,"__len__"): + large = np.where(log10w > logmean)[0] + + modlogw = np.copy(log10w) # ensures we don't change the original values + modlogw[large] = logmean # subs mean value in for values larger than the mean + else: + if log10w > logmean: + modlogw = logmean + else: + modlogw = log10w + result = lognormal(modlogw,logmean,logsigma) return result def constant(log10w,*args): diff --git a/zdm/survey_data.py b/zdm/survey_data.py index 7849322..17a95a7 100644 --- a/zdm/survey_data.py +++ b/zdm/survey_data.py @@ -146,8 +146,8 @@ class Telescope(data_class.myDataClass): 'Notation': '', }) NBINS: int = field( - default=0, - metadata={'help': "Number of bins for width analysis", + default=5, + metadata={'help': "Number of bins for beam analysis", 'unit': '', 'Notation': '', }) From c6a3734a3d223cff177f33387dbd15aff115cc23 Mon Sep 17 00:00:00 2001 From: Clancy James Date: Fri, 20 Jun 2025 09:02:13 +0800 Subject: [PATCH 09/20] Debugged SKA, updated C RAFT 892 MHz file, fixed likelihood for tau and w --- papers/SKA_science/make_zdists.py | 12 +- papers/SKA_science/plot_zdm_dists.py | 111 ++++++++ .../Scattering/MCMC_inputs/scat_w_only.json | 3 +- papers/Scattering/nonparametric.py | 161 +++++++++++ papers/Scattering/plot_scat_dist.py | 172 ++++++++++++ papers/Scattering/run_MCMC.sh | 21 +- papers/Scattering/visualise_mcmc.py | 19 +- papers/Scattering/width_scat_dists.py | 265 ++++++++++++++++++ zdm/data/Surveys/CRAFT_ICS_1300.ecsv | 41 +-- zdm/data/Surveys/CRAFT_ICS_1632.ecsv | 9 +- zdm/data/Surveys/CRAFT_ICS_892.ecsv | 40 +-- zdm/figures.py | 26 ++ zdm/iteration.py | 18 +- zdm/misc_functions.py | 47 ++++ zdm/scripts/Scattering/width_scat_dists.py | 50 +--- zdm/survey.py | 54 ++-- zdm/survey_data.py | 4 +- 17 files changed, 919 insertions(+), 134 deletions(-) create mode 100644 papers/SKA_science/plot_zdm_dists.py create mode 100644 papers/Scattering/nonparametric.py create mode 100644 papers/Scattering/plot_scat_dist.py create mode 100644 papers/Scattering/width_scat_dists.py diff --git a/papers/SKA_science/make_zdists.py b/papers/SKA_science/make_zdists.py index 79eac2a..23a4089 100644 --- a/papers/SKA_science/make_zdists.py +++ b/papers/SKA_science/make_zdists.py @@ -137,9 +137,9 @@ def generate_sensitivity_plot(infile,state,zDMgrid, zvals, dmvals, label, freq, np.save("sysplotdir/zvals.npy",zvals) np.save("sysplotdir/dmvals.npy",dmvals) - exit() + #load=True - load=True + load=False verbose=True for i in range(samples.shape[0]): @@ -161,15 +161,16 @@ def generate_sensitivity_plot(infile,state,zDMgrid, zvals, dmvals, label, freq, mask = pcosmic.get_dm_mask(dmvals, (state.host.lmean, state.host.lsigma), zvals, plot=False) # normalise number of FRBs to the CRAFT Fly's Eye survey - s = survey.load_survey(survey_name, state, dmvals, zvals=zvals) + s = survey.load_survey("CRAFT_class_I_and_II", state, dmvals, zvals=zvals) g = zdm_grid.Grid(s, copy.deepcopy(state), zDMgrid, zvals, dmvals, mask, wdist=True) # we expect np.sum(g.rates)*s.TOBS * C = s.NORM_FRB norm = s.NORM_FRB/(s.TOBS*np.sum(g.rates)) + print("norm is ",norm,s.NORM_FRB,s.TOBS) s = survey.load_survey(survey_name, state, dmvals, zvals=zvals, survey_dict=survey_dict) g = zdm_grid.Grid(s, copy.deepcopy(state), zDMgrid, zvals, dmvals, mask, wdist=True) - scale = TOBS[i] * norm + scale = TOBS[ibest] * norm if verbose: print("Finished iteration ",i," norm is ",norm) @@ -231,10 +232,11 @@ def get_samples(infile,nsets): # Read in the MCMC results without the burnin #infile = os.path.join(args.directory, args.infile) reader = emcee.backends.HDFBackend(infile) - sample = reader.get_chain(discard=50, flat=True) + sample = reader.get_chain(discard=500, flat=True) # Thin the results step = len(sample) // nsets + sample = sample[::step,:] # Get the corresponding parameters diff --git a/papers/SKA_science/plot_zdm_dists.py b/papers/SKA_science/plot_zdm_dists.py new file mode 100644 index 0000000..70f2556 --- /dev/null +++ b/papers/SKA_science/plot_zdm_dists.py @@ -0,0 +1,111 @@ +""" +This script creates plots of p(z) and p(dm) for different SKA configs + +""" +import numpy as np +from matplotlib import pyplot as plt + +def main(): + """ + Plots outputs of simulations + + """ + + + ####### Loop over input files ######### + # these set the frequencies in MHz and bandwidths in MHz + names = ["SKA_mid","SKA_mid","SKA_low"] + + for i,tel in enumerate(["Band1", "Band2", "Low"]): + # sets frequency and bandwidth for each instrument + for telconfig in ["AA4","AAstar"]: + label = tel+"_"+telconfig + make_plots(label) + + +def make_plots(label): + """ + + Args: + label (string): string label identifying the band and config + of the SKA data to load, and tag to apply to the + output files + + """ + + # load redshift and dm values + zvals = np.load("zvals.npy") + dmvals = np.load("dmvals.npy") + + # load survey-specific outputs + Ns = np.load(label+"_sys_N.npy") + meanN = np.sum(Ns)/Ns.size + print("Mean annual evenbt rate for ",label," is ",meanN) + + pzs = np.load(label+"_sys_pz.npy") + pdms = np.load(label+"_sys_pdm.npy") + + make_pz_plots(zvals,pzs,label) + make_pdm_plots(dmvals,pdms,label) + + +def make_pz_plots(zvals,pzs,label): + """ + Make plots of p(z) for each systematic simulation + """ + + Nparams,NZ = pzs.shape + + # this scales from the "per z bin" to "per z", + # i.e. to make the units N per year per dz + scale = 1./(zvals[1]-zvals[0]) + + mean = np.sum(pzs,axis=0)/Nparams + + # make un-normalised plots + plt.figure() + plt.xlabel("z") + plt.ylabel("N(z) per year") + plt.xlim(0,5) + + for i in np.arange(Nparams): + plt.plot(zvals,pzs[i,:],color="grey",linestyle="-") + + plt.plot(zvals,mean,color="black",linestyle="-",linewidth=2,label="Simulation mean") + + plt.legend() + plt.tight_layout() + plt.savefig(label+"_pz.png") + plt.close() + + +def make_pdm_plots(dmvals,pdms,label): + """ + Make plots of p(DM) for each systematic simulation + """ + + Nparams,NDM = pdms.shape + + # this scales from the "per z bin" to "per z", + # i.e. to make the units N per year per dz + scale = 1./(dmvals[1]-dmvals[0]) + + mean = np.sum(pdms,axis=0)/Nparams + + # make un-normalised plots + plt.figure() + plt.xlabel("DM [pc cm$^{-3}$]") + plt.ylabel("N(DM) per year") + plt.xlim(0,5000) + + for i in np.arange(Nparams): + plt.plot(dmvals,pdms[i,:],color="grey",linestyle="-") + + plt.plot(dmvals,mean,color="black",linestyle="-",linewidth=2,label="Simulation mean") + + plt.legend() + plt.tight_layout() + plt.savefig(label+"_pdm.png") + plt.close() + +main() diff --git a/papers/Scattering/MCMC_inputs/scat_w_only.json b/papers/Scattering/MCMC_inputs/scat_w_only.json index c61d78b..0697b2c 100644 --- a/papers/Scattering/MCMC_inputs/scat_w_only.json +++ b/papers/Scattering/MCMC_inputs/scat_w_only.json @@ -15,7 +15,8 @@ "alpha": 1.5, "WNbins": 33, "WNInternalBins": 1000, - "ScatFunction": 1 + "ScatFunction": 1, + "Sfnorm": 1000 }, "Wlogmean": { "DC": "width", diff --git a/papers/Scattering/nonparametric.py b/papers/Scattering/nonparametric.py new file mode 100644 index 0000000..f0d2824 --- /dev/null +++ b/papers/Scattering/nonparametric.py @@ -0,0 +1,161 @@ +import numpy as np +from matplotlib import pyplot as plt +import scipy as sp +import matplotlib + +import sys +import os + + +import pandas + +defaultsize=16 +ds=4 +font = {'family' : 'Helvetica', + 'weight' : 'normal', + 'size' : defaultsize} +matplotlib.rc('font', **font) + +def main(): + ##### does scatz plot ##### + #infile="table_scatz.dat" + #scat,scatmid,err,errl,errh,z,DM,DMG,SNR,Class = read_data(infile) + #scat,scaterr,z,DM,DMG,SNR,Class = read_data(infile) + + tresinfo = np.loadtxt("treslist.dat",dtype="str") + names=tresinfo[:,0] + + for i in np.arange(names.size): + names[i] = names[i][0:8] # truncates the letter + + dataframe = pandas.read_csv("CRAFT_ICS_HTR_Catalogue1.csv") + for key in dataframe.keys(): + print(key) + + ERR=9999. + tns = dataframe.TNS + tauobs = dataframe.TauObs + w95 = dataframe.W95 + wsnr = dataframe.Wsnr + z = dataframe.Z + snr = dataframe.SNdet + freq = dataframe.NUTau + DM = dataframe.DM + OK = getOK([tns,tauobs,w95,wsnr,z,snr,freq,DM]) + tns = tns[OK] + tauobs= tauobs[OK] + w95 = w95[OK] + wsnr = wsnr[OK] + z = z[OK] + snr = snr[OK] + freq = freq[OK] + DM = DM[OK] + NFRB = len(OK) + tres = np.zeros([NFRB]) + + for i,name in enumerate(tns): + j = np.where(name[0:8] == names)[0] + if len(j) != 1: + print("Cannot find tres info for FRB ",name) + tres[i] = float(tresinfo[j,1]) + + print(tres) + +def find_max_tau(wsnr,tauobs,DM,freq,snr,tres,nu_res=1.): + """ + + """ + + tauvals = np.linspace(0.1,100.,1001) + k_DM = 4.149 # leading constant: ms per pccc GHz^2 + dmsmear = 2*(nu_res/1.e3)*k_DM*DM_frb/(fbar/1e3)**3 + + + + totalw = (uw**2 + dm_smearing**2/3. + t_res**2/3.)**0.5 + + + print(OK) + exit() + + + +def getOK(arrays,ERR=9999.): + """ + Find indices where all arrays have good values + """ + OK = np.where(arrays[0] != ERR) + for array in arrays[1:]: + OK1 = np.where(array != ERR) + OK = np.intersect1d(OK,OK1) + return OK + +def make_cum_dist(vals): + """ + Makes a cumulative distributiuon for plotting purposes + """ + + # orders the vals smallest to largest + ovals = np.sort(vals) + + Nvals = ovals.size + Npts = 2*Nvals+2 + xs=np.zeros([Npts]) + ys=np.zeros([Npts]) + # begins at 0,0 + + for i in np.arange(Nvals): + xs[2*i+1] = ovals[i] + xs[2*i+2] = ovals[i] + ys[2*i+1] = i / Nvals + ys[2*i+2] = (i+1) / Nvals + + if np.max(xs) >1: + themax = np.max(xs) + else: + themax = 1. + xs[-1] = themax + ys[-1] = 1 + return xs,ys + +def read_chime_scat(infile = "chime_scat.dat"): + """ + gets chime scat err data + """ + scats=[] + errs=[] + with open(infile) as file: + for line in file: + fields = line.split() + scatstring = fields[1].replace(" ", "") + if scatstring[0] == "<": + scat = float(scatstring[1:])/2. + err = scat + elif scatstring[0] == "~": + scat = float(scatstring[1:]) + err = float(fields[2].replace(" ", "")) + else: + scat = float(scatstring) + err = float(fields[2].replace(" ", "")) + scats.append(scat) + errs.append(err) + scats = np.array(scats) + errs = np.array(errs) + return scats,errs + +def cutz(DM,DMG,z,scat,err): + """ + cuts on z>0 + """ + + + OK = np.where(z>0.)[0] + DM = DM[OK] + DMG = DMG[OK] + z = z[OK] + scat = scat[OK] + err = err[OK] + return DM,DMG,z,scat,err + + +main() diff --git a/papers/Scattering/plot_scat_dist.py b/papers/Scattering/plot_scat_dist.py new file mode 100644 index 0000000..c50b535 --- /dev/null +++ b/papers/Scattering/plot_scat_dist.py @@ -0,0 +1,172 @@ +import numpy as np +from matplotlib import pyplot as plt +import scipy as sp +import matplotlib + +import sys +import os + + +import pandas + +defaultsize=16 +ds=4 +font = {'family' : 'Helvetica', + 'weight' : 'normal', + 'size' : defaultsize} +matplotlib.rc('font', **font) + +def main(): + ##### does scatz plot ##### + #infile="table_scatz.dat" + #scat,scatmid,err,errl,errh,z,DM,DMG,SNR,Class = read_data(infile) + #scat,scaterr,z,DM,DMG,SNR,Class = read_data(infile) + + dataframe = pandas.read_csv("CRAFT_ICS_HTR_Catalogue1.csv") + tauobs = dataframe.TauObs + w95 = dataframe.W95 + wsnr = dataframe.wsnr + + FRB2,taus,tauerrs,ascat,aerr,alphas,alphaerrs = read_scat_table() + + #cscat,cerr = read_chime_scat() + + clx,cly = make_cum_dist(cscat-cerr) + cmx,cmy = make_cum_dist(cscat) + cux,cuy = make_cum_dist(cscat+cerr) + + alx,aly = make_cum_dist(ascat-aerr) + amx,amy = make_cum_dist(ascat) + aux,auy = make_cum_dist(ascat+aerr) + + a2lx,a2ly = make_cum_dist(taus-tauerrs) + a2mx,a2my = make_cum_dist(taus) + a2ux,a2uy = make_cum_dist(taus+tauerrs) + + exponent = 0 + plt.figure() + plt.plot(cmx*(0.6**exponent),cmy,label="CHIME 600 MHz", linewidth=2) + plt.plot(clx*(0.6**exponent),cly,linestyle="-.",color=plt.gca().lines[-1].get_color()) + plt.plot(cux*(0.6**exponent),cuy,linestyle="-.",color=plt.gca().lines[-1].get_color()) + + plt.plot(amx,amy,label="ASKAP 1 GHz",linestyle="--", linewidth=2) + plt.plot(alx,aly,linestyle=":",color=plt.gca().lines[-1].get_color()) + plt.plot(aux,auy,linestyle=":",color=plt.gca().lines[-1].get_color()) + plt.xscale('log') + #plt.xlabel("$\\tau_{\\rm 1 GHz}$ [ms]") + plt.xlabel("$\\tau$ [ms]") + plt.ylabel("Cumulative distribution") + plt.ylim(0,1) + plt.legend() + plt.tight_layout() + plt.savefig("scattering_comparison_exp"+str(exponent)+".png") + plt.close() + + exponent = 4 + plt.figure() + plt.plot(cmx*(0.6**exponent),cmy,label="CHIME 600 MHz", linewidth=2) + plt.plot(clx*(0.6**exponent),cly,linestyle="-.",color=plt.gca().lines[-1].get_color()) + plt.plot(cux*(0.6**exponent),cuy,linestyle="-.",color=plt.gca().lines[-1].get_color()) + + plt.plot(amx,amy,label="ASKAP 1 GHz",linestyle="--", linewidth=2) + plt.plot(alx,aly,linestyle=":",color=plt.gca().lines[-1].get_color()) + plt.plot(aux,auy,linestyle=":",color=plt.gca().lines[-1].get_color()) + plt.xscale('log') + #plt.xlabel("$\\tau_{\\rm 1 GHz}$ [ms]") + plt.xlabel("$\\tau$ [ms]") + plt.ylabel("Cumulative distribution") + plt.ylim(0,1) + plt.legend() + plt.tight_layout() + plt.savefig("scattering_comparison_exp"+str(exponent)+".png") + plt.close() + + + plt.figure() + plt.plot(cmx*(0.6**exponent),cmy,label="CHIME 600 MHz") + plt.plot(clx*(0.6**exponent),cly,linestyle=":",color=plt.gca().lines[-1].get_color()) + plt.plot(cux*(0.6**exponent),cuy,linestyle=":",color=plt.gca().lines[-1].get_color()) + + plt.plot(a2mx,a2my,label="ASKAP unscaled",linestyle="--") + plt.plot(a2lx,a2ly,linestyle="-.",color=plt.gca().lines[-1].get_color()) + plt.plot(a2ux,a2uy,linestyle="-.",color=plt.gca().lines[-1].get_color()) + plt.xscale('log') + #plt.xlabel("$\\tau_{\\rm 1 GHz}$ [ms]") + plt.xlabel("$\\tau$ [ms]") + plt.ylabel("Cumulative distribution") + plt.legend() + plt.ylim(0,1) + plt.tight_layout() + plt.savefig("scattering_comparison_noscaling.png") + plt.close() + + +def make_cum_dist(vals): + """ + Makes a cumulative distributiuon for plotting purposes + """ + + # orders the vals smallest to largest + ovals = np.sort(vals) + + Nvals = ovals.size + Npts = 2*Nvals+2 + xs=np.zeros([Npts]) + ys=np.zeros([Npts]) + # begins at 0,0 + + for i in np.arange(Nvals): + xs[2*i+1] = ovals[i] + xs[2*i+2] = ovals[i] + ys[2*i+1] = i / Nvals + ys[2*i+2] = (i+1) / Nvals + + if np.max(xs) >1: + themax = np.max(xs) + else: + themax = 1. + xs[-1] = themax + ys[-1] = 1 + return xs,ys + +def read_chime_scat(infile = "chime_scat.dat"): + """ + gets chime scat err data + """ + scats=[] + errs=[] + with open(infile) as file: + for line in file: + fields = line.split() + scatstring = fields[1].replace(" ", "") + if scatstring[0] == "<": + scat = float(scatstring[1:])/2. + err = scat + elif scatstring[0] == "~": + scat = float(scatstring[1:]) + err = float(fields[2].replace(" ", "")) + else: + scat = float(scatstring) + err = float(fields[2].replace(" ", "")) + scats.append(scat) + errs.append(err) + scats = np.array(scats) + errs = np.array(errs) + return scats,errs + +def cutz(DM,DMG,z,scat,err): + """ + cuts on z>0 + """ + + + OK = np.where(z>0.)[0] + DM = DM[OK] + DMG = DMG[OK] + z = z[OK] + scat = scat[OK] + err = err[OK] + return DM,DMG,z,scat,err + + +main() diff --git a/papers/Scattering/run_MCMC.sh b/papers/Scattering/run_MCMC.sh index a494402..f3b0ecb 100755 --- a/papers/Scattering/run_MCMC.sh +++ b/papers/Scattering/run_MCMC.sh @@ -2,14 +2,16 @@ # script to run MCMC for CRAFT width parameters +####### TESTING ####### + #files="CRAFT_ICS_892 CRAFT_ICS_1300 CRAFT_ICS_1632" # use this for a halflognormal distribution #opfile="v3_mcmc_halflognormal" #pfile="MCMC_inputs/scat_w_only_halflog.json" # use this for a lognormal distribution -opfile="v6_mcmc_lognormal" # v4 is done with 300x300 nz ndm bins -pfile="MCMC_inputs/scat_w_only.json" +#opfile="v6_mcmc_lognormal" # v4 is done with 300x300 nz ndm bins +#pfile="MCMC_inputs/scat_w_only.json" # LOG # files="CRAFT_ICS_892 CRAFT_ICS_1300 CRAFT_ICS_1632" @@ -25,20 +27,25 @@ pfile="MCMC_inputs/scat_w_only.json" #opfile="v6_mcmc_lognormal" # for v7 - only one survey. Faster! Turns off the P(w) function -files="modCRAFT_ICS_1300" -opfile="MCMC_outputs/v7_mcmc_lognormal" +#files="modCRAFT_ICS_1300" +#opfile="MCMC_outputs/v7_mcmc_lognormal" # for v8 - turns off the Pscat | w function. (in p(2D) only) #files="modCRAFT_ICS_1300" # takes away 1D as well, only 2D #opfile="v8_mcmc_lognormal" # for v8 - has 1000 internal bins in width, and 33 evaluation bins -files="modCRAFT_ICS_1300" # takes away 1D as well, only 2D -opfile="MCMC_outputs/v9_mcmc_lognormal" +#files="modCRAFT_ICS_1300" # takes away 1D as well, only 2D +#opfile="MCMC_outputs/v9_mcmc_lognormal" + +####### ACTUAL RUN ####### +pfile="MCMC_inputs/scat_w_only.json" +files="CRAFT_ICS_892 CRAFT_ICS_1300 CRAFT_ICS_1632" +opfile="MCMC_outputs/mcmc_lognormal_v1" Pn=False ptauw=True -steps=1000 +steps=2000 walkers=14 Nz=100 diff --git a/papers/Scattering/visualise_mcmc.py b/papers/Scattering/visualise_mcmc.py index 3f164ca..fbf9f07 100644 --- a/papers/Scattering/visualise_mcmc.py +++ b/papers/Scattering/visualise_mcmc.py @@ -53,24 +53,9 @@ # this name gets added to all produced plots prefix="MCMC_Plots/halflognormal_" else: - filenames = ['MCMC_outputs/v2_mcmc_lognormal','MCMC_outputs/v3_mcmc_lognormal'] - # this name gets added to all produced plots - prefix="MCMC_Plots/lognormal_" # 100x100 zDM points - - filenames = ['MCMC_outputs/v4_mcmc_lognormal'] # 300 x 300 zdm points - prefix = "MCMC_Plots/v4lognormal_" - - filenames = ['MCMC_outputs/v5_mcmc_lognormal'] # 15 beam values - prefix = "MCMC_Plots/v5lognormal_" - - filenames = ['MCMC_outputs/v6_mcmc_lognormal'] # 15 beam values - prefix = "MCMC_Plots/v6lognormal_" - - filenames = ['MCMC_outputs/v7_mcmc_lognormal'] #turn off p(w) - prefix = "MCMC_Plots/v7lognormal_" - filenames = ['MCMC_outputs/v8_mcmc_lognormal'] # turn off p(scat|w) - prefix = "MCMC_Plots/v8lognormal_" + filenames = ['MCMC_outputs/mcmc_lognormal_v1'] # turn off p(scat|w) + prefix = "MCMC_Plots/lognormal_v1" samples = [] diff --git a/papers/Scattering/width_scat_dists.py b/papers/Scattering/width_scat_dists.py new file mode 100644 index 0000000..cc7cbec --- /dev/null +++ b/papers/Scattering/width_scat_dists.py @@ -0,0 +1,265 @@ +""" +This script plots observed and fitted width and scattering distributions + + +""" + +import os + +from zdm import cosmology as cos +from zdm import figures +from zdm import parameters +from zdm import survey +from zdm import pcosmic +from zdm import misc_functions as mf +from zdm import iteration as it +from zdm import loading +from zdm import io +from zdm import figures as fig +from pkg_resources import resource_filename +import numpy as np +from matplotlib import pyplot as plt + +import matplotlib + +defaultsize=14 +ds=4 +font = {'family' : 'Helvetica', + 'weight' : 'normal', + 'size' : defaultsize} +matplotlib.rc('font', **font) + + +def main(): + """ + + """ + + # in case you wish to switch to another output directory + opdir = "Plots/" + if not os.path.exists(opdir): + os.mkdir(opdir) + + # directory where the survey files are located. The below is the default - + # you can leave this out, or change it for a different survey file location. + sdir = os.path.join(resource_filename('zdm', 'data'), 'Surveys') + + # make this into a list to initialise multiple surveys art once + names = ["CRAFT_ICS_892","CRAFT_ICS_1300","CRAFT_ICS_1632"] # for example + + repeaters=False + # sets plotting limits + zmax = 2. + dmmax = 2000 + + survey_dict = {"WMETHOD": 3} + state_dict = {} + state_dict["scat"] = {} + state_dict["scat"]["Sbackproject"] = True # turns on backprojection of tau and width for our model + state_dict["width"] = {} + state_dict["width"]["WNInternalBins"] = 1000 # sets it to a small quantity + state_dict["width"]["WNbins"] = 33 # set to large number for this analysis + + surveys, grids = loading.surveys_and_grids(survey_names = names,\ + repeaters=repeaters, sdir=sdir,nz=70,ndm=140, + survey_dict = survey_dict, state_dict = state_dict) + + # we now generate plots for the FRBs with known tau and width values + # iwidths + ilist = [] + wlist = [] + tlist = [] + + # sets up a figure to do a scatter plot of frequency dependence of these widths + plt.figure() + plt.xlabel("Frequency [MHz]") + plt.ylabel("$\\log_{10} t$ [ms]") + for i,s in enumerate(surveys): + + nfrb = len(s.OKTAU) + + + taus = s.TAUs[s.OKTAU] + widths = s.frbs['WIDTH'].values[s.OKTAU] + iwidths = s.IWIDTHs[s.OKTAU] + + logtaus = np.log10(taus) + logwidths = np.log10(widths) + logis = np.log10(iwidths) + + taubar = np.average(logtaus) + taurms = (np.sum((logtaus - taubar)**2) / (nfrb-1))**0.5 + + + ibar = np.average(logis) + irms = (np.sum((logis - ibar)**2) / (nfrb-1))**0.5 + + + wbar = np.average(logwidths) + wrms = (np.sum((logwidths - wbar)**2) / (nfrb-1))**0.5 + + freqs = s.frbs['FBAR'][s.OKTAU] + fbar = np.average(freqs) + + if i==0: + l1,=plt.plot(freqs, logwidths,marker='s',label="$w_{\\rm app}$",linestyle="") + l2,=plt.plot(freqs, logtaus,marker="+",label="$\\tau$",linestyle="") + l3,=plt.plot(freqs, logis,marker="x",label="$w_i$",linestyle="") + else: + plt.plot(freqs, logwidths,marker='s',color=l1.get_color(),linestyle="") + plt.plot(freqs, logtaus,marker="+",color=l2.get_color(),linestyle="") + plt.plot(freqs, logis,marker="x",color=l3.get_color(),linestyle="") + + plt.errorbar([fbar-20],[wbar],yerr=[wrms],color=l1.get_color(),linewidth=2,capsize=5) + plt.errorbar([fbar],[taubar],yerr=[taurms],color=l2.get_color(),linewidth=2,capsize=6) + plt.errorbar([fbar+20],[ibar],yerr=[irms],color=l3.get_color(),linewidth=2,capsize=7) + + for index in s.OKTAU: + ilist.append(s.IWIDTHs[index]) + tlist.append(s.TAUs[index]) + wlist.append(s.frbs['WIDTH'].values[index]) + + + plt.legend() + plt.tight_layout() + # we do not save, because the figure is already done + #plt.savefig(opdir+"freq_scat.png") + plt.close() + + ####### We now plot against expectations ######## + # For each of w,i,and tau, we plot the true modelled distribution, + # the de-biased distribution, and the observed distribution + # We do this as both a cumulative distribution, and a pdf + # This is also done for all bursts summed together. + + # generate cumulative distributions of these quantities + xi,yi = fig.gen_cdf_hist(iwidths) + xw,yw = fig.gen_cdf_hist(widths) + xt,yt = fig.gen_cdf_hist(taus) + + + for i,s in enumerate(surveys): + g =grids[i] + ################ Generates some plots ################ + + ##### gets the expected, de-biased distributions ##### + # extracts the p(W) distribution + Rw,Rtau,Nw,Nwz,Nwdm = mf.get_width_stats(s,g) + + # normalise each survey to actual number of FRBs + nfrb = len(s.OKTAU) + + + # these need to be normalised by the internal bin width + logbinwidth = s.internal_logwvals[-1] - s.internal_logwvals[-2] + + # divide these histograms by the width of the log bins to get probability distributions + if i==0: + # intrinsic width, ptau, total width + SumRw = Rw / np.sum(Rw) * nfrb / logbinwidth #has Ninternalbins + SumRtau = Rtau / np.sum(Rtau) * nfrb / logbinwidth + + SumNw = Nw / np.sum(Nw) * nfrb / s.dlogw # has NWbins + else: + SumRw += Rw / np.sum(Rw) * nfrb + SumRtau += Rtau / np.sum(Rtau) * nfrb + SumNw += Nw / np.sum(Nw) * nfrb / s.dlogw + + ##### gets the underlying means ####### + # values at z=0 + WidthArgs = (s.wlogmean,s.wlogsigma) + ScatArgs = (s.slogmean,s.slogsigma) + + pw = s.WidthFunction(s.internal_logwvals, *WidthArgs)#*s.dlogw #/logbinwidth + ptau = s.ScatFunction(s.internal_logwvals, *ScatArgs)#*s.dlogw #/logbinwidth + + if i==0: + # intrinsic width, ptau, total width + Sumpw = pw / np.sum(pw) * nfrb + Sumptau = ptau / np.sum(ptau) * nfrb + else: + Sumpw += pw / np.sum(pw) * nfrb + Sumptau += ptau / np.sum(ptau) * nfrb + + # we now plot these differential distributions against histograms of the observations + + #### Plot 1: Intrinsic vs detected distributions ##### + plt.figure() + plt.xscale("log") + plt.yscale("log") + plt.ylim(1e-1,100) + plt.xlim(1e-2,1e2) + + l1,=plt.plot(s.wlist,SumNw,label="Total widths") + l2,=plt.plot(10**s.internal_logwvals,SumRtau,label="Scattering",linestyle="-") + l3,=plt.plot(10**s.internal_logwvals,SumRw,label="Intrinsic width",linestyle=":") + + bins=np.logspace(-2.01,2.01,9) + lbw = np.log10(bins[1]/bins[0]) + #normalisation: p per log bin * Nfrb. + weights = np.full([len(wlist)],1./lbw) + alpha=1.0 + plt.hist(wlist,bins=bins,weights=weights,alpha=alpha,facecolor = l1.get_color(),edgecolor = l1.get_color(),linewidth=2,histtype='step') + plt.hist(tlist,bins=bins,weights=weights,alpha=alpha,facecolor = l2.get_color(),edgecolor = l2.get_color(),linewidth=2,histtype='step') + plt.hist(ilist,bins=bins,weights=weights,alpha=alpha,facecolor = l3.get_color(),edgecolor = l3.get_color(),linewidth=2,histtype='step') + + + plt.legend() + plt.tight_layout() + plt.savefig(opdir+"differential.png") + + plt.close() + exit() + + plt.plot(ws,s.wplist/n1,label="Total width (z=0)",linestyle="-") + plt.plot(ws,Nw/n2,label="Detected total",linestyle=":",color=plt.gca().lines[-1].get_color()) + + plt.plot(10**s.internal_logwvals,ptau/n3,label="Scattering (z=0)",linestyle="-") + plt.plot(10**s.internal_logwvals,Rtau/n4,label="Detected scattering",linestyle=":",color=plt.gca().lines[-1].get_color()) + + plt.plot(10**s.internal_logwvals,pw/n5,label="Intrinsic widths (z=0)",linestyle="-") + plt.plot(10**s.internal_logwvals,Rw/n6,label="Detected width",linestyle=":",color=plt.gca().lines[-1].get_color()) + + plt.xlabel("width [ms]") + plt.ylabel("$\\rm p(w) d\\log_{10} w$") + plt.legend() + plt.tight_layout() + plt.savefig(opdir+"pw.png") + plt.close() + + + #### Plot 2: Redsift dependence ##### + + plt.figure() + plt.xscale("log") + plt.yscale("log") + plt.ylim(1e-3,None) + plt.plot(ws,s.wplist/np.sum(s.wplist),label="Intrinsic width",color="black") + plt.plot(ws,Nw/np.sum(Nw),label="Detected FRBs",linestyle="--") + plt.plot(ws,Nwz[:,4]/np.sum(Nwz[:,4]),label=" (z=0.25)",linestyle="--") + plt.plot(ws,Nwz[:,18]/np.sum(Nwz[:,18]),label=" (z=1.25)",linestyle=":") + plt.xlabel("FRB effective width [ms]") + plt.ylabel("FRBs/day") + plt.legend() + plt.tight_layout() + plt.savefig(opdir+"pw_zdep.png") + plt.close() + + #### Plot 3: DM dependence ##### + + plt.figure() + plt.xscale("log") + plt.yscale("log") + plt.ylim(1e-3,None) + plt.plot(ws,s.wplist/np.sum(s.wplist),label="Intrinsic width",color="black") + plt.plot(ws,Nw/np.sum(Nw),label="Detected FRBs",linestyle="--") + plt.plot(ws,Nwdm[:,3]/np.sum(Nwdm[:,3]),label=" (DM=125)",linestyle="--") + plt.plot(ws,Nwdm[:,21]/np.sum(Nwdm[:,21]),label=" (DM=1025)",linestyle=":") + plt.xlabel("FRB effective width [ms]") + plt.ylabel("FRBs/day") + plt.legend() + plt.tight_layout() + plt.savefig(opdir+"pw_dmdep.png") + plt.close() + +main() diff --git a/zdm/data/Surveys/CRAFT_ICS_1300.ecsv b/zdm/data/Surveys/CRAFT_ICS_1300.ecsv index 82a17d6..eb715d2 100644 --- a/zdm/data/Surveys/CRAFT_ICS_1300.ecsv +++ b/zdm/data/Surveys/CRAFT_ICS_1300.ecsv @@ -14,6 +14,7 @@ # - {name: THRESH, datatype: float64} # - {name: TRES, datatype: float64} # - {name: WIDTH, datatype: float64} +# - {name: XW95, datatype: float64} # - {name: TAU, datatype: float64} # - {name: DEC, datatype: string} # - {name: RA, datatype: string} @@ -22,23 +23,23 @@ # - {survey_data: '{"observing": {"NORM_FRB": 19, "TOBS": 165.506756761, "MAX_IDT": 4096, "MAX_IW": 12, "MAXWMETH": 1}, # "telescope": {"BEAM": "ASKAP_1300", "DIAM": 12.0, "NBEAMS": 36, "NBINS": 5, "WDATA": 1}}'} # schema: astropy-2.0 -TNS BW DM DMG FBAR FRES Gb Gl SNR SNRTHRESH THRESH TRES WIDTH TAU DEC RA Z -20180924B 336.0 362.4 40.5 1297.5 1.0 -74.40520121494983 277.20651893408893 21.1 9.0 4.4 0.864 2 0.59 -40:54:00.1 21:44:25.255 0.3214 -20181112A 336.0 589.0 40.2 1297.5 1.0 -63.30983709852826 290.87390392674445 19.3 9.0 4.4 0.864 0.8 0.023 -52:58:15.39 21:49:23.630 0.4755 -20190102C 336.0 364.5 57.3 1271.5 1.0 -37.519392943553534 300.95052401722796 14.0 9.0 4.4 0.864 1.25 0.027 -79:28:32.2845 21:29:39.70836 0.291 -20190608B 336.0 339.5 37.2 1271.5 1.0 -67.23724646562972 88.21721604792883 16.1 9.0 4.4 1.728 10.8 3.8 -07:53:53.6 22:16:04.7472 0.1178 -20190611B 336.0 322.2 57.6 1271.5 1.0 -37.59976893568559 300.9594501909617 9.3 9.0 4.4 1.728 1.59 0.03 -79:23:51.284 21:22:58.372 0.378 -20190711A 336.0 594.6 56.6 1271.5 1.0 -36.63629532677801 301.03976293370494 23.8 9.0 4.4 1.728 11.0 0.0076 -80:21:28.18 21:57:40.012 0.522 -20190714A 336.0 504.7 38.5 1271.5 1.0 -75.88144720209479 120.55805492153455 10.7 9.0 4.4 1.728 3.0 0.422 -13:01:14.36 12:15:55.081 0.2365 -20191228A 336.0 297.5 32.9 1271.5 1.0 -80.77822140033614 230.79855541687724 22.9 9.0 4.4 1.728 13.6 5.85 -29:35:37.85 22:57:43.269 0.243 -20210117A 336.0 730.0 34.4 1271.5 1.0 -75.7801432700954 164.65014968696988 27.1 9.0 4.4 1.182 3.6 0.25 -16:11:25.2 22:39:36.0 0.214 -20210214A 336.0 398.3 31.9 1271.5 1.0 -65.65372930742399 91.72782990931984 11.6 9.0 4.4 1.182 3.5 -1 -05:49:56 00:27:43 -1.0 -20210407E 336.0 1785.3 154.0 1271.5 1.0 -35.320866474063905 114.6146256941771 19.1 9.0 4.4 1.182 1.62 0.09 27:03:30.24 05:14:36.202 -1.0 -20210912A 336.0 1234.5 30.9 1271.5 1.0 -80.16655338584705 235.42165843951094 31.7 9.0 4.4 1.182 1.61 0.048 -30:29:33.1 23:24:40.3 -1.0 -20211127I 336.0 234.83 42.5 1271.5 1.0 -81.68554744714709 125.94306109583985 37.9 9.0 4.4 1.182 0.48 0.025 -18:49:28.4 13:19:09.5 0.046946 -20220531A 336.0 727.0 70.0 1271.5 1.0 -56.509199987039224 296.83938685003784 9.7 9.0 4.4 1.182 11.0 -1 -60:17:48.2 19:38:50.2 -1.0 -20220610A 336.0 1458.1 31.0 1271.5 1.0 -78.89122591551634 250.56280818953536 29.8 9.0 4.4 1.182 2.0 0.521 -33:30:49.87 23:24:17.559 1.016 -20220918A 336.0 656.8 40.7 1271.5 1.0 -45.81623556809721 308.4134807482381 26.4 9.0 4.4 1.182 13.9 7.66 -70:48:40.5 01:10:22.1 0.45 -20230526A 336.0 316.4 50.0 1271.5 1.0 -62.994316139408156 318.1591960546148 22.1 9.0 4.4 1.182 2.7 1.16 -52:46:07.7 01:29:27.5 0.157 -20230718A 336.0 477.0 396.0 1271.5 1.0 -75.66933767869608 316.30925962515585 10.9 9.0 4.4 1.182 0.7 0.117 -41:00:12.8 08:30:27.1 -1.0 -20230731A 336.0 701.1 547.0 1271.5 1.0 -60.14372530213189 304.262204790738 16.6 9.0 4.4 1.182 2.7 0.45 -56:58:19.1 11:38:40.1 -1.0 +TNS BW DM DMG FBAR FRES Gb Gl SNR SNRTHRESH THRESH TRES WIDTH XW95 TAU DEC RA Z +20180924B 336.0 362.4 40.5 1297.5 1.0 -74.40520121494983 277.20651893408893 21.1 9.0 4.4 0.864 0.91 2 0.59 -40:54:00.1 21:44:25.255 0.3214 +20181112A 336.0 589.0 40.2 1297.5 1.0 -63.30983709852826 290.87390392674445 19.3 9.0 4.4 0.864 0.1 0.8 0.023 -52:58:15.39 21:49:23.630 0.4755 +20190102C 336.0 364.5 57.3 1271.5 1.0 -37.519392943553534 300.95052401722796 14.0 9.0 4.4 0.864 0.076 1.25 0.027 -79:28:32.2845 21:29:39.70836 0.291 +20190608B 336.0 339.5 37.2 1271.5 1.0 -67.23724646562972 88.21721604792883 16.1 9.0 4.4 1.728 4.95 10.8 3.8 -07:53:53.6 22:16:04.7472 0.1178 +20190611B 336.0 322.2 57.6 1271.5 1.0 -37.59976893568559 300.9594501909617 9.3 9.0 4.4 1.728 0.076 1.59 0.03 -79:23:51.284 21:22:58.372 0.378 +20190711A 336.0 594.6 56.6 1271.5 1.0 -36.63629532677801 301.03976293370494 23.8 9.0 4.4 1.728 8.6 11.0 0.0076 -80:21:28.18 21:57:40.012 0.522 +20190714A 336.0 504.7 38.5 1271.5 1.0 -75.88144720209479 120.55805492153455 10.7 9.0 4.4 1.728 0.86 3.0 0.422 -13:01:14.36 12:15:55.081 0.2365 +20191228A 336.0 297.5 32.9 1271.5 1.0 -80.77822140033614 230.79855541687724 22.9 9.0 4.4 1.728 7.8 13.6 5.85 -29:35:37.85 22:57:43.269 0.243 +20210117A 336.0 730.0 34.4 1271.5 1.0 -75.7801432700954 164.65014968696988 27.1 9.0 4.4 1.182 1.24 3.58 0.25 -16:11:25.2 22:39:36.0 0.214 +20210214A 336.0 398.3 31.9 1271.5 1.0 -65.65372930742399 91.72782990931984 11.6 9.0 4.4 1.182 3.5 3.5 -1 -05:49:56 00:27:43 -1.0 +20210407E 336.0 1785.3 154.0 1271.5 1.0 -35.320866474063905 114.6146256941771 19.1 9.0 4.4 1.182 0.743 1.62 0.09 27:03:30.24 05:14:36.202 -1.0 +20210912A 336.0 1234.5 30.9 1271.5 1.0 -80.16655338584705 235.42165843951094 31.7 9.0 4.4 1.182 0.095 1.61 0.048 -30:29:33.1 23:24:40.3 -1.0 +20211127I 336.0 234.83 42.5 1271.5 1.0 -81.68554744714709 125.94306109583985 37.9 9.0 4.4 1.182 0.229 0.48 0.025 -18:49:28.4 13:19:09.5 0.046946 +20220531A 336.0 727.0 70.0 1271.5 1.0 -56.509199987039224 296.83938685003784 9.7 9.0 4.4 1.182 11.0 11.0 -1 -60:17:48.2 19:38:50.2 -1.0 +20220610A 336.0 1458.1 31.0 1271.5 1.0 -78.89122591551634 250.56280818953536 29.8 9.0 4.4 1.182 1.07 2.0 0.521 -33:30:49.87 23:24:17.559 1.016 +20220918A 336.0 656.8 40.7 1271.5 1.0 -45.81623556809721 308.4134807482381 26.4 9.0 4.4 1.182 11.43 13.9 7.66 -70:48:40.5 01:10:22.1 0.45 +20230526A 336.0 316.4 50.0 1271.5 1.0 -62.994316139408156 318.1591960546148 22.1 9.0 4.4 1.182 2 2.7 1.16 -52:46:07.7 01:29:27.5 0.157 +20230718A 336.0 477.0 396.0 1271.5 1.0 -75.66933767869608 316.30925962515585 10.9 9.0 4.4 1.182 0.55 0.695 0.117 -41:00:12.8 08:30:27.1 -1.0 +20230731A 336.0 701.1 547.0 1271.5 1.0 -60.14372530213189 304.262204790738 16.6 9.0 4.4 1.182 0.65 2.66 0.45 -56:58:19.1 11:38:40.1 -1.0 diff --git a/zdm/data/Surveys/CRAFT_ICS_1632.ecsv b/zdm/data/Surveys/CRAFT_ICS_1632.ecsv index 9780636..9042998 100644 --- a/zdm/data/Surveys/CRAFT_ICS_1632.ecsv +++ b/zdm/data/Surveys/CRAFT_ICS_1632.ecsv @@ -14,6 +14,7 @@ # - {name: THRESH, datatype: float64} # - {name: TRES, datatype: float64} # - {name: WIDTH, datatype: float64} +# - {name: XW95, datatype: float64} # - {name: TAU, datatype: float64} # - {name: DEC, datatype: string} # - {name: RA, datatype: string} @@ -22,7 +23,7 @@ # - {survey_data: '{"observing": {"NORM_FRB": 3, "TOBS": 50.866971336, "MAX_IDT": 4096, "MAX_IW": 12, "MAXWMETH": 1}, # "telescope": {"BEAM": "ASKAP_1632", "DIAM": 12.0, "NBEAMS": 1, "NBINS": 5}}'} # schema: astropy-2.0 -TNS BW DM DMG FBAR FRES Gb Gl SNR SNRTHRESH THRESH TRES WIDTH TAU DEC RA Z -20211212A 336.0 206.0 27.1 1631.5 1.0 -61.10615576177682 118.0697870677333 12.8 9.0 4.4 1.182 5.628 1.8 01:40:36.8 10:30:40.7 0.0715 -20220105A 336.0 583.0 22.0 1631.5 1.0 -40.39508677874226 124.21912132806385 9.8 9.0 4.4 1.182 2.25 0.43 22:27:57.8 13:55:13.01 0.2785 -20221106A 336.0 343.8 34.8 1631.5 1.0 -81.71820282415577 41.73623647983608 19.5 9.0 4.4 1.182 6.895 0.182 -25:34:10.5166 03:46:49.0982 0.204 +TNS BW DM DMG FBAR FRES Gb Gl SNR SNRTHRESH THRESH TRES WIDTH XW95 TAU DEC RA Z +20211212A 336.0 206.0 27.1 1631.5 1.0 -61.10615576177682 118.0697870677333 12.8 9.0 4.4 1.182 2.1 5.628 1.8 01:40:36.8 10:30:40.7 0.0715 +20220105A 336.0 583.0 22.0 1631.5 1.0 -40.39508677874226 124.21912132806385 9.8 9.0 4.4 1.182 0.95 2.25 0.43 22:27:57.8 13:55:13.01 0.2785 +20221106A 336.0 343.8 34.8 1631.5 1.0 -81.71820282415577 41.73623647983608 19.5 9.0 4.4 1.182 5.33 6.895 0.182 -25:34:10.5166 03:46:49.0982 0.204 diff --git a/zdm/data/Surveys/CRAFT_ICS_892.ecsv b/zdm/data/Surveys/CRAFT_ICS_892.ecsv index 3b6d18c..a785ab8 100644 --- a/zdm/data/Surveys/CRAFT_ICS_892.ecsv +++ b/zdm/data/Surveys/CRAFT_ICS_892.ecsv @@ -14,6 +14,7 @@ # - {name: THRESH, datatype: float64} # - {name: TRES, datatype: float64} # - {name: WIDTH, datatype: float64} +# - {name: XW95, datatype: float64} # - {name: TAU, datatype: float64} # - {name: DEC, datatype: string} # - {name: RA, datatype: string} @@ -22,19 +23,26 @@ # - {survey_data: '{"observing": {"NORM_FRB": 15, "TOBS": 317.293429793, "MAX_IDT": 4096, "MAX_IW": 12, "MAXWMETH": 1}, # "telescope": {"BEAM": "ASKAP_892", "DIAM": 12.0, "NBEAMS": 1, "NBINS": 5}}'} # schema: astropy-2.0 -TNS BW DM DMG FBAR FRES Gb Gl SNR SNRTHRESH THRESH TRES WIDTH TAU DEC RA Z -20191001A 336.0 506.92 44.2 919.5 1.0 -61.65860153281646 292.33820148500735 62.0 9.0 4.4 1.728 13.468 4.52 -54:44:54 21:33:24 0.23 -20200430A 336.0 380.1 27.0 864.5 1.0 -50.423867191684614 126.69623499918922 16.0 9.0 4.4 1.728 23.68 6.5 12:22:34.007 15:18:49.581 0.161 -20200627 336.0 294.0 40.0 920.5 1.0 -75.58714298569731 274.1944222920371 11.0 9.0 4.4 1.728 2.9 -1 -39:29:05.0 21:46:47.0 -1.0 -20200906A 336.0 577.8 35.9 864.5 1.0 -74.33639680729853 87.47153687885996 19.2 9.0 4.4 1.728 0.128 0.0315 -14:04:59.9136 03:33:58.9364 0.36879 -20210320C 336.0 384.8 42.2 864.5 1.0 -78.99989352050102 126.81646106581941 15.3 9.0 4.4 1.728 0.884 0.193 -16:09:05.1 13:37:50.08605 0.28 -20210807 336.0 251.9 121.2 920.5 1.0 -62.769666370514145 138.57666701461682 47.1 9.0 4.4 1.182 10.0 -1 -00:45:44.5 19:56:53.144 0.12969 -20210809 336.0 651.5 190.1 920.5 1.0 -61.10280652957891 133.77599069157813 16.8 9.0 4.4 1.182 14.2 -1 01:19:43.5 18:04:37.7 -1.0 -20211203C 336.0 636.2 63.4 920.5 1.0 -85.70736537183255 294.09236840807733 14.2 9.0 4.4 1.182 25.449 1.66 -31:22:04.0 13:37:52.8 0.34386 -20220501C 336.0 449.5 30.6 863.5 1.0 -79.3478690413585 245.52087739106628 16.1 9.0 4.4 1.182 6.9 0.35 -32:27:41.4 23:29:46.8 0.381 -20220725A 336.0 290.4 30.7 920.5 1.0 -77.2090157833712 260.2992248635945 12.7 9.0 4.4 1.182 8.016 2.29 -36:07:51.2 23:33:32.1 0.1926 -20230521A 336.0 640.2 41.8 831.5 1.0 -63.802137907452966 143.6440869115044 15.2 9.0 4.4 1.182 11.0 -1 -02:23:09.6 21:51:00.3 -1.0 -20230708A 336.0 411.5 50.2 920.5 1.0 -61.245461788949015 294.23527074045234 31.5 9.0 4.4 1.182 23.578 0.24 -55:22:59.4 20:12:56.9 0.105 -20230902A 336.0 440.1 34.3 831.5 1.0 -68.28916777429704 320.20535326767765 11.8 9.0 4.4 1.182 0.678 0.123 -47:33:45.5 03:29:28.1 -1.0 -20231006 336.0 509.7 67.5 863.5 1.0 -52.222367446784375 298.13333823887274 15.2 9.0 4.4 1.182 6.5 -1 -64:38:56.2 19:44:00.8 -1.0 -20231226A 336.0 329.9 38.0 863.5 1.0 -56.65392198775274 118.39332419712602 17.8 9.0 4.4 1.182 9.72 0.1 06:07:45.9 10:21:07.6 -1.0 +TNS BW DM DMG FBAR FRES Gb Gl SNR SNRTHRESH THRESH TRES WIDTH XW95 TAU DEC RA Z +20191001A 336.0 506.92 44.2 919.5 1.0 -61.65860153281646 292.33820148500735 62.0 9.0 4.4 1.728 5.3 13.468 4.52 -54:44:54 21:33:24 0.23 +20200430A 336.0 380.1 27.0 864.5 1.0 -50.423867191684614 126.69623499918922 16.0 9.0 4.4 1.728 11 22.68 6.5 12:22:34.007 15:18:49.581 0.161 +20200627 336.0 294.0 40.0 920.5 1.0 -75.58714298569731 274.1944222920371 11.0 9.0 4.4 1.728 2.9 2.9 -1 -39:29:05.0 21:46:47.0 -1.0 +20200906A 336.0 577.8 35.9 864.5 1.0 -74.33639680729853 87.47153687885996 19.2 9.0 4.4 1.728 0.057 0.128 0.0315 -14:04:59.9136 03:33:58.9364 0.36879 +20210320C 336.0 384.8 42.2 864.5 1.0 -78.99989352050102 126.81646106581941 15.3 9.0 4.4 1.728 0.381 0.884 0.193 -16:09:05.1 13:37:50.08605 0.28 +20210807 336.0 251.9 121.2 920.5 1.0 -62.769666370514145 138.57666701461682 47.1 9.0 4.4 1.182 10.0 10.0 -1 -00:45:44.5 19:56:53.144 0.12969 +20210809 336.0 651.5 190.1 920.5 1.0 -61.10280652957891 133.77599069157813 16.8 9.0 4.4 1.182 14.2 14.2 -1 01:19:43.5 18:04:37.7 -1.0 +20211203C 336.0 636.2 63.4 920.5 1.0 -85.70736537183255 294.09236840807733 14.2 9.0 4.4 1.182 12.4 25.449 1.66 -31:22:04.0 13:37:52.8 0.34386 +20220501C 336.0 449.5 30.6 863.5 1.0 -79.3478690413585 245.52087739106628 16.1 9.0 4.4 1.182 6.1 6.9 0.35 -32:27:41.4 23:29:46.8 0.381 +20220725A 336.0 290.4 30.7 920.5 1.0 -77.2090157833712 260.2992248635945 12.7 9.0 4.4 1.182 3.43 8.016 2.29 -36:07:51.2 23:33:32.1 0.1926 +20230521A 336.0 640.2 41.8 831.5 1.0 -63.802137907452966 143.6440869115044 15.2 9.0 4.4 1.182 11.0 11.0 -1 -02:23:09.6 21:51:00.3 -1.0 +20230708A 336.0 411.5 50.2 920.5 1.0 -61.245461788949015 294.23527074045234 31.5 9.0 4.4 1.182 1.14 23.578 0.24 -55:22:59.4 20:12:56.9 0.105 +20230902A 336.0 440.1 34.3 831.5 1.0 -68.28916777429704 320.20535326767765 11.8 9.0 4.4 1.182 0.229 0.678 0.123 -47:33:45.5 03:29:28.1 0.3619 +20231006 336.0 509.7 67.5 863.5 1.0 -52.222367446784375 298.13333823887274 15.2 9.0 4.4 1.182 6.5 6.5 -1 -64:38:56.2 19:44:00.8 -1.0 +20231226A 336.0 329.9 38.0 863.5 1.0 -56.65392198775274 118.39332419712602 17.8 9.0 4.4 1.182 5.3 9.72 0.1 06:07:45.9 10:21:07.6 0.1569 +20240201A 336.0 374.5 38 920.5 1.0 67.29517415215354 347.08291493850965 13.9 9.0 4.4 1.182 3.05 3.901 0.78 10:01:49.1 13:54:49 0.042729 +20240208A 336.0 260.2 98 863.5 1.0 -52.0180286372879 115.89194578232592 12.1 9.0 4.4 1.182 1.7 10 1.35 10:36:46.5 -00:33:50 -1 +20240210A 336.0 283.73 31 863.5 1.0 -36.19982407564526 55.957809969522174 11.6 9.0 4.4 1.182 0.305 1.539 0.1 00:39:55.0 -27:39:35 0.023686 +20240304A 336.0 652.6 30 832.5 1.0 22.379345521533207 214.02134339859478 12.3 9.0 4.4 1.182 8.57 19 2.51 09:05:19.3 -16:13:42 -1.0 +20240310A 336.0 601.8 36 902.5 1.0 -31.606263057321083 192.9022403619661 19.1 9.0 4.4 1.182 4.19 13.493 2.23 01:10:57.7 -44:24:05 0.127 +20240318A 336.0 256.4 37 902.5 1.0 69.83146076143834 337.11332396083634 13.2 9.0 4.4 1.182 0.286 0.837 0.163 10:01:50.6 37:36:49 -1.0 + diff --git a/zdm/figures.py b/zdm/figures.py index 760b5b7..fb8c44b 100644 --- a/zdm/figures.py +++ b/zdm/figures.py @@ -714,3 +714,29 @@ def ticks_pgrid(vals, everyn=5, fmt=None, these_vals=None): # Return return tvals, ticks + +def gen_cdf_hist(origx): + """ + Args: + origx (np.ndarray): x values of the data + + Returns: + xvals (np.ndarray): x values of cdf plot + yvals (np.ndarray): y values of cdf plot + """ + + xs = np.sort(origx) + + N = xs.size + newN = 2*N + xvals = np.zeros([newN]) + yvals = np.zeros([newN]) + + for i,x in enumerate(xs): + xvals[2*i] = x + xvals[2*i+1] = x + + yvals[2*i] = i/N + yvals[2*i+1] = (i+1)/N + return xvals,yvals + diff --git a/zdm/iteration.py b/zdm/iteration.py index 1bc6d7f..fd0d257 100644 --- a/zdm/iteration.py +++ b/zdm/iteration.py @@ -387,9 +387,14 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True, llptw = np.sum(np.log10(ptaus)) llpiw = np.sum(np.log10(piws)) - llsum += llptw - llsum += llpiw + # while we calculate llpiw, we don't add it to the sum + # this is because w and tau are not independent! + # p(iw|tau,w) = \delta(iw-(w**2 - tau**2)**0.5) + # However, numerical differences will affect this + # hence, we add half of eavh value here + llsum += 0.5*llpiw + llsum += 0.5*llptw lllist.append(llptw) lllist.append(llpiw) @@ -934,8 +939,13 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal ptaus[bad2] = 1e-10 llpiw = np.sum(np.log10(piws)) llptw = np.sum(np.log10(ptaus)) - llsum += llpiw - llsum += llptw + # while we calculate llpiw, we don't add it to the sum + # this is because w and tau are not independent! + # p(iw|tau,w) = \delta(iw-(w**2 - tau**2)**0.5) + # However, numerical differences will affect this + # Hence, we ad half of each value + llsum += 0.5*llpiw + llsum += 0.5*llptw lllist.append(llpiw) lllist.append(llptw) diff --git a/zdm/misc_functions.py b/zdm/misc_functions.py index 9651885..d748f12 100644 --- a/zdm/misc_functions.py +++ b/zdm/misc_functions.py @@ -28,6 +28,53 @@ from zdm import parameters + +def get_width_stats(s,g): + """ + gets the probability of detecting tau or w for + a given survey and grid + + Args: + s: survey + g: correspondiong grid object + + Returns: + Rw (np.ndarray): Rate (per day) of detecting intrinsic width w + Rtau (np.ndarray): Rate (per day) of detecting scattering time tau + Nw (np.ndarray): Total number of FRBs as a function of total width + Nwz (np.ndarray): Number of FRBs as a function of total width and redshift + Nwdm (np.ndarray): Number of FRBs as a function of total width and DM + """ + + # extracts the p(W) distribution + Nw,Nwz,Nwdm = g.get_pw_dist() + + if s.wplist.ndim > 1: + # plist is z-dependent + # get expected distribution at z=0 + wplist = s.wplist[:,0] + else: + wplist = s.wplist + + # these two arrays hold p(tau) and p(iw) values with dimensions: + # z, internal tau values, iwidth + # the normalisation is such that the sum over internal widths is unity + # that is, for a given tau, what is p(w). NOT for a given w, what is ptau! + # this is all a function of z + + Rtau = np.zeros([s.internal_logwvals.size]) + Rw = np.zeros([s.internal_logwvals.size]) + # calculates ptauw + for i,t in enumerate(s.internal_logwvals): + ptz = s.ptaus[:,i,:] # this is p(tau) given z and w + Rtz = ptz * Nwz.T + Rtau[i] = np.sum(Rtz) + + pwz = s.pws[:,i,:] # this is p(tau) given z and w + Rwz = pwz * Nwz.T + Rw[i] = np.sum(Rwz) + return Rw,Rtau,Nw,Nwz,Nwdm + def j2000_to_galactic(ra_deg, dec_deg): """ Convert Galactic coordinates to Equatorial J2000 coordinates. diff --git a/zdm/scripts/Scattering/width_scat_dists.py b/zdm/scripts/Scattering/width_scat_dists.py index 7ba3e8e..cc2a7b7 100644 --- a/zdm/scripts/Scattering/width_scat_dists.py +++ b/zdm/scripts/Scattering/width_scat_dists.py @@ -20,6 +20,7 @@ from zdm import parameters from zdm import survey from zdm import pcosmic +from zdm import misc_functions as mf from zdm import iteration as it from zdm import loading from zdm import io @@ -83,46 +84,19 @@ def main(): ################ Generates some plots ################ # extracts the p(W) distribution - Nw,Nwz,Nwdm = g.get_pw_dist() + Rw,Rtau,Nw,Nwz,Nwdm = mf.get_width_stats(s,g) - # gets the actual values of width - ws = s.wlist - - if s.wplist.ndim > 1: - # plist is z-dependent - # get expected distribution at z=0 - wplist = s.wplist[:,0] - else: - wplist = s.wplist + # values at z=0 + WidthArgs = (s.wlogmean,s.wlogsigma) + ScatArgs = (s.slogmean,s.slogsigma) - # generates some plots!!! # these need to be normalised by the internal bin width logbinwidth = s.internal_logwvals[-1] - s.internal_logwvals[-2] - # values at z=0 - WidthArgs = (s.wlogmean,s.wlogsigma) - ScatArgs = (s.slogmean,s.slogsigma) pw = s.WidthFunction(s.internal_logwvals, *WidthArgs)*s.dlogw #/logbinwidth ptau = s.ScatFunction(s.internal_logwvals, *ScatArgs)*s.dlogw #/logbinwidth - # these two arrays hold p(tau) and p(iw) values with dimensions: - # z, internal tau values, iwidth - # the normalisation is such that the sum over internal widths is unity - # that is, for a given tau, what is p(w). NOT for a given w, what is ptau! - # this is all a function of z - - Rtau = np.zeros([s.internal_logwvals.size]) - Rw = np.zeros([s.internal_logwvals.size]) - # calculates ptauw - for i,t in enumerate(s.internal_logwvals): - ptz = s.ptaus[:,i,:] # this is p(tau) given z and w - Rtz = ptz * Nwz.T - Rtau[i] = np.sum(Rtz) - - pwz = s.pws[:,i,:] # this is p(tau) given z and w - Rwz = pwz * Nwz.T - Rw[i] = np.sum(Rwz) - + ws = s.wlist norm=True if norm: @@ -131,7 +105,7 @@ def main(): # width in units of dlogw # n1,n2 are probability "per bin". hence, to convert to a p(w) dlogw, we diving by the bin width in logw - n1 = np.sum(wplist) * s.dlogw # n1 is probability within the bin. n3-6 are probability dlogp + n1 = np.sum(s.wplist) * s.dlogw # n1 is probability within the bin. n3-6 are probability dlogp n2 = np.sum(Nw) * s.dlogw n3 = np.sum(ptau) * logbinwidth @@ -154,7 +128,7 @@ def main(): plt.ylim(1e-2,1) plt.xlim(1e-2,1e2) - plt.plot(ws,wplist/n1,label="Total width (z=0)",linestyle="-") + plt.plot(ws,s.wplist/n1,label="Total width (z=0)",linestyle="-") plt.plot(ws,Nw/n2,label="Detected total",linestyle=":",color=plt.gca().lines[-1].get_color()) plt.plot(10**s.internal_logwvals,ptau/n3,label="Scattering (z=0)",linestyle="-") @@ -177,7 +151,7 @@ def main(): plt.xscale("log") plt.yscale("log") plt.ylim(1e-3,None) - plt.plot(ws,wplist/np.sum(wplist),label="Intrinsic width",color="black") + plt.plot(ws,s.wplist/np.sum(s.wplist),label="Intrinsic width",color="black") plt.plot(ws,Nw/np.sum(Nw),label="Detected FRBs",linestyle="--") plt.plot(ws,Nwz[:,4]/np.sum(Nwz[:,4]),label=" (z=0.25)",linestyle="--") plt.plot(ws,Nwz[:,18]/np.sum(Nwz[:,18]),label=" (z=1.25)",linestyle=":") @@ -188,13 +162,13 @@ def main(): plt.savefig(opdir+"pw_zdep.png") plt.close() - #### Plot 2: DM dependence ##### + #### Plot 3: DM dependence ##### plt.figure() plt.xscale("log") plt.yscale("log") plt.ylim(1e-3,None) - plt.plot(ws,wplist/np.sum(wplist),label="Intrinsic width",color="black") + plt.plot(ws,s.wplist/np.sum(s.wplist),label="Intrinsic width",color="black") plt.plot(ws,Nw/np.sum(Nw),label="Detected FRBs",linestyle="--") plt.plot(ws,Nwdm[:,3]/np.sum(Nwdm[:,3]),label=" (DM=125)",linestyle="--") plt.plot(ws,Nwdm[:,21]/np.sum(Nwdm[:,21]),label=" (DM=1025)",linestyle=":") @@ -204,5 +178,5 @@ def main(): plt.tight_layout() plt.savefig(opdir+"pw_dmdep.png") plt.close() - + main() diff --git a/zdm/survey.py b/zdm/survey.py index 461a618..eae41dd 100644 --- a/zdm/survey.py +++ b/zdm/survey.py @@ -827,21 +827,16 @@ def process_survey_file(self,filename:str, self.NORM_FRB=self.meta['NORM_FRB'] # calculates intrinsic widths - # likely needs to modify this because TAU is is the 1/e timescale, - # For a Gaussian burst, SNR max at 1.4 sigma, w95 at 1.9 sigma - # For exponential, SNR max at 1.3 tau, w95 at 3 tau - # Hence, should multiply tau by 3/1.3, calculate new w95, then divide by 1.9/1.4 to get SNR max width - # Note that none of this is satisfactory - it is dependent on pulse shape and DM, neither - # of which we can account for here - tscale = 3./1.3 - wscale = 1.4/1.9 + # Uses the model of James et al 2025 + # assumes we have S/N maximising widths + tscale = 2. # scale scattering time to total width at +- 1 sigma + TEMP = self.frbs['WIDTH'].values**2 - (tscale*self.frbs['TAU'].values)**2 self.OKTAU = np.where(self.frbs['TAU'].values != -1.)[0] # code for non-existent toolow = np.where(TEMP <= 0.) TEMP[toolow] = 0.01**2 # 10 microsecond width. Really could be anything up to tau - iwidths = wscale * TEMP**0.5 # scale to SNR max width assuming Gaussian shape + iwidths = TEMP**0.5 # scale to SNR max width assuming Gaussian shape self.IWIDTHs = iwidths - self.WIDTHs *= wscale # scales to SNR maximising width assuming Gaussian shape self.TAUs = self.frbs['TAU'].values # sets the 'beam' values to unity by default @@ -1112,7 +1107,7 @@ def calc_relative_sensitivity(DM_frb,DM,w,fbar,t_res,nu_res,Nchan=336,max_idt=No # calculates relative sensitivity to bursts as a function of DM # Check for Quadrature and Sammons - elif model == 'Quadrature' or model == 'Sammons': + elif model == 'Quadrature' or model == 'Sammons' or model=="StdDev": # constant of DM k_DM=4.149 #ms GHz^2 pc^-1 cm^3 @@ -1134,11 +1129,15 @@ def calc_relative_sensitivity(DM_frb,DM,w,fbar,t_res,nu_res,Nchan=336,max_idt=No # w represents the intrinsic width uw=w - totalw = (uw**2 + dm_smearing**2 + t_res**2)**0.5 - + if model == "StdDev": + totalw = (uw**2 + dm_smearing**2/3. + t_res**2/3.)**0.5 + nosmearw = (uw**2 + t_res**2/3.)**0.5 + else: + totalw = (uw**2 + dm_smearing**2 + t_res**2)**0.5 + nosmearw = (uw**2 + t_res**2)**0.5 # calculates relative sensitivity to bursts as a function of DM - if model=='Quadrature': + if model=='Quadrature' or model=="StdDev": sensitivity=totalw**-0.5 elif model=='Sammons': sensitivity=0.75*(0.93*dm_smearing + uw + 0.35*t_res)**-0.5 @@ -1146,10 +1145,22 @@ def calc_relative_sensitivity(DM_frb,DM,w,fbar,t_res,nu_res,Nchan=336,max_idt=No # implements max integer width cut. if max_meth != 0 and max_iw is not None: max_w = t_res*(max_iw+0.5) - toolong = np.where(totalw > max_w)[0] - if max_meth == 1: + + if max_meth == 1 or max_meth == 2: + # NOTE: for CRAFT, dm smearing already accounted for prior to width search + # this means that the smearing cut is DM independent + if nosmearw > max_w: + toolong = np.arange(dm_smearing.size) + else: + toolong = [] + #toolong = np.where(nosmearw > max_w)[0] + elif max_meth == 3 or max_meth == 4: + # NOTE: for CRAFT, dm smearing already accounted for prior to width search + toolong = np.where(totalw > max_w)[0] + + if max_meth == 1 or max_meth == 3: sensitivity[toolong] = MIN_THRESH # something close to zero - elif max_meth == 2: + elif max_meth == 2 or max_meth == 4: # we have already reduced it by \sqrt{t} # we thus add a further sqrt{t} factor sensitivity[toolong] *= (max_w / totalw[toolong])**0.5 @@ -1330,7 +1341,7 @@ def quadrature_convolution(width_function, width_args, scat_function, scat_args, h,b = np.histogram(totalwidths,bins=bins,weights=probs) hist += h - # calculate p(w) and p(tau) for each w fior this z + # calculate p(w) and p(tau) for each w for this z if backproject: # generate arrays to hold probabilities, so values of tau and # w can be fit @@ -1372,14 +1383,16 @@ def quadrature_convolution(width_function, width_args, scat_function, scat_args, taufracs = taufracs/tnorm # plot some examples. This code is kept here for internal analysis purposes - if False: + if True: plt.figure() plt.plot(internal_logvals,ptau,label="Intrinsic p(tau)") plt.plot(internal_logvals,pw,label="Intrinsic p(w)") for ib in np.arange(Nbins): - plt.plot(internal_logvals,taufracs[:,ib],label="width "+str(ib)) + plt.plot(internal_logvals,taufracs[:,ib],label="p(tau) width "+str(ib)) + plt.plot(internal_logvals,wfracs[:,ib],label="p(w) width "+str(ib)) plt.plot(np.log10([bins[ib],bins[ib]]),[0,1],linestyle=":",color=plt.gca().lines[-1].get_color()) plt.plot(np.log10([bins[ib+1],bins[ib+1]]),[0,1],linestyle=":",color=plt.gca().lines[-1].get_color()) + break plt.yscale("log") #plt.xscale("log") plt.xlabel("log10 Tau [ms]") @@ -1390,6 +1403,7 @@ def quadrature_convolution(width_function, width_args, scat_function, scat_args, plt.close() # exit now, to prevent very many such plots being generated exit() + return hist,wfracs,taufracs else: return hist diff --git a/zdm/survey_data.py b/zdm/survey_data.py index 17a95a7..d84fd60 100644 --- a/zdm/survey_data.py +++ b/zdm/survey_data.py @@ -164,8 +164,8 @@ class Telescope(data_class.myDataClass): 'Notation': '', }) WBIAS: str = field( - default="Quadrature", - metadata={'help': "Method to calculate width bias", + default="StdDev", + metadata={'help': "Method to calculate width bias. Quadrature, Sammons, or StdDev", 'unit': '', 'Notation': '', }) From 45e4a20b62351922341772c65c89c9d6252a29d1 Mon Sep 17 00:00:00 2001 From: Clancy James Date: Fri, 5 Sep 2025 16:39:13 +0800 Subject: [PATCH 10/20] Various minor updates to width and scattering behaviour for consistency with scattering paper --- papers/Scattering/MCMC_inputs/scat_w_sfr.json | 111 ++ papers/Scattering/functions.py | 220 ++++ papers/Scattering/nonparametric.py | 1125 ++++++++++++++++- papers/Scattering/run_MCMC_slurm_sfr.sh | 34 + papers/Scattering/visualise_mcmc.py | 23 +- zdm/MCMC.py | 22 +- zdm/figures.py | 20 +- zdm/iteration.py | 34 +- zdm/misc_functions.py | 24 +- zdm/scripts/Scattering/width_scat_dists.py | 5 +- zdm/survey.py | 32 +- 11 files changed, 1592 insertions(+), 58 deletions(-) create mode 100644 papers/Scattering/MCMC_inputs/scat_w_sfr.json create mode 100644 papers/Scattering/functions.py create mode 100755 papers/Scattering/run_MCMC_slurm_sfr.sh diff --git a/papers/Scattering/MCMC_inputs/scat_w_sfr.json b/papers/Scattering/MCMC_inputs/scat_w_sfr.json new file mode 100644 index 0000000..6426c7d --- /dev/null +++ b/papers/Scattering/MCMC_inputs/scat_w_sfr.json @@ -0,0 +1,111 @@ +{ + "mcmc": { + "parameter_order": ["Wlogmean", "Wlogsigma", "Slogmean", "Slogsigma","sfr_n"] + }, + "config": { + "luminosity_function": 2, + "alpha_method": 1, + "halo_method": 0, + "sigmaDMG": 0.0, + "sigmaHalo": 15.0, + "lmean": 2.02, + "lsigma": 0.46, + "sfr_n": 0.95, + "gamma": -1.16, + "alpha": 1.5, + "WNbins": 33, + "WNInternalBins": 1000, + "ScatFunction": 1, + "Sfnorm": 1000 + }, + "Wlogmean": { + "DC": "width", + "min": -3, + "max": 3 + }, + "Wlogsigma": { + "DC": "width", + "min": 0.2, + "max": 4 + }, + "Slogmean": { + "DC": "scat", + "min": -3, + "max": 3 + }, + "Slogsigma": { + "DC": "scat", + "min": 1, + "max": 8 + }, + "logF": { + "DC": "cosmo", + "min": -2.0, + "max": 0.0 + }, + "lEmax": { + "DC": "energy", + "min": 40.35, + "max": 45.0 + }, + "lEmin": { + "DC": "energy", + "min": 35.0, + "max": 40.35 + }, + "DMhalo": { + "DC": "MW", + "min": 0.0, + "max": 100 + }, + "sigmaDMG": { + "DC": "MW", + "min": 0.1, + "max": 0.5 + }, + "H0": { + "DC": "cosmo", + "min": 35.0, + "max": 110.0 + }, + "alpha": { + "DC": "energy", + "min": 0.5, + "max": 2.5 + }, + "gamma": { + "DC": "energy", + "min": -3.0, + "max": 0.0 + }, + "sfr_n": { + "DC": "FRBdemo", + "min": -2.0, + "max": 4.0 + }, + "lmean": { + "DC": "host", + "min": 1.0, + "max": 3.0 + }, + "lsigma": { + "DC": "host", + "min": 0.1, + "max": 1.5 + }, + "lRmin": { + "DC": "rep", + "min": -2.0, + "max": 1.0 + }, + "lRmax": { + "DC": "rep", + "min": 0.0, + "max": 4.0 + }, + "Rgamma": { + "DC": "rep", + "min": -3.0, + "max": -0.5 + } +} diff --git a/papers/Scattering/functions.py b/papers/Scattering/functions.py new file mode 100644 index 0000000..969e4d1 --- /dev/null +++ b/papers/Scattering/functions.py @@ -0,0 +1,220 @@ +""" +Set of functions used to fit scattering/width data +""" + +import numpy as np + + +def function_wrapper(ifunc,args,xvals=None,logxmin=-6,logxmax=6,N=1000,cspline=None): + """ + Wrapper for fitting functions, so they can be called by an ID, rather + than by name. + + Args: + ifunc [int]: integer ID of function + args [list]: aguments to the function + xvals [np.ndarray or None]: if provided, x values at which to + evaluate the function. If not, function evaluated between + logxmin, logxmax + logxmin [float]: min value at which to evaluate the function in log10 space + logxmax [flot]: max value at which to evaluate the function in log10 space + N [int]: number of points at which to evaluate the function + cspline [scipy spline object]: Spline object giving completeness + + Returns: + xs: x values at which function is evaluated [only if xvals not provided] + ys: value of function at xvals/xs + """ + + # modifies the likelihood function by the completeness, and normalises + xs = np.logspace(logxmin, logxmax, N) + logxs = np.linspace(logxmin, logxmax, N) + dlx = logxs[1]-logxs[0] + + a1 = args[0] + if ifunc != 3: + a2 = args[1] + # get expected distribution and hence cdf + A=1. + if ifunc==0: + ftaus = lognormal(xs,A,a1,a2) + elif ifunc==1: + ftaus = halflognormal(xs,A,a1,a2) + elif ifunc == 2: + ftaus = boxcar(xs,A,a1,a2) + elif ifunc == 3: + ftaus = logconstant(xs,A,a1) + elif ifunc == 4: + a3 = args[2] + ftaus = smoothboxcar(xs,A,a1,a2,a3) + elif ifunc == 5: + a3 = args[2] + ftaus = uppersmoothboxcar(xs,A,a1,a2,a3) + elif ifunc == 6: + a3 = args[2] + ftaus = lowersmoothboxcar(xs,A,a1,a2,a3) + elif ifunc==7: + ftaus = halflognormal2(xs,A,a1,a2) + + if cspline is not None: + ctaus = cspline(logxs) + ftaus *= ctaus + + if xvals is not None: + if ifunc==0: + ptaus = lognormal(xvals,A,a1,a2) + elif ifunc==1: + ptaus = halflognormal(xvals,A,a1,a2) + elif ifunc == 2: + ptaus = boxcar(xvals,A,a1,a2) + elif ifunc == 3: + ptaus = logconstant(xvals,A,a1) + elif ifunc == 4: + a3 = args[2] + ptaus = smoothboxcar(xvals,A,a1,a2,a3) + elif ifunc == 5: + a3 = args[2] + ptaus = uppersmoothboxcar(xvals,A,a1,a2,a3) + elif ifunc == 6: + a3 = args[2] + ptaus = lowersmoothboxcar(xvals,A,a1,a2,a3) + elif ifunc==47: + ptaus = halflognormal2(xvals,A,a1,a2) + + + # checks normalisation in log-space. + # This is such that \sum f(x) dlogx = 1 + norm = np.sum(ftaus)*dlx + if xvals is None: + ys = ftaus/norm + return xs,ys + else: + ys = ptaus/norm + return ys + + + + +def boxcar(x,*args): + """ + Constant above some min value + """ + logx = np.log10(x) + A = args[0] + xmin = args[1] + xmax = args[2] + y = np.zeros([logx.size]) + OK = np.where(logx > xmin) + y[OK] = A + notOK = np.where(logx > xmax) + y[notOK] = 0. + return y + + +def smoothboxcar(x,*args): + """ + Boxcar, with smooth Gaussian edges on both sides + """ + logx = np.log10(x) + A = args[0] + xmin = args[1] + xmax = args[2] + sigma = args[3] + y = np.full([logx.size],A) + large = np.where(logx > xmax) + y[large] = lognormal(x[large],A,xmax,sigma) + small = np.where(logx < xmin) + y[small] = lognormal(x[small],A,xmin,sigma) + + #y[notOK] = 0. + return y + +def uppersmoothboxcar(x,*args): + """ + Boxcar, with smooth Gaussian edge on upper side only + """ + logx = np.log10(x) + A = args[0] + xmin = args[1] + xmax = args[2] + sigma = args[3] + y = np.full([logx.size],A) + large = np.where(logx > xmax) + y[large] = lognormal(x[large],A,xmax,sigma) + small = np.where(logx < xmin) + y[small] = 0. + + #y[notOK] = 0. + return y + +def lowersmoothboxcar(x,*args): + """ + Boxcar, with smooth Gaussian edge on upper side only + """ + logx = np.log10(x) + A = args[0] + xmin = args[1] + xmax = args[2] + sigma = args[3] + y = np.full([logx.size],A) + large = np.where(logx > xmax) + y[large] = 0. + small = np.where(logx < xmin) + y[small] = lognormal(x[small],A,xmin,sigma) + + #y[notOK] = 0. + return y + +def logconstant(x,*args): + """ + Constant above some min value + """ + logx = np.log10(x) + A = args[0] + xmin = args[1] + y = np.zeros([logx.size]) + OK = np.where(logx > xmin) + y[OK] = A + return y + +def halflognormal(x,*args): + """ + Just like a lognormal, but only lower half + """ + logx = np.log10(x) + A=args[0] + mu = args[1] + sigma=args[2] + + y = A * np.exp(-0.5 * (logx-mu)**2/sigma**2) + OK = np.where(logx > mu) + y[OK] = A + + return y + +def halflognormal2(x,*args): + """ + Just like a lognormal, but only lower half + """ + logx = np.log10(x) + A=args[0] + mu = args[1] + sigma=args[2] + + y = A * np.exp(-0.5 * (logx-mu)**2/sigma**2) + OK = np.where(logx < mu) + y[OK] = A + + return y + +def lognormal(x,*args): + """ + lognormal, but just log10x + """ + logx = np.log10(x) + A=args[0] + mu = args[1] + sigma=args[2] + y = A * np.exp(-0.5 * (logx-mu)**2/sigma**2) + return y + diff --git a/papers/Scattering/nonparametric.py b/papers/Scattering/nonparametric.py index f0d2824..f757841 100644 --- a/papers/Scattering/nonparametric.py +++ b/papers/Scattering/nonparametric.py @@ -1,26 +1,1097 @@ import numpy as np from matplotlib import pyplot as plt +from zdm import misc_functions import scipy as sp import matplotlib - +from scipy import stats import sys import os - +import scipy as sp +from functions import * import pandas -defaultsize=16 +defaultsize=14 ds=4 font = {'family' : 'Helvetica', 'weight' : 'normal', 'size' : defaultsize} matplotlib.rc('font', **font) +# number of functions implemented +NFUNC=7 +FNAMES=["lognormal","half-lognormal","boxcar","log constant","smooth boxcar", "upper sb", "lower sb","hln"] +ARGS0=[[0.,1.],[0.,1.],[-2,2],[-2],[-1.,1,1.],[-2.,2,1.],[-2.,2,1.],[0.,1.]] +WARGS0=[[0.,1.],[0.,1.],[-3.5,2],[-3.5],[-1.,1,1.],[-3.5,2,1.],[-3.5,2,1.],[0.,1.]] + +# set up min/max ranges of integrations for Bayes factors +min_sigma = 0.3 +max_sigma = 4 +min_scat = 1e-6 +max_scat = 1000 +log_min_sigma = np.log10(min_sigma) +log_max_sigma = np.log10(max_sigma) +log_min_scat = np.log10(min_scat) +log_max_scat = np.log10(max_scat) +MIN_PRIOR = [ + [log_min_scat,log_min_sigma], + [log_min_scat,log_min_sigma], + [log_min_scat,log_min_scat], + [log_min_scat], + [log_min_scat,log_min_scat,log_min_sigma], + [log_min_scat,log_min_scat,log_min_sigma], + [log_min_scat,log_min_scat,log_min_sigma], + [log_min_scat,log_min_sigma] + ] + +MAX_PRIOR = [ + [log_max_scat,log_max_sigma], + [log_max_scat,log_max_sigma], + [log_max_scat,log_max_scat], + [log_max_scat], + [log_max_scat,log_max_scat,log_max_sigma], + [log_max_scat,log_max_scat,log_max_sigma], + [log_max_scat,log_max_scat,log_max_sigma], + [log_max_scat,log_max_sigma] + ] + + def main(): - ##### does scatz plot ##### - #infile="table_scatz.dat" - #scat,scatmid,err,errl,errh,z,DM,DMG,SNR,Class = read_data(infile) - #scat,scaterr,z,DM,DMG,SNR,Class = read_data(infile) + # does k-s test to scattering distribution + + plot_functions() + + tns,tauobs,w95,wsnr,z,snr,freq,DM,tres = get_data() + + # gets observed intrinsic widths + wobs = wsnr**2 - (tauobs/0.816)**2 + BAD = np.where(wobs < 0.) + # this is maybe a bit dodgy. Here, we set it to scattering/10. + #wobs[BAD] = 0.00001 + + wobs[BAD] = 0.01*tauobs[BAD]**2 + #wobs[BAD] = 1.e-2 + wobs = wobs**0.5 + + # generates scatter plot of tau and width + plt.figure() + plt.scatter(wsnr,tauobs) + plt.xlabel("$w_{\\rm SNR}$") + plt.ylabel("$\\tau_{\\rm obs}$") + plt.xscale("log") + plt.yscale("log") + slope = 0.816 + x=np.array([1e-2,20]) + y=slope*x + plt.plot(x,y,linestyle="--",color="black") + plt.tight_layout() + plt.savefig("scatter_w_tau.png") + plt.close() + + NFRB = snr.size + + maxtaus = np.zeros([NFRB]) + for i in np.arange(NFRB): + taumax = find_max_tau(tauobs[i],wsnr[i],snr[i],freq[i],DM[i],tres[i],plot=False) + maxtaus[i] = taumax + + + maxws = np.zeros([NFRB]) + for i in np.arange(NFRB): + + wmax = find_max_w(tauobs[i],wsnr[i],snr[i],freq[i],DM[i],tres[i],plot=False) + maxws[i] = wmax + + # scale taus to 1 GHz in host rest frame + alpha = -4 + host_tau = tauobs*(1+z)**3 * (freq/1e3)**-alpha + host_maxtaus = maxtaus * (1+z)**3 * (freq/1e3)**-alpha + + host_w = wobs/(1+z) + host_maxw = maxws/(1+z) + #for i,x in enumerate(host_maxw): + # print(i,z[i],maxws[i],host_maxw[i]) + + + + Nbins=11 + bins = np.logspace(-3.,2.,Nbins) + + + ################################################################## + ############################ TAU ############################# + ################################################################## + + ############## Observed Histogram ################# + + plt.figure() + #obshist,bins = np.histogram(tauobs,bins=bins) + #hosthist,bins = np.histogram(host_tau,bins=bins) + ax1=plt.gca() + plt.xscale("log") + plt.ylim(0,8) + l1 = plt.hist(tauobs,bins=bins,label="Observed",alpha=0.5) + + # makes a function of completeness + xvals,yvals = make_completeness_plot(maxtaus) + + tau_comp = get_completeness(tauobs,xvals,yvals) + + l2 = plt.hist(tauobs,bins=bins,weights = 1./tau_comp,label="Observed",alpha=0.5) + + + ax2 = ax1.twinx() + l3 = ax2.plot(xvals,yvals,label="Completeness") + plt.ylim(0,1) + plt.xlim(1e-2,1e3) + plt.ylabel("Completeness") + plt.sca(ax1) + plt.legend(handles=[l1[2],l3[0],l2[2]],labels=["$\\tau_{\\rm obs}$","Completeness","Corrected $\\tau_{\\rm obs}$"],fontsize=12) + plt.xlabel("$\\tau$ [ms]") + plt.ylabel("Number of FRBs") + plt.tight_layout() + plt.savefig("tau_observed_histogram.png") + plt.close() + + + + ############## 1 GHz Rest-frame Histogram ################# + + plt.figure() + #obshist,bins = np.histogram(tauobs,bins=bins) + #hosthist,bins = np.histogram(host_tau,bins=bins) + ax1=plt.gca() + plt.xscale("log") + plt.ylim(0,8) + l1 = plt.hist(host_tau,bins=bins,label="Observed",alpha=0.5) + + # makes a function of completeness + xvals,yvals = make_completeness_plot(host_maxtaus) + + # get completeness at points of measurement + tau_comp = get_completeness(host_tau,xvals,yvals) + + l2 = plt.hist(host_tau,bins=bins,weights = 1./tau_comp,label="Corrected",alpha=0.5) + + + ax2 = ax1.twinx() + l3 = ax2.plot(xvals,yvals,label="Completeness") + + plt.ylim(0,1) + plt.xlim(1e-2) + plt.ylabel("Completeness") + plt.sca(ax1) + plt.legend(handles=[l1[2],l3[0],l2[2]],labels=["Observed","Completeness","Corrected"],fontsize=12) + plt.xlabel("$\\tau_{\\rm host, 1\,GHz}$ [ms]") + plt.ylabel("Number of FRBs") + plt.tight_layout() + plt.savefig("tau_host_histogram.png") + + #### creates a copy of the above, for paper purposes + + plt.figure() + #obshist,bins = np.histogram(tauobs,bins=bins) + #hosthist,bins = np.histogram(host_tau,bins=bins) + ax1v2=plt.gca() + plt.xscale("log") + plt.ylim(0,8) + l1v2 = plt.hist(host_tau,bins=bins,label="Observed",alpha=0.5) + + # makes a function of completeness + xvals,yvals = make_completeness_plot(host_maxtaus) + + # get completeness at points of measurement + tau_comp = get_completeness(host_tau,xvals,yvals) + + l2v2 = plt.hist(host_tau,bins=bins,weights = 1./tau_comp,label="Corrected",alpha=0.5) + + + ax2v2 = ax1v2.twinx() + l3v2 = ax2v2.plot(xvals,yvals,label="Completeness") + + plt.ylim(0,1) + plt.xlim(1e-2) + plt.ylabel("Completeness") + plt.sca(ax1) + plt.xlabel("$\\tau_{\\rm host, 1\,GHz}$ [ms]") + plt.ylabel("Number of FRBs") + + # keeps open for later plotting + + + ####################################### TAU - CDF and fitting ################################ + print("\n\n KS test evaluation \n") + # amplitude, mean, and std dev of true distribution + ifunc=0 + args = (host_tau,xvals,yvals,ifunc) + #get_ks_stat(-1,1,*args) + + + ksbest = [] + # begins minimisation + for ifunc in np.arange(NFUNC): + if False: + print("skipping K-S test optimisation, remove this line to re-run") + continue + args = (host_tau,xvals,yvals,ifunc) + x0=ARGS0[ifunc] + + result = sp.optimize.minimize(get_ks_stat,x0=x0,args=args,method = 'Nelder-Mead') + psub1 = get_ks_stat(result.x,*args,plot=False) + ksbest.append(result.x) + #Best-fitting parameters are [0.85909445 1.45509687] with p-value 0.9202915513749959 + print("FUNCTION ",ifunc,",",FNAMES[ifunc]," Best-fitting parameters are ",result.x," with p-value ",1.-result.fun) + + + xtemp = xvals[::2] + ytemp = yvals[::2] + s = xtemp.size + xt = np.zeros([s+2]) + yt = np.zeros([s+2]) + xt[0] = xtemp[0] + yt[0] = ytemp[0] + xt[1] = 1. + yt[1] = 1. + xt[2:-1] = xtemp[1:] + yt[2:-1] = ytemp[1:] + xt[-1] = 1e5 + yt[-1] = 0. + cspline = sp.interpolate.make_interp_spline(np.log10(xt), yt,k=1) + # get a spline interpolation of completeness. Should be removed from function! + + make_cdf_plot(ksbest,host_tau,xvals,yvals,"bestfit_ks_scat_cumulative.png",cspline) + + ############################################### TAU - likelihood analysis ################################################# + print("\n\n Max Likelihood Calculation\n") + xbest=[] + for ifunc in np.arange(NFUNC): + if False: + print("skipping log-likelihood test optimisation, remove this line to re-run") + xbest=[[0.36494363, 1.04819578 ],[-1.38936953e+00, -4.12755731e-05] ,[-1.38516893, 1.62526839]] + continue + # make values for interpolation + + if False: + plt.figure() + plt.plot(np.logspace(-5,5,101),cspline(np.linspace(-5,5,101))) + plt.plot(xvals,yvals) + plt.xscale("log") + plt.savefig("spline_example.png") + plt.close() + + args = (host_tau,cspline,ifunc) + x0=ARGS0[ifunc] + + result = sp.optimize.minimize(get_ll_stat,x0=x0,args=args,method = 'Nelder-Mead') + #psub1 = get_ks_stat(result.x,*args,plot=True) + xbest.append(result.x) + # llbest returns negative ll + llbest = get_ll_stat(result.x,host_tau,cspline,ifunc) * -1 + print("FUNCTION ",ifunc,",",FNAMES[ifunc]," Best-fitting log-likelihood parameters are ",result.x," with p-value ",1.-result.fun) + print(" , BIC is ",2*np.log(host_tau.size) - len(x0)*llbest) + + make_cdf_plot(xbest,host_tau,xvals,yvals,"bestfit_ll_scat_cumulative.png",cspline) + + + ######## does plot with all fits added ######## + + plt.sca(ax1) + NFRB=host_tau.size + handles=[l1[2],l3[0],l2[2]] + labels=["$\\tau_{\\rm host, 1\,GHz}$","Completeness","Corrected $\\tau_{\\rm host, 1\,GHz}$"] + for i in np.arange(NFUNC): + print("plotting function ",i," with xbest ",xbest[i]) + xs,ys = function_wrapper(i,xbest[i])#cspline=None): + plotnorm = NFRB * (np.log10(bins[1])-np.log10(bins[0])) + l=plt.plot(xs,ys*plotnorm,label=FNAMES[i]) + handles.append(l[0]) + labels.append(FNAMES[i]) + xs,ys = function_wrapper(i,xbest[i],cspline=cspline) + plt.plot(xs,ys*plotnorm,linestyle=":",color=plt.gca().lines[-1].get_color()) + plt.xscale("log") + plt.xlim(1e-2,1e3) + plt.legend() + + plt.legend(handles=handles,labels=labels,fontsize=6) #fontsize=12) + + + plt.savefig("tau_host_histogram_fits.png") + plt.close() + + ######## plot for paper ######## + + plt.sca(ax1v2) + NFRB=host_tau.size + handles=[l1v2[2],l3v2[0],l2v2[2]] + labels=["Observed","Completeness","Corrected"] + for i in [0,2,3]: + print("plotting function ",i," with xbest ",xbest[i]) + xs,ys = function_wrapper(i,xbest[i])#cspline=None): + plotnorm = NFRB * (np.log10(bins[1])-np.log10(bins[0])) + l=plt.plot(xs,ys*plotnorm,label=FNAMES[i]) + handles.append(l[0]) + labels.append(FNAMES[i]) + xs,ys = function_wrapper(i,xbest[i],cspline=cspline) + plt.plot(xs,ys*plotnorm,linestyle=":",color=plt.gca().lines[-1].get_color()) + plt.xscale("log") + plt.xlim(1e-2,1e3) + plt.legend() + plt.xlabel("$\\tau_{\\rm host, 1\,GHz}$ [ms]") + plt.ylabel("Number of FRBs") + plt.legend(handles=handles,labels=labels,fontsize=10) #fontsize=12) + + plt.tight_layout() + plt.savefig("paper_tau_host_histogram_fits.png") + plt.close() + + + ############################################# TAU - bayes factor ####################################### + # priors + # uses Galactic pulsars as example + # min sigma: from pulsar scattering relation factor of + # max sigma: size of distribution + # min mean/min/max: size of distribution + # NOTE: Bayes factor is P(data|model1)/P(data|model2) + + print("\n\n Bayes Factor Calculation\n") + for ifunc in np.arange(NFUNC): + if True: + print("skipping log-likelihood test optimisation, remove this line to re-run") + #FUNCTION 0 has likelihood sum 2.6815961887322472e-21 now compute Bayes factor! + #FUNCTION 1 has likelihood sum 1.4462729739641349e-15 now compute Bayes factor! + #FUNCTION 2 has likelihood sum 5.364450880196842e-16 now compute Bayes factor! + #FUNCTION 3 has likelihood sum 1.2416558548887762e-15 now compute Bayes factor! + #FUNCTION 4 has likelihood sum 7.269113417017299e-16 now compute Bayes factor! + #FUNCTION 5 has likelihood sum 6.248172055823414e-16 now compute Bayes factor! + #FUNCTION 6 has likelihood sum 6.339647378300337e-16 now compute Bayes factor! + continue + + llsum=0. + N1=100 + N2=100 + N3=100 + # integrates likelihood over prior space + for p1 in np.linspace(MIN_PRIOR[ifunc][0],MAX_PRIOR[ifunc][0],N1): + if len(ARGS0[ifunc])>1: + for p2 in np.linspace(MIN_PRIOR[ifunc][1],MAX_PRIOR[ifunc][1],N2): + + if (ifunc == 2 or ifunc == 4 or ifunc==5 or ifunc==6) and p2 <= p1: + # skip if max is less than min + continue + + if len(ARGS0[ifunc])>2: + for p3 in np.linspace(MIN_PRIOR[ifunc][2],MAX_PRIOR[ifunc][2],N3): + ll = get_ll_stat([p1,p2,p3],host_tau,cspline,ifunc,log_min=-5, log_max = 5, NTau=600) + if np.isfinite(ll): + llsum += 10.**-ll + else: + ll = get_ll_stat([p1,p2],host_tau,cspline,ifunc,log_min=-5, log_max = 5, NTau=600) + if np.isfinite(ll): + llsum += 10.**-ll + else: + ll = get_ll_stat([p1],host_tau,cspline,ifunc,log_min=-5, log_max = 5, NTau=600) + if np.isfinite(ll): + llsum += 10.**-ll + # normalisation + llsum /= N1 + if len(ARGS0[ifunc])>1: + llsum /= N2 + if len(ARGS0[ifunc])>2: + llsum /= N3 + if (ifunc == 2 or ifunc == 4 or ifunc==5 or ifunc==6): + llsum *= 2 #because the parameter space is actually half that calculated above + + print("FUNCTION ",ifunc,",",FNAMES[ifunc]," has likelihood sum ",llsum, " now compute Bayes factor!") + + + + + ###################################################################################################### + ############################################## WIDTH ############################################### + ###################################################################################################### + + print("\n\n\n\n######### WIDTH #########\n") + ######################################### Observed Histogram ############################################ + + plt.figure() + ax1=plt.gca() + plt.xscale("log") + plt.ylim(0,8) + l1 = plt.hist(wobs,bins=bins,label="Observed",alpha=0.5) + + # makes a function of completeness + wxvals,wyvals = make_completeness_plot(maxws) + + wi_comp = get_completeness(wobs,wxvals,wyvals) + + l2 = plt.hist(wobs,bins=bins,weights = 1./wi_comp,label="Observed",alpha=0.5) + + + ax2 = ax1.twinx() + l3 = ax2.plot(wxvals,wyvals,label="Completeness") + plt.ylim(0,1) + plt.xlim(5e-3,1e2) + plt.ylabel("Completeness") + plt.sca(ax1) + plt.legend(handles=[l1[2],l3[0],l2[2]],labels=["$w_i$","Completeness","Corrected $w_i$"],fontsize=12) + plt.xlabel("$w_i$ [ms]") + plt.ylabel("Number of FRBs") + plt.tight_layout() + plt.savefig("w_observed_histogram.png") + plt.close() + + ################################ 1 GHz Rest-frame Histogram ########################## + + plt.figure() + ax1=plt.gca() + plt.xscale("log") + plt.ylim(0,8) + l1 = plt.hist(host_w,bins=bins,label="Host",alpha=0.5) + + # makes a function of completeness + wxvals,wyvals = make_completeness_plot(host_maxw) + + # get completeness at points of measurement + w_comp = get_completeness(host_w,wxvals,wyvals) + + l2 = plt.hist(host_w,bins=bins,weights = 1./w_comp,label="Observed",alpha=0.5) + + + ax2 = ax1.twinx() + l3 = ax2.plot(wxvals,wyvals,label="Completeness") + + plt.ylim(0,1) + plt.xlim(5e-3,1e2) + plt.ylabel("Completeness") + plt.sca(ax1) + plt.legend(handles=[l1[2],l3[0],l2[2]],labels=["Observed","Completeness","Corrected"],fontsize=12) + plt.xlabel("$w_{\\rm host}$ [ms]") + plt.ylabel("Number of FRBs") + plt.tight_layout() + plt.savefig("w_host_histogram.png") + + # keeps open for plottimng later + + #### new plot, for paper - just a copy of the above #### + + plt.figure() + ax1v2=plt.gca() + plt.xscale("log") + plt.ylim(0,8) + l1v2 = plt.hist(host_w,bins=bins,label="Host",alpha=0.5) + + # makes a function of completeness + wxvals,wyvals = make_completeness_plot(host_maxw) + + # get completeness at points of measurement + w_comp = get_completeness(host_w,wxvals,wyvals) + + l2v2 = plt.hist(host_w,bins=bins,weights = 1./w_comp,label="Observed",alpha=0.5) + + + ax2v2 = ax1v2.twinx() + l3v2 = ax2.plot(wxvals,wyvals,label="Completeness") + + plt.ylim(0,1) + plt.xlim(1e-3,1e3) + plt.ylabel("Completeness") + plt.sca(ax1) + plt.legend(handles=[l1[2],l3[0],l2[2]],labels=["Observed","Completeness","Corrected"],fontsize=12) + plt.xlabel("$w_{\\rm host}$ [ms]") + plt.ylabel("Number of FRBs") + + + ####################### W - likelihood maximisation ################# + + ### makes temporary values for completeness + xtemp = wxvals[::2] + ytemp = wyvals[::2] + s = xtemp.size + xt = np.zeros([s+2]) + yt = np.zeros([s+2]) + xt[0] = xtemp[0] + yt[0] = ytemp[0] + xt[1] = 0.1 + yt[1] = 1. + xt[2:-1] = xtemp[1:] + yt[2:-1] = ytemp[1:] + xt[-1] = 1e5 + yt[-1] = 0. + cspline = sp.interpolate.make_interp_spline(np.log10(xt), yt,k=1) + + xbest=[] + + for ifunc in np.arange(NFUNC): + if False: + print("skipping log-likelihood test optimisation, remove this line to re-run") + continue + + # make values for interpolation + # completeness for w in host frame + + # get a spline interpolation of completeness. Should be removed from function! + if False: + plt.figure() + plt.plot(np.logspace(-5,5,101),cspline(np.linspace(-5,5,101))) + plt.plot(xvals,yvals) + plt.xscale("log") + plt.savefig("spline_example.png") + plt.close() + + args = (host_w,cspline,ifunc) + x0=WARGS0[ifunc] + result = sp.optimize.minimize(get_ll_stat,x0=x0,args=args,method = 'Nelder-Mead') + #psub1 = get_ks_stat(result.x,*args,plot=True) + xbest.append(result.x) + print("width FUNCTION ",ifunc,",",FNAMES[ifunc]," Best-fitting log-likelihood parameters are ",result.x," with p-value ",1.-result.fun) + #FUNCTION 0 Best-fitting parameters are [1.54420422 1.77315156] with p-value -14.098744320572287 + #FUNCTION 1 Best-fitting parameters are [1.54415779 1.77312642] with p-value -14.098744320522364 + #FUNCTION 2 Best-fitting parameters are [-2.59375 4.753125] with p-value -16.16738087050264 + + make_cdf_plot(xbest,host_w,wxvals,wyvals,"bestfit_ll_width_cumulative.png",cspline,width=True) + + + ### does plot with fits added ### + + plt.sca(ax1) + NFRB=host_tau.size + handles=[l1[2],l3[0],l2[2]] + labels=["$w_{\\rm host}$","Completeness","Corrected $w_{\\rm host}$"] + for i in np.arange(NFUNC): + print("plotting function ",i," with xbest ",xbest[i]) + xs,ys = function_wrapper(i,xbest[i])#cspline=None): + plotnorm = NFRB * (np.log10(bins[1])-np.log10(bins[0])) + l=plt.plot(xs,ys*plotnorm,label=FNAMES[i]) + handles.append(l[0]) + labels.append(FNAMES[i]) + xs,ys = function_wrapper(i,xbest[i],cspline=cspline) + plt.plot(xs,ys*plotnorm,linestyle=":",color=plt.gca().lines[-1].get_color()) + plt.xscale("log") + plt.xlim(1e-4,1e3) + plt.legend() + + plt.legend(handles=handles,labels=labels,fontsize=6) #fontsize=12) + + + plt.savefig("w_host_histogram_fits.png") + plt.close() + + #### for paper #### + + + plt.sca(ax1v2) + plt.ylim(0,12) + NFRB=host_w.size + handles=[l1v2[2],l3v2[0],l2v2[2]] + labels=["Observed","Completeness","Corrected"] + for i in [0,1,3]: + print("plotting function ",i," with xbest ",xbest[i]) + xs,ys = function_wrapper(i,xbest[i])#cspline=None): + plotnorm = NFRB * (np.log10(bins[1])-np.log10(bins[0])) + l=plt.plot(xs,ys*plotnorm,label=FNAMES[i]) + handles.append(l[0]) + labels.append(FNAMES[i]) + xs,ys = function_wrapper(i,xbest[i],cspline=cspline) + plt.plot(xs,ys*plotnorm,linestyle=":",color=plt.gca().lines[-1].get_color()) + plt.xscale("log") + + plt.xlim(5e-3,1e2) + plt.legend() + plt.xlabel("$w_{\\rm host}$ [ms]") + plt.ylabel("Number of FRBs") + plt.legend(handles=handles,labels=labels,fontsize=12) #fontsize=12) + + plt.tight_layout() + plt.savefig("paper_w_host_histogram_fits.png") + plt.close() + + ############################################# WIDTH - bayes factor ####################################### + # priors + # uses Galactic pulsars as example + # min sigma: from pulsar scattering relation factor of + # max sigma: size of distribution + # min mean/min/max: size of distribution + # NOTE: Bayes factor is P(data|model1)/P(data|model2) + + print("\n\n Bayes Factor Calculation\n") + for ifunc in np.arange(NFUNC): + if True: + print("skipping log-likelihood test optimisation, remove this line to re-run") + #FUNCTION 0 , lognormal has likelihood sum 4.287511548315901e-15 now compute Bayes factor! + #FUNCTION 1 , half-lognormal has likelihood sum 1.3919340351669428e-12 now compute Bayes factor! + #FUNCTION 2 , boxcar has likelihood sum 3.1091474793575766e-13 now compute Bayes factor! + #FUNCTION 3 , log constant has likelihood sum 6.41635848895752e-13 now compute Bayes factor! + #FUNCTION 4 , smooth boxcar has likelihood sum 8.176332379444694e-13 now compute Bayes factor! + #FUNCTION 5 , upper sb has likelihood sum 3.595417631395158e-13 now compute Bayes factor! + #FUNCTION 6 , lower sb has likelihood sum 6.806166849685634e-13 now compute Bayes factor! + continue + + llsum=0. + N1=100 + N2=100 + N3=100 + # integrates likelihood over prior space + for p1 in np.linspace(MIN_PRIOR[ifunc][0],MAX_PRIOR[ifunc][0],N1): + if len(ARGS0[ifunc])>1: + for p2 in np.linspace(MIN_PRIOR[ifunc][1],MAX_PRIOR[ifunc][1],N2): + + if (ifunc == 2 or ifunc == 4 or ifunc==5 or ifunc==6) and p2 <= p1: + # skip if max is less than min + continue + + if len(ARGS0[ifunc])>2: + for p3 in np.linspace(MIN_PRIOR[ifunc][2],MAX_PRIOR[ifunc][2],N3): + ll = get_ll_stat([p1,p2,p3],host_w,cspline,ifunc,log_min=-3, log_max = 3, NTau=300) + if np.isfinite(ll): + llsum += 10.**-ll + else: + ll = get_ll_stat([p1,p2],host_w,cspline,ifunc,log_min=-3, log_max = 3, NTau=300) + if np.isfinite(ll): + llsum += 10.**-ll + else: + ll = get_ll_stat([p1],host_w,cspline,ifunc,log_min=-3, log_max = 3, NTau=300) + if np.isfinite(ll): + llsum += 10.**-ll + # normalisation + llsum /= N1 + if len(ARGS0[ifunc])>1: + llsum /= N2 + if len(ARGS0[ifunc])>2: + llsum /= N3 + if (ifunc == 2 or ifunc == 4 or ifunc==5 or ifunc==6): + llsum *= 2 #because the parameter space is actually half that calculated above + + print("FUNCTION ",ifunc,",",FNAMES[ifunc]," has likelihood sum ",llsum, " now compute Bayes factor!") + + + +def plot_functions(): + """ + Plots example functions for the paper + """ + plt.figure() + + tmin=-1.5 + tmax=1.5 + tbar=0 + sigma=1 + + + plt.plot([10**tmin,10**tmin],[0,5.8],color="black",linestyle=":") + plt.text(10**tmin*0.5,-0.3,"$t_{\\rm min}$") + + plt.plot([10**tmax,10**tmax],[0,5.8],color="black",linestyle=":") + plt.text(10**tmax*0.5,-0.3,"$t_{\\rm max}$") + + + plt.text(10**(tmax+sigma)*0.25,1.5,"$\\sigma_{t}$") + plt.arrow(10**(tmax)*5,1.5,10**(tmax+sigma)-10**tmax*5,0,color="black",linestyle=":", + shape="full",head_width=0.3,head_length=100,length_includes_head=True) + plt.arrow(10**(tmax)*2.5,1.5,-10**(tmax)*1.5,0,color="black",linestyle=":", + shape="full",head_width=0.3,head_length=15,length_includes_head=True) + + + + #plt.plot([10**(tmax),10**(tmax+sigma)],[1.5,1.5],color="black",linestyle=":",) + plt.plot([10**(tmax+sigma),10**(tmax+sigma)],[1.5,3.2],color="black",linestyle=":") + + + plt.text(10**tbar*0.7,6,"$\\mu_{t}$") + plt.plot([10**tbar,10**tbar],[6.3,8],color="black",linestyle=":") + + plot_args=[[tbar,sigma],[tbar,sigma],[tmin,tmax],[tmin],[tmin,tmax,sigma], + [tmin,tmax,sigma],[tmin,tmax,sigma],[0.,1.]] + styles=["-","--",":","-."] + xlabels=[1e-3,1e1,1e-1,1e-1,1e-1,1e-1,1e-1] + + for ifunc in np.arange(NFUNC): + args = plot_args[ifunc] + xs,ys = function_wrapper(ifunc,args,xvals=None,logxmin=-6,logxmax=6,N=1000,cspline=None) + plt.plot(xs,7+ys/np.max(ys)-1.1*ifunc,label=FNAMES[ifunc],linestyle=styles[ifunc%4]) + plt.text(xlabels[ifunc],7.5-1.1*ifunc,FNAMES[ifunc],color=plt.gca().lines[-1].get_color()) + + + + + plt.xlim(1e-4,1e4) + plt.xlabel("t [ms]") + plt.xscale("log") + #plt.legend() + plt.ylabel("f(t)") + plt.tight_layout() + plt.savefig("example_functions.png") + plt.close() + +def get_ll_stat(args,tau_obs,cspline,ifunc,plot=False, log_min=-5, log_max = 5, NTau=600): + #(tau_obs, completeness_x, completeness_y, prediction, log_min=-3, log_max = 3, NTau=600): + """ + Returns a likelihood, given: + + Args: + args[1]: logmean, or logmin, of distribution + aargs[2]: sigma, or logmax, of distribution + tau_obs: observed values of scattering (in host frame) + cspline: spline giving completeness as function of log_tau + ifunc: which function to use for modelling (0,1, or 2) + """ + + # modifies the likelihood function by the completeness, and normalises + taus = np.logspace(log_min, log_max, NTau) + logtaus = np.linspace(log_min, log_max, NTau) + dlx = logtaus[1]-logtaus[0] + ctaus = cspline(logtaus) + + log_tau_obs = np.log10(tau_obs) + + #a1 = args[0] + #a2 = args[1] + # get expected distribution and hence cdf + A=1. + + ftaus = function_wrapper(ifunc,args,taus,cspline=cspline) + ptaus = function_wrapper(ifunc,args,tau_obs,cspline=cspline) + + # checks normalisation in log-space + #modified = ctaus*ftaus + #norm = np.sum(modified) * dlx #/NTau # sets the posterior to the correct normalisation in log space + #print("Norm is ",norm) + + # already includes nornalisation + ls = ptaus + #ls = ptaus * cspline(log_tau_obs ) / norm + #print("ls are ",ls) + lls = np.log10(ls) + ll = np.sum(lls) + + return -ll + + +def get_ks_stat(args,taus,cxvals,cyvals,ifunc,plot=False): + """ + Args: + a1: logmean, or logmin, of distribution + a2: sigma, or logmax, of distribution + taus: observed values of scattering + cxvals: x-values at which completeness is calculated + cyvals: y-values at which completeness is calculated + ifunc: which function to use for modelling (0,1, or 2) + + + """ + #a1 = args[0] + #a2 = args[1] + + # Step 1: create an expected cumulative distribution + xvals = np.logspace(-4,4,101) + + yvals=function_wrapper(ifunc,args,xvals) + + # modify the expected distribution by the completeness + modyvals = yvals*get_completeness(xvals,cxvals,cyvals) + + # make a cdf + cum_dist = np.cumsum(modyvals) + cum_dist /= cum_dist[-1] + + # make these global variables to pass to the cdf + global xc,yc + xc = np.log10(xvals) + yc = cum_dist + + # does this in log10 space + result = stats.ks_1samp(np.log10(taus),cdf,alternative='two-sided', method='auto') + + pvalue = result.pvalue + + if plot: + plt.figure() + plt.xlabel("$\\tau$") + plt.ylabel("CDF") + plt.xscale("log") + plt.plot(xvals,yc,label="Model") + + # makes a function of completeness + xvals,yvals = make_completeness_plot(taus,reverse=False) + plt.plot(xvals,yvals,label="Observed") + + plt.legend() + plt.tight_layout() + plt.show() + + # we are trying to minimise 1.-p, hence maximise p-value + return 1.-pvalue + + +def make_cdf_plot(args,taus,cxvals,cyvals,outfile,cspline,plot=False,width=False): + """ + Args: + args1: arguments to lognormal + args2: arguments to halflognormal + args3: arguments to loguniform + taus: observed values of scattering + cxvals: x-values at which completeness is calculated + cyvals: y-values at which completeness is calculated + + + """ + + # Base plotting commands + plt.figure() + plt.xlim(1e-3,1e3) + plt.ylim(0,1) + if width: + plt.xlabel("$w_{\\rm rest}$") + else: + plt.xlabel("$\\tau_{\\rm 1 GHz,rest}$") + plt.ylabel("CDF") + plt.xscale("log") + + # Step 1: create an expected cumulative distribution + xvals = np.logspace(-3,3,101) + + ### Loop over functions ### + + #def function_wrapper(ifunc,args,logxmin=-6,logxmax=6,N=1000,cspline=None): + # get expected distribution and hence cdf + A=1. + nfunc=len(args) + yvals=[] + cum_dists=[] + for i in np.arange(nfunc): + xs,ys=function_wrapper(i,args[i],logxmin=-3,logxmax=3,N=101,cspline=cspline) + #ys *= cspline(xs) + yvals.append(ys) + + cum_dist = np.cumsum(ys) + cum_dist /= cum_dist[-1] + + plt.plot(xs,cum_dist,label=FNAMES[i],linestyle="--") + + #yvals1 = lognormal(xvals,A,*args1) + #yvals2 = halflognormal(xvals,A,*args2) + #yvals3=logconstant(xvals,A,*args3) + + # modify the expected distribution by the completeness + #completeness = get_completeness(xvals,cxvals,cyvals) + #modyvals1 = yvals1*completeness + #modyvals2 = yvals2*completeness + #modyvals3 = yvals3*completeness + + # make a cdf + + #cum_dist1 = np.cumsum(modyvals1) + #cum_dist1 /= cum_dist1[-1] + + #cum_dist2 = np.cumsum(modyvals2) + #cum_dist2 /= cum_dist2[-1] + + #cum_dist3 = np.cumsum(modyvals3) + #cum_dist3 /= cum_dist3[-1] + + + #plt.plot(xvals,cum_dist1,label="Lognormal fit",linestyle="--") + #plt.plot(xvals,cum_dist2,label="Half-lognormal fit",linestyle=":") + #plt.plot(xvals,cum_dist3,label="Log-uniform fit",linestyle="-.") + + # makes a function of completeness + xvals,yvals = make_completeness_plot(taus,reverse=False) + plt.plot(xvals,yvals,label="Observed",color="black") + + plt.legend() + plt.tight_layout() + plt.savefig(outfile) + plt.close() + + return + +def cdf(xs): + """ + cumulative distribution function used for KS test + + This is governed by global variables because ks_1samp + doesn't allow other arguments ot be passed to this function + """ + global xc # x values for cumulative distribution + global yc # y values for cumnulative distribution + + cs = np.zeros([xs.size]) + # gets the values of the cdf at a given point + for i,x in enumerate(xs): + ix1 = np.where(x > xc)[0][-1] + ix2 = ix1+2 + dx = xc[ix2] - xc[ix1] + kx2 = (x - xc[ix1])/dx + kx1 = 1.-kx2 + c = kx1*yc[ix1] + kx2*yc[ix2] + cs[i] = c + return cs + + + +def get_completeness(x,cx,cy): + """ + Looks up the value of completeness for data values x, + where completeness at values cx is defined by cy + """ + N = x.size + comp = np.zeros([N]) + for i in np.arange(N): + # gets first value where completeness x is larger than observed + ix = np.where(cx > x[i])[0][0] + # gets completeness at that value + comp[i] = cy[ix] + return comp + +def make_completeness_plot(data,reverse=True): + """ + gets x,y values for a completeness plot + this is just like a cumulative distribution, + but reversed + """ + xvals,yvals = misc_functions.make_cum_dist(data) + + xvals = np.concatenate((np.array([1e-5]),xvals,np.array([1e5]))) + yvals = np.concatenate((np.array([0.]),yvals,np.array([1.]))) + + if reverse: + yvals = 1.-yvals + + + return xvals,yvals + +def find_max_tau(tauobs,w,snrobs,fbar,DMobs,tres,nu_res = 1.,\ + SNRthresh=9.,max_iwidth=12, plot=False): + """ + Finds the maximum value of tau which is probed by this FRB, + assuming some basic calculations. + + Args: + tauobs (float): scattering time [ms] + w (float): total width, including scattering, in ms + snrobs (float): signal to noise ratio at detection + fbar (float): mean frequecy in MHz + DM (float): dispersion measure, in pc/cm3 + tres (float): time resolution in ms + nu_res (float): frequency resolution in MHz + SNR_thresh (float): threshold signal-to-noise for detection + max_iwidth (float): Maximum multiple of tres for width search + plot (bool): If True, make a plot + Returns: + Maximum value of tau which is probed by this FRB + """ + + tauvals = np.linspace(tauobs,100.,1001) + + k_DM = 4.149 # leading constant: ms per pccc GHz^2 + dmsmear = 2*(nu_res/1.e3)*k_DM*DMobs/(fbar/1e3)**3 + + + # square of intrinsic widht. This *can* be negative! Obviously + # not physically, by we assume width scales as the below, and + # forcing it to be positive can artificially inflate the total width + intrinsic_width_squared = (w**2 - (tauobs/0.816)**2) + simulated_widths_squared = intrinsic_width_squared + (0.816*tauvals)**2 + simulated_widths = simulated_widths_squared**0.5 + + # maximum width that is less than our cutoff of 12 times the integration time + + + imaxtau1 = np.where(simulated_widths < tres*(max_iwidth+0.5))[0][-1] + + totalw_obs_square = (w**2 + dmsmear**2/3. + tres**2/3.) + totalw_square = (simulated_widths_squared + dmsmear**2/3. + tres**2/3.) + + SNR = snrobs * (totalw_obs_square/totalw_square)**0.25 + imaxtau2 = np.where(SNR > SNRthresh)[0][-1] + + imaxtau = min(imaxtau1,imaxtau2) + maxtau = tauvals[imaxtau] + + + if plot: + plt.figure() + plt.plot(tauvals,SNR, label="Simulated S/N") + plt.plot(tauvals,simulated_widths,label="scat + widths [ms]") + plt.xlabel("Scattering time [ms]") + plt.legend() + plt.tight_layout() + plt.show() + plt.close() + + return maxtau + +def find_max_w(tauobs,wtot,snrobs,fbar,DMobs,tres,nu_res = 1.,\ + SNRthresh=9.,max_iwidth=12, plot=False): + """ + Finds the maximum value of tau which is probed by this FRB, + assuming some basic calculations. + + Args: + tauobs (float): scattering time [ms] + w (float): total width, including scattering, in ms + snrobs (float): signal to noise ratio at detection + fbar (float): mean frequecy in MHz + DM (float): dispersion measure, in pc/cm3 + tres (float): time resolution in ms + nu_res (float): frequency resolution in MHz + SNR_thresh (float): threshold signal-to-noise for detection + max_iwidth (float): Maximum multiple of tres for width search + plot (bool): If True, make a plot + Returns: + Maximum value of wi which is probed by this FRB + """ + + + + k_DM = 4.149 # leading constant: ms per pccc GHz^2 + dmsmear = 2*(nu_res/1.e3)*k_DM*DMobs/(fbar/1e3)**3 + + + # square of intrinsic widht. This *can* be negative! Obviously + # not physically, by we assume width scales as the below, and + # forcing it to be positive can artificially inflate the total width + intrinsic_width_squared = (wtot**2 - (tauobs/0.816)**2) + + intrinsic_width_squared = max(0,intrinsic_width_squared) + intrinsic_width = intrinsic_width_squared**0.5 + + # simulated widths = begin by adding times, min is zero + simulated_intrinsic_widths = np.linspace(intrinsic_width, 100.,1001) # adds up to 100 ms to this + + # simulated total width square + simulated_widths_squared = simulated_intrinsic_widths**2 + (tauobs/0.816)**2 + simulated_widths = simulated_widths_squared**0.5 + + # maximum width that is less than our cutoff of 12 times the integration time + + imaxw1 = np.where(simulated_widths < tres*(max_iwidth+0.5))[0][-1] + + totalw_obs_square = (wtot**2 + dmsmear**2/3. + tres**2/3.) + totalw_square = (simulated_widths_squared + dmsmear**2/3. + tres**2/3.) + + SNR = snrobs * (totalw_obs_square/totalw_square)**0.25 + imaxw2 = np.where(SNR > SNRthresh)[0][-1] + + imaxw = min(imaxw1,imaxw2) + maxw = simulated_intrinsic_widths[imaxw] + + + if plot: + plt.figure() + plt.plot(simulated_intrinsic_widths,SNR, label="Simulated S/N") + plt.plot(simulated_intrinsic_widths,simulated_widths,label="scat + widths [ms]") + plt.xlabel("Intrinsic width [ms]") + plt.legend() + plt.tight_layout() + plt.show() + plt.close() + + return maxw + +def get_data(): + # extract relevant data. + # this is the error code + tresinfo = np.loadtxt("treslist.dat",dtype="str") names=tresinfo[:,0] @@ -29,8 +1100,7 @@ def main(): names[i] = names[i][0:8] # truncates the letter dataframe = pandas.read_csv("CRAFT_ICS_HTR_Catalogue1.csv") - for key in dataframe.keys(): - print(key) + ERR=9999. tns = dataframe.TNS @@ -41,6 +1111,8 @@ def main(): snr = dataframe.SNdet freq = dataframe.NUTau DM = dataframe.DM + + # check which FRBs do not have nay errors in all columns OK = getOK([tns,tauobs,w95,wsnr,z,snr,freq,DM]) tns = tns[OK] tauobs= tauobs[OK] @@ -51,32 +1123,29 @@ def main(): freq = freq[OK] DM = DM[OK] NFRB = len(OK) - tres = np.zeros([NFRB]) + tres = np.zeros([NFRB]) for i,name in enumerate(tns): j = np.where(name[0:8] == names)[0] if len(j) != 1: print("Cannot find tres info for FRB ",name) - tres[i] = float(tresinfo[j,1]) - - print(tres) - -def find_max_tau(wsnr,tauobs,DM,freq,snr,tres,nu_res=1.): - """ - - """ - - tauvals = np.linspace(0.1,100.,1001) - k_DM = 4.149 # leading constant: ms per pccc GHz^2 - dmsmear = 2*(nu_res/1.e3)*k_DM*DM_frb/(fbar/1e3)**3 - - - - totalw = (uw**2 + dm_smearing**2/3. + t_res**2/3.)**0.5 + exit() + + tres[i] = float(tresinfo[j[0],1]) + # cats everything to numpy arrays, because pands throws + # some bonkers errors I can't debug + tns = np.array(tns) + tauobs = np.array(tauobs) + w95 = np.array(w95) + wsnr = np.array(wsnr) + z = np.array(z) + snr = np.array(snr) + freq = np.array(freq) + DM = np.array(DM) + tres = np.array(tres) - print(OK) - exit() + return tns,tauobs,w95,wsnr,z,snr,freq,DM,tres diff --git a/papers/Scattering/run_MCMC_slurm_sfr.sh b/papers/Scattering/run_MCMC_slurm_sfr.sh new file mode 100755 index 0000000..a26646a --- /dev/null +++ b/papers/Scattering/run_MCMC_slurm_sfr.sh @@ -0,0 +1,34 @@ +#!/bin/bash +#SBATCH --job-name=fit_scattering_test +#SBATCH --ntasks=10 +#SBATCH --time=24:00:00 +#SBATCH --export=NONE +#SBATCH --mem-per-cpu=700MB + +# activate python environment +source /fred/oz313/cwjames/zdm/python_env/bin/activate + + +export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK + +####### ACTUAL RUN ####### +version=$SLURM_ARRAY_TASK_ID +pfile="MCMC_inputs/scat_w_sfr.json" +files="CRAFT_ICS_892 CRAFT_ICS_1300 CRAFT_ICS_1632" +opfile="MCMC_outputs/v2mcmc_lognormal_sfr_v${version}" +Pn=False +ptauw=True +steps=500 +walkers=20 + +Nz=100 +Ndm=100 +zmax=2 +dmmax=2000 + +script="../../zdm/scripts/MCMC/MCMC_wrap.py" + +runcommand="python $script -f $files --opfile=$opfile --pfile=$pfile --ptauw -s $steps -w $walkers --Nz=$Nz --Ndm=$Ndm --zmax=$zmax --dmmax=$dmmax" + +echo $runcommand +$runcommand diff --git a/papers/Scattering/visualise_mcmc.py b/papers/Scattering/visualise_mcmc.py index fbf9f07..3af70c3 100644 --- a/papers/Scattering/visualise_mcmc.py +++ b/papers/Scattering/visualise_mcmc.py @@ -43,19 +43,32 @@ # labels = ["sfr_n", "alpha", "lmean", "lsigma", "lEmax", "lEmin", "gamma", "H0", "DMhalo"] # labels = [r"$n$", r"$\alpha$", r"log$\mu$", r"log$\sigma$", r"log$E_{\mathrm{max}}$", r"log$E_{\mathrm{min}}$", r"$\gamma$", r"$H_0$", "DMhalo"] -labels = [r"$\mu_w$", r"$\sigma_w$", r"$\mu_\tau$", r"$\sigma_\tau$"] -half = False +half = False +sfr=False if half: filenames = ['MCMC_outputs/v2_mcmc_halflognormal','MCMC_outputs/v3_mcmc_halflognormal'] # this name gets added to all produced plots prefix="MCMC_Plots/halflognormal_" + labels = [r"$\mu_w$", r"$\sigma_w$", r"$\mu_\tau$", r"$\sigma_\tau$"] +if sfr: + filenames = ['MCMC_outputs/mcmc_lognormal_sfr_v1', + 'MCMC_outputs/mcmc_lognormal_sfr_v2', + 'MCMC_outputs/mcmc_lognormal_sfr_v3', + 'MCMC_outputs/mcmc_lognormal_sfr_v4'] + # this name gets added to all produced plots + labels = [r"$\mu_w$", r"$\sigma_w$", r"$\mu_\tau$", r"$\sigma_\tau$", r"$n_{\rm sfr}$"] + prefix="MCMC_Plots/sfr_halflognormal_" else: - filenames = ['MCMC_outputs/mcmc_lognormal_v1'] # turn off p(scat|w) - prefix = "MCMC_Plots/lognormal_v1" + filenames = ['MCMC_outputs/mcmc_lognormal_v1', + 'MCMC_outputs/mcmc_lognormal_v2', + 'MCMC_outputs/mcmc_lognormal_v3', + 'MCMC_outputs/mcmc_lognormal_v4'] # turn off p(scat|w) + labels = [r"$\mu_w$", r"$\sigma_w$", r"$\mu_\tau$", r"$\sigma_\tau$"] + prefix = "MCMC_Plots/lognormal" samples = [] @@ -147,7 +160,7 @@ corner.corner(final_sample,labels=labels, show_titles=True, titles=titles, fig=fig,title_kwargs={"fontsize": 15},label_kwargs={"fontsize": 15}, quantiles=[0.16,0.5,0.84], truths=truths); - +print(prefix+"cornerplot.png") plt.savefig(prefix+"cornerplot.png") exit() #### LEAVE HERE ###### diff --git a/zdm/MCMC.py b/zdm/MCMC.py index 52a82de..0e6a90a 100644 --- a/zdm/MCMC.py +++ b/zdm/MCMC.py @@ -174,7 +174,8 @@ def calc_log_posterior(param_vals, state, params, surveys_sep, Pn=False, pNreps= #============================================================================== def mcmc_runner(logpf, outfile, state, params, surveys, nwalkers=10, nsteps=100, nthreads=1, Pn=False, - pNreps=True, ptauw = False, log_halo=False, lin_host=False, ind_surveys=False, g0info=None): + pNreps=True, ptauw = False, log_halo=False, lin_host=False, ind_surveys=False, g0info=None, + reset=False): """ Handles the MCMC running. @@ -210,9 +211,16 @@ def mcmc_runner(logpf, outfile, state, params, surveys, nwalkers=10, nsteps=100, print(key + " priors: " + str(val['min']) + "," + str(val['max'])) starting_guesses = np.array(starting_guesses).T - + + # we only reset the backend if specifically requested. + # This means that walkers will continue from a previous iteration backend = emcee.backends.HDFBackend(outfile+'.h5') - backend.reset(nwalkers, ndim) + exists = os.path.isfile(outfile+'.h5') + if reset: + backend.reset(nwalkers, ndim) + if exists: + print("WARNING: output file exists, will be writing new run to old file") + exists = False # if resetting, ignore that a file exists start = time.time() @@ -224,10 +232,14 @@ def mcmc_runner(logpf, outfile, state, params, surveys, nwalkers=10, nsteps=100, with Pool() as pool: # could add mp.Pool(ntrheads=5) or Pool = None - sampler = emcee.EnsembleSampler(nwalkers, ndim, logpf, args=[state, params, surveys, Pn, pNreps, ptauw, log_halo, lin_host, ind_surveys, g0info], backend=backend, pool=pool) - sampler.run_mcmc(starting_guesses, nsteps, progress=True) + if exists: + # start from last saved position + sampler.run_mcmc(None, nsteps, progress=True) + else: + # start from new random guesses + sampler.run_mcmc(starting_guesses, nsteps, progress=True) end = time.time() print("Total time taken: " + str(end - start)) diff --git a/zdm/figures.py b/zdm/figures.py index fb8c44b..a8fc98d 100644 --- a/zdm/figures.py +++ b/zdm/figures.py @@ -16,6 +16,7 @@ def plot_grid( name="temp.pdf", label='$\\log_{10}p(DM_{\\rm EG},z)$', ylabel="${\\rm DM}_{\\rm EG}$ (pc cm$^{-3}$)", + logrange=4, project=False, conts=False, FRBZs=None, @@ -35,7 +36,9 @@ def plot_grid( pdmgz=None, save=True, othergrids=None, - othernames=None + othernames=None, + c_cmap = None, + cont_clrs = None ): """ Very complicated routine for plotting 2D zdm grids @@ -56,6 +59,7 @@ def plot_grid( name (str, optional): Outfile name label (str, optional): Colourbar label ylabel (str,optional): Label on y axis of plot + logrange(float,optional): range in logspace of the z axis (defaults to 4) project (bool, optional): Add projections of P(z) and P(DM) conts (bool, optional): create contours in probability p(dm|z), at fractional levels set by conts. Defaults to False. @@ -85,6 +89,8 @@ def plot_grid( Aconts othernames (list of names) [None]: list of names for original *and* other grid. Used only if othergrids is not None. Must be length of othergrids +1. + c_cmap (string): Name of colormap used to plot "Acont" contours + cont_clrs (float, np.ndarray): list of colors in colourmap to use for contours """ if H0 is None: H0 = cos.cosmo.H0 @@ -104,12 +110,18 @@ def plot_grid( if Aconts: linestyles = ['--', '-.', ':', '-'] - c_cmap = cmr.arctic + if c_cmap is None: + c_cmap = cmr.arctic + else: + c_cmap = plt.get_cmap(c_cmap) if othergrids is not None: n_conts = len(Aconts) + len(othergrids) else: n_conts = len(Aconts) - cont_clrs = c_cmap(np.linspace(0.2, 0.8, n_conts)) + if cont_clrs is None: + cont_clrs = c_cmap(np.linspace(0.2, 0.8, n_conts)) + else: + cont_clrs = c_cmap(cont_clrs) # Make dictionary for the contours if cont_dicts == None: @@ -467,7 +479,7 @@ def plot_grid( if log: themax = np.nanmax(zDMgrid) - themin = int(themax - 4) + themin = int(themax - logrange) themax = int(themax) plt.clim(themin, themax) diff --git a/zdm/iteration.py b/zdm/iteration.py index fd0d257..45b2e28 100644 --- a/zdm/iteration.py +++ b/zdm/iteration.py @@ -385,6 +385,8 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True, piws[bad1] = 1e-10 ptaus[bad2] = 1e-10 + pbars = 0.5*ptaus + 0.5*piws # take the mean of these two + llptw = np.sum(np.log10(ptaus)) llpiw = np.sum(np.log10(piws)) @@ -393,8 +395,9 @@ def calc_likelihoods_1D(grid,survey,doplot=False,norm=True,psnr=True, # p(iw|tau,w) = \delta(iw-(w**2 - tau**2)**0.5) # However, numerical differences will affect this # hence, we add half of eavh value here - llsum += 0.5*llpiw - llsum += 0.5*llptw + #llsum += 0.5*llpiw + #llsum += 0.5*llptw + llsum += np.sum(np.log10(pbars)) lllist.append(llptw) lllist.append(llpiw) @@ -811,7 +814,7 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal llsum = -1e10 # llsum -= np.log10(norm)*Zobs.size # once per event - lllist=[llsum] + lllist=[llsum] # pz,DM #### calculates zdm components p(DM),p(z|DM),p(z),p(DM|z) # does this by using previous results for p(z,DM) and @@ -937,6 +940,25 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal bad2 = np.where(ptaus==0)[0] piws[bad1] = 1e-10 ptaus[bad2] = 1e-10 + pbars = 0.5*piws + 0.5*ptaus + + + # TESTING - I have left this in here, because it creates + # a useful plot showing the relative probabilities + # and how well they match up. Ideally, should be 1-1 + + TestProbabilities=False + if TestProbabilities: + for i,pb in enumerate(pbars): + print(i,Tauobs[i],Iwobs[i],pb,piws[i],ptaus[i]) + plt.scatter(piws/Iwobs,ptaus/Tauobs) + plt.xscale("log") + plt.yscale("log") + plt.show() + exit() + + + llpbar = np.sum(np.log10(pbars)) llpiw = np.sum(np.log10(piws)) llptw = np.sum(np.log10(ptaus)) # while we calculate llpiw, we don't add it to the sum @@ -944,11 +966,13 @@ def calc_likelihoods_2D(grid,survey,doplot=False,norm=True,psnr=True,printit=Fal # p(iw|tau,w) = \delta(iw-(w**2 - tau**2)**0.5) # However, numerical differences will affect this # Hence, we ad half of each value - llsum += 0.5*llpiw - llsum += 0.5*llptw + #llsum += 0.5*llpiw + #llsum += 0.5*llptw # this was summing in logspace + llsum += llpbar # now summing in linear space lllist.append(llpiw) lllist.append(llptw) + ############ Calculates p(s | z,DM) ############# # i.e. the probability of observing an FRB diff --git a/zdm/misc_functions.py b/zdm/misc_functions.py index d748f12..d420843 100644 --- a/zdm/misc_functions.py +++ b/zdm/misc_functions.py @@ -27,8 +27,28 @@ from zdm import pcosmic from zdm import parameters - - +def make_cum_dist(data): + """ + Gets cumulative distribution of the data ready for plotting + + Args: + data (list or np.ndarray): data for the distribution + Returns: + xvals (np.ndarray): x values of cumulative distribution + yvals (np.ndarray): x values of cumulative distribution + """ + + ordered = np.sort(data) + NDAT = len(data) + xvals = np.zeros([NDAT*2]) + yvals = np.zeros([NDAT*2]) + for i in np.arange(NDAT): + yvals[2*i] = i/NDAT + yvals[2*i+1] = (i+1.)/NDAT + xvals[2*i] = ordered[i] + xvals[2*i+1] = ordered[i] + return xvals,yvals + def get_width_stats(s,g): """ gets the probability of detecting tau or w for diff --git a/zdm/scripts/Scattering/width_scat_dists.py b/zdm/scripts/Scattering/width_scat_dists.py index cc2a7b7..23d7dac 100644 --- a/zdm/scripts/Scattering/width_scat_dists.py +++ b/zdm/scripts/Scattering/width_scat_dists.py @@ -105,7 +105,8 @@ def main(): # width in units of dlogw # n1,n2 are probability "per bin". hence, to convert to a p(w) dlogw, we diving by the bin width in logw - n1 = np.sum(s.wplist) * s.dlogw # n1 is probability within the bin. n3-6 are probability dlogp + # z is z=0 bin + n1 = np.sum(s.wplist[:,0]) * s.dlogw # n1 is probability within the bin. n3-6 are probability dlogp n2 = np.sum(Nw) * s.dlogw n3 = np.sum(ptau) * logbinwidth @@ -128,7 +129,7 @@ def main(): plt.ylim(1e-2,1) plt.xlim(1e-2,1e2) - plt.plot(ws,s.wplist/n1,label="Total width (z=0)",linestyle="-") + plt.plot(ws,s.wplist[:,0]/n1,label="Total width (z=0)",linestyle="-") plt.plot(ws,Nw/n2,label="Detected total",linestyle=":",color=plt.gca().lines[-1].get_color()) plt.plot(10**s.internal_logwvals,ptau/n3,label="Scattering (z=0)",linestyle="-") diff --git a/zdm/survey.py b/zdm/survey.py index eae41dd..bcae81c 100644 --- a/zdm/survey.py +++ b/zdm/survey.py @@ -829,12 +829,13 @@ def process_survey_file(self,filename:str, # calculates intrinsic widths # Uses the model of James et al 2025 # assumes we have S/N maximising widths - tscale = 2. # scale scattering time to total width at +- 1 sigma + # if scattering dominates total width, expect tau = 0.816 w + tscale = 1.225 # scale scattering time to total width at +- 1 sigma TEMP = self.frbs['WIDTH'].values**2 - (tscale*self.frbs['TAU'].values)**2 self.OKTAU = np.where(self.frbs['TAU'].values != -1.)[0] # code for non-existent toolow = np.where(TEMP <= 0.) - TEMP[toolow] = 0.01**2 # 10 microsecond width. Really could be anything up to tau + TEMP[toolow] = 0.01*self.frbs['TAU'].values[toolow]**2 # 10% of scattering width iwidths = TEMP**0.5 # scale to SNR max width assuming Gaussian shape self.IWIDTHs = iwidths self.TAUs = self.frbs['TAU'].values @@ -1120,9 +1121,12 @@ def calc_relative_sensitivity(DM_frb,DM,w,fbar,t_res,nu_res,Nchan=336,max_idt=No if dsmear==True: # width is the total width measured_dm_smearing=2*(nu_res/1.e3)*k_DM*DM_frb/(fbar/1e3)**3 #smearing factor of FRB in the band - uw=w**2-measured_dm_smearing**2-t_res**2 # uses the quadrature model to calculate intrinsic width uw + if model == "StdDev": + uw = w**2 - dm_smearing**2/3. - t_res**2/3. + else: + uw = w**2-measured_dm_smearing**2-t_res**2 # uses the quadrature model to calculate intrinsic width uw if uw < 0: - uw=0 + uw=1e-2 # replace this with some fraction of minimum width? else: uw=uw**0.5 else: @@ -1383,7 +1387,7 @@ def quadrature_convolution(width_function, width_args, scat_function, scat_args, taufracs = taufracs/tnorm # plot some examples. This code is kept here for internal analysis purposes - if True: + if False: plt.figure() plt.plot(internal_logvals,ptau,label="Intrinsic p(tau)") plt.plot(internal_logvals,pw,label="Intrinsic p(w)") @@ -1429,11 +1433,17 @@ def lognormal(log10w, *args): result = norm * np.exp(-0.5 * ((log10w - logmean) / logsigma) ** 2) return result -def halflognormal(log10w, *args):#logmean,logsigma,minw,maxw,nbins): +def halflognormal(log10w, *args,logmax=3):#logmean,logsigma,minw,maxw,nbins): """ Generates a parameterised half-lognormal distribution. This acts as a lognormal in the lower half, but keeps a constant per-log-bin width in the upper half + There is nor formal way to normalise this function. + + It can also be verified that changing the normalisation of these functions + only changes the P(N) calculation, which should in any case be + separately optimised. Note however that this changes the interpretation + of the log-constant, which may be incorrect to within such a normalisation factor. Args: log10w: log base 10 of widths @@ -1444,7 +1454,7 @@ def halflognormal(log10w, *args):#logmean,logsigma,minw,maxw,nbins): """ logmean = args[0] logsigma = args[1] - norm = (2.*np.pi)**-0.5/logsigma + norm = (2.*np.pi)**-0.5/logsigma #Currently no normalisation if hasattr(log10w,"__len__"): large = np.where(log10w > logmean)[0] @@ -1456,6 +1466,14 @@ def halflognormal(log10w, *args):#logmean,logsigma,minw,maxw,nbins): else: modlogw = log10w result = lognormal(modlogw,logmean,logsigma) + if logmax is not None: + # normalises the distribution. We note that the lower half + # is correctly normalised to 0.5 via the lognormal function + # the upper half spans the range [logmax-logmean] at amplitude + # norm. Hence, the total integral is + # 0.5 + [logmax-logmean]*norm + result /= (0.5 + (logmax-logmean)*norm) + return result def constant(log10w,*args): From c04a78a72b29044a240377832bc4a52211b0b797 Mon Sep 17 00:00:00 2001 From: Clancy James Date: Tue, 9 Sep 2025 07:19:15 +0800 Subject: [PATCH 11/20] various minor mods, mostly to doc strings --- .../Scattering/MCMC_inputs/halflog_w_sfr.json | 112 +++++++++++++++++ .../Scattering/MCMC_inputs/scat_w_only.json | 3 +- .../MCMC_inputs/scat_w_only_halflog.json | 4 +- papers/Scattering/MCMC_inputs/scat_w_sfr.json | 3 +- zdm/figures.py | 5 + zdm/grid.py | 23 +++- zdm/misc_functions.py | 117 ++++++++++++++++++ zdm/parameters.py | 10 +- zdm/scripts/Scattering/width_scat_dists.py | 108 +++++++++++++++- zdm/survey.py | 73 ++++++++--- 10 files changed, 426 insertions(+), 32 deletions(-) create mode 100644 papers/Scattering/MCMC_inputs/halflog_w_sfr.json diff --git a/papers/Scattering/MCMC_inputs/halflog_w_sfr.json b/papers/Scattering/MCMC_inputs/halflog_w_sfr.json new file mode 100644 index 0000000..1c89d31 --- /dev/null +++ b/papers/Scattering/MCMC_inputs/halflog_w_sfr.json @@ -0,0 +1,112 @@ +{ + "mcmc": { + "parameter_order": ["Wlogmean", "Wlogsigma", "Slogmean", "Slogsigma","sfr_n"] + }, + "config": { + "luminosity_function": 2, + "alpha_method": 1, + "halo_method": 0, + "sigmaDMG": 0.0, + "sigmaHalo": 15.0, + "lmean": 2.02, + "lsigma": 0.46, + "sfr_n": 0.95, + "gamma": -1.16, + "alpha": 1.5, + "WNbins": 33, + "WNInternalBins": 100, + "ScatFunction": 2, + "WidthFunction": 2, + "Sfnorm": 1000 + }, + "Wlogmean": { + "DC": "width", + "min": -3, + "max": 3 + }, + "Wlogsigma": { + "DC": "width", + "min": 0.2, + "max": 4 + }, + "Slogmean": { + "DC": "scat", + "min": -3, + "max": 3 + }, + "Slogsigma": { + "DC": "scat", + "min": 1, + "max": 8 + }, + "logF": { + "DC": "cosmo", + "min": -2.0, + "max": 0.0 + }, + "lEmax": { + "DC": "energy", + "min": 40.35, + "max": 45.0 + }, + "lEmin": { + "DC": "energy", + "min": 35.0, + "max": 40.35 + }, + "DMhalo": { + "DC": "MW", + "min": 0.0, + "max": 100 + }, + "sigmaDMG": { + "DC": "MW", + "min": 0.1, + "max": 0.5 + }, + "H0": { + "DC": "cosmo", + "min": 35.0, + "max": 110.0 + }, + "alpha": { + "DC": "energy", + "min": 0.5, + "max": 2.5 + }, + "gamma": { + "DC": "energy", + "min": -3.0, + "max": 0.0 + }, + "sfr_n": { + "DC": "FRBdemo", + "min": -2.0, + "max": 4.0 + }, + "lmean": { + "DC": "host", + "min": 1.0, + "max": 3.0 + }, + "lsigma": { + "DC": "host", + "min": 0.1, + "max": 1.5 + }, + "lRmin": { + "DC": "rep", + "min": -2.0, + "max": 1.0 + }, + "lRmax": { + "DC": "rep", + "min": 0.0, + "max": 4.0 + }, + "Rgamma": { + "DC": "rep", + "min": -3.0, + "max": -0.5 + } +} diff --git a/papers/Scattering/MCMC_inputs/scat_w_only.json b/papers/Scattering/MCMC_inputs/scat_w_only.json index 0697b2c..8b0f3e7 100644 --- a/papers/Scattering/MCMC_inputs/scat_w_only.json +++ b/papers/Scattering/MCMC_inputs/scat_w_only.json @@ -14,8 +14,9 @@ "gamma": -1.16, "alpha": 1.5, "WNbins": 33, - "WNInternalBins": 1000, + "WNInternalBins": 100, "ScatFunction": 1, + "WidthFunction": 1, "Sfnorm": 1000 }, "Wlogmean": { diff --git a/papers/Scattering/MCMC_inputs/scat_w_only_halflog.json b/papers/Scattering/MCMC_inputs/scat_w_only_halflog.json index a336a6b..45c302f 100644 --- a/papers/Scattering/MCMC_inputs/scat_w_only_halflog.json +++ b/papers/Scattering/MCMC_inputs/scat_w_only_halflog.json @@ -15,7 +15,9 @@ "alpha": 1.5, "WNbins": 33, "WNInternalBins": 100, - "ScatFunction": 2 + "ScatFunction": 2, + "WidthFunction": 2, + "Sfnorm": 1000 }, "Wlogmean": { "DC": "width", diff --git a/papers/Scattering/MCMC_inputs/scat_w_sfr.json b/papers/Scattering/MCMC_inputs/scat_w_sfr.json index 6426c7d..57b954b 100644 --- a/papers/Scattering/MCMC_inputs/scat_w_sfr.json +++ b/papers/Scattering/MCMC_inputs/scat_w_sfr.json @@ -14,8 +14,9 @@ "gamma": -1.16, "alpha": 1.5, "WNbins": 33, - "WNInternalBins": 1000, + "WNInternalBins": 100, "ScatFunction": 1, + "WidthFunction": 1, "Sfnorm": 1000 }, "Wlogmean": { diff --git a/zdm/figures.py b/zdm/figures.py index a8fc98d..46622fd 100644 --- a/zdm/figures.py +++ b/zdm/figures.py @@ -1,3 +1,8 @@ +""" +This file contains functions designed to produce plots, +and associated helper functions. +""" + import numpy as np import matplotlib.pyplot as plt import matplotlib diff --git a/zdm/grid.py b/zdm/grid.py index b2e6bf4..8db2bce 100644 --- a/zdm/grid.py +++ b/zdm/grid.py @@ -50,6 +50,7 @@ def __init__(self, survey, state, zDMgrid, zvals, dmvals, smear_mask, wdist=None self.beam_b = survey.beam_b self.beam_o = survey.beam_o self.b_fractions = None + self.w_fractions = None # State self.state = state self.MCinit = False @@ -342,7 +343,13 @@ def calc_pdv(self, beam_b=None, beam_o=None): self.b_fractions = np.zeros( [self.zvals.size, self.dmvals.size, self.beam_b.size] ) - + + # we can now access the width information later on + if (self.w_fractions is None): + self.w_fractions = np.zeros( + [self.zvals.size, self.dmvals.size, self.eff_weights.size] + ) + # for some arbitrary reason, we treat the beamshape slightly differently... no need to keep an intermediate product! main_beam_b = self.beam_b @@ -355,6 +362,7 @@ def calc_pdv(self, beam_b=None, beam_o=None): for i, b in enumerate(main_beam_b): # if eff_weights is 2D (i.e., z-dependent) then w is a vector of length NZ + # It is a probability - the detection efficiency is encapusalted by thresh" for j, w in enumerate(self.eff_weights): # using log10 space conversion if self.use_log10: @@ -365,12 +373,17 @@ def calc_pdv(self, beam_b=None, beam_o=None): # the below is to ensure this works when w is a vector of length nz w = np.array(w) - self.b_fractions[:, :, i] += ( - self.beam_o[i] + # this array gives the relative probability of detecting an FRB at this point + # in the beam, with this particular width. We may not have space to store this + # as a 4D (w,b,z,DM) array, hence we store two 3D arrays + temp_wb = self.beam_o[i] \ * (self.array_cum_lf( thresh, Emin, Emax, self.state.energy.gamma, self.use_log10 ).T * w.T).T - ) + + self.b_fractions[:, :, i] += temp_wb + + self.w_fractions[:, :, j] += temp_wb # here, b-fractions are unweighted according to the value of b. self.fractions = np.sum( self.b_fractions, axis=2 @@ -1152,3 +1165,5 @@ def chk_upd_param(self, param: str, vparams: dict, update=False): self.state.update_param(param, vparams[param]) # return updated + + diff --git a/zdm/misc_functions.py b/zdm/misc_functions.py index d420843..ae7e464 100644 --- a/zdm/misc_functions.py +++ b/zdm/misc_functions.py @@ -27,6 +27,123 @@ from zdm import pcosmic from zdm import parameters + +def get_w_tau_dist(grid,norm=True): + """ + This function determins the total width, tau, and intrinsic width + distributions predicted by zDM. It returns histograms of each of these + parameters, both 1D hists of total rates, and 2D hists projected onto + the redshift axis. + + + Args: + grid: zdm grid object. Not yet implemented for a repeating grid + + Returns: + wvals [np.ndarray]: array of width values + pw [np.ndarray]: relative probabilities of observing those widths + zvals [np.ndarray: nz]: grid.zvals + pwz [np.ndarray: nw x nz]: probability of observing each width as a function of z + dmvals [np.ndarray: ndm]: grid.dmvals + pwdm [np.ndarray: nw x ndm] + """ + + state=grid.state # for shorter variable names + survey = grid.survey + Wmethod = survey.meta["WMETHOD"] + if Wmethod == 0: + print("WARNING: trivial width distribution: all 1ms") + widths = np.array([1]) + pw = np.array([1]) + pwz = np.full([grid.zvals.size,1],1.) + pwdm = np.full([grid.dmvals.size,1],1.) + return widths,pw,grid.zvals,pwz,grid.dmvals,pwdm + elif Wmethod == 4: + print("WARNING: trivial width distribution: all ",survey.wlist[0]," ms") + widths = survey.wlist + pw = np.array([1]) + pwz = np.full([grid.zvals.size,1],1.) + pwdm = np.full([grid.dmvals.size,1],1.) + return widths,pw,grid.zvals,pwz,grid.dmvals,pwdm + + # if we get to here, the results make sense + # wlist is the list of total widths parameterised by the survey. + # It is the distribution against which efficiencies are evaluated + widths = survey.wlist + + # accesses grid weight fraction, as function of z and DM. + # We now need to weight by z and DM + + pw = np.zeros([widths.size]) + pwz = np.zeros([widths.size,grid.nz]) + pwdm = np.zeros([widths.size,grid.ndm]) + for iw,w in enumerate(widths): + # weights by volume elements along z axis + this_pdv = np.multiply(grid.w_fractions[:,:,iw].T,grid.dV).T + + #print("Threshold at z=0, dm=0 to widths ",widths[iw],grid.thresholds[iw,0,0]) + iz=40 + idm=0 + #print("this pdv at z=0, dm=0 to widths ",widths[iw],this_pdv[iz,idm],grid.eff_weights[iw,iz]) + + # multiplies by p(DM) distribution for a given z + this_rate = grid.sfr_smear * this_pdv + + pwz[iw,:] = np.sum(this_rate,axis=1) + + pwdm[iw,:] = np.sum(this_rate,axis=0) + + pw[iw] = np.sum(pwz[iw,:]) + + if norm: + pw /= np.sum(pw) + + thenorm = np.sum(pwz,axis=1) # probability is zero for all w at this z + zero = np.where(thenorm == 0) + pwz = (pwz.T/np.sum(pwz,axis=1)).T + pwz[zero,:] = 0. # resets the nan components + + thenorm = np.sum(pwdm,axis=1) # probability is zero for all w at this z + zero = np.where(thenorm == 0) + pwdm = (pwdm.T/np.sum(pwdm,axis=1)).T + pwdm[zero,:] = 0. # resets the nan components + + # we now estimate p(tau) and p(iw) based on the + # this gives the probability of a gives intrinsic width iw + # gives z and total width iw + # hence, we need to multiply this by pwz and sum + if Wmethod == 3: + a,b,c = survey.pws.shape + # we sum the distribution of intrinsic width in z, intrinsic width, total width + # space, by multiplying by the relative expected number in z, total width space + # since this is normalised to unity for each z, total width when integrating over ii + piis = np.copy(survey.pws) + for ib in np.arange(b): + piis[:,ib,:] *= pwz.T + piisz = np.sum(piis,axis=2) # summing over total width + piis = np.sum(piisz,axis=0) + + ptaus = np.copy(survey.ptaus) + for ib in np.arange(b): + + ptaus[:,ib,:] *= pwz.T + ptausz = np.sum(ptaus,axis=2) # summing over total width + ptaus = np.sum(ptausz,axis=0) + + if norm: + ptaus /= np.sum(ptaus) + ptausz = (ptausz.T/np.sum(ptausz,axis=1)).T + + piis /= np.sum(piis) + piisz = (piisz.T/np.sum(piisz, axis=1)).T + + return widths,pw,grid.zvals,pwz,grid.dmvals,pwdm,\ + 10**survey.internal_logwvals,ptaus,ptausz,piis,piisz + else: + return widths,pw,grid.zvals,pwz,grid.dmvals,pwdm + + + def make_cum_dist(data): """ Gets cumulative distribution of the data ready for plotting diff --git a/zdm/parameters.py b/zdm/parameters.py index f5339a2..c6ad04a 100644 --- a/zdm/parameters.py +++ b/zdm/parameters.py @@ -230,7 +230,7 @@ class WidthParams(data_class.myDataClass): WidthFunction: int = field( default=1, metadata={ - "help": "ID of function to describe width distribution. 0: log-constant, 1:log-normal", + "help": "ID of function to describe width distribution. 0: log-constant, 1:log-normal, 2: half-lognormal", "unit": "", "Notation": "", }, @@ -260,7 +260,7 @@ class WidthParams(data_class.myDataClass): }, ) WNbins: int = field( - default=11, + default=12, metadata={"help": "Number of bins for FRB width distribution", "unit": ""}, ) WNInternalBins: int = field( @@ -272,11 +272,11 @@ class WidthParams(data_class.myDataClass): }, ) WMin: int = field( - default=0.1, + default=0.01, metadata={"help": "Minimum width value to model", "unit": "ms"}, ) WMax: int = field( - default=1000, + default=100, metadata={"help": "Maximum width value to model", "unit": "ms"}, ) @@ -317,7 +317,7 @@ class ScatParams(data_class.myDataClass): }, ) Sfnorm: float = field( - default=600, + default=1000, metadata={ "help": "Frequency of scattering width", "unit": "MHz", diff --git a/zdm/scripts/Scattering/width_scat_dists.py b/zdm/scripts/Scattering/width_scat_dists.py index 23d7dac..6532cf9 100644 --- a/zdm/scripts/Scattering/width_scat_dists.py +++ b/zdm/scripts/Scattering/width_scat_dists.py @@ -10,6 +10,10 @@ p(w|z,DM) is independent of the tau and w that contributed to it. +NOTE: all the calculations for p(w) assume distributions in + log10 space, i.e. p(log10_w), and all bins are log-spaced + bins, regardless of whether or not the calculations are + performed in linear or log space. """ @@ -40,7 +44,7 @@ def main(): """ - + See main description at top of file. """ # in case you wish to switch to another output directory @@ -60,13 +64,46 @@ def main(): zmax = 2. dmmax = 2000 - survey_dict = {"WMETHOD": 3} + survey_dict = {"WMETHOD": 3} # 2 is no z-depence, 3 is + state_dict = {} state_dict["scat"] = {} state_dict["scat"]["Sbackproject"] = True # turns on backprojection of tau and width for our model state_dict["width"] = {} - state_dict["width"]["WNInternalBins"] = 1000 # sets it to a small quantity + state_dict["width"]["WNInternalBins"] = 100 # sets it to a small quantity state_dict["width"]["WNbins"] = 33 # set to large number for this analysis + # We set these to smaller values in order to show more- + # detectable observational effects + + wset = 2 + if wset == 1: # set for analytic fits to lognormal + state_dict["width"]["WidthFunction"] = 1 # lognormal + state_dict["width"]["ScatFunction"] = 1 # lognormal + state_dict["width"]["Wlogmean"] = 0.215 + state_dict["width"]["Wlogsigma"] = 0.88 + state_dict["scat"]["Slogmean"] = 0.36 + state_dict["scat"]["Slogsigma"] = 1.05 + elif wset == 2: # set for analytic fits to halflognormal + state_dict["width"]["WidthFunction"] = 2 # halflognormal + state_dict["width"]["ScatFunction"] = 2 # halflognormal + state_dict["width"]["Wlogmean"] = 0.29 # 2 + state_dict["width"]["Wlogsigma"] = 0.64 #2 + state_dict["scat"]["Slogmean"] = -1.3 #1 + state_dict["scat"]["Slogsigma"] = 0.01 #4 + elif wset==3: # set for MCMC fits to lognormal + state_dict["width"]["WidthFunction"] = 1 # lognormal + state_dict["width"]["ScatFunction"] = 1 # lognormal + state_dict["width"]["Wlogmean"] = 2 # 2 + state_dict["width"]["Wlogsigma"] = 1.6 #2 + state_dict["scat"]["Slogmean"] = 1.4 #1 + state_dict["scat"]["Slogsigma"] = 4 #4 + elif wset==4: # set for MCMC fits to halflognormal + state_dict["width"]["WidthFunction"] = 2 # halflognormal + state_dict["width"]["ScatFunction"] = 2 # halflognormal + state_dict["width"]["Wlogmean"] = 1 # 2 + state_dict["width"]["Wlogsigma"] = 1.5 #2 + state_dict["scat"]["Slogmean"] = 1.5 #1 + state_dict["scat"]["Slogsigma"] = 4 #4 surveys, grids = loading.surveys_and_grids(survey_names = names,\ repeaters=repeaters, sdir=sdir,nz=70,ndm=140, @@ -179,5 +216,68 @@ def main(): plt.tight_layout() plt.savefig(opdir+"pw_dmdep.png") plt.close() - + + ###### Plot 4: modelled p(w) ##### + + widths,pw,zvals,pwz,dmvals,pwdm,finewidths,ptaus,ptausz,piis,piisz = mf.get_w_tau_dist(g) + dw = np.log10(widths[3]/widths[2]) + bins = 10.**(np.arange(31)*dw-2) + NFRB = s.frbs['WIDTH'].values.size + + plt.figure() + plt.plot(widths,pw*NFRB) + plt.hist(s.frbs['WIDTH'].values,bins=bins) + plt.xlabel("Total width w [ms]") + plt.ylabel("Probability, p(w)") + plt.xscale("log") + plt.yscale("log") + plt.ylim(0.1,10) + plt.tight_layout() + plt.savefig(opdir+"detected_pw.png") + plt.close() + + dfine = np.log10(finewidths[3]/finewidths[2]) + rel_width = dw/dfine + plt.figure() + plt.plot(finewidths,ptaus*rel_width*NFRB) + #plt.hist(s.TAUs,bins=bins) + plt.hist(s.TAUs,bins=s.wbins) + plt.plot(finewidths,s.ScatFunction(np.log10(finewidths),s.slogmean,s.slogsigma)) + plt.xlabel("Total width w [ms]") + plt.ylabel("Probability, p(w)") + plt.xscale("log") + plt.yscale("log") + plt.ylim(0.1,1000) + plt.tight_layout() + plt.savefig(opdir+"detected_ptau.png") + plt.close() + + plt.figure() + plt.plot(finewidths,piis*rel_width*NFRB) + plt.hist(s.IWIDTHs,bins=bins) + plt.xlabel("Total width w [ms]") + plt.ylabel("Probability, p(w)") + plt.xscale("log") + plt.yscale("log") + plt.ylim(0.1,10) + plt.tight_layout() + plt.savefig(opdir+"detected_piw.png") + plt.close() + + + exit() + # The below code shows how to plot p(tau|w) distributions for particlar widths + # and redshifts. + plt.figure() + iw=10 + for iz in np.arange(6): + z = g.zvals[iz] + w = (s.wbins[iw]*s.wbins[iw+1])**0.5 + plt.plot(s.internal_logwvals,s.ptaus[iz,:,iw]/np.max(s.ptaus[iz,:,iw]), + label="total width "+str(w)[0:5]+" at z="+str(z)[0:3]) + plt.legend() + plt.ylim(1e-3,10) + plt.yscale("log") + plt.show() + main() diff --git a/zdm/survey.py b/zdm/survey.py index bcae81c..02f864e 100644 --- a/zdm/survey.py +++ b/zdm/survey.py @@ -160,15 +160,16 @@ def init_widths(self,state=None): ###### calculate width bins. We fix these here ###### # Unless w and tau are explicitly being fit, it is not actually necessary - # to have constant bin values over z a d DM - # ensures the first bin begins at 0 + # to have constant bin values over z and DM. But best to do so! + # Here, wbins are the bin edges, w list the midpoint values used for calculations + # Nbins describes the number of bins, so Nedges is Nbins+1 wbins = np.zeros([self.NWbins+1]) if self.NWbins > 1: - wbins[0] = 1.e-10 # set to a tiny value, to ensure we capture all small widths - wbins[1:] = np.logspace(np.log10(self.WMin),np.log10(self.WMax),self.NWbins) + wbins = np.logspace(np.log10(self.WMin),np.log10(self.WMax),self.NWbins+1) dlogw = np.log10(wbins[2]/wbins[1]) + #wbins[0] = wbins[1]-dlogw # no longer tint: 1.e-10 # set to a tiny value, to ensure we capture all small widths # offsets the mean by half the log-spacing for each - wlist = np.logspace(np.log10(self.WMin)-dlogw/2.,np.log10(self.WMax)-dlogw/2.,self.NWbins) + wlist = np.logspace(np.log10(self.WMin)+dlogw/2.,np.log10(self.WMax)-dlogw/2.,self.NWbins) else: wbins[0] = np.log10(self.WMin) wbins[1] = np.log10(self.WMax) @@ -179,9 +180,10 @@ def init_widths(self,state=None): self.dlogw = dlogw ####### generates internal width values of numerical calculation purposes ##### - minval = np.min([self.wlogmean - self.maxsigma*self.wlogsigma, - self.slogmean - self.maxsigma*self.slogsigma, - np.log10(self.WMin)]) + #minval = np.min([self.wlogmean - self.maxsigma*self.wlogsigma, + # self.slogmean - self.maxsigma*self.slogsigma, + # np.log10(self.WMin)]) + minval = np.log10(self.WMin) maxval = np.log10(self.WMax) # I haven't decided yet where to put the value of 1000 internal bins # in terms of parameters. @@ -228,7 +230,27 @@ def do_efficiencies(self): model=self.meta['WBIAS'], edir=self.edir, iz=None) def make_widths(self,iz=None): """ - Used to be an exterior method, now interior + This routine calculates width distributions of FRBs, which + is required to estimate detection efficiency, and (potentially) + calculate p(w,tau). + + The functionality depends on self.meta['WMETHOD'], which + is set via + + WMETHOD [int: 0-4]: + 0: No distribution - calculations performed at total width = 1ms + 1: FRB widths are a simple lognormal of self.wlogmean, self.wlogsigma + histogrammed into self.wbins. Describes widths as "total width", + i.e. intrinsic plus scattering. + 2: FRB widths convolved with frequency-dependent scattering distribution. + Calculates lognormal intrinsic width distribution, and convolves + this with a lognormal scattering distribution. NOTE: if wmethod=2, + iz must be called with iz=None. + 3: As above, but allows for redshift dependence of both intrinsic + width and scattering. Distributions both scale width redshift: + width with 1+z, scattering with (1+z)^-3 + 4: Use a specific width of a particular FRB. Used for detailed calcs + for individual FRB-by-FRB analyses. """ if self.meta['WMETHOD'] == 0: @@ -250,6 +272,10 @@ def make_widths(self,iz=None): #gets cumulative hist and bin edges if iz is not None: + if self.meta['WMETHOD'] == 2: + print("Trying to calculate redshift dependence for redshift ",\ + "independent WMETHOD=2. Internally inconsistent.") + exit() z = self.zvals[iz] else: z=0. @@ -284,12 +310,12 @@ def make_widths(self,iz=None): elif self.meta['WMETHOD'] == 4: # use specific width of FRB. This requires there to be only a single FRB in the survey if s.meta['NFRB'] != 1: - raise ValueError("If width method in make_widths is 3 only one FRB should be specified in the survey but ", str(s.meta['NFRB']), " FRBs were specified") + raise ValueError("If width method in make_widths is 4 only one FRB should be specified in the survey but ", str(s.meta['NFRB']), " FRBs were specified") else: self.wplist[0] = 1. self.wlist[0] = s.frbs['WIDTH'][0] else: - raise ValueError("Width method in make_widths must be 0, 1 or 2, not ",width_method) + raise ValueError("Width method in make_widths must be 0-4, not ",width_method) def init_repeaters(self): @@ -436,7 +462,9 @@ def get_w_coeffs(self,wlist): # convert to log-widths - the bins are in log10 space logwlist = np.log10(wlist) - kws=(logwlist-np.log10(self.WMin))/self.dlogw + 0.5 + # the below assumes that + kws=(logwlist-np.log10(self.WMin))/self.dlogw # now will assume it begins at Wmin+dlogw + 0.5 + # forces any low values to zero Bin0 = np.where(kws < 0.)[0] kws[Bin0] = 0. @@ -1327,16 +1355,19 @@ def quadrature_convolution(width_function, width_args, scat_function, scat_args, pw = width_function(internal_logvals, *width_args)*logbinwidth ptau = scat_function(internal_logvals, *scat_args)*logbinwidth - # adds extra bits onto the lowest bin. Assumes exp(-10) is small enough! + + # adds extra bits onto the lowest bin. Does this by integrating in + # log space. Assumes exp(-10) is small enough! lowest = internal_logvals[0] - logbinwidth/2. extrapw,err = quad(width_function,lowest-10,lowest,args=width_args) extraptau,err = quad(scat_function,lowest-10,lowest,args=scat_args) + pw[0] += extrapw ptau[0] += extraptau linvals = 10**internal_logvals - # calculate widths - all done in linear domain + # calculate total widths - all done in linear domain Nbins = bins.size-1 hist = np.zeros([Nbins]) for i,x1 in enumerate(linvals): @@ -1358,12 +1389,17 @@ def quadrature_convolution(width_function, width_args, scat_function, scat_args, # arrays have dimensions(Nw,Ntotal_width) so for each total # width, we get the probability for i,x1 in enumerate(linvals): + # total widths corresponding to linvals for tau/iw x1 totalwidths = (x1**2 + linvals**2)**0.5 + # ptau is the probability of achieving linvals, so combined + # probability is pw[i]*ptau probs = pw[i]*ptau h,b = np.histogram(totalwidths,bins=bins,weights=probs) wfracs[i,:] = h + # pw is the probability of achieving linvals, so combined + # probability is ptau[i] * pw probs = ptau[i]*pw h,b = np.histogram(totalwidths,bins=bins,weights=probs) taufracs[i,:] = h @@ -1465,7 +1501,9 @@ def halflognormal(log10w, *args,logmax=3):#logmean,logsigma,minw,maxw,nbins): modlogw = logmean else: modlogw = log10w + result = lognormal(modlogw,logmean,logsigma) + if logmax is not None: # normalises the distribution. We note that the lower half # is correctly normalised to 0.5 via the lognormal function @@ -1490,7 +1528,10 @@ def constant(log10w,*args): Returns: result: p(logw) d logw """ - nvals = log10w.size - result = np.fill([nvals],1.) + if hasattr(log10w,"__len__"): + nvals = log10w.size + result = np.full([nvals],1.) + else: + result = np.array([1]) return result From bb37cdc8e30a23c131385e4cd6c02066bdcdd66b Mon Sep 17 00:00:00 2001 From: Clancy James Date: Fri, 19 Sep 2025 10:23:21 +0800 Subject: [PATCH 12/20] Final files corresponding to circulated scattering paper --- .../Scattering/MCMC_inputs/halflog_w_sfr.json | 4 +- papers/Scattering/MCMC_inputs/scat_w_sfr.json | 4 +- ...nparametric.py => fit_scattering_width.py} | 211 ++++++--- papers/Scattering/run_MCMC_slurm.sh | 34 ++ papers/Scattering/run_MCMC_slurm_halflog.sh | 34 ++ papers/Scattering/run_MCMC_slurm_hl_sfr.sh | 34 ++ papers/Scattering/run_MCMC_slurm_sfr.sh | 6 +- papers/Scattering/visualise_mcmc.py | 426 ++++-------------- zdm/MCMC_analysis.py | 17 +- 9 files changed, 351 insertions(+), 419 deletions(-) rename papers/Scattering/{nonparametric.py => fit_scattering_width.py} (87%) create mode 100755 papers/Scattering/run_MCMC_slurm.sh create mode 100755 papers/Scattering/run_MCMC_slurm_halflog.sh create mode 100755 papers/Scattering/run_MCMC_slurm_hl_sfr.sh diff --git a/papers/Scattering/MCMC_inputs/halflog_w_sfr.json b/papers/Scattering/MCMC_inputs/halflog_w_sfr.json index 1c89d31..8af76e3 100644 --- a/papers/Scattering/MCMC_inputs/halflog_w_sfr.json +++ b/papers/Scattering/MCMC_inputs/halflog_w_sfr.json @@ -32,11 +32,11 @@ "Slogmean": { "DC": "scat", "min": -3, - "max": 3 + "max": 6 }, "Slogsigma": { "DC": "scat", - "min": 1, + "min": 0.1, "max": 8 }, "logF": { diff --git a/papers/Scattering/MCMC_inputs/scat_w_sfr.json b/papers/Scattering/MCMC_inputs/scat_w_sfr.json index 57b954b..c3600d7 100644 --- a/papers/Scattering/MCMC_inputs/scat_w_sfr.json +++ b/papers/Scattering/MCMC_inputs/scat_w_sfr.json @@ -32,11 +32,11 @@ "Slogmean": { "DC": "scat", "min": -3, - "max": 3 + "max": 6 }, "Slogsigma": { "DC": "scat", - "min": 1, + "min": 0.1, "max": 8 }, "logF": { diff --git a/papers/Scattering/nonparametric.py b/papers/Scattering/fit_scattering_width.py similarity index 87% rename from papers/Scattering/nonparametric.py rename to papers/Scattering/fit_scattering_width.py index f757841..2a7a3bc 100644 --- a/papers/Scattering/nonparametric.py +++ b/papers/Scattering/fit_scattering_width.py @@ -1,3 +1,31 @@ +""" +This routine fits scattering and width distributions to CRAFT FRBs. + +It loads data from the HTR and ICS survey papers, and performs the following +analysis: + +- Calculation of completeness +- Best-fit using the KS-test +- Best-fit using maximum likelihood test +- Calculation of Bayes factors by integrating + over distributions of priors + +For each, we fit a range of functions defined at global level (yes, it's lazy!) +These are indexed according to the routine "function_wrapper". + +Plots that get generated, for each of tau and scattering, are: +- histograms in observer frame +- histograms in host frame showing best fit functions +- histograms in host frames showing only best fits relevant to paper +- cumulative distributions with best fit KS functions +- cumulative distributions with best-fit likelihood functions +- spline fits to completeness + +A scatter vs width plot +A plot showing function examples +""" + + import numpy as np from matplotlib import pyplot as plt from zdm import misc_functions @@ -18,7 +46,11 @@ 'size' : defaultsize} matplotlib.rc('font', **font) -# number of functions implemented +################################################################################################### +######### These define the kinds of fitting functions, and their initial fit first guesses ######## +################################################################################################### + +# number of functions implemented, function names, and initial guesses NFUNC=7 FNAMES=["lognormal","half-lognormal","boxcar","log constant","smooth boxcar", "upper sb", "lower sb","hln"] ARGS0=[[0.,1.],[0.,1.],[-2,2],[-2],[-1.,1,1.],[-2.,2,1.],[-2.,2,1.],[0.,1.]] @@ -33,6 +65,8 @@ log_max_sigma = np.log10(max_sigma) log_min_scat = np.log10(min_scat) log_max_scat = np.log10(max_scat) + +# priors for Bayes factors MIN_PRIOR = [ [log_min_scat,log_min_sigma], [log_min_scat,log_min_sigma], @@ -55,11 +89,23 @@ [log_max_scat,log_max_sigma] ] +# CHIME values +# raw were +# scaled to 1GHz and log10 are +# $(\mu_w,\sigma_w) = (\log_{10} 1.0 {\rm ms},0.42)$ +# $(\mu_\tau,\sigma_\tau) = (\log_{10} 0.262 {\rm ms},0.75) +CHIME_muw = 0. +CHIME_sw = 0.42 +CHIME_mut = 0.3 +CHIME_st = 0.75 -def main(): +def main(outdir="Fitting_Outputs/"): # does k-s test to scattering distribution - plot_functions() + if not os.path.exists(outdir): + os.mkdir(outdir) + + plot_functions(outdir=outdir) tns,tauobs,w95,wsnr,z,snr,freq,DM,tres = get_data() @@ -85,7 +131,7 @@ def main(): y=slope*x plt.plot(x,y,linestyle="--",color="black") plt.tight_layout() - plt.savefig("scatter_w_tau.png") + plt.savefig(outdir+"scatter_w_tau.png") plt.close() NFRB = snr.size @@ -146,11 +192,12 @@ def main(): plt.xlim(1e-2,1e3) plt.ylabel("Completeness") plt.sca(ax1) - plt.legend(handles=[l1[2],l3[0],l2[2]],labels=["$\\tau_{\\rm obs}$","Completeness","Corrected $\\tau_{\\rm obs}$"],fontsize=12) - plt.xlabel("$\\tau$ [ms]") + plt.legend(handles=[l1[2],l3[0],l2[2]],labels=["Observed","Completeness","Corrected"],fontsize=12) + plt.xlabel("$\\tau_{\\rm obs}$ [ms]") plt.ylabel("Number of FRBs") + plt.text(2e-3,8,"(a)",fontsize=18) plt.tight_layout() - plt.savefig("tau_observed_histogram.png") + plt.savefig(outdir+"tau_observed_histogram.png") plt.close() @@ -176,6 +223,7 @@ def main(): ax2 = ax1.twinx() l3 = ax2.plot(xvals,yvals,label="Completeness") + use_this_color = l3[0].get_color() plt.ylim(0,1) plt.xlim(1e-2) @@ -184,8 +232,10 @@ def main(): plt.legend(handles=[l1[2],l3[0],l2[2]],labels=["Observed","Completeness","Corrected"],fontsize=12) plt.xlabel("$\\tau_{\\rm host, 1\,GHz}$ [ms]") plt.ylabel("Number of FRBs") + + plt.text(2e-3,8,"(b)",fontsize=18) plt.tight_layout() - plt.savefig("tau_host_histogram.png") + plt.savefig(outdir+"tau_host_histogram.png") #### creates a copy of the above, for paper purposes @@ -215,24 +265,16 @@ def main(): plt.sca(ax1) plt.xlabel("$\\tau_{\\rm host, 1\,GHz}$ [ms]") plt.ylabel("Number of FRBs") - - # keeps open for later plotting + # keeps open for later plotting - don't close this here ####################################### TAU - CDF and fitting ################################ - print("\n\n KS test evaluation \n") + print("\n\n KS test evaluation for tau \n") # amplitude, mean, and std dev of true distribution - ifunc=0 - args = (host_tau,xvals,yvals,ifunc) - #get_ks_stat(-1,1,*args) - ksbest = [] - # begins minimisation + # begins minimisation for KS statistic for ifunc in np.arange(NFUNC): - if False: - print("skipping K-S test optimisation, remove this line to re-run") - continue args = (host_tau,xvals,yvals,ifunc) x0=ARGS0[ifunc] @@ -259,26 +301,22 @@ def main(): cspline = sp.interpolate.make_interp_spline(np.log10(xt), yt,k=1) # get a spline interpolation of completeness. Should be removed from function! - make_cdf_plot(ksbest,host_tau,xvals,yvals,"bestfit_ks_scat_cumulative.png",cspline) + # do a test plot of the spline + if True: + plt.figure() + plt.plot(np.logspace(-5,5,101),cspline(np.linspace(-5,5,101))) + plt.plot(xvals,yvals) + plt.xscale("log") + plt.savefig(outdir+"tau_spline_example.png") + plt.close() + + # make a cdf plot of the best fits + make_cdf_plot(ksbest,host_tau,xvals,yvals,outdir+"bestfit_ks_scat_cumulative.png",cspline) ############################################### TAU - likelihood analysis ################################################# - print("\n\n Max Likelihood Calculation\n") + print("\n\n Max Likelihood Calculation for tau\n") xbest=[] for ifunc in np.arange(NFUNC): - if False: - print("skipping log-likelihood test optimisation, remove this line to re-run") - xbest=[[0.36494363, 1.04819578 ],[-1.38936953e+00, -4.12755731e-05] ,[-1.38516893, 1.62526839]] - continue - # make values for interpolation - - if False: - plt.figure() - plt.plot(np.logspace(-5,5,101),cspline(np.linspace(-5,5,101))) - plt.plot(xvals,yvals) - plt.xscale("log") - plt.savefig("spline_example.png") - plt.close() - args = (host_tau,cspline,ifunc) x0=ARGS0[ifunc] @@ -287,10 +325,16 @@ def main(): xbest.append(result.x) # llbest returns negative ll llbest = get_ll_stat(result.x,host_tau,cspline,ifunc) * -1 + + print("FUNCTION ",ifunc,",",FNAMES[ifunc]," Best-fitting log-likelihood parameters are ",result.x," with p-value ",1.-result.fun) print(" , BIC is ",2*np.log(host_tau.size) - len(x0)*llbest) + if ifunc == 0: + llCHIME = get_ll_stat([CHIME_mut,CHIME_st],host_tau,cspline,ifunc) * -1 + print("Compare with CHIME ",llCHIME) - make_cdf_plot(xbest,host_tau,xvals,yvals,"bestfit_ll_scat_cumulative.png",cspline) + + make_cdf_plot(xbest,host_tau,xvals,yvals,outdir+"bestfit_ll_scat_cumulative.png",cspline) ######## does plot with all fits added ######## @@ -298,7 +342,7 @@ def main(): plt.sca(ax1) NFRB=host_tau.size handles=[l1[2],l3[0],l2[2]] - labels=["$\\tau_{\\rm host, 1\,GHz}$","Completeness","Corrected $\\tau_{\\rm host, 1\,GHz}$"] + labels=["Observed","Completeness","Corrected$"] for i in np.arange(NFUNC): print("plotting function ",i," with xbest ",xbest[i]) xs,ys = function_wrapper(i,xbest[i])#cspline=None): @@ -315,7 +359,7 @@ def main(): plt.legend(handles=handles,labels=labels,fontsize=6) #fontsize=12) - plt.savefig("tau_host_histogram_fits.png") + plt.savefig(outdir+"tau_host_histogram_fits.png") plt.close() ######## plot for paper ######## @@ -335,13 +379,15 @@ def main(): plt.plot(xs,ys*plotnorm,linestyle=":",color=plt.gca().lines[-1].get_color()) plt.xscale("log") plt.xlim(1e-2,1e3) + + plt.text(1e-3,8,"(b)",fontsize=18) plt.legend() plt.xlabel("$\\tau_{\\rm host, 1\,GHz}$ [ms]") plt.ylabel("Number of FRBs") plt.legend(handles=handles,labels=labels,fontsize=10) #fontsize=12) plt.tight_layout() - plt.savefig("paper_tau_host_histogram_fits.png") + plt.savefig(outdir+"paper_tau_host_histogram_fits.png") plt.close() @@ -356,7 +402,7 @@ def main(): print("\n\n Bayes Factor Calculation\n") for ifunc in np.arange(NFUNC): if True: - print("skipping log-likelihood test optimisation, remove this line to re-run") + print("skipping Bayes factor calculation for Tau, remove this line to re-run") #FUNCTION 0 has likelihood sum 2.6815961887322472e-21 now compute Bayes factor! #FUNCTION 1 has likelihood sum 1.4462729739641349e-15 now compute Bayes factor! #FUNCTION 2 has likelihood sum 5.364450880196842e-16 now compute Bayes factor! @@ -433,11 +479,13 @@ def main(): plt.xlim(5e-3,1e2) plt.ylabel("Completeness") plt.sca(ax1) - plt.legend(handles=[l1[2],l3[0],l2[2]],labels=["$w_i$","Completeness","Corrected $w_i$"],fontsize=12) + plt.legend(handles=[l1[2],l3[0],l2[2]],labels=["Observed","Completeness","Corrected $w_i$"],fontsize=12) plt.xlabel("$w_i$ [ms]") plt.ylabel("Number of FRBs") + + plt.text(1e-3,8,"(a)",fontsize=18) plt.tight_layout() - plt.savefig("w_observed_histogram.png") + plt.savefig(outdir+"w_observed_histogram.png") plt.close() ################################ 1 GHz Rest-frame Histogram ########################## @@ -468,18 +516,18 @@ def main(): plt.xlabel("$w_{\\rm host}$ [ms]") plt.ylabel("Number of FRBs") plt.tight_layout() - plt.savefig("w_host_histogram.png") + plt.savefig(outdir+"w_host_histogram.png") - # keeps open for plottimng later + # keeps open for plotting later #### new plot, for paper - just a copy of the above #### plt.figure() - ax1v2=plt.gca() plt.xscale("log") plt.ylim(0,8) l1v2 = plt.hist(host_w,bins=bins,label="Host",alpha=0.5) + ax1v2=plt.gca() # makes a function of completeness wxvals,wyvals = make_completeness_plot(host_maxw) @@ -488,21 +536,37 @@ def main(): l2v2 = plt.hist(host_w,bins=bins,weights = 1./w_comp,label="Observed",alpha=0.5) - ax2v2 = ax1v2.twinx() - l3v2 = ax2.plot(wxvals,wyvals,label="Completeness") + l3v2 = ax2v2.plot(wxvals,wyvals,label="Completeness")#,color=use_this_color) plt.ylim(0,1) plt.xlim(1e-3,1e3) plt.ylabel("Completeness") - plt.sca(ax1) - plt.legend(handles=[l1[2],l3[0],l2[2]],labels=["Observed","Completeness","Corrected"],fontsize=12) + plt.sca(ax1v2) + plt.legend(handles=[l1v2[2],l3v2[0],l2v2[2]],labels=["Observed","Completeness","Corrected"],fontsize=12) plt.xlabel("$w_{\\rm host}$ [ms]") plt.ylabel("Number of FRBs") ####################### W - likelihood maximisation ################# + ####################################### Width - CDF and fitting ################################ + print("\n\n KS test evaluation for width \n") + # amplitude, mean, and std dev of true distribution + + ksbest = [] + # begins minimisation for KS statistic + for ifunc in np.arange(NFUNC): + args = (host_w,wxvals,wyvals,ifunc) + x0=WARGS0[ifunc] + + result = sp.optimize.minimize(get_ks_stat,x0=x0,args=args,method = 'Nelder-Mead') + psub1 = get_ks_stat(result.x,*args,plot=False) + ksbest.append(result.x) + print("FUNCTION ",ifunc,",",FNAMES[ifunc]," Best-fitting parameters are ",result.x," with p-value ",1.-result.fun) + + + print("\n\n Maximum likelihood for width \n") ### makes temporary values for completeness xtemp = wxvals[::2] ytemp = wyvals[::2] @@ -518,37 +582,37 @@ def main(): xt[-1] = 1e5 yt[-1] = 0. cspline = sp.interpolate.make_interp_spline(np.log10(xt), yt,k=1) + # do a test plot of the spline? + if True: + plt.figure() + plt.plot(np.logspace(-5,5,101),cspline(np.linspace(-5,5,101))) + plt.plot(xvals,yvals) + plt.xscale("log") + plt.savefig(outdir+"width_spline_example.png") + plt.close() + # make a cdf plot of the best fits + make_cdf_plot(ksbest,host_tau,xvals,yvals,outdir+"bestfit_ks_width_cumulative.png",cspline) + ####################################### Width - max likelihood ################################ + print("\n\n Likelhiood maximasation for width \n") + # amplitude, mean, and std dev of true distribution + xbest=[] + # iterate over functions to calculate max likelihood for ifunc in np.arange(NFUNC): - if False: - print("skipping log-likelihood test optimisation, remove this line to re-run") - continue - # make values for interpolation - # completeness for w in host frame - - # get a spline interpolation of completeness. Should be removed from function! - if False: - plt.figure() - plt.plot(np.logspace(-5,5,101),cspline(np.linspace(-5,5,101))) - plt.plot(xvals,yvals) - plt.xscale("log") - plt.savefig("spline_example.png") - plt.close() - args = (host_w,cspline,ifunc) x0=WARGS0[ifunc] result = sp.optimize.minimize(get_ll_stat,x0=x0,args=args,method = 'Nelder-Mead') #psub1 = get_ks_stat(result.x,*args,plot=True) xbest.append(result.x) print("width FUNCTION ",ifunc,",",FNAMES[ifunc]," Best-fitting log-likelihood parameters are ",result.x," with p-value ",1.-result.fun) - #FUNCTION 0 Best-fitting parameters are [1.54420422 1.77315156] with p-value -14.098744320572287 - #FUNCTION 1 Best-fitting parameters are [1.54415779 1.77312642] with p-value -14.098744320522364 - #FUNCTION 2 Best-fitting parameters are [-2.59375 4.753125] with p-value -16.16738087050264 + if ifunc == 0: + llCHIME = get_ll_stat([CHIME_muw,CHIME_sw],host_tau,cspline,ifunc) * -1 + print("Compare with CHIME ",llCHIME) - make_cdf_plot(xbest,host_w,wxvals,wyvals,"bestfit_ll_width_cumulative.png",cspline,width=True) + make_cdf_plot(xbest,host_w,wxvals,wyvals,outdir+"bestfit_ll_width_cumulative.png",cspline,width=True) ### does plot with fits added ### @@ -573,7 +637,7 @@ def main(): plt.legend(handles=handles,labels=labels,fontsize=6) #fontsize=12) - plt.savefig("w_host_histogram_fits.png") + plt.savefig(outdir+"w_host_histogram_fits.png") plt.close() #### for paper #### @@ -595,6 +659,7 @@ def main(): plt.plot(xs,ys*plotnorm,linestyle=":",color=plt.gca().lines[-1].get_color()) plt.xscale("log") + plt.text(1e-3,12,"(b)",fontsize=18) plt.xlim(5e-3,1e2) plt.legend() plt.xlabel("$w_{\\rm host}$ [ms]") @@ -602,7 +667,7 @@ def main(): plt.legend(handles=handles,labels=labels,fontsize=12) #fontsize=12) plt.tight_layout() - plt.savefig("paper_w_host_histogram_fits.png") + plt.savefig(outdir+"paper_w_host_histogram_fits.png") plt.close() ############################################# WIDTH - bayes factor ####################################### @@ -616,7 +681,7 @@ def main(): print("\n\n Bayes Factor Calculation\n") for ifunc in np.arange(NFUNC): if True: - print("skipping log-likelihood test optimisation, remove this line to re-run") + print("skipping Bayes factor calculation for width, remove this line to re-run") #FUNCTION 0 , lognormal has likelihood sum 4.287511548315901e-15 now compute Bayes factor! #FUNCTION 1 , half-lognormal has likelihood sum 1.3919340351669428e-12 now compute Bayes factor! #FUNCTION 2 , boxcar has likelihood sum 3.1091474793575766e-13 now compute Bayes factor! @@ -665,7 +730,7 @@ def main(): -def plot_functions(): +def plot_functions(outdir=""): """ Plots example functions for the paper """ @@ -719,7 +784,7 @@ def plot_functions(): #plt.legend() plt.ylabel("f(t)") plt.tight_layout() - plt.savefig("example_functions.png") + plt.savefig(outdir+"example_functions.png") plt.close() def get_ll_stat(args,tau_obs,cspline,ifunc,plot=False, log_min=-5, log_max = 5, NTau=600): diff --git a/papers/Scattering/run_MCMC_slurm.sh b/papers/Scattering/run_MCMC_slurm.sh new file mode 100755 index 0000000..196357e --- /dev/null +++ b/papers/Scattering/run_MCMC_slurm.sh @@ -0,0 +1,34 @@ +#!/bin/bash +#SBATCH --job-name=fit_scattering_test +#SBATCH --ntasks=10 +#SBATCH --time=24:00:00 +#SBATCH --export=NONE +#SBATCH --mem-per-cpu=700MB + +# activate python environment +source /fred/oz313/cwjames/zdm/python_env/bin/activate + + +export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK + +####### ACTUAL RUN ####### +version=$SLURM_ARRAY_TASK_ID +pfile="MCMC_inputs/scat_w_only.json" +files="CRAFT_ICS_892 CRAFT_ICS_1300 CRAFT_ICS_1632" +opfile="MCMC_outputs/mcmc_lognormal_v${version}" +Pn=False +ptauw=True +steps=500 +walkers=20 + +Nz=100 +Ndm=100 +zmax=2 +dmmax=2000 + +script="../../zdm/scripts/MCMC/MCMC_wrap.py" + +runcommand="python $script -f $files --opfile=$opfile --pfile=$pfile --ptauw -s $steps -w $walkers --Nz=$Nz --Ndm=$Ndm --zmax=$zmax --dmmax=$dmmax" + +echo $runcommand +$runcommand diff --git a/papers/Scattering/run_MCMC_slurm_halflog.sh b/papers/Scattering/run_MCMC_slurm_halflog.sh new file mode 100755 index 0000000..e5a7428 --- /dev/null +++ b/papers/Scattering/run_MCMC_slurm_halflog.sh @@ -0,0 +1,34 @@ +#!/bin/bash +#SBATCH --job-name=fit_scattering_test +#SBATCH --ntasks=10 +#SBATCH --time=24:00:00 +#SBATCH --export=NONE +#SBATCH --mem-per-cpu=700MB + +# activate python environment +source /fred/oz313/cwjames/zdm/python_env/bin/activate + + +export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK + +####### ACTUAL RUN ####### +version=$SLURM_ARRAY_TASK_ID +pfile="MCMC_inputs/scat_w_only_halflog.json" +files="CRAFT_ICS_892 CRAFT_ICS_1300 CRAFT_ICS_1632" +opfile="MCMC_outputs/mcmc_lognormal_hl_v${version}" +Pn=False +ptauw=True +steps=500 +walkers=20 + +Nz=100 +Ndm=100 +zmax=2 +dmmax=2000 + +script="../../zdm/scripts/MCMC/MCMC_wrap.py" + +runcommand="python $script -f $files --opfile=$opfile --pfile=$pfile --ptauw -s $steps -w $walkers --Nz=$Nz --Ndm=$Ndm --zmax=$zmax --dmmax=$dmmax" + +echo $runcommand +$runcommand diff --git a/papers/Scattering/run_MCMC_slurm_hl_sfr.sh b/papers/Scattering/run_MCMC_slurm_hl_sfr.sh new file mode 100755 index 0000000..e1b2804 --- /dev/null +++ b/papers/Scattering/run_MCMC_slurm_hl_sfr.sh @@ -0,0 +1,34 @@ +#!/bin/bash +#SBATCH --job-name=fit_scattering_test +#SBATCH --ntasks=10 +#SBATCH --time=10:00:00 +#SBATCH --export=NONE +#SBATCH --mem-per-cpu=700MB + +# activate python environment +source /fred/oz313/cwjames/zdm/python_env/bin/activate + + +export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK + +####### ACTUAL RUN ####### +version=$SLURM_ARRAY_TASK_ID +pfile="MCMC_inputs/halflog_w_sfr.json" +files="CRAFT_ICS_892 CRAFT_ICS_1300 CRAFT_ICS_1632" +opfile="MCMC_outputs/mcmc_lognormal_hl_sfr_v${version}" +Pn=False +ptauw=True +steps=2000 +walkers=20 + +Nz=100 +Ndm=100 +zmax=2 +dmmax=2000 + +script="../../zdm/scripts/MCMC/MCMC_wrap.py" + +runcommand="python $script -f $files --opfile=$opfile --pfile=$pfile --ptauw -s $steps -w $walkers --Nz=$Nz --Ndm=$Ndm --zmax=$zmax --dmmax=$dmmax" + +echo $runcommand +$runcommand diff --git a/papers/Scattering/run_MCMC_slurm_sfr.sh b/papers/Scattering/run_MCMC_slurm_sfr.sh index a26646a..e5dd8b6 100755 --- a/papers/Scattering/run_MCMC_slurm_sfr.sh +++ b/papers/Scattering/run_MCMC_slurm_sfr.sh @@ -1,7 +1,7 @@ #!/bin/bash #SBATCH --job-name=fit_scattering_test #SBATCH --ntasks=10 -#SBATCH --time=24:00:00 +#SBATCH --time=10:00:00 #SBATCH --export=NONE #SBATCH --mem-per-cpu=700MB @@ -15,10 +15,10 @@ export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK version=$SLURM_ARRAY_TASK_ID pfile="MCMC_inputs/scat_w_sfr.json" files="CRAFT_ICS_892 CRAFT_ICS_1300 CRAFT_ICS_1632" -opfile="MCMC_outputs/v2mcmc_lognormal_sfr_v${version}" +opfile="MCMC_outputs/mcmc_lognormal_sfr_v${version}" Pn=False ptauw=True -steps=500 +steps=2000 walkers=20 Nz=100 diff --git a/papers/Scattering/visualise_mcmc.py b/papers/Scattering/visualise_mcmc.py index 3af70c3..65728cd 100644 --- a/papers/Scattering/visualise_mcmc.py +++ b/papers/Scattering/visualise_mcmc.py @@ -33,370 +33,124 @@ plt.rcParams['font.size'] = 16 -# # Load files -# -# - labels = list of parameters (in order) -# - filenames = list of .h5 files to use (without .h5 extension) - -# In[ ]: - - -# labels = ["sfr_n", "alpha", "lmean", "lsigma", "lEmax", "lEmin", "gamma", "H0", "DMhalo"] -# labels = [r"$n$", r"$\alpha$", r"log$\mu$", r"log$\sigma$", r"log$E_{\mathrm{max}}$", r"log$E_{\mathrm{min}}$", r"$\gamma$", r"$H_0$", "DMhalo"] - - -half = False -sfr=False -if half: - - filenames = ['MCMC_outputs/v2_mcmc_halflognormal','MCMC_outputs/v3_mcmc_halflognormal'] - # this name gets added to all produced plots - prefix="MCMC_Plots/halflognormal_" - labels = [r"$\mu_w$", r"$\sigma_w$", r"$\mu_\tau$", r"$\sigma_\tau$"] -if sfr: - filenames = ['MCMC_outputs/mcmc_lognormal_sfr_v1', - 'MCMC_outputs/mcmc_lognormal_sfr_v2', - 'MCMC_outputs/mcmc_lognormal_sfr_v3', - 'MCMC_outputs/mcmc_lognormal_sfr_v4'] - # this name gets added to all produced plots - labels = [r"$\mu_w$", r"$\sigma_w$", r"$\mu_\tau$", r"$\sigma_\tau$", r"$n_{\rm sfr}$"] - prefix="MCMC_Plots/sfr_halflognormal_" -else: +def main(filenames,labels,prefix): + """ + Main function to process MCMC output - filenames = ['MCMC_outputs/mcmc_lognormal_v1', - 'MCMC_outputs/mcmc_lognormal_v2', - 'MCMC_outputs/mcmc_lognormal_v3', - 'MCMC_outputs/mcmc_lognormal_v4'] # turn off p(scat|w) - labels = [r"$\mu_w$", r"$\sigma_w$", r"$\mu_\tau$", r"$\sigma_\tau$"] - prefix = "MCMC_Plots/lognormal" - -samples = [] - -# Q1: why multiple files? Are these independent MCMC runs that can be added together for a larger data-set? -for i, filename in enumerate(filenames): - reader = emcee.backends.HDFBackend(filename + '.h5') - samples.append(reader.get_chain()) + Args: + Filenames [list of strings]: specifies the MCMC output (.h5 files) from e.g. a slurm job. + Labels [list of strings]: defined the latex labvels for plotting of MCMC variables + Prefix [string]: prefix to prepend the plot files with + """ + samples = [] + # Q1: why multiple files? Are these independent MCMC runs that can be added together for a larger data-set? + for i, filename in enumerate(filenames): + reader = emcee.backends.HDFBackend(filename + '.h5') + samples.append(reader.get_chain()) -# # Negate $\alpha$ -# -# - In our code we assume $\alpha$ is negative and so $\alpha=2$ here corresponds to a negative spectral index. -# - So here, we change that to a negative for clarity - -# Make alpha negative -a=-1 -for i, x in enumerate(labels): - if x == r"$\alpha$": - a = i - break - -if a != -1: - for sample in samples: - sample[:,:,a] = -sample[:,:,a] - -# plot walkers -analysis.plot_walkers(samples,labels,prefix+"raw_walkers.png") - - -######## Define burnin sample ########## -# here are many ways to do this. But visually tends to be -# the best. Here, a hard-coded value of 200 is used. -# Please inspect walker -good_samples = analysis.std_rej(samples, burnin=200) + # # Negate $\alpha$ + # + # - In our code we assume $\alpha$ is negative and so $\alpha=2$ here corresponds to a negative spectral index. + # - So here, we change that to a negative for clarity -analysis.plot_autocorrelations(good_samples,prefix+"autocorrelation_times.png") + # Make alpha negative + a=-1 + for i, x in enumerate(labels): + if x == r"$\alpha$": + a = i + break + if a != -1: + for sample in samples: + sample[:,:,a] = -sample[:,:,a] -# # Implement burnin and change priors + # plot walkers + analysis.plot_walkers(samples,labels,prefix+"raw_walkers.png",legend=False) -burnin = 250 # define this by inspecting the raw walkers -# keep burnin the same over all samples - in theory, this could be different though -# (but probably should not be!) -burnin = (np.ones(len(good_samples)) * burnin).astype(int) -# removes the burnin from each sample + ######## Define burnin sample ########## + # here are many ways to do this. But visually tends to be + # the best. Here, a hard-coded value of 200 is used. + # Please inspect walker + good_samples = analysis.std_rej(samples, burnin=200) -analysis.plot_walkers(good_samples,labels,prefix+"final_walkers.png",burnin=burnin) -# NOTE - there is no current way to plot the final walkers with bad points removed + analysis.plot_autocorrelations(good_samples,prefix+"autocorrelation_times.png") -# Get the final sample without burnin and without bad walkers -final_sample = [[] for i in range(samples[0].shape[2])] -# we now remove the burnin from each -for j,sample in enumerate(good_samples): - for i in range(sample.shape[2]): - final_sample[i].append(sample[burnin[j]:,:,i].flatten()) -final_sample = np.array([np.hstack(final_sample[i]) for i in range(len(final_sample))]).T + # # Implement burnin and change priors -# - Changes prior to discard samples outside the specified prior range -# - Implements the burnin using either the predefined burnin or a constant specified -# e.g.: -# final_sample = analysis.change_priors(final_sample, 5, min=38) -# final_sample = analysis.change_priors(final_sample, 7, max=110.0) -# final_sample = analysis.change_priors(final_sample, 9, max=80.0) -# final_sample = analysis.change_priors(final_sample, 1, max=1.0, min=-3.5) + burnin = 250 # define this by inspecting the raw walkers + # keep burnin the same over all samples - in theory, this could be different though + # (but probably should not be!) + burnin = (np.ones(len(good_samples)) * burnin).astype(int) + # removes the burnin from each sample -######## Cornerplot ######### - -# use the below to show other lines on this plot. E.g. to show a standard H0 value. -# Typically used to show "correct", i.e. true, values, against the MCMC estimates -truth = False -if truth: - lmean = 2.27 - DMhalo = np.log10(50.0) - param_dict={'logF': np.log10(0.32), 'sfr_n': 1.13, 'alpha': 1.5, 'lmean': lmean, 'lsigma': 0.55, - 'lEmax': 41.26, 'lEmin': 39.5, 'gamma': -0.95, 'DMhalo': DMhalo, 'H0': 73, - 'min_lat': None} - truths = [param_dict[param] for param in labels] -else: - truths = None - -fig = plt.figure(figsize=(12,12)) - -titles = ['' for i in range(final_sample.shape[1])] -corner.corner(final_sample,labels=labels, show_titles=True, titles=titles, - fig=fig,title_kwargs={"fontsize": 15},label_kwargs={"fontsize": 15}, - quantiles=[0.16,0.5,0.84], truths=truths); -print(prefix+"cornerplot.png") -plt.savefig(prefix+"cornerplot.png") -exit() -#### LEAVE HERE ###### - -# # Point estimates -# -# - Use finer histogram binning than the corner plot -# - Obtain point estimates and confidence intervals using median / mode -# In[64]: + analysis.plot_walkers(good_samples,labels,prefix+"final_walkers.png",burnin=burnin,legend=False) + xlim=[1000,1200] + analysis.plot_walkers(good_samples,labels,prefix+"final_walkers_zoom.png",burnin=burnin,legend=False,xlim=xlim) -nBins = 20 -win_len = int(nBins/10) -CL = 0.68 + # NOTE - there is no current way to plot the final walkers with bad points removed -best_fit = {} + # Get the final sample without burnin and without bad walkers + final_sample = [[] for i in range(samples[0].shape[2])] -for i in range(len(labels)): - fig = plt.figure(figsize=(6,4)) - ax = fig.add_subplot(1,1,1) - hist, bin_edges, _ = ax.hist(final_sample[:,i], bins=nBins, density=True) - bin_width = bin_edges[1] - bin_edges[0] - bins = -np.diff(bin_edges)/2.0 + bin_edges[1:] + # we now remove the burnin from each + for j,sample in enumerate(good_samples): + for i in range(sample.shape[2]): + final_sample[i].append(sample[burnin[j]:,:,i].flatten()) + final_sample = np.array([np.hstack(final_sample[i]) for i in range(len(final_sample))]).T - ax.set_xlabel(labels[i]) - ax.set_ylabel("P("+labels[i]+")") + # - Changes prior to discard samples outside the specified prior range + # - Implements the burnin using either the predefined burnin or a constant specified + # e.g.: + # final_sample = analysis.change_priors(final_sample, 5, min=38) + # final_sample = analysis.change_priors(final_sample, 7, max=110.0) + # final_sample = analysis.change_priors(final_sample, 9, max=80.0) + # final_sample = analysis.change_priors(final_sample, 1, max=1.0, min=-3.5) - # Use mode ordered - # ordered_idxs = np.argsort(hist) - # sum = hist[ordered_idxs[0]] * bin_width - # j = 1 - # while(sum < 1-CL): - # sum += hist[ordered_idxs[j]] * bin_width - # j = j+1 + ######## Cornerplot ######### - # best = bins[ordered_idxs[-1]] - # lower = bins[np.min(ordered_idxs[j:])] - # upper = bins[np.max(ordered_idxs[j:])] + # use the below to show other lines on this plot. E.g. to show a standard H0 value. + # Typically used to show "correct", i.e. true, values, against the MCMC estimates + truth = False + if truth: + lmean = 2.27 + DMhalo = np.log10(50.0) + param_dict={'logF': np.log10(0.32), 'sfr_n': 1.13, 'alpha': 1.5, 'lmean': lmean, 'lsigma': 0.55, + 'lEmax': 41.26, 'lEmin': 39.5, 'gamma': -0.95, 'DMhalo': DMhalo, 'H0': 73, + 'min_lat': None} + truths = [param_dict[param] for param in labels] + else: + truths = None - # Use median - best = np.quantile(final_sample[:,i], 0.5) - # best = bins[np.argmax(hist)] - lower = np.quantile(final_sample[:,i], 0.16) - upper = np.quantile(final_sample[:,i], 0.84) + fig = plt.figure(figsize=(12,12)) - best_fit[labels[i]] = best - u_lower = best - lower - u_upper = upper - best - ax.axvline(lower, color='r') - ax.axvline(best, color='r') - ax.axvline(upper, color='r') - # print(labels[i] + ": " + str(best) + " (-" + str(u_lower) + "/+" + str(u_upper) + ")") - print(rf'{labels[i]}: {best} (-{u_lower}/+{u_upper})') + titles = ['' for i in range(final_sample.shape[1])] + corner.corner(final_sample,labels=labels, show_titles=True, titles=titles, + fig=fig,title_kwargs={"fontsize": 15},label_kwargs={"fontsize": 15}, + quantiles=[0.16,0.5,0.84], truths=truths); + print(prefix+"cornerplot.png") + plt.savefig(prefix+"cornerplot.png") -print(best_fit) +sfr = "sfr_" +labels = [r"$\mu_w$", r"$\sigma_w$", r"$\mu_{\tau}$", r"$\sigma_\tau$"] +if sfr != "": + labels.append(r"$n_{\rm sfr}$") +for half in ["","hl_"]: -# In[65]: - - -import scipy.stats as st - - -# In[66]: - - -# nsamps = np.linspace(3, np.log10(final_sample.shape[0]/10), 30) -# nsamps = [int(10**x) for x in nsamps] -# print("Number of samps: " + str(nsamps)) - -# for i in range(len(labels)): -# # nsamps = [] -# std = [] -# for j in range(len(nsamps)): -# best = [] -# nruns = int(final_sample.shape[0] / nsamps[j]) -# for k in range(nruns): -# # best.append(np.quantile(final_sample[nsamps[j]*k:nsamps[j]*(k+1),i], 0.5)) -# step = int(final_sample.shape[0]/nsamps[j]) -# best.append(np.quantile(final_sample[k::step,i], 0.5)) -# std.append(np.std(best)) - -# # print(labels[i] + ": " + str(std)) - -# line = st.linregress(np.log10(nsamps),np.log10(std)) -# x = np.linspace(nsamps[0], nsamps[-1], 50) -# y = 1/np.sqrt(x) -# y = y / y[0] * std[0] -# y = 10**(line.slope*np.log10(x) + line.intercept) -# # print(line.slope) -# print(labels[i] + ": " + str(10**(line.slope*np.log10(final_sample.shape[0]) + line.intercept))) -# print(str(line.slope)) -# fig = plt.figure(figsize=(6,4)) -# ax = fig.add_subplot(1,1,1) - -# ax.plot(nsamps, std) -# ax.loglog(x,y) -# ax.set_xlabel("Number of samples") -# ax.set_ylabel("Standard deviation") -# ax.set_title(labels[i]) - - -# # Load surveys and grids -# -# - Loads the surveys and grids with the best fit parameters from above. -# - Plots P(DM) and DMEG weights for each FRB - -# In[67]: - - -s_names = [ - # "FAST2", - # "FAST2_old" - # "DSA", - "FAST", - # "CRAFT_class_I_and_II", - # "private_CRAFT_ICS_892_14", - # "private_CRAFT_ICS_1300_14", - # "private_CRAFT_ICS_1632_14", - # "parkes_mb_class_I_and_II" -] -# rs_names = ["CHIME/CHIME_decbin_0_of_6", -# "CHIME/CHIME_decbin_1_of_6", -# "CHIME/CHIME_decbin_2_of_6", -# "CHIME/CHIME_decbin_3_of_6", -# "CHIME/CHIME_decbin_4_of_6", -# "CHIME/CHIME_decbin_5_of_6"] -rs_names = [] - -state = parameters.State() -state.set_astropy_cosmo(Planck18) -# state.update_params(best_fit) -# state.update_param('luminosity_function', 2) -# state.update_param('alpha_method', 0) -# state.update_param('sfr_n', 1.36) -# state.update_param('alpha', 1.5) -# state.update_param('lmean', 1.97) -# state.update_param('lsigma', 0.92) -# state.update_param('lEmax', 41.3) -# state.update_param('gamma', -0.63) -# state.update_param('H0', 70.0) -# state.update_param('DMhalo', 50.0) - -if len(s_names) != 0: - surveys, grids = loading.surveys_and_grids(survey_names = s_names, init_state=state, repeaters=False, nz=500, ndm=1400) -else: - surveys = [] - grids = [] - -if len(rs_names) != 0: - rep_surveys, rep_grids = loading.surveys_and_grids(survey_names = rs_names, init_state=state, repeaters=True, nz=500, ndm=1400) - for s,g in zip(rep_surveys, rep_grids): - surveys.append(s) - grids.append(g) - - -# In[68]: - - -newC, llc = it.minimise_const_only(None, grids, surveys) -llsum = 0 -for s,g in zip(surveys, grids): - - g.state.FRBdemo.lC = newC - - # Calc pdm - rates=g.rates - dmvals=g.dmvals - pdm=np.sum(rates,axis=0) - - # # Calc psnr - # min = s.SNRTHRESHs[0] - # max = np.max(s.SNRs) - # snrs = np.linspace(min,max, 50) - # psnr = get_psnr(snrs, s, g) - - # Plot pdm + snr - fig = plt.figure(figsize=(10,4)) - ax = fig.add_subplot(1,2,1) - ax.set_title(s.name) - ax.set_xlabel("DM") - ax.set_ylabel("P(DM)") - ax.set_xlim(xmax=3000) - ax.plot(dmvals, pdm) - ax.vlines(s.DMEGs, np.zeros(len(s.DMs)), np.max(pdm)*np.ones(len(s.DMs)), ls='--', colors='r') - - # ax = fig.add_subplot(1,2,2) - # ax.set_xlabel("log SNR") - # ax.set_ylabel("log P(SNR)") - # ax.plot(np.log10(snrs), np.log10(psnr)) - # ax.vlines(np.log10(s.SNRs), np.min(np.log10(psnr))*np.ones(len(s.SNRs)), np.max(np.log10(psnr))*np.ones(len(s.SNRs)), ls='--', colors='r') - - # Get expected and observed - expected=it.CalculateIntegral(g.rates,s) - expected *= 10**g.state.FRBdemo.lC - observed=s.NORM_FRB - - print(s.name + " - expected, observed: " + str(expected) + ", " + str(observed)) - - llsum += it.get_log_likelihood(g,s,Pn=True) - - -# In[ ]: - - -uDMGs = 0.5 -# DMhalo = 100.0 - -fig = plt.figure(figsize=(6,4*len(s_names))) - -for j,(s,g) in enumerate(zip(surveys, grids)): - ax = fig.add_subplot(len(surveys),1,j+1) - plt.title(s.name) - ax.set_xlabel('DM') - ax.set_ylabel('Weight') - - # s.DMhalo = DMhalo - # s.init_DMEG(DMhalo) - - dmvals=g.dmvals - DMobs=s.DMEGs - - # calc_DMG_weights(DMEGs, DMhalos, DM_ISMs, dmvals, sigma_ISM=0.5, sigma_halo=15.0, percent_ISM=True) - dm_weights, iweights = it.calc_DMG_weights(DMobs, s.DMhalos, s.DMGs, dmvals, uDMGs) - - pdm = np.sum(g.rates, axis=0) - pdm = pdm / np.max(pdm) * np.max(dm_weights[0]) - - for i in range(len(DMobs)): - ax.plot(dmvals[iweights[i]], dm_weights[i], '.-', label=s.frbs["TNS"][i] + " " + str(s.DMGs[i])) - - # ax.plot(dmvals, pdm) # Upper limit is not correct because grid has not been updated so efficiencies have not been recalc'd - ax.set_xlim(right=3000) - # ax.legend() - - + filenames = ['MCMC_outputs/mcmc_lognormal_'+half+sfr+'v1', + 'MCMC_outputs/mcmc_lognormal_'+half+sfr+'v2', + 'MCMC_outputs/mcmc_lognormal_'+half+sfr+'v3', + 'MCMC_outputs/mcmc_lognormal_'+half+sfr+'v4'] + # this name gets added to all produced plots + prefix="MCMC_Plots/"+half+sfr + main(filenames,labels,prefix) diff --git a/zdm/MCMC_analysis.py b/zdm/MCMC_analysis.py index 3216efd..775b1a6 100644 --- a/zdm/MCMC_analysis.py +++ b/zdm/MCMC_analysis.py @@ -11,11 +11,20 @@ # Here are different plotting functions -def plot_walkers(samples,labels,outfile,burnin=None): +def plot_walkers(samples,labels,outfile,burnin=None,legend=True,xlim=None): """ Puts all walkers from all samples on one plot If you want different samples per plot, call this function multiple times + + + Args: + + outfile (string): name of outfile to print to + burnin (int): length of initial burnin to discard + legend (bool): Plot legend or not + xlim (None or [xmin,xmax] flots): apply x limit, to zoom in for + visibility purposes """ plt.rcParams['font.size'] = 16 # get the number of parameters @@ -30,9 +39,11 @@ def plot_walkers(samples,labels,outfile,burnin=None): ax.plot(sample[burnin[j]:,k,i], '.-', label=str(k)) ax.set_ylabel(labels[i]) - + if xlim is not None: + ax.set_xlim(xlim[0],xlim[1]) axes[-1].set_xlabel("Step number") - axes[-1].legend() + if legend: + axes[-1].legend() plt.tight_layout() plt.savefig(outfile) From 7ae3ee4020bd725ed8379f5f67fc09fa995f6f36 Mon Sep 17 00:00:00 2001 From: Clancy James Date: Mon, 22 Sep 2025 09:19:59 +0800 Subject: [PATCH 13/20] Added documentation to scripts, and added directory for the tau-dm relation figure 1 --- papers/Scattering/fit_scattering_width.py | 84 +++++------------------ papers/Scattering/width_scat_dists.py | 26 +++++-- papers/TauDMrelation/README.txt | 3 + papers/TauDMrelation/plot_fig_1.py | 67 ++++++++++++++++++ papers/TauDMrelation/state.json | 68 ++++++++++++++++++ 5 files changed, 175 insertions(+), 73 deletions(-) create mode 100644 papers/TauDMrelation/README.txt create mode 100644 papers/TauDMrelation/plot_fig_1.py create mode 100644 papers/TauDMrelation/state.json diff --git a/papers/Scattering/fit_scattering_width.py b/papers/Scattering/fit_scattering_width.py index 2a7a3bc..6275adb 100644 --- a/papers/Scattering/fit_scattering_width.py +++ b/papers/Scattering/fit_scattering_width.py @@ -931,33 +931,7 @@ def make_cdf_plot(args,taus,cxvals,cyvals,outfile,cspline,plot=False,width=False cum_dist /= cum_dist[-1] plt.plot(xs,cum_dist,label=FNAMES[i],linestyle="--") - - #yvals1 = lognormal(xvals,A,*args1) - #yvals2 = halflognormal(xvals,A,*args2) - #yvals3=logconstant(xvals,A,*args3) - - # modify the expected distribution by the completeness - #completeness = get_completeness(xvals,cxvals,cyvals) - #modyvals1 = yvals1*completeness - #modyvals2 = yvals2*completeness - #modyvals3 = yvals3*completeness - - # make a cdf - - #cum_dist1 = np.cumsum(modyvals1) - #cum_dist1 /= cum_dist1[-1] - - #cum_dist2 = np.cumsum(modyvals2) - #cum_dist2 /= cum_dist2[-1] - - #cum_dist3 = np.cumsum(modyvals3) - #cum_dist3 /= cum_dist3[-1] - - #plt.plot(xvals,cum_dist1,label="Lognormal fit",linestyle="--") - #plt.plot(xvals,cum_dist2,label="Half-lognormal fit",linestyle=":") - #plt.plot(xvals,cum_dist3,label="Log-uniform fit",linestyle="-.") - # makes a function of completeness xvals,yvals = make_completeness_plot(taus,reverse=False) plt.plot(xvals,yvals,label="Observed",color="black") @@ -1154,10 +1128,18 @@ def find_max_w(tauobs,wtot,snrobs,fbar,DMobs,tres,nu_res = 1.,\ return maxw def get_data(): + """ + Loads relevant data for this analysis + + Returns list of FRBs with both HTR data and redshift, + giving following info: + tns,tauobs,w95,wsnr,z,snr,freq,DM,tres + + """ # extract relevant data. # this is the error code - + # loads time resolutions tresinfo = np.loadtxt("treslist.dat",dtype="str") names=tresinfo[:,0] @@ -1217,6 +1199,14 @@ def get_data(): def getOK(arrays,ERR=9999.): """ Find indices where all arrays have good values + Returns indices where every array has a valid value + + Args: + arrays: some list of vectors + ERR is the error code + + Retusn: + OK: list of indices """ OK = np.where(arrays[0] != ERR) for array in arrays[1:]: @@ -1251,45 +1241,5 @@ def make_cum_dist(vals): xs[-1] = themax ys[-1] = 1 return xs,ys - -def read_chime_scat(infile = "chime_scat.dat"): - """ - gets chime scat err data - """ - scats=[] - errs=[] - with open(infile) as file: - for line in file: - fields = line.split() - scatstring = fields[1].replace(" ", "") - if scatstring[0] == "<": - scat = float(scatstring[1:])/2. - err = scat - elif scatstring[0] == "~": - scat = float(scatstring[1:]) - err = float(fields[2].replace(" ", "")) - else: - scat = float(scatstring) - err = float(fields[2].replace(" ", "")) - scats.append(scat) - errs.append(err) - scats = np.array(scats) - errs = np.array(errs) - return scats,errs - -def cutz(DM,DMG,z,scat,err): - """ - cuts on z>0 - """ - - - OK = np.where(z>0.)[0] - DM = DM[OK] - DMG = DMG[OK] - z = z[OK] - scat = scat[OK] - err = err[OK] - return DM,DMG,z,scat,err - main() diff --git a/papers/Scattering/width_scat_dists.py b/papers/Scattering/width_scat_dists.py index cc7cbec..be9354b 100644 --- a/papers/Scattering/width_scat_dists.py +++ b/papers/Scattering/width_scat_dists.py @@ -1,6 +1,9 @@ """ -This script plots observed and fitted width and scattering distributions +This script plots observed and fitted width and scattering distributions from the zdm code +i.e. it lods in the CRAFT ICS surveys, models the width and scattering distributions +that are best-fit from the scattering paper, then generates "Plots/differential.png" +whcih compares the observed and modelled distributions. """ @@ -32,7 +35,7 @@ def main(): """ - + Main function """ # in case you wish to switch to another output directory @@ -60,6 +63,15 @@ def main(): state_dict["width"]["WNInternalBins"] = 1000 # sets it to a small quantity state_dict["width"]["WNbins"] = 33 # set to large number for this analysis + # best-fit results for half-log-normal distributions + state_dict["width"]["WidthFunction"] = 2 + state_dict["scat"]["ScatFunction"] = 2 + state_dict["width"]["Wlogmean"] = 1.3 + state_dict["width"]["Wlogsigma"] = 1.3 + state_dict["scat"]["Slogmean"] = 2 + state_dict["scat"]["Slogsigma"] = 2.2 + + surveys, grids = loading.surveys_and_grids(survey_names = names,\ repeaters=repeaters, sdir=sdir,nz=70,ndm=140, survey_dict = survey_dict, state_dict = state_dict) @@ -149,7 +161,6 @@ def main(): # normalise each survey to actual number of FRBs nfrb = len(s.OKTAU) - # these need to be normalised by the internal bin width logbinwidth = s.internal_logwvals[-1] - s.internal_logwvals[-2] @@ -199,9 +210,12 @@ def main(): #normalisation: p per log bin * Nfrb. weights = np.full([len(wlist)],1./lbw) alpha=1.0 - plt.hist(wlist,bins=bins,weights=weights,alpha=alpha,facecolor = l1.get_color(),edgecolor = l1.get_color(),linewidth=2,histtype='step') - plt.hist(tlist,bins=bins,weights=weights,alpha=alpha,facecolor = l2.get_color(),edgecolor = l2.get_color(),linewidth=2,histtype='step') - plt.hist(ilist,bins=bins,weights=weights,alpha=alpha,facecolor = l3.get_color(),edgecolor = l3.get_color(),linewidth=2,histtype='step') + plt.hist(wlist,bins=bins,weights=weights,alpha=alpha,facecolor = l1.get_color(), + edgecolor = l1.get_color(),linewidth=2,histtype='step', label="Observed total") + plt.hist(tlist,bins=bins,weights=weights,alpha=alpha,facecolor = l2.get_color(), + edgecolor = l2.get_color(),linewidth=2,histtype='step', label="Observed scattering") + plt.hist(ilist,bins=bins,weights=weights,alpha=alpha,facecolor = l3.get_color(), + edgecolor = l3.get_color(),linewidth=2,histtype='step', label="Observed intrinsic") plt.legend() diff --git a/papers/TauDMrelation/README.txt b/papers/TauDMrelation/README.txt new file mode 100644 index 0000000..e4778de --- /dev/null +++ b/papers/TauDMrelation/README.txt @@ -0,0 +1,3 @@ +This directory gives code for the paper "A tau-DM relation for FRB hosts?" by Lous Malba et al + +It simply produces Figure 1 of that text. diff --git a/papers/TauDMrelation/plot_fig_1.py b/papers/TauDMrelation/plot_fig_1.py new file mode 100644 index 0000000..5d02e7d --- /dev/null +++ b/papers/TauDMrelation/plot_fig_1.py @@ -0,0 +1,67 @@ +""" +This script creates zdm grids and plots localised FRBs + +It can also generate a summed histogram from all CRAFT data + +""" +import os + +from astropy.cosmology import Planck18 +from zdm import cosmology as cos +from zdm import figures +from zdm import parameters +from zdm import survey +from zdm import pcosmic +from zdm import iteration as it +from zdm import loading +from zdm import io + +import numpy as np +from zdm import survey +from matplotlib import pyplot as plt +from pkg_resources import resource_filename +import time +import json + +def main(): + + # in case you wish to switch to another output directory + opdir = "./" + if not os.path.exists(opdir): + os.mkdir(opdir) + + # Initialise surveys and grids + sdir = os.path.join(resource_filename('zdm', 'data'), 'Surveys') + names = ["CRAFT_ICS_1300"] + + # loads state variables + with open('state.json') as json_file: + oldstate = json.load(json_file) + + + # essentially turns off DM host and sets all FRB widths to ~0 (or close enough) + state_dict = {'lmean': 0.01, 'lsigma': 0.4, 'Wlogmean': -1,'WNbins': 1, + 'Wlogsigma': 0.1, 'Slogmean': -2,'Slogsigma': 0.1,'H0': 70,'logF': -0.495, + 'alpha': 0.65,'sfr_n': 0.73,'lEmax': 41.4,'lEmin': 30.,'gamma': -1.01, + 'alpha_method': 1} + state = parameters.State() + state.set_astropy_cosmo(Planck18) + state.update_params(state_dict) + + + surveys, grids = loading.surveys_and_grids(survey_names = names, + repeaters=False, sdir=sdir,init_state=state) + + # plots it + g=grids[0] + + figures.plot_grid(g.rates,g.zvals,g.dmvals, + name='figure1.png',norm=3,log=True, + label='$\\log_{10} p({\\rm DM}_{\\rm cosmic},z)$ [a.u.]', + ylabel='${\\rm DM}_{\\rm cosmic}$', + project=False, + logrange=5, + zmax=2.5,DMmax=2500,cmap="Oranges",Aconts=[0.01,0.1,0.5], + cont_clrs=[0.,0.,0.]) + +main() diff --git a/papers/TauDMrelation/state.json b/papers/TauDMrelation/state.json new file mode 100644 index 0000000..2b47103 --- /dev/null +++ b/papers/TauDMrelation/state.json @@ -0,0 +1,68 @@ +{ + "FRBdemo": { + "alpha_method": 1, + "lC": 1, + "sfr_n": 0.73, + "source_evolution": 0 + }, + "IGM": { + "logF": -0.494850021680094 + }, + "MW": { + "DMhalo": 50, + "ISM": 35.0, + "halo_method": 0, + "logu": false, + "sigmaDMG": 0.0, + "sigmaHalo": 15.0 + }, + "analysis": { + "DMG_cut": null, + "NewGrids": true, + "min_lat": -1.0, + "sprefix": "Std" + }, + "cosmo": { + "H0": 67.66, + "Omega_b": 0.04897, + "Omega_b_h2": 0.0224178568132, + "Omega_k": 0.0, + "Omega_lambda": 0.6888463055445441, + "Omega_m": 0.30966, + "fix_Omega_b_h2": true + }, + "energy": { + "alpha": 0.65, + "gamma": -1.01, + "lEmax": 41.4, + "lEmin": 30, + "luminosity_function": 2 + }, + "host": { + "lmean": 2.18, + "lsigma": 0.48 + }, + "rep": { + "RC": 0.01, + "RE0": 1e+39, + "Rgamma": -2.375, + "lRmax": 1.0, + "lRmin": -3.0 + }, + "scat": { + "ScatDist": 2, + "Sfnorm": 600, + "Sfpower": -4.0, + "Slogmean": 0.7, + "Slogsigma": 1.9, + "Smaxsigma": 3.0 + }, + "width": { + "WMax": 100, + "WMin": 0.1, + "WNbins": 10, + "Wlogmean": 1.70267, + "Wlogsigma": 0.899148, + "Wthresh": 0.5 + } +} From d2244511d315adad7c56f3d14c053296fdeb681e Mon Sep 17 00:00:00 2001 From: Clancy James Date: Mon, 22 Sep 2025 09:28:20 +0800 Subject: [PATCH 14/20] updated files with latest fit values --- papers/Scattering/width_scat_dists.py | 8 ++++---- zdm/parameters.py | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/papers/Scattering/width_scat_dists.py b/papers/Scattering/width_scat_dists.py index be9354b..a7201fe 100644 --- a/papers/Scattering/width_scat_dists.py +++ b/papers/Scattering/width_scat_dists.py @@ -66,10 +66,10 @@ def main(): # best-fit results for half-log-normal distributions state_dict["width"]["WidthFunction"] = 2 state_dict["scat"]["ScatFunction"] = 2 - state_dict["width"]["Wlogmean"] = 1.3 - state_dict["width"]["Wlogsigma"] = 1.3 - state_dict["scat"]["Slogmean"] = 2 - state_dict["scat"]["Slogsigma"] = 2.2 + state_dict["width"]["Wlogmean"] = -0.29 + state_dict["width"]["Wlogsigma"] = 0.65 + state_dict["scat"]["Slogmean"] = -1.3 + state_dict["scat"]["Slogsigma"] = 0.2 surveys, grids = loading.surveys_and_grids(survey_names = names,\ diff --git a/zdm/parameters.py b/zdm/parameters.py index c6ad04a..0d5414f 100644 --- a/zdm/parameters.py +++ b/zdm/parameters.py @@ -228,7 +228,7 @@ class IGMParams(data_class.myDataClass): @dataclass class WidthParams(data_class.myDataClass): WidthFunction: int = field( - default=1, + default=2, metadata={ "help": "ID of function to describe width distribution. 0: log-constant, 1:log-normal, 2: half-lognormal", "unit": "", @@ -236,7 +236,7 @@ class WidthParams(data_class.myDataClass): }, ) Wlogmean: float = field( - default=1.70267, + default=-0.29, metadata={ "help": "$\log_{10}$ mean of intrinsic width distribution in ms", "unit": "ms", @@ -244,7 +244,7 @@ class WidthParams(data_class.myDataClass): }, ) Wlogsigma: float = field( - default=0.899148, + default=0.65, metadata={ "help": "$\log_{10}$ sigma of intrinsic width distribution in ms", "unit": "ms", @@ -285,7 +285,7 @@ class WidthParams(data_class.myDataClass): @dataclass class ScatParams(data_class.myDataClass): ScatFunction: int = field( - default=1, + default=2, metadata={ "help": "Which scattering function to use. 0: log-constant. 1: lognormal. 2: half log-normal", "unit": "", @@ -293,7 +293,7 @@ class ScatParams(data_class.myDataClass): }, ) Slogmean: float = field( - default=0.7, + default=-1.3, metadata={ "help": "Mean of log-scattering distribution at 600\,Mhz", "unit": "ms", @@ -301,7 +301,7 @@ class ScatParams(data_class.myDataClass): }, ) Slogsigma: float = field( - default=1.9, + default=0.2, metadata={ "help": " Standard deviation of log-scattering distribution at 600\,MHz ", "unit": "ms", From 501275060dc732a743cf31f082844c4079914c53 Mon Sep 17 00:00:00 2001 From: Clancy James Date: Fri, 3 Oct 2025 14:39:03 +0800 Subject: [PATCH 15/20] fixed a scattering bug, and updated SKA plots --- papers/SKA_science/make_zdists.py | 1 - papers/SKA_science/plot_zdm_dists.py | 99 +++++-- papers/Scattering/compare_width_effects.py | 197 ++++++++++++++ papers/Scattering/fit_scattering_width.py | 286 +++++++++++++++++++-- papers/Scattering/width_scat_dists.py | 279 -------------------- zdm/data/Surveys/SKA_low.ecsv | 2 +- zdm/data/Surveys/SKA_mid.ecsv | 2 +- zdm/loading.py | 7 +- zdm/survey.py | 58 +++-- 9 files changed, 593 insertions(+), 338 deletions(-) create mode 100644 papers/Scattering/compare_width_effects.py delete mode 100644 papers/Scattering/width_scat_dists.py diff --git a/papers/SKA_science/make_zdists.py b/papers/SKA_science/make_zdists.py index 23a4089..d6d6d37 100644 --- a/papers/SKA_science/make_zdists.py +++ b/papers/SKA_science/make_zdists.py @@ -115,7 +115,6 @@ def generate_sensitivity_plot(infile,state,zDMgrid, zvals, dmvals, label, freq, thresh_Jyms = np.load(opfile6) # finds the best - survey_name='SKA_mid' ibest = np.argmax(oldNs) survey_dict = {"THRESH": thresh_Jyms[ibest], "TOBS": TOBS[ibest], "FBAR": freq, "BW": bw} diff --git a/papers/SKA_science/plot_zdm_dists.py b/papers/SKA_science/plot_zdm_dists.py index 70f2556..968eceb 100644 --- a/papers/SKA_science/plot_zdm_dists.py +++ b/papers/SKA_science/plot_zdm_dists.py @@ -5,6 +5,15 @@ import numpy as np from matplotlib import pyplot as plt +import matplotlib + +defaultsize=14 +ds=4 +font = {'family' : 'Helvetica', + 'weight' : 'normal', + 'size' : defaultsize} +matplotlib.rc('font', **font) + def main(): """ Plots outputs of simulations @@ -16,14 +25,36 @@ def main(): # these set the frequencies in MHz and bandwidths in MHz names = ["SKA_mid","SKA_mid","SKA_low"] + datadir = "sys_outputs/" + plotdir = "sysplotdir/" + + + plt.figure() + plt.xlabel("z") + plt.ylabel("N(z) per year") + plt.xlim(0,5) + for i,tel in enumerate(["Band1", "Band2", "Low"]): # sets frequency and bandwidth for each instrument for telconfig in ["AA4","AAstar"]: label = tel+"_"+telconfig - make_plots(label) - + print("\n\n\n DOING ",label," ########") + zvals,plow,pmid,phigh = make_plots(label,datadir=datadir) + if telconfig == "AA4": + if i==2: + plotlabel="SKA Low "+telconfig + else: + plotlabel="SKA Mid "+tel+" "+telconfig + plt.fill_between(zvals,plow,phigh,linestyle=":",linewidth=1,label=plotlabel,alpha=0.5) + plt.plot(zvals,pmid,linestyle="-",linewidth=2) + plt.ylim(1,1e4) + plt.legend() + plt.yscale("log") + plt.tight_layout() + plt.savefig("all_pz.png") + plt.close() -def make_plots(label): +def make_plots(label,datadir="sys_outputs/",plotdir="sysplotdir/"): """ Args: @@ -34,21 +65,23 @@ def make_plots(label): """ # load redshift and dm values - zvals = np.load("zvals.npy") - dmvals = np.load("dmvals.npy") + zvals = np.load(datadir+"zvals.npy") + dmvals = np.load(datadir+"dmvals.npy") # load survey-specific outputs - Ns = np.load(label+"_sys_N.npy") + Ns = np.load(datadir+label+"_sys_N.npy") meanN = np.sum(Ns)/Ns.size - print("Mean annual evenbt rate for ",label," is ",meanN) + print("Mean annual event rate for ",label," is ",meanN) - pzs = np.load(label+"_sys_pz.npy") - pdms = np.load(label+"_sys_pdm.npy") + pzs = np.load(datadir+label+"_sys_pz.npy") + pdms = np.load(datadir+label+"_sys_pdm.npy") - make_pz_plots(zvals,pzs,label) - make_pdm_plots(dmvals,pdms,label) - + + plow,pmid,phigh = make_pz_plots(zvals,pzs,plotdir+label) + make_pdm_plots(dmvals,pdms,plotdir+label) + return zvals,plow,pmid,phigh + def make_pz_plots(zvals,pzs,label): """ Make plots of p(z) for each systematic simulation @@ -62,23 +95,52 @@ def make_pz_plots(zvals,pzs,label): mean = np.sum(pzs,axis=0)/Nparams + # total estimates + Ntots = np.sum(pzs,axis=1) + Nordered = np.sort(Ntots) + Nbar = np.sum(Ntots)/Nparams + sigma1 = Nordered[15] + sigma2 = Nordered[83] + print("Range for Ntotal is ",sigma1-Nbar,Nbar,sigma2-Nbar) + + # constructs intervals - does this on a per-z basis + # first sorts over the axis of different simulations + zordered = np.sort(pzs,axis=0) + pzlow = zordered[15,:] + pzhigh = zordered[83,:] + # make un-normalised plots plt.figure() plt.xlabel("z") plt.ylabel("N(z) per year") plt.xlim(0,5) + themax = np.max(pzs) + plt.ylim(0,themax*scale) for i in np.arange(Nparams): - plt.plot(zvals,pzs[i,:],color="grey",linestyle="-") + plt.plot(zvals,pzs[i,:]*scale,color="grey",linestyle="-") - plt.plot(zvals,mean,color="black",linestyle="-",linewidth=2,label="Simulation mean") + plt.plot(zvals,mean*scale,color="black",linestyle="-",linewidth=2,label="Simulation mean") plt.legend() plt.tight_layout() plt.savefig(label+"_pz.png") plt.close() - + # prints some summary statistics in z-space + # calculates mean z + zbar = zvals * mean / np.sum(mean) + last=0. + tot = np.sum(mean) + for z in np.arange(5)+1: + OK = np.where(zvals < z) + Nthis = np.sum(mean[OK]) + N = Nthis -last + print("FRBs from ",z-1," to ",z,": ",N/tot," %") + last = Nthis + + return pzlow*scale,mean*scale,pzhigh*scale + def make_pdm_plots(dmvals,pdms,label): """ Make plots of p(DM) for each systematic simulation @@ -98,10 +160,13 @@ def make_pdm_plots(dmvals,pdms,label): plt.ylabel("N(DM) per year") plt.xlim(0,5000) + themax = np.max(pdms) + plt.ylim(0,themax*scale) + for i in np.arange(Nparams): - plt.plot(dmvals,pdms[i,:],color="grey",linestyle="-") + plt.plot(dmvals,pdms[i,:]*scale,color="grey",linestyle="-") - plt.plot(dmvals,mean,color="black",linestyle="-",linewidth=2,label="Simulation mean") + plt.plot(dmvals,mean*scale,color="black",linestyle="-",linewidth=2,label="Simulation mean") plt.legend() plt.tight_layout() diff --git a/papers/Scattering/compare_width_effects.py b/papers/Scattering/compare_width_effects.py new file mode 100644 index 0000000..5ea2a01 --- /dev/null +++ b/papers/Scattering/compare_width_effects.py @@ -0,0 +1,197 @@ +""" +This script compares the p(z,DM) distributions +created for the CRAFT ICS survey for different +treatments of scattering/width. + +""" + +import os + +from zdm import cosmology as cos +from zdm import figures +from zdm import parameters +from zdm import survey +from zdm import pcosmic +from zdm import misc_functions as mf +from zdm import iteration as it +from zdm import loading +from zdm import io +from pkg_resources import resource_filename +import numpy as np +from matplotlib import pyplot as plt + +import matplotlib + +defaultsize=14 +ds=4 +font = {'family' : 'Helvetica', + 'weight' : 'normal', + 'size' : defaultsize} +matplotlib.rc('font', **font) + + +def main(): + """ + See main description at top of file. + """ + + # in case you wish to switch to another output directory + opdir = "Comparisons/" + if not os.path.exists(opdir): + os.mkdir(opdir) + + # directory where the survey files are located. The below is the default - + # you can leave this out, or change it for a different survey file location. + sdir = os.path.join(resource_filename('zdm', 'data'), 'Surveys') + + # make this into a list to initialise multiple surveys art once. But here, we just use the standard + # CRAFT ICS survey + names = ["CRAFT_ICS_1300"] # for example + + # gets four cases of relevance + survey_dicts,state_dicts = get_defined_states() + + # We ignore repetition... for now! + repeaters=False + + # sets limits for calculations - both plotting, nad intrinsic. We want things to go quickly! + zmax = 2. + dmmax = 2000 + + rates = [] + for i,state_dict in enumerate(state_dicts): + + survey_dict = survey_dicts[i] + + surveys, grids = loading.surveys_and_grids(survey_names = names,\ + repeaters=repeaters, sdir=sdir,nz=400,ndm=400,zmax=2,dmmax=2000, + survey_dict = survey_dict, state_dict = state_dict) + s = surveys[0] + g = grids[0] + + rates.append(g.rates) + + + # now we have rates, let's look at the effects! + + labels=["No width/scattering","CHIME Catalogue 1","lognormals (this work)","Best fit (this work)"] + styles=[":","-.","--","-"] + + # generates a plot of p(z) + plt.figure() + ax1 = plt.gca() + + # generates a plot of p(dm) + plt.figure() + ax2 = plt.gca() + + dz = g.zvals[1]-g.zvals[0] + ddm = g.dmvals[1]-g.dmvals[0] + + for i,r in enumerate(rates): + + zdist = np.sum(r,axis=1) + dmdist = np.sum(r,axis=0) + + ax1.plot(g.zvals,zdist/dz,label=labels[i],linestyle=styles[i]) + ax2.plot(g.dmvals,dmdist/ddm,label=labels[i],linestyle=styles[i]) + + plt.sca(ax1) + plt.legend() + plt.xlabel("z") + plt.ylabel("p(z)") + plt.yscale("log") + #plt.ylim(0,1e-3) + plt.xlim(0,2) + plt.tight_layout() + plt.savefig(opdir+"zdists.png") + plt.close() + + plt.sca(ax2) + plt.legend() + plt.xlabel("DM [pc cm$^{-3}$]") + plt.ylabel("p(DM)") + plt.tight_layout() + plt.savefig(opdir+"dmdists.png") + plt.close() + + +def get_defined_states(): + """ + This function just contains hard-coded state and survey dicts + corresponding to four cases of interest + """ + # we now set up three examples. These are: + #1: no width/scattering distributions. Treat width=1ms (nominal) + #2: CHIME width/scattering distributions (no z-dependence_ + #3: bestfit lognormal function of this work (with z-dependence) + #4: bestfit functions of this work (with z-dependence) + + # sets up survey dictionaries for width methods + survey_dict1 = {"WMETHOD": 0} # ignore it - just 1ms + survey_dict2 = {"WMETHOD": 2} # scattering and width + survey_dict3 = {"WMETHOD": 3} # z-dependence + survey_dict4 = {"WMETHOD": 3} # z-dependence + + + # now sets up state dictionaries + state_dict1 = {} + state_dict1["scat"] = {} + state_dict1["width"] = {} + + state_dict2 = {} + state_dict2["scat"] = {} + state_dict2["width"] = {} + state_dict2["width"]["WNbins"] = 100 # set to large number for this analysis + state_dict2["width"]["WidthFunction"] = 1 # lognormal + state_dict2["width"]["ScatFunction"] = 1 # lognormal + state_dict2["width"]["Wlogmean"] = 0. # 1ms + state_dict2["width"]["Wlogsigma"] = 0.42 # 0.97 in ln + state_dict2["scat"]["Slogmean"] = 0.3 # 2.02 ms# + state_dict2["scat"]["Slogsigma"] = 0.74 # 1.72 in ln + state_dict2["scat"]["Sfnorm"] = 600 # 600 MHz normalisation - for clarity + + state_dict3 = {} + state_dict3["scat"] = {} + state_dict3["width"] = {} + state_dict3["width"]["WNbins"] = 100 # set to large number for this analysis + state_dict3["width"]["WidthFunction"] = 1 # lognormal + state_dict3["width"]["ScatFunction"] = 1 # lognormal + state_dict3["width"]["Wlogmean"] = 0.22 + state_dict3["width"]["Wlogsigma"] = 0.88 + state_dict3["scat"]["Slogmean"] = 0.36 + state_dict3["scat"]["Slogsigma"] = 1.05 + state_dict3["scat"]["Sfnorm"] = 1000 # 1 GHz normalisation + + if False: + state_dict3 = {} + state_dict3["scat"] = {} + state_dict3["width"] = {} + state_dict3["width"]["WNbins"] = 100 # set to large number for this analysis + state_dict3["width"]["WidthFunction"] = 2 # half-lognormal + state_dict3["width"]["ScatFunction"] = 2 # log-uniform true value 0 + state_dict3["width"]["Wlogmean"] = -0.29 # -0.29 + state_dict3["width"]["Wlogsigma"] = 0.65 #0.65 + state_dict3["scat"]["Slogmean"] = -0.38 # true valiue -1.38 + state_dict3["scat"]["Slogsigma"] = 0.01 # not actually used in the log-constant distribution + state_dict3["scat"]["Sfnorm"] = 1000 # 1 GHz normalisation + + + state_dict4 = {} + state_dict4["scat"] = {} + state_dict4["width"] = {} + state_dict4["width"]["WNbins"] = 100 # set to large number for this analysis + state_dict4["width"]["WidthFunction"] = 2 # half-lognormal + state_dict4["width"]["ScatFunction"] = 0 # log-uniform true value 0 + state_dict4["width"]["Wlogmean"] = -0.29 # -0.29 + state_dict4["width"]["Wlogsigma"] = 0.65 #0.65 + state_dict4["scat"]["Slogmean"] = -1.38 # true valiue -1.38 + state_dict4["scat"]["Slogsigma"] = 0.01 # not actually used in the log-constant distribution + state_dict4["scat"]["Sfnorm"] = 1000 # 1 GHz normalisation + + state_dicts = [state_dict1, state_dict2, state_dict3, state_dict4] + survey_dicts = [survey_dict1, survey_dict2, survey_dict3, survey_dict4] + + return survey_dicts,state_dicts + +main() diff --git a/papers/Scattering/fit_scattering_width.py b/papers/Scattering/fit_scattering_width.py index 6275adb..d3cf22b 100644 --- a/papers/Scattering/fit_scattering_width.py +++ b/papers/Scattering/fit_scattering_width.py @@ -52,10 +52,18 @@ # number of functions implemented, function names, and initial guesses NFUNC=7 -FNAMES=["lognormal","half-lognormal","boxcar","log constant","smooth boxcar", "upper sb", "lower sb","hln"] -ARGS0=[[0.,1.],[0.,1.],[-2,2],[-2],[-1.,1,1.],[-2.,2,1.],[-2.,2,1.],[0.,1.]] +FNAMES=["lognormal","half-lognormal","boxcar","log-constant","smooth boxcar", "upper sb", "lower sb","hln"] +ARGS0=[[0.,1.],[0.,1.],[-2,2],[-2],[-2.,2,1.],[-2.,2,1.],[-2.,2,1.],[0.,1.]] WARGS0=[[0.,1.],[0.,1.],[-3.5,2],[-3.5],[-1.,1,1.],[-3.5,2,1.],[-3.5,2,1.],[0.,1.]] +latextau = ["\\mu_\\tau, \\sigma_\\tau ", "\\mu_\\tau, \\sigma_\\tau ", "\\tau_{\\rm min},\\tau_{\\rm max} ", + "\\tau_{\\rm min} ","\\tau_{\\rm min},\\tau_{\\rm max},\sigma_\\tau ", + "\\tau_{\\rm min},\\tau_{\\rm max},\sigma_\\tau ","\\tau_{\\rm min},\\tau_{\\rm max},\sigma_\\tau "] + +latexw = ["\\mu_w, \\sigma_w ", "\\mu_w, \\sigma_w ", "w_{\\rm min},w_{\\rm max} ", + "w_{\\rm min} ","w_{\\rm min},w_{\\rm max},\sigma_w ", + "w_{\\rm min},w_{\\rm max},\sigma_w ","w_{\\rm min},w_{\\rm max},\sigma_w "] + # set up min/max ranges of integrations for Bayes factors min_sigma = 0.3 max_sigma = 4 @@ -160,6 +168,28 @@ def main(outdir="Fitting_Outputs/"): + + + + + # generates a table + print("Table of FRB properties for latex") + for i,name in enumerate(tns): + string=name + # detection properties + string += " & " + str(int(DM[i]+0.5)) + " & " + str(z[i])[0:6] + " & " + str(snr[i])[0:4] + string += " & " + str(freq[i])[0:6] + " & " + str(tres[i])[0:6] + " & " + str(w95[i])[0:4] + + # scattering + string += " & " + str(tauobs[i])[0:4] + " & " + str(maxtaus[i])[0:4] + " & " + str(host_tau[i])[0:4] + " & " + str(host_maxtaus[i])[0:4] + + # width + string += " & " + str(wobs[i])[0:4] + " & " + str(maxws[i])[0:4] + " & " + str(host_w[i])[0:4] + " & " + str(host_maxw[i])[0:4] + + # newline + string += " \\\\" + print(string) + print("\n\n\n\n") Nbins=11 bins = np.logspace(-3.,2.,Nbins) @@ -237,8 +267,10 @@ def main(outdir="Fitting_Outputs/"): plt.tight_layout() plt.savefig(outdir+"tau_host_histogram.png") - #### creates a copy of the above, for paper purposes + #### creates a copy of the above, for paper purposes. Does this three times! Once + # for "everything", once for intrinsic, once for observed + # "everything" figures # plt.figure() #obshist,bins = np.histogram(tauobs,bins=bins) #hosthist,bins = np.histogram(host_tau,bins=bins) @@ -268,6 +300,49 @@ def main(outdir="Fitting_Outputs/"): # keeps open for later plotting - don't close this here + # "observed" figures # + plt.figure() + #obshist,bins = np.histogram(tauobs,bins=bins) + #hosthist,bins = np.histogram(host_tau,bins=bins) + ax1v3=plt.gca() + plt.xscale("log") + plt.ylim(0,8) + l1v3 = plt.hist(host_tau,bins=bins,label="Observed",alpha=0.5) + + ax2v3 = ax1v3.twinx() + l3v3 = ax2v3.plot(xvals,yvals,label="Completeness") + + plt.ylim(0,1) + plt.xlim(1e-2) + plt.ylabel("Completeness") + plt.sca(ax1) + plt.xlabel("$\\tau_{\\rm host, 1\,GHz}$ [ms]") + plt.ylabel("Number of FRBs") + # keeps open for later plotting - don't close this here + + # "intrinsic" figures # + plt.figure() + #obshist,bins = np.histogram(tauobs,bins=bins) + #hosthist,bins = np.histogram(host_tau,bins=bins) + ax1v4=plt.gca() + plt.xscale("log") + plt.ylim(0,8) + l2v4 = plt.hist(host_tau,bins=bins,weights = 1./tau_comp,label="Corrected",alpha=0.5) + + + ax2v4 = ax1v4.twinx() + l3v4 = ax2v4.plot(xvals,yvals,label="Completeness") + + plt.ylim(0,1) + plt.xlim(1e-2) + plt.ylabel("Completeness") + plt.sca(ax1) + plt.xlabel("$\\tau_{\\rm host, 1\,GHz}$ [ms]") + plt.ylabel("Intrinsic number of FRBs") + # keeps open for later plotting - don't close this here + + + ####################################### TAU - CDF and fitting ################################ print("\n\n KS test evaluation for tau \n") # amplitude, mean, and std dev of true distribution @@ -278,12 +353,13 @@ def main(outdir="Fitting_Outputs/"): args = (host_tau,xvals,yvals,ifunc) x0=ARGS0[ifunc] - result = sp.optimize.minimize(get_ks_stat,x0=x0,args=args,method = 'Nelder-Mead') + result = sp.optimize.minimize(get_ks_stat,x0=x0,args=args)#,method = 'Nelder-Mead') + psub1 = get_ks_stat(result.x,*args,plot=False) ksbest.append(result.x) #Best-fitting parameters are [0.85909445 1.45509687] with p-value 0.9202915513749959 print("FUNCTION ",ifunc,",",FNAMES[ifunc]," Best-fitting parameters are ",result.x," with p-value ",1.-result.fun) - + xtemp = xvals[::2] ytemp = yvals[::2] @@ -316,6 +392,7 @@ def main(outdir="Fitting_Outputs/"): ############################################### TAU - likelihood analysis ################################################# print("\n\n Max Likelihood Calculation for tau\n") xbest=[] + llbests = [] for ifunc in np.arange(NFUNC): args = (host_tau,cspline,ifunc) x0=ARGS0[ifunc] @@ -325,14 +402,24 @@ def main(outdir="Fitting_Outputs/"): xbest.append(result.x) # llbest returns negative ll llbest = get_ll_stat(result.x,host_tau,cspline,ifunc) * -1 + llbests.append(llbest) - - print("FUNCTION ",ifunc,",",FNAMES[ifunc]," Best-fitting log-likelihood parameters are ",result.x," with p-value ",1.-result.fun) + print("FUNCTION ",ifunc,",",FNAMES[ifunc]," Best-fitting log-likelihood parameters are ",result.x," with likelihood ",-result.fun) print(" , BIC is ",2*np.log(host_tau.size) - len(x0)*llbest) if ifunc == 0: llCHIME = get_ll_stat([CHIME_mut,CHIME_st],host_tau,cspline,ifunc) * -1 print("Compare with CHIME ",llCHIME) + print("\n\nLATEX TABLE") + for ifunc in np.arange(NFUNC): + string = FNAMES[ifunc] + " & $" + latextau[ifunc] + f"$ & ${xbest[ifunc][0]:.2f}" + for iarg, arg in enumerate(xbest[ifunc]): + if iarg==0: + continue + string += f", {xbest[ifunc][iarg]:.2f}" + string += f" $ & {llbests[ifunc]:.2f} \\\\" + print(string) + print("\n\n\n\n") make_cdf_plot(xbest,host_tau,xvals,yvals,outdir+"bestfit_ll_scat_cumulative.png",cspline) @@ -362,8 +449,9 @@ def main(outdir="Fitting_Outputs/"): plt.savefig(outdir+"tau_host_histogram_fits.png") plt.close() - ######## plot for paper ######## + ######## plots for paper ######## + #"everything" plt.sca(ax1v2) NFRB=host_tau.size handles=[l1v2[2],l3v2[0],l2v2[2]] @@ -390,6 +478,59 @@ def main(outdir="Fitting_Outputs/"): plt.savefig(outdir+"paper_tau_host_histogram_fits.png") plt.close() + #"observed" + plt.sca(ax1v3) + NFRB=host_tau.size + handles=[l1v3[2],l3v3[0]] + labels=["Observed","Completeness"] + for i in [0,2,3]: + print("plotting function ",i," with xbest ",xbest[i]) + xs,ys = function_wrapper(i,xbest[i],logxmax=2)#cspline=None): + plotnorm = NFRB * (np.log10(bins[1])-np.log10(bins[0])) + #l=plt.plot(xs,ys*plotnorm,label=FNAMES[i]) + + xs,ys = function_wrapper(i,xbest[i],cspline=cspline) + l=plt.plot(xs,ys*plotnorm,label=FNAMES[i]) + handles.append(l[0]) + labels.append(FNAMES[i]) + + plt.xscale("log") + plt.xlim(1e-2,1e3) + + plt.text(1e-3,8,"(a)",fontsize=18) + plt.legend() + plt.xlabel("$\\tau_{\\rm host, 1\,GHz}$ [ms]") + plt.ylabel("Observed number of FRBs") + plt.legend(handles=handles,labels=labels,fontsize=10) #fontsize=12) + + plt.tight_layout() + plt.savefig(outdir+"observed_paper_tau_host_histogram_fits.png") + plt.close() + + # "intrinsic" + plt.sca(ax1v4) + NFRB=host_tau.size + handles=[l2v4[2],l3v4[0]] + labels=["Adjusted","Completeness"] + for i in [0,2,3]: + print("plotting function ",i," with xbest ",xbest[i]) + xs,ys = function_wrapper(i,xbest[i],logxmax=2)#cspline=None): + plotnorm = NFRB * (np.log10(bins[1])-np.log10(bins[0])) + l=plt.plot(xs,ys*plotnorm,label=FNAMES[i]) + handles.append(l[0]) + labels.append(FNAMES[i]) + plt.xscale("log") + plt.xlim(1e-2,1e3) + + plt.text(1e-3,8,"(b)",fontsize=18) + plt.legend() + plt.xlabel("$\\tau_{\\rm host, 1\,GHz}$ [ms]") + plt.ylabel("Intrinsic number of FRBs") + plt.legend(handles=handles,labels=labels,fontsize=10) #fontsize=12) + + plt.tight_layout() + plt.savefig(outdir+"intrinsic_paper_tau_host_histogram_fits.png") + plt.close() ############################################# TAU - bayes factor ####################################### # priors @@ -513,7 +654,7 @@ def main(outdir="Fitting_Outputs/"): plt.ylabel("Completeness") plt.sca(ax1) plt.legend(handles=[l1[2],l3[0],l2[2]],labels=["Observed","Completeness","Corrected"],fontsize=12) - plt.xlabel("$w_{\\rm host}$ [ms]") + plt.xlabel("$w_{i,\\rm host}$ [ms]") plt.ylabel("Number of FRBs") plt.tight_layout() plt.savefig(outdir+"w_host_histogram.png") @@ -522,6 +663,7 @@ def main(outdir="Fitting_Outputs/"): #### new plot, for paper - just a copy of the above #### + #"everything" plt.figure() plt.xscale("log") plt.ylim(0,8) @@ -540,14 +682,52 @@ def main(outdir="Fitting_Outputs/"): l3v2 = ax2v2.plot(wxvals,wyvals,label="Completeness")#,color=use_this_color) plt.ylim(0,1) - plt.xlim(1e-3,1e3) plt.ylabel("Completeness") plt.sca(ax1v2) plt.legend(handles=[l1v2[2],l3v2[0],l2v2[2]],labels=["Observed","Completeness","Corrected"],fontsize=12) - plt.xlabel("$w_{\\rm host}$ [ms]") + plt.xlabel("$w_{i,\\rm host}$ [ms]") plt.ylabel("Number of FRBs") + # "observed" figures # + plt.figure() + #obshist,bins = np.histogram(tauobs,bins=bins) + #hosthist,bins = np.histogram(host_tau,bins=bins) + ax1v3=plt.gca() + plt.xscale("log") + plt.ylim(0,11) + l1v3 = plt.hist(host_w,bins=bins,label="Observed",alpha=0.5) + + ax2v3 = ax1v3.twinx() + l3v3 = ax2v3.plot(wxvals,wyvals,label="Completeness") + + plt.ylim(0,1) + plt.ylabel("Completeness") + plt.sca(ax1) + plt.xlabel("$w_{i,\\rm host}$ [ms]") + plt.ylabel("Number of FRBs") + # keeps open for later plotting - don't close this here + + # "intrinsic" figures # + plt.figure() + #obshist,bins = np.histogram(tauobs,bins=bins) + #hosthist,bins = np.histogram(host_tau,bins=bins) + ax1v4=plt.gca() + plt.xscale("log") + plt.ylim(0,11) + l2v4 = plt.hist(host_w,bins=bins,weights = 1./w_comp,label="Corrected",alpha=0.5) + + + ax2v4 = ax1v4.twinx() + l3v4 = ax2v4.plot(wxvals,wyvals,label="Completeness") + + plt.ylim(0,1) + plt.ylabel("Completeness") + plt.sca(ax1) + plt.xlabel("$w_{\\rm host}$ [ms]") + plt.ylabel("Intrinsic number of FRBs") + # keeps open for later plotting - don't close this here + ####################### W - likelihood maximisation ################# ####################################### Width - CDF and fitting ################################ @@ -563,8 +743,9 @@ def main(outdir="Fitting_Outputs/"): result = sp.optimize.minimize(get_ks_stat,x0=x0,args=args,method = 'Nelder-Mead') psub1 = get_ks_stat(result.x,*args,plot=False) ksbest.append(result.x) - print("FUNCTION ",ifunc,",",FNAMES[ifunc]," Best-fitting parameters are ",result.x," with p-value ",1.-result.fun) - + print("FUNCTION ",ifunc,",",FNAMES[ifunc]," Best-fitting parameters are ",result.x," with p-value ",-result.fun) + + print("\n\n Maximum likelihood for width \n") ### makes temporary values for completeness @@ -582,6 +763,7 @@ def main(outdir="Fitting_Outputs/"): xt[-1] = 1e5 yt[-1] = 0. cspline = sp.interpolate.make_interp_spline(np.log10(xt), yt,k=1) + # do a test plot of the spline? if True: plt.figure() @@ -598,13 +780,15 @@ def main(outdir="Fitting_Outputs/"): # amplitude, mean, and std dev of true distribution xbest=[] - + llbests = [] # iterate over functions to calculate max likelihood for ifunc in np.arange(NFUNC): args = (host_w,cspline,ifunc) x0=WARGS0[ifunc] result = sp.optimize.minimize(get_ll_stat,x0=x0,args=args,method = 'Nelder-Mead') + llbest = get_ll_stat(result.x,host_w,cspline,ifunc) * -1 + llbests.append(llbest) #psub1 = get_ks_stat(result.x,*args,plot=True) xbest.append(result.x) print("width FUNCTION ",ifunc,",",FNAMES[ifunc]," Best-fitting log-likelihood parameters are ",result.x," with p-value ",1.-result.fun) @@ -612,6 +796,20 @@ def main(outdir="Fitting_Outputs/"): llCHIME = get_ll_stat([CHIME_muw,CHIME_sw],host_tau,cspline,ifunc) * -1 print("Compare with CHIME ",llCHIME) + + print("\n\nLATEX TABLE") + for ifunc in np.arange(NFUNC): + string = FNAMES[ifunc] + " & $" + latexw[ifunc] + f"$ & ${xbest[ifunc][0]:.2f}" + for iarg, arg in enumerate(xbest[ifunc]): + if iarg==0: + continue + string += f", {xbest[ifunc][iarg]:.2f}" + string += f" $ & {llbests[ifunc]:.2f} \\\\" + print(string) + print("\n\n\n\n") + + + make_cdf_plot(xbest,host_w,wxvals,wyvals,outdir+"bestfit_ll_width_cumulative.png",cspline,width=True) @@ -635,8 +833,6 @@ def main(outdir="Fitting_Outputs/"): plt.legend() plt.legend(handles=handles,labels=labels,fontsize=6) #fontsize=12) - - plt.savefig(outdir+"w_host_histogram_fits.png") plt.close() @@ -670,6 +866,62 @@ def main(outdir="Fitting_Outputs/"): plt.savefig(outdir+"paper_w_host_histogram_fits.png") plt.close() + + #"observed" + plt.sca(ax1v3) + NFRB=host_tau.size + handles=[l1v3[2],l3v3[0]] + labels=["Observed","Completeness"] + for i in [0,1,3]: + print("plotting function ",i," with xbest ",xbest[i]) + xs,ys = function_wrapper(i,xbest[i],logxmax=2)#cspline=None): + plotnorm = NFRB * (np.log10(bins[1])-np.log10(bins[0])) + #l=plt.plot(xs,ys*plotnorm,label=FNAMES[i]) + + xs,ys = function_wrapper(i,xbest[i],cspline=cspline) + l=plt.plot(xs,ys*plotnorm,label=FNAMES[i]) + handles.append(l[0]) + labels.append(FNAMES[i]) + + plt.xscale("log") + plt.xlim(5e-3,1e2) + + plt.text(1e-3,10.5,"(a)",fontsize=18) + plt.legend() + plt.xlabel("$w_{\\rm host}$ [ms]") + plt.ylabel("Observed number of FRBs") + plt.legend(handles=handles,labels=labels,fontsize=10) #fontsize=12) + + plt.tight_layout() + plt.savefig(outdir+"observed_paper_width_host_histogram_fits.png") + plt.close() + + # "intrinsic" + plt.sca(ax1v4) + NFRB=host_tau.size + handles=[l2v4[2],l3v4[0]] + labels=["Adjusted","Completeness"] + for i in [0,1,3]: + print("plotting function ",i," with xbest ",xbest[i]) + xs,ys = function_wrapper(i,xbest[i],logxmax=2)#cspline=None): + plotnorm = NFRB * (np.log10(bins[1])-np.log10(bins[0])) + l=plt.plot(xs,ys*plotnorm,label=FNAMES[i]) + handles.append(l[0]) + labels.append(FNAMES[i]) + plt.xscale("log") + plt.xlim(5e-3,1e2) + + plt.text(1e-3,10.5,"(b)",fontsize=18) + plt.legend() + plt.xlabel("$w_{\\rm host}$ [ms]") + plt.ylabel("Intrinsic number of FRBs") + plt.legend(handles=handles,labels=labels,fontsize=10) #fontsize=12) + + plt.tight_layout() + plt.savefig(outdir+"intrinsic_paper_width_host_histogram_fits.png") + plt.close() + + ############################################# WIDTH - bayes factor ####################################### # priors # uses Galactic pulsars as example @@ -814,7 +1066,7 @@ def get_ll_stat(args,tau_obs,cspline,ifunc,plot=False, log_min=-5, log_max = 5, A=1. ftaus = function_wrapper(ifunc,args,taus,cspline=cspline) - ptaus = function_wrapper(ifunc,args,tau_obs,cspline=cspline) + ptaus = function_wrapper(ifunc,args,tau_obs,cspline=cspline) #normalisation should happen here # checks normalisation in log-space #modified = ctaus*ftaus diff --git a/papers/Scattering/width_scat_dists.py b/papers/Scattering/width_scat_dists.py deleted file mode 100644 index a7201fe..0000000 --- a/papers/Scattering/width_scat_dists.py +++ /dev/null @@ -1,279 +0,0 @@ -""" -This script plots observed and fitted width and scattering distributions from the zdm code - -i.e. it lods in the CRAFT ICS surveys, models the width and scattering distributions -that are best-fit from the scattering paper, then generates "Plots/differential.png" -whcih compares the observed and modelled distributions. - -""" - -import os - -from zdm import cosmology as cos -from zdm import figures -from zdm import parameters -from zdm import survey -from zdm import pcosmic -from zdm import misc_functions as mf -from zdm import iteration as it -from zdm import loading -from zdm import io -from zdm import figures as fig -from pkg_resources import resource_filename -import numpy as np -from matplotlib import pyplot as plt - -import matplotlib - -defaultsize=14 -ds=4 -font = {'family' : 'Helvetica', - 'weight' : 'normal', - 'size' : defaultsize} -matplotlib.rc('font', **font) - - -def main(): - """ - Main function - """ - - # in case you wish to switch to another output directory - opdir = "Plots/" - if not os.path.exists(opdir): - os.mkdir(opdir) - - # directory where the survey files are located. The below is the default - - # you can leave this out, or change it for a different survey file location. - sdir = os.path.join(resource_filename('zdm', 'data'), 'Surveys') - - # make this into a list to initialise multiple surveys art once - names = ["CRAFT_ICS_892","CRAFT_ICS_1300","CRAFT_ICS_1632"] # for example - - repeaters=False - # sets plotting limits - zmax = 2. - dmmax = 2000 - - survey_dict = {"WMETHOD": 3} - state_dict = {} - state_dict["scat"] = {} - state_dict["scat"]["Sbackproject"] = True # turns on backprojection of tau and width for our model - state_dict["width"] = {} - state_dict["width"]["WNInternalBins"] = 1000 # sets it to a small quantity - state_dict["width"]["WNbins"] = 33 # set to large number for this analysis - - # best-fit results for half-log-normal distributions - state_dict["width"]["WidthFunction"] = 2 - state_dict["scat"]["ScatFunction"] = 2 - state_dict["width"]["Wlogmean"] = -0.29 - state_dict["width"]["Wlogsigma"] = 0.65 - state_dict["scat"]["Slogmean"] = -1.3 - state_dict["scat"]["Slogsigma"] = 0.2 - - - surveys, grids = loading.surveys_and_grids(survey_names = names,\ - repeaters=repeaters, sdir=sdir,nz=70,ndm=140, - survey_dict = survey_dict, state_dict = state_dict) - - # we now generate plots for the FRBs with known tau and width values - # iwidths - ilist = [] - wlist = [] - tlist = [] - - # sets up a figure to do a scatter plot of frequency dependence of these widths - plt.figure() - plt.xlabel("Frequency [MHz]") - plt.ylabel("$\\log_{10} t$ [ms]") - for i,s in enumerate(surveys): - - nfrb = len(s.OKTAU) - - - taus = s.TAUs[s.OKTAU] - widths = s.frbs['WIDTH'].values[s.OKTAU] - iwidths = s.IWIDTHs[s.OKTAU] - - logtaus = np.log10(taus) - logwidths = np.log10(widths) - logis = np.log10(iwidths) - - taubar = np.average(logtaus) - taurms = (np.sum((logtaus - taubar)**2) / (nfrb-1))**0.5 - - - ibar = np.average(logis) - irms = (np.sum((logis - ibar)**2) / (nfrb-1))**0.5 - - - wbar = np.average(logwidths) - wrms = (np.sum((logwidths - wbar)**2) / (nfrb-1))**0.5 - - freqs = s.frbs['FBAR'][s.OKTAU] - fbar = np.average(freqs) - - if i==0: - l1,=plt.plot(freqs, logwidths,marker='s',label="$w_{\\rm app}$",linestyle="") - l2,=plt.plot(freqs, logtaus,marker="+",label="$\\tau$",linestyle="") - l3,=plt.plot(freqs, logis,marker="x",label="$w_i$",linestyle="") - else: - plt.plot(freqs, logwidths,marker='s',color=l1.get_color(),linestyle="") - plt.plot(freqs, logtaus,marker="+",color=l2.get_color(),linestyle="") - plt.plot(freqs, logis,marker="x",color=l3.get_color(),linestyle="") - - plt.errorbar([fbar-20],[wbar],yerr=[wrms],color=l1.get_color(),linewidth=2,capsize=5) - plt.errorbar([fbar],[taubar],yerr=[taurms],color=l2.get_color(),linewidth=2,capsize=6) - plt.errorbar([fbar+20],[ibar],yerr=[irms],color=l3.get_color(),linewidth=2,capsize=7) - - for index in s.OKTAU: - ilist.append(s.IWIDTHs[index]) - tlist.append(s.TAUs[index]) - wlist.append(s.frbs['WIDTH'].values[index]) - - - plt.legend() - plt.tight_layout() - # we do not save, because the figure is already done - #plt.savefig(opdir+"freq_scat.png") - plt.close() - - ####### We now plot against expectations ######## - # For each of w,i,and tau, we plot the true modelled distribution, - # the de-biased distribution, and the observed distribution - # We do this as both a cumulative distribution, and a pdf - # This is also done for all bursts summed together. - - # generate cumulative distributions of these quantities - xi,yi = fig.gen_cdf_hist(iwidths) - xw,yw = fig.gen_cdf_hist(widths) - xt,yt = fig.gen_cdf_hist(taus) - - - for i,s in enumerate(surveys): - g =grids[i] - ################ Generates some plots ################ - - ##### gets the expected, de-biased distributions ##### - # extracts the p(W) distribution - Rw,Rtau,Nw,Nwz,Nwdm = mf.get_width_stats(s,g) - - # normalise each survey to actual number of FRBs - nfrb = len(s.OKTAU) - - # these need to be normalised by the internal bin width - logbinwidth = s.internal_logwvals[-1] - s.internal_logwvals[-2] - - # divide these histograms by the width of the log bins to get probability distributions - if i==0: - # intrinsic width, ptau, total width - SumRw = Rw / np.sum(Rw) * nfrb / logbinwidth #has Ninternalbins - SumRtau = Rtau / np.sum(Rtau) * nfrb / logbinwidth - - SumNw = Nw / np.sum(Nw) * nfrb / s.dlogw # has NWbins - else: - SumRw += Rw / np.sum(Rw) * nfrb - SumRtau += Rtau / np.sum(Rtau) * nfrb - SumNw += Nw / np.sum(Nw) * nfrb / s.dlogw - - ##### gets the underlying means ####### - # values at z=0 - WidthArgs = (s.wlogmean,s.wlogsigma) - ScatArgs = (s.slogmean,s.slogsigma) - - pw = s.WidthFunction(s.internal_logwvals, *WidthArgs)#*s.dlogw #/logbinwidth - ptau = s.ScatFunction(s.internal_logwvals, *ScatArgs)#*s.dlogw #/logbinwidth - - if i==0: - # intrinsic width, ptau, total width - Sumpw = pw / np.sum(pw) * nfrb - Sumptau = ptau / np.sum(ptau) * nfrb - else: - Sumpw += pw / np.sum(pw) * nfrb - Sumptau += ptau / np.sum(ptau) * nfrb - - # we now plot these differential distributions against histograms of the observations - - #### Plot 1: Intrinsic vs detected distributions ##### - plt.figure() - plt.xscale("log") - plt.yscale("log") - plt.ylim(1e-1,100) - plt.xlim(1e-2,1e2) - - l1,=plt.plot(s.wlist,SumNw,label="Total widths") - l2,=plt.plot(10**s.internal_logwvals,SumRtau,label="Scattering",linestyle="-") - l3,=plt.plot(10**s.internal_logwvals,SumRw,label="Intrinsic width",linestyle=":") - - bins=np.logspace(-2.01,2.01,9) - lbw = np.log10(bins[1]/bins[0]) - #normalisation: p per log bin * Nfrb. - weights = np.full([len(wlist)],1./lbw) - alpha=1.0 - plt.hist(wlist,bins=bins,weights=weights,alpha=alpha,facecolor = l1.get_color(), - edgecolor = l1.get_color(),linewidth=2,histtype='step', label="Observed total") - plt.hist(tlist,bins=bins,weights=weights,alpha=alpha,facecolor = l2.get_color(), - edgecolor = l2.get_color(),linewidth=2,histtype='step', label="Observed scattering") - plt.hist(ilist,bins=bins,weights=weights,alpha=alpha,facecolor = l3.get_color(), - edgecolor = l3.get_color(),linewidth=2,histtype='step', label="Observed intrinsic") - - - plt.legend() - plt.tight_layout() - plt.savefig(opdir+"differential.png") - - plt.close() - exit() - - plt.plot(ws,s.wplist/n1,label="Total width (z=0)",linestyle="-") - plt.plot(ws,Nw/n2,label="Detected total",linestyle=":",color=plt.gca().lines[-1].get_color()) - - plt.plot(10**s.internal_logwvals,ptau/n3,label="Scattering (z=0)",linestyle="-") - plt.plot(10**s.internal_logwvals,Rtau/n4,label="Detected scattering",linestyle=":",color=plt.gca().lines[-1].get_color()) - - plt.plot(10**s.internal_logwvals,pw/n5,label="Intrinsic widths (z=0)",linestyle="-") - plt.plot(10**s.internal_logwvals,Rw/n6,label="Detected width",linestyle=":",color=plt.gca().lines[-1].get_color()) - - plt.xlabel("width [ms]") - plt.ylabel("$\\rm p(w) d\\log_{10} w$") - plt.legend() - plt.tight_layout() - plt.savefig(opdir+"pw.png") - plt.close() - - - #### Plot 2: Redsift dependence ##### - - plt.figure() - plt.xscale("log") - plt.yscale("log") - plt.ylim(1e-3,None) - plt.plot(ws,s.wplist/np.sum(s.wplist),label="Intrinsic width",color="black") - plt.plot(ws,Nw/np.sum(Nw),label="Detected FRBs",linestyle="--") - plt.plot(ws,Nwz[:,4]/np.sum(Nwz[:,4]),label=" (z=0.25)",linestyle="--") - plt.plot(ws,Nwz[:,18]/np.sum(Nwz[:,18]),label=" (z=1.25)",linestyle=":") - plt.xlabel("FRB effective width [ms]") - plt.ylabel("FRBs/day") - plt.legend() - plt.tight_layout() - plt.savefig(opdir+"pw_zdep.png") - plt.close() - - #### Plot 3: DM dependence ##### - - plt.figure() - plt.xscale("log") - plt.yscale("log") - plt.ylim(1e-3,None) - plt.plot(ws,s.wplist/np.sum(s.wplist),label="Intrinsic width",color="black") - plt.plot(ws,Nw/np.sum(Nw),label="Detected FRBs",linestyle="--") - plt.plot(ws,Nwdm[:,3]/np.sum(Nwdm[:,3]),label=" (DM=125)",linestyle="--") - plt.plot(ws,Nwdm[:,21]/np.sum(Nwdm[:,21]),label=" (DM=1025)",linestyle=":") - plt.xlabel("FRB effective width [ms]") - plt.ylabel("FRBs/day") - plt.legend() - plt.tight_layout() - plt.savefig(opdir+"pw_dmdep.png") - plt.close() - -main() diff --git a/zdm/data/Surveys/SKA_low.ecsv b/zdm/data/Surveys/SKA_low.ecsv index 1bf4761..b7a1f77 100644 --- a/zdm/data/Surveys/SKA_low.ecsv +++ b/zdm/data/Surveys/SKA_low.ecsv @@ -23,4 +23,4 @@ # \ \"DIAM\": 40,\n \"BMETHOD\": 0,\n \"NBEAMS\": 1,\n \"NBINS\": 10\n }\n}"} # schema: astropy-2.0 TNS BW DM DMG FBAR FRES Gb Gl NREP SNR SNRTHRESH THRESH TRES WIDTH XDec XRA Z -0 770 200 35.0 1284 0.0145 "" "" 1 10. 10. 0.2 0.069 1.0 "" "" 0.276 +0 120 200 35.0 1284 0.0145 "" "" 1 10. 10. 0.2 0.069 1.0 "" "" 0.276 diff --git a/zdm/data/Surveys/SKA_mid.ecsv b/zdm/data/Surveys/SKA_mid.ecsv index 476400c..4fd949c 100644 --- a/zdm/data/Surveys/SKA_mid.ecsv +++ b/zdm/data/Surveys/SKA_mid.ecsv @@ -23,4 +23,4 @@ # \ \"DIAM\": 15,\n \"BMETHOD\": 0,\n \"NBEAMS\": 1,\n \"NBINS\": 10\n }\n}"} # schema: astropy-2.0 TNS BW DM DMG FBAR FRES Gb Gl NREP SNR SNRTHRESH THRESH TRES WIDTH XDec XRA Z -0 770 200 35.0 1284 0.1075 "" "" 1 10. 10. 0.2 0.069 1.0 "" "" 0.276 +0 300 200 35.0 1284 0.1075 "" "" 1 10. 10. 0.2 0.069 1.0 "" "" 0.276 diff --git a/zdm/loading.py b/zdm/loading.py index ae6a018..8f704e8 100644 --- a/zdm/loading.py +++ b/zdm/loading.py @@ -178,6 +178,7 @@ def load_CHIME(Nbin:int=6, make_plots:bool=False, opdir='CHIME/',\ def surveys_and_grids(init_state=None, alpha_method=1, survey_names=None, nz:int=500, ndm:int=1400, + zmax=5, dmmax=7000, NFRB=None, repeaters=False, sdir=None, edir=None, rand_DMG=False, discard_empty=False, @@ -199,6 +200,10 @@ def surveys_and_grids(init_state=None, alpha_method=1, Number of redshift bins ndm (int, optional): Number of DM bins + zmax (float,optional): + Maximum value of redshift + dmmax (float,optional): + Maximum value of dispersion measure edir (string, optional): Directory containing efficiency files if using FRB-specific responses rand_DMG (bool, optional): @@ -229,7 +234,7 @@ def surveys_and_grids(init_state=None, alpha_method=1, # get the grid of p(DM|z) zDMgrid, zvals,dmvals = misc_functions.get_zdm_grid( state, new=True, plot=False, method='analytic', - nz=nz, ndm=ndm, + nz=nz, ndm=ndm, zmax=zmax, dmmax=dmmax, datdir=resource_filename('zdm', 'GridData')) ############## Initialise surveys ############## diff --git a/zdm/survey.py b/zdm/survey.py index 02f864e..a75337b 100644 --- a/zdm/survey.py +++ b/zdm/survey.py @@ -170,6 +170,7 @@ def init_widths(self,state=None): #wbins[0] = wbins[1]-dlogw # no longer tint: 1.e-10 # set to a tiny value, to ensure we capture all small widths # offsets the mean by half the log-spacing for each wlist = np.logspace(np.log10(self.WMin)+dlogw/2.,np.log10(self.WMax)-dlogw/2.,self.NWbins) + wbins[0] -= 3 # ensures we capture low values! else: wbins[0] = np.log10(self.WMin) wbins[1] = np.log10(self.WMax) @@ -183,7 +184,7 @@ def init_widths(self,state=None): #minval = np.min([self.wlogmean - self.maxsigma*self.wlogsigma, # self.slogmean - self.maxsigma*self.slogsigma, # np.log10(self.WMin)]) - minval = np.log10(self.WMin) + minval = np.log10(self.WMin)-3 maxval = np.log10(self.WMax) # I haven't decided yet where to put the value of 1000 internal bins # in terms of parameters. @@ -285,9 +286,15 @@ def make_widths(self,iz=None): wlogsigma = self.wlogsigma slogmean = self.slogmean - 3.*np.log10(1+z) slogsigma = self.slogsigma + # we need these for normalisation purposes. Otherwise, the + # scattering distribution gets diluted with a constant max value + # these are only used for normalisation purposes of otherwise + # unnormalisable functions. + smax = np.log10(self.WMax) - 3.*np.log10(1+z) + wmax = np.log10(self.WMax) + np.log10(1+z) - WidthArgs = (wlogmean,wlogsigma) - ScatArgs = (slogmean,slogsigma) + WidthArgs = (wlogmean,wlogsigma,wmax) + ScatArgs = (slogmean,slogsigma,smax) if self.backproject: dist,pw,ptau = quadrature_convolution(self.WidthFunction, WidthArgs, self.ScatFunction, ScatArgs, @@ -296,6 +303,7 @@ def make_widths(self,iz=None): dist = quadrature_convolution(self.WidthFunction, WidthArgs, self.ScatFunction, ScatArgs, self.internal_logwvals, self.wbins,backproject=self.backproject) + if iz is not None: self.wplist[:,iz] = dist if self.backproject: @@ -1346,16 +1354,16 @@ def quadrature_convolution(width_function, width_args, scat_function, scat_args, ''' - #draw from both distributions - np.random.seed(1234) - # these need to be normalised by the internal bin width logbinwidth = internal_logvals[-1] - internal_logvals[-2] + # these functions should *not* be normalsid, since some true distribution + # may extend beyond the range of interest. But it means that the below functions + # absolutely should be correctly normalised + pw = width_function(internal_logvals, *width_args)*logbinwidth ptau = scat_function(internal_logvals, *scat_args)*logbinwidth - # adds extra bits onto the lowest bin. Does this by integrating in # log space. Assumes exp(-10) is small enough! lowest = internal_logvals[0] - logbinwidth/2. @@ -1467,9 +1475,10 @@ def lognormal(log10w, *args): logsigma = args[1] norm = (2.*np.pi)**-0.5/logsigma result = norm * np.exp(-0.5 * ((log10w - logmean) / logsigma) ** 2) + return result -def halflognormal(log10w, *args,logmax=3):#logmean,logsigma,minw,maxw,nbins): +def halflognormal(log10w, *args):#logmean,logsigma,minw,maxw,nbins): """ Generates a parameterised half-lognormal distribution. This acts as a lognormal in the lower half, but @@ -1483,13 +1492,14 @@ def halflognormal(log10w, *args,logmax=3):#logmean,logsigma,minw,maxw,nbins): Args: log10w: log base 10 of widths - args: vector of [logmean,logsigma] mean and std dev + args: vector of [logmean,logsigma] mean and std dev and max value Returns: result: p(log10w) d log10w """ logmean = args[0] logsigma = args[1] + logmax = args[2] norm = (2.*np.pi)**-0.5/logsigma #Currently no normalisation if hasattr(log10w,"__len__"): large = np.where(log10w > logmean)[0] @@ -1504,34 +1514,40 @@ def halflognormal(log10w, *args,logmax=3):#logmean,logsigma,minw,maxw,nbins): result = lognormal(modlogw,logmean,logsigma) - if logmax is not None: - # normalises the distribution. We note that the lower half - # is correctly normalised to 0.5 via the lognormal function - # the upper half spans the range [logmax-logmean] at amplitude - # norm. Hence, the total integral is - # 0.5 + [logmax-logmean]*norm - result /= (0.5 + (logmax-logmean)*norm) + # normalises the distribution. We note that the lower half + # is correctly normalised to 0.5 via the lognormal function + # the upper half spans the range [logmax-logmean] at amplitude + # norm. Hence, the total integral is + # 0.5 + [logmax-logmean]*norm + result /= (0.5 + (logmax-logmean)*norm) return result def constant(log10w,*args): """ - Dummy function that returns a constant of unity. + Dummy function that returns a constant of unity, down + to a certain minimum, below which it is zero. NOTE: to include 1+z scaling here, one will need to reduce the minimum width argument with z. Feature to be added. Maybe have args also contain min and max values? Args: log10w: log base 10 of widths - args: vector of [logmean,logsigma] mean and std dev + args: vector of [logmean,logsigma] mean and std dev and max width Returns: result: p(logw) d logw """ + + width = args[2] - args[0] if hasattr(log10w,"__len__"): - nvals = log10w.size - result = np.full([nvals],1.) + good = np.where(log10w > args[0])[0] + result = np.zeros([log10w.size]) + result[good] = 1./width else: - result = np.array([1]) + if log10w < args[0]: + result = np.array([0]) + else: + result = np.array([1./width]) return result From 98977b386c3855c0c304d7ceffe7565845763529 Mon Sep 17 00:00:00 2001 From: Clancy James Date: Wed, 8 Oct 2025 09:38:37 +0800 Subject: [PATCH 16/20] Minor mods to plotting scripts. Corresponds to arXiv version --- papers/Scattering/compare_width_effects.py | 147 +++++++++++++++++-- papers/Scattering/estimate_2d_effect.py | 161 +++++++++++++++++++++ papers/Scattering/fit_scattering_width.py | 10 +- 3 files changed, 299 insertions(+), 19 deletions(-) create mode 100644 papers/Scattering/estimate_2d_effect.py diff --git a/papers/Scattering/compare_width_effects.py b/papers/Scattering/compare_width_effects.py index 5ea2a01..e3c8ebe 100644 --- a/papers/Scattering/compare_width_effects.py +++ b/papers/Scattering/compare_width_effects.py @@ -35,8 +35,17 @@ def main(): See main description at top of file. """ + Load = True + # in case you wish to switch to another output directory - opdir = "Comparisons/" + + if False: + opdir = "FastComparisons/" + names = ["FAST"] + else: + opdir = "Comparisons/" + names = ["CRAFT_ICS_1300"] # for example + if not os.path.exists(opdir): os.mkdir(opdir) @@ -46,7 +55,8 @@ def main(): # make this into a list to initialise multiple surveys art once. But here, we just use the standard # CRAFT ICS survey - names = ["CRAFT_ICS_1300"] # for example + #names = ["CRAFT_ICS_1300"] # for example + # gets four cases of relevance survey_dicts,state_dicts = get_defined_states() @@ -55,27 +65,40 @@ def main(): repeaters=False # sets limits for calculations - both plotting, nad intrinsic. We want things to go quickly! - zmax = 2. - dmmax = 2000 + zmax = 5. + dmmax = 5000 + ndm=1000 + nz=1000 rates = [] for i,state_dict in enumerate(state_dicts): - + if Load: + continue survey_dict = survey_dicts[i] surveys, grids = loading.surveys_and_grids(survey_names = names,\ - repeaters=repeaters, sdir=sdir,nz=400,ndm=400,zmax=2,dmmax=2000, + repeaters=repeaters, sdir=sdir,nz=nz,ndm=ndm,zmax=zmax,dmmax=dmmax, survey_dict = survey_dict, state_dict = state_dict) s = surveys[0] g = grids[0] rates.append(g.rates) - + + if Load: + rates = np.load(opdir+"rates.npy") + dmvals = np.load(opdir+"dmvals.npy") + zvals = np.load(opdir+"zvals.npy") + else: + np.save(opdir+"rates.npy",rates) + dmvals = g.dmvals + zvals = g.zvals + np.save(opdir+"dmvals.npy",dmvals) + np.save(opdir+"zvals.npy",zvals) # now we have rates, let's look at the effects! - labels=["No width/scattering","CHIME Catalogue 1","lognormals (this work)","Best fit (this work)"] - styles=[":","-.","--","-"] + labels=["No width/scattering","CHIME Catalogue 1","Lognormals (this work)","Best fit (this work)","(numerical approx)"] + styles=[":","-.","--","-",":"] # generates a plot of p(z) plt.figure() @@ -85,16 +108,39 @@ def main(): plt.figure() ax2 = plt.gca() - dz = g.zvals[1]-g.zvals[0] - ddm = g.dmvals[1]-g.dmvals[0] + # generates a plot of p(z) relative to "truth" + plt.figure() + ax3 = plt.gca() + + # generates a plot of p(dm) relativer to "truth" + plt.figure() + ax4 = plt.gca() + + dz = zvals[1]-zvals[0] + ddm = dmvals[1]-dmvals[0] + + zdists=[] + dmdists=[] for i,r in enumerate(rates): + r /= np.sum(r) zdist = np.sum(r,axis=1) dmdist = np.sum(r,axis=0) + zdists.append(zdist) + dmdists.append(dmdist) + + + inorm=3 # this is the "correct" one + for i,r in enumerate(rates): + zdist = zdists[i] + dmdist = dmdists[i] + + ax1.plot(zvals,zdist/dz,label=labels[i],linestyle=styles[i]) + ax2.plot(dmvals,dmdist/ddm,label=labels[i],linestyle=styles[i]) - ax1.plot(g.zvals,zdist/dz,label=labels[i],linestyle=styles[i]) - ax2.plot(g.dmvals,dmdist/ddm,label=labels[i],linestyle=styles[i]) + ax3.plot(zvals,zdist/zdists[inorm],label=labels[i],linestyle=styles[i]) + ax4.plot(dmvals,dmdist/dmdists[inorm],label=labels[i],linestyle=styles[i]) plt.sca(ax1) plt.legend() @@ -111,10 +157,67 @@ def main(): plt.legend() plt.xlabel("DM [pc cm$^{-3}$]") plt.ylabel("p(DM)") + plt.yscale("log") plt.tight_layout() plt.savefig(opdir+"dmdists.png") + plt.close() + + + #### relative plots #### + plt.sca(ax3) + plt.legend() + plt.xlabel("$z$") + plt.ylabel("$p(z)/p_{\\rm best}(z)$") + #plt.yscale("log") + #plt.ylim(0,1e-3) + plt.xlim(0,2) + plt.tight_layout() + plt.savefig(opdir+"rel_zdists.png") + plt.close() + + plt.sca(ax4) + plt.legend() + plt.xlabel("DM [pc cm$^{-3}$]") + plt.ylabel("p(DM)") + plt.tight_layout() + plt.savefig(opdir+"rel_dmdists.png") plt.close() + + ##### Plots of p(DM) for specific z, illustrating the z-dependence of DM bias ##### + plt.figure() + + pzvals = [0.01,1]#0.5,1,1.5,2] + colors=[] + jnorm=3 + for i,z in enumerate(pzvals): + # gets index + iz = np.where(zvals <= z)[0][-1] + + for j,g in enumerate(rates): + + if i==0: + label=labels[j] + else: + label=None + pdm = rates[j][iz,:] + relpdm = pdm/rates[jnorm][iz,:] + if i==0: + color=None + else: + color=colors[j] + p=plt.plot(dmvals,relpdm+i*0.4,label=label,linestyle=styles[j],color=color) + if i==0: + colors.append(p[-1].get_color()) + plt.xlabel("DM [pc cm$^{-3}$]") + plt.ylabel("$p({\\rm DM})/p_{\\rm best}({\\rm DM})$") + plt.legend(fontsize=6) + plt.tight_layout() + plt.savefig(opdir+"pdm_vs_z_methods.png") + plt.close() + + + def get_defined_states(): """ @@ -132,6 +235,7 @@ def get_defined_states(): survey_dict2 = {"WMETHOD": 2} # scattering and width survey_dict3 = {"WMETHOD": 3} # z-dependence survey_dict4 = {"WMETHOD": 3} # z-dependence + survey_dict5 = {"WMETHOD": 3} # z-dependence # now sets up state dictionaries @@ -189,8 +293,21 @@ def get_defined_states(): state_dict4["scat"]["Slogsigma"] = 0.01 # not actually used in the log-constant distribution state_dict4["scat"]["Sfnorm"] = 1000 # 1 GHz normalisation - state_dicts = [state_dict1, state_dict2, state_dict3, state_dict4] - survey_dicts = [survey_dict1, survey_dict2, survey_dict3, survey_dict4] + state_dict5 = {} + state_dict5["scat"] = {} + state_dict5["width"] = {} + state_dict5["width"]["WNbins"] = 5 # set to large number for this analysis + state_dict5["width"]["WidthFunction"] = 2 # half-lognormal + state_dict5["width"]["ScatFunction"] = 0 # log-uniform true value 0 + state_dict5["width"]["Wlogmean"] = -0.29 # -0.29 + state_dict5["width"]["Wlogsigma"] = 0.65 #0.65 + state_dict5["scat"]["Slogmean"] = -1.38 # true valiue -1.38 + state_dict5["scat"]["Slogsigma"] = 0.01 # not actually used in the log-constant distribution + state_dict5["scat"]["Sfnorm"] = 1000 # 1 GHz normalisation + + + state_dicts = [state_dict1, state_dict2, state_dict3, state_dict4, state_dict5] + survey_dicts = [survey_dict1, survey_dict2, survey_dict3, survey_dict4, survey_dict5] return survey_dicts,state_dicts diff --git a/papers/Scattering/estimate_2d_effect.py b/papers/Scattering/estimate_2d_effect.py new file mode 100644 index 0000000..48cb7e3 --- /dev/null +++ b/papers/Scattering/estimate_2d_effect.py @@ -0,0 +1,161 @@ +""" +This script estimates the effect of bias in 2D vs 1D, +e.g. what is the difference between +efficiency = f(z,DM) and f1(z)f2(DM) +""" + +import os + +from zdm import cosmology as cos +from zdm import figures +from zdm import parameters +from zdm import survey +from zdm import pcosmic +from zdm import misc_functions as mf +from zdm import iteration as it +from zdm import loading +from zdm import io +from pkg_resources import resource_filename +import numpy as np +from matplotlib import pyplot as plt + +import matplotlib + +defaultsize=14 +ds=4 +font = {'family' : 'Helvetica', + 'weight' : 'normal', + 'size' : defaultsize} +matplotlib.rc('font', **font) + + +def main(): + """ + See main description at top of file. + """ + + Load = False + + # in case you wish to switch to another output directory + + opdir = "2Deffect/" + names = ["CRAFT_ICS_1300"] # for example + + if not os.path.exists(opdir): + os.mkdir(opdir) + + # directory where the survey files are located. The below is the default - + # you can leave this out, or change it for a different survey file location. + sdir = os.path.join(resource_filename('zdm', 'data'), 'Surveys') + + # make this into a list to initialise multiple surveys art once. But here, we just use the standard + # CRAFT ICS survey + #names = ["CRAFT_ICS_1300"] # for example + + + # gets single case of relevance + survey_dict,state_dict = get_defined_states() + + # We ignore repetition... for now! + repeaters=False + + # sets limits for calculations - both plotting, nad intrinsic. We want things to go quickly! + zmax = 2. + dmmax = 2000 + ndm=200 + nz=200 + + if not Load: + surveys, grids = loading.surveys_and_grids(survey_names = names,\ + repeaters=repeaters, sdir=sdir,nz=nz,ndm=ndm,zmax=zmax,dmmax=dmmax, + survey_dict = survey_dict, state_dict = state_dict) + s = surveys[0] + g = grids[0] + + rates = g.rates + np.save(opdir+"rates.npy",rates) + dmvals = g.dmvals + zvals = g.zvals + np.save(opdir+"dmvals.npy",dmvals) + np.save(opdir+"zvals.npy",zvals) + np.save(opdir+"efficiencies.npy",s.efficiencies) + + + else: + rates = np.load(opdir+"rates.npy") + dmvals = np.load(opdir+"dmvals.npy") + zvals = np.load(opdir+"zvals.npy") + efficiencies = np.load("efficiencies.npy") + + + + # now we have rates, let's look at the effects! + + labels=["No width/scattering","CHIME Catalogue 1","Lognormals (this work)","Best fit (this work)","(numerical approx)"] + styles=[":","-.","--","-",":"] + + pzvals = [0.01,0.5,1,1.5,2]#0.5,1,1.5,2] + colors=[] + jnorm=3 + + #pzvals=[0.01,0.5,1,1.5,2] + pzvals=[0.01,2,1.5,1,0.5] + plt.figure() + weffs=[] + for i,z in enumerate(pzvals): + iz = np.where(zvals <= z)[0][-1] + + # weights this over all wieght bins + weighted_eff = np.array(s.efficiencies[:,iz,:]).T*np.array(s.wplist[:,iz]) + #now we sum this over those bins + weighted_eff = np.sum(weighted_eff,axis=1) # summed over weight bins + weffs.append(weighted_eff) + + for i,z in enumerate(pzvals): + if i==0: + continue + plt.plot(g.dmvals,weffs[i]/weffs[0],label="z = "+str(z)[0:3],linestyle=styles[i-1]) + + plt.xlabel("DM [pc cm$^{-3}$]") + plt.ylabel("$\\frac{\\epsilon({\\rm DM}|z)}{\\epsilon({\\rm DM}|z=0)}$") + + plt.xlim(0,2000) + plt.ylim(1.08,1.2) + plt.legend(loc="upper left") + plt.tight_layout() + plt.savefig(opdir+"efficiency.png") + plt.close() + + + +def get_defined_states(): + """ + This function just contains hard-coded state and survey dicts + corresponding to four cases of interest + """ + # we now set up three examples. These are: + #1: no width/scattering distributions. Treat width=1ms (nominal) + #2: CHIME width/scattering distributions (no z-dependence_ + #3: bestfit lognormal function of this work (with z-dependence) + #4: bestfit functions of this work (with z-dependence) + + # sets up survey dictionaries for width methods + survey_dict4 = {"WMETHOD": 3} # z-dependence + + + state_dict4 = {} + state_dict4["scat"] = {} + state_dict4["width"] = {} + state_dict4["width"]["WNbins"] = 100 # set to large number for this analysis + state_dict4["width"]["WidthFunction"] = 2 # half-lognormal + state_dict4["width"]["ScatFunction"] = 0 # log-uniform true value 0 + state_dict4["width"]["Wlogmean"] = -0.29 # -0.29 + state_dict4["width"]["Wlogsigma"] = 0.65 #0.65 + state_dict4["scat"]["Slogmean"] = -1.38 # true valiue -1.38 + state_dict4["scat"]["Slogsigma"] = 0.01 # not actually used in the log-constant distribution + state_dict4["scat"]["Sfnorm"] = 1000 # 1 GHz normalisation + + + return survey_dict4,state_dict4 + +main() diff --git a/papers/Scattering/fit_scattering_width.py b/papers/Scattering/fit_scattering_width.py index d3cf22b..af1623d 100644 --- a/papers/Scattering/fit_scattering_width.py +++ b/papers/Scattering/fit_scattering_width.py @@ -430,6 +430,7 @@ def main(outdir="Fitting_Outputs/"): NFRB=host_tau.size handles=[l1[2],l3[0],l2[2]] labels=["Observed","Completeness","Corrected$"] + styles=["-.","--","--",":"] for i in np.arange(NFUNC): print("plotting function ",i," with xbest ",xbest[i]) xs,ys = function_wrapper(i,xbest[i])#cspline=None): @@ -490,7 +491,7 @@ def main(outdir="Fitting_Outputs/"): #l=plt.plot(xs,ys*plotnorm,label=FNAMES[i]) xs,ys = function_wrapper(i,xbest[i],cspline=cspline) - l=plt.plot(xs,ys*plotnorm,label=FNAMES[i]) + l=plt.plot(xs,ys*plotnorm,label=FNAMES[i],linestyle=styles[i]) handles.append(l[0]) labels.append(FNAMES[i]) @@ -516,7 +517,7 @@ def main(outdir="Fitting_Outputs/"): print("plotting function ",i," with xbest ",xbest[i]) xs,ys = function_wrapper(i,xbest[i],logxmax=2)#cspline=None): plotnorm = NFRB * (np.log10(bins[1])-np.log10(bins[0])) - l=plt.plot(xs,ys*plotnorm,label=FNAMES[i]) + l=plt.plot(xs,ys*plotnorm,label=FNAMES[i],linestyle=styles[i]) handles.append(l[0]) labels.append(FNAMES[i]) plt.xscale("log") @@ -879,7 +880,7 @@ def main(outdir="Fitting_Outputs/"): #l=plt.plot(xs,ys*plotnorm,label=FNAMES[i]) xs,ys = function_wrapper(i,xbest[i],cspline=cspline) - l=plt.plot(xs,ys*plotnorm,label=FNAMES[i]) + l=plt.plot(xs,ys*plotnorm,label=FNAMES[i],linestyle=styles[i]) handles.append(l[0]) labels.append(FNAMES[i]) @@ -901,11 +902,12 @@ def main(outdir="Fitting_Outputs/"): NFRB=host_tau.size handles=[l2v4[2],l3v4[0]] labels=["Adjusted","Completeness"] + for i in [0,1,3]: print("plotting function ",i," with xbest ",xbest[i]) xs,ys = function_wrapper(i,xbest[i],logxmax=2)#cspline=None): plotnorm = NFRB * (np.log10(bins[1])-np.log10(bins[0])) - l=plt.plot(xs,ys*plotnorm,label=FNAMES[i]) + l=plt.plot(xs,ys*plotnorm,label=FNAMES[i],linestyle=styles[i]) handles.append(l[0]) labels.append(FNAMES[i]) plt.xscale("log") From b97d31e6306bfde0ec0805f81258b59e2dad38f2 Mon Sep 17 00:00:00 2001 From: Clancy James Date: Wed, 8 Oct 2025 11:34:42 +0800 Subject: [PATCH 17/20] Making changes according to X's comments on github --- zdm/MCMC.py | 5 +++-- zdm/data/Surveys/SKA_low.ecsv | 2 +- zdm/data/Surveys/SKA_mid.ecsv | 2 +- zdm/{scripts/MCMC/check_emcee.py => tests/test_mcmc.py} | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) rename zdm/{scripts/MCMC/check_emcee.py => tests/test_mcmc.py} (96%) diff --git a/zdm/MCMC.py b/zdm/MCMC.py index 0e6a90a..42d00fc 100644 --- a/zdm/MCMC.py +++ b/zdm/MCMC.py @@ -13,7 +13,7 @@ import numpy as np import zdm.iteration as it -from pkg_resources import resource_filename +import importlib.resources as resources import emcee import scipy.stats as st @@ -125,9 +125,10 @@ def calc_log_posterior(param_vals, state, params, surveys_sep, Pn=False, pNreps= # gets new zDM grid if F and H0 in the param_dict if 'H0' in param_dict or 'logF' in param_dict or g0info is None: + datdir = resources.files('zdm').joinpath('GridData') zDMgrid, zvals,dmvals = mf.get_zdm_grid( state, new=True, plot=False, method='analytic', - datdir=resource_filename('zdm', 'GridData')) + datdir=datdir) g0info = [zDMgrid, zvals,dmvals] if len(surveys_sep[0]) != 0: diff --git a/zdm/data/Surveys/SKA_low.ecsv b/zdm/data/Surveys/SKA_low.ecsv index b7a1f77..067dfed 100644 --- a/zdm/data/Surveys/SKA_low.ecsv +++ b/zdm/data/Surveys/SKA_low.ecsv @@ -23,4 +23,4 @@ # \ \"DIAM\": 40,\n \"BMETHOD\": 0,\n \"NBEAMS\": 1,\n \"NBINS\": 10\n }\n}"} # schema: astropy-2.0 TNS BW DM DMG FBAR FRES Gb Gl NREP SNR SNRTHRESH THRESH TRES WIDTH XDec XRA Z -0 120 200 35.0 1284 0.0145 "" "" 1 10. 10. 0.2 0.069 1.0 "" "" 0.276 +DUMMY 120 200 35.0 1284 0.0145 "" "" 1 10. 10. 0.2 0.069 1.0 "" "" 0.276 diff --git a/zdm/data/Surveys/SKA_mid.ecsv b/zdm/data/Surveys/SKA_mid.ecsv index 4fd949c..d4f06b8 100644 --- a/zdm/data/Surveys/SKA_mid.ecsv +++ b/zdm/data/Surveys/SKA_mid.ecsv @@ -23,4 +23,4 @@ # \ \"DIAM\": 15,\n \"BMETHOD\": 0,\n \"NBEAMS\": 1,\n \"NBINS\": 10\n }\n}"} # schema: astropy-2.0 TNS BW DM DMG FBAR FRES Gb Gl NREP SNR SNRTHRESH THRESH TRES WIDTH XDec XRA Z -0 300 200 35.0 1284 0.1075 "" "" 1 10. 10. 0.2 0.069 1.0 "" "" 0.276 +DUMMY 300 200 35.0 1284 0.1075 "" "" 1 10. 10. 0.2 0.069 1.0 "" "" 0.276 diff --git a/zdm/scripts/MCMC/check_emcee.py b/zdm/tests/test_mcmc.py similarity index 96% rename from zdm/scripts/MCMC/check_emcee.py rename to zdm/tests/test_mcmc.py index 632b469..079aae6 100644 --- a/zdm/scripts/MCMC/check_emcee.py +++ b/zdm/tests/test_mcmc.py @@ -2,7 +2,7 @@ This is a function to test if emcee is running on your computer It defines a simple probability (log_prob) and -submits this a a pooled job over your cpus. +submits this as a pooled job over your cpus. Original test code taken from https://emcee.readthedocs.io/en/stable/tutorials/parallel/ From 13e5f8eb9bde2911ac67c1a2d68d04467a371e65 Mon Sep 17 00:00:00 2001 From: profxj Date: Wed, 8 Oct 2025 02:49:45 -0700 Subject: [PATCH 18/20] small edit --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 6c373ac..b8270df 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,7 +27,7 @@ classifiers = zip_safe = False use_2to3=False packages = find: -python_requires = >=3.8 +python_requires = >=3.10 setup_requires = setuptools_scm include_package_data = True install_requires = From ecc5e9f3860eabbf2b5aeaf07c96422a682f933e Mon Sep 17 00:00:00 2001 From: profxj Date: Wed, 8 Oct 2025 11:43:14 -0700 Subject: [PATCH 19/20] test fix --- zdm/tests/test_mcmc.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/zdm/tests/test_mcmc.py b/zdm/tests/test_mcmc.py index 079aae6..8742f96 100644 --- a/zdm/tests/test_mcmc.py +++ b/zdm/tests/test_mcmc.py @@ -12,6 +12,10 @@ import time import numpy as np +import emcee +import os +import multiprocessing as mp +from multiprocessing import cpu_count def log_prob(theta): @@ -21,29 +25,25 @@ def log_prob(theta): break return -0.5 * np.sum(theta**2) -import emcee +def test_mcmc(): + np.random.seed(42) + initial = np.random.randn(32, 5) + nwalkers, ndim = initial.shape + nsteps = 20 -np.random.seed(42) -initial = np.random.randn(32, 5) -nwalkers, ndim = initial.shape -nsteps = 20 + os.environ["OMP_NUM_THREADS"] = "1" -import os -os.environ["OMP_NUM_THREADS"] = "1" - -import multiprocessing as mp -# this mod is required for running on mac osx -Pool = mp.get_context('fork').Pool -from multiprocessing import cpu_count + # this mod is required for running on mac osx + Pool = mp.get_context('fork').Pool -ncpu = cpu_count() -print("{0} CPUs".format(ncpu)) + ncpu = cpu_count() + print("{0} CPUs".format(ncpu)) -with Pool() as pool: - sampler = emcee.EnsembleSampler(nwalkers, ndim, log_prob,pool=pool) - start = time.time() - sampler.run_mcmc(initial, nsteps, progress=True) - end = time.time() - serial_time = end - start - print("MP took {0:.1f} seconds".format(serial_time)) + with Pool() as pool: + sampler = emcee.EnsembleSampler(nwalkers, ndim, log_prob,pool=pool) + start = time.time() + sampler.run_mcmc(initial, nsteps, progress=True) + end = time.time() + serial_time = end - start + print("MP took {0:.1f} seconds".format(serial_time)) From e1ea7ded2e257a1794596511f925fca0569e8f7b Mon Sep 17 00:00:00 2001 From: profxj Date: Wed, 8 Oct 2025 12:13:10 -0700 Subject: [PATCH 20/20] rename --- ...test_scattering_properties.py => tst_scattering_properties.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename zdm/scripts/Scattering/{test_scattering_properties.py => tst_scattering_properties.py} (100%) diff --git a/zdm/scripts/Scattering/test_scattering_properties.py b/zdm/scripts/Scattering/tst_scattering_properties.py similarity index 100% rename from zdm/scripts/Scattering/test_scattering_properties.py rename to zdm/scripts/Scattering/tst_scattering_properties.py