package pnEvents; use strict; use English; use Carp; use Regexp::Common qw(boolean); use vars qw(@ISA $VERSION $name $author $date $version @instList $numberOfStreams); @ISA = qw(Module); use ModuleResources; # Declare identity, version, author, date, etc. $name = __PACKAGE__; $VERSION = '2.07'; $version = $VERSION; $author = 'Dean Hinshaw,Richard West,Duncan John Fyfe,Ian Stewart'; $date = '2006-04-04'; # # ChangeLog # ========= # # version 2.07 - 2009-01-12 DLG # # + Changed "withtempcorrection" to "Y" following advice from Hermann Brunner # # # version 2.06 - 2006-07-10 DJF # ------------ # # + Fix for darck quadrant problem applied to OAL. No longer need OBT_WARN test. # Replaced test with an echo of the OBT_WARN value to the log file. # # version 2.05 - 2006-04-04 DJF # ------------ # # + Test for OBT_WARN keyword in epframes output. Return ignore if it is found to be true. # # version 2.04 - 2005-11-16 IMS # ------------ # # + Incorporated DJF's shortened intermediate file names. # # version 2.03 - 2005-10-14 IMS # ------------ # # + TSTART obtained from flare time series header and subtracted from the times of the flare PDF plot. # + Flare plot Y axis now RATE not COUNTS. Calculation of RATE is thus moved forward a bit, and all flare productions now moved within the test for existence of RATE extension. # # version 2.02 - 2005-08-12 RGW # ------------ # + changes to flare plotting to work around 80-character filename limit # in 'fplot' # # version 2.01 - 2005-08-02 IMS # ------------ # # + PDF product now made from flare bkg time series. # + Slightly changed event selection expression (events with PI < 150 eV are no longer kept; also the flag filter mask is given explicitly, not as XMMEA_EP). # # version 2.00 - 2005-05-09 DJF # ------------ # # + Add explicit perl module headers. Previously these were supplied # at compile time. This will make debugging and extending the modules # through additional perl libraries easier. # # + Code layout made more uniform with perltidy # + Standerdized date format (ccyy-mm-dd) # + Standerdized author list to use comma separator # + Make use of perl $VERSION magic. Now $Version = version = '' ## # Version 1.69 - 16-Mar-2004 (DJF) # ------------ # # + Added dummy badpixset=temporary file to first call to badpixfind # to stop it creating it's default. # # Version 1.68 - 14-Jan-2004 (DJF) # ------------ # # + Use tabgtigen parameter mingitsize rather than evselect to exclude # small GTI from flare background # # Version 1.67 - 10-12-2003 (DJF) # ------------ # # + Adapted doCommand to use anonymous lists for list parameters # + Fixed filter expression. PI<-150 should have been PI<=150 # # Version 1.66 - 05-Sept-2003 (DJF) # ------------------------------- # + Fixed backgroundrate in second call to badpixfind. Was 0.0011 should have been 0.00011 # # Version 1.65 - 11-Nov-2002 (DJF) # ------------------------------- # # + Updated evlistcomb columns to include OFFSETX # + Added DLIMAP to evlistcomb othertables option # # Version 1.63 - 11-Nov-2002 (DJF) # ------------------------------- # # + Updated backgroundrate (0.0001 -> 0.0011) and hienergythresh (10.0 -> 12.0) # in second call to badpixfind # # Version 1.62 - 17-Oct-2002 (DJF) # ------------------------------- # # + Moved OFFSETS from othertables to maintable option # # Version 1.61 - 19-Feb-2002 (DH) # ------------------------------- # # + Add OFFSETS to the list of tables for evlistcomb to propagate. # # Version 1.60 - 15-Feb-2002 (DH) # ------------------------------- # # + Bug fixes for 1.60 . # # Version 1.59 - 7-Feb-2002 (DH) # ------------------------------- # # + Remove RAWY>10 cut on event lists in imaging mode. # # Version 1.58 - 6-Nov-2001 (DH) # ------------------------------- # # + New parameters for badpixfind for making of the background # mask. Values as recommended by Andy Read, via email from Michael # Freyburg dated 10/10/01. # # Version 1.57 - 9-Oct-2001 (DH) # ------------------------------- # # + Change energy thresholds in badpixfind for making of the background # mask. Was 0.16/10.0, now 7.0/15.0 . # # Version 1.56 - 4-Oct-2001 (DH) # ------------------------------- # # + Fix bug that made flare screening only work for full frame modes. # + Change threshold on number for good background pixels from 0 to 100. # # Version 1.55 - 2-Oct-2001 (DH) # ------------------------------- # # + Put back flare gti creation for small window mode. Instead # inhibit flare gti creation if the number of suitable background # pixels found in a field is zero. # + Increase flare gti cut from 6 to 10. # # Version 1.54 - 1-Oct-2001 (DH) # ------------------------------- # # + Scale integration time for flare histograms ($timebinsize) # to 2 times nominal for large window mode, and 35 times for # small window mode. # + Do not create flare gti file for small window mode, though # a flare histogram is still created. # # Version 1.53 - 1-Oct-2001 (DH) # ------------------------------- # # + Replace dslatts call with the hasFITSExtension function. # # Version 1.52 - 2001-08-09 (DJF) # ------------ # # + Changed Flare light curve from intermediate to final product. # # Version 1.51 - 2001-06-15 # ------------ # # + Change withtempcorrection parameter back to 'N'. # # Version 1.50 - 2001-06-06 # ------------ # # + Change withtempcorrection=N parameter of epevents to 'Y'. # # Version 1.49 - 2001-06-05 # ------------ # # + Re-work merging of HK and CCD specific gti data. gtialign # was not being called properly. # # Version 1.48 - 2001-05-22 # ------------ # # + Only apply RAWY>10 only for imaging mode data. # + Add withtempcorrection=N parameter to epevents, in case the # default changes. # # Version 1.47 - 2001-05-16 # ------------ # # + Bug fixes and refinements for changes made in version 1.46 # # Version 1.46 - 2001-05-16 (DH) # ------------ # # + Remove unwanted columns from the EXPOSUnn extensions # of the pn event lists. # # Version 1.45 - 2001-04-17 (DH) # ------------ # # + Changes to main badpixfind call: # - Change loenergythresh from 0.16 to 0.14 # - Change hicolthresh from 0.0015 to 0.00105 # # Version 1.44 - 2001-03-16 (DH) # ------------ # # + Print out version number in performAction() for # tracking purposes. # # Version 1.43 - 2001-02-01 (DH) # ------------ # # + Change threshold for flare rejection rate cutoff from 36.4 to 6 # # Version 1.42 - 2001-01-26 (DH) # ------------ # # + Restrict event energy range for flare rejection to 7-15 keV # # Version 1.41 - 2001-01-19 (DH) # ------------ # # + Change narrowerthanpsf parameter in standard call # to badpixfind from 1.25 to 1.5. # # Version 1.40 - 2001-01-19 (DH) # ------------ # # + hicolthesh values for the two calls to badpixfind # were not set correctly. In particular, value for the # standard call was off by a factor of 10. Values now # set as per Andy Read's email dated 13-Dec-2000. # # Version 1.39 - 2001-01-11 (DH) # ------------ # # + Remove GlobalHK from ignore rule. # # + Add InstrumentHK to start rule. # # Version 1.38 - 2000-12-19 (DH) # ------------ # # + Change hicolthesh in badpixfind masking back # to its nominal value. # # Version 1.37 - 2000-12-13 (DH) # ------------ # # + First production version. # # Declare list of instruments this module is interested in @instList = qw(epn); # Number of streams sub numberOfStreams { return numberOfExposures(); } # Rules method sub evaluateRules { # return ignore() if ignored( module => 'ExpChooser' , instrument => thisInstrument , stream => thisStream ); # start() if complete( module => 'ExpChooser' , instrument => thisInstrument , stream => thisStream ) and complete( module => 'InstrumentHK' , instrument => thisInstrument , stream => 1 ) and complete( module => 'GlobalHK' , instrument => 'all' , stream => 1 ); } # Action method sub performAction { info("Module version number: $version"); my @extraopts; # Work out which exposure this stream maps on to my $exp_id = exposureID( instrument => thisInstrument , stream => thisStream ); info("Exposure ID: $exp_id"); # my $filter = getExposureProperty( exp_id => $exp_id , name => 'filter' ); my $mode = getExposureProperty( exp_id => $exp_id , name => 'mode' ); info("Filter: $filter"); info("Instrument mode: $mode"); # Find event files for this exposure my @evFiles = findFile( class => 'ODF' , instrument => thisInstrument , exp_id => $exp_id , content => '*events' ); # Return immediately if there are no event lists return success() if $#evFiles < 0; # Find attitude HK file my $attFile = findFile( class => 'product' , instrument => 'all' , content => 'Attitude time series' , required => 'true' ); # Find HK GTI file my $hkGTIFile = findFile( class => 'intermediate' , instrument => thisInstrument , content => 'HK GTI' ); # my ( @finalEvents, @gtiFilter, @maskFilter, $datamode ); my $goodPixels = 0; # Time binning to use when making background light curves # my $timebinsize = 10.0; # Loop through each event file foreach my $evFile0 (@evFiles) { # Find CCD and node numbers my $ccdid = readFITSKeyword( file => $evFile0 , extension => 2 , keyword => "CCDID" ); my $quadrant = readFITSKeyword( file => $evFile0 , extension => 2 , keyword => "QUADRANT" ); my $ccd = $quadrant * 3 + $ccdid + 1; $datamode = readFITSKeyword( file => $evFile0 , extension => 2 , keyword => "DATATYPE" ); # my $ccdmode = readFITSKeyword( # file => $evFile0 # , extension => 2 # , keyword => "CCDMODE" # ); # info("Processing events for CCD $ccd, mode $datamode."); # Unused. IMS 2005-08-03 my $evFile1 = newFile( class => 'intermediate' , instrument => thisInstrument , exp_id => $exp_id , ccd => $ccd , content => 'Processed events' , level => 1 ); my $gtiFile1 = newFile( class => 'intermediate' , instrument => thisInstrument , exp_id => $exp_id , ccd => $ccd , content => 'CCD GTI' , level => 1 ); # Run epframes # doCommand( 'epframes' , set => $evFile0 , wrongpixlimit => 20 , srcposition => 190 , eventset => $evFile1 , gtiset => $gtiFile1 , mipmethod => 'onboard' , mipthreshold => 3000 , setupbpx => 'nom3' ) or return exception(); my %obt_warn = ( file => $evFile1 , extension => 'EXPOSURE' , keyword => 'OBT_WARN' ); if (hasFITSKeyword( %obt_warn )) { my $opt_warn = readFITSKeyword( %obt_warn ); if ( $opt_warn =~ /$RE{boolean}{true}/ ) { info("OBT_WARN True"); } else { info("OBT_WARN False"); } } # Run badpixfind to create a mask to be used for # generating a background lightcurve # my $dummyBadPixels = newFile( class => 'intermediate' , instrument => thisInstrument , exp_id => $exp_id , ccd => $ccd , content => 'Ignore This Bad pixels' ); my $badPixMask = newFile( class => 'intermediate' , instrument => thisInstrument , exp_id => $exp_id , ccd => $ccd , content => 'Bright pixels mask' ); doCommand( 'badpixfind', eventset => $evFile1 , badpixmap => $badPixMask , badpixset => $dummyBadPixels , withfovmask => 'Y' , threshabovebackground => 'Y' , hithresh => 0.00025 , hicolthresh => 0.00007 , narrowerthanpsf => 0 , withbadpixmap => 'Y' , withsearchbadcolumn => 'Y' , loenergythresh => 7.0 , hienergythresh => 15.0 , backgroundrate => -1 , thresholdlabel => 'rate' ) or return exception(); $goodPixels += readFITSKeyword( file => $badPixMask , extension => 1 , keyword => "GOODPIX" ); push( @maskFilter , "(CCDNR==${ccd} && MASK($badPixMask,1,1,RAWX,RAWY))" ); # Run badpixfind # @extraopts = (); if ( $datamode eq "IMAGE.EL" ) { my $newBadPixels = newFile( class => 'intermediate' , instrument => thisInstrument , exp_id => $exp_id , ccd => $ccd , content => 'Bad pixels' ); doCommand( 'badpixfind' , eventset => $evFile1 , badpixset => $newBadPixels , hithresh => 0.0045 , flickertimesteps => 1 , thresholdlabel => 'rate' , withsearchbadcolumn => 'Y' , hicolthresh => 0.00105 , backgroundrate => 0.00011 , loenergythresh => 0.14 , hienergythresh => 12.0 , narrowerthanpsf => 1.5 ) or return exception(); @extraopts = ( getnewbadpix => 'Y' , badpixset => $newBadPixels ); } # Flag bad pixels # doCommand( 'badpix' , eventset => $evFile1 , getuplnkbadpix => 'Y' , getotherbadpix => 'Y' , windowfilter => 'N' , @extraopts ) or return exception(); # Run epevents # my $evFile2 = newFile( class => 'intermediate' , instrument => thisInstrument , exp_id => $exp_id , ccd => $ccd , content => 'Processed events' , level => 2 ); doCommand( 'epevents' , eventset => $evFile1 , gainctiaccuracy => 2 , reemissionthresh => 0 , randomizeposition => 'Y' , randomizeenergy => 'Y' , withtempcorrection => 'Y' , outset => $evFile2 ) or return exception(); # Apply attitude correction # doCommand( 'attcalc' , eventset => $evFile2 , refpointlabel => 'pnt' , attitudelabel => 'ahf' , withmedianpnt => 'Y' , atthkset => $attFile , -w => 10 ) or return exception(); # Filter events before final merging my $evFile3 = newFile( class => 'intermediate' , instrument => thisInstrument , exp_id => $exp_id , ccd => $ccd , content => 'Processed events' , level => 3 ); # my $expression = '#XMMEA_EP && (PI>150 || PI<=150)'; my $expression = '(FLAG & 0x0fa0000)==0 && PI>150'; doCommand( 'evselect' , table => "${evFile2}:EVENTS" , withfilteredset => 'Y' , filteredset => $evFile3 , destruct => 'Y' , expression => $expression , writedss => 'Y' , updateexposure => 'Y' , keepfilteroutput => 'Y' ) or return exception(); # Append to list of processed event files # push( @finalEvents, $evFile3 ); # Accumulate expression for GTI filter # push( @gtiFilter, "(CCDNR==${ccd} && GTI($gtiFile1,TIME))" ); } # Combine per-CCD event lists into single list # @extraopts = (); my $tmpEvents; if ( $datamode eq "IMAGE.EL" ) { $tmpEvents = newFile( class => 'intermediate' , instrument => thisInstrument , exp_id => $exp_id , content => 'Merged imaging events' ); @extraopts = ( imagingset => $tmpEvents , epnimgcolnames => [ qw(TIME RAWX RAWY DETX DETY X Y PHA PI FLAG PATTERN PAT_ID PAT_SEQ OFFSETX) ] , epnimgcoltypes => [ qw(double int16 int16 int16 int16 int32 int32 int16 int16 int32 int8 int16 int8 int16) ] ); } elsif ( $datamode eq "TIME.EL" || $datamode eq "BURST.EL" ) { $tmpEvents = newFile( class => 'intermediate' , instrument => thisInstrument , exp_id => $exp_id , content => 'Merged timing events' ); @extraopts = ( timingset => $tmpEvents , epntimcolnames => [ qw(TIME RAWX RAWY DETX DETY X Y PHA PI FLAG PATTERN PAT_ID PAT_SEQ OFFSETX) ] , epntimcoltypes => [ qw(double int16 int16 int16 int16 int32 int32 int16 int16 int32 int8 int16 int8 int16) ] ); } doCommand( 'evlistcomb' , eventsets => [@finalEvents] , instrument => 'epn' , maintable => [qw(EVENTS OFFSETS)] , othertables => [qw(BADPIX EXPOSURE DLIMAP)] , @extraopts ) or return exception(); # Remove unwanted columns from EXPOSUnn extensions my @badColumns = (qw(FRAMELIM NDSCLIN1 NDSCLIN2 NDSCLIN3 NDSCLIN4)); my @badObjects; for ( my $ic = 1 ; $ic <= 12 ; $ic++ ) { my $ext = sprintf 'EXPOSU%02d', $ic; info("Checking for extension $ext"); if ( hasFITSExtension( file => $tmpEvents, extension => $ext ) ) { my @columns = @badColumns; push @badObjects , map { $_ = "$tmpEvents:$ext:$_"; } @columns; } } if (@badObjects) { doCommand( 'dsrm', objects => [@badObjects] ) or return exception(); } # Fold in the HK gti files # my $hkGtiFileAligned = newFile( class => 'intermediate' , instrument => thisInstrument , exp_id => $exp_id , content => 'Aligned HK GTIs' ); if ( fileExists( file => $hkGTIFile ) ) { doCommand( "gtialign" , gtitable => $hkGTIFile . ":STDGTI" , eventset => $tmpEvents , outset => $hkGtiFileAligned ) or return exception(); foreach my $gtiFile ( findFile( class => 'intermediate' , instrument => thisInstrument , exp_id => $exp_id , content => 'CCD GTI' , level => 1 ) ) { my $fileInf = fileInfo( class => 'intermediate', name => $gtiFile ); my $gtiExt = sprintf 'STDGTI%02d', $$fileInf{'ccd'}; doCommand( "gtimerge" , tables => [ "$gtiFile", "$hkGtiFileAligned:$gtiExt" ] , mergemode => 'and' , withgtitable => 'N' ) or return exception(); } } # Filter event list on GTI # my $gtiExpr = join( " || ", @gtiFilter ); my $finEvents; if ( $datamode eq "IMAGE.EL" ) { $finEvents = newFile( class => 'product' , instrument => thisInstrument , exp_id => $exp_id , content => 'EPIC PN imaging mode event list' ); } elsif ( $datamode eq "TIME.EL" || $datamode eq "BURST.EL" ) { $finEvents = newFile( class => 'product' , instrument => thisInstrument , exp_id => $exp_id , content => 'EPIC timing mode event list' ); } doCommand( 'evselect' , table => $tmpEvents , filteredset => $finEvents , withfilteredset => 'Y' , expression => $gtiExpr , keepfilteroutput => 'Y' , writedss => 'Y' , updateexposure => 'Y' , destruct => 'true' ) or return exception(); # Copy CIF into event list # my $cifFile = findFile( class => 'product' , content => 'Calibration index file' ); doCommand( 'dscopyblock' , blocks => "${cifFile}:CALINDEX" , to => $finEvents ) or return exception(); # Make flare GTI # if ( $datamode eq "IMAGE.EL" ) { $timebinsize *= 2 if $mode eq "PrimeLargeWindow"; $timebinsize *= 35 if $mode eq "PrimeSmallWindow"; my $flareExpr = "($gtiExpr)"; if (@maskFilter) { $flareExpr .= " && (" . join( " || ", @maskFilter ) . ")"; } # Put in energy range for flare rejection $flareExpr .= " && (PI IN [7000:15000])"; my $backRates = newFile( class => 'product' , instrument => thisInstrument , exp_id => $exp_id , content => 'EPIC flare background timeseries' ); my $flareGti = newFile( class => 'intermediate' , instrument => thisInstrument , exp_id => $exp_id , content => 'flare GTI' ); doCommand( 'evselect' , table => $finEvents , maketimecolumn => 'Y' , timebinsize => $timebinsize , withrateset => 'Y' , rateset => $backRates , writedss => 'N' , updateexposure => 'N' , expression => $flareExpr ) or return exception(); # # 1 pixel ~ 17 arcsec**2 ~ 0.00474 arcmin**2, convert to per [ks * arcmin**2] # my $scalefactor = 0.0047355e-3; # unit = [arcmin**2/pixel ks/s] # If rate file is empty, no rate extension is created. Trap this. info("Number of good background pixels found: $goodPixels"); if ( hasFITSExtension( file => $backRates, extension => "RATE" ) && $goodPixels > 100 ) { doCommand( 'tabcalc' , tables => "$backRates:RATE" , columntype => 'real32' , column => 'RATE' , expression => "(COUNTS/($timebinsize*$goodPixels*$scalefactor))" ) or return exception(); # Now subtract TSTART from the TIME values and leave the result in a new column T_ELAPSED: my $start_time=readFITSKeyword( file => $backRates , extension => 'RATE' , keyword => "TSTART" ); doCommand( 'tabcalc' , tables => "$backRates:RATE" , columntype => 'real64' , column => 'T_ELAPSED' , expression => "TIME-$start_time" # Use the perl variable rather than the string '#TSTART' in case we decide later to use some other time as offset. ) or return exception(); my $fplotCommandFile = newFile(class => 'intermediate' , content => 'Flare bkg ts pco file' , format => 'ASCII' , extension => 'pco' ); writeASCIIFile( name => $fplotCommandFile , text => [join("\n", "LA X TIME-$start_time (s)", 'LA G2 RATE (cts/s)', 'PLOT', 'QUIT', '')] ); my $flareBkgPs = newFile( class => 'intermediate' , instrument => thisInstrument , exp_id => $exp_id , content => 'Flare bkg ts' , format => 'PS' ); doCommand( 'fplot' , infile => $backRates , xparm => 'T_ELAPSED' , yparm => 'RATE' , rows => '-' , device => "${flareBkgPs}/cps" , pltcmd => '@'."$fplotCommandFile" ) or return exception(); my $flareBkgPdf = newFile( class => 'product' , instrument => thisInstrument , exp_id => $exp_id , content => 'EPIC flare background timeseries' , format => 'PDF' ); PStoPDF( source => $flareBkgPs , destination => $flareBkgPdf ) or return exception(); # # 1 pixel ~ 17 arcsec**2 ~ 0.00474 arcmin**2, convert to per [ks * arcmin**2] # # my $scalefactor = 0.0047355e-3; # unit = [arcmin**2/pixel ks/s] # # If rate file is empty, no rate extension is created. Trap this. # info("Number for good background pixels found: $goodPixels"); # if ( hasFITSExtension( file => $backRates, extension => "RATE" ) # && $goodPixels > 100 ) # { # doCommand( # 'tabcalc' # , tables => "$backRates:RATE" # , columntype => 'real32' # , column => 'RATE' # , expression => # "(COUNTS/($timebinsize*$goodPixels*$scalefactor))" # ) # or return exception(); doCommand( 'tabgtigen', table => $backRates , gtiset => $flareGti , expression => 'RATE<10.0' , mingtisize => 100 ) or return exception(); } } # Everything OK return success(); } 1;