package MOSEvents;
use strict;
use English;
use Carp;
use vars
  qw(@ISA $VERSION $name $author $date $version @instList $numberOfStreams);
@ISA = qw(Module);
use ModuleResources;

# Declare identity, version, author, date, etc.
$name=__PACKAGE__;
$VERSION='3.06';
$version = $VERSION;
$author='Dean Hinshaw,Richard West,Duncan John Fyfe,Ian Stewart';
$date='2005-05-09';

#
# ChangeLog
# =========
#
# Version 3.06 - 2005-11-16 IMS
# ------------
#
# + Incorporated DJF's shortened intermediate file names.
#
# Version 3.05 - 2005-10-31 IMS
# ------------
#
# + Parameter --getotherbadpix of badpix call changed from 'no' to 'yes' on advice from JB (should get rid of bright pixels in ccd 4, mos1.)
#
# Version 3.04 - 2005-10-14 IMS
# ------------
#
# + TSTART obtained from flare time series header and subtracted from the times of the flare PDF plot.
# + Flare plot Y axis now RATE not COUNTS. Calculation of RATE is thus moved forward a bit.
# + Added 'PStoPDF', 'writeASCIIFile' to the list of methods called from ModuleResources.
#
# Version 3.03 - 2005-09-30 RGW
# ------------
#
# + OFFSETS table is propagated by evlistcomb
#
# version 3.02 - 2005-08-12 RGW
# ------------
#
# + trimmed filenames in flare plotting to work around FPLOT filename length
#   restriction
#
# version 3.01 - 2005-08-09 IMS
# ------------
#
# + PDF product now made from flare bkg time series.
# + Calls to fmedian removed because embadpixfind>2.0 now calculates this internally. Parameter --medianset of embadpixfind has likewise been removed.
#
# version 3.00 - 2005-05-09 DJF
# ------------
#
# + Add explicit perl module headers.  Previously these were supplied
#   at compile time.  This will make debugging and extending the modules
#   through additional perl libraries easier.
#
# Version 2.33 - 2004-03-19 (DJF)
# ------------
#
# + Added 'FLICKERING' and 'ON_BADOFFSET' to flareFlags and eventlistFlags.
#
# Version 2.32 - 2004-03-17 (DJF)
# ------------
#
# + Added Explicit use of withimageset=>'N' for evselect.
#
# Version 2.31 - 2004-02-12 (DJF)
# ------------
#
# + Use new PCMS function gtiTimeThreshold() to return threshold
#
# Version 2.30 - 2004-01-14 (DJF)
# ------------
#
# + Use tabgitgen parameter mingtisize rather than evselect to remove 
#   short intervals from bad pixel and flare background GTI.
#
# Version 2.29 - 2003-12-09 (DJF)
# ------------
#
# + Adapted doCommand to use anonymous lists for list parameters
#
# Version 2.28 - 2003-07-01 (DJF)
# ------------
#
# + #XMMEA_EM selection had been applied to wrong evselect.  This caused
#   removal of GATTI events before  flare background construction.
#   Try again.  #XMMEA_EM selection applied to final event list
#
# Version 2.27 - 2003-07-01 (DJF)
# ------------
#
# + XMMEA_EM selection still not working, more testing.
#
# Version 2.26 - 2003-07-01 (DJF)
# ------------
#
# + Fixed use of #XMMEA_EM flag selection in good event filtering.
#   From (FLAG & #XMMEA_EM)==0 to (#XMMEA_EM)==0
#
# Version 2.25 - 2003-07-01 (DJF)
# ------------
#
# + Corrected good event filtering.  Had missed out the ==0 from the flag filter.
#
# Version 2.24 - 2003-06-30 (DJF)
# ------------
#
# + Added emosdatamodes => 'IMAGING' to first evlistcomb call.
#
# Version 2.23 - 2003-04-11 (DJF)
# ------------
#
# + Fixed command line to evlistcomb (instrument= 'emos' rather than thisInstrument)
#
# Version 2.22 - 2003-04-08 (DJF)
# ------------
#
# + Added Jean Ballet's newer recipy for improved flare screening
#
# Version 2.21 - 2002-07-01 (DJF)
# ------------
#
# + Changed image creation flags to XMMEA_EM
#
# Version 2.20 - 2002-03-21 (DH)
# ------------
#
# + Bug fix for version 2.19.
#
# Version 2.19 - 2002-03-20 (DH)
# ------------
#
# + Put in removal of short (<100s) flare GTIs for bad pixel
#   finding.
# + Use offset variance files in first call to emevents.
# + Add bitmask fitlering to final event list, so it makes it in
#   the data subspace.
#
# Version 2.18 - 2002-03-13 (DH)
# ------------
#
# + Fix bugs in identifying and handling of CCDs in low gain mode.
#
# Version 2.17 - 2002-03-11 (DH)
# ------------
#
# + Bug fix: was using wrong file name for output flare gti file.
#
# Version 2.16 - 2002-03-05 (DH)
# ------------
#
# + Fix error in background rate calculation.  Do rate calculation
#   based on the ccd area available, rather than the number of CCDs.
#
# Version 2.15 - 2002-02-20 (DH)
# ------------
#
# + Back out change in 2.14, as the MOS software is not yet ready to
#   handle this.
#
# Version 2.14 - 2002-02-19 (DH)
# ------------
#
# + Add OFFSETS to the list of tables for evlistcomb to propagate.
#
# Version 2.13 - 2002-02-18 (DH)
# ------------
#
# + Explicitly list all parameters for the fmedian ftool call.
# + More bug fixes
#
# Version 2.12 - 2002-02-15 (DH)
# ------------
#
# + Bug fixes for 2.11 .
#
# Version 2.11 - 2002-02-07 (DH)
# ------------
#
# + New method for specifying bitmask filters for event lists, in order
#   to make it more clear which flags are being used.  
# + Add in the UNDERSHOOT flag, for event list and flare screening.
# + The flare background timeseries is now generated as a rates file, not
#   a histogram.  The RATE column is normalized to 1/[ks*acrmin**2], and 
#   the GTI cut is made at 2.0 .
# + Check for GAIN_CCD keyword in frame file.  If set to 'LOW', do not
#   process events for this ccd.
#
# Version 2.10 - 2002-01-08 (DH)
# ------------
#
# + Test for XMMEA_22 in event list before making flare histograms.
#
# Version 2.09 - 2001-11-14 (DH)
# ------------
#
# + Fix bug in flare GTI application introduced in 2.07.
#
# Version 2.08 - 2001-11-06 (DH)
# ------------
#
# + Slight change to flare GTI test to more exactly mimic MakeImage.pm .
#
# Version 2.07 - 2001-10-31 (DH)
# ------------
#
# + Test flare GTI as in MakeImage.pm to see if it is above a time threshold before using it.
#
# Version 2.06 - 2001-08-20 (DJF)
# ------------
#
# + Added test for empty GTI when filtering out flares before searching for bright pixels.
#
# Version 2.05 - 2001-08-15 (DJF)
# ------------
#
# + Added test for column 'HISTO' to first call to tabgtigen. 
#
# Version 2.04 - 2001-08-14 (DJF)
# ------------
#
# + Remove flagtruncatede1=N flag from first call to emevents (to fix XMMEA_22 failure fo evselect)
# + Add flag findbright=N to first call to embadpixfind as suggesteg by Jean Ballet. 
#
# Version 2.03 - 2001-08-13 (DJF)
# ------------
#
# + Another fix for histogram intermadiate to product change.
#
# Version 2.02 - 2001-08-13 (DJF)
# ------------
#
# + Fix to flare histogram intermadiate to product change.
#
#
# Version 2.01 - 2001-08-09 (DJF)
# ------------
#
# + Brought up to date with changes to previous versions of MOSEvents.
# + Changed Flare Histogram from intermediate to final product.
# - Removed supurfluous findFile at end of module v1.13
#
#############################################
##  Changes Applied to v1.10 before     #####
##  implimentation of v 2.0.1           #####
##
## Version 1.13 - 2001-06-07 (DH)
## ------------
##
## + Take out changes in version 1.12 and move test for
##   empty event list to MakeImage module.
##
## Version 1.12 - 2001-06-07 (DH)
## ------------
##
## + Set ignore flag if final imaging event list is empty.
##
## Version 1.11 - 2001-06-07 (DH)
## ------------
##
## + Do not make flare gti if flare histogram is empty.
##
############################################
#
# Version 2.00 - 2001-05-29 (DH)
# ------------
#
# + Complete re-work of bad pixel finding using the new embadpixfind package.
#
# Version 1.10 - 2001-05-21 (DH)
# ------------
#
# + Changes made as instructed by Jean Ballet:
#   - changes to rejection of e3 and e4 events due to task changes
#   - Add call to emenergy before badpixfind in Imaging mode only
#   - Change to emenergy options for TIME and CTIME modes
#
# Version 1.09 - 2001-03-16 (DH)
# ------------
#
# + Print out version number in performAction() for
#   tracking purposes.
#
# Version 1.08 - 2001-01-11 (DH)
# ------------
#
# + Remove GlobalHK from ignore rule.
#
# + Add InstrumentHK to start rule.
#
# Version 1.07 - 2000-11-23 (DH)
# ------------
#
# + First production version.
#

# Declare list of instruments this module is interested in
@instList=qw(emos1 emos2);

# Number of streams
sub numberOfStreams {
    return numberOfExposures();
}

sub evaluateRules {
    
    # If upstream module has been ignored, so if this one
    return ignore() if ignored(module => 'ExpChooser',
			       instrument => thisInstrument,
			       stream => thisStream);

    # Start conditions
    start()
      if complete(module => 'ExpChooser',
		  instrument => thisInstrument,
		  stream => thisStream)
	and
	  complete(module => 'InstrumentHK',
		   instrument => thisInstrument,
		   stream => 1)
	and
	  complete(module => 'GlobalHK',
		   instrument => 'all',
		   stream => 1);
}

my $flareFlags =   ['UNDERSHOOT'
	, 'BAD_E3E4'
	, 'ON_BADROW'
	, 'OUTSIDE_THRESHOLDS'
	, 'OUT_OF_CCD_WINDOW'
	, 'ON_BADPIX'
	, 'COSMIC_RAY'
	, 'IN_BAD_FRAME'
	, 'OUT_OF_FOV'
	, 'FLICKERING'
	, 'ON_BADOFFSET'
];

my $eventlistFlags =['UNDERSHOOT'
	, 'BAD_E3E4'
	, 'ON_BADROW'
	, 'OUTSIDE_THRESHOLDS'
	, 'OUT_OF_CCD_WINDOW'
	, 'ON_BADPIX'
	, 'COSMIC_RAY'
	, 'IN_BAD_FRAME'
	, 'FLICKERING'
	, 'ON_BADOFFSET'
];

sub performAction {
	info("Module version number: $version");
	# Misc. variables
	my @extraopts;

	# Work out which exposure this stream maps on to
	my $exp_id=exposureID(instrument => thisInstrument
		,stream => thisStream
	);
	info("Exposure ID: $exp_id");

	#
	my $filter=getExposureProperty(exp_id => $exp_id
		,name => 'filter'
	);
	
	my $mode=getExposureProperty(exp_id => $exp_id
		,name => 'mode'
	);

	info("Filter: $filter");
	info("Instrument mode: $mode");

	# Find event files for this exposure
	my @evFiles=findFile(class => 'ODF'
		,instrument => thisInstrument
		,exp_id => $exp_id
		,content => '*events'
	);

	# Return immediately if there are no event files
	if ($#evFiles<0)
	{
		ppd("000005");
		return success();
	}
	my @evFiles2;

	# Find auxiliary file for this exposure in the ODF
	my $auxFile=findFile(class => 'ODF'
		,instrument => thisInstrument
		,exp_id => $exp_id
		,content => 'Auxiliary data'
		,required => 'true'
	);

	# Find attitude HK file
	my $attFile=findFile(class => 'product'
		,instrument => 'all'
		,content => 'Attitude time series'
		,required => 'true'
	);

	# Find HK GTI file
	my $hkGTIFile=findFile(class => 'intermediate'
		,instrument => thisInstrument
		,content => 'HK GTI'
	);

	# Loop through each event file
	my @filter;
	my $fovArea = 0;

    foreach my $evFile0 (@evFiles) 
	{

		# Find CCD and node numbers
		my $ccd=readFITSKeyword(file => $evFile0,
			extension => 2,
			keyword => "CCDID"
		);
		my $node=readFITSKeyword(file => $evFile0,
			extension => 2,
			keyword => "CCDNODE"
		);
		my $datamode=readFITSKeyword(file => $evFile0,
			extension => 2,
			keyword => "DATATYPE"
		);
		info("First pass of events for CCD $ccd, node $node, mode=$datamode");

		# Run emframes
		#
		my $frameFile=newFile(class => 'intermediate',
			instrument => thisInstrument,
			exp_id => $exp_id,
			node => $node, 'ccd' => $ccd,
			content => 'Rectified frames'
	 	);
		my $gtiFile1=newFile(class => 'intermediate',
			instrument => thisInstrument,
			exp_id => $exp_id,
			node => $node,
			ccd => $ccd,
			content => 'CCD GTI',
			level => 1
		);
		my $evFile1=newFile(class => 'intermediate',
			instrument => thisInstrument,
			exp_id => $exp_id,
			node => $node,
			ccd => $ccd,
			content => 'Processed events',
			level => 1
		);
		@extraopts=();
		@extraopts=(flagbadtimes => 'yes', ingtiset => $hkGTIFile)
			if $hkGTIFile
		;

		doCommand('emframes',
			  auxiliaryset => $auxFile,
			  frameset => $frameFile,
			  withodfeventset => 'Y',
			  odfeventset => $evFile0,
			  neweventset => 'Y',
			  outeventset => $evFile1,
			  writegtiset => 'Y',
			  outgtiset => $gtiFile1,
			  @extraopts
		) or return exception();

		if( hasFITSKeyword(file=>$frameFile, extension=>'FRAMES', keyword=>'GAIN_CCD')
		   && readFITSKeyword(file=>$frameFile, extension=>'FRAMES', keyword=>'GAIN_CCD') =~ /^LOW/ 
		){
			info("CCD $ccd is in low gain mode.  Will not process events for this ccd.");
			ppd(id => "000007" , ccd => $ccd , node => $node );
			next;
		}

		push @evFiles2, $evFile0;

			#
		@extraopts=();
		if( $datamode eq "IMAGE.EL" || $datamode eq "RIMAGE.EL" )
		{
			my $evFile2=newFile(class => 'intermediate',
				instrument => thisInstrument,
				exp_id => $exp_id,
				node => $node,
				ccd => $ccd,
				content => 'Processed events',
				level => 2
			);

			@extraopts=();
			@extraopts=( rejectbade3 => 'N' )
			unless ($datamode eq "IMAGE.EL");

			# Check if an offset-variance map is available for this
			# instrument/CCD/node, if so add to the arguments for 'emevents'
			#
			my @ovfile=findFile(class => 'ODF',
					instrument => thisInstrument,
					ccd => $ccd,
					node => $node,
					content => 'Offset/variance mode events'
			);
			push(@extraopts, withoffvarsets => 'Y', offvarsets => join(" ",@ovfile))
			if @ovfile;
			ppd(id => "000006" , ccd => $ccd , node => $node) if @ovfile;

			doCommand('emevents',
				  odfeventset => $evFile1,
				  eventset => $evFile2,
				  analysepatterns => 'N',
				  flagbadpixels => 'N',
				  splitdiagonals => 'N',
				  withframeset => 'Y',
				  frameset => $frameFile,
				  randomizeposition => 'N',
				  @extraopts)
			or return exception();

			doCommand('emenergy',
				  ineventset => $evFile2,
				  correctcti => 'N',
				  correctgain => 'N',
				  randomizeenergy => 'N')
			or return exception();

			$fovArea += readFITSKeyword(file => $evFile2
						,extension => 'EVENTS'
						,keyword => 'IN_FOV'
			) if ( hasFITSKeyword(file => $evFile2
				,extension => 'EVENTS'
				,keyword => 'XMMEA_22'
				) 
			);
		}
    }

    # Make flare gti for use with bad pixel searching

    my $hasBadpixGti = 0;
    my $bpFlareGti;
    my $backgrndBinsize = 52;
    my $GTI_time_threshold=gtiTimeThreshold();

	my $fovAreaThreshold = 10;
    if( $fovArea > $fovAreaThreshold )
	{
		ppd(id => "000008" , data => { fovarea => $fovArea , threshold => $fovAreaThreshold } );
		my $bpEvFile=newFile(class => 'intermediate',
					 instrument => thisInstrument,
					 exp_id => $exp_id,
					 content => 'Bad pix find event list'
		);

		my @outerEvFiles=findFile(class => 'intermediate'
			,instrument => thisInstrument
			,exp_id => $exp_id
			,content => 'Processed events'
			,level => 2
		);

		doCommand('evlistcomb'
			,eventsets => [@outerEvFiles]
			,imagingset => $bpEvFile
			,instrument => 'emos'
			,emosdatamodes => 'IMAGING'
		);
		
		$bpFlareGti=newFile(class => 'intermediate',
					instrument => thisInstrument,
					exp_id => $exp_id,
					content => 'Bad pix flare GTI');
			my $histoFile=newFile(class => 'intermediate',
								  instrument => thisInstrument,
								  exp_id => $exp_id,
								  content => 'Flare rates file',
								  level => 1);

		if( hasFITSKeyword(file => $bpEvFile, extension => 'EVENTS', keyword => 'XMMEA_22') )
		{
			my $flareBitmask = getEventBitmask(file=>$bpEvFile, masks=>$flareFlags);
			ppd(id => "000009" , data => { flarebitmask => $flareBitmask } );

			doCommand('evselect'
				,table => $bpEvFile
				,writedss => 'N'
				,updateexposure => 'N'
				,withrateset => 'Y'
				,rateset => $histoFile
				,maketimecolumn => 'Y'
				,timebinsize => $backgrndBinsize
				,timecolumn => 'TIME'
				,expression => "(PATTERN==0) && #XMMEA_22 && ((FLAG & $flareBitmask) == 0)"
				,withimageset => 'N'
			) or return exception();
		
			if( numberFITSRows(file=>$histoFile, extension=>'RATE') > 0 )
			{
				doCommand('tabcalc',
					  tables => "$histoFile:RATE",
					  columntype => 'real32',
					  column => 'RATE',
					  expression => "1000*COUNTS/($backgrndBinsize*$fovArea)" )
				or return exception();


				doCommand('tabgtigen'
					  ,table => $histoFile
					  ,gtiset => $bpFlareGti
					  ,expression => 'RATE<2.0'
					  ,mingtisize => 100
				) or return exception();
				
				my $GTI_time=0;
				if ($GTI_time_threshold) 
				{
					$GTI_time = readFITSKeyword(file => $bpFlareGti
								,extension => "STDGTI"
								,keyword => "ONTIME"
					);
					ppd(id => "000010" , data => { ontime => $GTI_time , threshold => $GTI_time_threshold } );
					info("Flare screening GTI_Time: $GTI_time, GTI_time_threshold: $GTI_time_threshold");
				}
				
				if  ( numberFITSRows(file =>  $bpFlareGti, extension => 'STDGTI') > 0
					&& ($GTI_time > $GTI_time_threshold || !$GTI_time_threshold) 
				)
				{
					ppd( id => "000011" );
					$hasBadpixGti = 1;
				}
			}
		}
    }

    $fovArea = 0;

    # Second loop over CCDs

    foreach my $evFile0 (@evFiles2) 
	{

		# Find CCD and node numbers
		my $ccd=readFITSKeyword(file => $evFile0,
					extension => 2,
					keyword => "CCDID");
		my $node=readFITSKeyword(file => $evFile0,
					 extension => 2,
					 keyword => "CCDNODE");
		my $datamode=readFITSKeyword(file => $evFile0,
						 extension => 2,
						 keyword => "DATATYPE");
		info("Second pass of events for CCD $ccd, node $node, mode=$datamode");

		my $frameFile=findFile(class => 'intermediate',
					   instrument => thisInstrument,
					   exp_id => $exp_id,
					   node => $node, 'ccd' => $ccd,
					   content => 'Rectified frames');

		my $evFile1=findFile(class => 'intermediate',
					 instrument => thisInstrument,
					 exp_id => $exp_id,
					 node => $node,
					 ccd => $ccd,
					 content => 'Processed events',
					 level => 1);

		my $gtiFile1=findFile(class => 'intermediate',
					  instrument => thisInstrument,
					  exp_id => $exp_id,
					  node => $node,
					  ccd => $ccd,
					  content => 'CCD GTI',
					  level => 1);

		my $evFile3=newFile(class => 'intermediate',
					instrument => thisInstrument,
					exp_id => $exp_id,
					node => $node,
					ccd => $ccd,
					content => 'Processed events',
					level => 3);

		# Do bad pixel finding if imaging mode

		@extraopts=();

		if( $datamode eq "IMAGE.EL" || $datamode eq "RIMAGE.EL" )
		{
			my $evFile2=findFile(class => 'intermediate',
					instrument => thisInstrument,
					exp_id => $exp_id,
					node => $node,
					ccd => $ccd,
					content => 'Processed events',
					level => 2);

			my $badFile=newFile(class => 'intermediate',
					instrument => thisInstrument,
					exp_id => $exp_id,
					node => $node,
					ccd => $ccd,
					content => 'Bad pixel list');

			my $projFile=newFile(class => 'intermediate',
					instrument => thisInstrument,
					exp_id => $exp_id,
					node => $node,
					ccd => $ccd,
					content => 'Projected image');

			# Search for dark pixels, and bright pixels if no flare gti

			@extraopts=(findbright => 'N') if $hasBadpixGti;

			doCommand('emeventsproj',
				  eventset => $evFile2,
				  rejectbadevents => 'Y',
				  evimageset => $projFile)
			or return exception();

			doCommand('embadpixfind',
				  evimageset => $projFile,
				  badpixset => $badFile,
				  includedeadpixels => 'Y',
					  @extraopts)
			or return exception();

			# Search for bright pixels after filtering out flares if the falre gti is above the threshold

			if($hasBadpixGti)
			{
			
				doCommand('evselect'
					  ,writedss => 'N'
					  ,updateexposure => 'N'
					  ,table => $evFile2
					  ,keepfilteroutput => 'Y'
					  ,expression => "gti($bpFlareGti:STDGTI,TIME)"
					  ,withimageset => 'N'
				) or return exception();

				doCommand('emeventsproj',
					  eventset => $evFile2,
					  rejectbadevents => 'Y',
					  evimageset => $projFile)
					or return exception();

				doCommand('embadpixfind',
					  evimageset => $projFile,
					  badpixset => $badFile,
					  finddead => 'N',
					  incremental => 'Y')
					or return exception();
			}

			@extraopts=(getnewbadpix => 'Y'
				,badpixset => $badFile
				,getuplnkbadpix => 'Y'
				,getotherbadpix => 'Y'
			);
		}

		# run badpix on the event file
		#

		doCommand('badpix',
			  eventset => $evFile1,
			  withoutset => 'Y',
			  outset => $evFile3,
			  windowfilter => 'Y',
			  @extraopts)
			or return exception();

		# Check if an offset-variance map is available for this
		# instrument/CCD/node, if so add to the arguments for 'emevents'
		#
		@extraopts=();
		my @ovfile=findFile(class => 'ODF',
					instrument => thisInstrument,
					ccd => $ccd,
					node => $node,
					content => 'Offset/variance mode events');
	    if ( @ovfile )
		{
			ppd( id => "000012" );
			push(@extraopts, withoffvarsets => 'Y', offvarsets => join(" ",@ovfile));
		}


		#
		push(@extraopts, splitdiagonals => 'N')
			  if $datamode eq "TIME.EL" || $datamode eq "CTIME.EL";

		push(@extraopts, flagtruncatede1 => 'N')
			  if $datamode eq "CTIME.EL";

		#
		push(@extraopts, rejectbade3 => 'N')
		  unless $datamode eq "IMAGE.EL";


		my $evFile4=newFile(class => 'intermediate',
					instrument => thisInstrument,
					exp_id => $exp_id,
					node => $node,
					ccd => $ccd,
					content => 'Processed events',
					level => 4
		);

		doCommand("emevents",
			  withframeset => 'Y',
			  frameset => $frameFile,
			  odfeventset => $evFile3,
			  eventset => $evFile4,
			  randomizeposition => 'Y',
			  randomizetime => 'Y',
			  @extraopts
		) or return exception();


		# Align global gti file with event list and merge with ccd gti
			# Some files may not have been created, so need to check
			#
		my $hkGtiFileAligned=newFile(class => 'intermediate',
						 instrument => thisInstrument,
						 exp_id => $exp_id,
						 node => $node,
						 ccd => $ccd,
						 content => 'Aligned HK GTI');
		my $gtiFile2=newFile(class => 'intermediate',
					 instrument => thisInstrument,
					 exp_id => $exp_id,
					 node => $node,
					 ccd => $ccd,
					 content => 'CCD GTI',
					 level => 2);

		if( fileExists(file => $hkGTIFile) )
		{
			ppd("000013");
			if( ($datamode eq "IMAGE.EL" || $datamode eq "RIMAGE.EL") )
			{
				doCommand("gtialign",
					gtitable => $hkGTIFile . ":STDGTI",
				  	eventset => $evFile4,
				  	outset => $hkGtiFileAligned)
				or return exception();
			}
			else
			{
				copyFile(source => $hkGTIFile, destination => $hkGtiFileAligned); 
				doCommand("dscp",
					  from => "$hkGtiFileAligned:STDGTI",
					  to => "$hkGtiFileAligned:STDGTI" . $node . $ccd)
				  or return exception();
				doCommand("dsrm",
					  objects => "$hkGtiFileAligned:STDGTI")
				  or return exception();
			}

			doCommand("gtimerge",
				  tables => "$hkGtiFileAligned $gtiFile1",
				  mergemode => 'and',
				  gtitable => "$gtiFile2:STDGTI" . $node . $ccd)
			or return exception();
		}
		else
		{
			copyFile(source => $gtiFile1, destination => $gtiFile2);
		}

		# Add GTI file to evselect expression
		push(@filter,"(CCDNR==${node}${ccd} && GTI($gtiFile2,TIME))");

		# Apply attitude correction to events
		#
		doCommand("attcalc",
			  eventset => $evFile4,
			  refpointlabel => 'pnt',
			  withmedianpnt => 'true',
			  attitudelabel => 'ahf',
			  atthkset => $attFile)
			or return exception();


		# Rectify event energies
		#
		@extraopts=();

		# Background subtraction disabled in non-imaging mode
		push(@extraopts, getccdbkg => 'N', rejectbade3e4 => 'N')
		  unless $datamode eq "IMAGE.EL";

		doCommand('emenergy',
			  ineventset => $evFile4,
			  randomizeenergy => 'Y',
			  @extraopts)
		  or return exception();
		  
		$fovArea += readFITSKeyword(file => $evFile4
					,extension => 'EVENTS'
					,keyword => 'IN_FOV'
		) if ( hasFITSKeyword(file => $evFile4
			,extension => 'EVENTS'
			,keyword => 'XMMEA_22'
			) 
		);

		# Filter out unwanted events before lists are merged
		#
		my $eventBitmask = getEventBitmask(file=>$evFile4, masks=>$eventlistFlags);

		doCommand('evselect'
			  ,table => "${evFile4}:EVENTS"
			  ,destruct => 'Y'
			  ,keepfilteroutput => 'Y'
			  ,writedss => 'Y'
			  ,updateexposure => 'N'
			  ,expression => "(FLAG & $eventBitmask)==0"
			  ,withimageset => 'N'
		) or return exception();
    }


    # Merge per-CCD event lists into one per camera (for imaging and
    # timing mode)
    #
    @evFiles=findFile(class => 'intermediate',
		      instrument => thisInstrument,
		      exp_id => $exp_id,
		      content => 'Processed events',
		      level => 4);
    my $tmpimgEvents=newFile(class => 'intermediate',
			     instrument => thisInstrument,
			     exp_id => $exp_id,
			     content => 'imaging mode event list');
    my $tmptimEvents=newFile(class => 'intermediate',
			     instrument => thisInstrument,
			     exp_id => $exp_id,
			     content => 'timing mode event list');

    doCommand('evlistcomb', 
	      eventsets => [@evFiles],
	      imagingset => $tmpimgEvents, 
	      timingset => $tmptimEvents,
	      instrument => 'emos',
	      maintable => 'EVENTS OFFSETS',
	      othertables => 'EXPOSURE STDGTI BADPIX' )
	or return exception();


    # Create filter expression for event lists
    my $expr=join("||", @filter);


    # Find calibration index file
    my $cifFile=findFile(class => 'product',
			 content => 'Calibration index file');


    # Filter good imaging events into final product files
    if(fileExists(file => $tmpimgEvents))
	{
		my $imgEvents=newFile(class => 'product',
					  instrument => thisInstrument,
					  exp_id => $exp_id,
					  content => 'EPIC MOS imaging mode event list');
		my $eventBitmask = getEventBitmask(file=>$tmpimgEvents, masks=>$eventlistFlags);

		doCommand('evselect'
			  ,table => $tmpimgEvents
			  ,filteredset => "${imgEvents}:EVENTS"
			  ,withfilteredset => 'true'
			  ,expression => "($expr) && (FLAG & $eventBitmask)==0"
			  ,writedss => 'true'
			  ,cleandss => 'true'
			  ,updateexposure => 'true'
			  ,keepfilteroutput => 'true'
			  ,destruct => 'true'
			  ,withimageset => 'N'
		  ) or return exception();

		# Copy CIF into event list
		doCommand('dscopyblock',
			  blocks => "${cifFile}:CALINDEX",
			  to => $imgEvents)
		  or return exception();

		# Create flare gti
		# Works as soon as the GATTI was correctly reconstructed for one CCD
		#
		if( $fovArea > 10 ){
			my $flareGti=newFile(class => 'intermediate',
					 instrument => thisInstrument,
					 exp_id => $exp_id,
					 content => 'flare GTI');
			my $histoFile=newFile(class => 'product',
					  instrument => thisInstrument,
					  exp_id => $exp_id,
					  content => 'EPIC flare background timeseries');

			if( hasFITSKeyword(file => $imgEvents, extension => 'EVENTS', keyword => 'XMMEA_22') ){

			my $flareBitmask = getEventBitmask(file=>$imgEvents, masks=>$flareFlags);
			doCommand('evselect'
				,table => $imgEvents
				,writedss => 'N'
				,updateexposure => 'N'
				,withrateset => 'Y'
				,rateset => $histoFile
				,timebinsize => $backgrndBinsize
				,maketimecolumn => 'Y'
				,timecolumn => 'TIME'
				,expression => "(PATTERN==0) && #XMMEA_22 && ((FLAG & $flareBitmask) == 0)"
				,withimageset => 'N'
			) or return exception();

			doCommand(
				'tabcalc'
				, tables => "$histoFile:RATE"
				, columntype => 'real32'
				, column => 'RATE'
				, expression => "1000*COUNTS/($backgrndBinsize*$fovArea)"
			  )
			  or return exception();

		# Now subtract TSTART from the TIME values and leave the result in a new column T_ELAPSED:
		my $start_time=readFITSKeyword(
				file => $histoFile
				, extension => 'RATE'
				, keyword => "TSTART"
			  );
			doCommand(
				'tabcalc'
				, tables => "$histoFile:RATE"
				, columntype => 'real64'
				, column => 'T_ELAPSED'
				, expression => "TIME-$start_time"
	# Use the perl variable rather than the string '#TSTART' in case we decide later to use some other time as offset.
			  )
			  or return exception();
			my $fplotCommandFile = newFile(
				class => 'intermediate'
				, content => 'Flare bkg ts pco file'
				, format => 'ASCII'
			, extension => 'pco'
			);
			writeASCIIFile(
				name => $fplotCommandFile
				, text => [join("\n", "LA X TIME-$start_time (s)", 'LA G2 RATE (cts/s)', 'PLOT', 'QUIT', '')]
			);
			my $flareBkgPs = newFile(
				class => 'intermediate'
				, instrument => thisInstrument
				, exp_id => $exp_id
				, content => 'F bkg ts'
				, format => 'PS'
			);
			doCommand(
				'fplot'
				, infile => $histoFile
				, xparm => 'T_ELAPSED'
				, yparm => 'RATE'
				, rows => '-'
				, device => "$flareBkgPs/cps"
				, pltcmd => '@'."$fplotCommandFile"
			  )
			  or return exception();
			my $flareBkgPdf = newFile(
				class => 'product'
				, instrument => thisInstrument
				, exp_id => $exp_id
				, content => 'EPIC flare background timeseries'
				, format => 'PDF'
			);
			PStoPDF(
				source => $flareBkgPs
				, destination => $flareBkgPdf
			  )
			  or return exception();

				if( numberFITSRows(file=>$histoFile, extension=>'RATE') > 0 )
				{

		#			doCommand('tabcalc'
		#			      ,tables => "$histoFile:RATE"
		#			      ,columntype => 'real32'
		#			      ,column => 'RATE'
		#			      ,expression => "1000*COUNTS/($backgrndBinsize*$fovArea)" 
		#			) or return exception();

					doCommand('tabgtigen'
						  ,table => $histoFile
						  ,gtiset => $flareGti
						  ,expression => 'RATE<2.0'
						  ,mingtisize => 100
					) or return exception();

				}
			}
		}
    }


    # Filter good timing events into final product files
    if(fileExists(file => $tmptimEvents)){
	my $timEvents=newFile(class => 'product',
			      instrument => thisInstrument,
			      exp_id => $exp_id,
			      content => 'EPIC timing mode event list');
	doCommand('evselect'
		  ,table => $tmptimEvents
		  ,filteredset => $timEvents
		  ,withfilteredset => 'true'
		  ,keepfilteroutput => 'true'
		  ,updateexposure => 'true'
		  ,writedss => 'true'
		  ,cleandss => 'true'
		  ,#expression => "($expr) && #XMMEA_EM"
		  ,expression => $expr
		  ,destruct => 'true'
		  ,withimageset => 'N'
	) or return exception();

	# Copy CIF into event list
	doCommand('dscopyblock',
		  blocks => "${cifFile}:CALINDEX",
		  to => $timEvents)
	  or return exception();
    }


    return success();
}

1;