#!/usr/bin/env python
# -*- python -*-

r'''
Genvalobs command line utility.

Uses the lofarobsxml library to compose an XML file containing LOFAR test
observations that can be submitted to MoM.
'''

from __future__ import print_function


import sys
import copy
import argparse

import ephem

from lofarobsxml import Folder, SourceCatalogue, lofar_sidereal_time
from lofarobsxml import Stokes, TiedArrayBeams, BackendProcessing, Beam, Observation
from lofarobsxml import station_list, radec_from_lm, parse_subband_list
from lofarobsxml import TargetSource
from lofarobsxml import xml
from lofarobsxml import SourceSpecificationError, InvalidStationSetError
from lofarobsxml import NoSuitableSourceError
from lofarobsxml import __version__
from lofarobsxml.pipelines import NDPPP, AveragingPipeline


#
# Various messages
#

def prerror(message):
    """
    Print the string in *message* to standard error.
    """
    return sys.stderr.write('genvalobs error: '+message+'\n')


def print_short_help():
    """
    Print a short help message to standard output.
    """
    return sys.stdout.write('genvalobs: use "genvalobs --help" for help\n')


def uppercase_string(string_var):
    r'''
    Function that converts string_var to upper case. Used in
    parse_arguments() as a data type.

    **Paremeters**

    string_var : string
        The string to convert to upper case.

    **Returns**

    An upper case string.

    **Examples**

    >>> [uppercase_string(s) for s in ['asd', 'aBc', '123']]
    ['ASD', 'ABC', '123']
    '''
    return string_var.upper()


def parse_arguments():
    r'''
    Parse all command line arguments. Uses Python 2.7's argparse module.
    '''
    parser = argparse.ArgumentParser(
        description=r'''
The (optional) source name must be enclosed in single or double
quotes if it contains spaces. The following sources are recommended:

- "3C 48"  /  48: LST 22:30--04:30
- "3C 147" / 147: LST 02:30--08:30
- "3C 196" / 196: LST 05:00--11:00
- "3C 295" / 295: LST 11:00--17:00
- "Cyg A"  / cyg: LST 16:00--24:00

If no source is specified, the program chooses the source that is
closest to the meridian at the central LST of the observing
sequence. The program has separate calibrator lists for LBA
and HBA observations and pulsars.

Although genvalobs has its own default sequence of observations, it is
possible to read a custom set from an ASCII file with a fairly simple
format. The specification consists of newline-separated observations
where each observation is specified in a white-space separated line
with format:

    <ANTENNA_SET> <BAND> <SUBBANDS> <CLOCK> <BIT_MODE> <DATA_PRODUCTS>

    - ANTENNA_SET: one of LBA_INNER, LBA_OUTER, LBA_SPARSE_ODD, 
                   LBA_SPARSE_EVEN, HBA_ZERO, HBA_ONE,
                   HBA_DUAL, HBA_JOINED, HBA_ZERO_INNER,
                   HBA_ONE_INNER, HBA_DUAL_INNER

    - FREQUENCY_BAND: LBA_LOW, LBA_HIGH, HBA_LOW, HBA_MID, HBA_HIGH

    - SUBBAND_LIST: comma-separated list of sub band ranges. Note: NO
                    SPACES ALLOWED! Examples:
                      - 12..499
                      - 12..22,112..122,212..222
    - CLOCK_MHZ: either 200 or 160

    - BIT_MODE: either 4, 8, or 16

    - DATA_PRODUCTS: white-space separated list of products. Allowed:
                     - XC      (cross-correlation)
                     - FE      (fly's eye)
                     - CS_I    (coherent stokes I)
                     - CS_IQUV (coherent stokes IQUV)
                     - IS_I    (incoherent stokes I)
                     - IS_IQUV (incoherent stokes IQUV)
                     - TR      (TAB rings)

    Empty lines are ignored, comments start with # end run until the
    end of the line.

    Example file:

        # Antennaset    Band      Subbands   Clock  Bits   Products  Pipeline (avg freq; avg time; demix freq-window; demix time-window)
        LBA_OUTER       LBA_LOW   12..499    200     8     XC
        HBA_DUAL        HBA_LOW   12..499    200     8     XC        Preprocessing 64 2 64 10
        LBA_INNER       LBA_HIGH  156..399   200    16     FE
        HBA_DUAL        HBA_LOW   77..320    200    16     IS_I
        HBA_DUAL        HBA_LOW   77..320    200    16     CS_I
        HBA_DUAL        HBA_LOW   77..320    200    16     CS_IQUV
        HBA_DUAL_INNER  HBA_MID   66..309    160    16     XC IS_I

Note that the custom sequence is subject to the same filtering by
options -m, -c, etc. as the default observing sequence. The last entry
is using the 160 MHz clock. It will only be used in the observation
sequence if the 160 MHz is selected using the -c / --clock option.

''',
        formatter_class=argparse.RawDescriptionHelpFormatter)

    #usage='%(prog)s [options] \'source name\'')
    arg = parser.add_argument

    arg('-o', '--output', metavar='FILENAME',
        help='Name of the output file [%(default)s].',
        default=('lofar-validation-%04d%02d%02d-%02d%02d%02d.xml' %
                 ephem.Observer().date.tuple()))

    arg('-m', '--mode', metavar='CORRELATOR_MODE',
        type=uppercase_string,
        choices=['XC', 'FE', 'CS', 'CS_I', 'CS_IQUV',
                 'IS', 'IS_I', 'IS_IQUV', 'TR'],
        action='append',
        help='''
Correlator modes to test. Choose one of XC (crosscorrelation), FE
(Fly\'s eye), IS (incoherent stokes), or CS (coherent stokes). One can
specify more than one mode. If this argument is not spedified, all
        modes will be tested [XC, FE, CS_I, CS_IQUV, IS_I, IS_IQUV, TR].''',
        default=None)

    arg('-d', '--duration', metavar='SECONDS',
        help='Duration of individual observations in seconds [%(default)r].',
        type=int, default=120)

    arg('-g', '--gap', metavar='SECONDS',
        help='Gap between observations in seconds [%(default)r].',
        type=int, default=60)

    arg('-s', '--stations', metavar='STATION_SET',
        choices=['superterp', 'core', 'remote', 'nl', 'eu', 'all', 'none'],
        help='''
One of superterp, core, remote, nl, eu, all, or none. EU stations
that conflict with certain HBA1 core fields are excluded from the
HBA_ONE, HBA_ONE_INNER, HBA_DUAL, and HBA_DUAL_INNER
observations. They are taken along in all other observations [%(default)s].''',
        default='nl')

    arg('-i', '--include', metavar='STATION_NAMES',
        help='''Comma separated list of station names to include. No
        spaces allowed in the list. Example: -i cs013,de601,rs106''')

    arg('-e', '--exclude', metavar='STATION_NAMES',
        help='''Comma separated list of station names to exclude. No
        spaces allowed in the list. Example: -e cs013,de601,rs106''')


    arg('--min-alt', metavar='DEGREES',
        help='Minimum elevation for target sources [%(default)5.2f].',
        type=float, default=40.0)

    arg('--max-alt', metavar='DEGREES',
        help='Maximum elevation for target sources [%(default)5.2f].',
        type=float, default=70.0)

    arg('-c', '--clocks', metavar='CLOCK_MHZ', action='append',
        choices=[160, 200],
        help='''Allowed clock frequencies. Choose 160 or 200. Option can be
provided multiple times if more than one clock frequency is
required.''',
        type=int, default=None)

    arg('-w', '--wait', metavar='SECONDS',
        help='Number of seconds until the first observation [%(default)r].',
        type=int, default=540)

    arg('-t', '--start-date', metavar='DATE_STRING',
        help='''Specify an exact date and time to start the first
        observation. Format: "yyyy/mm/dd hh:mm:ss.s"''')

    arg('-p', '--project', metavar='MOM_NAME',
        help='Name of the observations\' MoM project [%(default)r].',
        default='2017LOFAROBS')

    arg('-f', '--from-file', metavar='FILE_NAME',
        help='Read a custom observing sequence from FILE_NAME.',
        default=None)

    arg('-v', '--version', action='version',
        version='%(prog)s ' +__version__+
        ' (c) 2010-2017 M.A. Brentjens <brentjens@astron.nl>',
        help='Print version number and exit.')

    arg('-a', '--approved', action='store_true',
        help="Set initial status of observations and pipelines to 'approved' instead of 'opened'.",
        default=False)

    arg('--max-subbands', metavar='N',
        help='''Set the maximum allowed number of sub bands per
observation. When minimal resources are required, use for example
10. If limited, it will take the first N sub bands.''',
        type=int, default=488)

    arg('--storage-cluster', metavar='CLUSTER_NAME',
        help='''The cluster to which data are being stored and
at which pipelines are being run.''',
        type=str, default='CEP4')

    arg('--storage-partition', metavar='PARTITION',
        help='''The directory/partition to which data are being stored.''',
        type=str, default='')

    arg('source', nargs='?',
        type=str,
        help='''
Force a source to be used for all observations, bypassing genvalobs\'
own heuristics.''',
        default=None)

    args = parser.parse_args()
    if args.clocks is None:
        args.clocks = [200]

    if args.mode is None:
        args.mode = ['XC', 'FE', 'CS', 'CS_I', 'CS_IQUV', 'IS', 'IS_I', 'IS_IQUV', 'TR', 'BM']
    args.mode = [mode.upper() for mode in args.mode]
    if args.storage_partition == '':
        args.storage_partition = None
    return  args





def lookup_source(source_name=None):
    """
    Look up *source_name* in a table of standard calibrator
    sources. Raise a SourceSpecificationError if the source is not
    found. Otherwise, return a TargetSource object.
    """
    return SourceCatalogue().find_source(source_name)


def read_custom_observation_sequence(file_contents):
    r'''
    Read a custom scheduled from string ``file_contents``. The
    specification consists of newline-separated observationsm where
    each observation is specified in a white-space separated line with format:

        <ANTENNA_SET>  <FREQ_BAND>  <SUBBANDS>  <CHANNELS> <CLOCK_MHZ>  <BITS>  <PRODUCTS>

    - ANTENNA_SET: one of LBA_INNER, LBA_OUTER, LBA_SPARSE_ODD,
                   LBA_SPARSE_EVEN, HBA_ZERO, HBA_ONE,
                   HBA_DUAL, HBA_JOINED, HBA_ZERO_INNER,
                   HBA_ONE_INNER, HBA_DUAL_INNER

    - FREQUENCY_BAND: LBA_LOW, LBA_HIGH, HBA_LOW, HBA_MID, HBA_HIGH

    - SUBBAND_LIST: comma-separated list of sub band ranges. Note: NO
                    SPACES ALLOWED! Examples:
                      - 12..499
                      - 12..22,112..122,212..222
    - CHANNELS: number of channels per sub band. Usually 64, sometimes 
      16 or 256.

    - CLOCK_MHZ: either 200 or 160

    - BIT_MODE: either 4, 8, or 16

    - DATA_PRODUCTS: white-space separated list of products. Allowed:
                     - XC (cross-correlation)
                     - BM (5x5 multi-beam XC)
                     - FE (fly's eye)
                     - IS_I    (incoherent stokes)
                     - IS_IQUV (incoherent stokes)
                     - CS_I    (coherent stokes)
                     - CS_IQUV (coherent stokes)
                     - CV (complex voltage)
                     - TR (tab rings)

    Empty lines are ignored, comments start with # end run until the
    end of the line.

    Example file:

        # Antennaset    Band      Subbands   Channels Clock  Bits   Products
        LBA_OUTER       LBA_LOW   12..499    64       200     8     XC
        HBA_DUAL        HBA_LOW   12..499    64       200     8     XC       Preprocessing 64 2 64 10
        LBA_INNER       LBA_HIGH  156..399   64       200    16     FE
        HBA_DUAL        HBA_LOW   77..320    64       200    16     IS_I
        HBA_DUAL        HBA_LOW   77..320    64       200    16     CS_I
        HBA_DUAL        HBA_LOW   77..320    64       200    16     CS_IQUV
        HBA_DUAL_INNER  HBA_MID   66..309    64       160    16     XC IS_I

    Note that the last entry is using the 160 MHz clock. It will only be
    used in the observation sequence if the 160 MHz is selected using
    the -c / --clock option. See the genvalobs --help for details.

    **Parameters**

    file_contents : string
        Contents of the specification file.

    **Raises**

    ValueError:
        If an error occurs while reading the file.

    '''
    lines = [line.strip() for line in file_contents.split('\n')]
    no_comments = [line.split('#')[0].strip() for line in lines]
    no_empty_lines = [line.strip()
                      for line in no_comments
                      if line.strip() != '']

    valid_antenna_sets = ['LBA_INNER', 'LBA_OUTER',
                          'LBA_SPARSE_ODD', 'LBA_SPARSE_EVEN',
                          'HBA_ZERO', 'HBA_ONE', 'HBA_DUAL', 'HBA_JOINED',
                          'HBA_ZERO_INNER', 'HBA_ONE_INNER',
                          'HBA_DUAL_INNER']
    valid_bands = ['LBA_LOW', 'LBA_HIGH',
                   'HBA_LOW', 'HBA_MID', 'HBA_HIGH']
    valid_clocks = [160, 200]
    valid_channels = [1]+ [2**n for n in range(4, 12)]
    valid_bits = [4, 8, 16]
    valid_products = ['XC', 'FE', 'IS', 'IS_I', 'IS_IQUV',
                      'CS', 'CS_I', 'CS_IQUV', 'CV', 'TR', 'BM']
    valid_pipelines = ['Preprocessing']

    pipeline = None

    def parse_pipeline_spec(words):
        pipeline_def = [words[0]]
        if pipeline_def[0] == 'Preprocessing':
            if len(words) >= 3:
                pipeline_def.append(int(words[1]))
                pipeline_def.append(int(words[2]))
            if len(words) >= 5:
                pipeline_def.append(int(words[3]))
                pipeline_def.append(int(words[4]))
            else:
                pipeline_def.append(64)
                pipeline_def.append(10)
            if len(words) > 5:
                pipeline_def.append(' '.join(words[5:]))
        else:
            raise ValueError('Unknown pipeline %r' % words[0])
        return pipeline_def

    def record_from_line(line):
        'Split a line (at its whitespace) into a list of fields'
        fields = line.split()
        data_products = []
        pipeline = None

        if len(fields) < 7: # or len(fields) > 9:
            raise ValueError('Line %r is malformed' % line)

        if fields[0].upper() not in valid_antenna_sets:
            raise ValueError('Antenna set %s not in %r' %
                             (fields[0], valid_antenna_sets))

        if fields[1].upper() not in valid_bands:
            raise ValueError('Band %s not in %r' %
                             (fields[1], valid_bands))

        if int(fields[3]) not in valid_channels:
            raise ValueError('Number of channels %s per subband not in %r' %
                             (fields[3], valid_channels))

        if int(fields[4]) not in valid_clocks:
            raise ValueError('Clock frequency %r not in %r' %
                             (int(fields[4]), valid_clocks))

        if int(fields[5]) not in valid_bits:
            raise ValueError('Bit mode %r not in %r' %
                             (int(fields[5]), valid_bits))

        for offset, data_product in enumerate(fields[6:]):
            if data_product.upper() not in valid_products:
                if data_product in valid_pipelines:
                    pipeline = parse_pipeline_spec(fields[6+offset:])
                    break
                else:
                    raise ValueError('Data product %r not in %r' %
                                     (data_product, valid_products))
            elif data_product.upper() in valid_products:
                data_products.append(data_product.upper())
            else:
                raise ValueError('Do not know what to do with %r' % data_product)

        return [fields[0].upper(),
                fields[1].upper(),
                fields[2],
                int(fields[3]),
                int(fields[4]),
                int(fields[5]),
                data_products,
                pipeline]

    return [record_from_line(line) for line in no_empty_lines]




def plan_observing_sequence(job_description):
    r'''
    Actually compute the observing sequence. This function contains
    all of the policy.
    '''
    # Possible products:
    #  'XC'             , 'IS'             , 'CS'           , 'FE'       'BM' for
    #  cross correlation, incoherent stokes, coherent stokes, Fly's eye, 5x5 beam scans
    # pylint: disable=bad-whitespace
    full_schedule = [
        ['LBA_OUTER'     , 'LBA_LOW' , '12..499' , 64, 200,  8, ['XC'],
         ('Preprocessing', 8, 1, 64, 10, 'CygA CasA')],
        ['HBA_DUAL'      , 'HBA_LOW' , '12..499' , 64, 200,  8, ['XC'],
         ('Preprocessing', 16, 5, 64, 10)],
        ['LBA_OUTER'     , 'LBA_HIGH' ,
         ','.join(['%d' % sb for sb in range(150, 331, 10)]), 64, 200,  8, ['BM'],
         ('Preprocessing', 64, 2, 64, 10)],
        ['HBA_DUAL'      , 'HBA_LOW' ,
         ','.join(['%d' % sb for sb in range(77, 348, 15)]) , 64, 200,  8, ['BM'],
         ('Preprocessing' , 64, 5, 64, 10)],
        ['LBA_INNER'      , 'LBA_HIGH', '156..399', 16, 200, 16, ['FE'], None],
        ['LBA_OUTER'      , 'LBA_HIGH', '156..399', 16, 200, 16, ['FE'], None],
        ['HBA_DUAL'       , 'HBA_LOW' , '77..320' , 16, 200, 16, ['FE'], None],
        ['HBA_DUAL'       , 'HBA_LOW' , '77..320' , 16, 200, 16, ['IS_I'], None],
        ['HBA_DUAL'       , 'HBA_LOW' , '77..320' , 16, 200, 16, ['CS_I'], None],
        ['HBA_DUAL'       , 'HBA_LOW' , '77..320' , 16, 200, 16, ['CS_IQUV'], None],
        ['HBA_DUAL'       , 'HBA_LOW' , '77..320' ,  1, 200, 16, ['CV'], None],
        ['HBA_DUAL'       , 'HBA_LOW' , '12..499' , 16, 200,  8, ['TR'], None],
        ['HBA_DUAL'       , 'HBA_LOW' , '77..320' , 256, 200, 16, ['XC'], None],
        ['HBA_DUAL'       , 'HBA_LOW' , '77..320' , 16, 200, 16, ['XC', 'CS_IQUV'], None],
        ['LBA_INNER'      , 'LBA_LOW' , '54..297' , 64, 200, 16, ['XC'], None],
        ['LBA_OUTER'      , 'LBA_LOW' , '54..297' , 64, 200, 16, ['XC'], None],
        ['LBA_INNER'      , 'LBA_HIGH', '156..399', 64, 200, 16, ['XC'], None],
        ['LBA_OUTER'      , 'LBA_HIGH', '156..399', 64, 200, 16, ['XC'], None],
        ['LBA_SPARSE_ODD' , 'LBA_HIGH', '156..399', 64, 200, 16, ['XC'], None],
        ['LBA_SPARSE_EVEN', 'LBA_HIGH', '156..399', 64, 200, 16, ['XC'], None],
        ['HBA_ZERO'       , 'HBA_LOW' , '77..320' , 64, 200, 16, ['XC'], None],
        ['HBA_ONE'        , 'HBA_LOW' , '77..320' , 64, 200, 16, ['XC'], None],
        ['HBA_DUAL'       , 'HBA_LOW' , '77..320' , 64, 200, 16, ['XC'], None],
        ['HBA_DUAL_INNER' , 'HBA_LOW' , '77..320' , 64, 200, 16, ['XC'], None],
        ['HBA_JOINED'     , 'HBA_LOW' , '77..320' , 64, 200, 16, ['XC'], None],
        ['HBA_ZERO'       , 'HBA_MID' , '66..309' , 64, 160, 16, ['XC'], None],
        ['HBA_ONE'        , 'HBA_MID' , '66..309' , 64, 160, 16, ['XC'], None],
        ['HBA_DUAL'       , 'HBA_MID' , '66..309' , 64, 160, 16, ['XC'], None],
        ['HBA_DUAL_INNER' , 'HBA_MID' , '66..309' , 64, 160, 16, ['XC'], None],
        ['HBA_JOINED'     , 'HBA_MID' , '66..309' , 64, 160, 16, ['XC'], None],
        ['HBA_ZERO'       , 'HBA_HIGH', '52..255' , 64, 200, 16, ['XC'], None],
        ['HBA_ONE'        , 'HBA_HIGH', '52..255' , 64, 200, 16, ['XC'], None],
        ['HBA_DUAL'       , 'HBA_HIGH', '52..255' , 64, 200, 16, ['XC'], None],
        ['HBA_DUAL_INNER' , 'HBA_HIGH', '52..255' , 64, 200, 16, ['XC'], None],
        ['HBA_JOINED'     , 'HBA_HIGH', '52..255' , 64, 200, 16, ['XC'], None]]
    # pylint: enable=bad-whitespace

    if job_description.from_file is not None:
        full_schedule = read_custom_observation_sequence(
            open(job_description.from_file).read())
        print('='*60)
        print('Using custom sequence (%s)' %  job_description.from_file)
        for observation in full_schedule:
            print(observation)
        print('='*60)
    else:
        print('using default sequence')
        for observation in full_schedule:
            print(observation)

    max_el_deg = job_description.max_alt
    min_el_deg = job_description.min_alt

    schedule = [obs for obs in full_schedule
                if obs[4] in job_description.clocks]
    now = ephem.Observer().date
    start_date = ephem.Date(now + ephem.second*job_description.wait)
    if job_description.start_date:
        start_date = ephem.Date(job_description.start_date)

    print('LST at start observation: %s' % str(lofar_sidereal_time(start_date)))

    mean_date = (start_date +
                 ephem.second*len(schedule)*(job_description.duration +
                                             job_description.gap)*0.5)

    mean_lst = lofar_sidereal_time(mean_date)
    print('Mean date: ' + repr(mean_date))
    print('Mean LST: '+str(mean_lst))
    print('Mean LST rad: %f' % float(mean_lst))

    include = job_description.include
    include = include and include.split(',')
    exclude = job_description.exclude
    exclude = exclude and exclude.split(',')
    stations = station_list(job_description.stations,
                            include=include,
                            exclude=exclude,)
    initial_status = 'opened'
    if job_description.approved:
        initial_status = 'approved'

    observations = []
    previous_clock = 200
    for (antenna_set,
         frequency_range,
         subband_spec,
         channels_per_subband,
         clock_mhz,
         bit_mode,
         data_products,
         pipeline) in schedule:
        station_set = copy.deepcopy(stations)
        target_source = job_description.source
        data_products = [data_product.upper()
                         for data_product in data_products
                         if data_product in job_description.mode]

        if not data_products:
            continue # No data products to write, no point to observe.
        if target_source is None:
            if (any([dprod in data_products
                     for dprod in ['FE', 'CS', 'CS_I', 'CS_IQUV', 'IS', 'IS_I', 'IS_IQUV', 'TR']])):
                target_source = SourceCatalogue().psr_source(
                    mean_date, antenna_set[0:3],
                    min_elevation_deg=min_el_deg,
                    max_elevation_deg=max_el_deg)
            else:
                if frequency_range[0:3] == 'LBA':
                    target_source = SourceCatalogue().cal_source(
                        mean_date, antenna_set[0:3])
                else:
                    target_source = SourceCatalogue().cal_source(
                        mean_date, antenna_set[0:3],
                        min_elevation_deg=min_el_deg,
                        max_elevation_deg=max_el_deg)
        else:
            target_source = lookup_source(target_source)

        print(','.join(data_products) + ' ' + antenna_set)
        print(target_source)
        # If clock switches, allow for two minutes extra
        if previous_clock != clock_mhz:
            start_date = ephem.Date(start_date + 2*ephem.minute)

        coherent_stokes_data = None
        incoherent_stokes_data = None
        tied_array_beams = None
        duration_seconds = job_description.duration
        if 'TR' in data_products:
            station_subset = set(station_list('superterp'))
            station_set = set(station_set).intersection(station_subset)
            coherent_stokes_data = Stokes('coherent',
                                          stokes_downsampling_steps=16) #3^{0,1}*2^{0..12}
            tied_array_beams = TiedArrayBeams(flyseye=False,
                                              beams_ra_dec_rad=None,
                                              nr_tab_rings=5,
                                              tab_ring_size=0.0042799)
            print('TAB ring test is experimental')
        if 'FE' in data_products:
            coherent_stokes_data = Stokes('coherent',
                                          stokes_downsampling_steps=128)
            tied_array_beams = TiedArrayBeams(flyseye=True,
                                              beams_ra_dec_rad=None)
            if antenna_set[0:3] == 'LBA':
                duration_seconds = 600.0
        if 'CS' in data_products or 'CS_I' in data_products:
            coherent_stokes_data = Stokes('coherent', polarizations='I',
                                          stokes_downsampling_steps=128)
            if tied_array_beams is None:
                tied_array_beams = TiedArrayBeams(flyseye=False,
                                                  beams_ra_dec_rad=[target_source.ra_dec_rad()])
            else:
                tied_array_beams.beams_ra_dec_rad = [target_source.ra_dec_rad()]
            core = set(station_list('core'))
            station_set = set(station_set).intersection(core)
        if 'CS_IQUV' in data_products:
            coherent_stokes_data = Stokes('coherent', polarizations='IQUV',
                                          stokes_downsampling_steps=128)
            if tied_array_beams is None:
                tied_array_beams = TiedArrayBeams(flyseye=False,
                                                  beams_ra_dec_rad=[target_source.ra_dec_rad()])
            else:
                tied_array_beams.beams_ra_dec_rad = [target_source.ra_dec_rad()]
            core = set(station_list('core'))
            station_set = set(station_set).intersection(core)
        if 'CV' in data_products:
            coherent_stokes_data = Stokes('coherent', polarizations='XXYY',
                                          number_collapsed_channels=1,
                                          stokes_downsampling_steps=1),
            if tied_array_beams is None:
                tied_array_beams = TiedArrayBeams(flyseye=False,
                                                  beams_ra_dec_rad=[target_source.ra_dec_rad()])
            else:
                tied_array_beams.beams_ra_dec_rad = [target_source.ra_dec_rad()]
            core = set(station_list('core'))
            station_set = set(station_set).intersection(core)
        if 'IS' in data_products or 'IS_I' in data_products:
            incoherent_stokes_data = Stokes('incoherent', polarizations='I',
                                            stokes_downsampling_steps=128)
        if 'IS_IQUV' in data_products:
            incoherent_stokes_data = Stokes('incoherent', polarizations='IQUV',
                                            stokes_downsampling_steps=128)

        good_stations = sorted(list(station_set))

        backend = BackendProcessing(
            integration_time_seconds=1,
            correlated_data=('XC' in data_products) or ('BM' in data_products),
            channels_per_subband=channels_per_subband,
            coherent_stokes_data=coherent_stokes_data,
            incoherent_stokes_data=incoherent_stokes_data,
            tied_array_beams=tied_array_beams)

        sb_list = sorted(parse_subband_list(subband_spec))
        subband_spec = ','.join([str(sb) for sb in sb_list[0:max(job_description.max_subbands, 1)]])
        multibeam_subband_spec = ','.join([str(sb) for sb in sb_list[0:max(int(job_description.max_subbands/25.0 + 0.5), 1)]])
        beam_list = [Beam(
            sap_id=0,
            target_source=target_source,
            subband_spec=subband_spec,
            storage_cluster=job_description.storage_cluster,
            storage_partition=job_description.storage_partition)]
        if 'BM' in data_products:
            max_sb = sorted(parse_subband_list(subband_spec))[-1]
            max_freq_hz = clock_mhz*1e6*(max_sb/1024.0)
            if 'HBA' in antenna_set:
                all_stations = ''.join(good_stations)
                if 'DE' in all_stations or \
                   'FR' in all_stations or \
                   'UK' in all_stations or \
                   'PL' in all_stations or \
                   'FI' in all_stations:
                    station_size_m = 11*5
                elif 'RS' in all_stations and 'INNER' not in antenna_set:
                    station_size_m = 8*5
                else:
                    station_size_m = 6*5
            if 'LBA_OUTER' in antenna_set or \
               'LBA_SPARSE' in antenna_set or \
               'LBA_X' in antenna_set or \
               'LBA_Y' in antenna_set:
                station_size_m = 80.0
            if 'LBA_INNER' in antenna_set:
                if 'DE' in all_stations or \
                   'FR' in all_stations or \
                   'UK' in all_stations or \
                   'PL' in all_stations or \
                   'FI' in all_stations:
                    station_size_m = 80.0
                else:
                    station_size_m = 30.0
            nyquist_spacing_rad = 0.5*299792458.0/(max_freq_hz*station_size_m)
            beam_id = 1
            for offset_m in range(-2, 3):
                for offset_l in range(-2, 3):
                    if not (offset_m == 0 and offset_l == 0):
                        ra, dec = radec_from_lm(offset_l*nyquist_spacing_rad/1.1,
                                                offset_m*nyquist_spacing_rad/1.1,
                                                target_source.ra_angle.as_rad(),
                                                target_source.dec_angle.as_rad())
                        aux_field = TargetSource('Aux-%03d' % beam_id,
                                                 ra_angle=ra, dec_angle=dec)
                        beam_list.append(Beam(sap_id=beam_id,
                                              target_source=aux_field,
                                              subband_spec=multibeam_subband_spec,
                                              storage_partition=job_description.storage_partition,
                                              storage_cluster=job_description.storage_cluster))
                        beam_id += 1
        obs = Observation(
            name=','.join(data_products)+str(channels_per_subband)+antenna_set,
            antenna_set=antenna_set,
            frequency_range=frequency_range,
            start_date=start_date.tuple(),
            duration_seconds=duration_seconds,
            stations=good_stations,
            clock_mhz=clock_mhz,
            backend=backend,
            bit_mode=bit_mode,
            beam_list=beam_list,
            initial_status=initial_status)
        observations.append(obs)
        if pipeline is not None:
            if pipeline[0] == 'Preprocessing':
                avg_freq_step, avg_time_step = pipeline[1:3]
                demix_freq_step, demix_time_step = pipeline[3:5]
                demix_always = None
                demix_if_needed = None
                if len(pipeline) >= 6:
                    demix_always = pipeline[5].split()
                if len(pipeline) >= 7:
                    demix_if_needed = pipeline[6].split()
                beams = [child for child in obs.children if isinstance(child, Beam)]
                for beam in beams:
                    if 'cyg' in beam.target_source.name.lower():
                        if demix_always is not None:
                            demix_always = [source for source in demix_always
                                            if 'cyg' not in  source]
                        if demix_if_needed is not None:
                            demix_if_needed = [
                                source for source in demix_if_needed
                                if 'cyg' not in  source]
                    pl = AveragingPipeline(
                        name='Preproc.'+antenna_set,
                        ndppp=NDPPP(avg_freq_step, avg_time_step,
                                    demix_freq_step, demix_time_step,
                                    demix_always, demix_if_needed),
                        input_data=[beam],
                        duration_s=duration_seconds*2,
                        start_date=ephem.Date(start_date\
                                              + duration_seconds*ephem.second\
                                              + 5*ephem.minute).tuple(),
                        initial_status=initial_status,
                        processing_cluster=job_description.storage_cluster,
                        processing_partition=job_description.storage_partition,
                        processing_nr_tasks=int((len(parse_subband_list(beam.subband_spec))/3)+1))
                    observations[-1].append_child(pl)
        start_date = ephem.Date(start_date
                                + ephem.second*duration_seconds
                                + ephem.second*job_description.gap)
        previous_clock = clock_mhz
    return observations






def main():
    r'''
    Main function. Connects command line parsing to actual observing
    sequence, creates a set of folders, and produces the XML file.
    '''
    print(parse_arguments())
    job_description = parse_arguments()

    observations = plan_observing_sequence(job_description)
    folder_date = observations[0].start_date

    sub_folder = Folder(
        name='%04d-%02d-%02d %02d:%02d' % folder_date[:-1],
        description=('Validation observations %04d-%02d-%02d %02d:%02d' %
                     folder_date[:-1]),
        children=observations,
        grouping_parent=True)
    val_obs_folder = Folder(name='Validation Obs',
                            children=[sub_folder],
                            update_folder=True)
    out = open(job_description.output, 'w')
    out.write(xml([val_obs_folder],
                  project=job_description.project))
    out.close()

    return 0





if __name__ == '__main__':
    try:
        print('Current LST: '+str(lofar_sidereal_time(ephem.Observer().date)))
        exit(main())
    except (OSError, IOError, SourceSpecificationError,
            InvalidStationSetError, NoSuitableSourceError):
        prerror(sys.exc_info()[1].args[0])
        print_short_help()
        exit(-1)
