<commit_before>"""
Functions to interrogate and extract buoy data from the buoys.db SQLite3
database.

"""

from __future__ import print_function

from datetime import datetime
from pathlib import Path
from warnings import warn

import numpy as np

try:
    import sqlite3
    use_sqlite = True
except ImportError:
    warn('No sqlite standard library found in this python '
         'installation. Some functions will be disabled.')
    use_sqlite = False

def _split_lines(line, remove_empty=False, remove_trailing=False):
    """
    Quick function to tidy up lines in an ASCII file (split on a given separator (default space)).

    Parameters
    ----------
    line : str
        String to split.
    remove_empty : bool, optional
        Set to True to remove empty columns. Defaults to leaving them in.
    remove_trailing : bool, optional
        Set to True to remove trailing empty columns. Defaults to leaving them in.

    Returns
    -------
    y : list
        The split string.

    """

    delimiters = (';', ',', '\t', ' ')
    delimiter = None
    for d in delimiters:
        if d in line:
            delimiter = d
            break

    # Clear out newlines.
    line = line.strip('\n')

    if remove_trailing:
        line = delimiter.join(line.rstrip(delimiter).split(delimiter))

    y = line.split(delimiter)

    if remove_empty:
        y = [i.strip() for i in line.split(delimiter) if i]

    return y

def get_buoy_metadata(db):
    """
    Extracts the meta data from the buoy database.

    Parameters
    ----------
    db : str
        Full path to the buoy data SQLite database.

    Returns
    -------
    meta_info : list
        List of dicts with keys based on the field names from the Stations
        table. Returns [False] if there is an error.

    """

    if not use_sqlite:
        raise RuntimeError('No sqlite standard library found in this python '
                           'installation. This function (get_buoy_metadata) '
                           'is unavailable.')

    def _dict_factory(cursor, row):
        d = {}
        for idx, col in enumerate(cursor.description):
            d[col[0]] = row[idx]
        return d

    try:
        con = sqlite3.connect(db)

        con.row_factory = _dict_factory

        c = con.cursor()

        out = c.execute('SELECT * from Stations')

        meta_info = out.fetchall()

    except sqlite3.Error as e:
        if con:
            con.close()
            print('Error %s:' % e.args[0])
            meta_info = [False]

    finally:
        if con:
            con.close()

    return meta_info

def get_buoy_data(db, table, fields, noisy=False):
    """
    Extract the buoy from the SQLite database for a given site.  Specify the
    database (db), the table name (table) of the station of interest.

    Parameters
    ----------
    db : str
        Full path to the buoy data SQLite database.
    table : str
        Name of the table to be extracted (e.g. 'hastings_wavenet_site').
    fields : list
        List of names of fields to extract for the given table, such as
        ['Depth', 'Temperature']. Where no data exists, a column of NaNs will
        be returned (actually Nones, but numpy does the conversion for you).
    noisy : bool, optional
        Set to True to enable verbose output.

    Returns
    -------
    data : ndarray
        Array of the fields requested from the table specified.

    See Also
    --------
    buoy.get_observed_metadata : extract metadata for a buoy time series.

    Notes
    -----
    Search is case insensitive (b0737327 is equal to B0737327).

    """

    if not use_sqlite:
        raise RuntimeError('No sqlite standard library found in this python '
                           'installation. This function (get_buoy_data) is '
                           'unavailable.')

    if noisy:
        print('Getting data for {} from the database...'.format(table),
              end=' ')

    try:
        con = sqlite3.connect(db)

        with con:
            c = con.cursor()
            # I know, using a string is Bad. But it works and it's only me
            # working with this.
            c.execute('SELECT {} FROM {}'.format(','.join(fields), table))

            # Now get the data in a format we might actually want to use
            data = np.asarray(c.fetchall())

        if noisy:
            print('done.')

    except sqlite3.Error as e:
        if con:
            con.close()
            print('Error %s:' % e.args[0])
            data = np.asarray([False])

    finally:
        if con:
            con.close()

    return data.astype(float)

class Buoy:
    """ Generic class for buoy data (i.e. surface time series). """

    def __init__(self, filename, position=None, station=None, missing_value=None, noisy=False):
        """
        Create a buoy object from the given file name.

        Parameters
        ----------
        filename : str, pathlib.Path
            The file name to read in.
        position : tuple, list, optional
            Longitude/latitude of the current buoy. If omitted, position is np.nan/np.nan.
        station : str, optional
            The name of the current buoy. If omitted, set to None.
        missing_value : float
            Supply a value which is the missing value for this buoy time series.
        noisy : bool, optional
            If True, verbose output is printed to screen. Defaults to False.

        """

        self._file = Path(filename)

        self._debug = False
        self._noisy = noisy
        self._locations = position
        self._site = station
        self._missing = missing_value
        self._time_header = ['Year', 'Serial', 'Jd', 'Time', 'Time_GMT', 'Date_YYMMDD', 'Time_HHMMSS', 'Date/Time_GMT']
        self.data = None
        self.position = None
        self.time = None

        # Get the metadata read in.
        self._slurp_file()
        self.header, self.header_length, self.header_indices = _read_header(self._lines, self._time_header)

    def _slurp_file(self):
        """
        Read in the contents of the file into self so we don't have to read each file multiple times.

        Provides
        --------
        self._lines : list
            The lines in the file, stripped of newlines and leading/trailing whitespace and split based on a trying a
            few common delimiters.

        """

        extension = self._file.suffix
        # Ignore crappy characters by forcing everything to ASCII.
        with self._file.open('r', encoding='ascii', errors='ignore') as f:
            empty = False
            trailing = False
            if extension == '.csv':
                # Probably CEFAS data which has empty columns, so we need to leave them in place. However, we need to
                # remove the trailing empty columns as the headers don't account for those.
                trailing = True
            elif extension == '.txt':
                # Probably WCO data, which is usually space separated, so nuke duplicate spaces.
                empty = True
            elif 'L4' in str(self._file):
                # The other WCO data format.
                empty = False
                trailing = False
            elif 'E1' in str(self._file):
                # The other WCO data format.
                empty = False
                trailing = False

            self._lines = f.readlines()
            self._lines = [_split_lines(i, remove_empty=empty, remove_trailing=trailing) for i in self._lines]

            # If we've left empty columns in, replace them with NaNs. This is not elegant.
            if not empty:
                new_lines = []
                for line in self._lines:
                    new_lines.append([np.nan if i == '' else i for i in line])
                self._lines = new_lines

    def load(self):
        """
        Parse the header and extract the data for the loaded file.

        Provides
        --------
        Adds data and time objects, with the variables and time loaded respectively. The time object also has a
        datetime attribute.

        """

        # Add times.
        self.time = self._ReadTime(self._lines)

        if not any(self.time.datetime):
            return

        # Add positions
        self.position = self._ReadPosition(self._locations, self._site)

        # Grab the data.
        self.data = self._ReadData(self._lines)

        # Replace missing values with NaNs.
        if self._missing is not None:
            for name in dir(self.data):
                if not name.startswith('_'):
                    values = getattr(self.data, name)
                    values[values == self._missing] = np.nan
                    setattr(self.data, name, values)

    class _Read(object):
        def __init__(self, lines, noisy=False):
            """
            Initialise parsing the buoy time series data so we can subclass this for the header and data reading.

            Parameters
            ----------
            lines : list
                The data to parse, read in by Buoys._slurp.
            noisy : bool, optional
                If True, verbose output is printed to screen. Defaults to False.

            Provides
            --------
            Attributes in self which are named for each variable found in `lines'. Each attribute contains a single
            time series as a numpy array.

            """

            self._debug = False
            self._noisy = noisy
            self._lines = lines
            self._time_header = ['Year', 'Serial', 'Jd', 'Time', 'Time_GMT', 'Date_YYMMDD', 'Time_HHMMSS', 'Date/Time_GMT']

            self._header, self._header_length, self._header_indices = _read_header(self._lines, self._time_header)
            self._read()

    class _ReadData(_Read):
        """ Read time series data from a given WCO file. This is meant to be called by the Buoy class. """

        def _read(self):
            """
            Parse the data in self._lines for each of the time series.

            Provides
            --------
            Attributes in self which are named for each variable found in `self._lines'. Each attribute contains a single
            time series as a numpy array.

            """

            # We want everything bar the time column names.
            num_lines = len(self._lines) - self._header_length
            num_columns = len(self._header)
            if num_lines > 1:
                for name in self._header:
                    if name not in self._time_header:
                        name_index = self._header_indices[name]
                        data = []
                        for line in self._lines[self._header_length:]:
                            # Only keep values where we've got as many columns as headers.
                            if len(line) == num_columns:
                                data.append(line[name_index])
                        try:
                            setattr(self, name.strip().replace(' ', '_').replace('(', '').replace(')', ''), np.asarray(data, dtype=float))
                        except ValueError:
                            # Probably strings so just leave as is.
                            setattr(self, name.strip().replace(' ', '_').replace('(', '').replace(')', ''), np.asarray(data))

    class _ReadTime(_Read):
        """ Extract the time from the given WCO file. This is meant to be called by the Buoy class. """

        def _read(self):
            """
            Parse the data in self._lines for each of the time series.

            Provides
            --------
            Attributes in self which are named for each variable found in `self._lines'. Each attribute contains some
            time data. We also create a datetime attribute which has the times as datetime objects.

            """

            # Try everything in self._time_header values.
            self.time_header = []
            num_lines = len(self._lines) - self._header_length
            num_columns = len(self._header)
            if num_lines > 1:
                for name in self._header:
                    if name in self._time_header:
                        self.time_header.append(name)
                        name_index = self._header_indices[name]
                        data = []
                        for line in self._lines[self._header_length:]:
                            # Only keep values where we've got as many columns as headers.
                            if len(line) == num_columns:
                                data.append(line[name_index])
                        setattr(self, name, np.asarray(data))

            # Now make datetime objects from the time.
            self.datetime = []
            if hasattr(self, 'Year') and hasattr(self, 'Serial') and hasattr(self, 'Time'):
                # First Western Channel Observatory format.
                for year, doy, time in zip(self.Year, self.Serial, self.Time):
                    self.datetime.append(datetime.strptime('{y}{doy} {hm}'.format(y=year, doy=doy, hm=time), '%Y%j %H.%M'))
            elif hasattr(self, 'Year') and hasattr(self, 'Jd') and hasattr(self, 'Time'):
                # Different Western Channel Observatory format.
                for year, doy, time in zip(self.Year, self.Jd, self.Time):
                    try:
                        self.datetime.append(datetime.strptime('{y}{doy} {hm}'.format(y=year, doy=doy, hm=time), '%Y%j %H.%M'))
                    except ValueError:
                        self.datetime.append(np.nan)
            elif hasattr(self, 'Date_YYMMDD') and hasattr(self, 'Time_HHMMSS'):
                # Another different Western Channel Observatory format.
                for date, time in zip(getattr(self, 'Date_YYMMDD'), getattr(self, 'Time_HHMMSS')):
                    if date == 'nan' and time == 'nan':
                        self.datetime.append(np.nan)
                    else:
                        self.datetime.append(datetime.strptime('{date} {time}'.format(date=date, time=time), '%y%m%d %H%M%S'))
            elif hasattr(self, 'Time_GMT'):
                # CEFAS format.
                for date in getattr(self, 'Time_GMT'):
                    self.datetime.append(datetime.strptime(date, '%Y-%m-%d %H:%M:%S'))
            elif hasattr(self, 'Date/Time_GMT'):
                # CCO format.
                for date in getattr(self, 'Date/Time_GMT'):
                    self.datetime.append(datetime.strptime(date, '%d-%b-%Y %H:%M:%S'))

    class _ReadPosition:
        """ Add the position for the buoy. """

        def __init__(self, location, site):
            """
            Grab the data for the given buoy site.

            Parameters
            ----------
            location : tuple, list
                The longitude/latitude of the buoy.
            site : str
                The name of the site we're working on.

            Provides
            --------
            lon, lat : float
                The longitude and latitude of the site.

            """

            self.lon, self.lat = location
            self.name = site

def _read_header(lines, header_names):
    """
    Extract the header columns. Accounts for duplicates.

    Parameters
    ----------
    lines : list
        List of the lines from the file.
    header_names : list
        Header time variable names to search for which define the header.

    Returns
    -------
    header : list
        List of the header names.
    header_length : int
        Number of lines in the header.
    header_indices : dict
        Indices of each header name in `header' with the name as the key.

    """
    header_length = 0
    for count, line in enumerate(lines):
        if any(time in line for time in header_names):
            header_length = count
        else:
            break

    header_length += 1

    header = lines[header_length - 1]

    header_indices = {i: header.index(i) for i in header}

    return header, header_length, header_indices
<commit_msg>Remove annoying characters from variable names.<commit_after>"""
Functions to interrogate and extract buoy data from the buoys.db SQLite3
database.

"""

from __future__ import print_function

from datetime import datetime
from pathlib import Path
from warnings import warn

import numpy as np

try:
    import sqlite3
    use_sqlite = True
except ImportError:
    warn('No sqlite standard library found in this python '
         'installation. Some functions will be disabled.')
    use_sqlite = False

def _split_lines(line, remove_empty=False, remove_trailing=False):
    """
    Quick function to tidy up lines in an ASCII file (split on a given separator (default space)).

    Parameters
    ----------
    line : str
        String to split.
    remove_empty : bool, optional
        Set to True to remove empty columns. Defaults to leaving them in.
    remove_trailing : bool, optional
        Set to True to remove trailing empty columns. Defaults to leaving them in.

    Returns
    -------
    y : list
        The split string.

    """

    delimiters = (';', ',', '\t', ' ')
    delimiter = None
    for d in delimiters:
        if d in line:
            delimiter = d
            break

    # Clear out newlines.
    line = line.strip('\n')

    if remove_trailing:
        line = delimiter.join(line.rstrip(delimiter).split(delimiter))

    y = line.split(delimiter)

    if remove_empty:
        y = [i.strip() for i in line.split(delimiter) if i]

    return y

def get_buoy_metadata(db):
    """
    Extracts the meta data from the buoy database.

    Parameters
    ----------
    db : str
        Full path to the buoy data SQLite database.

    Returns
    -------
    meta_info : list
        List of dicts with keys based on the field names from the Stations
        table. Returns [False] if there is an error.

    """

    if not use_sqlite:
        raise RuntimeError('No sqlite standard library found in this python '
                           'installation. This function (get_buoy_metadata) '
                           'is unavailable.')

    def _dict_factory(cursor, row):
        d = {}
        for idx, col in enumerate(cursor.description):
            d[col[0]] = row[idx]
        return d

    try:
        con = sqlite3.connect(db)

        con.row_factory = _dict_factory

        c = con.cursor()

        out = c.execute('SELECT * from Stations')

        meta_info = out.fetchall()

    except sqlite3.Error as e:
        if con:
            con.close()
            print('Error %s:' % e.args[0])
            meta_info = [False]

    finally:
        if con:
            con.close()

    return meta_info

def get_buoy_data(db, table, fields, noisy=False):
    """
    Extract the buoy from the SQLite database for a given site.  Specify the
    database (db), the table name (table) of the station of interest.

    Parameters
    ----------
    db : str
        Full path to the buoy data SQLite database.
    table : str
        Name of the table to be extracted (e.g. 'hastings_wavenet_site').
    fields : list
        List of names of fields to extract for the given table, such as
        ['Depth', 'Temperature']. Where no data exists, a column of NaNs will
        be returned (actually Nones, but numpy does the conversion for you).
    noisy : bool, optional
        Set to True to enable verbose output.

    Returns
    -------
    data : ndarray
        Array of the fields requested from the table specified.

    See Also
    --------
    buoy.get_observed_metadata : extract metadata for a buoy time series.

    Notes
    -----
    Search is case insensitive (b0737327 is equal to B0737327).

    """

    if not use_sqlite:
        raise RuntimeError('No sqlite standard library found in this python '
                           'installation. This function (get_buoy_data) is '
                           'unavailable.')

    if noisy:
        print('Getting data for {} from the database...'.format(table),
              end=' ')

    try:
        con = sqlite3.connect(db)

        with con:
            c = con.cursor()
            # I know, using a string is Bad. But it works and it's only me
            # working with this.
            c.execute('SELECT {} FROM {}'.format(','.join(fields), table))

            # Now get the data in a format we might actually want to use
            data = np.asarray(c.fetchall())

        if noisy:
            print('done.')

    except sqlite3.Error as e:
        if con:
            con.close()
            print('Error %s:' % e.args[0])
            data = np.asarray([False])

    finally:
        if con:
            con.close()

    return data.astype(float)

class Buoy:
    """ Generic class for buoy data (i.e. surface time series). """

    def __init__(self, filename, position=None, station=None, missing_value=None, noisy=False):
        """
        Create a buoy object from the given file name.

        Parameters
        ----------
        filename : str, pathlib.Path
            The file name to read in.
        position : tuple, list, optional
            Longitude/latitude of the current buoy. If omitted, position is np.nan/np.nan.
        station : str, optional
            The name of the current buoy. If omitted, set to None.
        missing_value : float
            Supply a value which is the missing value for this buoy time series.
        noisy : bool, optional
            If True, verbose output is printed to screen. Defaults to False.

        """

        self._file = Path(filename)

        self._debug = False
        self._noisy = noisy
        self._locations = position
        self._site = station
        self._missing = missing_value
        self._time_header = ['Year', 'Serial', 'Jd', 'Time', 'Time_GMT', 'Date_YYMMDD', 'Time_HHMMSS', 'Date/Time_GMT']
        self.data = None
        self.position = None
        self.time = None

        # Get the metadata read in.
        self._slurp_file()
        self.header, self.header_length, self.header_indices = _read_header(self._lines, self._time_header)

    def _slurp_file(self):
        """
        Read in the contents of the file into self so we don't have to read each file multiple times.

        Provides
        --------
        self._lines : list
            The lines in the file, stripped of newlines and leading/trailing whitespace and split based on a trying a
            few common delimiters.

        """

        extension = self._file.suffix
        # Ignore crappy characters by forcing everything to ASCII.
        with self._file.open('r', encoding='ascii', errors='ignore') as f:
            empty = False
            trailing = False
            if extension == '.csv':
                # Probably CEFAS data which has empty columns, so we need to leave them in place. However, we need to
                # remove the trailing empty columns as the headers don't account for those.
                trailing = True
            elif extension == '.txt':
                # Probably WCO data, which is usually space separated, so nuke duplicate spaces.
                empty = True
            elif 'L4' in str(self._file):
                # The other WCO data format.
                empty = False
                trailing = False
            elif 'E1' in str(self._file):
                # The other WCO data format.
                empty = False
                trailing = False

            self._lines = f.readlines()
            self._lines = [_split_lines(i, remove_empty=empty, remove_trailing=trailing) for i in self._lines]

            # If we've left empty columns in, replace them with NaNs. This is not elegant.
            if not empty:
                new_lines = []
                for line in self._lines:
                    new_lines.append([np.nan if i == '' else i for i in line])
                self._lines = new_lines

    def load(self):
        """
        Parse the header and extract the data for the loaded file.

        Provides
        --------
        Adds data and time objects, with the variables and time loaded respectively. The time object also has a
        datetime attribute.

        """

        # Add times.
        self.time = self._ReadTime(self._lines)

        if not any(self.time.datetime):
            return

        # Add positions
        self.position = self._ReadPosition(self._locations, self._site)

        # Grab the data.
        self.data = self._ReadData(self._lines)

        # Replace missing values with NaNs.
        if self._missing is not None:
            for name in dir(self.data):
                if not name.startswith('_'):
                    values = getattr(self.data, name)
                    values[values == self._missing] = np.nan
                    setattr(self.data, name, values)

    class _Read(object):
        def __init__(self, lines, noisy=False):
            """
            Initialise parsing the buoy time series data so we can subclass this for the header and data reading.

            Parameters
            ----------
            lines : list
                The data to parse, read in by Buoys._slurp.
            noisy : bool, optional
                If True, verbose output is printed to screen. Defaults to False.

            Provides
            --------
            Attributes in self which are named for each variable found in `lines'. Each attribute contains a single
            time series as a numpy array.

            """

            self._debug = False
            self._noisy = noisy
            self._lines = lines
            self._time_header = ['Year', 'Serial', 'Jd', 'Time', 'Time_GMT', 'Date_YYMMDD', 'Time_HHMMSS', 'Date/Time_GMT']

            self._header, self._header_length, self._header_indices = _read_header(self._lines, self._time_header)
            self._read()

    class _ReadData(_Read):
        """ Read time series data from a given WCO file. This is meant to be called by the Buoy class. """

        def _read(self):
            """
            Parse the data in self._lines for each of the time series.

            Provides
            --------
            Attributes in self which are named for each variable found in `self._lines'. Each attribute contains a single
            time series as a numpy array.

            """

            # We want everything bar the time column names.
            num_lines = len(self._lines) - self._header_length
            num_columns = len(self._header)
            if num_lines > 1:
                for name in self._header:
                    if name not in self._time_header:
                        name_index = self._header_indices[name]
                        data = []
                        for line in self._lines[self._header_length:]:
                            # Only keep values where we've got as many columns as headers.
                            if len(line) == num_columns:
                                data.append(line[name_index])
                        try:
                            setattr(self, name.strip().replace(' ', '_').replace('(', '').replace(')', ''), np.asarray(data, dtype=float))
                        except ValueError:
                            # Probably strings so just leave as is.
                            setattr(self, name.strip().replace(' ', '_').replace('(', '').replace(')', ''), np.asarray(data))

    class _ReadTime(_Read):
        """ Extract the time from the given WCO file. This is meant to be called by the Buoy class. """

        def _read(self):
            """
            Parse the data in self._lines for each of the time series.

            Provides
            --------
            Attributes in self which are named for each variable found in `self._lines'. Each attribute contains some
            time data. We also create a datetime attribute which has the times as datetime objects.

            """

            # Try everything in self._time_header values.
            self.time_header = []
            num_lines = len(self._lines) - self._header_length
            num_columns = len(self._header)
            if num_lines > 1:
                for name in self._header:
                    if name in self._time_header:
                        self.time_header.append(name)
                        name_index = self._header_indices[name]
                        data = []
                        for line in self._lines[self._header_length:]:
                            # Only keep values where we've got as many columns as headers.
                            if len(line) == num_columns:
                                data.append(line[name_index])
                        setattr(self, name.strip().replace(' ', '_').replace('(', '').replace(')', ''), np.asarray(data))

            # Now make datetime objects from the time.
            self.datetime = []
            if hasattr(self, 'Year') and hasattr(self, 'Serial') and hasattr(self, 'Time'):
                # First Western Channel Observatory format.
                for year, doy, time in zip(self.Year, self.Serial, self.Time):
                    self.datetime.append(datetime.strptime('{y}{doy} {hm}'.format(y=year, doy=doy, hm=time), '%Y%j %H.%M'))
            elif hasattr(self, 'Year') and hasattr(self, 'Jd') and hasattr(self, 'Time'):
                # Different Western Channel Observatory format.
                for year, doy, time in zip(self.Year, self.Jd, self.Time):
                    try:
                        self.datetime.append(datetime.strptime('{y}{doy} {hm}'.format(y=year, doy=doy, hm=time), '%Y%j %H.%M'))
                    except ValueError:
                        self.datetime.append(np.nan)
            elif hasattr(self, 'Date_YYMMDD') and hasattr(self, 'Time_HHMMSS'):
                # Another different Western Channel Observatory format.
                for date, time in zip(getattr(self, 'Date_YYMMDD'), getattr(self, 'Time_HHMMSS')):
                    if date == 'nan' and time == 'nan':
                        self.datetime.append(np.nan)
                    else:
                        self.datetime.append(datetime.strptime('{date} {time}'.format(date=date, time=time), '%y%m%d %H%M%S'))
            elif hasattr(self, 'Time_GMT'):
                # CEFAS format.
                for date in getattr(self, 'Time_GMT'):
                    self.datetime.append(datetime.strptime(date, '%Y-%m-%d %H:%M:%S'))
            elif hasattr(self, 'Date/Time_GMT'):
                # CCO format.
                for date in getattr(self, 'Date/Time_GMT'):
                    self.datetime.append(datetime.strptime(date, '%d-%b-%Y %H:%M:%S'))

    class _ReadPosition:
        """ Add the position for the buoy. """

        def __init__(self, location, site):
            """
            Grab the data for the given buoy site.

            Parameters
            ----------
            location : tuple, list
                The longitude/latitude of the buoy.
            site : str
                The name of the site we're working on.

            Provides
            --------
            lon, lat : float
                The longitude and latitude of the site.

            """

            self.lon, self.lat = location
            self.name = site

def _read_header(lines, header_names):
    """
    Extract the header columns. Accounts for duplicates.

    Parameters
    ----------
    lines : list
        List of the lines from the file.
    header_names : list
        Header time variable names to search for which define the header.

    Returns
    -------
    header : list
        List of the header names.
    header_length : int
        Number of lines in the header.
    header_indices : dict
        Indices of each header name in `header' with the name as the key.

    """
    header_length = 0
    for count, line in enumerate(lines):
        if any(time in line for time in header_names):
            header_length = count
        else:
            break

    header_length += 1

    # Remove annoying characters from header names.
    header = [i.strip().replace(' ', '_').replace('(', '').replace(')', '') for i in lines[header_length - 1]]

    header_indices = {i: header.index(i) for i in header}

    return header, header_length, header_indices
<|endoftext|>


<commit_before>import types
import textwrap
import inspect
import sys
import importlib
import simplejson as json
import time
import cmd
from bson.json_util import dumps
from cmd3.shell import command
from cloudmesh.user.cm_user import cm_user
from cloudmesh.cm_mongo import cm_mongo
from cloudmesh.config.cm_config import cm_config
from pprint import pprint
from prettytable import PrettyTable
from cloudmesh.util.logger import LOGGER
import docopt

log = LOGGER(__file__)

class cm_shell_defaults:

    def createDefaultDict(self, cloudName):
        defaultDict = {}
        config = cm_config()
        pprint(config.cloud(cloudName))

    def activate_cm_shell_defaults(self):
        try:
            config = cm_config()
            self.user = config.username()
            self.mongoClass = cm_mongo()
            self.mongoClass.activate(cm_user_id=self.user)
        except Exception, e:
            print e
            print "Please check if mongo service is running."
            sys.exit()
	@command
    def do_defaults(self, args, arguments):
        """
        Usage:
               defaults [-v] clean
               defaults [-v] load CLOUD
               defaults [options] info
               defaults list [options] CLOUD

        Manages the defaults

        Arguments:

          NAME           The name of a service or server
          N              The number of defaultss to be started
          CLOUD          The name of Cloud

        Options:

           -v             verbose mode
           -j --json      json output

        """
        if arguments["clean"]:
            log.info ("clean the vm")
            print arguments['-v']
            return

        if arguments["load"] and arguments["CLOUD"]:
            createDefaultDict(arguments["CLOUD"])
            return
<commit_msg>slight format improvement adding print arguments for debugging<commit_after>import types
import textwrap
import inspect
import sys
import importlib
import simplejson as json
import time
import cmd
from bson.json_util import dumps
from cmd3.shell import command
from cloudmesh.user.cm_user import cm_user
from cloudmesh.cm_mongo import cm_mongo
from cloudmesh.config.cm_config import cm_config
from pprint import pprint
from prettytable import PrettyTable
from cloudmesh.util.logger import LOGGER
import docopt

log = LOGGER(__file__)

class cm_shell_defaults:

    def createDefaultDict(self, cloudName):
        defaultDict = {}
        config = cm_config()
        pprint(config.cloud(cloudName))

    def activate_cm_shell_defaults(self):
        try:
            config = cm_config()
            self.user = config.username()
            self.mongoClass = cm_mongo()
            self.mongoClass.activate(cm_user_id=self.user)
        except Exception, e:
            print e
            print "Please check if mongo service is running."
            sys.exit()

    @command
    def do_defaults(self, args, arguments):
        """
        Usage:
               defaults [-v] clean
               defaults [-v] load CLOUD
               defaults [options] info
               defaults list [options] CLOUD

        Manages the defaults

        Arguments:

          NAME           The name of a service or server
          N              The number of defaultss to be started
          CLOUD          The name of Cloud

        Options:

           -v             verbose mode
           -j --json      json output

        """

        print arguments

        if arguments["clean"]:
            log.info ("clean the vm")
            print arguments['-v']
            return

        if arguments["load"] and arguments["CLOUD"]:
            createDefaultDict(arguments["CLOUD"])
            return
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function

# based on https://gist.github.com/jasoncpatton/e51debc6cd579e9ecb219afc72a78e67

import dask.dataframe as dd
import pandas as pd
import logging
import time

logging.basicConfig(level=10)

from distributed import Client
from dask_condor import HTCondorCluster

sites = ['MSN', 'MKE', 'GRB', 'EAU', 'LSE', 'LNR', 'VOK', 'JVL', 'MTW', 'CWA', 'OSH', 'RHI', 'CMY', 'AUW']
#sites = ['MSN']
year_start = 2007
year_end = 2016

cluster = HTCondorCluster(worker_tarball='dask_condor_worker_base.SL6.tar.gz')
client = Client(cluster)
cluster.start_workers(n=len(sites))

csvfile = "{0}-{1}-to-{2}.csv"
ddfs = {}
for site in sites:
    print("Scheduling site: " + site)
    # dask doesn't support the 'index' parameter so load in pandas instead
    pdf = pd.read_csv(
        #url.format(site, year_start, year_end),
        csvfile.format(site, year_start, year_end),
        index_col = 1, # index on timestamp
        skipinitialspace = True, comment = '#', na_values = ['M'],
        parse_dates = True, infer_datetime_format = True,
    )
    ddf = dd.from_pandas(pdf, chunksize=128)
    ddf = ddf.resample('1H', label='right').last()
    ddfs[site] = ddf

for site, ddf in ddfs.items():
    print("Calculating site {0} with cluster {1!s}:".format(site, cluster))
    st = time.time()
    # compute the whole data frame (I think head might only compute the first n lines?)
    ddf.compute()
    print(ddf.head())
    print("{0:.1f}s".format(time.time() - st))
    print()
<commit_msg>Use Adaptive in weather test<commit_after>#!/usr/bin/env python
from __future__ import print_function

# based on https://gist.github.com/jasoncpatton/e51debc6cd579e9ecb219afc72a78e67

import dask.dataframe as dd
import pandas as pd
import logging
import time

logging.basicConfig(level=10)

from distributed import Client
from dask_condor import HTCondorCluster
from distributed.deploy.adaptive import Adaptive

sites = ['MSN', 'MKE', 'GRB', 'EAU', 'LSE', 'LNR', 'VOK', 'JVL', 'MTW', 'CWA', 'OSH', 'RHI', 'CMY', 'AUW']
#sites = ['MSN']
year_start = 2007
year_end = 2016

cluster = HTCondorCluster(worker_tarball='dask_condor_worker_base.SL6.tar.gz')
client = Client(cluster)
adaptive = Adaptive(cluster.scheduler, cluster, interval=20000)

csvfile = "{0}-{1}-to-{2}.csv"
ddfs = {}
for site in sites:
    print("Scheduling site: " + site)
    # dask doesn't support the 'index' parameter so load in pandas instead
    pdf = pd.read_csv(
        #url.format(site, year_start, year_end),
        csvfile.format(site, year_start, year_end),
        index_col = 1, # index on timestamp
        skipinitialspace = True, comment = '#', na_values = ['M'],
        parse_dates = True, infer_datetime_format = True,
    )
    ddf = dd.from_pandas(pdf, chunksize=128)
    ddf = ddf.resample('1H', label='right').last()
    ddfs[site] = ddf

for site, ddf in ddfs.items():
    print("Calculating site {0} with cluster {1!s}:".format(site, cluster))
    st = time.time()
    # compute the whole data frame (I think head might only compute the first n lines?)
    ddf.compute()
    print(ddf.head())
    print("{0:.1f}s".format(time.time() - st))
    print()
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-

"""
Take spectrum library, and list some basic stellar parameters and photometry for the spectra within.
"""

from os import path as os_path
import argparse
import re
import numpy as np

from fourgp_speclib import SpectrumLibrarySqlite

# Read input parameters
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--library', required=True, dest='library',
                    help="Spectrum library we should export, with additional constraints if wanted.")
parser.add_argument('--workspace', dest='workspace', default="",
                    help="Directory where we expect to find spectrum libraries.")
args = parser.parse_args()

# Helper for opening input SpectrumLibrary(s)
def open_input_libraries(library_spec, extra_constraints):
    test = re.match("([^\[]*)\[(.*)\]$", library_spec)
    constraints = {}
    if test is None:
        library_name = library_spec
    else:
        library_name = test.group(1)
        for constraint in test.group(2).split(","):
            words_1 = constraint.split("=")
            words_2 = constraint.split("<")
            if len(words_1) == 2:
                constraint_name = words_1[0]
                try:
                    constraint_value = float(words_1[1])
                except ValueError:
                    constraint_value = words_1[1]
                constraints[constraint_name] = constraint_value
            elif len(words_2) == 3:
                constraint_name = words_2[1]
                try:
                    constraint_value_a = float(words_2[0])
                    constraint_value_b = float(words_2[2])
                except ValueError:
                    constraint_value_a = words_2[0]
                    constraint_value_b = words_2[2]
                constraints[constraint_name] = (constraint_value_a, constraint_value_b)
            else:
                assert False, "Could not parse constraint <{}>".format(constraint)
    constraints.update(extra_constraints)
    constraints["continuum_normalised"] = 0  # All input spectra must be continuum normalised
    library_path = os_path.join(workspace, library_name)
    input_library = SpectrumLibrarySqlite(path=library_path, create=False)
    library_items = input_library.search(**constraints)
    return {
        "library": input_library,
        "items": library_items
    }

# Set path to workspace where we expect to find libraries of spectra
our_path = os_path.split(os_path.abspath(__file__))[0]
workspace = args.workspace if args.workspace else os_path.join(our_path, "..", "workspace")

# Open spectrum library we're going to plot
input_library_info = open_input_libraries(args.library, {})
input_library, library_items = [input_library_info[i] for i in ("library", "items")]
library_ids = [i["specId"] for i in library_items]
library_spectra = input_library.open(ids=library_ids)

# Write out spectra one by one
for i in range(len(library_spectra)):
    metadata = library_spectra.get_metadata(i)

    if "Starname" not in metadata:
        metadata["Starname"] = "untitled"

    spectrum = library_spectra.extract_item(i)

    name = metadata["Starname"]
    if metadata.get("exposure", np.nan) is None:
        metadata["exposure"] = np.nan
    exposure = float(metadata.get("exposure", np.nan))
    mag_4fs = float(metadata.get("magnitude", np.nan))
    reddening = float(metadata.get("e_bv", np.nan))
    snr = float(metadata.get("SNR", np.nan))
    snr_defn = metadata.get("snr_definition", "--")
    snr_per_unit = "pix" if (metadata.get("SNR_per", None) != "A") else "A"

    r = spectrum.photometry("SDSS_r")
    g = spectrum.photometry("SDSS_g")
    u = spectrum.photometry("SDSS_u")

    if ((("lrs" in args.library) and (int(mag_4fs) in [13, 16]) or (int(snr) in [100, 150])) or
            (("hrs" in args.library) and (int(mag_4fs) in [15, 19]) or (int(snr) in [10, 50]))):
        continue

    if i == 0:
        line = "# {:13s} {:8s} {:8s} {:8s} {:8s} {:8s} {:8s} {:8s} {:10s}". \
            format("Object", "Exposure", "Mag(4FS)", "E(B-V)", "Teff", "log(g)", "[Fe/H]",
                   "SNR/" + snr_per_unit, "SNR defn")
        # line += " {:8s} {:8s} {:8s}".format("SDSS_r", "SDSS_g", "SDSS_u")
        print line

    line = "{:15s} {:8.1f} {:8.2f} {:8.3f} {:8.3f} {:8.3f} {:8.3f} {:8.1f} {:10s}". \
        format(name, exposure, mag_4fs, reddening, metadata["Teff"], metadata["logg"], metadata["[Fe/H]"],
               snr, snr_defn)
    # line += " {:8.3f} {:8.3f} {:8.3f}".format(r, g, u)
    print line
<commit_msg>Added missing brackets in logic<commit_after>#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-

"""
Take spectrum library, and list some basic stellar parameters and photometry for the spectra within.
"""

from os import path as os_path
import argparse
import re
import numpy as np

from fourgp_speclib import SpectrumLibrarySqlite

# Read input parameters
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--library', required=True, dest='library',
                    help="Spectrum library we should export, with additional constraints if wanted.")
parser.add_argument('--workspace', dest='workspace', default="",
                    help="Directory where we expect to find spectrum libraries.")
args = parser.parse_args()

# Helper for opening input SpectrumLibrary(s)
def open_input_libraries(library_spec, extra_constraints):
    test = re.match("([^\[]*)\[(.*)\]$", library_spec)
    constraints = {}
    if test is None:
        library_name = library_spec
    else:
        library_name = test.group(1)
        for constraint in test.group(2).split(","):
            words_1 = constraint.split("=")
            words_2 = constraint.split("<")
            if len(words_1) == 2:
                constraint_name = words_1[0]
                try:
                    constraint_value = float(words_1[1])
                except ValueError:
                    constraint_value = words_1[1]
                constraints[constraint_name] = constraint_value
            elif len(words_2) == 3:
                constraint_name = words_2[1]
                try:
                    constraint_value_a = float(words_2[0])
                    constraint_value_b = float(words_2[2])
                except ValueError:
                    constraint_value_a = words_2[0]
                    constraint_value_b = words_2[2]
                constraints[constraint_name] = (constraint_value_a, constraint_value_b)
            else:
                assert False, "Could not parse constraint <{}>".format(constraint)
    constraints.update(extra_constraints)
    constraints["continuum_normalised"] = 0  # All input spectra must be continuum normalised
    library_path = os_path.join(workspace, library_name)
    input_library = SpectrumLibrarySqlite(path=library_path, create=False)
    library_items = input_library.search(**constraints)
    return {
        "library": input_library,
        "items": library_items
    }

# Set path to workspace where we expect to find libraries of spectra
our_path = os_path.split(os_path.abspath(__file__))[0]
workspace = args.workspace if args.workspace else os_path.join(our_path, "..", "workspace")

# Open spectrum library we're going to plot
input_library_info = open_input_libraries(args.library, {})
input_library, library_items = [input_library_info[i] for i in ("library", "items")]
library_ids = [i["specId"] for i in library_items]
library_spectra = input_library.open(ids=library_ids)

# Write out spectra one by one
for i in range(len(library_spectra)):
    metadata = library_spectra.get_metadata(i)

    if "Starname" not in metadata:
        metadata["Starname"] = "untitled"

    spectrum = library_spectra.extract_item(i)

    name = metadata["Starname"]
    if metadata.get("exposure", np.nan) is None:
        metadata["exposure"] = np.nan
    exposure = float(metadata.get("exposure", np.nan))
    mag_4fs = float(metadata.get("magnitude", np.nan))
    reddening = float(metadata.get("e_bv", np.nan))
    snr = float(metadata.get("SNR", np.nan))
    snr_defn = metadata.get("snr_definition", "--")
    snr_per_unit = "pix" if (metadata.get("SNR_per", None) != "A") else "A"

    r = spectrum.photometry("SDSS_r")
    g = spectrum.photometry("SDSS_g")
    u = spectrum.photometry("SDSS_u")

    if ((("lrs" in args.library) and ((int(mag_4fs) in [13, 16]) or (int(snr) in [100, 150]))) or
            (("hrs" in args.library) and ((int(mag_4fs) in [15, 19]) or (int(snr) in [10, 50])))):
        continue

    if i == 0:
        line = "# {:13s} {:8s} {:8s} {:8s} {:8s} {:8s} {:8s} {:8s} {:10s}". \
            format("Object", "Exposure", "Mag(4FS)", "E(B-V)", "Teff", "log(g)", "[Fe/H]",
                   "SNR/" + snr_per_unit, "SNR defn")
        # line += " {:8s} {:8s} {:8s}".format("SDSS_r", "SDSS_g", "SDSS_u")
        print line

    line = "{:15s} {:8.1f} {:8.2f} {:8.3f} {:8.3f} {:8.3f} {:8.3f} {:8.1f} {:10s}". \
        format(name, exposure, mag_4fs, reddening, metadata["Teff"], metadata["logg"], metadata["[Fe/H]"],
               snr, snr_defn)
    # line += " {:8.3f} {:8.3f} {:8.3f}".format(r, g, u)
    print line
<|endoftext|>


<commit_before>import gzip
import json
import os
import tempfile
import re

from exporters.readers.base_reader import BaseReader
from exporters.records.base_record import BaseRecord
from exporters.default_retries import retry_long
from exporters.exceptions import ConfigurationError

class S3Reader(BaseReader):
    """
    Reads items from s3 keys with a common prefix.

        - batch_size (int)
            Number of items to be returned in each batch

        - bucket (str)
            Name of the bucket to retrieve items from.

        - aws_access_key_id (str)
            Public access key to the s3 bucket.

        - aws_secret_access_key (str)
            Secret access key to the s3 bucket.

        - prefix (str)
            Prefix of s3 keys to be read.

        - prefix_pointer (str)
            Prefix pointing to the last version of dataset.

    """

    # List of options to set up the reader
    supported_options = {
        'batch_size': {'type': int, 'default': 10000},
        'bucket': {'type': basestring},
        'aws_access_key_id': {'type': basestring, 'env_fallback': 'EXPORTERS_S3READER_AWS_KEY'},
        'aws_secret_access_key': {'type': basestring, 'env_fallback': 'EXPORTERS_S3READER_AWS_SECRET'},
        'prefix': {'type': basestring, 'default': ''},
        'prefix_pointer': {'type': basestring, 'default': None},
        'pattern': {'type': basestring, 'default': None}
    }

    def __init__(self, options):
        import boto

        super(S3Reader, self).__init__(options)
        self.batch_size = self.read_option('batch_size')
        self.connection = boto.connect_s3(self.read_option('aws_access_key_id'),
                                          self.read_option('aws_secret_access_key'))
        self.bucket = self.connection.get_bucket(self.read_option('bucket'))
        self.prefix = self.read_option('prefix')
        self.prefix_pointer = self.read_option('prefix_pointer')
        self.pattern = self.read_option('pattern')

        if self.prefix and self.prefix_pointer:
            raise ConfigurationError("prefix and prefix_pointer options cannot be used together")

        if self.prefix_pointer:
            self.prefix = self._download_pointer(self.prefix_pointer)

        self.keys = []
        for key in self.bucket.list(prefix=self.prefix):
            if self.pattern:
                self._add_key_if_matches(key)
            else:
                self.keys.append(key.key)
        self.read_keys = []
        self.current_key = None
        self.last_line = 0
        self.logger.info('S3Reader has been initiated')
        self.tmp_folder = tempfile.mkdtemp()

    def _add_key_if_matches(self, key):
        if re.match(os.path.join(self.prefix, self.pattern), key.name):
            self.keys.append(key.key)

    @retry_long
    def _download_pointer(self, prefix_pointer):
        return self.bucket.get_key(prefix_pointer).get_contents_as_string().strip()

    @retry_long
    def get_key(self, file_path):
        self.bucket.get_key(self.current_key).get_contents_to_filename(file_path)

    def get_next_batch(self):
        file_path = '{}/ds_dump.gz'.format(self.tmp_folder)
        if not self.current_key:
            self.current_key = self.keys[0]
            self.get_key(file_path)
            self.last_line = 0

        dump_file = gzip.open(file_path, 'r')
        lines = dump_file.readlines()
        if self.last_line + self.batch_size <= len(lines):
            last_item = self.last_line + self.batch_size
        else:
            last_item = len(lines)
            self.read_keys.append(self.current_key)
            self.keys.remove(self.current_key)
            self.current_key = None
            if len(self.keys) == 0:
                self.finished = True
                os.remove(file_path)
        for line in lines[self.last_line:last_item]:
            line = line.replace("\n", '')
            item = BaseRecord(json.loads(line))
            self.stats['read_items'] += 1
            yield item
        dump_file.close()

        self.last_line += self.batch_size

        self.last_position['keys'] = self.keys
        self.last_position['read_keys'] = self.read_keys
        self.last_position['current_key'] = self.current_key
        self.last_position['last_line'] = self.last_line

        self.logger.debug('Done reading batch')

    def set_last_position(self, last_position):
        if last_position is None:
            self.last_position = {}
            self.last_position['keys'] = self.keys
            self.last_position['read_keys'] = self.read_keys
            self.last_position['current_key'] = None
            self.last_position['last_line'] = 0
        else:
            self.last_position = last_position
            self.keys = self.last_position['keys']
            self.read_keys = self.last_position['read_keys']
            file_path = '{}/ds_dump.gz'.format(self.tmp_folder)
            if self.last_position['current_key']:
                self.current_key = self.last_position['current_key']
            else:
                self.current_key = self.keys[0]
                self.bucket.get_key(self.current_key).get_contents_to_filename(file_path)
                self.last_line = 0
            self.last_line = self.last_position['last_line']

<commit_msg>making s3 reader slightly more verbose<commit_after>import gzip
import json
import os
import tempfile
import re

from exporters.readers.base_reader import BaseReader
from exporters.records.base_record import BaseRecord
from exporters.default_retries import retry_long
from exporters.exceptions import ConfigurationError

class S3Reader(BaseReader):
    """
    Reads items from s3 keys with a common prefix.

        - batch_size (int)
            Number of items to be returned in each batch

        - bucket (str)
            Name of the bucket to retrieve items from.

        - aws_access_key_id (str)
            Public access key to the s3 bucket.

        - aws_secret_access_key (str)
            Secret access key to the s3 bucket.

        - prefix (str)
            Prefix of s3 keys to be read.

        - prefix_pointer (str)
            Prefix pointing to the last version of dataset.

    """

    # List of options to set up the reader
    supported_options = {
        'batch_size': {'type': int, 'default': 10000},
        'bucket': {'type': basestring},
        'aws_access_key_id': {'type': basestring, 'env_fallback': 'EXPORTERS_S3READER_AWS_KEY'},
        'aws_secret_access_key': {'type': basestring, 'env_fallback': 'EXPORTERS_S3READER_AWS_SECRET'},
        'prefix': {'type': basestring, 'default': ''},
        'prefix_pointer': {'type': basestring, 'default': None},
        'pattern': {'type': basestring, 'default': None}
    }

    def __init__(self, options):
        import boto

        super(S3Reader, self).__init__(options)
        self.batch_size = self.read_option('batch_size')
        self.connection = boto.connect_s3(self.read_option('aws_access_key_id'),
                                          self.read_option('aws_secret_access_key'))
        self.bucket = self.connection.get_bucket(self.read_option('bucket'))
        self.prefix = self.read_option('prefix')
        self.prefix_pointer = self.read_option('prefix_pointer')
        self.pattern = self.read_option('pattern')

        if self.prefix and self.prefix_pointer:
            raise ConfigurationError("prefix and prefix_pointer options cannot be used together")

        if self.prefix_pointer:
            self.prefix = self._download_pointer(self.prefix_pointer)

        self.keys = []
        for key in self.bucket.list(prefix=self.prefix):
            if self.pattern:
                self._add_key_if_matches(key)
            else:
                self.keys.append(key.key)
        self.read_keys = []
        self.current_key = None
        self.last_line = 0
        self.logger.info('S3Reader has been initiated')
        self.tmp_folder = tempfile.mkdtemp()

    def _add_key_if_matches(self, key):
        if re.match(os.path.join(self.prefix, self.pattern), key.name):
            self.keys.append(key.key)

    @retry_long
    def _download_pointer(self, prefix_pointer):
        self.logger.info('Downloading prefix pointer from key: %s' % prefix_pointer)
        return self.bucket.get_key(prefix_pointer).get_contents_as_string().strip()

    @retry_long
    def get_key(self, file_path):
        self.logger.info('Downloading key: %s' % self.current_key)
        self.bucket.get_key(self.current_key).get_contents_to_filename(file_path)

    def get_next_batch(self):
        file_path = '{}/ds_dump.gz'.format(self.tmp_folder)
        if not self.current_key:
            self.current_key = self.keys[0]
            self.get_key(file_path)
            self.last_line = 0

        dump_file = gzip.open(file_path, 'r')
        lines = dump_file.readlines()
        if self.last_line + self.batch_size <= len(lines):
            last_item = self.last_line + self.batch_size
        else:
            last_item = len(lines)
            self.read_keys.append(self.current_key)
            self.keys.remove(self.current_key)
            self.current_key = None
            if len(self.keys) == 0:
                self.finished = True
                os.remove(file_path)
        for line in lines[self.last_line:last_item]:
            line = line.replace("\n", '')
            item = BaseRecord(json.loads(line))
            self.stats['read_items'] += 1
            yield item
        dump_file.close()

        self.last_line += self.batch_size

        self.last_position['keys'] = self.keys
        self.last_position['read_keys'] = self.read_keys
        self.last_position['current_key'] = self.current_key
        self.last_position['last_line'] = self.last_line

        self.logger.debug('Done reading batch')

    def set_last_position(self, last_position):
        if last_position is None:
            self.last_position = {}
            self.last_position['keys'] = self.keys
            self.last_position['read_keys'] = self.read_keys
            self.last_position['current_key'] = None
            self.last_position['last_line'] = 0
        else:
            self.last_position = last_position
            self.keys = self.last_position['keys']
            self.read_keys = self.last_position['read_keys']
            file_path = '{}/ds_dump.gz'.format(self.tmp_folder)
            if self.last_position['current_key']:
                self.current_key = self.last_position['current_key']
            else:
                self.current_key = self.keys[0]
                self.bucket.get_key(self.current_key).get_contents_to_filename(file_path)
                self.last_line = 0
            self.last_line = self.last_position['last_line']

<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import sys
import os.path
import bz2
import gzip

from csvkit import CSVKitReader
from csvkit.exceptions import ColumnIdentifierError

class CSVFileType(object):
    """
    An argument factory like argparse.FileType with compression support.
    """

    def __init__(self, mode = "rb"):
        """
        Initialize the factory.
        """

        self._mode = mode

    def __call__(self, path):
        """
        Build a file-like object from the specified path.
        """

        if path == "-":
            if "r" in self._mode:
                return sys.stdin
            elif "w" in self._mode:
                return sys.stdout
            else:
                raise ValueError("invalid path \"-\" with mode {0}".format(self._mode))
        else:
            (_, extension) = os.path.splitext(path)

            if extension == ".gz":
                return gzip.open(path, self._mode)
            if extension == ".bz2":
                return bz2.BZ2File(path, self._mode)
            else:
                return open(path, self._mode)

class CSVKitUtility(object):
    description = ''
    epilog = ''
    override_flags = ''

    def __init__(self):
        """
        Perform argument processing and other setup for a CSVKitUtility.
        """
        self._init_common_parser()
        self.add_arguments()
        self.args = self.argparser.parse_args()

        self.reader_kwargs = self._extract_csv_reader_kwargs()
        self.writer_kwargs = self._extract_csv_writer_kwargs()

        self._install_exception_handler()

    def add_arguments(self):
        """
        Called upon initialization once the parser for common arguments has been constructed.

        Should be overriden by individual utilities.
        """
        raise NotImplementedError('add_arguments must be provided by each subclass of CSVKitUtility.')

    def main(self):
        """
        Main loop of the utility.

        Should be overriden by individual utilities and explicitly called by the executing script.
        """
        raise NotImplementedError(' must be provided by each subclass of CSVKitUtility.')

    def _init_common_parser(self):
        """
        Prepare a base argparse argument parser so that flags are consistent across different shell command tools.
        If you want to constrain which common args are present, you can pass a string for 'omitflags'. Any argument
        whose single-letter form is contained in 'omitflags' will be left out of the configured parser. Use 'f' for
        file.
        """
        self.argparser = argparse.ArgumentParser(description=self.description, epilog=self.epilog)

        # Input
        if 'f' not in self.override_flags:
            self.argparser.add_argument('file', metavar="FILE", nargs='?', type=CSVFileType(), default=sys.stdin,
                                help='The CSV file to operate on. If omitted, will accept input on STDIN.')
        if 'd' not in self.override_flags:
            self.argparser.add_argument('-d', '--delimiter', dest='delimiter',
                                help='Delimiting character of the input CSV file.')
        if 't' not in self.override_flags:
            self.argparser.add_argument('-t', '--tabs', dest='tabs', action='store_true',
                                help='Specifies that the input CSV file is delimited with tabs. Overrides "-d".')
        if 'q' not in self.override_flags:
            self.argparser.add_argument('-q', '--quotechar', dest='quotechar',
                                help='Character used to quote strings in the input CSV file.')
        if 'u' not in self.override_flags:
            self.argparser.add_argument('-u', '--quoting', dest='quoting', type=int, choices=[0,1,2,3],
                                help='Quoting style used in the input CSV file. 0 = Quote Minimal, 1 = Quote All, 2 = Quote Non-numeric, 3 = Quote None.')
        if 'b' not in self.override_flags:
            self.argparser.add_argument('-b', '--doublequote', dest='doublequote', action='store_true',
                                help='Whether or not double quotes are doubled in the input CSV file.')
        if 'p' not in self.override_flags:
            self.argparser.add_argument('-p', '--escapechar', dest='escapechar',
                                help='Character used to escape the delimiter if quoting is set to "Quote None" and the quotechar if doublequote is not specified.')
        if 'e' not in self.override_flags:
            self.argparser.add_argument('-e', '--encoding', dest='encoding', default='utf-8',
                                help='Specify the encoding the input file.')
        if 'v' not in self.override_flags:
            self.argparser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
                                help='Print detailed tracebacks when errors occur.')

        # Output
        if 'l' not in self.override_flags:
            self.argparser.add_argument('-l', '--linenumbers', dest='line_numbers', action='store_true',
                                help='Insert a column of line numbers at the front of the output. Useful when piping to grep or as a simple primary key.')

    def _extract_csv_reader_kwargs(self):
        """
        Extracts those from the command-line arguments those would should be passed through to the input CSV reader(s).
        """
        kwargs = {}
        if self.args.encoding:
            kwargs['encoding'] = self.args.encoding

        if self.args.tabs:
            kwargs['delimiter'] = '\t'
        elif self.args.delimiter:
            kwargs['delimiter'] = self.args.delimiter

        if self.args.quotechar:
            kwargs['quotechar'] = self.args.quotechar

        if self.args.quoting:
            kwargs['quoting'] = self.args.quoting

        if self.args.doublequote:
            kwargs['doublequote'] = self.args.doublequote

        if self.args.escapechar:
            kwargs['escapechar'] = self.args.escapechar

        return kwargs

    def _extract_csv_writer_kwargs(self):
        """
        Extracts those from the command-line arguments those would should be passed through to the output CSV writer.
        """
        kwargs = {}

        if 'l' not in self.override_flags and self.args.line_numbers:
            kwargs['line_numbers'] = True

        return kwargs

    def _install_exception_handler(self):
        """
        Installs a replacement for sys.excepthook, which handles pretty-printing uncaught exceptions.
        """
        def handler(t, value, traceback):
            if self.args.verbose:
                sys.__excepthook__(t, value, traceback)
            else:
                print value

        sys.excepthook = handler

def match_column_identifier(column_names, c):
    """
    Determine what column a single column id (name or index) matches in a series of column names.
    """
    if c in column_names:
        return column_names.index(c)
    else:
        try:
            c = int(c) - 1
        # Fail out if neither a column name nor an integer
        except:
            raise ColumnIdentifierError('Column identifier "%s" is neither a index, nor a existing column\'s name.' % c)

        # Fail out if index is 0-based
        if c < 0:
            raise ColumnIdentifierError('Columns 0 is not valid; columns are 1-based.')

        # Fail out if index is out of range
        if c >= len(column_names):
            raise ColumnIdentifierError('Index %i is beyond the last named column, "%s" at index %i.' % (c, column_names[-1], len(column_names) - 1))

    return c

def parse_column_identifiers(ids, column_names):
    """
    Parse a comma-separated list of column indices AND/OR names into a list of integer indices.

    Note: Column indices are 1-based.
    """
    # If not specified, return all columns
    if not ids:
        return range(len(column_names))

    columns = []

    for c in ids.split(','):
        columns.append(match_column_identifier(column_names, c.strip()))

    return columns

def print_column_names(f, output, **reader_kwargs):
    """
    Pretty-prints the names and indices of all columns to a file-like object (usually sys.stdout).
    """
    rows = CSVKitReader(f, **reader_kwargs)
    column_names = rows.next()

    for i, c in enumerate(column_names):
        output.write('%3i: %s\n' % (i + 1, c))

    sys.exit()
<commit_msg>Shuffle dependencies to sate OCD urges.<commit_after>#!/usr/bin/env python

import argparse
import bz2
import gzip
import os.path
import sys

from csvkit import CSVKitReader
from csvkit.exceptions import ColumnIdentifierError

class CSVFileType(object):
    """
    An argument factory like argparse.FileType with compression support.
    """

    def __init__(self, mode = "rb"):
        """
        Initialize the factory.
        """
        self._mode = mode

    def __call__(self, path):
        """
        Build a file-like object from the specified path.
        """
        if path == "-":
            if "r" in self._mode:
                return sys.stdin
            elif "w" in self._mode:
                return sys.stdout
            else:
                raise ValueError("invalid path \"-\" with mode {0}".format(self._mode))
        else:
            (_, extension) = os.path.splitext(path)

            if extension == ".gz":
                return gzip.open(path, self._mode)
            if extension == ".bz2":
                return bz2.BZ2File(path, self._mode)
            else:
                return open(path, self._mode)

class CSVKitUtility(object):
    description = ''
    epilog = ''
    override_flags = ''

    def __init__(self):
        """
        Perform argument processing and other setup for a CSVKitUtility.
        """
        self._init_common_parser()
        self.add_arguments()
        self.args = self.argparser.parse_args()

        self.reader_kwargs = self._extract_csv_reader_kwargs()
        self.writer_kwargs = self._extract_csv_writer_kwargs()

        self._install_exception_handler()

    def add_arguments(self):
        """
        Called upon initialization once the parser for common arguments has been constructed.

        Should be overriden by individual utilities.
        """
        raise NotImplementedError('add_arguments must be provided by each subclass of CSVKitUtility.')

    def main(self):
        """
        Main loop of the utility.

        Should be overriden by individual utilities and explicitly called by the executing script.
        """
        raise NotImplementedError(' must be provided by each subclass of CSVKitUtility.')

    def _init_common_parser(self):
        """
        Prepare a base argparse argument parser so that flags are consistent across different shell command tools.
        If you want to constrain which common args are present, you can pass a string for 'omitflags'. Any argument
        whose single-letter form is contained in 'omitflags' will be left out of the configured parser. Use 'f' for
        file.
        """
        self.argparser = argparse.ArgumentParser(description=self.description, epilog=self.epilog)

        # Input
        if 'f' not in self.override_flags:
            self.argparser.add_argument('file', metavar="FILE", nargs='?', type=CSVFileType(), default=sys.stdin,
                                help='The CSV file to operate on. If omitted, will accept input on STDIN.')
        if 'd' not in self.override_flags:
            self.argparser.add_argument('-d', '--delimiter', dest='delimiter',
                                help='Delimiting character of the input CSV file.')
        if 't' not in self.override_flags:
            self.argparser.add_argument('-t', '--tabs', dest='tabs', action='store_true',
                                help='Specifies that the input CSV file is delimited with tabs. Overrides "-d".')
        if 'q' not in self.override_flags:
            self.argparser.add_argument('-q', '--quotechar', dest='quotechar',
                                help='Character used to quote strings in the input CSV file.')
        if 'u' not in self.override_flags:
            self.argparser.add_argument('-u', '--quoting', dest='quoting', type=int, choices=[0,1,2,3],
                                help='Quoting style used in the input CSV file. 0 = Quote Minimal, 1 = Quote All, 2 = Quote Non-numeric, 3 = Quote None.')
        if 'b' not in self.override_flags:
            self.argparser.add_argument('-b', '--doublequote', dest='doublequote', action='store_true',
                                help='Whether or not double quotes are doubled in the input CSV file.')
        if 'p' not in self.override_flags:
            self.argparser.add_argument('-p', '--escapechar', dest='escapechar',
                                help='Character used to escape the delimiter if quoting is set to "Quote None" and the quotechar if doublequote is not specified.')
        if 'e' not in self.override_flags:
            self.argparser.add_argument('-e', '--encoding', dest='encoding', default='utf-8',
                                help='Specify the encoding the input file.')
        if 'v' not in self.override_flags:
            self.argparser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
                                help='Print detailed tracebacks when errors occur.')

        # Output
        if 'l' not in self.override_flags:
            self.argparser.add_argument('-l', '--linenumbers', dest='line_numbers', action='store_true',
                                help='Insert a column of line numbers at the front of the output. Useful when piping to grep or as a simple primary key.')

    def _extract_csv_reader_kwargs(self):
        """
        Extracts those from the command-line arguments those would should be passed through to the input CSV reader(s).
        """
        kwargs = {}
        if self.args.encoding:
            kwargs['encoding'] = self.args.encoding

        if self.args.tabs:
            kwargs['delimiter'] = '\t'
        elif self.args.delimiter:
            kwargs['delimiter'] = self.args.delimiter

        if self.args.quotechar:
            kwargs['quotechar'] = self.args.quotechar

        if self.args.quoting:
            kwargs['quoting'] = self.args.quoting

        if self.args.doublequote:
            kwargs['doublequote'] = self.args.doublequote

        if self.args.escapechar:
            kwargs['escapechar'] = self.args.escapechar

        return kwargs

    def _extract_csv_writer_kwargs(self):
        """
        Extracts those from the command-line arguments those would should be passed through to the output CSV writer.
        """
        kwargs = {}

        if 'l' not in self.override_flags and self.args.line_numbers:
            kwargs['line_numbers'] = True

        return kwargs

    def _install_exception_handler(self):
        """
        Installs a replacement for sys.excepthook, which handles pretty-printing uncaught exceptions.
        """
        def handler(t, value, traceback):
            if self.args.verbose:
                sys.__excepthook__(t, value, traceback)
            else:
                print value

        sys.excepthook = handler

def match_column_identifier(column_names, c):
    """
    Determine what column a single column id (name or index) matches in a series of column names.
    """
    if c in column_names:
        return column_names.index(c)
    else:
        try:
            c = int(c) - 1
        # Fail out if neither a column name nor an integer
        except:
            raise ColumnIdentifierError('Column identifier "%s" is neither a index, nor a existing column\'s name.' % c)

        # Fail out if index is 0-based
        if c < 0:
            raise ColumnIdentifierError('Columns 0 is not valid; columns are 1-based.')

        # Fail out if index is out of range
        if c >= len(column_names):
            raise ColumnIdentifierError('Index %i is beyond the last named column, "%s" at index %i.' % (c, column_names[-1], len(column_names) - 1))

    return c

def parse_column_identifiers(ids, column_names):
    """
    Parse a comma-separated list of column indices AND/OR names into a list of integer indices.

    Note: Column indices are 1-based.
    """
    # If not specified, return all columns
    if not ids:
        return range(len(column_names))

    columns = []

    for c in ids.split(','):
        columns.append(match_column_identifier(column_names, c.strip()))

    return columns

def print_column_names(f, output, **reader_kwargs):
    """
    Pretty-prints the names and indices of all columns to a file-like object (usually sys.stdout).
    """
    rows = CSVKitReader(f, **reader_kwargs)
    column_names = rows.next()

    for i, c in enumerate(column_names):
        output.write('%3i: %s\n' % (i + 1, c))

    sys.exit()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import datetime
import re
from types import NoneType

from sqlalchemy import Column, MetaData, Table, create_engine
from sqlalchemy import Boolean, Date, DateTime, Float, Integer, String, Time
from sqlalchemy.schema import CreateTable

DIALECTS = {
    'access': 'access.base',
    'firebird': 'firebird.kinterbasdb',
    'informix': 'informix.informixdb',
    'maxdb': 'maxdb.sapdb',
    'mssql': 'mssql.pyodbc',
    'mysql': 'mysql.mysqlconnector',
    'oracle': 'oracle.cx_oracle',
    'postgresql': 'postgresql.psycopg2',
    'sqlite': 'sqlite.pysqlite',
    'sybase': 'sybase.pyodbc'
}

NULL_COLUMN_MAX_LENGTH = 32

def make_column(column):
    """
    Creates a sqlalchemy column from a csvkit Column.
    """
    sql_column_kwargs = {'nullable': False}
    sql_type_kwargs = {}

    column_types = {
        bool: Boolean,
        int: Integer,
        float: Float,
        datetime.datetime: DateTime,
        datetime.date: Date,
        datetime.time: Time,
        NoneType: String,
        unicode: String
        }

    max_lengths = {
        NoneType: NULL_COLUMN_MAX_LENGTH,
        unicode: column.max_length
        }

    if column.type in column_types:
        sql_column_type = column_types[column.type]
    else:
        raise ValueError('Unexpected normalized column type: %s' % column.type)

    if column.type in max_lengths:
        sql_type_kwargs['length'] = max_lengths[column.type]

    sql_column_kwargs['nullable'] = column.nullable

    return Column(column.name, sql_column_type(**sql_type_kwargs), **sql_column_kwargs)

def get_connection(connection_string):
    engine = create_engine(connection_string)
    metadata = MetaData(engine)

    return engine, metadata

def make_table(csv_table, name='table_name', metadata=None):
    """
    Creates a sqlalchemy table from a csvkit Table.
    """
    if not metadata:
        metadata = MetaData()

    sql_table = Table(csv_table.name, metadata)

    for column in csv_table:
        sql_table.append_column(make_column(column))

    return sql_table

def make_create_table_statement(sql_table, dialect=None):
    """
    Generates a CREATE TABLE statement for a sqlalchemy table.
    """
    if dialect:
        module = __import__('sqlalchemy.dialects.%s' % DIALECTS[dialect], fromlist=['dialect'])
        sql_dialect = module.dialect()
    else:
        sql_dialect = None

    return unicode(CreateTable(sql_table).compile(dialect=sql_dialect)).strip() + ';'

def de_parameterize_insert_query(statement, values, dialect=None):
    """
    Insert row values in a parameterized insert statement generated by sqlalchemy.

    Note: all the hard-coded processing in this function sucks, but sqlalchemy doesn't expose a way of doing this as part of the dialect-specific query processing.
    """
    # All dialects use VALUES to set off the start of the parameter clause
    start_params = statement.index('VALUES (') + 7
    params_clause = statement[start_params:]

    # Get an iterator for the values (which are likely a tuple)
    values = iter(values)

    # Use probably ill-advised closure magic to format values in way SQL understands
    def value_formatter(*args):
        v = values.next()

        if v is None:
            return 'NULL'
        elif type(v) == bool:
            return unicode(v).upper()
        elif type(v) == unicode:
            return u'\'%s\'' % v
        else:
            return unicode(v)

    # ':column_name' syntax
    if dialect in ['access', 'firebird', 'maxdb', 'mssql', 'oracle', 'sybase', None]:
        params_clause = re.sub('(:"?.+?"?)(?=[,)])', value_formatter, params_clause)
    # '?' syntax
    elif dialect in ['informix', 'sqlite']:
        params_clause = re.sub('(\?)(?=[,)])', value_formatter, params_clause)
    # '%s' syntax
    elif dialect in ['mysql']:
        params_clause = re.sub('(%s)(?=[,)])', value_formatter, params_clause)
    # '%(column_name)s' syntax
    elif dialect in ['postgresql']:
        params_clause = re.sub('(%("?.+?"?)s)(?=[,)])', value_formatter, params_clause)

    return statement[:start_params] + params_clause

def make_insert_statement(sql_table, values, dialect=None):
    """
    Generates INSERT statements for data in a Table using a sqlalchemy dialect.
    """
    if dialect:
        module = __import__('sqlalchemy.dialects.%s' % DIALECTS[dialect], fromlist=['dialect'])
        sql_dialect = module.dialect()
    else:
        sql_dialect = None

    insert = unicode(sql_table.insert().compile(dialect=sql_dialect))
    statement = de_parameterize_insert_query(insert, values, dialect=dialect)
    return statement.strip() + ';'
    return statement.strip() + ';'

<commit_msg>Remove sql generation cruft now that inserts are directly executed on the database. Closes #126.<commit_after>#!/usr/bin/env python

import datetime
import re
from types import NoneType

from sqlalchemy import Column, MetaData, Table, create_engine
from sqlalchemy import Boolean, Date, DateTime, Float, Integer, String, Time
from sqlalchemy.schema import CreateTable

DIALECTS = {
    'access': 'access.base',
    'firebird': 'firebird.kinterbasdb',
    'informix': 'informix.informixdb',
    'maxdb': 'maxdb.sapdb',
    'mssql': 'mssql.pyodbc',
    'mysql': 'mysql.mysqlconnector',
    'oracle': 'oracle.cx_oracle',
    'postgresql': 'postgresql.psycopg2',
    'sqlite': 'sqlite.pysqlite',
    'sybase': 'sybase.pyodbc'
}

NULL_COLUMN_MAX_LENGTH = 32

def make_column(column):
    """
    Creates a sqlalchemy column from a csvkit Column.
    """
    sql_column_kwargs = {'nullable': False}
    sql_type_kwargs = {}

    column_types = {
        bool: Boolean,
        int: Integer,
        float: Float,
        datetime.datetime: DateTime,
        datetime.date: Date,
        datetime.time: Time,
        NoneType: String,
        unicode: String
        }

    max_lengths = {
        NoneType: NULL_COLUMN_MAX_LENGTH,
        unicode: column.max_length
        }

    if column.type in column_types:
        sql_column_type = column_types[column.type]
    else:
        raise ValueError('Unexpected normalized column type: %s' % column.type)

    if column.type in max_lengths:
        sql_type_kwargs['length'] = max_lengths[column.type]

    sql_column_kwargs['nullable'] = column.nullable

    return Column(column.name, sql_column_type(**sql_type_kwargs), **sql_column_kwargs)

def get_connection(connection_string):
    engine = create_engine(connection_string)
    metadata = MetaData(engine)

    return engine, metadata

def make_table(csv_table, name='table_name', metadata=None):
    """
    Creates a sqlalchemy table from a csvkit Table.
    """
    if not metadata:
        metadata = MetaData()

    sql_table = Table(csv_table.name, metadata)

    for column in csv_table:
        sql_table.append_column(make_column(column))

    return sql_table

def make_create_table_statement(sql_table, dialect=None):
    """
    Generates a CREATE TABLE statement for a sqlalchemy table.
    """
    if dialect:
        module = __import__('sqlalchemy.dialects.%s' % DIALECTS[dialect], fromlist=['dialect'])
        sql_dialect = module.dialect()
    else:
        sql_dialect = None

    return unicode(CreateTable(sql_table).compile(dialect=sql_dialect)).strip() + ';'

<|endoftext|>


<commit_before># -*- coding=utf -*-

import os.path
import json
from collections import namedtuple
from .extensions import get_namespace, initialize_namespace
from .browser import Cell
from .errors import *
from .common import read_json_file, sorted_dependencies

__all__ = (
    "create_authorizer",
    "Authorizer",
    "SimpleAuthorizer",
    "NotAuthorized"
)

class NotAuthorized(UserError):
    """Raised when user is not authorized for the request."""
    # Note: This is not called NotAuthorizedError as it is not in fact an
    # error, it is just type of signal.

def create_authorizer(name, **options):
    """Gets a new instance of an authorizer with name `name`."""

    ns = get_namespace("authorizers")
    if not ns:
        ns = initialize_namespace("authorizers", root_class=Authorizer,
                                  suffix="_authorizer",
                                  option_checking=True)

    try:
        factory = ns[name]
    except KeyError:
        raise ConfigurationError("Unknown authorizer '%s'" % name)

    return factory(**options)

class Authorizer(object):
    def cube_is_allowed(self, token):
        """Return a list of allowed cubes for authorization `token`."""
        raise NotImplementedError

    def authorize(self, token, cube):
        """Returns appropriated `cell` within `cube` for authorization token
        `token`. The `token` is specific to concrete authorizer object.

        If `token` does not authorize access to the cube a NotAuthorized exception is
        raised."""
        raise NotImplementedError

    def restricted_cell(self, token, cube, cell=None):
        """Restricts the `cell` for `cube` according to authorization by
        `token`. If no cell is provided or the cell is empty then returns
        the restriction cell. If there is no restriction, returns the original
        `cell` if provided or `None`.
        """
        return cell

class NoopAuthorizer(Authorizer):
    def __init__(self):
        super(NoopAuthorizer, self).__init__()

    def authorize(self, token, cube, cell=None):
        return None

    def restricted_cell(self, token, cube, cell):
        return None

class _SimpleAccessRight(object):
    def __init__(self, roles, allow_cubes, deny_cubes, cube_restrictions):
        self.roles = roles or []
        self.allow_cubes = set(allow_cubes) if allow_cubes else set()
        self.deny_cubes = set(deny_cubes) if deny_cubes else set()
        self.cube_restrictions = cube_restrictions or {}

    def merge(self, other):
        """Merge `right` with the receiver:

        * `allow_cubes` are merged (union)
        * `deny_cubes` are merged (union)
        * `cube_restrictions` from `other` with same cube replace restrictions
          from the receiver"""

        self.allow_cubes |= other.allow_cubes
        self.deny_cubes |= other.deny_cubes

        for cube, restrictions in other.cube_restrictions:
            if not cube in self.cube_restrictions:
                self.cube_restrictions = list(other.cube_restrictions)
            else:
                mine = self.cube_restrictions.get(cube)
                mine += restritions

class SimpleAuthorizer(Authorizer):
    __options__ = [
        {
            "name": "rights_file",
            "description": "JSON file with access rights",
            "type": "string"
        },
        {
            "name": "roles_file",
            "description": "JSON file with access right roles",
            "type": "string"
        },

    ]

    def __init__(self, rights_file=None, roles_file=None, roles=None,
                 rights=None, **options):
        """Creates a simple JSON-file based authorizer. Reads data from
        `rights_file` and `roles_file` and merge them with `roles` and
        `rights` dictionaries respectively."""

        super(SimpleAuthorizer, self).__init__()

        roles = roles or {}
        rights = rights or {}

        if roles_file:
            content = read_json_file(roles_file, "access roles")
            roles.update(content)

        if rights_file:
            content = read_json_file(rights_file, "access rights")
            rights.update(content)

        self.roles = {}
        self.rights = {}

        # Process the roles
        for key, info in roles.items():
            role = _SimpleAccessRight(roles=info.get("roles"),
                                      allow_cubes=info.get("allow_cubes"),
                                      deny_cubes=info.get("deny_cubes"),
                                      cube_restrictions=info.get("cube_restrictions"))
            self.roles[key] = role

        deps = dict((name, role.roles) for name, role in self.roles.items())
        order = sorted_dependencies(deps)

        for name in order:
            role = self.roles[name]
            for parent_name in role.roles:
                parent = self.roles[parent_name]
                role.merge(parent)

        # Process rights
        for key, info in rights.items():
            right = _SimpleAccessRight(roles=info.get("roles"),
                                       allow_cubes=info.get("allow_cubes"),
                                       deny_cubes=info.get("deny_cubes"),
                                       cube_restrictions=info.get("cube_restrictions"))
            self.rights[key] = right

            for role_name in right.roles:
                role = self.roles[role_name]
                right.merge(role)

    def right(self, token):
        try:
            right = self.rights[token]
        except KeyError:
            raise NotAuthorized("Unknown access right '%s'" % token)
        return right

    def authorize(self, token, cube):
        right = self.right(token)

        cube_name = str(cube)

        if (right.allow_cubes and cube_name not in right.allow_cubes) \
                or (right.deny_cubes and cube_name in right.deny_cubes):
            raise NotAuthorized("Unauthorized cube '%s' for '%s'"
                                % (cube_name, token))

    def restricted_cell(self, token, cube, cell):
        right = self.right(token)

        cuts = right.cube_restrictions.get(cube.name)

        if cuts:
            cuts = [cut_from_dict(cut) for cut in cuts]
            restriction = Cell(cube, cuts)
        else:
            restriction = None

        if not restriction:
            return cell
        elif cell:
            return cell + restriction
        else:
            return restriction

<commit_msg>Don't use verb: allow → allowed, deny → denied<commit_after># -*- coding=utf -*-

import os.path
import json
from collections import namedtuple
from .extensions import get_namespace, initialize_namespace
from .browser import Cell
from .errors import *
from .common import read_json_file, sorted_dependencies

__all__ = (
    "create_authorizer",
    "Authorizer",
    "SimpleAuthorizer",
    "NotAuthorized"
)

class NotAuthorized(UserError):
    """Raised when user is not authorized for the request."""
    # Note: This is not called NotAuthorizedError as it is not in fact an
    # error, it is just type of signal.

def create_authorizer(name, **options):
    """Gets a new instance of an authorizer with name `name`."""

    ns = get_namespace("authorizers")
    if not ns:
        ns = initialize_namespace("authorizers", root_class=Authorizer,
                                  suffix="_authorizer",
                                  option_checking=True)

    try:
        factory = ns[name]
    except KeyError:
        raise ConfigurationError("Unknown authorizer '%s'" % name)

    return factory(**options)

class Authorizer(object):
    def cube_is_allowed(self, token):
        """Return a list of allowed cubes for authorization `token`."""
        raise NotImplementedError

    def authorize(self, token, cube):
        """Returns appropriated `cell` within `cube` for authorization token
        `token`. The `token` is specific to concrete authorizer object.

        If `token` does not authorize access to the cube a NotAuthorized exception is
        raised."""
        raise NotImplementedError

    def restricted_cell(self, token, cube, cell=None):
        """Restricts the `cell` for `cube` according to authorization by
        `token`. If no cell is provided or the cell is empty then returns
        the restriction cell. If there is no restriction, returns the original
        `cell` if provided or `None`.
        """
        return cell

class NoopAuthorizer(Authorizer):
    def __init__(self):
        super(NoopAuthorizer, self).__init__()

    def authorize(self, token, cube, cell=None):
        return None

    def restricted_cell(self, token, cube, cell):
        return None

class _SimpleAccessRight(object):
    def __init__(self, roles, allow_cubes, deny_cubes, cube_restrictions):
        self.roles = roles or []
        self.allow_cubes = set(allow_cubes) if allow_cubes else set()
        self.deny_cubes = set(deny_cubes) if deny_cubes else set()
        self.cube_restrictions = cube_restrictions or {}

    def merge(self, other):
        """Merge `right` with the receiver:

        * `allow_cubes` are merged (union)
        * `deny_cubes` are merged (union)
        * `cube_restrictions` from `other` with same cube replace restrictions
          from the receiver"""

        self.allow_cubes |= other.allow_cubes
        self.deny_cubes |= other.deny_cubes

        for cube, restrictions in other.cube_restrictions:
            if not cube in self.cube_restrictions:
                self.cube_restrictions = list(other.cube_restrictions)
            else:
                mine = self.cube_restrictions.get(cube)
                mine += restritions

class SimpleAuthorizer(Authorizer):
    __options__ = [
        {
            "name": "rights_file",
            "description": "JSON file with access rights",
            "type": "string"
        },
        {
            "name": "roles_file",
            "description": "JSON file with access right roles",
            "type": "string"
        },

    ]

    def __init__(self, rights_file=None, roles_file=None, roles=None,
                 rights=None, **options):
        """Creates a simple JSON-file based authorizer. Reads data from
        `rights_file` and `roles_file` and merge them with `roles` and
        `rights` dictionaries respectively."""

        super(SimpleAuthorizer, self).__init__()

        roles = roles or {}
        rights = rights or {}

        if roles_file:
            content = read_json_file(roles_file, "access roles")
            roles.update(content)

        if rights_file:
            content = read_json_file(rights_file, "access rights")
            rights.update(content)

        self.roles = {}
        self.rights = {}

        # Process the roles
        for key, info in roles.items():
            role = _SimpleAccessRight(roles=info.get("roles"),
                                      allow_cubes=info.get("allowed_cubes"),
                                      deny_cubes=info.get("denied_cubes"),
                                      cube_restrictions=info.get("cube_restrictions"))
            self.roles[key] = role

        deps = dict((name, role.roles) for name, role in self.roles.items())
        order = sorted_dependencies(deps)

        for name in order:
            role = self.roles[name]
            for parent_name in role.roles:
                parent = self.roles[parent_name]
                role.merge(parent)

        # Process rights
        for key, info in rights.items():
            right = _SimpleAccessRight(roles=info.get("roles"),
                                       allow_cubes=info.get("allowed_cubes"),
                                       deny_cubes=info.get("denied_cubes"),
                                       cube_restrictions=info.get("cube_restrictions"))
            self.rights[key] = right

            for role_name in right.roles:
                role = self.roles[role_name]
                right.merge(role)

    def right(self, token):
        try:
            right = self.rights[token]
        except KeyError:
            raise NotAuthorized("Unknown access right '%s'" % token)
        return right

    def authorize(self, token, cube):
        right = self.right(token)

        cube_name = str(cube)

        if (right.allow_cubes and cube_name not in right.allow_cubes) \
                or (right.deny_cubes and cube_name in right.deny_cubes):
            raise NotAuthorized("Unauthorized cube '%s' for '%s'"
                                % (cube_name, token))

    def restricted_cell(self, token, cube, cell):
        right = self.right(token)

        cuts = right.cube_restrictions.get(cube.name)

        if cuts:
            cuts = [cut_from_dict(cut) for cut in cuts]
            restriction = Cell(cube, cuts)
        else:
            restriction = None

        if not restriction:
            return cell
        elif cell:
            return cell + restriction
        else:
            return restriction

<|endoftext|>


<commit_before>"""
SAGA package version
"""
__version__ = "0.46.2"
<commit_msg>version increment<commit_after>"""
SAGA package version
"""
__version__ = "0.46.3"
<|endoftext|>