<commit_before># pylint: disable=C0111,R0903

"""Copy passwords from a password store into the clipboard (currently supports only "pass")

Many thanks to [@bbernhard](https://github.com/bbernhard) for the idea!

Parameters:
    * vault.duration: Duration until password is cleared from clipboard (defaults to 30)
    * vault.location: Location of the password store (defaults to ~/.password-store)
    * vault.offx: x-axis offset of popup menu (defaults to 0)
    * vault.offy: y-axis offset of popup menu (defaults to 0)
"""

# TODO:
# - support multiple backends by abstracting the menu structure into a tree
# - build the menu and the actions based on that abstracted tree
#

import os
import time
import threading
import bumblebee.util
import bumblebee.popup_v2
import bumblebee.input
import bumblebee.output
import bumblebee.engine

def build_menu(parent, current_directory, callback):
    with os.scandir(current_directory) as it:
        for entry in it:
            if entry.name.startswith("."): continue
            if entry.is_file():
                name = entry.name[:entry.name.rfind(".")]
                parent.add_menuitem(name, callback=lambda : callback(os.path.join(current_directory, name)))

            else:
                submenu = bumblebee.popup_v2.PopupMenu(parent, leave=False)
                build_menu(submenu, os.path.join(current_directory, entry.name), callback)
                parent.add_cascade(entry.name, submenu)

class Module(bumblebee.engine.Module):
    def __init__(self, engine, config):
        super(Module, self).__init__(engine, config,
            bumblebee.output.Widget(full_text=self.text)
        )
        self._duration = int(self.parameter("duration", 30))
        self._offx = int(self.parameter("offx", 0))
        self._offy = int(self.parameter("offy", 0))
        self._path = os.path.expanduser(self.parameter("location", "~/.password-store/"))
        self._reset()
        engine.input.register_callback(self, button=bumblebee.input.LEFT_MOUSE,
           cmd=self.popup)

    def popup(self, widget):
        menu = bumblebee.popup_v2.PopupMenu(leave=False)

        build_menu(menu, self._path, self._callback)
        menu.show(widget, offset_x=self._offx, offset_y=self._offy)

    def _reset(self):
        self._timer = None
        self._text = str(self.parameter("text", "<click-for-password>"))

    def _callback(self, secret_name):
        secret_name = secret_name.replace(self._path, "") # remove common path
        if self._timer:
            self._timer.cancel()
        # bumblebee.util.execute hangs for some reason
        os.system("PASSWORD_STORE_CLIP_TIME={} pass -c {} > /dev/null 2>&1".format(self._duration, secret_name))
        self._timer = threading.Timer(self._duration, self._reset)
        self._timer.start()
        self._start = int(time.time())
        self._text = secret_name

    def text(self, widget):
        if self._timer:
            return "{} ({}s)".format(self._text, self._duration - (int(time.time()) - self._start))
        return self._text

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<commit_msg>[modules/vault] quotes<commit_after># pylint: disable=C0111,R0903

"""Copy passwords from a password store into the clipboard (currently supports only 'pass')

Many thanks to [@bbernhard](https://github.com/bbernhard) for the idea!

Parameters:
    * vault.duration: Duration until password is cleared from clipboard (defaults to 30)
    * vault.location: Location of the password store (defaults to ~/.password-store)
    * vault.offx: x-axis offset of popup menu (defaults to 0)
    * vault.offy: y-axis offset of popup menu (defaults to 0)
"""

# TODO:
# - support multiple backends by abstracting the menu structure into a tree
# - build the menu and the actions based on that abstracted tree
#

import os
import time
import threading
import bumblebee.util
import bumblebee.popup_v2
import bumblebee.input
import bumblebee.output
import bumblebee.engine

def build_menu(parent, current_directory, callback):
    with os.scandir(current_directory) as it:
        for entry in it:
            if entry.name.startswith('.'): continue
            if entry.is_file():
                name = entry.name[:entry.name.rfind('.')]
                parent.add_menuitem(name, callback=lambda : callback(os.path.join(current_directory, name)))

            else:
                submenu = bumblebee.popup_v2.PopupMenu(parent, leave=False)
                build_menu(submenu, os.path.join(current_directory, entry.name), callback)
                parent.add_cascade(entry.name, submenu)

class Module(bumblebee.engine.Module):
    def __init__(self, engine, config):
        super(Module, self).__init__(engine, config,
            bumblebee.output.Widget(full_text=self.text)
        )
        self._duration = int(self.parameter('duration', 30))
        self._offx = int(self.parameter('offx', 0))
        self._offy = int(self.parameter('offy', 0))
        self._path = os.path.expanduser(self.parameter('location', '~/.password-store/'))
        self._reset()
        engine.input.register_callback(self, button=bumblebee.input.LEFT_MOUSE,
           cmd=self.popup)

    def popup(self, widget):
        menu = bumblebee.popup_v2.PopupMenu(leave=False)

        build_menu(menu, self._path, self._callback)
        menu.show(widget, offset_x=self._offx, offset_y=self._offy)

    def _reset(self):
        self._timer = None
        self._text = str(self.parameter('text', '<click-for-password>'))

    def _callback(self, secret_name):
        secret_name = secret_name.replace(self._path, '') # remove common path
        if self._timer:
            self._timer.cancel()
        # bumblebee.util.execute hangs for some reason
        os.system('PASSWORD_STORE_CLIP_TIME={} pass -c {} > /dev/null 2>&1'.format(self._duration, secret_name))
        self._timer = threading.Timer(self._duration, self._reset)
        self._timer.start()
        self._start = int(time.time())
        self._text = secret_name

    def text(self, widget):
        if self._timer:
            return '{} ({}s)'.format(self._text, self._duration - (int(time.time()) - self._start))
        return self._text

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<|endoftext|>


<commit_before>"""Handle gain."""
import logging
import subprocess
import sys
from collections import OrderedDict, defaultdict, namedtuple

from jinja2 import Template
from matrixscreener.experiment import attribute
from pkg_resources import resource_filename

from camacq.command import cam_com, gain_com
from camacq.const import (BLUE, COORD_FILE, END_10X, END_40X, END_63X,
                          FIELD_NAME, FIRST_JOB, FOV_NAME, GREEN, IMAGING_DIR,
                          INIT_GAIN, JOB_ID, LAST_WELL, RED, TEMPLATE_FILE,
                          WELL, WELL_NAME, YELLOW)
from camacq.helper import read_csv

_LOGGER = logging.getLogger(__name__)
# Make these constants configurable.

NA = 'NA'
TEN_X = '10x'
FORTY_X = '40x'
SIXTYTHREE_X = '63x'
CAM = 'CAM'
CHANNELS = [GREEN, BLUE, YELLOW, RED, ]
DETECTOR_MAP = {GREEN: '1', BLUE: '1', YELLOW: '2', RED: '2', }
DX_PX = 'dxPx'
DY_PX = 'dyPx'
FOV = 'fov'
GAIN_DEFAULT = {
    TEN_X: {GREEN: 1000, BLUE: 985, YELLOW: 805, RED: 705},
    FORTY_X: {GREEN: 800, BLUE: 585, YELLOW: 655, RED: 630},
    SIXTYTHREE_X: {GREEN: 800, BLUE: 505, YELLOW: 655, RED: 630},
}
GAIN_OFFSET_BLUE = 0
GAIN_OFFSET_RED = 25
GAIN_SCAN = 'gain_scan'
GAIN_FROM_WELL = 'gain_from_well'
JOB_MAP = {GREEN: 0, BLUE: 1, YELLOW: 1, RED: 2, }
TEMPLATE = 'template'

def process_output(well, output, well_map):
    """Process output from the R script."""
    channels = output.split()
    for index, gain in enumerate(channels):
        well_map[well].update({CHANNELS[index]: int(gain)})
    return well_map

def get_gain_com(commands, channels, job_list):
    """Return a list of command lists to set gain for all channels."""
    for channel, gain in channels.iteritems():
        job = job_list[JOB_MAP[channel]]
        detector = DETECTOR_MAP[channel]
        gain = str(gain.gain)
        commands.append(gain_com(exp=job, num=detector, value=gain))
    return commands

class Channel(object):
    """Represent a channel with gain.

    Parameters
    ----------
    channel : str
        Name of the channel.
    gain : int
        Gain value.

    Attributes
    ----------
    channel : str
        Return name of the channel.
    """

    # pylint: disable=too-few-public-methods

    __slots__ = ['channel', '_gain', ]

    def __init__(self, channel, gain):
        """Set up instance."""
        self.channel = channel
        self._gain = gain

    def __repr__(self):
        """Return the representation."""
        return "<Channel {}: gain {}>".format(self.channel, self._gain)

    @property
    def gain(self):
        """:int: Return gain value.

        :setter: Set the gain value and convert to int.
        """
        return self._gain

    @gain.setter
    def gain(self, value):
        """Set gain."""
        self._gain = int(value)

class Field(namedtuple('Field', 'X Y dX dY gain_field img_ok')):
    """Represent a field.

    Parameters
    ----------
    X : int
        Coordinate of field in X.
    Y : int
        Coordinate of field in Y.
    dX : int
        Pixel coordinate of region of interest within image field in X.
    dY : int
        Pixel coordinate of region of interest within image field in Y.
    gain_field : bool
        True if field should run gain selection analysis.
    img_ok : bool
        True if field has acquired an ok image.
    """

class Well(object):
    """Represent a well with fields and gain.

    Parameters
    ----------
    name : str
        Name of the well in format 'U00-V00'.

    Attributes
    ----------
    U : int
        Number showing the U coordinate of the well, from 0.
    V : int
        Number showing the V coordinate of the well, from 0.
    channels : dict
        Dict where keys are color channels and values are Gain instances.
    """

    # pylint: disable=too-few-public-methods

    def __init__(self, name):
        """Set up instance."""
        # pylint: disable=invalid-name
        self.U = attribute('--{}'.format(name), 'U')
        self.V = attribute('--{}'.format(name), 'V')
        self._field = Field(0, 0, 0, 0, False, False)
        self._fields = OrderedDict()
        self.channels = {}

    def __repr__(self):
        """Return the representation."""
        return "<Well {}: channels {}>".format(
            WELL_NAME.format(int(self.U), int(self.V)), self.channels)

    def add_field(
            self, xcoord, ycoord, dxpx=0, dypx=0,
            gain_field=False, img_ok=False):
        """Add a field to the well."""
        # pylint: disable=too-many-arguments
        self._fields.update({
            FIELD_NAME.format(xcoord, ycoord):
            self._field._make(
                (xcoord, ycoord, dxpx, dypx, gain_field, img_ok))})

    @property
    def fields(self):  # noqa D301, D207
        """:dict: Return a dict of field coordinates as named tuples.

        :setter: Sets the coordinates of multiple fields.
            Should be a sequence or iterable of tuples or lists.

        Example
        -------
        ::

            >>> well = Well('U00--V00')
            >>> well.fields = [[1, 3, 0, 1, True, False], ]
            >>> well.fields
            {'X01--Y03': Field(X=1, Y=3, dX=0, dY=1, \
gain_field=True, img_ok=False)}
        """
        return self._fields

    @fields.setter
    def fields(self, fields):
        """Set the fields."""
        self._fields = {
            FIELD_NAME.format(field[0], field[1]): self._field._make(field)
            for field in fields}

    @property
    def img_ok(self):
        """:bool: Return True if all fields of the well are imaged ok."""
        return all(field.img_ok for field in self.fields.values())

class GainMap(object):
    """
    Contain the information and methods to calculate gain for each well.

    Parameters
    ----------
    config : dict
        Dict with configuration key value pairs.
    job_info : tuple
        Tuple of job_list, pattern_g, and pattern, which will be attributes.

    Attributes
    ----------
    config : dict
        Dict with configuration key value pairs.
    job_list : list
        List of names of the jobs for the objective and experiment.
    pattern_g : str
        Name of the pattern for the gain job.
    pattern : str
        Name of the pattern for the experiment job.
    template : collections.defaultdict
        A defaultdict of dicts that maps wells and gain analysis wells,
        where to run the experiment.
    coords : collections.defaultdict
        A defaultdict of dicts that map wells and selected positions with
        pixel coordinates. The coordinates represent where to acquire the
        images in the wells, relative to the center of each field.
        The pixel coordinates have the same size as the chosen objective.
    wells: dict
        A dict where the keys are the wells and each value is Well object.
    """

    def __init__(self, config, job_info):
        """Set up instance."""
        self.config = config
        self.job_list, self.pattern_g, self.pattern = job_info
        if config[TEMPLATE_FILE] is None:
            self.template = None
        else:
            self.template = read_csv(config[TEMPLATE_FILE], WELL)
            self.config[LAST_WELL] = sorted(self.template.keys())[-1]
        if config[COORD_FILE] is None:
            self.coords = defaultdict(dict)
        else:
            self.coords = read_csv(config[COORD_FILE], FOV)
        self.wells = {}

    def __repr__(self):
        """Return the representation."""
        return "<Wells {}>".format(self.wells)

    def calc_gain(self, bases, wells):
        """Run R scripts and calculate gain values for the wells."""
        # Get a unique set of filebases from the csv paths.
        gain_dict = defaultdict(dict)
        filebases = sorted(set(bases))
        # Get a unique set of names of the experiment wells.
        fin_wells = sorted(set(wells))
        r_script = resource_filename(__name__, 'data/gain.r')
        if self.config[END_10X]:
            init_gain = resource_filename(__name__, 'data/10x_gain.csv')
        elif self.config[END_40X]:
            init_gain = resource_filename(__name__, 'data/40x_gain.csv')
        elif self.config[END_63X]:
            init_gain = resource_filename(__name__, 'data/63x_gain.csv')
        if self.config[INIT_GAIN]:
            init_gain = self.config[INIT_GAIN]
        for fbase, well in zip(filebases, fin_wells):
            _LOGGER.info('WELL: %s', well)
            try:
                _LOGGER.info('Starting R...')
                r_output = subprocess.check_output(['Rscript',
                                                    r_script,
                                                    self.config[IMAGING_DIR],
                                                    fbase,
                                                    init_gain])
                gain_dict = process_output(well, r_output, gain_dict)
            except OSError as exc:
                _LOGGER.error('Execution failed: %s', exc)
                sys.exit()
            except subprocess.CalledProcessError as exc:
                _LOGGER.error(
                    'Subprocess returned a non-zero exit status: %s', exc)
                sys.exit()
            _LOGGER.debug(r_output)
        return gain_dict

    def sanitize_gain(self, channel, gain):
        """Make sure all channels have a reasonable gain value."""
        if self.config[END_10X]:
            obj = TEN_X
        elif self.config[END_40X]:
            obj = FORTY_X
        elif self.config[END_63X]:
            obj = SIXTYTHREE_X
        if gain == NA:
            gain = GAIN_DEFAULT[obj][channel]
        if channel == GREEN:
            # Round gain values to multiples of 10 in green channel
            if self.config[END_63X]:
                gain = int(min(
                    round(int(gain), -1), GAIN_DEFAULT[SIXTYTHREE_X][GREEN]))
            else:
                gain = int(round(int(gain), -1))
        # Add gain offset to blue and red channel.
        if channel == BLUE:
            gain += GAIN_OFFSET_BLUE
        if channel == RED:
            gain += GAIN_OFFSET_RED
        return gain

    def set_gain(self, well, channel, gain):
        """Set gain in a channel in a well.

        Create a Well instance if well not already exists.
        """
        if well not in self.wells:
            self.wells[well] = Well(well)
        self.wells[well].channels.update({channel: Channel(channel, gain)})

    def distribute_gain(self, gain_dict):
        """Collate gain values and distribute them to the wells."""
        for gain_well, channels in gain_dict.iteritems():
            for channel, gain in channels.iteritems():
                gain = self.sanitize_gain(channel, gain)
                if self.template:
                    # Add template class with functions and options, later.
                    tmpl = Template(self.template[gain_well][TEMPLATE])
                    try:
                        gain = int(tmpl.render(gain=gain))
                    except ValueError:
                        pass
                    wells = [
                        well for well, settings in self.template.iteritems()
                        if settings[GAIN_FROM_WELL] == gain_well]
                    for well in wells:
                        self.set_gain(well, channel, gain)
                else:
                    self.set_gain(gain_well, channel, gain)

    def set_fields(self, well, fields_x, fields_y):
        """Set fields."""
        for i in range(fields_y):
            for j in range(fields_x):
                # Only add selected fovs from file (arg) to cam list
                fov = FOV_NAME.format(well.U, well.V, j, i)
                if fov in self.coords.keys():
                    dxcoord = self.coords[fov][DX_PX]
                    dycoord = self.coords[fov][DY_PX]
                    well.add_field(
                        j, i, dxcoord, dycoord,
                        j == 0 and i == 0 or j == 1 and i == 1)
                elif not self.coords:
                    well.add_field(
                        j, i, 0, 0, j == 0 and i == 0 or j == 1 and i == 1)

    def get_com(self, fields_x, fields_y):
        """Get command."""
        # Lists for storing command strings.
        com_list = []
        end_com_list = []
        # FIXME: CHECK GAINS AND RUN JOBS SMART  pylint: disable=fixme
        # Ie use one job for multiple wells where the gain is the same
        # or similar.
        for well in self.wells.values():
            self.set_fields(well, fields_x, fields_y)
            if well.img_ok:
                # Only get commands for wells that are not imaged ok.
                continue
            end_com = []
            com = get_gain_com([], well.channels, self.job_list)
            for field in well.fields.values():
                com.append(cam_com(
                    self.pattern, well.U, well.V, field.X, field.Y, field.dX,
                    field.dY))
                end_com = [
                    CAM, WELL_NAME.format(well.U, well.V),
                    JOB_ID.format(self.config[FIRST_JOB] + 2),
                    FIELD_NAME.format(field.X, field.Y)]
            # Store the commands in lists.
            com_list.append(com)
            end_com_list.append(end_com)
        return {'com': com_list, 'end_com': end_com_list}

    def get_init_com(self):
        """Get command for gain analysis."""
        wells = []
        if self.template:
            # Selected wells from template file.
            for well, row in self.template.iteritems():
                if 'true' in row[GAIN_SCAN]:
                    wells.append(well)
        else:
            # All wells.
            for ucoord in range(
                    attribute('--{}'.format(
                        self.config[LAST_WELL]), 'U') + 1):
                for vcoord in range(attribute(
                        '--{}'.format(self.config[LAST_WELL]), 'V') + 1):
                    wells.append(WELL_NAME.format(ucoord, vcoord))
        # Lists and strings for storing command strings.
        com_list = []
        end_com_list = []
        end_com = []
        # Selected objective gain job cam command in wells.
        for well in sorted(wells):
            com = []
            well_u_id = attribute('--{}'.format(well), 'U')
            well_v_id = attribute('--{}'.format(well), 'V')
            for i in range(2):
                com.append(cam_com(
                    self.pattern_g, well_u_id, well_v_id, i, i, '0', '0'))
                end_com = [
                    CAM, well, JOB_ID.format(2), FIELD_NAME.format(i, i)]
            com_list.append(com)
            end_com_list.append(end_com)

        # Join the list of lists of command lists into a list of a command
        # list if dry a objective is used.
        if self.config[END_10X] or self.config[END_40X]:
            com_list_bak = list(com_list)
            com_list = []
            for com in com_list_bak:
                com_list.extend(com)
            com_list = [com_list]
            end_com_list = [end_com_list[-1]]
        return {'com': com_list, 'end_com': end_com_list}
<commit_msg>Fix template rendering (#19)<commit_after>"""Handle gain."""
import logging
import subprocess
import sys
from collections import OrderedDict, defaultdict, namedtuple

from jinja2 import Template
from matrixscreener.experiment import attribute
from pkg_resources import resource_filename

from camacq.command import cam_com, gain_com
from camacq.const import (BLUE, COORD_FILE, END_10X, END_40X, END_63X,
                          FIELD_NAME, FIRST_JOB, FOV_NAME, GREEN, IMAGING_DIR,
                          INIT_GAIN, JOB_ID, LAST_WELL, RED, TEMPLATE_FILE,
                          WELL, WELL_NAME, YELLOW)
from camacq.helper import read_csv

_LOGGER = logging.getLogger(__name__)
# Make these constants configurable.

NA = 'NA'
TEN_X = '10x'
FORTY_X = '40x'
SIXTYTHREE_X = '63x'
CAM = 'CAM'
CHANNELS = [GREEN, BLUE, YELLOW, RED, ]
DETECTOR_MAP = {GREEN: '1', BLUE: '1', YELLOW: '2', RED: '2', }
DX_PX = 'dxPx'
DY_PX = 'dyPx'
FOV = 'fov'
GAIN_DEFAULT = {
    TEN_X: {GREEN: 1000, BLUE: 985, YELLOW: 805, RED: 705},
    FORTY_X: {GREEN: 800, BLUE: 585, YELLOW: 655, RED: 630},
    SIXTYTHREE_X: {GREEN: 800, BLUE: 505, YELLOW: 655, RED: 630},
}
GAIN_OFFSET_BLUE = 0
GAIN_OFFSET_RED = 25
GAIN_SCAN = 'gain_scan'
GAIN_FROM_WELL = 'gain_from_well'
JOB_MAP = {GREEN: 0, BLUE: 1, YELLOW: 1, RED: 2, }
TEMPLATE = 'template'

def process_output(well, output, well_map):
    """Process output from the R script."""
    channels = output.split()
    for index, gain in enumerate(channels):
        well_map[well].update({CHANNELS[index]: int(gain)})
    return well_map

def get_gain_com(commands, channels, job_list):
    """Return a list of command lists to set gain for all channels."""
    for channel, gain in channels.iteritems():
        job = job_list[JOB_MAP[channel]]
        detector = DETECTOR_MAP[channel]
        gain = str(gain.gain)
        commands.append(gain_com(exp=job, num=detector, value=gain))
    return commands

class Channel(object):
    """Represent a channel with gain.

    Parameters
    ----------
    channel : str
        Name of the channel.
    gain : int
        Gain value.

    Attributes
    ----------
    channel : str
        Return name of the channel.
    """

    # pylint: disable=too-few-public-methods

    __slots__ = ['channel', '_gain', ]

    def __init__(self, channel, gain):
        """Set up instance."""
        self.channel = channel
        self._gain = gain

    def __repr__(self):
        """Return the representation."""
        return "<Channel {}: gain {}>".format(self.channel, self._gain)

    @property
    def gain(self):
        """:int: Return gain value.

        :setter: Set the gain value and convert to int.
        """
        return self._gain

    @gain.setter
    def gain(self, value):
        """Set gain."""
        self._gain = int(value)

class Field(namedtuple('Field', 'X Y dX dY gain_field img_ok')):
    """Represent a field.

    Parameters
    ----------
    X : int
        Coordinate of field in X.
    Y : int
        Coordinate of field in Y.
    dX : int
        Pixel coordinate of region of interest within image field in X.
    dY : int
        Pixel coordinate of region of interest within image field in Y.
    gain_field : bool
        True if field should run gain selection analysis.
    img_ok : bool
        True if field has acquired an ok image.
    """

class Well(object):
    """Represent a well with fields and gain.

    Parameters
    ----------
    name : str
        Name of the well in format 'U00-V00'.

    Attributes
    ----------
    U : int
        Number showing the U coordinate of the well, from 0.
    V : int
        Number showing the V coordinate of the well, from 0.
    channels : dict
        Dict where keys are color channels and values are Gain instances.
    """

    # pylint: disable=too-few-public-methods

    def __init__(self, name):
        """Set up instance."""
        # pylint: disable=invalid-name
        self.U = attribute('--{}'.format(name), 'U')
        self.V = attribute('--{}'.format(name), 'V')
        self._field = Field(0, 0, 0, 0, False, False)
        self._fields = OrderedDict()
        self.channels = {}

    def __repr__(self):
        """Return the representation."""
        return "<Well {}: channels {}>".format(
            WELL_NAME.format(int(self.U), int(self.V)), self.channels)

    def add_field(
            self, xcoord, ycoord, dxpx=0, dypx=0,
            gain_field=False, img_ok=False):
        """Add a field to the well."""
        # pylint: disable=too-many-arguments
        self._fields.update({
            FIELD_NAME.format(xcoord, ycoord):
            self._field._make(
                (xcoord, ycoord, dxpx, dypx, gain_field, img_ok))})

    @property
    def fields(self):  # noqa D301, D207
        """:dict: Return a dict of field coordinates as named tuples.

        :setter: Sets the coordinates of multiple fields.
            Should be a sequence or iterable of tuples or lists.

        Example
        -------
        ::

            >>> well = Well('U00--V00')
            >>> well.fields = [[1, 3, 0, 1, True, False], ]
            >>> well.fields
            {'X01--Y03': Field(X=1, Y=3, dX=0, dY=1, \
gain_field=True, img_ok=False)}
        """
        return self._fields

    @fields.setter
    def fields(self, fields):
        """Set the fields."""
        self._fields = {
            FIELD_NAME.format(field[0], field[1]): self._field._make(field)
            for field in fields}

    @property
    def img_ok(self):
        """:bool: Return True if all fields of the well are imaged ok."""
        return all(field.img_ok for field in self.fields.values())

class GainMap(object):
    """
    Contain the information and methods to calculate gain for each well.

    Parameters
    ----------
    config : dict
        Dict with configuration key value pairs.
    job_info : tuple
        Tuple of job_list, pattern_g, and pattern, which will be attributes.

    Attributes
    ----------
    config : dict
        Dict with configuration key value pairs.
    job_list : list
        List of names of the jobs for the objective and experiment.
    pattern_g : str
        Name of the pattern for the gain job.
    pattern : str
        Name of the pattern for the experiment job.
    template : collections.defaultdict
        A defaultdict of dicts that maps wells and gain analysis wells,
        where to run the experiment.
    coords : collections.defaultdict
        A defaultdict of dicts that map wells and selected positions with
        pixel coordinates. The coordinates represent where to acquire the
        images in the wells, relative to the center of each field.
        The pixel coordinates have the same size as the chosen objective.
    wells: dict
        A dict where the keys are the wells and each value is Well object.
    """

    def __init__(self, config, job_info):
        """Set up instance."""
        self.config = config
        self.job_list, self.pattern_g, self.pattern = job_info
        if config[TEMPLATE_FILE] is None:
            self.template = None
        else:
            self.template = read_csv(config[TEMPLATE_FILE], WELL)
            self.config[LAST_WELL] = sorted(self.template.keys())[-1]
        if config[COORD_FILE] is None:
            self.coords = defaultdict(dict)
        else:
            self.coords = read_csv(config[COORD_FILE], FOV)
        self.wells = {}

    def __repr__(self):
        """Return the representation."""
        return "<Wells {}>".format(self.wells)

    def calc_gain(self, bases, wells):
        """Run R scripts and calculate gain values for the wells."""
        # Get a unique set of filebases from the csv paths.
        gain_dict = defaultdict(dict)
        filebases = sorted(set(bases))
        # Get a unique set of names of the experiment wells.
        fin_wells = sorted(set(wells))
        r_script = resource_filename(__name__, 'data/gain.r')
        if self.config[END_10X]:
            init_gain = resource_filename(__name__, 'data/10x_gain.csv')
        elif self.config[END_40X]:
            init_gain = resource_filename(__name__, 'data/40x_gain.csv')
        elif self.config[END_63X]:
            init_gain = resource_filename(__name__, 'data/63x_gain.csv')
        if self.config[INIT_GAIN]:
            init_gain = self.config[INIT_GAIN]
        for fbase, well in zip(filebases, fin_wells):
            _LOGGER.info('WELL: %s', well)
            try:
                _LOGGER.info('Starting R...')
                r_output = subprocess.check_output(['Rscript',
                                                    r_script,
                                                    self.config[IMAGING_DIR],
                                                    fbase,
                                                    init_gain])
                gain_dict = process_output(well, r_output, gain_dict)
            except OSError as exc:
                _LOGGER.error('Execution failed: %s', exc)
                sys.exit()
            except subprocess.CalledProcessError as exc:
                _LOGGER.error(
                    'Subprocess returned a non-zero exit status: %s', exc)
                sys.exit()
            _LOGGER.debug(r_output)
        return gain_dict

    def sanitize_gain(self, channel, gain):
        """Make sure all channels have a reasonable gain value."""
        if self.config[END_10X]:
            obj = TEN_X
        elif self.config[END_40X]:
            obj = FORTY_X
        elif self.config[END_63X]:
            obj = SIXTYTHREE_X
        if gain == NA:
            gain = GAIN_DEFAULT[obj][channel]
        if channel == GREEN:
            # Round gain values to multiples of 10 in green channel
            if self.config[END_63X]:
                gain = int(min(
                    round(int(gain), -1), GAIN_DEFAULT[SIXTYTHREE_X][GREEN]))
            else:
                gain = int(round(int(gain), -1))
        # Add gain offset to blue and red channel.
        if channel == BLUE:
            gain += GAIN_OFFSET_BLUE
        if channel == RED:
            gain += GAIN_OFFSET_RED
        return gain

    def set_gain(self, well, channel, gain):
        """Set gain in a channel in a well.

        Create a Well instance if well not already exists.
        """
        if well not in self.wells:
            self.wells[well] = Well(well)
        self.wells[well].channels.update({channel: Channel(channel, gain)})

    def distribute_gain(self, gain_dict):
        """Collate gain values and distribute them to the wells."""
        for gain_well, channels in gain_dict.iteritems():
            for channel, gain in channels.iteritems():
                gain = self.sanitize_gain(channel, gain)
                if self.template:
                    # Add template class with functions and options, later.
                    tmpl = self.template[gain_well][TEMPLATE]
                    tmpl = Template(tmpl) if tmpl else Template('{{ gain }}')
                    try:
                        gain = int(tmpl.render(gain=gain))
                    except ValueError:
                        _LOGGER.error('Failed to render template')
                    wells = [
                        well for well, settings in self.template.iteritems()
                        if settings[GAIN_FROM_WELL] == gain_well]
                    for well in wells:
                        self.set_gain(well, channel, gain)
                else:
                    self.set_gain(gain_well, channel, gain)

    def set_fields(self, well, fields_x, fields_y):
        """Set fields."""
        for i in range(fields_y):
            for j in range(fields_x):
                # Only add selected fovs from file (arg) to cam list
                fov = FOV_NAME.format(well.U, well.V, j, i)
                if fov in self.coords.keys():
                    dxcoord = self.coords[fov][DX_PX]
                    dycoord = self.coords[fov][DY_PX]
                    well.add_field(
                        j, i, dxcoord, dycoord,
                        j == 0 and i == 0 or j == 1 and i == 1)
                elif not self.coords:
                    well.add_field(
                        j, i, 0, 0, j == 0 and i == 0 or j == 1 and i == 1)

    def get_com(self, fields_x, fields_y):
        """Get command."""
        # Lists for storing command strings.
        com_list = []
        end_com_list = []
        # FIXME: CHECK GAINS AND RUN JOBS SMART  pylint: disable=fixme
        # Ie use one job for multiple wells where the gain is the same
        # or similar.
        for well in self.wells.values():
            self.set_fields(well, fields_x, fields_y)
            if well.img_ok:
                # Only get commands for wells that are not imaged ok.
                continue
            end_com = []
            com = get_gain_com([], well.channels, self.job_list)
            for field in well.fields.values():
                com.append(cam_com(
                    self.pattern, well.U, well.V, field.X, field.Y, field.dX,
                    field.dY))
                end_com = [
                    CAM, WELL_NAME.format(well.U, well.V),
                    JOB_ID.format(self.config[FIRST_JOB] + 2),
                    FIELD_NAME.format(field.X, field.Y)]
            # Store the commands in lists.
            com_list.append(com)
            end_com_list.append(end_com)
        return {'com': com_list, 'end_com': end_com_list}

    def get_init_com(self):
        """Get command for gain analysis."""
        wells = []
        if self.template:
            # Selected wells from template file.
            for well, row in self.template.iteritems():
                if 'true' in row[GAIN_SCAN]:
                    wells.append(well)
        else:
            # All wells.
            for ucoord in range(
                    attribute('--{}'.format(
                        self.config[LAST_WELL]), 'U') + 1):
                for vcoord in range(attribute(
                        '--{}'.format(self.config[LAST_WELL]), 'V') + 1):
                    wells.append(WELL_NAME.format(ucoord, vcoord))
        # Lists and strings for storing command strings.
        com_list = []
        end_com_list = []
        end_com = []
        # Selected objective gain job cam command in wells.
        for well in sorted(wells):
            com = []
            well_u_id = attribute('--{}'.format(well), 'U')
            well_v_id = attribute('--{}'.format(well), 'V')
            for i in range(2):
                com.append(cam_com(
                    self.pattern_g, well_u_id, well_v_id, i, i, '0', '0'))
                end_com = [
                    CAM, well, JOB_ID.format(2), FIELD_NAME.format(i, i)]
            com_list.append(com)
            end_com_list.append(end_com)

        # Join the list of lists of command lists into a list of a command
        # list if dry a objective is used.
        if self.config[END_10X] or self.config[END_40X]:
            com_list_bak = list(com_list)
            com_list = []
            for com in com_list_bak:
                com_list.extend(com)
            com_list = [com_list]
            end_com_list = [end_com_list[-1]]
        return {'com': com_list, 'end_com': end_com_list}
<|endoftext|>


<commit_before>import os
import shutil
import warnings
from functools import wraps
from . import urls

# Make sure that DeprecationWarning gets printed
warnings.simplefilter("always", DeprecationWarning)

def get_id(obj, suffix=None):
    """Get a unique id for the object"""
    objid = str(os.getpid()) + str(id(obj))
    if suffix:
        objid += str(suffix)
    return objid

def deprecated(func, old_name, new_name):
    """Decorator to mark functions as deprecated."""
    @wraps(func)
    def new_func(*args, **kwargs):
        warnings.warn(("{0} is deprecated and will be removed.  "
                       "Use {1} instead".format(old_name, new_name)),
                      category=DeprecationWarning)
        return func(*args, **kwargs)
    new_func.__doc__ = ("*%s is deprecated: use %s instead*\n\n    "
                        % (old_name, new_name)) + new_func.__doc__
    return new_func

def write_js_libs(location=None, d3_src=None, mpld3_src=None):
    """
    Write the mpld3 and d3 javascript libraries to the given file location.

    This utility is used by the IPython notebook tools to enable easy use
    of mpld3 with no web connection.

    Parameters
    ----------
    location : string (optioal)
        the directory in which the d3 and mpld3 javascript libraries will be
        written. If not specified, the IPython nbextensions directory will be
        used. If IPython doesn't support nbextensions (< 2.0),
        the current working directory will be used.
    d3_src : string (optional)
        the source location of the d3 library. If not specified, the standard
        path in mpld3.urls.D3_LOCAL will be used.
    mpld3_src : string (optional)
        the source location of the mpld3 library. If not specified, the
        standard path in mpld3.urls.MPLD3_LOCAL will be used.

    Returns
    -------
    d3_url, mpld3_url : string
        The URLs to be used for loading these js files.
    """
    if location is None:
        try:
            from IPython.html import install_nbextension
        except ImportError:
            location = os.getcwd()
            nbextension = False
        else:
            nbextension = True
    else:
        nbextension = False

    if d3_src is None:
        d3_src = urls.D3_LOCAL
    if mpld3_src is None:
        mpld3_src = urls.MPLD3_LOCAL

    d3js = os.path.basename(d3_src)
    mpld3js = os.path.basename(mpld3_src)

    if not os.path.exists(d3_src):
        raise ValueError("d3 src not found at '{0}'".format(d3_src))
    if not os.path.exists(mpld3_src):
        raise ValueError("mpld3 src not found at '{0}'".format(mpld3_src))

    if nbextension:
        install_nbextension([d3_src, mpld3_src])
        prefix = 'nbextensions/'
    else:
        # IPython < 2.0 or explicit path.
        # This won't work if users have changed the kernel directory.
        d3_dest = os.path.join(location, d3js)
        mpld3_dest = os.path.join(location, mpld3js)

        shutil.copyfile(d3_src, d3_dest)
        shutil.copyfile(mpld3_src, mpld3_dest)

        prefix = '/files/'

    return prefix + d3js, prefix + mpld3js

<commit_msg>BUG: fix invalid HTML4 ids<commit_after>import os
import re
import shutil
import warnings
from functools import wraps
from . import urls

# Make sure that DeprecationWarning gets printed
warnings.simplefilter("always", DeprecationWarning)

def html_id_ok(objid, html5=False):
    """Check whether objid is valid as an HTML id attribute.

    If html5 == True, then use the more liberal html5 rules.
    """
    if html5:
        return not re.search('\s', objid)
    else:
        return bool(re.match("^[a-zA-Z][a-zA-Z0-9\-\.\:\_]*$", objid))

def get_id(obj, suffix="", prefix="el", warn_on_invalid=True):
    """Get a unique id for the object"""
    if not suffix:
        suffix = ""
    if not prefix:
        prefix = ""

    objid = prefix + str(os.getpid()) + str(id(obj)) + suffix

    if warn_on_invalid and not html_id_ok(objid):
        warnings.warn('"{0}" is not a valid html ID. This may cause problems')

    return objid

def deprecated(func, old_name, new_name):
    """Decorator to mark functions as deprecated."""
    @wraps(func)
    def new_func(*args, **kwargs):
        warnings.warn(("{0} is deprecated and will be removed.  "
                       "Use {1} instead".format(old_name, new_name)),
                      category=DeprecationWarning)
        return func(*args, **kwargs)
    new_func.__doc__ = ("*%s is deprecated: use %s instead*\n\n    "
                        % (old_name, new_name)) + new_func.__doc__
    return new_func

def write_js_libs(location=None, d3_src=None, mpld3_src=None):
    """
    Write the mpld3 and d3 javascript libraries to the given file location.

    This utility is used by the IPython notebook tools to enable easy use
    of mpld3 with no web connection.

    Parameters
    ----------
    location : string (optioal)
        the directory in which the d3 and mpld3 javascript libraries will be
        written. If not specified, the IPython nbextensions directory will be
        used. If IPython doesn't support nbextensions (< 2.0),
        the current working directory will be used.
    d3_src : string (optional)
        the source location of the d3 library. If not specified, the standard
        path in mpld3.urls.D3_LOCAL will be used.
    mpld3_src : string (optional)
        the source location of the mpld3 library. If not specified, the
        standard path in mpld3.urls.MPLD3_LOCAL will be used.

    Returns
    -------
    d3_url, mpld3_url : string
        The URLs to be used for loading these js files.
    """
    if location is None:
        try:
            from IPython.html import install_nbextension
        except ImportError:
            location = os.getcwd()
            nbextension = False
        else:
            nbextension = True
    else:
        nbextension = False

    if d3_src is None:
        d3_src = urls.D3_LOCAL
    if mpld3_src is None:
        mpld3_src = urls.MPLD3_LOCAL

    d3js = os.path.basename(d3_src)
    mpld3js = os.path.basename(mpld3_src)

    if not os.path.exists(d3_src):
        raise ValueError("d3 src not found at '{0}'".format(d3_src))
    if not os.path.exists(mpld3_src):
        raise ValueError("mpld3 src not found at '{0}'".format(mpld3_src))

    if nbextension:
        install_nbextension([d3_src, mpld3_src])
        prefix = 'nbextensions/'
    else:
        # IPython < 2.0 or explicit path.
        # This won't work if users have changed the kernel directory.
        d3_dest = os.path.join(location, d3js)
        mpld3_dest = os.path.join(location, mpld3js)

        shutil.copyfile(d3_src, d3_dest)
        shutil.copyfile(mpld3_src, mpld3_dest)

        prefix = '/files/'

    return prefix + d3js, prefix + mpld3js
<|endoftext|>


<commit_before>import random
import time
import sys

import paho.mqtt.client as mqtt
from locust import Locust
from locust import task
from locust import TaskSet
from locust import events

def time_delta(t1, t2):
    return int((t2 - t1) * 1000)

def fire_locust_failure(**kwargs):
    events.request_failure.fire(**kwargs)

def fire_locust_success(**kwargs):
    events.request_success.fire(**kwargs)

class LocustError(Exception):
    pass

class TimeoutError(ValueError):
    pass

class Message(object):

    def __init__(self, topic, payload, start_time, timeout):
        self.topic = topic
        self.payload = payload
        self.start_time = start_time
        self.timeout = timeout

    def timed_out(self, total_time):
        return self.timeout is not None and total_time > self.timeout

class MQTTClient(mqtt.Client):

    def __init__(self, *args, **kwargs):
        super(MQTTClient, self).__init__(*args, **kwargs)
        self.on_publish = self._on_publish

    def publish(self, topic, payload=None, repeat=1, **kwargs):
        timeout = kwargs.pop('timeout', 5)
        if not hasattr(self, 'mmap'):
            self.mmap = {}
        for i in range(repeat):
            start_time = time.time()
            try:
                err, mid = super(MQTTClient, self).publish(
                    topic,
                    payload=payload,
                    **kwargs
                )
                if err:
                    raise ValueError(err)
                self.mmap[mid] = Message(topic, payload, start_time, timeout)
            except Exception as e:
                total_time = time_delta(start_time, time.time())
                fire_locust_failure(
                    request_type='mqtt',
                    name='publish',
                    response_time=total_time,
                    exception=e,
                )

    def _on_publish(self, client, userdata, mid):
        end_time = time.time()
        message = self.mmap.pop(mid, None)
        if message is None:
            return
        total_time = end_time - message.start_time
        if message.timed_out(total_time):
            fire_locust_failure(
                request_type='mqtt',
                name='publish',
                response_time=total_time,
                exception=TimeoutError("publish timed out"),
            )
        else:
            fire_locust_success(
                request_type='mqtt',
                name='publish',
                response_time=total_time,
                response_length=len(message.payload),
            )
        self.check_for_locust_timeouts(end_time)

    def check_for_locust_timeouts(self, end_time):
        timed_out = [mid for mid, msg in dict(self.mmap).iteritems()
                     if msg.timed_out(end_time - msg.start_time)]
        for mid in timed_out:
            msg = self.mmap.pop(mid)
            total_time = end_time - msg.start_time
            print(total_time)
            fire_locust_failure(
                request_type='mqtt',
                name='publish',
                response_time=total_time,
                exception=TimeoutError(
                    "message not received in %s s" % msg.timeout
                    ),
            )

class MQTTLocust(Locust):

    def __init__(self, *args, **kwargs):
        super(Locust, self).__init__(*args, **kwargs)
        if self.host is None:
            raise LocustError("You must specify a host")
        self.client = MQTTClient()
        try:
            [host, port] = self.host.split(":")
        except:
            host, port = self.host, 1883
        self.client.connect(host, port=port)
        self.client.loop_start()
<commit_msg>Another timing fix. Note that time.time() returns the current time in seconds (as a float), so we can just do math on it.  timedelta is useful for working with DateTime objects<commit_after>import random
import time
import sys

import paho.mqtt.client as mqtt
from locust import Locust
from locust import task
from locust import TaskSet
from locust import events

def time_delta(t1, t2):
    return int((t2 - t1) * 1000)

def fire_locust_failure(**kwargs):
    events.request_failure.fire(**kwargs)

def fire_locust_success(**kwargs):
    events.request_success.fire(**kwargs)

class LocustError(Exception):
    pass

class TimeoutError(ValueError):
    pass

class Message(object):

    def __init__(self, topic, payload, start_time, timeout):
        self.topic = topic
        self.payload = payload
        self.start_time = start_time
        self.timeout = timeout

    def timed_out(self, total_time):
        return self.timeout is not None and total_time > self.timeout

class MQTTClient(mqtt.Client):

    def __init__(self, *args, **kwargs):
        super(MQTTClient, self).__init__(*args, **kwargs)
        self.on_publish = self._on_publish

    def publish(self, topic, payload=None, repeat=1, **kwargs):
        timeout = kwargs.pop('timeout', 5)
        if not hasattr(self, 'mmap'):
            self.mmap = {}
        for i in range(repeat):
            start_time = time.time()
            try:
                err, mid = super(MQTTClient, self).publish(
                    topic,
                    payload=payload,
                    **kwargs
                )
                if err:
                    raise ValueError(err)
                self.mmap[mid] = Message(topic, payload, start_time, timeout)
            except Exception as e:
                total_time = time.time() - start_time
                fire_locust_failure(
                    request_type='mqtt',
                    name='publish',
                    response_time=total_time,
                    exception=e,
                )

    def _on_publish(self, client, userdata, mid):
        end_time = time.time()
        message = self.mmap.pop(mid, None)
        if message is None:
            return
        total_time = end_time - message.start_time
        if message.timed_out(total_time):
            fire_locust_failure(
                request_type='mqtt',
                name='publish',
                response_time=total_time,
                exception=TimeoutError("publish timed out"),
            )
        else:
            fire_locust_success(
                request_type='mqtt',
                name='publish',
                response_time=total_time,
                response_length=len(message.payload),
            )
        self.check_for_locust_timeouts(end_time)

    def check_for_locust_timeouts(self, end_time):
        timed_out = [mid for mid, msg in dict(self.mmap).iteritems()
                     if msg.timed_out(end_time - msg.start_time)]
        for mid in timed_out:
            msg = self.mmap.pop(mid)
            total_time = end_time - msg.start_time
            print(total_time)
            fire_locust_failure(
                request_type='mqtt',
                name='publish',
                response_time=total_time,
                exception=TimeoutError(
                    "message not received in %s s" % msg.timeout
                    ),
            )

class MQTTLocust(Locust):

    def __init__(self, *args, **kwargs):
        super(Locust, self).__init__(*args, **kwargs)
        if self.host is None:
            raise LocustError("You must specify a host")
        self.client = MQTTClient()
        try:
            [host, port] = self.host.split(":")
        except:
            host, port = self.host, 1883
        self.client.connect(host, port=port)
        self.client.loop_start()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import logging
import glob
import os
import os.path
import sys
import re
import importlib
import shlex
import subprocess
import datetime

import psycopg2
import psycopg2.extensions
import psycopg2.extras

import click

log = logging.getLogger('schematool')

DEFAULT_CONFIG_MODULE_NAME = 'mschematool_config'

MIGRATION_PATTERN_SQL = 'm*.sql'
MIGRATION_PATTERN_PY = 'm*.py'

### Loading configuration

class Config(object):

    def __init__(self, verbose, config_module=None):
        self.verbose = verbose
        self.config_module = config_module
        self._module = None

    def setup_pythonpath_for_migrations(self, dbnick):
        sys.path.append(self.module.DATABASES[dbnick]['migrations_dir'])

    def _setup_logging(self):
        global log
        log.setLevel(logging.DEBUG)
        formatter = logging.Formatter('%(asctime)-15s %(message)s')
        console_handler = logging.StreamHandler(sys.stderr)
        console_handler.setFormatter(formatter)
        console_handler.setLevel(logging.DEBUG)
        if self.verbose:
            log.addHandler(console_handler)
        if hasattr(self.module, 'LOG_FILE'):
            file_handler = logging.FileHandler(self.module.LOG_FILE)
            file_handler.setLevel(logging.DEBUG)
            file_handler.setFormatter(formatter)
            log.addHandler(file_handler)

    def _load_config(self):
        if self._module is not None:
            return

        module_from_env = os.getenv('MSCHEMATOOL_CONFIG_MODULE')
        if self.config_module is not None:
            module_name = self.config_module
        elif module_from_env:
            log.info('Importing config module from env. variable MSCHEMATOOL_CONFIG_MODULE: %s',
                    module_from_env)
            module_name = module_from_env
        else:
            log.info('Importing default config module %s', DEFAULT_CONFIG_MODULE_NAME)
            module_name = DEFAULT_CONFIG_MODULE_NAME

        try:
            self._module = importlib.import_module(module_name)
        except ImportError:
            msg = 'Cannot import mschematool config module'
            sys.stderr.write(msg + '\n')
            log.critical(msg)
            raise

        self._setup_logging()

    @property
    def module(self):
        self._load_config()
        return self._module

### Utility functions

def _simplify_whitespace(s):
    return ' '.join(s.split())

#### Database-independent interface for migration-related operations

class MigrationsBase(object):

    engine = 'unknown'

    def __init__(self, db_config): raise NotImplementedError()

    def initialize_db(self): raise NotImplementedError()

    def fetch_executed_migrations(self): raise NotImplementedError()

    def execute_python_migration(self, migration_file, module): raise NotImplementedError()

    def execute_sql_migration(self, migration_file): raise NotImplementedError()

    def execute_migration(self, migration_file_relative):
        migration_file = os.path.join(self.db_config['migrations_dir'], migration_file_relative)
        if migration_file.endswith('.sql'):
            return self.execute_sql_migration(migration_file)
        if migration_file.endswith('.py'):
            module = importlib.import_module(migration_file[:-len('.py')])
            return self.execute_python_migration(migration_file, module)
        assert False, 'Unknown migration type %s' % migration_file

## Postgres

class PostgresLoggingDictCursor(psycopg2.extras.DictCursor):
    """Log all SQL executed in the database.
    """

    def __init__(self, *args, **kwargs):
        psycopg2.extras.DictCursor.__init__(self, *args, **kwargs)

    def execute(self, sql, args=None):
        global log
        if log.isEnabledFor(logging.INFO):
            realsql = self.mogrify(sql, args)
            log.info('Executing SQL: <<%s>>', _simplify_whitespace(realsql))
        try:
            psycopg2.extras.DictCursor.execute(self, sql, args)
        except:
            log.exception('Exception while executing SQL')
            raise

class PostgresMigrations(MigrationsBase):

    engine = 'postgres'

    def __init__(self, db_config):
        self.db_config = db_config
        self.conn = psycopg2.connect(self.db_config['dsn'])

    def cursor(self):
        return self.conn.cursor(cursor_factory=PostgresLoggingDictCursor)

    def initialize_db(self):
        with self.cursor() as cur:
            cur.execute("""CREATE TABLE schemamigration (
                migration_file TEXT,
                execution_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )""")
            cur.connection.commit()

    def fetch_executed_migrations(self):
        with self.cursor() as cur:
            cur.execute("""SELECT migration_file FROM schemamigration
            ORDER BY execution_time""")
            return [row[0] for row in cur.fetchall()]

    def _migration_success(self, migration_file):
        with self.cursor() as cur:
            cur.execute("""INSERT INTO schemamigration (migration_file) VALUES (%s)""",
                    [migration_file])

    def execute_python_migration(self, migration_file, module):
        assert hasattr(module, 'do'), 'Python module must have `do` function accepting ' \
            'database connection'
        module.do(self.conn)
        self._migration_success(migration_file)
        self.conn.commit()

    # https://bitbucket.org/andrewgodwin/south/src/74742a1ba41ce6e9ea56cc694c824b7a93934ac6/south/db/generic.py?at=default
    def _sqlfile_to_statements(self, sql, regex=r"(?mx) ([^';]* (?:'[^']*'[^';]*)*)",
            comment_regex=r"(?mx) (?:^\s*$)|(?:--.*$)"):
        """
        Takes a SQL file and executes it as many separate statements.
        (Some backends, such as Postgres, don't work otherwise.)
        """
        # Be warned: This function is full of dark magic. Make sure you really
        # know regexes before trying to edit it.
        # First, strip comments
        sql = "\n".join([x.strip().replace("%", "%%") for x in re.split(comment_regex, sql) if x.strip()])
        # Now execute each statement
        return re.split(regex, sql)[1:][::2]

    def execute_sql_migration(self, migration_file):
        with open(migration_file) as f:
            content = f.read()
        for statement in self._sqlfile_to_statements(content):
            with self.cursor() as cur:
                cur.execute(statement)
        self._migration_success(migration_file)
        self.conn.commit()

#### Parsing options

MIGRATIONS_IMPLS = [PostgresMigrations]
ENGINE_TO_IMPL = {m.engine: m for m in MIGRATIONS_IMPLS}

class RunContext(object):

    def __init__(self, config, dbnick):
        assert dbnick in config.module.DATABASES, 'Not found in DATABASES in config: %s' % dbnick
        self.config = config
        self.dbnick = dbnick
        self.db_config = config.module.DATABASES[dbnick]
        self.migrations = ENGINE_TO_IMPL[self.db_config['engine']](self.db_config)
        config.setup_pythonpath_for_migrations(self.dbnick)

    def not_executed_migration_files(self):
        executed = self.migrations.fetch_executed_migrations()
        available = self.get_filenames()
        not_executed = set(available) - set(executed)
        not_executed = sorted(not_executed)
        return not_executed

    def _get_all_filenames(self, base_dir):
        filenames = glob.glob(os.path.join(base_dir, MIGRATION_PATTERN_SQL)) + \
            glob.glob(os.path.join(base_dir, MIGRATION_PATTERN_PY))
        filenames.sort()
        return filenames

    def get_filenames(self):
        return self._get_all_filenames(self.db_config['migrations_dir'])

    def execute_after_sync(self):
        after_sync = self.db_config.get('after_sync')
        if not after_sync:
            return
        msg = 'Executing after_sync command %r' % after_sync
        log.info(msg)
        click.echo(msg)
        os.system(after_sync)

#### Commands

HELP = """Example usage:

$ ./mschematool.py my_db initdb

A database nickname defined in the configuration module must be passed as the first argument.
After it a command must be specified.

"""

@click.group(help=HELP)
@click.option('--config', type=str, envvar='MSCHEMATOOL', help='Configuration module, e.g. "mypkg.mschematool_config". Environment variable MSCHEMATOOL_CONFIG can be specified instead.')
@click.option('--verbose/--no-verbose', default=False, help='Print executed SQL? Default: no.')
@click.argument('dbnick', type=str)
@click.pass_context
def main(ctx, config, verbose, dbnick):
    config_obj = Config(verbose, config)
    run_ctx = RunContext(config_obj, dbnick)
    ctx.obj = run_ctx

@main.command(help='Create tables used for tracking migrations.')
@click.pass_context
def init_db(ctx):
    ctx.obj.migrations.initialize_db()

@main.command(help='Show synced migrations.')
@click.pass_context
def synced(ctx):
    migrations = ctx.obj.migrations.fetch_executed_migrations()
    for migration in migrations:
        click.echo(migration)

@main.command(help='Show migrations available for syncing.')
@click.pass_context
def to_sync(ctx):
    migrations = ctx.obj.not_executed_migration_files()
    for migration in migrations:
        click.echo(migration)

@main.command(help='Sync all available migrations.')
@click.pass_context
def sync(ctx):
    to_execute = ctx.obj.not_executed_migration_files()
    if not to_execute:
        click.echo('No migrations to sync')
        return
    for migration_file in to_execute:
        msg = 'Executing migrations %s' % migration_file
        log.info(msg)
        click.echo(msg)
        ctx.obj.migrations.execute_migration(migration_file)
    ctx.obj.execute_after_sync()

@main.command(help='Sync a single migration, without syncing older ones.')
@click.argument('migration_file', type=str)
@click.pass_context
def force_sync_single(ctx, migration_file):
    msg = 'Force executing %s' % migration_file
    log.info(msg)
    click.echo(msg)
    ctx.obj.migrations.execute_migration(migration_file)
    ctx.obj.execute_after_sync()

@main.command(help='Print a filename for a new migration.')
@click.argument('name', type=str)
@click.pass_context
def print_new(ctx, name):
    '''Prints filename of a new migration'''
    print os.path.join(ctx.obj.db_config['migrations_dir'],
            'm{datestr}_{name}.sql'.format(
                datestr=datetime.datetime.now().strftime('%Y%m%d%H%M%S'),
                name=name.replace(' ', '_')))

@main.command(help='Show latest synced migration.')
@click.pass_context
def latest_synced(ctx):
    migrations = ctx.obj.migrations.fetch_executed_migrations()
    if not migrations:
        click.echo('No synced migrations')
    else:
        click.echo(executed[-1])

if __name__ == '__main__':
    main()

<commit_msg>Add CWD to sys.path<commit_after>#!/usr/bin/env python

import logging
import glob
import os
import os.path
import sys
import re
import importlib
import shlex
import subprocess
import datetime

import psycopg2
import psycopg2.extensions
import psycopg2.extras

import click

log = logging.getLogger('schematool')

DEFAULT_CONFIG_MODULE_NAME = 'mschematool_config'

MIGRATION_PATTERN_SQL = 'm*.sql'
MIGRATION_PATTERN_PY = 'm*.py'

# Look for a config module in cwd also
sys.path.append(os.getcwd())

### Loading configuration

class Config(object):

    def __init__(self, verbose, config_module=None):
        self.verbose = verbose
        self.config_module = config_module
        self._module = None

    def setup_pythonpath_for_migrations(self, dbnick):
        sys.path.append(self.module.DATABASES[dbnick]['migrations_dir'])

    def _setup_logging(self):
        global log
        log.setLevel(logging.DEBUG)
        formatter = logging.Formatter('%(asctime)-15s %(message)s')
        console_handler = logging.StreamHandler(sys.stderr)
        console_handler.setFormatter(formatter)
        console_handler.setLevel(logging.DEBUG)
        if self.verbose:
            log.addHandler(console_handler)
        if hasattr(self.module, 'LOG_FILE'):
            file_handler = logging.FileHandler(self.module.LOG_FILE)
            file_handler.setLevel(logging.DEBUG)
            file_handler.setFormatter(formatter)
            log.addHandler(file_handler)

    def _load_config(self):
        if self._module is not None:
            return

        module_from_env = os.getenv('MSCHEMATOOL_CONFIG_MODULE')
        if self.config_module is not None:
            module_name = self.config_module
        elif module_from_env:
            log.info('Importing config module from env. variable MSCHEMATOOL_CONFIG_MODULE: %s',
                    module_from_env)
            module_name = module_from_env
        else:
            log.info('Importing default config module %s', DEFAULT_CONFIG_MODULE_NAME)
            module_name = DEFAULT_CONFIG_MODULE_NAME

        try:
            self._module = importlib.import_module(module_name)
        except ImportError:
            msg = 'Cannot import mschematool config module'
            sys.stderr.write(msg + '\n')
            log.critical(msg)
            raise

        self._setup_logging()

    @property
    def module(self):
        self._load_config()
        return self._module

### Utility functions

def _simplify_whitespace(s):
    return ' '.join(s.split())

#### Database-independent interface for migration-related operations

class MigrationsBase(object):

    engine = 'unknown'

    def __init__(self, db_config): raise NotImplementedError()

    def initialize_db(self): raise NotImplementedError()

    def fetch_executed_migrations(self): raise NotImplementedError()

    def execute_python_migration(self, migration_file, module): raise NotImplementedError()

    def execute_sql_migration(self, migration_file): raise NotImplementedError()

    def execute_migration(self, migration_file_relative):
        migration_file = os.path.join(self.db_config['migrations_dir'], migration_file_relative)
        if migration_file.endswith('.sql'):
            return self.execute_sql_migration(migration_file)
        if migration_file.endswith('.py'):
            module = importlib.import_module(migration_file[:-len('.py')])
            return self.execute_python_migration(migration_file, module)
        assert False, 'Unknown migration type %s' % migration_file

## Postgres

class PostgresLoggingDictCursor(psycopg2.extras.DictCursor):
    """Log all SQL executed in the database.
    """

    def __init__(self, *args, **kwargs):
        psycopg2.extras.DictCursor.__init__(self, *args, **kwargs)

    def execute(self, sql, args=None):
        global log
        if log.isEnabledFor(logging.INFO):
            realsql = self.mogrify(sql, args)
            log.info('Executing SQL: <<%s>>', _simplify_whitespace(realsql))
        try:
            psycopg2.extras.DictCursor.execute(self, sql, args)
        except:
            log.exception('Exception while executing SQL')
            raise

class PostgresMigrations(MigrationsBase):

    engine = 'postgres'

    def __init__(self, db_config):
        self.db_config = db_config
        self.conn = psycopg2.connect(self.db_config['dsn'])

    def cursor(self):
        return self.conn.cursor(cursor_factory=PostgresLoggingDictCursor)

    def initialize_db(self):
        with self.cursor() as cur:
            cur.execute("""CREATE TABLE schemamigration (
                migration_file TEXT,
                execution_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )""")
            cur.connection.commit()

    def fetch_executed_migrations(self):
        with self.cursor() as cur:
            cur.execute("""SELECT migration_file FROM schemamigration
            ORDER BY execution_time""")
            return [row[0] for row in cur.fetchall()]

    def _migration_success(self, migration_file):
        with self.cursor() as cur:
            cur.execute("""INSERT INTO schemamigration (migration_file) VALUES (%s)""",
                    [migration_file])

    def execute_python_migration(self, migration_file, module):
        assert hasattr(module, 'do'), 'Python module must have `do` function accepting ' \
            'database connection'
        module.do(self.conn)
        self._migration_success(migration_file)
        self.conn.commit()

    # https://bitbucket.org/andrewgodwin/south/src/74742a1ba41ce6e9ea56cc694c824b7a93934ac6/south/db/generic.py?at=default
    def _sqlfile_to_statements(self, sql, regex=r"(?mx) ([^';]* (?:'[^']*'[^';]*)*)",
            comment_regex=r"(?mx) (?:^\s*$)|(?:--.*$)"):
        """
        Takes a SQL file and executes it as many separate statements.
        (Some backends, such as Postgres, don't work otherwise.)
        """
        # Be warned: This function is full of dark magic. Make sure you really
        # know regexes before trying to edit it.
        # First, strip comments
        sql = "\n".join([x.strip().replace("%", "%%") for x in re.split(comment_regex, sql) if x.strip()])
        # Now execute each statement
        return re.split(regex, sql)[1:][::2]

    def execute_sql_migration(self, migration_file):
        with open(migration_file) as f:
            content = f.read()
        for statement in self._sqlfile_to_statements(content):
            with self.cursor() as cur:
                cur.execute(statement)
        self._migration_success(migration_file)
        self.conn.commit()

#### Parsing options

MIGRATIONS_IMPLS = [PostgresMigrations]
ENGINE_TO_IMPL = {m.engine: m for m in MIGRATIONS_IMPLS}

class RunContext(object):

    def __init__(self, config, dbnick):
        assert dbnick in config.module.DATABASES, 'Not found in DATABASES in config: %s' % dbnick
        self.config = config
        self.dbnick = dbnick
        self.db_config = config.module.DATABASES[dbnick]
        self.migrations = ENGINE_TO_IMPL[self.db_config['engine']](self.db_config)
        config.setup_pythonpath_for_migrations(self.dbnick)

    def not_executed_migration_files(self):
        executed = self.migrations.fetch_executed_migrations()
        available = self.get_filenames()
        not_executed = set(available) - set(executed)
        not_executed = sorted(not_executed)
        return not_executed

    def _get_all_filenames(self, base_dir):
        filenames = glob.glob(os.path.join(base_dir, MIGRATION_PATTERN_SQL)) + \
            glob.glob(os.path.join(base_dir, MIGRATION_PATTERN_PY))
        filenames.sort()
        return filenames

    def get_filenames(self):
        return self._get_all_filenames(self.db_config['migrations_dir'])

    def execute_after_sync(self):
        after_sync = self.db_config.get('after_sync')
        if not after_sync:
            return
        msg = 'Executing after_sync command %r' % after_sync
        log.info(msg)
        click.echo(msg)
        os.system(after_sync)

#### Commands

HELP = """Example usage:

$ ./mschematool.py my_db initdb

A database nickname defined in the configuration module must be passed as the first argument.
After it a command must be specified.

"""

@click.group(help=HELP)
@click.option('--config', type=str, envvar='MSCHEMATOOL', help='Configuration module, e.g. "mypkg.mschematool_config". Environment variable MSCHEMATOOL_CONFIG can be specified instead.')
@click.option('--verbose/--no-verbose', default=False, help='Print executed SQL? Default: no.')
@click.argument('dbnick', type=str)
@click.pass_context
def main(ctx, config, verbose, dbnick):
    config_obj = Config(verbose, config)
    run_ctx = RunContext(config_obj, dbnick)
    ctx.obj = run_ctx

@main.command(help='Create tables used for tracking migrations.')
@click.pass_context
def init_db(ctx):
    ctx.obj.migrations.initialize_db()

@main.command(help='Show synced migrations.')
@click.pass_context
def synced(ctx):
    migrations = ctx.obj.migrations.fetch_executed_migrations()
    for migration in migrations:
        click.echo(migration)

@main.command(help='Show migrations available for syncing.')
@click.pass_context
def to_sync(ctx):
    migrations = ctx.obj.not_executed_migration_files()
    for migration in migrations:
        click.echo(migration)

@main.command(help='Sync all available migrations.')
@click.pass_context
def sync(ctx):
    to_execute = ctx.obj.not_executed_migration_files()
    if not to_execute:
        click.echo('No migrations to sync')
        return
    for migration_file in to_execute:
        msg = 'Executing migrations %s' % migration_file
        log.info(msg)
        click.echo(msg)
        ctx.obj.migrations.execute_migration(migration_file)
    ctx.obj.execute_after_sync()

@main.command(help='Sync a single migration, without syncing older ones.')
@click.argument('migration_file', type=str)
@click.pass_context
def force_sync_single(ctx, migration_file):
    msg = 'Force executing %s' % migration_file
    log.info(msg)
    click.echo(msg)
    ctx.obj.migrations.execute_migration(migration_file)
    ctx.obj.execute_after_sync()

@main.command(help='Print a filename for a new migration.')
@click.argument('name', type=str)
@click.pass_context
def print_new(ctx, name):
    '''Prints filename of a new migration'''
    print os.path.join(ctx.obj.db_config['migrations_dir'],
            'm{datestr}_{name}.sql'.format(
                datestr=datetime.datetime.now().strftime('%Y%m%d%H%M%S'),
                name=name.replace(' ', '_')))

@main.command(help='Show latest synced migration.')
@click.pass_context
def latest_synced(ctx):
    migrations = ctx.obj.migrations.fetch_executed_migrations()
    if not migrations:
        click.echo('No synced migrations')
    else:
        click.echo(executed[-1])

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Licensed under a 3-clause BSD style license - see LICENSE.rst

"""
This package defines the astrophysics-specific units.  They are also
available in the `astropy.units` namespace.

The ``mag`` unit is provided for compatibility with the FITS unit string
standard.  However, it is not very useful as-is since it is "orphaned"
and can not be converted to any other unit.  A future astropy
magnitudes library is planned to address this shortcoming.

"""

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

from . import si
from ..constants import si as _si
from .core import UnitBase, def_unit, si_prefixes, binary_prefixes

import numpy as _numpy

_ns = globals()

###########################################################################
# LENGTH

def_unit((['AU', 'au'], []), _si.au.value * si.m, namespace=_ns, prefixes=True,
         doc="astronomical unit: approximately the mean Earth--Sun "
         "distance.")

def_unit(['pc', 'parsec'], _si.pc.value * si.m, namespace=_ns, prefixes=True,
         doc="parsec: approximately 3.26 light-years.")

def_unit(['solRad', 'R_sun', 'Rsun'], _si.R_sun.value * si.m, namespace=_ns,
         doc="Solar radius", prefixes=True,
         format={'latex': r'R_{\odot}', 'unicode': 'R⊙'})
def_unit(['lyr', 'lightyear'], _si.c.value * si.yr.to(si.s) * si.m,
         namespace=_ns, prefixes=True, doc="Light year")

###########################################################################
# AREAS

def_unit(['barn'], 10 ** -28 * si.m ** 2, namespace=_ns, prefixes=True,
         doc="barn: unit of area used in HEP")

###########################################################################
# ANGULAR MEASUREMENTS

def_unit(['cycle', 'cy'], 2.0 * _numpy.pi * si.rad,
         namespace=_ns, prefixes=False,
         doc="cycle: angular measurement, a full turn or rotation")

###########################################################################
# MASS

def_unit(['solMass', 'M_sun', 'Msun'], _si.M_sun.value * si.kg, namespace=_ns,
         prefixes=True, doc="Solar mass",
         format={'latex': r'M_{\odot}', 'unicode': 'M⊙'})
def_unit(['M_p'], _si.m_p.value * si.kg, namespace=_ns,
         doc="Proton mass",
         format={'latex': r'M_{p}', 'unicode': 'Mₚ'})
def_unit(['M_e'], _si.m_e.value * si.kg, namespace=_ns,
         doc="Electron mass",
         format={'latex': r'M_{e}', 'unicode': 'Mₑ'})
# Unified atomic mass unit
def_unit(['u', 'Da', 'Dalton'], 1.6605387e-27 * si.kg, namespace=_ns,
         prefixes=True, exclude_prefixes=['a', 'da'],
         doc="Unified atomic mass unit")

##########################################################################
# ENERGY

def_unit(['Ry', 'rydberg'], 13.605692 * si.eV, namespace=_ns,
         prefixes=True,
         doc="Rydberg: Energy of a photon whose wavenumber is the Rydberg "
         "constant",
         format={'latex': r'R_{\infty}', 'unicode': 'R∞'})

###########################################################################
# ILLUMINATION

def_unit(['solLum', 'L_sun', 'Lsun'], _si.L_sun.value * si.W, namespace=_ns,
         prefixes=True, doc="Solar luminance",
         format={'latex': r'L_{\odot}', 'unicode': 'L⊙'})

###########################################################################
# SPECTRAL DENSITY

def_unit((['ph', 'photon'], ['photon']),
         format={'ogip': 'photon', 'vounit': 'photon'},
         namespace=_ns, prefixes=True)
def_unit(['Jy', 'Jansky', 'jansky'], 1e-26 * si.W / si.m ** 2 / si.Hz,
         namespace=_ns, prefixes=True,
         doc="Jansky: spectral flux density")
def_unit(['R', 'Rayleigh', 'rayleigh'],
         (1e10 / (4 * _numpy.pi)) *
         ph * si.m ** -2 * si.s ** -1 * si.sr ** -1,
         namespace=_ns, prefixes=True,
         doc="Rayleigh: photon flux")

###########################################################################
# MISCELLANEOUS

# Some of these are very FITS-specific and perhaps considered a mistake.
# Maybe they should be moved into the FITS format class?
# TODO: This is defined by the FITS standard as "relative to the sun".
# Is that mass, volume, what?
def_unit(['Sun'], namespace=_ns)

###########################################################################
# EVENTS

def_unit((['ct', 'count'], ['count']),
         format={'fits': 'count', 'ogip': 'count', 'vounit': 'count'},
         namespace=_ns, prefixes=True, exclude_prefixes=['p'])
def_unit((['pix', 'pixel'], ['pixel']),
         format={'ogip': 'pixel', 'vounit': 'pixel'},
         namespace=_ns, prefixes=True)

###########################################################################
# MISCELLANEOUS

def_unit(['chan'], namespace=_ns, prefixes=True)
def_unit(['bin'], namespace=_ns, prefixes=True)
def_unit((['vox', 'voxel'], ['voxel']),
         format={'fits': 'voxel', 'ogip': 'voxel', 'vounit': 'voxel'},
         namespace=_ns, prefixes=True)
def_unit((['bit', 'b'], ['bit']), namespace=_ns,
         prefixes=si_prefixes + binary_prefixes)
def_unit((['byte', 'B'], ['byte']), 8 * bit, namespace=_ns,
         format={'vounit': 'byte'},
         prefixes=si_prefixes + binary_prefixes,
         exclude_prefixes=['d'])
def_unit(['adu'], namespace=_ns, prefixes=True)
def_unit(['beam'], namespace=_ns, prefixes=True)
def_unit(['electron'], doc="Number of electrons", namespace=_ns,
         format={'latex': r'e^{-}', 'unicode': 'e⁻'})

###########################################################################
# Logarithmic units

def_unit(['dex'], namespace=_ns, prefixes=False,
         doc="Dex: Base 10 logarithmic unit")

def_unit(['dB', 'decibel'], 0.1 * dex, namespace=_ns, prefixes=False,
         doc="Decibel: ten per base 10 logarithmic unit")

def_unit(['mag'], -0.4 * dex, namespace=_ns, prefixes=True,
         doc="Astronomical magnitude: -2.5 per base 10 logarithmic unit")

###########################################################################
# CLEANUP

del UnitBase
del def_unit
del si

###########################################################################
# DOCSTRING

# This generates a docstring for this module that describes all of the
# standard units defined here.
from .utils import generate_unit_summary as _generate_unit_summary
__doc__ += _generate_unit_summary(globals())
<commit_msg>add long name for Astronomical Unit<commit_after># -*- coding: utf-8 -*-
# Licensed under a 3-clause BSD style license - see LICENSE.rst

"""
This package defines the astrophysics-specific units.  They are also
available in the `astropy.units` namespace.

The ``mag`` unit is provided for compatibility with the FITS unit string
standard.  However, it is not very useful as-is since it is "orphaned"
and can not be converted to any other unit.  A future astropy
magnitudes library is planned to address this shortcoming.

"""

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

from . import si
from ..constants import si as _si
from .core import UnitBase, def_unit, si_prefixes, binary_prefixes

import numpy as _numpy

_ns = globals()

###########################################################################
# LENGTH

def_unit((['AU', 'au'], ['AstronomicalUnit']), _si.au.value * si.m, namespace=_ns, prefixes=True,
         doc="astronomical unit: approximately the mean Earth--Sun "
         "distance.")

def_unit(['pc', 'parsec'], _si.pc.value * si.m, namespace=_ns, prefixes=True,
         doc="parsec: approximately 3.26 light-years.")

def_unit(['solRad', 'R_sun', 'Rsun'], _si.R_sun.value * si.m, namespace=_ns,
         doc="Solar radius", prefixes=True,
         format={'latex': r'R_{\odot}', 'unicode': 'R⊙'})
def_unit(['lyr', 'lightyear'], _si.c.value * si.yr.to(si.s) * si.m,
         namespace=_ns, prefixes=True, doc="Light year")

###########################################################################
# AREAS

def_unit(['barn'], 10 ** -28 * si.m ** 2, namespace=_ns, prefixes=True,
         doc="barn: unit of area used in HEP")

###########################################################################
# ANGULAR MEASUREMENTS

def_unit(['cycle', 'cy'], 2.0 * _numpy.pi * si.rad,
         namespace=_ns, prefixes=False,
         doc="cycle: angular measurement, a full turn or rotation")

###########################################################################
# MASS

def_unit(['solMass', 'M_sun', 'Msun'], _si.M_sun.value * si.kg, namespace=_ns,
         prefixes=True, doc="Solar mass",
         format={'latex': r'M_{\odot}', 'unicode': 'M⊙'})
def_unit(['M_p'], _si.m_p.value * si.kg, namespace=_ns,
         doc="Proton mass",
         format={'latex': r'M_{p}', 'unicode': 'Mₚ'})
def_unit(['M_e'], _si.m_e.value * si.kg, namespace=_ns,
         doc="Electron mass",
         format={'latex': r'M_{e}', 'unicode': 'Mₑ'})
# Unified atomic mass unit
def_unit(['u', 'Da', 'Dalton'], 1.6605387e-27 * si.kg, namespace=_ns,
         prefixes=True, exclude_prefixes=['a', 'da'],
         doc="Unified atomic mass unit")

##########################################################################
# ENERGY

def_unit(['Ry', 'rydberg'], 13.605692 * si.eV, namespace=_ns,
         prefixes=True,
         doc="Rydberg: Energy of a photon whose wavenumber is the Rydberg "
         "constant",
         format={'latex': r'R_{\infty}', 'unicode': 'R∞'})

###########################################################################
# ILLUMINATION

def_unit(['solLum', 'L_sun', 'Lsun'], _si.L_sun.value * si.W, namespace=_ns,
         prefixes=True, doc="Solar luminance",
         format={'latex': r'L_{\odot}', 'unicode': 'L⊙'})

###########################################################################
# SPECTRAL DENSITY

def_unit((['ph', 'photon'], ['photon']),
         format={'ogip': 'photon', 'vounit': 'photon'},
         namespace=_ns, prefixes=True)
def_unit(['Jy', 'Jansky', 'jansky'], 1e-26 * si.W / si.m ** 2 / si.Hz,
         namespace=_ns, prefixes=True,
         doc="Jansky: spectral flux density")
def_unit(['R', 'Rayleigh', 'rayleigh'],
         (1e10 / (4 * _numpy.pi)) *
         ph * si.m ** -2 * si.s ** -1 * si.sr ** -1,
         namespace=_ns, prefixes=True,
         doc="Rayleigh: photon flux")

###########################################################################
# MISCELLANEOUS

# Some of these are very FITS-specific and perhaps considered a mistake.
# Maybe they should be moved into the FITS format class?
# TODO: This is defined by the FITS standard as "relative to the sun".
# Is that mass, volume, what?
def_unit(['Sun'], namespace=_ns)

###########################################################################
# EVENTS

def_unit((['ct', 'count'], ['count']),
         format={'fits': 'count', 'ogip': 'count', 'vounit': 'count'},
         namespace=_ns, prefixes=True, exclude_prefixes=['p'])
def_unit((['pix', 'pixel'], ['pixel']),
         format={'ogip': 'pixel', 'vounit': 'pixel'},
         namespace=_ns, prefixes=True)

###########################################################################
# MISCELLANEOUS

def_unit(['chan'], namespace=_ns, prefixes=True)
def_unit(['bin'], namespace=_ns, prefixes=True)
def_unit((['vox', 'voxel'], ['voxel']),
         format={'fits': 'voxel', 'ogip': 'voxel', 'vounit': 'voxel'},
         namespace=_ns, prefixes=True)
def_unit((['bit', 'b'], ['bit']), namespace=_ns,
         prefixes=si_prefixes + binary_prefixes)
def_unit((['byte', 'B'], ['byte']), 8 * bit, namespace=_ns,
         format={'vounit': 'byte'},
         prefixes=si_prefixes + binary_prefixes,
         exclude_prefixes=['d'])
def_unit(['adu'], namespace=_ns, prefixes=True)
def_unit(['beam'], namespace=_ns, prefixes=True)
def_unit(['electron'], doc="Number of electrons", namespace=_ns,
         format={'latex': r'e^{-}', 'unicode': 'e⁻'})

###########################################################################
# Logarithmic units

def_unit(['dex'], namespace=_ns, prefixes=False,
         doc="Dex: Base 10 logarithmic unit")

def_unit(['dB', 'decibel'], 0.1 * dex, namespace=_ns, prefixes=False,
         doc="Decibel: ten per base 10 logarithmic unit")

def_unit(['mag'], -0.4 * dex, namespace=_ns, prefixes=True,
         doc="Astronomical magnitude: -2.5 per base 10 logarithmic unit")

###########################################################################
# CLEANUP

del UnitBase
del def_unit
del si

###########################################################################
# DOCSTRING

# This generates a docstring for this module that describes all of the
# standard units defined here.
from .utils import generate_unit_summary as _generate_unit_summary
__doc__ += _generate_unit_summary(globals())
<|endoftext|>


<commit_before>import glob
import os
from tkinter import *
from PIL import Image, ImageTk
import csv

from Archive import Archive
from S3 import S3Bucket

def carclicker(fname, scale=2):
    # Loading the image
    org_image = Image.open(fname)
    width, height = org_image.size
    width *= scale
    height *= scale
    org_image = org_image.resize((width, height))  # resize image

    # Setting up a tkinter canvas with image on it
    root = Tk()
    canvas = Canvas(root, width=width, height=height)
    canvas.pack()
    img = ImageTk.PhotoImage(org_image)
    canvas.create_image(0, 0, image=img, anchor="nw")

    coords = []

    def logcoords(event):
        # Append the x, y click coords to coords list
        nonlocal coords, scale
        x = event.x
        y = event.y

        drawbox(x, y)

        column = x // 2
        row = y // 2

        coords.append((row, column))

    def drawbox(x, y):
        # Draws a rectangle around the x, y position
        nonlocal canvas
        size = 80
        x0 = x - size//2
        x1 = x0 + size
        y0 = y - size//2
        y1 = y0 + size
        canvas.create_rectangle(x0, y0, x1, y1, width=2, outline='white')

    # Bind left click on mouse to logcoords
    canvas.bind("<Button 1>", logcoords)

    root.mainloop()

    return coords

def main():
    dat_file_list = []

    img_files = glob.glob('google_image_dump/*.png')

    for img_file in img_files:
        coords = carclicker(img_file)

        dat_file = img_file + '.dat'
        with open(dat_file, 'w', newline='') as f:
            writer = csv.writer(f)
            writer.writerows(coords)

        dat_file_list.append(dat_file)

    archive = Archive(dat_file_list)
    buffer = archive.streamtargz()

    s3bucket = S3Bucket('zepto-archive')
    s3bucket.uploadstream(buffer, archive.name)

if __name__ == "__main__":
    # execute only if run as a script
    main()
<commit_msg>Added rectangle preview with left-click hold and remove last click with right-click<commit_after>import glob
import os
from tkinter import *
from PIL import Image, ImageTk
import csv

from Archive import Archive
from S3 import S3Bucket

def carclicker(fname, scale=2):
    # Loading the image
    org_image = Image.open(fname)
    width, height = org_image.size
    width *= scale
    height *= scale
    org_image = org_image.resize((width, height))  # resize image

    # Setting up a tkinter canvas with image on it
    root = Tk()
    canvas = Canvas(root, width=width, height=height)
    canvas.pack()
    img = ImageTk.PhotoImage(org_image)
    canvas.create_image(0, 0, image=img, anchor="nw")

    coords = []
    coord_rects = []
    temp_rect = None

    def drawbox(x, y, colour='white'):
        # Draws a rectangle around the x, y position
        nonlocal canvas
        size = 80
        x0 = x - size//2
        x1 = x0 + size
        y0 = y - size//2
        y1 = y0 + size
        id = canvas.create_rectangle(x0, y0, x1, y1, width=2, outline=colour)
        return id

    def tempbox(event):
        nonlocal canvas, temp_rect
        x = event.x
        y = event.y
        # Remove the last rectangle
        if temp_rect is not None:
            canvas.delete(temp_rect)
        # Draw the new rectangle
        temp_rect = drawbox(x, y, 'white')

    def logcoords(event):
        # Append the x, y click coords to coords list
        nonlocal coords, coord_rects, temp_rect, scale
        x = event.x
        y = event.y
        # Remove the temporary rectangle
        if temp_rect is not None:
            canvas.delete(temp_rect)
            temp_rect = None
        # Draw a rectangle around the coords
        rect_id = drawbox(x, y, 'green')
        # Scale x and y coords to row column of original image
        column = x // scale
        row = y // scale
        # Record the coords and rect id to the lsit
        coords.append((row, column))
        coord_rects.append(rect_id)

    def removelastcoord(event):
        nonlocal canvas, coords, coord_rects
        # Pop last coord and rect id from list
        rect_id = coord_rects.pop()
        coords.pop()
        # Delete the rectangle
        canvas.delete(rect_id)

    # Bind left click motion to draw a temporary rectangle
    canvas.bind("<B1-Motion>", tempbox)
    # Bind release left click to log the coord
    canvas.bind("<ButtonRelease-1>", logcoords)
    # Bind right click to remove the last coord
    canvas.bind("<Button-3>", removelastcoord)

    root.mainloop()

    return coords

def main():
    dat_file_list = []

    img_files = glob.glob('google_image_dump/*.png')

    for img_file in img_files:
        coords = carclicker(img_file)

        dat_file = img_file + '.dat'
        with open(dat_file, 'w', newline='') as f:
            writer = csv.writer(f)
            writer.writerows(coords)

        dat_file_list.append(dat_file)

    archive = Archive(dat_file_list)
    buffer = archive.streamtargz()

    s3bucket = S3Bucket('zepto-archive')
    s3bucket.uploadstream(buffer, archive.name)

if __name__ == "__main__":
    # execute only if run as a script
    main()
<|endoftext|>


<commit_before># stdlib
from typing import Any
from typing import Dict
from typing import List

# third party
from sqlalchemy.engine import Engine
from sqlalchemy.orm import Query

# relative
from ..exceptions import RoleNotFoundError
from ..node_table.roles import Role
from .database_manager import DatabaseManager

class RoleManager(DatabaseManager):
    schema = Role

    def __init__(self, database: Engine) -> None:
        super().__init__(schema=RoleManager.schema, db=database)

    @property
    def ds_role(self) -> Role:
        return self.first(name="Data Scientist")

    @property
    def owner_role(self) -> Role:
        return self.first(name="Owner")

    @property
    def compliance_officer_role(self) -> Role:
        return self.first(name="Compliance Officer")

    @property
    def admin_role(self) -> Role:
        return self.first(name="Administrator")

    def _common_roles(self) -> Query:
        return self.db.session.query(self._schema).filter_by(
            can_make_data_requests=True,
            can_triage_data_requests=False,
            can_manage_privacy_budget=False,
            can_create_users=False,
            can_manage_users=False,
            can_edit_roles=False,
            can_manage_infrastructure=False,
            can_upload_data=False,
            can_upload_legal_document=False,
            can_edit_domain_settings=False,
        )

    @property
    def common_roles(self) -> List[Role]:
        return self._common_roles().all()

    @property
    def org_roles(self) -> List[Role]:
        return self.db.session.query(self._schema).except_(self._common_roles).all()

    def first(self, **kwargs: Any) -> Role:
        result = super().first(**kwargs)
        if not result:
            raise RoleNotFoundError
        return result

    def query(self, **kwargs: Any) -> Query:
        results = super().query(**kwargs)
        if len(results) == 0:
            raise RoleNotFoundError
        return results

    def set(self, role_id: int, params: Dict[Any, Any]) -> None:
        if self.contain(id=role_id):
            self.modify({"id": role_id}, params)
        else:
            raise RoleNotFoundError
<commit_msg>CREATE can_create_users method<commit_after># stdlib
from typing import Any
from typing import Dict
from typing import List

# third party
from sqlalchemy.engine import Engine
from sqlalchemy.orm import Query

# relative
from ..exceptions import RoleNotFoundError
from ..node_table.roles import Role
from .database_manager import DatabaseManager

class RoleManager(DatabaseManager):
    schema = Role

    def __init__(self, database: Engine) -> None:
        super().__init__(schema=RoleManager.schema, db=database)

    @property
    def ds_role(self) -> Role:
        return self.first(name="Data Scientist")

    @property
    def owner_role(self) -> Role:
        return self.first(name="Owner")

    @property
    def compliance_officer_role(self) -> Role:
        return self.first(name="Compliance Officer")

    @property
    def admin_role(self) -> Role:
        return self.first(name="Administrator")

    def _common_roles(self) -> Query:
        return self.db.session.query(self._schema).filter_by(
            can_make_data_requests=True,
            can_triage_data_requests=False,
            can_manage_privacy_budget=False,
            can_create_users=False,
            can_manage_users=False,
            can_edit_roles=False,
            can_manage_infrastructure=False,
            can_upload_data=False,
            can_upload_legal_document=False,
            can_edit_domain_settings=False,
        )

    def can_create_users(self, role_id: int) -> bool:
        role = super().first(id=role_id)
        if role:
            return role.can_create_users
        else:
            return False

    @property
    def common_roles(self) -> List[Role]:
        return self._common_roles().all()

    @property
    def org_roles(self) -> List[Role]:
        return self.db.session.query(self._schema).except_(self._common_roles).all()

    def first(self, **kwargs: Any) -> Role:
        result = super().first(**kwargs)
        if not result:
            raise RoleNotFoundError
        return result

    def query(self, **kwargs: Any) -> Query:
        results = super().query(**kwargs)
        if len(results) == 0:
            raise RoleNotFoundError
        return results

    def set(self, role_id: int, params: Dict[Any, Any]) -> None:
        if self.contain(id=role_id):
            self.modify({"id": role_id}, params)
        else:
            raise RoleNotFoundError
<|endoftext|>