<commit_before>"""
Define the Symbol class

Separate file because it depends on many other files, so we try to simplify
the circular dependencies.

:author: Roberto Roos
:license: MIT, see license file or https://opensource.org/licenses/MIT
:created on: 2020-11-16

"""

import re
from ctypes import sizeof
from typing import TYPE_CHECKING, Any, Optional, List, Tuple, Callable, Union, Type

from . import constants  # To access all constants, use package notation
from .constants import PLCDataType
from .pyads_ex import adsGetSymbolInfo
from .structs import NotificationAttrib

# ads.Connection relies on structs.AdsSymbol (but in type hints only), so use
# this 'if' to only include it when type hinting (False during execution)
if TYPE_CHECKING:
    from .ads import Connection, StructureDef  # pragma: no cover

class AdsSymbol:
    """Object that points to an ADS variable

    Contains index group, index offset, name, symbol type, comment of ADS
    symbol. Also remembers a reference to a Connection to be able to
    read/write directly.

    The virtual property `value` can be used to read from and write to
    the symbol.

    :ivar index_group: Index group of symbol
    :ivar index_offset: Index offset of symbol
    :ivar name: Name of symbol
    :ivar symbol_type: String representation of symbol type (PLC-style,
        e.g. "LREAL")
    :ivar plc_type: ctypes type of variable (from constants.PLCTYPE_*)
    :ivar comment: Comment of symbol
    :ivar value: Buffered value, i.e. the most recently read or written
        value for this symbol
    """

    # Regex for array - e.g. "ARRAY [1..10] OF DINT"
    _regex_array = re.compile(r"ARRAY \[(\d+)..(\d+)\] OF (.*)")
    # Regex for matrix - e.g. "matrix_10_int32"
    _regex_matrix = re.compile(r"matrix_(\d+)_(.*)_T")
    # Regex for list - e.g. "DINT(10)"
    _regex_list = re.compile(r"(.*)\((\d+)\)")

    def __init__(
            self,
            plc: "Connection",
            name: Optional[str] = None,
            index_group: Optional[int] = None,
            index_offset: Optional[int] = None,
            symbol_type: Optional[Union[str, Type[PLCDataType]]] = None,
            comment: Optional[str] = None,
            auto_update: bool = False,
            structure_def: Optional["StructureDef"] = None,
    ) -> None:
        """Create AdsSymbol instance.

        Specify either the variable name or the index_group **and**
        index_offset so the symbol can be located.
        If the name was specified but not all other attributes were,
        the other attributes will be looked up from the connection.

        `symbol_type` should be a type constant like `pyads.PLCTYPE_*`.
        Alternatively, it can be a string representation a PLC type (e.g.
        'LREAL').

        :param plc: Connection instance
        :param name:
        :param index_group:
        :param index_offset:
        :param symbol_type: PLC variable type (e.g. `pyads.PLCTYPE_DINT`)
        :param comment:
        :param auto_update: Create notification to update buffer (same as
            `set_auto_update(True)`)
        :param Optional[StructureDef] structure_def: tuple defining the structure and types contained within at
            according to PLCTYPE constants, must match the structure defined in the PLC, PLC structure must be defined
            with {attribute 'pack_mode' := '1'}
        """
        self._plc = plc
        self._handles_list: List[Tuple[int, int]] = []  # Notification handles
        self._auto_update_handle: Optional[Tuple[int, int]] = None

        # Check if the required info is present:
        missing_info = index_group is None or index_offset is None or symbol_type is None

        if missing_info:
            if name is None:
                raise ValueError(
                    "Please specify either `name`, or `index_group`, "
                    "`index_offset` and plc_type"
                )

        self.name = name
        self.index_offset = index_offset
        self.index_group = index_group
        self.symbol_type = symbol_type
        self.comment = comment
        self._value: Any = None

        # structure information
        self.structure_def = structure_def
        self._structure_size = 0
        if self.structure_def is not None:
            from .ads import size_of_structure
            self._structure_size = size_of_structure(self.structure_def)

        if missing_info:
            self._create_symbol_from_info()  # Perform remote lookup

        # Now `self.symbol_type` should have a value, find the actual PLCTYPE
        # from it.
        # This is relevant for both lookup and full user definition.

        self.plc_type: Optional[Type[PLCDataType]] = None
        if self.symbol_type is not None:
            if isinstance(self.symbol_type, str):  # Perform lookup if string
                self.plc_type = AdsSymbol.get_type_from_str(self.symbol_type)
            else:  # Otherwise `symbol_type` is probably a pyads.PLCTYPE_* constant
                self.plc_type = self.symbol_type

        self.auto_update = auto_update

    def _create_symbol_from_info(self) -> None:
        """Look up remaining info from the remote

        The name must already be present.
        Other values will already have a default value (mostly None).
        """
        info = adsGetSymbolInfo(self._plc._port, self._plc._adr, self.name)

        self.index_group = info.iGroup
        self.index_offset = info.iOffs
        if info.comment:
            self.comment = info.comment

        # info.dataType is an integer mapping to a type in
        # constants.ads_type_to_ctype.
        # However, this type ignores whether the variable is really an array!
        # So are not going to be using this and instead rely on the textual
        # type
        self.symbol_type = info.symbol_type  # Save the type as string

    def _check_for_open_connection(self) -> None:
        """Assert the current object is ready to read from/write to.

        This checks only if the Connection is open.
        """
        if not self._plc.is_open:
            raise ValueError(
                "Cannot read or write data with missing or closed Connection"
            )

    def read(self) -> Any:
        """Read the current value of this symbol.

        The new read value is also saved in the buffer.
        """
        self._check_for_open_connection()

        if self.is_structure:
            self._value = self._plc.read_structure_by_name(self.name, self.structure_def,
                                                           structure_size=self._structure_size)
        else:
            self._value = self._plc.read(self.index_group, self.index_offset, self.plc_type)

        return self._value

    def write(self, new_value: Optional[Any] = None) -> None:
        """Write a new value or the buffered value to the symbol.

        When a new value was written, the buffer is updated.

        :param new_value    Value to be written to symbol (if None,
                            the buffered value is send instead)
        """
        self._check_for_open_connection()

        if new_value is None:
            new_value = self._value  # Send buffered value instead
        else:
            self._value = new_value  # Update buffer with new value

        if self.is_structure:
            self._plc.write_structure_by_name(self.name, new_value, self.structure_def,
                                              structure_size=self._structure_size)
        else:
            self._plc.write(self.index_group, self.index_offset, new_value, self.plc_type)

    def __repr__(self) -> str:
        """Debug string"""
        t = type(self)
        return "<{}.{} object at {}, name: {}, type: {}>".format(
            t.__module__, t.__qualname__, hex(id(self)), self.name, self.symbol_type
        )

    def __del__(self) -> None:
        """Destructor"""
        self.clear_device_notifications()

    def add_device_notification(
            self,
            callback: Callable[[Any, Any], None],
            attr: Optional[NotificationAttrib] = None,
            user_handle: Optional[int] = None,
    ) -> Optional[Tuple[int, int]]:
        """Add on-change callback to symbol.

        See Connection.add_device_notification(...).

        When `attr` is omitted, the default will be used.

        The notification handles are returned but also stored locally. When
        this symbol is destructed any notifications will be freed up
        automatically.
        """

        if attr is None:
            attr = NotificationAttrib(length=sizeof(self.plc_type))

        handles = self._plc.add_device_notification(
            (self.index_group, self.index_offset), attr, callback, user_handle
        )

        self._handles_list.append(handles)

        return handles

    def clear_device_notifications(self) -> None:
        """Remove all registered notifications"""
        if self._handles_list:
            for handles in self._handles_list:
                self._plc.del_device_notification(*handles)
            self._handles_list = []  # Clear the list

        self._auto_update_handle = None  # If auto-update was enabled,
        # it won't work anymore

    def del_device_notification(self, handles: Tuple[int, int]) -> None:
        """Remove a single device notification by handles"""
        if handles in self._handles_list:
            self._plc.del_device_notification(*handles)
            self._handles_list.remove(handles)

    def _value_callback(self, notification: Any, data_name: Any) -> None:
        """Internal callback used by auto-update"""

        _handle, _datetime, value = self._plc.parse_notification(
            notification, self.plc_type
        )
        self._value = value

    @staticmethod
    def get_type_from_str(type_str: str) -> Optional[Type[PLCDataType]]:
        """Get PLCTYPE_* from PLC name string

        If PLC name could not be mapped, return None. This is done on
        purpose to prevent a program from crashing when an unusable symbol
        is found. Instead, exceptions will be thrown when this unmapped
        symbol is read/written.
        """

        # If simple scalar
        plc_name = "PLCTYPE_" + type_str
        if hasattr(constants, plc_name):
            # Map e.g. 'LREAL' to 'PLCTYPE_LREAL' directly based on the name
            return getattr(constants, plc_name)

        # If ARRAY
        reg_match = AdsSymbol._regex_array.match(type_str)
        if reg_match is not None:

            groups = reg_match.groups()
            size = int(groups[1]) + 1 - int(groups[0])  # Estimate the size
            scalar_type_str = groups[2]

            # Find scalar type
            scalar_type = AdsSymbol.get_type_from_str(scalar_type_str)

            if scalar_type:
                return scalar_type * size

            # Fall to method default instead

        # If array/matrix (an 1D array is also called a matrix)
        reg_match = AdsSymbol._regex_matrix.match(type_str)
        if reg_match is not None:

            groups = reg_match.groups()
            size = int(groups[0])
            scalar_type_str = groups[1]

            if scalar_type_str in constants.PLC_ARRAY_MAP:
                return constants.PLC_ARRAY_MAP[scalar_type_str](size)

        # If list
        reg_match = AdsSymbol._regex_list.match(type_str)
        if reg_match is not None:

            groups = reg_match.groups()
            scalar_type_str = groups[0]
            size = int(groups[1])

            scalar_type = AdsSymbol.get_type_from_str(scalar_type_str)

            if scalar_type:
                return scalar_type * size

        # We allow unmapped types at this point - Instead we will throw  an
        # error when they are being addressed

        return None

    @property
    def auto_update(self) -> Any:
        """Return True if auto_update is enabled for this symbol."""
        return self._auto_update_handle is not None

    @auto_update.setter
    def auto_update(self, value: bool) -> None:
        """Enable or disable auto-update of the buffered value.

        This automatic update is done through a device notification. This
        can be efficient when a remote variables changes its values less often
        than your code run.

        Clearing all device notifications will also disable auto-update.
        Automatic update is disabled by default.
        """
        if value and self._auto_update_handle is None:
            self._auto_update_handle = self.add_device_notification(
                self._value_callback
            )
        elif not value and self._auto_update_handle is not None:
            self.del_device_notification(self._auto_update_handle)
            self._auto_update_handle = None

    @property
    def value(self) -> Any:
        """Return the current value of the symbol."""
        return self._value

    @value.setter
    def value(self, val: Any) -> None:
        """Set the current value of the symbol.

        If auto_update is True then the the write command will be called automatically.

        """
        self._value = val

        # write value to plc if auto_update is enabled
        if self.auto_update:
            self.write(val)

    @property
    def is_structure(self) -> bool:
        """Return True if the symbol object represents a structure.

        This is the case if a structure_def has been passed during initialization.
        """
        return self.structure_def is not None
<commit_msg>Prevent type expansion in docs<commit_after>"""
Define the Symbol class

Separate file because it depends on many other files, so we try to simplify
the circular dependencies.

:author: Roberto Roos
:license: MIT, see license file or https://opensource.org/licenses/MIT
:created on: 2020-11-16

"""
from __future__ import annotations

import re
from ctypes import sizeof
from typing import TYPE_CHECKING, Any, Optional, List, Tuple, Callable, Union, Type

from . import constants  # To access all constants, use package notation
from .constants import PLCDataType
from .pyads_ex import adsGetSymbolInfo
from .structs import NotificationAttrib

# ads.Connection relies on structs.AdsSymbol (but in type hints only), so use
# this 'if' to only include it when type hinting (False during execution)
if TYPE_CHECKING:
    from .ads import Connection, StructureDef  # pragma: no cover

class AdsSymbol:
    """Object that points to an ADS variable

    Contains index group, index offset, name, symbol type, comment of ADS
    symbol. Also remembers a reference to a Connection to be able to
    read/write directly.

    The virtual property `value` can be used to read from and write to
    the symbol.

    :ivar index_group: Index group of symbol
    :ivar index_offset: Index offset of symbol
    :ivar name: Name of symbol
    :ivar symbol_type: String representation of symbol type (PLC-style,
        e.g. "LREAL")
    :ivar plc_type: ctypes type of variable (from constants.PLCTYPE_*)
    :ivar comment: Comment of symbol
    :ivar value: Buffered value, i.e. the most recently read or written
        value for this symbol
    """

    # Regex for array - e.g. "ARRAY [1..10] OF DINT"
    _regex_array = re.compile(r"ARRAY \[(\d+)..(\d+)\] OF (.*)")
    # Regex for matrix - e.g. "matrix_10_int32"
    _regex_matrix = re.compile(r"matrix_(\d+)_(.*)_T")
    # Regex for list - e.g. "DINT(10)"
    _regex_list = re.compile(r"(.*)\((\d+)\)")

    def __init__(
            self,
            plc: "Connection",
            name: Optional[str] = None,
            index_group: Optional[int] = None,
            index_offset: Optional[int] = None,
            symbol_type: Optional[Union[str, Type["PLCDataType"]]] = None,
            comment: Optional[str] = None,
            auto_update: bool = False,
            structure_def: Optional["StructureDef"] = None,
    ) -> None:
        """Create AdsSymbol instance.

        Specify either the variable name or the index_group **and**
        index_offset so the symbol can be located.
        If the name was specified but not all other attributes were,
        the other attributes will be looked up from the connection.

        `symbol_type` should be a type constant like `pyads.PLCTYPE_*`.
        Alternatively, it can be a string representation a PLC type (e.g.
        'LREAL').

        :param plc: Connection instance
        :param name:
        :param index_group:
        :param index_offset:
        :param symbol_type: PLC variable type (e.g. `pyads.PLCTYPE_DINT`)
        :param comment:
        :param auto_update: Create notification to update buffer (same as
            `set_auto_update(True)`)
        :param Optional[StructureDef] structure_def: tuple defining the structure and types contained within at
            according to PLCTYPE constants, must match the structure defined in the PLC, PLC structure must be defined
            with {attribute 'pack_mode' := '1'}
        """
        self._plc = plc
        self._handles_list: List[Tuple[int, int]] = []  # Notification handles
        self._auto_update_handle: Optional[Tuple[int, int]] = None

        # Check if the required info is present:
        missing_info = index_group is None or index_offset is None or symbol_type is None

        if missing_info:
            if name is None:
                raise ValueError(
                    "Please specify either `name`, or `index_group`, "
                    "`index_offset` and plc_type"
                )

        self.name = name
        self.index_offset = index_offset
        self.index_group = index_group
        self.symbol_type = symbol_type
        self.comment = comment
        self._value: Any = None

        # structure information
        self.structure_def = structure_def
        self._structure_size = 0
        if self.structure_def is not None:
            from .ads import size_of_structure
            self._structure_size = size_of_structure(self.structure_def)

        if missing_info:
            self._create_symbol_from_info()  # Perform remote lookup

        # Now `self.symbol_type` should have a value, find the actual PLCTYPE
        # from it.
        # This is relevant for both lookup and full user definition.

        self.plc_type: Optional[Type[PLCDataType]] = None
        if self.symbol_type is not None:
            if isinstance(self.symbol_type, str):  # Perform lookup if string
                self.plc_type = AdsSymbol.get_type_from_str(self.symbol_type)
            else:  # Otherwise `symbol_type` is probably a pyads.PLCTYPE_* constant
                self.plc_type = self.symbol_type

        self.auto_update = auto_update

    def _create_symbol_from_info(self) -> None:
        """Look up remaining info from the remote

        The name must already be present.
        Other values will already have a default value (mostly None).
        """
        info = adsGetSymbolInfo(self._plc._port, self._plc._adr, self.name)

        self.index_group = info.iGroup
        self.index_offset = info.iOffs
        if info.comment:
            self.comment = info.comment

        # info.dataType is an integer mapping to a type in
        # constants.ads_type_to_ctype.
        # However, this type ignores whether the variable is really an array!
        # So are not going to be using this and instead rely on the textual
        # type
        self.symbol_type = info.symbol_type  # Save the type as string

    def _check_for_open_connection(self) -> None:
        """Assert the current object is ready to read from/write to.

        This checks only if the Connection is open.
        """
        if not self._plc.is_open:
            raise ValueError(
                "Cannot read or write data with missing or closed Connection"
            )

    def read(self) -> Any:
        """Read the current value of this symbol.

        The new read value is also saved in the buffer.
        """
        self._check_for_open_connection()

        if self.is_structure:
            self._value = self._plc.read_structure_by_name(self.name, self.structure_def,
                                                           structure_size=self._structure_size)
        else:
            self._value = self._plc.read(self.index_group, self.index_offset, self.plc_type)

        return self._value

    def write(self, new_value: Optional[Any] = None) -> None:
        """Write a new value or the buffered value to the symbol.

        When a new value was written, the buffer is updated.

        :param new_value    Value to be written to symbol (if None,
                            the buffered value is send instead)
        """
        self._check_for_open_connection()

        if new_value is None:
            new_value = self._value  # Send buffered value instead
        else:
            self._value = new_value  # Update buffer with new value

        if self.is_structure:
            self._plc.write_structure_by_name(self.name, new_value, self.structure_def,
                                              structure_size=self._structure_size)
        else:
            self._plc.write(self.index_group, self.index_offset, new_value, self.plc_type)

    def __repr__(self) -> str:
        """Debug string"""
        t = type(self)
        return "<{}.{} object at {}, name: {}, type: {}>".format(
            t.__module__, t.__qualname__, hex(id(self)), self.name, self.symbol_type
        )

    def __del__(self) -> None:
        """Destructor"""
        self.clear_device_notifications()

    def add_device_notification(
            self,
            callback: Callable[[Any, Any], None],
            attr: Optional[NotificationAttrib] = None,
            user_handle: Optional[int] = None,
    ) -> Optional[Tuple[int, int]]:
        """Add on-change callback to symbol.

        See Connection.add_device_notification(...).

        When `attr` is omitted, the default will be used.

        The notification handles are returned but also stored locally. When
        this symbol is destructed any notifications will be freed up
        automatically.
        """

        if attr is None:
            attr = NotificationAttrib(length=sizeof(self.plc_type))

        handles = self._plc.add_device_notification(
            (self.index_group, self.index_offset), attr, callback, user_handle
        )

        self._handles_list.append(handles)

        return handles

    def clear_device_notifications(self) -> None:
        """Remove all registered notifications"""
        if self._handles_list:
            for handles in self._handles_list:
                self._plc.del_device_notification(*handles)
            self._handles_list = []  # Clear the list

        self._auto_update_handle = None  # If auto-update was enabled,
        # it won't work anymore

    def del_device_notification(self, handles: Tuple[int, int]) -> None:
        """Remove a single device notification by handles"""
        if handles in self._handles_list:
            self._plc.del_device_notification(*handles)
            self._handles_list.remove(handles)

    def _value_callback(self, notification: Any, data_name: Any) -> None:
        """Internal callback used by auto-update"""

        _handle, _datetime, value = self._plc.parse_notification(
            notification, self.plc_type
        )
        self._value = value

    @staticmethod
    def get_type_from_str(type_str: str) -> Optional[Type[PLCDataType]]:
        """Get PLCTYPE_* from PLC name string

        If PLC name could not be mapped, return None. This is done on
        purpose to prevent a program from crashing when an unusable symbol
        is found. Instead, exceptions will be thrown when this unmapped
        symbol is read/written.
        """

        # If simple scalar
        plc_name = "PLCTYPE_" + type_str
        if hasattr(constants, plc_name):
            # Map e.g. 'LREAL' to 'PLCTYPE_LREAL' directly based on the name
            return getattr(constants, plc_name)

        # If ARRAY
        reg_match = AdsSymbol._regex_array.match(type_str)
        if reg_match is not None:

            groups = reg_match.groups()
            size = int(groups[1]) + 1 - int(groups[0])  # Estimate the size
            scalar_type_str = groups[2]

            # Find scalar type
            scalar_type = AdsSymbol.get_type_from_str(scalar_type_str)

            if scalar_type:
                return scalar_type * size

            # Fall to method default instead

        # If array/matrix (an 1D array is also called a matrix)
        reg_match = AdsSymbol._regex_matrix.match(type_str)
        if reg_match is not None:

            groups = reg_match.groups()
            size = int(groups[0])
            scalar_type_str = groups[1]

            if scalar_type_str in constants.PLC_ARRAY_MAP:
                return constants.PLC_ARRAY_MAP[scalar_type_str](size)

        # If list
        reg_match = AdsSymbol._regex_list.match(type_str)
        if reg_match is not None:

            groups = reg_match.groups()
            scalar_type_str = groups[0]
            size = int(groups[1])

            scalar_type = AdsSymbol.get_type_from_str(scalar_type_str)

            if scalar_type:
                return scalar_type * size

        # We allow unmapped types at this point - Instead we will throw  an
        # error when they are being addressed

        return None

    @property
    def auto_update(self) -> Any:
        """Return True if auto_update is enabled for this symbol."""
        return self._auto_update_handle is not None

    @auto_update.setter
    def auto_update(self, value: bool) -> None:
        """Enable or disable auto-update of the buffered value.

        This automatic update is done through a device notification. This
        can be efficient when a remote variables changes its values less often
        than your code run.

        Clearing all device notifications will also disable auto-update.
        Automatic update is disabled by default.
        """
        if value and self._auto_update_handle is None:
            self._auto_update_handle = self.add_device_notification(
                self._value_callback
            )
        elif not value and self._auto_update_handle is not None:
            self.del_device_notification(self._auto_update_handle)
            self._auto_update_handle = None

    @property
    def value(self) -> Any:
        """Return the current value of the symbol."""
        return self._value

    @value.setter
    def value(self, val: Any) -> None:
        """Set the current value of the symbol.

        If auto_update is True then the the write command will be called automatically.

        """
        self._value = val

        # write value to plc if auto_update is enabled
        if self.auto_update:
            self.write(val)

    @property
    def is_structure(self) -> bool:
        """Return True if the symbol object represents a structure.

        This is the case if a structure_def has been passed during initialization.
        """
        return self.structure_def is not None
<|endoftext|>


<commit_before>"""pyang plugin handling"""

import os
import sys

plugins = []
"""List of registered PyangPlugin instances"""

def init(plugindirs=[]):
    """Initialize the plugin framework"""

    # initialize the builtin plugins
    from .translators import yang,yin,dsdl
    yang.pyang_plugin_init()
    yin.pyang_plugin_init()
    dsdl.pyang_plugin_init()

    # search for plugins in std directory
    basedir = os.path.split(sys.modules['pyang'].__file__)[0]
    plugindirs.insert(0, basedir + "/plugins")

    # add paths from env
    pluginpath = os.getenv('PYANG_PLUGINPATH')
    if pluginpath is not None:
        plugindirs.extend(pluginpath.split(os.pathsep))

    syspath = sys.path
    for plugindir in plugindirs:
        sys.path = [plugindir] + syspath
        try:
            fnames = os.listdir(plugindir)
        except OSError:
            continue
        for fname in fnames:
            if fname.endswith(".py") and fname != '__init__.py':
                pluginmod = __import__(fname[:-3])
                try:
                    pluginmod.pyang_plugin_init()
                except AttributeError as s:
                    print(pluginmod.__dict__)
                    raise AttributeError(pluginmod.__file__ + ': ' + str(s))
        sys.path = syspath

def register_plugin(plugin):
    """Call this to register a pyang plugin. See class PyangPlugin
    for more info.
    """
    plugins.append(plugin)

class PyangPlugin(object):
    """Abstract base class for pyang plugins

    A pyang plugin is a module found in the plugins directory of the
    pyang installation, or in the dynamic pluginpath.

    Such a module must export a function 'pyang_plugin_init()', which
    may call pyang.plugin.register_plugin() with an instance of a class
    derived from this class as argument.

    A plugin can extend the base pyang library functions, or the pyang
    front-end program, or both.
    """

    def __init__(self):
        self.multiple_modules = False
        self.handle_comments = False

    ## pyang front-end program methods

    def add_output_format(self, fmts):
        """Add an output format to the pyang program.

        `fmts` is a dict which maps the format name string to a plugin
        instance.

        Override this method and update `fmts` with the output format
        name.
        """
        return
    def add_opts(self, optparser):
        """Add command line options to the pyang program.

        Override this method and add the plugin related options as an
        option group.
        """
        return

    ## library methods

    def setup_ctx(self, ctx):
        """Modify the Context at setup time.  Called for all plugins.

        Override this method to modify the Context before the module
        repository is accessed.
        """
        return

    def setup_fmt(self, ctx):
        """Modify the Context at setup time.  Called for the selected plugin.

        Override this method to modify the Context before the module
        repository is accessed.
        """
        return

    def pre_load_modules(self, ctx):
        """Called for the selected plugin, before any modules are loaded"""
        return

    def pre_validate_ctx(self, ctx, modules):
        """Called for all plugins, before the modules are validated"""
        return

    def pre_validate(self, ctx, modules):
        """Called for the selected plugin, before the modules are validated"""
        return

    def post_validate(self, ctx, modules):
        """Called for the selected plugin, after the modules
        have been validated"""
        return

    def post_validate_ctx(self, ctx, modules):
        """Called for all plugins, after the modules
        have been validated"""
        return

    def emit(self, ctx, modules, writef):
        """Produce the plugin output.

        Override this method to perform the output conversion.
        `writef` is a function that takes one string to print as argument.

        Raise error.EmitError on failure.
        """
        return
<commit_msg>Prevent importing temporary Emacs files<commit_after>"""pyang plugin handling"""

import os
import sys

plugins = []
"""List of registered PyangPlugin instances"""

def init(plugindirs=[]):
    """Initialize the plugin framework"""

    # initialize the builtin plugins
    from .translators import yang,yin,dsdl
    yang.pyang_plugin_init()
    yin.pyang_plugin_init()
    dsdl.pyang_plugin_init()

    # search for plugins in std directory
    basedir = os.path.split(sys.modules['pyang'].__file__)[0]
    plugindirs.insert(0, basedir + "/plugins")

    # add paths from env
    pluginpath = os.getenv('PYANG_PLUGINPATH')
    if pluginpath is not None:
        plugindirs.extend(pluginpath.split(os.pathsep))

    syspath = sys.path
    for plugindir in plugindirs:
        sys.path = [plugindir] + syspath
        try:
            fnames = os.listdir(plugindir)
        except OSError:
            continue
        for fname in fnames:
            if not fname.startswith(".#") and fname.endswith(".py") and \
               fname != '__init__.py':
                pluginmod = __import__(fname[:-3])
                try:
                    pluginmod.pyang_plugin_init()
                except AttributeError as s:
                    print(pluginmod.__dict__)
                    raise AttributeError(pluginmod.__file__ + ': ' + str(s))
        sys.path = syspath

def register_plugin(plugin):
    """Call this to register a pyang plugin. See class PyangPlugin
    for more info.
    """
    plugins.append(plugin)

class PyangPlugin(object):
    """Abstract base class for pyang plugins

    A pyang plugin is a module found in the plugins directory of the
    pyang installation, or in the dynamic pluginpath.

    Such a module must export a function 'pyang_plugin_init()', which
    may call pyang.plugin.register_plugin() with an instance of a class
    derived from this class as argument.

    A plugin can extend the base pyang library functions, or the pyang
    front-end program, or both.
    """

    def __init__(self):
        self.multiple_modules = False
        self.handle_comments = False

    ## pyang front-end program methods

    def add_output_format(self, fmts):
        """Add an output format to the pyang program.

        `fmts` is a dict which maps the format name string to a plugin
        instance.

        Override this method and update `fmts` with the output format
        name.
        """
        return
    def add_opts(self, optparser):
        """Add command line options to the pyang program.

        Override this method and add the plugin related options as an
        option group.
        """
        return

    ## library methods

    def setup_ctx(self, ctx):
        """Modify the Context at setup time.  Called for all plugins.

        Override this method to modify the Context before the module
        repository is accessed.
        """
        return

    def setup_fmt(self, ctx):
        """Modify the Context at setup time.  Called for the selected plugin.

        Override this method to modify the Context before the module
        repository is accessed.
        """
        return

    def pre_load_modules(self, ctx):
        """Called for the selected plugin, before any modules are loaded"""
        return

    def pre_validate_ctx(self, ctx, modules):
        """Called for all plugins, before the modules are validated"""
        return

    def pre_validate(self, ctx, modules):
        """Called for the selected plugin, before the modules are validated"""
        return

    def post_validate(self, ctx, modules):
        """Called for the selected plugin, after the modules
        have been validated"""
        return

    def post_validate_ctx(self, ctx, modules):
        """Called for all plugins, after the modules
        have been validated"""
        return

    def emit(self, ctx, modules, writef):
        """Produce the plugin output.

        Override this method to perform the output conversion.
        `writef` is a function that takes one string to print as argument.

        Raise error.EmitError on failure.
        """
        return
<|endoftext|>


<commit_before>try:
    from urlparse import urlparse, urlunparse
except ImportError:
    from urllib.parse import urlparse, urlunparse

import uuid
from datetime import datetime
from functools import update_wrapper
from werkzeug.utils import redirect
from flask import Flask
from flask.json import jsonify
from flask.templating import render_template
from flask.helpers import url_for, make_response
from flask.globals import session, request

from pybitid import bitid
from models.user import User
from models.nonce import Nonce
from services.fake_user_db_service import FakeUserDbService
from services.fake_tx_db_service import FakeTxDbService
from services.fake_nonce_db_service import FakeNonceDbService

# Constant indicating if we run the app against Bitcoin test network or main network
USE_TESTNET = False

# Initializes the flask app
app = Flask(__name__)

# Initializes a secret key used to encrypt data in cookies
app.secret_key = '\xfd{H\xe5<\x95\xf9\xe3\x96.5\xd1\x01O<!\xd5\xa2\xa0\x9fR"\xa1\xa8'

# Initializes services to access databases or services
# For this toy project we use fake dbs storing data in memory
nonce_db_service = FakeNonceDbService()
user_db_service = FakeUserDbService()
tx_db_service = FakeTxDbService()

# NoCache decorator
# Required to fix a problem with IE which caches all XMLHttpRequest responses
def nocache(f):
    def add_nocache(*args, **kwargs):
        resp = make_response(f(*args, **kwargs))
        resp.headers.add('Last-Modified', datetime.now())
        resp.headers.add('Cache-Control', 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0')
        resp.headers.add('Pragma', 'no-cache')
        return resp
    return update_wrapper(add_nocache, f)

@app.route("/", methods=["GET"])
@app.route("/home", methods=["GET"])
def home():
    params_tpl = {}
    # Checks if user is logged
    if not session.get("uid", None) is None:
        # Gets the user from db
        user = user_db_service.get_user_by_uid(session["uid"])
        if not user is None: params_tpl["user_address"] = user.address
    # Renders the home page
    return render_template('index.html', params_tpl=params_tpl)

@app.route("/login", methods=["GET"])
def login():
    '''
    This function initializes the authentication process
    It builds a challenge which is sent to the client
    '''
    # Initializes a new session id and stores it in the session cookie
    # If user was authenticated, it will be similar to a log out
    session["sid"]  = str(uuid.uuid4())
    session["uid"] = None
    # Creates a new nonce associated to this session
    nonce = Nonce(session["sid"])
    # Stores the nonce in database
    nonce_db_service.create_nonce(nonce)
    # Gets the callback uri
    callback_uri = get_callback_uri()
    # Builds the challenge (bitid uri)
    bitid_uri = bitid.build_uri(callback_uri, nonce.nid)
    # Gets the qrcode uri
    qrcode = bitid.qrcode(bitid_uri)
    # Renders the login page
    params_tpl = {"callback_uri": callback_uri, "bitid_uri": bitid_uri, "qrcode": qrcode}
    return render_template('login.html', params_tpl=params_tpl)

@app.route("/callback", methods=["POST"])
def callback():
    '''
    This function validates the response sent by the client about the challenge
    This is the route called by the bitcoin wallet when the challenge has been signed
    '''
    # Retrieves the callback uri
    callback_uri = get_callback_uri()
    # Extracts data from the posted request (from form of from json data according to the method used by the client)
    container = request.get_json(False, True, False) if request.mimetype == "application/json" else request.form
    bitid_uri = container["uri"]
    signature = container["signature"]
    address   = container["address"]

    #
    # Let's start by a bunch of validations
    #

    # Checks the address
    if not bitid.address_valid(address, USE_TESTNET):
        return jsonify(message = "Address is invalid or not legal"), 401
    # Checks the bitid uri
    if not bitid.uri_valid(bitid_uri, callback_uri):
        return jsonify(message = "BitID URI is invalid or not legal"), 401
    # Checks the signature
    if not bitid.signature_valid(address, signature, bitid_uri, callback_uri, USE_TESTNET):
        return jsonify(message = "Signature is incorrect"), 401

    # Note that the previous 3 steps could also be done in 1 step with following code:
    # if not bitid.challenge_valid(address, signature, bitid_uri, callback_uri, USE_TESTNET):
    #    return jsonify(message = "Sorry but something does not match"), 401

    # Checks the nonce
    nid = bitid.extract_nonce(bitid_uri)
    # Tries to retrieve the nonce from db
    nonce = nonce_db_service.get_nonce_by_nid(nid)
    if nonce is None:
        return jsonify(message = "NONCE is illegal"), 401
    elif nonce.has_expired():
        nonce_db_service.delete_nonce(nonce)
        return jsonify(message = "NONCE has expired"), 401

    #
    # So good so far, everything seems ok
    # It's time to check if we have a sign out or a sign in
    #

    # Checks if a user with the given address has already been registered in db (sign in)
    user = user_db_service.get_user_by_address(address)
    # If we have a sign out
    if user is None:
        # Here, we have an important check to do in order to avoid flooding of the users db
        # Let's check for a proof of goodwill (@see pybitid_demo.services.fake_tx_db_service)
        if not tx_db_service.check_proof_of_goodwill(address):
            return jsonify(message = "Address is invalid or not legal"), 401
        else:
            # Creates a new user and stores it in db
            user = User(address)
            if not user_db_service.create_user(user):
                return jsonify(message = "Ooops ! Something went wrong but we work on it"), 500

    # To finalize the authentication, let's set the user id in the nonce and update it in db
    nonce.uid = user.uid
    if not nonce_db_service.update_nonce(nonce):
        return jsonify(message = "Ooops ! Something went wrong but we work on it"), 500
    # Everything was ok: user is authenticated
    return jsonify(address = address, nonce = nonce.sid)

@app.route("/auth", methods=["GET"])
@nocache
def auth():
    '''
    This function checks if a challenge associated to a given address has been validated
    '''
    # Checks that session id is set
    if not session["sid"]:
        return jsonify(auth = 0)
    # Gets the nonce associated to the session id
    nonce = nonce_db_service.get_nonce_by_sid(session["sid"])
    if nonce is None:
        return jsonify(auth = 0)
    # Checks if nonce has an associated user id
    if nonce.uid is None:
        return jsonify(auth = 0)
    # Gets the user from db
    user = user_db_service.get_user_by_uid(nonce.uid)
    if user is None:
        return jsonify(auth = 0)
    # Let's increase the sign_in counter in user object (for demo purpose only)
    user.signin_count += 1
    user_db_service.update_user(user)
    # Everything is ok, let's finalize the authentication
    session["uid"] = user.uid
    nonce_db_service.delete_nonce(nonce)
    return jsonify(auth = 1)

@app.route("/user", methods=["GET"])
@nocache
def user():
    # Checks if user is logged
    if session["uid"] is None:
        return redirect(url_for("login")), 401
    user_address = ""
    user_signin_count = 0
    # Gets the user from db
    user = user_db_service.get_user_by_uid(session["uid"])
    if not user is None:
        user_address = user.address
        user_signin_count = user.signin_count
    params_tpl = {"user_address": user_address, "user_signin_count": user_signin_count}
    return render_template("user.html", params_tpl=params_tpl)

@app.route("/sign_out", methods=["GET"])
@nocache
def sign_out():
    session.pop("uid", None)
    return redirect(url_for("home"))

'''
Utils functions
'''
@app.template_filter("escape_slash")
def escape_slash_filter(s):
    return str.replace(s, "/", "\\/")

def get_callback_uri():
    '''
    Returns the callback uri for this site
    '''
    callback_uri = url_for("home", _external=True)
    parsed = urlparse(callback_uri)
    scheme = parsed.scheme
    netloc = parsed.netloc
    path = "/callback"
    return urlunparse((scheme, netloc, path, "", "", ""))

if __name__ == "__main__":
    # Comment/uncomment following lines to switch production / debug mode
    # Note that using BitId with a smartphone is not possible in debug mode (server only accessible by local machine)
    app.run(host='0.0.0.0')
    #app.run(debug=True)<commit_msg>Fixed pb of cache with IE<commit_after>try:
    from urlparse import urlparse, urlunparse
except ImportError:
    from urllib.parse import urlparse, urlunparse

import uuid
from datetime import datetime
from functools import update_wrapper
from werkzeug.utils import redirect
from flask import Flask
from flask.json import jsonify
from flask.templating import render_template
from flask.helpers import url_for, make_response
from flask.globals import session, request

from pybitid import bitid
from models.user import User
from models.nonce import Nonce
from services.fake_user_db_service import FakeUserDbService
from services.fake_tx_db_service import FakeTxDbService
from services.fake_nonce_db_service import FakeNonceDbService

# Constant indicating if we run the app against Bitcoin test network or main network
USE_TESTNET = False

# Initializes the flask app
app = Flask(__name__)

# Initializes a secret key used to encrypt data in cookies
app.secret_key = '\xfd{H\xe5<\x95\xf9\xe3\x96.5\xd1\x01O<!\xd5\xa2\xa0\x9fR"\xa1\xa8'

# Initializes services to access databases or services
# For this toy project we use fake dbs storing data in memory
nonce_db_service = FakeNonceDbService()
user_db_service = FakeUserDbService()
tx_db_service = FakeTxDbService()

# NoCache decorator
# Required to fix a problem with IE which caches all XMLHttpRequest responses
def nocache(f):
    def add_nocache(*args, **kwargs):
        resp = make_response(f(*args, **kwargs))
        resp.headers.add('Last-Modified', datetime.now())
        resp.headers.add('Cache-Control', 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0')
        resp.headers.add('Pragma', 'no-cache')
        return resp
    return update_wrapper(add_nocache, f)

@app.route("/", methods=["GET"])
@app.route("/home", methods=["GET"])
@nocache
def home():
    params_tpl = {}
    # Checks if user is logged
    if not session.get("uid", None) is None:
        # Gets the user from db
        user = user_db_service.get_user_by_uid(session["uid"])
        if not user is None: params_tpl["user_address"] = user.address
    # Renders the home page
    return render_template('index.html', params_tpl=params_tpl)

@app.route("/login", methods=["GET"])
@nocache
def login():
    '''
    This function initializes the authentication process
    It builds a challenge which is sent to the client
    '''
    # Initializes a new session id and stores it in the session cookie
    # If user was authenticated, it will be similar to a log out
    session["sid"]  = str(uuid.uuid4())
    session["uid"] = None
    # Creates a new nonce associated to this session
    nonce = Nonce(session["sid"])
    # Stores the nonce in database
    nonce_db_service.create_nonce(nonce)
    # Gets the callback uri
    callback_uri = get_callback_uri()
    # Builds the challenge (bitid uri)
    bitid_uri = bitid.build_uri(callback_uri, nonce.nid)
    # Gets the qrcode uri
    qrcode = bitid.qrcode(bitid_uri)
    # Renders the login page
    params_tpl = {"callback_uri": callback_uri, "bitid_uri": bitid_uri, "qrcode": qrcode}
    return render_template('login.html', params_tpl=params_tpl)

@app.route("/callback", methods=["POST"])
@nocache
def callback():
    '''
    This function validates the response sent by the client about the challenge
    This is the route called by the bitcoin wallet when the challenge has been signed
    '''
    # Retrieves the callback uri
    callback_uri = get_callback_uri()
    # Extracts data from the posted request (from form of from json data according to the method used by the client)
    container = request.get_json(False, True, False) if request.mimetype == "application/json" else request.form
    bitid_uri = container["uri"]
    signature = container["signature"]
    address   = container["address"]

    #
    # Let's start by a bunch of validations
    #

    # Checks the address
    if not bitid.address_valid(address, USE_TESTNET):
        return jsonify(message = "Address is invalid or not legal"), 401
    # Checks the bitid uri
    if not bitid.uri_valid(bitid_uri, callback_uri):
        return jsonify(message = "BitID URI is invalid or not legal"), 401
    # Checks the signature
    if not bitid.signature_valid(address, signature, bitid_uri, callback_uri, USE_TESTNET):
        return jsonify(message = "Signature is incorrect"), 401

    # Note that the previous 3 steps could also be done in 1 step with following code:
    # if not bitid.challenge_valid(address, signature, bitid_uri, callback_uri, USE_TESTNET):
    #    return jsonify(message = "Sorry but something does not match"), 401

    # Checks the nonce
    nid = bitid.extract_nonce(bitid_uri)
    # Tries to retrieve the nonce from db
    nonce = nonce_db_service.get_nonce_by_nid(nid)
    if nonce is None:
        return jsonify(message = "NONCE is illegal"), 401
    elif nonce.has_expired():
        nonce_db_service.delete_nonce(nonce)
        return jsonify(message = "NONCE has expired"), 401

    #
    # So good so far, everything seems ok
    # It's time to check if we have a sign out or a sign in
    #

    # Checks if a user with the given address has already been registered in db (sign in)
    user = user_db_service.get_user_by_address(address)
    # If we have a sign out
    if user is None:
        # Here, we have an important check to do in order to avoid flooding of the users db
        # Let's check for a proof of goodwill (@see pybitid_demo.services.fake_tx_db_service)
        if not tx_db_service.check_proof_of_goodwill(address):
            return jsonify(message = "Address is invalid or not legal"), 401
        else:
            # Creates a new user and stores it in db
            user = User(address)
            if not user_db_service.create_user(user):
                return jsonify(message = "Ooops ! Something went wrong but we work on it"), 500

    # To finalize the authentication, let's set the user id in the nonce and update it in db
    nonce.uid = user.uid
    if not nonce_db_service.update_nonce(nonce):
        return jsonify(message = "Ooops ! Something went wrong but we work on it"), 500
    # Everything was ok: user is authenticated
    return jsonify(address = address, nonce = nonce.sid)

@app.route("/auth", methods=["GET"])
@nocache
def auth():
    '''
    This function checks if a challenge associated to a given address has been validated
    '''
    # Checks that session id is set
    if not session["sid"]:
        return jsonify(auth = 0)
    # Gets the nonce associated to the session id
    nonce = nonce_db_service.get_nonce_by_sid(session["sid"])
    if nonce is None:
        return jsonify(auth = 0)
    # Checks if nonce has an associated user id
    if nonce.uid is None:
        return jsonify(auth = 0)
    # Gets the user from db
    user = user_db_service.get_user_by_uid(nonce.uid)
    if user is None:
        return jsonify(auth = 0)
    # Let's increase the sign_in counter in user object (for demo purpose only)
    user.signin_count += 1
    user_db_service.update_user(user)
    # Everything is ok, let's finalize the authentication
    session["uid"] = user.uid
    nonce_db_service.delete_nonce(nonce)
    return jsonify(auth = 1)

@app.route("/user", methods=["GET"])
@nocache
def user():
    # Checks if user is logged
    if session["uid"] is None:
        return redirect(url_for("login")), 401
    user_address = ""
    user_signin_count = 0
    # Gets the user from db
    user = user_db_service.get_user_by_uid(session["uid"])
    if not user is None:
        user_address = user.address
        user_signin_count = user.signin_count
    params_tpl = {"user_address": user_address, "user_signin_count": user_signin_count}
    return render_template("user.html", params_tpl=params_tpl)

@app.route("/sign_out", methods=["GET"])
@nocache
def sign_out():
    session.pop("uid", None)
    return redirect(url_for("home"))

'''
Utils functions
'''
@app.template_filter("escape_slash")
def escape_slash_filter(s):
    return str.replace(s, "/", "\\/")

def get_callback_uri():
    '''
    Returns the callback uri for this site
    '''
    callback_uri = url_for("home", _external=True)
    parsed = urlparse(callback_uri)
    scheme = parsed.scheme
    netloc = parsed.netloc
    path = "/callback"
    return urlunparse((scheme, netloc, path, "", "", ""))

if __name__ == "__main__":
    # Comment/uncomment following lines to switch production / debug mode
    # Note that using BitId with a smartphone is not possible in debug mode (server only accessible by local machine)
    app.run(host='0.0.0.0')
    #app.run(debug=True)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Fri Nov 28 20:37:58 2014

@author: micha
"""

# The makefile takes these substitutions:
# 0 the modeldir
# 1 the GeNNPATH
# 2 a date as a timestamp
makefile = """
##--------------------------------------------------------------------------
##   Author: Thomas Nowotny
##
##   Institute: Center for Computational Neuroscience and Robotics
##              University of Sussex
##            Falmer, Brighton BN1 9QJ, UK
##
##   email to:  T.Nowotny@sussex.ac.uk
##
##   initial version: 2010-02-07
##
##--------------------------------------------------------------------------

ROOTDIR		:= {0}
EXECUTABLE	:= PyNNGeNN_model
SOURCES		:= PyNNGeNN_model.cu

INCLUDE_FLAGS	:=
LINK_FLAGS	:=
CCFLAGS		:=
NVCCFLAGS	:=

ifdef COMSPEC
	include	{1}/lib/include/makefile_common_cygwin.mk
else
	include	{1}/lib/include/makefile_common.mk
endif

## {2}
"""
<commit_msg>switched from {} syntax to python templates<commit_after># -*- coding: utf-8 -*-
"""
Created on Fri Nov 28 20:37:58 2014

@author: micha
"""

# The makefile takes these substitutions (all strings):
# $modeldir  - the modeldir
# $gp - the GeNNPATH
# $timestamp - a date as a timestamp
makefile = """
##--------------------------------------------------------------------------
##   Author: Thomas Nowotny
##
##   Institute: Center for Computational Neuroscience and Robotics
##              University of Sussex
##            Falmer, Brighton BN1 9QJ, UK
##
##   email to:  T.Nowotny@sussex.ac.uk
##
##   initial version: 2010-02-07
##
##--------------------------------------------------------------------------

ROOTDIR		:= $modeldir
EXECUTABLE	:= PyNNGeNN_model
SOURCES		:= PyNNGeNN_model.cu

INCLUDE_FLAGS	:=
LINK_FLAGS	:=
CCFLAGS		:=
NVCCFLAGS	:=

ifdef COMSPEC
	include	$gp/lib/include/makefile_common_cygwin.mk
else
	include	$gp/lib/include/makefile_common.mk
endif

## $timestamp
"""

<|endoftext|>


<commit_before>import monkey
<commit_msg>Add __version__ string in module. Fix #20<commit_after>import monkey
from ghpythonremote.version import __version__
<|endoftext|>


<commit_before># Copyright (c) 2006-2019  Andrey Golovigin
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Miscellaneous small utils."""
from __future__ import print_function, unicode_literals

import itertools
from collections import (
    MutableMapping, MutableSet, OrderedDict, Sequence, deque
)
from functools import wraps
from types import GeneratorType

from six.moves import zip_longest

from .py3compat import fix_unicode_literals_in_doctest

def deprecated(since, reason=None):
    def decorator(f):
        @wraps(f)
        def new_f(*args, **kwargs):
            import warnings
            message = u'{0}() is deprecated since {1}'.format(f.__name__, since)
            if reason:
                message += ': {0}'.format(reason)
            warnings.warn(message, DeprecationWarning, stacklevel=2)
            return f(*args, **kwargs)
        return new_f
    return decorator

def memoize(f, capacity=1024):
    memory = {}
    history = deque()

    @wraps(f)
    def new_f(*args):
        if args not in memory:
            if len(history) >= capacity:
                del memory[history.popleft()]
            memory[args] = f(*args)
            history.append(args)
        return memory[args]
    return new_f

def collect_iterable(f):
    @wraps(f)
    def new_f(*args, **kwargs):
        return list(f(*args, **kwargs))
    return new_f

def pairwise(iterable):
    a, b = itertools.tee(iterable)
    next(b, None)
    return zip_longest(a, b)

@fix_unicode_literals_in_doctest
class CaseInsensitiveDict(MutableMapping):
    """A dict with case-insensitive lookup.

    >>> d = CaseInsensitiveDict(TesT='passed')
    >>> d
    CaseInsensitiveDict({'TesT': u'passed'})
    >>> d.lower()
    CaseInsensitiveDict({'test': u'passed'})
    >>> print(d['TesT'])
    passed
    >>> print(d['test'])
    passed
    >>> print(d['Test'])
    passed
    >>> print(d.get('test'))
    passed
    >>> print(d.get('Test'))
    passed

    >>> d['Test'] = 'passed again'
    >>> print(d['test'])
    passed again
    >>> 'test' in d
    True
    >>> 'Test' in d
    True
    >>> list(d.keys())
    [u'Test']
    >>> for key in d:
    ...     print(key)
    Test
    >>> for key, value in d.items():
    ...     print(key, value)
    Test passed again
    >>> bool(d)
    True
    >>> len(d)
    1

    >>> del d['test']
    >>> len(d)
    0
    >>> bool(d)
    False
    >>> 'test' in d
    False
    >>> 'Test' in d
    False
    >>> print(d.get('test'))
    None
    >>> print(d.get('Test'))
    None
    >>> print(d.get('Test', 'failed'))
    failed

    >>> CaseInsensitiveDict(
    ...     (key, value) for key, value in [('a', 'b')]
    ... )
    CaseInsensitiveDict({u'a': u'b'})

    """

    def __init__(self, *args, **kwargs):
        initial = dict(*args, **kwargs)
        self._dict = dict((key.lower(), value) for key, value in initial.items())
        self._keys = dict((key.lower(), key) for key in initial)

    def __len__(self):
        return len(self._dict)

    def __iter__(self):
        return iter(self._keys.values())

    def __setitem__(self, key, value):
        """To implement lowercase keys."""
        key_lower = key.lower()
        self._dict[key_lower] = value
        self._keys[key_lower] = key

    def __getitem__(self, key):
        return self._dict[key.lower()]

    def __delitem__(self, key):
        key_lower = key.lower()
        del self._dict[key_lower]
        del self._keys[key_lower]

    def __contains__(self, key):
        return key.lower() in self._dict

    def __repr__(self):
        """A caselessDict version of __repr__ """
        dct = dict((key, self[key]) for key in self)
        return '{0}({1})'.format(
            type(self).__name__, repr(dct),
        )

    def items_lower(self):
        return ((key.lower(), value) for key, value in self.items())

    def lower(self):
        return type(self)(self.items_lower())

class CaseInsensitiveDefaultDict(CaseInsensitiveDict):
    """CaseInseisitiveDict with default factory, like collections.defaultdict

    >>> d = CaseInsensitiveDefaultDict(int)
    >>> d['a']
    0
    >>> d['a'] += 1
    >>> d['a']
    1
    >>> d['A']
    1
    >>> d['a'] = 3
    >>> d['a']
    3
    >>> d['B'] += 10
    >>> d['b']
    10

    """
    def __init__(self, default_factory):
        super(CaseInsensitiveDefaultDict, self).__init__()
        self.default_factory = default_factory

    def __getitem__(self, key):
        try:
            return super(CaseInsensitiveDefaultDict, self).__getitem__(key)
        except KeyError:
            return self.default_factory()

@fix_unicode_literals_in_doctest
class OrderedCaseInsensitiveDict(CaseInsensitiveDict):
    """ An (incomplete) ordered case-insensitive dict.

    >>> d = OrderedCaseInsensitiveDict([
    ...     ('Uno', 1),
    ...     ('Dos', 2),
    ...     ('Tres', 3),
    ... ])
    >>> d
    OrderedCaseInsensitiveDict({u'Uno': 1, u'Dos': 2, u'Tres': 3})
    >>> d.lower()
    OrderedCaseInsensitiveDict({u'uno': 1, u'dos': 2, u'tres': 3})
    >>> list(d.keys())
    [u'Uno', u'Dos', u'Tres']
    >>> list(d.items())
    [(u'Uno', 1), (u'Dos', 2), (u'Tres', 3)]
    >>> list(d.values())
    [1, 2, 3]
    >>> d['Cuatro'] = 4
    >>> list(d.keys())
    [u'Uno', u'Dos', u'Tres', u'Cuatro']
    >>> list(d.items())
    [(u'Uno', 1), (u'Dos', 2), (u'Tres', 3), (u'Cuatro', 4)]
    >>> list(d.values())
    [1, 2, 3, 4]
    >>> list(d)
    [u'Uno', u'Dos', u'Tres', u'Cuatro']
    >>> 'Uno' in d
    True
    >>> 'uno' in d
    True
    >>> d['Uno']
    1
    >>> d['uno']
    1
    >>> d['UNO']
    1
    >>> 'Cuatro' in d
    True
    >>> 'CUATRO' in d
    True
    >>> d['Cuatro']
    4
    >>> d['cuatro']
    4
    >>> d['UNO'] = 'one'
    >>> d['uno']
    u'one'
    >>> d['Uno']
    u'one'
    >>> list(d.keys())
    [u'Uno', u'Dos', u'Tres', u'Cuatro']
    >>> d['cuatro'] = 'four'
    >>> d['Cuatro']
    u'four'
    >>> d['cuatro']
    u'four'
    >>> list(d.keys())
    [u'Uno', u'Dos', u'Tres', u'Cuatro']
    >>> list(d.values())
    [u'one', 2, 3, u'four']
    >>> del d['dos']
    >>> list(d.keys())
    [u'UNO', u'Tres', u'cuatro']
    >>> list(d.values())
    [u'one', 3, u'four']
    """

    def __init__(self, *args, **kwargs):
        initial = OrderedDict(*args, **kwargs)
        self._dict = dict((key.lower(), value) for key, value in initial.items())
        self._keys = OrderedDict((key.lower(), key) for key in initial)

@fix_unicode_literals_in_doctest
class CaseInsensitiveSet(MutableSet):
    """A very basic case-insensitive set.

    >>> s = CaseInsensitiveSet()
    >>> len(s)
    0
    >>> 'a' in s
    False
    >>> list(CaseInsensitiveSet(['aaa', 'Aaa', 'AAA']))
    [u'aaa']
    >>> s = CaseInsensitiveSet(['Aaa', 'Bbb'])
    >>> s
    CaseInsensitiveSet([u'Aaa', u'Bbb'])
    >>> s.lower()
    CaseInsensitiveSet([u'aaa', u'bbb'])
    >>> len(s)
    2
    >>> 'aaa' in s
    True
    >>> 'Aaa' in s
    True
    >>> 'AAA' in s
    True
    >>> 'bbb' in s
    True
    >>> 'Bbb' in s
    True
    >>> 'abc' in s
    False
    >>> s.add('ccc')
    >>> len(s)
    3
    >>> 'aaa' in s
    True
    >>> 'ccc' in s
    True
    >>> s.remove('AAA')
    >>> len(s)
    2
    >>> 'aaa' in s
    False

    >>> bool(CaseInsensitiveSet(['a']))
    True
    >>> bool(CaseInsensitiveSet([]))
    False
    >>> bool(CaseInsensitiveSet())
    False

    """

    def __init__(self, iterable=()):
        self._set = set()
        self._keys = dict()
        for item in iterable:
            self.add(item)

    def __contains__(self, key):
        return key.lower() in self._set

    def __iter__(self):
        return iter(self._set)

    def __len__(self):
        return len(self._set)

    def __repr__(self):
        """A caselessDict version of __repr__ """
        return '{0}({1})'.format(
            type(self).__name__, repr(sorted(self._keys.values()))
        )

    def add(self, key):
        key_lower = key.lower()
        self._set.add(key_lower)
        self._keys[key_lower] = key

    def discard(self, key):
        key_lower = key.lower()
        self._set.discard(key_lower)
        self._keys.pop(key_lower, None)

    def get_canonical_key(self, key):
        return self._keys[key.lower()]

    def lower(self):
        return type(self)(self._set)
<commit_msg>Fix tests<commit_after># Copyright (c) 2006-2019  Andrey Golovigin
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Miscellaneous small utils."""
from __future__ import print_function, unicode_literals

import itertools
from collections import (
    MutableMapping, MutableSet, OrderedDict, Sequence, deque
)
from functools import wraps
from types import GeneratorType

from six.moves import zip_longest

from .py3compat import fix_unicode_literals_in_doctest

def deprecated(since, reason=None):
    def decorator(f):
        @wraps(f)
        def new_f(*args, **kwargs):
            import warnings
            message = u'{0}() is deprecated since {1}'.format(f.__name__, since)
            if reason:
                message += ': {0}'.format(reason)
            warnings.warn(message, DeprecationWarning, stacklevel=2)
            return f(*args, **kwargs)
        return new_f
    return decorator

def memoize(f, capacity=1024):
    memory = {}
    history = deque()

    @wraps(f)
    def new_f(*args):
        if args not in memory:
            if len(history) >= capacity:
                del memory[history.popleft()]
            memory[args] = f(*args)
            history.append(args)
        return memory[args]
    return new_f

def collect_iterable(f):
    @wraps(f)
    def new_f(*args, **kwargs):
        return list(f(*args, **kwargs))
    return new_f

def pairwise(iterable):
    a, b = itertools.tee(iterable)
    next(b, None)
    return zip_longest(a, b)

@fix_unicode_literals_in_doctest
class CaseInsensitiveDict(MutableMapping):
    """A dict with case-insensitive lookup.

    >>> d = CaseInsensitiveDict(TesT='passed')
    >>> d
    CaseInsensitiveDict({'TesT': u'passed'})
    >>> d.lower()
    CaseInsensitiveDict({'test': u'passed'})
    >>> print(d['TesT'])
    passed
    >>> print(d['test'])
    passed
    >>> print(d['Test'])
    passed
    >>> print(d.get('test'))
    passed
    >>> print(d.get('Test'))
    passed

    >>> d['Test'] = 'passed again'
    >>> print(d['test'])
    passed again
    >>> 'test' in d
    True
    >>> 'Test' in d
    True
    >>> list(d.keys())
    [u'Test']
    >>> for key in d:
    ...     print(key)
    Test
    >>> for key, value in d.items():
    ...     print(key, value)
    Test passed again
    >>> bool(d)
    True
    >>> len(d)
    1

    >>> del d['test']
    >>> len(d)
    0
    >>> bool(d)
    False
    >>> 'test' in d
    False
    >>> 'Test' in d
    False
    >>> print(d.get('test'))
    None
    >>> print(d.get('Test'))
    None
    >>> print(d.get('Test', 'failed'))
    failed

    >>> CaseInsensitiveDict(
    ...     (key, value) for key, value in [('a', 'b')]
    ... )
    CaseInsensitiveDict({u'a': u'b'})

    """

    def __init__(self, *args, **kwargs):
        initial = dict(*args, **kwargs)
        self._dict = dict((key.lower(), value) for key, value in initial.items())
        self._keys = dict((key.lower(), key) for key in initial)

    def __len__(self):
        return len(self._dict)

    def __iter__(self):
        return iter(self._keys.values())

    def __setitem__(self, key, value):
        """To implement lowercase keys."""
        key_lower = key.lower()
        self._dict[key_lower] = value
        self._keys[key_lower] = key

    def __getitem__(self, key):
        return self._dict[key.lower()]

    def __delitem__(self, key):
        key_lower = key.lower()
        del self._dict[key_lower]
        del self._keys[key_lower]

    def __contains__(self, key):
        return key.lower() in self._dict

    def __repr__(self):
        """A caselessDict version of __repr__ """
        dct = dict((key, self[key]) for key in self)
        return '{0}({1})'.format(
            type(self).__name__, repr(dct),
        )

    def items_lower(self):
        return ((key.lower(), value) for key, value in self.items())

    def lower(self):
        return type(self)(self.items_lower())

class CaseInsensitiveDefaultDict(CaseInsensitiveDict):
    """CaseInseisitiveDict with default factory, like collections.defaultdict

    >>> d = CaseInsensitiveDefaultDict(int)
    >>> d['a']
    0
    >>> d['a'] += 1
    >>> d['a']
    1
    >>> d['A']
    1
    >>> d['a'] = 3
    >>> d['a']
    3
    >>> d['B'] += 10
    >>> d['b']
    10

    """
    def __init__(self, default_factory):
        super(CaseInsensitiveDefaultDict, self).__init__()
        self.default_factory = default_factory

    def __getitem__(self, key):
        try:
            return super(CaseInsensitiveDefaultDict, self).__getitem__(key)
        except KeyError:
            return self.default_factory()

@fix_unicode_literals_in_doctest
class OrderedCaseInsensitiveDict(CaseInsensitiveDict):
    """ An (incomplete) ordered case-insensitive dict.

    >>> d = OrderedCaseInsensitiveDict([
    ...     ('Uno', 1),
    ...     ('Dos', 2),
    ...     ('Tres', 3),
    ... ])
    >>> d
    OrderedCaseInsensitiveDict({u'Uno': 1, u'Dos': 2, u'Tres': 3})
    >>> d.lower()
    OrderedCaseInsensitiveDict({u'uno': 1, u'dos': 2, u'tres': 3})
    >>> list(d.keys())
    [u'Uno', u'Dos', u'Tres']
    >>> list(d.items())
    [(u'Uno', 1), (u'Dos', 2), (u'Tres', 3)]
    >>> list(d.values())
    [1, 2, 3]
    >>> d['Cuatro'] = 4
    >>> list(d.keys())
    [u'Uno', u'Dos', u'Tres', u'Cuatro']
    >>> list(d.items())
    [(u'Uno', 1), (u'Dos', 2), (u'Tres', 3), (u'Cuatro', 4)]
    >>> list(d.values())
    [1, 2, 3, 4]
    >>> list(d)
    [u'Uno', u'Dos', u'Tres', u'Cuatro']
    >>> 'Uno' in d
    True
    >>> 'uno' in d
    True
    >>> d['Uno']
    1
    >>> d['uno']
    1
    >>> d['UNO']
    1
    >>> 'Cuatro' in d
    True
    >>> 'CUATRO' in d
    True
    >>> d['Cuatro']
    4
    >>> d['cuatro']
    4
    >>> d['UNO'] = 'one'
    >>> d['uno']
    u'one'
    >>> d['Uno']
    u'one'
    >>> list(d.keys())
    [u'UNO', u'Dos', u'Tres', u'Cuatro']
    >>> d['cuatro'] = 'four'
    >>> d['Cuatro']
    u'four'
    >>> d['cuatro']
    u'four'
    >>> list(d.keys())
    [u'UNO', u'Dos', u'Tres', u'cuatro']
    >>> list(d.values())
    [u'one', 2, 3, u'four']
    >>> del d['dos']
    >>> list(d.keys())
    [u'UNO', u'Tres', u'cuatro']
    >>> list(d.values())
    [u'one', 3, u'four']
    """

    def __init__(self, *args, **kwargs):
        initial = OrderedDict(*args, **kwargs)
        self._dict = dict((key.lower(), value) for key, value in initial.items())
        self._keys = OrderedDict((key.lower(), key) for key in initial)

@fix_unicode_literals_in_doctest
class CaseInsensitiveSet(MutableSet):
    """A very basic case-insensitive set.

    >>> s = CaseInsensitiveSet()
    >>> len(s)
    0
    >>> 'a' in s
    False
    >>> list(CaseInsensitiveSet(['aaa', 'Aaa', 'AAA']))
    [u'aaa']
    >>> s = CaseInsensitiveSet(['Aaa', 'Bbb'])
    >>> s
    CaseInsensitiveSet([u'Aaa', u'Bbb'])
    >>> s.lower()
    CaseInsensitiveSet([u'aaa', u'bbb'])
    >>> len(s)
    2
    >>> 'aaa' in s
    True
    >>> 'Aaa' in s
    True
    >>> 'AAA' in s
    True
    >>> 'bbb' in s
    True
    >>> 'Bbb' in s
    True
    >>> 'abc' in s
    False
    >>> s.add('ccc')
    >>> len(s)
    3
    >>> 'aaa' in s
    True
    >>> 'ccc' in s
    True
    >>> s.remove('AAA')
    >>> len(s)
    2
    >>> 'aaa' in s
    False

    >>> bool(CaseInsensitiveSet(['a']))
    True
    >>> bool(CaseInsensitiveSet([]))
    False
    >>> bool(CaseInsensitiveSet())
    False

    """

    def __init__(self, iterable=()):
        self._set = set()
        self._keys = dict()
        for item in iterable:
            self.add(item)

    def __contains__(self, key):
        return key.lower() in self._set

    def __iter__(self):
        return iter(self._set)

    def __len__(self):
        return len(self._set)

    def __repr__(self):
        """A caselessDict version of __repr__ """
        return '{0}({1})'.format(
            type(self).__name__, repr(sorted(self._keys.values()))
        )

    def add(self, key):
        key_lower = key.lower()
        self._set.add(key_lower)
        self._keys[key_lower] = key

    def discard(self, key):
        key_lower = key.lower()
        self._set.discard(key_lower)
        self._keys.pop(key_lower, None)

    def get_canonical_key(self, key):
        return self._keys[key.lower()]

    def lower(self):
        return type(self)(self._set)
<|endoftext|>


<commit_before>from __future__ import print_function
import re

r_line = re.compile(r"^(syn keyword vimCommand contained|syn keyword vimOption "
                    r"contained|syn keyword vimAutoEvent contained)\s+(.*)")
r_item = re.compile(r"(\w+)(?:\[(\w+)\])?")

def getkw(input, output):
    out = file(output, 'w')

    output_info = {'command': [], 'option': [], 'auto': []}
    for line in file(input):
        m = r_line.match(line)
        if m:
            # Decide which output gets mapped to d
            if 'vimCommand' in m.group(1):
                d = output_info['command']
            elif 'AutoEvent' in m.group(1):
                d = output_info['auto']
            else:
                d = output_info['option']

            # Extract all the shortened versions
            for i in r_item.finditer(m.group(2)):
                d.append('(%r,%r)' %
                         (i.group(1), "%s%s" % (i.group(1), i.group(2) or '')))

    output_info['option'].append("('nnoremap','nnoremap')")
    output_info['option'].append("('inoremap','inoremap')")
    output_info['option'].append("('vnoremap','vnoremap')")

    for a, b in output_info.items():
        b.sort()
        print('%s=[%s]' % (a, ','.join(b)), file=out)

def is_keyword(w, keywords):
    for i in range(len(w), 0, -1):
        if w[:i] in keywords:
            return keywords[w[:i]][:len(w)] == w
    return False

if __name__ == "__main__":
    getkw("/usr/share/vim/vim73/syntax/vim.vim", "temp.py")
<commit_msg>New version of the vim keyword script, using format_lines<commit_after>from __future__ import print_function

import re

from pygments.util import format_lines

r_line = re.compile(r"^(syn keyword vimCommand contained|syn keyword vimOption "
                    r"contained|syn keyword vimAutoEvent contained)\s+(.*)")
r_item = re.compile(r"(\w+)(?:\[(\w+)\])?")

HEADER = '''\
# This file is autogenerated by scripts/get_vimkw.py

# Split up in multiple functions so it's importable by jython, which has a
# per-method size limit.
'''

METHOD = '''\
def _get%(key)s():
%(body)s
    return var
%(key)s = _get%(key)s()
'''

def getkw(input, output):
    out = file(output, 'w')

    # Copy template from an existing file.
    print(HEADER, file=out)

    output_info = {'command': [], 'option': [], 'auto': []}
    for line in file(input):
        m = r_line.match(line)
        if m:
            # Decide which output gets mapped to d
            if 'vimCommand' in m.group(1):
                d = output_info['command']
            elif 'AutoEvent' in m.group(1):
                d = output_info['auto']
            else:
                d = output_info['option']

            # Extract all the shortened versions
            for i in r_item.finditer(m.group(2)):
                d.append('(%r,%r)' %
                         (i.group(1), "%s%s" % (i.group(1), i.group(2) or '')))

    output_info['option'].append("('nnoremap','nnoremap')")
    output_info['option'].append("('inoremap','inoremap')")
    output_info['option'].append("('vnoremap','vnoremap')")

    for key, keywordlist in output_info.items():
        keywordlist.sort()
        body = format_lines('var', keywordlist, raw=True, indent_level=1)
        print(METHOD % locals(), file=out)

def is_keyword(w, keywords):
    for i in range(len(w), 0, -1):
        if w[:i] in keywords:
            return keywords[w[:i]][:len(w)] == w
    return False

if __name__ == "__main__":
    getkw("/usr/share/vim/vim74/syntax/vim.vim",
          "pygments/lexers/_vimbuiltins.py")
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# tools.py
#
# Reimplementations of Santi's trim_longitudes.py and blastclust_lst2fasta.py
# scripts/functions, and other miscellaneous functions
#
# (c) The James Hutton Institute 2016
# Author: Leighton Pritchard and Peter Thorpe

import os

import gzip
from subprocess import check_output, CalledProcessError
import hashlib
import sys

from collections import defaultdict
from optparse import OptionParser

from Bio import SeqIO
from Bio.SeqRecord import SeqRecord

class NotExecutableError(Exception):
    """Exception raised when expected executable is not executable"""
    def __init__(self, message):
        self.message = message

def is_exe(filename):
    """Returns True if path is to an executable file"""
    if os.path.isfile(filename) and os.access(filename, os.X_OK):
        return True
    else:
        try:
            exefile = check_output(["which", filename]).strip()
        except CalledProcessError:
            raise NotExecutableError("{0} does not exist".format(filename))
    return os.path.isfile(exefile) and os.access(exefile, os.X_OK)

# Defaults  settings are for the current ITS1 phy
# primers and the development ITS1 Phy database. DON'T BLINDLY USE THESE
# The filenames, we now provide directly.
def trim_seq(infname, outfname, lclip=53, rclip=0, minlen=100):
    """Trims all FASTA sequences in infname by lclip and rclip on the
    'left'- and 'right'-hand ends, respectively and writes the results to
    outfname. Any clipped sequence with length < minlen is not written.

    Defaults are equivalent to the default settings are for the current ITS1
    phy primers and the development ITS1 Phy database. PLEASE DONT BLINDLY
    USE THESE. The filenames, we now provide directly.
    """
    # ensure these are int, as they may have been passed as a string.
    lclip = int(lclip)
    rclip = int(rclip)
    with open(infname, 'r') as fh:
        # rclip needs to be set up to allow for zero values, using logic:
        # rclip_coordinate = len(s) - rclip
        # Use generators to save memory
        s_trim = (s[lclip:(len(s) - rclip)] for s in SeqIO.parse(fh, 'fasta'))
        return SeqIO.write((s for s in s_trim if len(s) >= minlen),
                           outfname, 'fasta')

def convert_fq_to_fa(in_file, out_file):
    """ Function to convert fq to fa
    -i the fastq file to be converted
    -o the desired out fasta file name
    the function will auto detect if the file is .gz
    requires biopython
    """
    # if the file is .gz we need to deal with it
    if in_file.endswith(".gz"):
        in_file = gzip.open(in_file, mode='rt', compresslevel=9,
                            encoding=None, errors=None,
                            newline=None)
    SeqIO.convert(in_file, "fastq", out_file, "fasta")

def deduplicate_and_rename(seqlist):
    """Removes duplicates from the passed SeqRecords, and replaces sequence
    IDs with the md5 hash of the sequence, suffixed with the number of
    sequences that were replaced. Returns a tuple (sequences, names_old_to_new)
    where names_old_to_new is a list of tuples of (old_name, hash) pairs.

    - sequences   Iterable of Bio.SeqRecord objects
    """
    names_old_to_new = list()
    abundance = defaultdict(int)
    hash_to_seq = defaultdict(str)
    hash_to_name = defaultdict(str)
    # compile hashed sequence data - the loop is necessary because we need
    # to run through completely once to obtain abundance info. If memory
    # to store this becomes an issue, we might want to try an alternative
    # approach
    for seq in seqlist:
        seqhash = hashlib.md5(str(seq.seq).encode()).hexdigest()
        abundance[seqhash] += 1
        hash_to_seq[seqhash] = seq.seq
        hash_to_name[seqhash] = seq.id
        database_output = ("{0}\t{1}\n".format(seq.id, seqhash))
        names_old_to_new.append(database_output)
    # generate list of sequences, named for the hash, including abundance data,
    # and return with ID:hash lookup
    seqlist = list()
    for name, abundance_val in abundance.items():
        seqname = "{0}_{1}".format(name, abundance_val)
        seqlist.append(SeqRecord(id=seqname, description="",
                                 seq=hash_to_seq[name]))
    return(seqlist, names_old_to_new, hash_to_seq)

def dereplicate_name(fasta, database_out, out):
    """function to dereplicate the seq. Thus generating an
    abundance of that seq. Rename the seq to something
    swarm will work with.
    - fasta     - fasta file of assembled seq
    - database_out   - ouput old name to new file.
    -out        - outfile
    """
    # open files to write to
    fasta_out = open(out, 'w')
    name_out = open(database_out, "w")
    # convert the file to a list of Seqrecord objects
    seqlist = list(SeqIO.parse(fasta, 'fasta'))
    seqlist, names_old_to_new, hash_to_seq = deduplicate_and_rename(seqlist)
    for i in (names_old_to_new):
        name_out.write(i)
    for seq_record in (seqlist):
        SeqIO.write(seq_record, fasta_out, "fasta")
    # close the open files
    fasta_out.close()
    name_out.close()

def check_OTU_db_abundance_val(OTUfasta):
    """this is a function to check that the OTU database has _1
    at the end of the title, this is required for swamr clustering
    takes in a fasta and if required, writes out a new one"""
    # we will iterate through first to see if we need to rewrite it.
    bad_count = 0
    for seq_record in SeqIO.parse(OTUfasta, "fasta"):
        if not seq_record.id.endswith("_1"):
            # print ("OTU db %s missing abundance value" % seq_record.id)
            bad_count = bad_count + 1
    if bad_count > 0:
        # only if we need to write to a new file, we will.
        outfile = OTUfasta.split(".fa")[0] + "abundance.fasta"
        f_out = open(outfile, 'w')
        for seq_record in SeqIO.parse(OTUfasta, "fasta"):
            seq_record.id = seq_record.id + "_1"
            seq_record.description = ""
            SeqIO.write(seq_record, f_out, "fasta")
        f_out.close()
        return outfile
    else:
        return OTUfasta

# Function replacing Santi's blastclust_lst2fasta.py script
def blastclust_to_fasta(infname, seqfname, outdir):
    """Converts input BLASTCLUST output list to a subdirectory of FASTA files,
    each of which contains all sequences from a single cluster. The sequences
    matching the IDs listed in BLASTCLUST output should all be in the file
    seqfname.
    """
    outdirname = os.path.join(outdir, "blastclust_OTUs")
    if not os.path.exists(outdirname):
        os.makedirs(outdirname)
    seqdict = SeqIO.index(seqfname, 'fasta')
    with open(infname, 'r') as fh:
        otu_id = 0
        for line in fh:
            otu_id += 1
            outfname = os.path.join(outdirname,
                                    "blastclust_OTU_%06d.fasta" % otu_id)
            SeqIO.write((seqdict[key] for key in line.split()),
                        outfname, 'fasta')
    return outdirname

# the following three function are to rename the clusters back to their
# original names
def coded_name_to_species(database_file):
    """functiong takes the already generated tab separated
    database of coded name to species file. Returns a dic
    of coded_name to species"""
    with open(database_file) as file:
        data = file.read().split("\n")
    coded_name_to_species_dict = dict()
    for line in data:
        if not line.strip():
            continue  # if the last line is blank
        if line.startswith("#"):
            continue
        coded_name, species = line.split("\t")
        coded_name_to_species_dict[coded_name.rstrip()] = species.rstrip()
    return coded_name_to_species_dict

def parse_tab_file_get_clusters(filename1, database, out_file):
    """function to open up a tab or space separeted clustering
    output and rename according to the name in the database file.
    Abundance is also appended to the name"""
    # call the function to get the dictionary
    # populated with the database
    coded_name_to_species_dict = coded_name_to_species(database)
    cluster_file = open(filename1, "r")
    summary_out_file = open(out_file, "w")

    count = int(0)
    # iterate through the cluster file. One line per cluster
    for line in cluster_file:
        if not line.strip():
            continue  # if the last line is blank
        if line.startswith("#"):  # dont want comment lines
            continue
        output_str = ""
        if "\t" in line:
            cluster_line = line.rstrip("\n").split("\t")
        else:
            # different clustering program?
            cluster_line = line.rstrip("\n").split()
        count += 1
        for member in cluster_line:
            try:
                # data would be: 2f1454d16278fda2d44f26ebf7a0ed05_310
                # _abundance value
                split_name = member.split("_")[:-1]
                species = ("_").join(split_name)
                print (species)
                abundance = member.split("_")[-1]
                # print (abundance)
            except:
                KeyError
                errmsg = """something went wrong with decoding the names maybe
                your names were not separated in your database? If so, the
                block of code above this staement need adjusting accordingly.
                """
                sys.exit(os.system(errmsg))
            # add the info to a str, we will write at the
            # end of the cluster line
            cluster_summary = "%s_abundance=%s\t" % (species, abundance)
            output_str = output_str + cluster_summary
        summary_out_file.write(output_str+"\n")
    # close the files
    cluster_file.close()
    summary_out_file.close()
<commit_msg>removed print statement<commit_after>#!/usr/bin/env python
#
# tools.py
#
# Reimplementations of Santi's trim_longitudes.py and blastclust_lst2fasta.py
# scripts/functions, and other miscellaneous functions
#
# (c) The James Hutton Institute 2016
# Author: Leighton Pritchard and Peter Thorpe

import os

import gzip
from subprocess import check_output, CalledProcessError
import hashlib
import sys

from collections import defaultdict
from optparse import OptionParser

from Bio import SeqIO
from Bio.SeqRecord import SeqRecord

class NotExecutableError(Exception):
    """Exception raised when expected executable is not executable"""
    def __init__(self, message):
        self.message = message

def is_exe(filename):
    """Returns True if path is to an executable file"""
    if os.path.isfile(filename) and os.access(filename, os.X_OK):
        return True
    else:
        try:
            exefile = check_output(["which", filename]).strip()
        except CalledProcessError:
            raise NotExecutableError("{0} does not exist".format(filename))
    return os.path.isfile(exefile) and os.access(exefile, os.X_OK)

# Defaults  settings are for the current ITS1 phy
# primers and the development ITS1 Phy database. DON'T BLINDLY USE THESE
# The filenames, we now provide directly.
def trim_seq(infname, outfname, lclip=53, rclip=0, minlen=100):
    """Trims all FASTA sequences in infname by lclip and rclip on the
    'left'- and 'right'-hand ends, respectively and writes the results to
    outfname. Any clipped sequence with length < minlen is not written.

    Defaults are equivalent to the default settings are for the current ITS1
    phy primers and the development ITS1 Phy database. PLEASE DONT BLINDLY
    USE THESE. The filenames, we now provide directly.
    """
    # ensure these are int, as they may have been passed as a string.
    lclip = int(lclip)
    rclip = int(rclip)
    with open(infname, 'r') as fh:
        # rclip needs to be set up to allow for zero values, using logic:
        # rclip_coordinate = len(s) - rclip
        # Use generators to save memory
        s_trim = (s[lclip:(len(s) - rclip)] for s in SeqIO.parse(fh, 'fasta'))
        return SeqIO.write((s for s in s_trim if len(s) >= minlen),
                           outfname, 'fasta')

def convert_fq_to_fa(in_file, out_file):
    """ Function to convert fq to fa
    -i the fastq file to be converted
    -o the desired out fasta file name
    the function will auto detect if the file is .gz
    requires biopython
    """
    # if the file is .gz we need to deal with it
    if in_file.endswith(".gz"):
        in_file = gzip.open(in_file, mode='rt', compresslevel=9,
                            encoding=None, errors=None,
                            newline=None)
    SeqIO.convert(in_file, "fastq", out_file, "fasta")

def deduplicate_and_rename(seqlist):
    """Removes duplicates from the passed SeqRecords, and replaces sequence
    IDs with the md5 hash of the sequence, suffixed with the number of
    sequences that were replaced. Returns a tuple (sequences, names_old_to_new)
    where names_old_to_new is a list of tuples of (old_name, hash) pairs.

    - sequences   Iterable of Bio.SeqRecord objects
    """
    names_old_to_new = list()
    abundance = defaultdict(int)
    hash_to_seq = defaultdict(str)
    hash_to_name = defaultdict(str)
    # compile hashed sequence data - the loop is necessary because we need
    # to run through completely once to obtain abundance info. If memory
    # to store this becomes an issue, we might want to try an alternative
    # approach
    for seq in seqlist:
        seqhash = hashlib.md5(str(seq.seq).encode()).hexdigest()
        abundance[seqhash] += 1
        hash_to_seq[seqhash] = seq.seq
        hash_to_name[seqhash] = seq.id
        database_output = ("{0}\t{1}\n".format(seq.id, seqhash))
        names_old_to_new.append(database_output)
    # generate list of sequences, named for the hash, including abundance data,
    # and return with ID:hash lookup
    seqlist = list()
    for name, abundance_val in abundance.items():
        seqname = "{0}_{1}".format(name, abundance_val)
        seqlist.append(SeqRecord(id=seqname, description="",
                                 seq=hash_to_seq[name]))
    return(seqlist, names_old_to_new, hash_to_seq)

def dereplicate_name(fasta, database_out, out):
    """function to dereplicate the seq. Thus generating an
    abundance of that seq. Rename the seq to something
    swarm will work with.
    - fasta     - fasta file of assembled seq
    - database_out   - ouput old name to new file.
    -out        - outfile
    """
    # open files to write to
    fasta_out = open(out, 'w')
    name_out = open(database_out, "w")
    # convert the file to a list of Seqrecord objects
    seqlist = list(SeqIO.parse(fasta, 'fasta'))
    seqlist, names_old_to_new, hash_to_seq = deduplicate_and_rename(seqlist)
    for i in (names_old_to_new):
        name_out.write(i)
    for seq_record in (seqlist):
        SeqIO.write(seq_record, fasta_out, "fasta")
    # close the open files
    fasta_out.close()
    name_out.close()

def check_OTU_db_abundance_val(OTUfasta):
    """this is a function to check that the OTU database has _1
    at the end of the title, this is required for swamr clustering
    takes in a fasta and if required, writes out a new one"""
    # we will iterate through first to see if we need to rewrite it.
    bad_count = 0
    for seq_record in SeqIO.parse(OTUfasta, "fasta"):
        if not seq_record.id.endswith("_1"):
            # print ("OTU db %s missing abundance value" % seq_record.id)
            bad_count = bad_count + 1
    if bad_count > 0:
        # only if we need to write to a new file, we will.
        outfile = OTUfasta.split(".fa")[0] + "abundance.fasta"
        f_out = open(outfile, 'w')
        for seq_record in SeqIO.parse(OTUfasta, "fasta"):
            seq_record.id = seq_record.id + "_1"
            seq_record.description = ""
            SeqIO.write(seq_record, f_out, "fasta")
        f_out.close()
        return outfile
    else:
        return OTUfasta

# Function replacing Santi's blastclust_lst2fasta.py script
def blastclust_to_fasta(infname, seqfname, outdir):
    """Converts input BLASTCLUST output list to a subdirectory of FASTA files,
    each of which contains all sequences from a single cluster. The sequences
    matching the IDs listed in BLASTCLUST output should all be in the file
    seqfname.
    """
    outdirname = os.path.join(outdir, "blastclust_OTUs")
    if not os.path.exists(outdirname):
        os.makedirs(outdirname)
    seqdict = SeqIO.index(seqfname, 'fasta')
    with open(infname, 'r') as fh:
        otu_id = 0
        for line in fh:
            otu_id += 1
            outfname = os.path.join(outdirname,
                                    "blastclust_OTU_%06d.fasta" % otu_id)
            SeqIO.write((seqdict[key] for key in line.split()),
                        outfname, 'fasta')
    return outdirname

# the following three function are to rename the clusters back to their
# original names
def coded_name_to_species(database_file):
    """functiong takes the already generated tab separated
    database of coded name to species file. Returns a dic
    of coded_name to species"""
    with open(database_file) as file:
        data = file.read().split("\n")
    coded_name_to_species_dict = dict()
    for line in data:
        if not line.strip():
            continue  # if the last line is blank
        if line.startswith("#"):
            continue
        coded_name, species = line.split("\t")
        coded_name_to_species_dict[coded_name.rstrip()] = species.rstrip()
    return coded_name_to_species_dict

def parse_tab_file_get_clusters(filename1, database, out_file):
    """function to open up a tab or space separeted clustering
    output and rename according to the name in the database file.
    Abundance is also appended to the name"""
    # call the function to get the dictionary
    # populated with the database
    coded_name_to_species_dict = coded_name_to_species(database)
    cluster_file = open(filename1, "r")
    summary_out_file = open(out_file, "w")

    count = int(0)
    # iterate through the cluster file. One line per cluster
    for line in cluster_file:
        if not line.strip():
            continue  # if the last line is blank
        if line.startswith("#"):  # dont want comment lines
            continue
        output_str = ""
        if "\t" in line:
            cluster_line = line.rstrip("\n").split("\t")
        else:
            # different clustering program?
            cluster_line = line.rstrip("\n").split()
        count += 1
        for member in cluster_line:
            try:
                # data would be: 2f1454d16278fda2d44f26ebf7a0ed05_310
                # _abundance value
                split_name = member.split("_")[:-1]
                species = ("_").join(split_name)
                abundance = member.split("_")[-1]
                # print (abundance)
            except:
                KeyError
                errmsg = """something went wrong with decoding the names maybe
                your names were not separated in your database? If so, the
                block of code above this staement need adjusting accordingly.
                """
                sys.exit(os.system(errmsg))
            # add the info to a str, we will write at the
            # end of the cluster line
            cluster_summary = "%s_abundance=%s\t" % (species, abundance)
            output_str = output_str + cluster_summary
        summary_out_file.write(output_str+"\n")
    # close the files
    cluster_file.close()
    summary_out_file.close()
<|endoftext|>