<commit_before>"""
Parse adapter specifications
"""
import re
import logging
from pathlib import Path
from typing import Type, Optional, List, Tuple, Iterator, Any, Dict
from xopen import xopen
from dnaio.readers import FastaReader
from .adapters import (
    Adapter, FrontAdapter, NonInternalFrontAdapter, BackAdapter, NonInternalBackAdapter,
    AnywhereAdapter, PrefixAdapter, SuffixAdapter, LinkedAdapter, InvalidCharacter
)

logger = logging.getLogger(__name__)

class AdapterSpecification:
    """  # noqa: E501
    Description of a single non-linked adapter.

    These are the attributes:

    - name (None or str)
    - restriction (None, 'anchored', or 'noninternal')
    - sequence (nucleotide sequence as string)
    - parameters (dict with extra parameters such as 'max_error_rate', 'min_overlap')
    - cmdline_type ('front' for -a, 'back' for -g and 'anywhere' for -b)

    >>> AdapterSpecification.parse('a_name=ACGT;anywhere', 'back')
    AdapterSpecification(name='a_name', restriction=None, sequence='ACGT', parameters={'anywhere': True}, cmdline_type='back')
    """

    def __init__(
        self,
        name: str,
        restriction: Optional[str],
        sequence: str,
        parameters,
        cmdline_type: str,
    ):
        assert restriction in (None, "anchored", "noninternal")
        assert cmdline_type in ("front", "back", "anywhere")
        self.name = name
        self.restriction = restriction
        self.sequence = sequence
        self.parameters = parameters
        self.cmdline_type = cmdline_type

    @classmethod
    def parse(cls, spec: str, cmdline_type: str):
        """Factory for creating an instance from a string specification"""
        name, restriction, sequence, parameters = cls._parse(spec, cmdline_type)
        return cls(name, restriction, sequence, parameters, cmdline_type)

    def __repr__(self):
        return '{}(name={!r}, restriction={!r}, sequence={!r}, parameters={!r}, cmdline_type={!r})'.format(
            self.__class__.__name__, self.name, self.restriction, self.sequence, self.parameters, self.cmdline_type)

    def __eq__(self, other):
        return (
            self.name == other.name
            and self.restriction == other.restriction
            and self.sequence == other.sequence
            and self.parameters == other.parameters
            and self.cmdline_type == other.cmdline_type
        )

    @staticmethod
    def expand_braces(sequence: str) -> str:
        """
        Replace all occurrences of ``x{n}`` (where x is any character) with n
        occurrences of x. Raise ValueError if the expression cannot be parsed.

        >>> AdapterSpecification.expand_braces('TGA{5}CT')
        'TGAAAAACT'
        """
        # Simple DFA with four states, encoded in prev
        result = ''
        prev = None
        for s in re.split('([{}])', sequence):
            if s == '':
                continue
            if prev is None:
                if s == '{':
                    raise ValueError('"{" must be used after a character')
                if s == '}':
                    raise ValueError('"}" cannot be used here')
                prev = s
                result += s
            elif prev == '{':
                prev = int(s)
                if not 0 <= prev <= 10000:
                    raise ValueError('Value {} invalid'.format(prev))
            elif isinstance(prev, int):
                if s != '}':
                    raise ValueError('"}" expected')
                result = result[:-1] + result[-1] * prev
                prev = None
            else:
                if s != '{':
                    raise ValueError('Expected "{"')
                prev = '{'
        # Check if we are in a non-terminating state
        if isinstance(prev, int) or prev == '{':
            raise ValueError("Unterminated expression")
        return result

    @staticmethod
    def _extract_name(spec: str) -> Tuple[Optional[str], str]:
        """
        Parse an adapter specification given as 'name=adapt' into 'name' and 'adapt'.
        """
        fields = spec.split('=', 1)
        name = None  # type: Optional[str]
        if len(fields) > 1:
            name, spec = fields
            name = name.strip()
        spec = spec.strip()
        return name, spec

    allowed_parameters = {
        # abbreviations
        'e': 'max_error_rate',
        'error_rate': 'max_error_rate',
        'o': 'min_overlap',

        # allowed parameters
        'max_error_rate': None,
        'min_overlap': None,
        'anywhere': None,
        'required': None,
        'optional': None,  # If this is specified, 'required' will be set to False
    }

    @classmethod
    def _parse_parameters(cls, spec: str):
        """Parse key=value;key=value;key=value into a dict"""

        fields = spec.split(';')
        result = dict()  # type: Dict[str,Any]
        for field in fields:
            field = field.strip()
            if not field:
                continue
            key, equals, value = field.partition('=')  # type: (str, str, Any)
            if equals == '=' and value == '':
                raise ValueError('No value given')
            key = key.strip()
            if key not in cls.allowed_parameters:
                raise KeyError('Unknown parameter {}'.format(key))
            # unabbreviate
            while cls.allowed_parameters[key] is not None:
                key = cls.allowed_parameters[key]  # type: ignore
            value = value.strip()
            if value == '':
                value = True
            else:
                try:
                    value = int(value)
                except ValueError:
                    value = float(value)
            if key in result:
                raise KeyError('Key {} specified twice'.format(key))
            result[key] = value
        if 'optional' in result and 'required' in result:
            raise ValueError("'optional' and 'required' cannot be specified at the same time")
        if 'optional' in result:
            result['required'] = False
            del result['optional']
        return result

    @classmethod
    def _parse(cls, spec, cmdline_type):
        """
        Parse an adapter specification for a non-linked adapter (without '...')

        Allow:
        'back' and ADAPTER
        'back' and ADAPTERX
        'back' and ADAPTER$
        'front' and ADAPTER
        'front' and XADAPTER
        'front' and ^ADAPTER
        'anywhere' and ADAPTER
        """
        if cmdline_type not in ("front", "back", "anywhere"):
            raise ValueError("cmdline_type must be front, back or anywhere")
        error = ValueError(
            "You cannot use multiple placement restrictions for an adapter at the same time. "
            "Choose one of ^ADAPTER, ADAPTER$, XADAPTER or ADAPTERX")
        spec, middle, parameters_spec = spec.partition(';')
        name, spec = cls._extract_name(spec)
        spec = spec.strip()
        parameters = cls._parse_parameters(parameters_spec)
        spec = cls.expand_braces(spec)

        # Special case for adapters consisting of only X characters:
        # This needs to be supported for backwards-compatibilitity
        if len(spec.strip('X')) == 0:
            return name, None, spec, {}

        front_restriction = None
        if spec.startswith('^'):
            front_restriction = 'anchored'
            spec = spec[1:]
        if spec.upper().startswith('X'):
            if front_restriction is not None:
                raise error
            front_restriction = 'noninternal'
            spec = spec.lstrip('xX')

        back_restriction = None
        if spec.endswith('$'):
            back_restriction = 'anchored'
            spec = spec[:-1]
        if spec.upper().endswith('X'):
            if back_restriction is not None:
                raise error
            back_restriction = 'noninternal'
            spec = spec.rstrip('xX')

        n_placement_restrictions = int(bool(front_restriction)) + int(bool(back_restriction))
        if n_placement_restrictions > 1:
            raise error

        if cmdline_type == 'front' and back_restriction:
            raise ValueError(
                "Allowed placement restrictions for a 5' adapter are XADAPTER and ^ADAPTER")
        if cmdline_type == 'back' and front_restriction:
            raise ValueError(
                "Allowed placement restrictions for a 3' adapter are ADAPTERX and ADAPTER$")

        assert front_restriction is None or back_restriction is None
        if front_restriction is not None:
            restriction = front_restriction
        else:
            restriction = back_restriction

        if cmdline_type == 'anywhere' and restriction is not None:
            raise ValueError(
                "Placement restrictions (with X, ^, $) not supported for 'anywhere' (-b) adapters")

        return name, restriction, spec, parameters

    @staticmethod
    def _restriction_to_class(cmdline_type, restriction):
        if cmdline_type == 'front':
            if restriction is None:
                return FrontAdapter
            elif restriction == 'anchored':
                return PrefixAdapter
            elif restriction == 'noninternal':
                return NonInternalFrontAdapter
            else:
                raise ValueError(
                    'Value {} for a front restriction not allowed'.format(restriction))
        elif cmdline_type == 'back':
            if restriction is None:
                return BackAdapter
            elif restriction == 'anchored':
                return SuffixAdapter
            elif restriction == 'noninternal':
                return NonInternalBackAdapter
            else:
                raise ValueError(
                    'Value {} for a back restriction not allowed'.format(restriction))
        else:
            assert cmdline_type == 'anywhere'
            if restriction is None:
                return AnywhereAdapter
            else:
                raise ValueError('No placement may be specified for "anywhere" adapters')

    def adapter_class(self):
        return self._restriction_to_class(self.cmdline_type, self.restriction)

class AdapterParser:
    """
    Factory for Adapter classes that all use the same default parameters (error rate,
    indels etc.). The given **kwargs will be passed to the Adapter constructors.
    """
    def __init__(self, **kwargs):
        # kwargs: max_error_rate, min_overlap, read_wildcards, adapter_wildcards, indels
        self.default_parameters = kwargs

    def _parse(self, spec: str, cmdline_type: str = "back", name: Optional[str] = None) -> Adapter:
        """
        Parse an adapter specification not using ``file:`` notation and return
        an object of an appropriate Adapter class.

        name -- Adapter name if not included as part of the spec. (If spec is
        'name=ADAPTER', name will be 'name'.)

        cmdline_type -- describes which commandline parameter was used (``-a``
        is 'back', ``-b`` is 'anywhere', and ``-g`` is 'front').
        """
        if cmdline_type not in ('front', 'back', 'anywhere'):
            raise ValueError('cmdline_type cannot be {!r}'.format(cmdline_type))
        spec1, middle, spec2 = spec.partition('...')
        if middle == '...' and spec1 and spec2:
            return self._parse_linked(spec1, spec2, name, cmdline_type)

        if middle == '...':
            spec, cmdline_type = self._normalize_ellipsis(spec1, spec2, cmdline_type)
        else:
            spec = spec1
        return self._parse_not_linked(spec, name, cmdline_type)

    @staticmethod
    def _normalize_ellipsis(spec1: str, spec2: str, cmdline_type) -> Tuple[str, str]:
        if not spec1:
            if cmdline_type == 'back':
                # -a ...ADAPTER
                spec = spec2
            else:
                # -g ...ADAPTER
                raise ValueError('Invalid adapter specification')
        elif not spec2:
            if cmdline_type == 'back':
                # -a ADAPTER...
                cmdline_type = 'front'
                spec = spec1
            else:
                # -g ADAPTER...
                spec = spec1
        else:
            raise ValueError("Expected either spec1 or spec2")
        return spec, cmdline_type

    def _parse_not_linked(self, spec: str, name: Optional[str], cmdline_type: str) -> Adapter:
        aspec = AdapterSpecification.parse(spec, cmdline_type)
        adapter_class = aspec.adapter_class()  # type: Type[Adapter]
        if not name:
            name = aspec.name
        if aspec.parameters.pop('anywhere', False) and adapter_class in (FrontAdapter, BackAdapter):
            aspec.parameters['force_anywhere'] = True
        parameters = self.default_parameters.copy()
        parameters.update(aspec.parameters)

        return adapter_class(sequence=aspec.sequence, name=name, **parameters)

    def _parse_linked(self, spec1: str, spec2: str, name: Optional[str], cmdline_type: str) -> LinkedAdapter:
        """Return a linked adapter from two specification strings"""

        if cmdline_type == 'anywhere':
            raise ValueError("'anywhere' (-b) adapters may not be linked")
        front_spec = AdapterSpecification.parse(spec1, 'front')
        back_spec = AdapterSpecification.parse(spec2, 'back')
        if not name:
            name = front_spec.name

        if cmdline_type == 'back' and front_spec.restriction is None:
            import textwrap
            logger.warning('\n'.join(textwrap.wrap(
                "You specified a linked adapter as '-a ADAPTER1...ADAPTER2'. "
                "The interpretation of what this means has changed in Cutadapt 2.0. "
                "(The 5' adapter is now no longer anchored by default.) "
                "To get results consist with the old behavior, you need to anchor "
                "the 5' adapter explicitly as in '-a ^ADAPTER1...ADAPTER2'."
            )))

        front_anchored = front_spec.restriction is not None
        back_anchored = back_spec.restriction is not None

        front_parameters = self.default_parameters.copy()
        front_parameters.update(front_spec.parameters)
        back_parameters = self.default_parameters.copy()
        back_parameters.update(back_spec.parameters)

        if cmdline_type == 'front':
            # -g requires both adapters to be present
            front_required = True
            back_required = True
        else:
            # -a requires only the anchored adapters to be present
            front_required = front_anchored
            back_required = back_anchored

        # Handle parameters overriding whether an adapter is required
        front_required = front_parameters.pop('required', front_required)
        back_required = back_parameters.pop('required', back_required)

        front_adapter = front_spec.adapter_class()(front_spec.sequence, name=None,
            **front_parameters)
        back_adapter = back_spec.adapter_class()(back_spec.sequence, name=None,
            **back_parameters)

        return LinkedAdapter(
            front_adapter=front_adapter,
            back_adapter=back_adapter,
            front_required=front_required,
            back_required=back_required,
            name=name,
        )

    def parse(self, spec: str, cmdline_type: str = 'back') -> Iterator[Adapter]:
        """
        Parse an adapter specification and yield appropriate Adapter classes.
        This works like the _parse_no_file() function above, but also supports the
        ``file:`` notation for reading adapters from an external FASTA
        file. Since a file can contain multiple adapters, this
        function is a generator.
        """
        if spec.startswith('file:'):
            # read adapter sequences from a file
            with xopen(spec[5:], mode="rb", threads=0) as f:
                fasta = FastaReader(f)
                for record in fasta:
                    name = record.name.split(None, 1)
                    name = name[0] if name else None
                    yield self._parse(record.sequence, cmdline_type, name=name)
        else:
            try:
                yield self._parse(spec, cmdline_type, name=None)
            except InvalidCharacter as e:
                if Path(spec).exists():
                    extra_message = "A file exists named '{}'. ".format(spec) +\
                        "To use the sequences in that file as adapter sequences, write 'file:' " +\
                        "before the path, as in 'file:{}'.".format(spec)
                    raise InvalidCharacter(e.args[0] + "\n" + extra_message)
                else:
                    raise

    def parse_multi(self, type_spec_pairs: List[Tuple[str, str]]) -> List[Adapter]:
        """
        Parse all three types of commandline options that can be used to
        specify adapters. adapters must be a list of (str, str) pairs, where the first is
        the adapter type (either 'front', 'back' or 'anywhere') and the second is the
        adapter specification given on the commandline

        Return a list of appropriate Adapter classes.
        """
        adapters = []  # type: List[Adapter]
        for cmdline_type, spec in type_spec_pairs:
            if cmdline_type not in {'front', 'back', 'anywhere'}:
                raise ValueError('adapter type must be front, back or anywhere')
            adapters.extend(self.parse(spec, cmdline_type))
        return adapters
<commit_msg>Remove a 2.0 incompatibility warning<commit_after>"""
Parse adapter specifications
"""
import re
import logging
from pathlib import Path
from typing import Type, Optional, List, Tuple, Iterator, Any, Dict
from xopen import xopen
from dnaio.readers import FastaReader
from .adapters import (
    Adapter, FrontAdapter, NonInternalFrontAdapter, BackAdapter, NonInternalBackAdapter,
    AnywhereAdapter, PrefixAdapter, SuffixAdapter, LinkedAdapter, InvalidCharacter
)

logger = logging.getLogger(__name__)

class AdapterSpecification:
    """  # noqa: E501
    Description of a single non-linked adapter.

    These are the attributes:

    - name (None or str)
    - restriction (None, 'anchored', or 'noninternal')
    - sequence (nucleotide sequence as string)
    - parameters (dict with extra parameters such as 'max_error_rate', 'min_overlap')
    - cmdline_type ('front' for -a, 'back' for -g and 'anywhere' for -b)

    >>> AdapterSpecification.parse('a_name=ACGT;anywhere', 'back')
    AdapterSpecification(name='a_name', restriction=None, sequence='ACGT', parameters={'anywhere': True}, cmdline_type='back')
    """

    def __init__(
        self,
        name: str,
        restriction: Optional[str],
        sequence: str,
        parameters,
        cmdline_type: str,
    ):
        assert restriction in (None, "anchored", "noninternal")
        assert cmdline_type in ("front", "back", "anywhere")
        self.name = name
        self.restriction = restriction
        self.sequence = sequence
        self.parameters = parameters
        self.cmdline_type = cmdline_type

    @classmethod
    def parse(cls, spec: str, cmdline_type: str):
        """Factory for creating an instance from a string specification"""
        name, restriction, sequence, parameters = cls._parse(spec, cmdline_type)
        return cls(name, restriction, sequence, parameters, cmdline_type)

    def __repr__(self):
        return '{}(name={!r}, restriction={!r}, sequence={!r}, parameters={!r}, cmdline_type={!r})'.format(
            self.__class__.__name__, self.name, self.restriction, self.sequence, self.parameters, self.cmdline_type)

    def __eq__(self, other):
        return (
            self.name == other.name
            and self.restriction == other.restriction
            and self.sequence == other.sequence
            and self.parameters == other.parameters
            and self.cmdline_type == other.cmdline_type
        )

    @staticmethod
    def expand_braces(sequence: str) -> str:
        """
        Replace all occurrences of ``x{n}`` (where x is any character) with n
        occurrences of x. Raise ValueError if the expression cannot be parsed.

        >>> AdapterSpecification.expand_braces('TGA{5}CT')
        'TGAAAAACT'
        """
        # Simple DFA with four states, encoded in prev
        result = ''
        prev = None
        for s in re.split('([{}])', sequence):
            if s == '':
                continue
            if prev is None:
                if s == '{':
                    raise ValueError('"{" must be used after a character')
                if s == '}':
                    raise ValueError('"}" cannot be used here')
                prev = s
                result += s
            elif prev == '{':
                prev = int(s)
                if not 0 <= prev <= 10000:
                    raise ValueError('Value {} invalid'.format(prev))
            elif isinstance(prev, int):
                if s != '}':
                    raise ValueError('"}" expected')
                result = result[:-1] + result[-1] * prev
                prev = None
            else:
                if s != '{':
                    raise ValueError('Expected "{"')
                prev = '{'
        # Check if we are in a non-terminating state
        if isinstance(prev, int) or prev == '{':
            raise ValueError("Unterminated expression")
        return result

    @staticmethod
    def _extract_name(spec: str) -> Tuple[Optional[str], str]:
        """
        Parse an adapter specification given as 'name=adapt' into 'name' and 'adapt'.
        """
        fields = spec.split('=', 1)
        name = None  # type: Optional[str]
        if len(fields) > 1:
            name, spec = fields
            name = name.strip()
        spec = spec.strip()
        return name, spec

    allowed_parameters = {
        # abbreviations
        'e': 'max_error_rate',
        'error_rate': 'max_error_rate',
        'o': 'min_overlap',

        # allowed parameters
        'max_error_rate': None,
        'min_overlap': None,
        'anywhere': None,
        'required': None,
        'optional': None,  # If this is specified, 'required' will be set to False
    }

    @classmethod
    def _parse_parameters(cls, spec: str):
        """Parse key=value;key=value;key=value into a dict"""

        fields = spec.split(';')
        result = dict()  # type: Dict[str,Any]
        for field in fields:
            field = field.strip()
            if not field:
                continue
            key, equals, value = field.partition('=')  # type: (str, str, Any)
            if equals == '=' and value == '':
                raise ValueError('No value given')
            key = key.strip()
            if key not in cls.allowed_parameters:
                raise KeyError('Unknown parameter {}'.format(key))
            # unabbreviate
            while cls.allowed_parameters[key] is not None:
                key = cls.allowed_parameters[key]  # type: ignore
            value = value.strip()
            if value == '':
                value = True
            else:
                try:
                    value = int(value)
                except ValueError:
                    value = float(value)
            if key in result:
                raise KeyError('Key {} specified twice'.format(key))
            result[key] = value
        if 'optional' in result and 'required' in result:
            raise ValueError("'optional' and 'required' cannot be specified at the same time")
        if 'optional' in result:
            result['required'] = False
            del result['optional']
        return result

    @classmethod
    def _parse(cls, spec, cmdline_type):
        """
        Parse an adapter specification for a non-linked adapter (without '...')

        Allow:
        'back' and ADAPTER
        'back' and ADAPTERX
        'back' and ADAPTER$
        'front' and ADAPTER
        'front' and XADAPTER
        'front' and ^ADAPTER
        'anywhere' and ADAPTER
        """
        if cmdline_type not in ("front", "back", "anywhere"):
            raise ValueError("cmdline_type must be front, back or anywhere")
        error = ValueError(
            "You cannot use multiple placement restrictions for an adapter at the same time. "
            "Choose one of ^ADAPTER, ADAPTER$, XADAPTER or ADAPTERX")
        spec, middle, parameters_spec = spec.partition(';')
        name, spec = cls._extract_name(spec)
        spec = spec.strip()
        parameters = cls._parse_parameters(parameters_spec)
        spec = cls.expand_braces(spec)

        # Special case for adapters consisting of only X characters:
        # This needs to be supported for backwards-compatibilitity
        if len(spec.strip('X')) == 0:
            return name, None, spec, {}

        front_restriction = None
        if spec.startswith('^'):
            front_restriction = 'anchored'
            spec = spec[1:]
        if spec.upper().startswith('X'):
            if front_restriction is not None:
                raise error
            front_restriction = 'noninternal'
            spec = spec.lstrip('xX')

        back_restriction = None
        if spec.endswith('$'):
            back_restriction = 'anchored'
            spec = spec[:-1]
        if spec.upper().endswith('X'):
            if back_restriction is not None:
                raise error
            back_restriction = 'noninternal'
            spec = spec.rstrip('xX')

        n_placement_restrictions = int(bool(front_restriction)) + int(bool(back_restriction))
        if n_placement_restrictions > 1:
            raise error

        if cmdline_type == 'front' and back_restriction:
            raise ValueError(
                "Allowed placement restrictions for a 5' adapter are XADAPTER and ^ADAPTER")
        if cmdline_type == 'back' and front_restriction:
            raise ValueError(
                "Allowed placement restrictions for a 3' adapter are ADAPTERX and ADAPTER$")

        assert front_restriction is None or back_restriction is None
        if front_restriction is not None:
            restriction = front_restriction
        else:
            restriction = back_restriction

        if cmdline_type == 'anywhere' and restriction is not None:
            raise ValueError(
                "Placement restrictions (with X, ^, $) not supported for 'anywhere' (-b) adapters")

        return name, restriction, spec, parameters

    @staticmethod
    def _restriction_to_class(cmdline_type, restriction):
        if cmdline_type == 'front':
            if restriction is None:
                return FrontAdapter
            elif restriction == 'anchored':
                return PrefixAdapter
            elif restriction == 'noninternal':
                return NonInternalFrontAdapter
            else:
                raise ValueError(
                    'Value {} for a front restriction not allowed'.format(restriction))
        elif cmdline_type == 'back':
            if restriction is None:
                return BackAdapter
            elif restriction == 'anchored':
                return SuffixAdapter
            elif restriction == 'noninternal':
                return NonInternalBackAdapter
            else:
                raise ValueError(
                    'Value {} for a back restriction not allowed'.format(restriction))
        else:
            assert cmdline_type == 'anywhere'
            if restriction is None:
                return AnywhereAdapter
            else:
                raise ValueError('No placement may be specified for "anywhere" adapters')

    def adapter_class(self):
        return self._restriction_to_class(self.cmdline_type, self.restriction)

class AdapterParser:
    """
    Factory for Adapter classes that all use the same default parameters (error rate,
    indels etc.). The given **kwargs will be passed to the Adapter constructors.
    """
    def __init__(self, **kwargs):
        # kwargs: max_error_rate, min_overlap, read_wildcards, adapter_wildcards, indels
        self.default_parameters = kwargs

    def _parse(self, spec: str, cmdline_type: str = "back", name: Optional[str] = None) -> Adapter:
        """
        Parse an adapter specification not using ``file:`` notation and return
        an object of an appropriate Adapter class.

        name -- Adapter name if not included as part of the spec. (If spec is
        'name=ADAPTER', name will be 'name'.)

        cmdline_type -- describes which commandline parameter was used (``-a``
        is 'back', ``-b`` is 'anywhere', and ``-g`` is 'front').
        """
        if cmdline_type not in ('front', 'back', 'anywhere'):
            raise ValueError('cmdline_type cannot be {!r}'.format(cmdline_type))
        spec1, middle, spec2 = spec.partition('...')
        if middle == '...' and spec1 and spec2:
            return self._parse_linked(spec1, spec2, name, cmdline_type)

        if middle == '...':
            spec, cmdline_type = self._normalize_ellipsis(spec1, spec2, cmdline_type)
        else:
            spec = spec1
        return self._parse_not_linked(spec, name, cmdline_type)

    @staticmethod
    def _normalize_ellipsis(spec1: str, spec2: str, cmdline_type) -> Tuple[str, str]:
        if not spec1:
            if cmdline_type == 'back':
                # -a ...ADAPTER
                spec = spec2
            else:
                # -g ...ADAPTER
                raise ValueError('Invalid adapter specification')
        elif not spec2:
            if cmdline_type == 'back':
                # -a ADAPTER...
                cmdline_type = 'front'
                spec = spec1
            else:
                # -g ADAPTER...
                spec = spec1
        else:
            raise ValueError("Expected either spec1 or spec2")
        return spec, cmdline_type

    def _parse_not_linked(self, spec: str, name: Optional[str], cmdline_type: str) -> Adapter:
        aspec = AdapterSpecification.parse(spec, cmdline_type)
        adapter_class = aspec.adapter_class()  # type: Type[Adapter]
        if not name:
            name = aspec.name
        if aspec.parameters.pop('anywhere', False) and adapter_class in (FrontAdapter, BackAdapter):
            aspec.parameters['force_anywhere'] = True
        parameters = self.default_parameters.copy()
        parameters.update(aspec.parameters)

        return adapter_class(sequence=aspec.sequence, name=name, **parameters)

    def _parse_linked(self, spec1: str, spec2: str, name: Optional[str], cmdline_type: str) -> LinkedAdapter:
        """Return a linked adapter from two specification strings"""

        if cmdline_type == 'anywhere':
            raise ValueError("'anywhere' (-b) adapters may not be linked")
        front_spec = AdapterSpecification.parse(spec1, 'front')
        back_spec = AdapterSpecification.parse(spec2, 'back')
        if not name:
            name = front_spec.name

        front_anchored = front_spec.restriction is not None
        back_anchored = back_spec.restriction is not None

        front_parameters = self.default_parameters.copy()
        front_parameters.update(front_spec.parameters)
        back_parameters = self.default_parameters.copy()
        back_parameters.update(back_spec.parameters)

        if cmdline_type == 'front':
            # -g requires both adapters to be present
            front_required = True
            back_required = True
        else:
            # -a requires only the anchored adapters to be present
            front_required = front_anchored
            back_required = back_anchored

        # Handle parameters overriding whether an adapter is required
        front_required = front_parameters.pop('required', front_required)
        back_required = back_parameters.pop('required', back_required)

        front_adapter = front_spec.adapter_class()(front_spec.sequence, name=None,
            **front_parameters)
        back_adapter = back_spec.adapter_class()(back_spec.sequence, name=None,
            **back_parameters)

        return LinkedAdapter(
            front_adapter=front_adapter,
            back_adapter=back_adapter,
            front_required=front_required,
            back_required=back_required,
            name=name,
        )

    def parse(self, spec: str, cmdline_type: str = 'back') -> Iterator[Adapter]:
        """
        Parse an adapter specification and yield appropriate Adapter classes.
        This works like the _parse_no_file() function above, but also supports the
        ``file:`` notation for reading adapters from an external FASTA
        file. Since a file can contain multiple adapters, this
        function is a generator.
        """
        if spec.startswith('file:'):
            # read adapter sequences from a file
            with xopen(spec[5:], mode="rb", threads=0) as f:
                fasta = FastaReader(f)
                for record in fasta:
                    name = record.name.split(None, 1)
                    name = name[0] if name else None
                    yield self._parse(record.sequence, cmdline_type, name=name)
        else:
            try:
                yield self._parse(spec, cmdline_type, name=None)
            except InvalidCharacter as e:
                if Path(spec).exists():
                    extra_message = "A file exists named '{}'. ".format(spec) +\
                        "To use the sequences in that file as adapter sequences, write 'file:' " +\
                        "before the path, as in 'file:{}'.".format(spec)
                    raise InvalidCharacter(e.args[0] + "\n" + extra_message)
                else:
                    raise

    def parse_multi(self, type_spec_pairs: List[Tuple[str, str]]) -> List[Adapter]:
        """
        Parse all three types of commandline options that can be used to
        specify adapters. adapters must be a list of (str, str) pairs, where the first is
        the adapter type (either 'front', 'back' or 'anywhere') and the second is the
        adapter specification given on the commandline

        Return a list of appropriate Adapter classes.
        """
        adapters = []  # type: List[Adapter]
        for cmdline_type, spec in type_spec_pairs:
            if cmdline_type not in {'front', 'back', 'anywhere'}:
                raise ValueError('adapter type must be front, back or anywhere')
            adapters.extend(self.parse(spec, cmdline_type))
        return adapters
<|endoftext|>


<commit_before>import unittest
from fontParts.base import FontPartsError

class TestFeatures(unittest.TestCase):

    pass
<commit_msg>Test features.<commit_after>import unittest
from fontParts.base import FontPartsError

class TestFeatures(unittest.TestCase):

    def getFeatures_generic(self):
        features, unreferenced = self.objectGenerator("features")
        features.text = "# test"
        return features, unreferenced

    # ----
    # Text
    # ----

    def test_text(self):
        features, unreferenced = self.getFeatures_generic()
        # get
        self.assertEqual(
            features.text,
            "# test"
        )
        # set: valid
        features.text = "# foo"
        self.assertEqual(
            features.text,
            "# foo"
        )
        features.text = None
        self.assertIsNone(features.text)
        # set: invalid
        with self.assertRaises(FontPartsError):
            features.text = 123<|endoftext|>


<commit_before>#!/usr/bin/env python
# Common tools:
#	pip/pip3 : install packages
#	pylint/pychecker : verify syntax
#	unittest package for xUnit.
""" This module helps setup a fresh install the way I like it. """

# Imports
from __future__ import print_function
import argparse
import os
import subprocess
import shutil
try:
    import apt
except ImportError:
    print('not on debian system, don\'t use option install linux packages')

# Packages to install follow, broken down into categories.

PROGRAMS = """ \
    htop pkg-config aptitude synaptic dos2unix oxygen-molecule \
    kubuntu-restricted-extras kubuntu-restricted-addons \
    brother-cups-wrapper-laser brother-cups-wrapper-laser1 \
    brother-lpr-drivers-laser brother-lpr-drivers-laser1 \
    k3b kid3 krita kolourpaint4 kchmviewer yakuake plasma-widget-quickaccess \
    chromium-browser firefox rekonq \
    abiword baobab dvdrip easytag chm2pdf dia catdoc \
    gimp gimp-plugin-registry \
    fuse gparted quicksynergy tree dfc \
    libnotify-bin libnotify-dev \
    redshift geoclue geoclue-hostip numlockx \
    samba samba-doc samba-tools wireshark wondershaper \
    ubuntuone-client ubuntuone-control-panel-qt \
    vlc ffmpeg ffmpeg-doc mplayer mencoder \
    p7zip-full rar zip unzip gzip \
    virtualbox-qt wine \
    ttf-xfree86-nonfree"""

KEYRINGS = """ \
    debian-keyring debian-archive-keyring gnome-keyring \
    debian-ports-archive-keyring python-gnomekeyring python-keyring \
    ubuntu-keyring"""

PROGRAMMING = """ \
    build-essential debianutils ubuntu-dev-tools mesa-utils \
    automake automake1.9-doc ant ant-doc checkinstall checkbox \
    dkms docbook make-doc lynx kdiff3 kdiff3-doc patch rpm2cpio rpm \
    codeblocks kdevelop qt-sdk \
    colormake colordiff colorgcc jq \
    vim vim-doc vim-gtk vim-rails vim-syntax-go vim-syntax-gtk vim-doc \
    flex flex-doc bison bison-doc graphviz exuberant-ctags \
    clisp clisp-doc clisp-dev clisp-module-gdbm \
    erlang erlang-eunit \
    gcc gcc-doc gcc-4.7-source libcunit1 gdb gdb-doc cgdb xxgdb ccache \
    libboost-all-dev libglm-dev libglew-dev libglfw-dev \
    libncursesw5-dev libpcre3-dev zlib1g-dev liblzma-dev \
    openmpi-bin openmpi-checkpoint openmpi-common \
    gfortran \
    ghc ghc-doc ghc-haddock ghc-prof haskell-debian-utils \
    haskell-devscripts haskell-doc cabal-install \
    junit junit-doc maven openjdk-7-doc openjdk-7-jdk openjdk-7-dbg \
    openjdk-7-source openjdk-7-demo icedtea-7-plugin \
    lua5.2 lua5.2-doc luadoc \
    perl perl-doc perl-modules libpadwalker-perl libfile-next-perl \
    php5 php5-mysql phpunit php5-dev \
    nodejs nodejs-dev nodejs-legacy npm \
    python python-doc python3-doc python-pip python3-pip jython jython-doc \
    pychecker pylint pep8 python-autopep8 ruby1.9.1-full shunit2 \
    bzr bzr-builddeb bzr-doc python-bzrlib bzrtools git git-gui git-doc \
    mercurial subversion cvs"""

BABUN = """ \
    clisp lua perl python python3 ruby make cmake colordiff colorgcc \
    p7zip unzip zip gzip ncurses pcre mercurial bzr subversion \
    libboost-devel libboost_python-devel cppunit
    """

CABAL = "buildwrapper scion-browser hoogle terminfo happy hlint"

PY_PACKS = "argcomplete trash-cli"

class NotSudo(Exception):
    """ Throw this if we aren't sudo but need to be. """
    pass

# Functions

def get_code(command, target):
    """ Wrapper function to clone repos.
    Protects against overwriting if target exists.
    command: The command that would run in bash.
    target: Where to clone to.
    """
    cmd = command.split()
    cmd.append(target)
    if not os.path.exists(target):
        subprocess.call(cmd)

def make_cmd(src, dst):
    """ Generator for helper. """
    def cmd_when_dst_empty(files, command, opts=[]):
        """ Execute a command if destination empty. """
        for fil in files:
            sfile = src + fil
            dfile = dst + fil
            if not os.path.exists(dfile):
                command(sfile, dfile, *opts)
    return cmd_when_dst_empty

def home_config():
    """ Setup the dev environment, stuff goes in the user's home folder. """
    script_path = os.path.realpath(__file__)
    script_dir = script_path[0:script_path.rindex(os.sep)]

    # Leaving trailing sep for ease later.
    src = script_dir + os.sep + 'dot_files' + os.sep
    dst = os.path.expanduser('~') + os.sep

    # Helper function
    helper = make_cmd(src, dst)

    # Copy files that get user details in plain text
    helper(['.bazaar'], shutil.copytree, [True])
    helper(['.gitconfig', '.hgrc'], shutil.copy)

    # Link to config files, and vim folder
    files = ['.bash_aliases', '.gitignore_global', '.hgignore_global',
             '.inputrc', '.vim', '.vimrc', '.ycm_extra_conf.py']
    helper(files, os.symlink)

    # Init vundle for vim plugin install.
    ddir = dst + '.vim' + os.sep + 'bundle' + os.sep
    if not os.path.exists(ddir):
        os.mkdir(ddir)
    get_code('git clone https://github.com/gmarik/Vundle.vim.git',
            ddir + 'Vundle.vim')

    # Setup git/hg prompt.
    get_code('hg clone http://bitbucket.org/sjl/hg-prompt/',
            dst + '.hg-prompt')
    get_code('git clone https://github.com/magicmonty/bash-git-prompt.git',
            dst + '.bash-git-prompt')

    # Highlighter to replace grepping a pipe
    get_code('git clone https://github.com/starcraftman/hhighlighter.git',
            dst + '.hhighlighter')

    # Custom bins go here, ensure it exists
    ddir = dst + '.optSoftware' + os.sep + 'bin'
    if not os.path.exists(ddir):
        os.makedirs(ddir)

    # Ag silver, repo package is old
    ddir = dst + '.ag'
    if os.name == 'posix' and not os.path.exists(ddir):
        get_code('git clone https://github.com/ggreer/the_silver_searcher.git',
                ddir)
        cmd = (ddir + os.sep + 'build.sh').split()
        subprocess.call(cmd)
        sfile = ddir + os.sep + 'ag'
        dfile = dst + '.optSoftware' + os.sep + 'bin'
        shutil.copy(sfile, dfile)

    # Ack, may sometimes be preferred over ag
    ddir = dst + '.ack'
    if not os.path.exists(ddir):
        get_code('git clone https://github.com/petdance/ack2.git', ddir)
        os.chdir(ddir)
        cmd = 'perl Makefile.PL'.split()
        subprocess.call(cmd)
        cmd = 'make ack-standalone'.split()
        subprocess.call(cmd)
        os.chdir(dst)
        sfile = ddir + os.sep + 'ack-standalone'
        dfile = dst + '.optSoftware' + os.sep + 'bin' + os.sep + 'ack'
        shutil.copy(sfile, dfile)

    # Setup powerline fonts if not done.
    ddir = dst + '.fonts'
    dpow = ddir + os.sep + 'powerline-fonts'
    if not os.path.exists(ddir):
        os.mkdir(ddir)
        get_code('git clone https://github.com/Lokaltog/powerline-fonts', dpow)
        cmd = ('fc-cache -vf' + ddir).split()
        subprocess.call(cmd)

    # Create dir for ccache
    ddir = dst + '.ccache'
    if not os.path.exists(ddir):
        os.mkdir(ddir)

def packs_babun():
    """ Setup a fresh babun install. """
    # Install packages
    cmd = ('pact install ' + BABUN).split()
    subprocess.call(cmd)

    # Now prepare then invoke regular setup link common files
    dst = os.path.expanduser('~') + os.sep

    # Make empty directories to ignore parts of linux setup
    for name in ['.ag', '.ack', '.fonts']:
        ddir = dst + name
        if not os.path.exists(ddir):
            os.mkdir(ddir)

    # Backup defaults and allow for new ones
    for name in ['.gitconfig', '.vim']:
        dfile = dst + name
        dfile_bak = dfile + '_bak'
        if os.path.exists(dfile) and not os.path.exists(dfile_bak):
            os.rename(dfile, dfile_bak)

    home_config()

def packs_cabal():
    """ Installs haskell packages for Eclipse Haskell plugin. """
    cmd = 'cabal update'.split()
    subprocess.call(cmd)

    cmd = ('cabal install ' + CABAL).split()
    subprocess.call(cmd)

def packs_linux():
    """ Install packages on the current system. """
    if os.getuid() != 0:
        raise NotSudo

    packages = (PROGRAMS + PROGRAMMING + KEYRINGS).split()

    cache = apt.Cache()
    print("One moment while we update cache.")
    cache.update()
    cache.open(None)
    print("Update done.")

    cmd = 'sudo apt-get install'.split()
    for pack in packages:
        try:
            package = cache[pack]
            if not package.is_installed:
                cmd.append(pack)
        except KeyError:
            print("Package couldn't be selected: %s" % pack)

    print("Please wait, running: " + " ".join(cmd))
    subprocess.call(cmd)

def packs_py():
    """ Installs python packages using pip. """
    # Use python package manager.
    cmd = ('pip install' + PY_PACKS).split()
    subprocess.call(cmd)

    # Install python completion to system bash_completion.d.
    cmd = 'activate-global-python-argcomplete'.split()
    subprocess.call(cmd)

def install_jshint():
    """ Setup jshint for progrmaming javascript with vim. """
    cmd = 'npm install jshint -g'.split()
    subprocess.call(cmd)

def install_pipelight():
    """ Silverlight plugin for firefox/chrome on linux.
    http://www.webupd8.org/2013/08/pipelight-use-silverlight-in-your-linux.html
    """
    if os.getuid() != 0:
        raise NotSudo

    cmds = ['sudo apt-get remove flashplugin-installer',
            'sudo apt-add-repository ppa:pipelight/stable',
            'sudo apt-get update',
            'sudo apt-get install pipelight-multi',
            'pipelight-plugin --enable silverlight',
            'pipelight-plugin --enable flash',]
    cmds = [x.split() for x in cmds]
    [subprocess.call(x) for x in cmds]
    print("Installation over, remember to use a useragent switcher.")

def main():
    """ Main function. """
    mesg = """This script sets up a dev environment.

    choice      effect
    ------------------------------------------------------
    linux       Install debian packages.
    babun       Install babun packages.
    python      Install python libraries via pip.
    cabal       Install haskell packages for eclipse.
    jshint      Install jshint via npm for javascript vim.
    pipelight   Install pipelight flash & silverlight.\
    """
    parser = argparse.ArgumentParser(description=mesg,
            formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('choice', nargs='+', action='append', help='the stages')

    args = parser.parse_args()  # Default parses argv[1:]
    choices = args.choice[0]

    # Use a dict of funcs instead of a case switch
    actions = {'linux': packs_linux,
                'babun': packs_babun,
                'home': home_config,
                'python': packs_py,
                'cabal': packs_cabal,
                'jshint': install_jshint,
                'pipelight': install_pipelight
                }

    try:
        [actions[x]() for x in choices]
    except IOError as exc:
        print('Failed to install: {}'.format(exc))
    except NotSudo:
        print("Rerun this part of script with sudo.")

if __name__ == '__main__':
    main()
<commit_msg>Shouldn't use mutable list in default.<commit_after>#!/usr/bin/env python
# Common tools:
#	pip/pip3 : install packages
#	pylint/pychecker : verify syntax
#	unittest package for xUnit.
""" This module helps setup a fresh install the way I like it. """

# Imports
from __future__ import print_function
import argparse
import os
import subprocess
import shutil
try:
    import apt
except ImportError:
    print('not on debian system, don\'t use option install linux packages')

# Packages to install follow, broken down into categories.

PROGRAMS = """ \
    htop pkg-config aptitude synaptic dos2unix oxygen-molecule \
    kubuntu-restricted-extras kubuntu-restricted-addons \
    brother-cups-wrapper-laser brother-cups-wrapper-laser1 \
    brother-lpr-drivers-laser brother-lpr-drivers-laser1 \
    k3b kid3 krita kolourpaint4 kchmviewer yakuake plasma-widget-quickaccess \
    chromium-browser firefox rekonq \
    abiword baobab dvdrip easytag chm2pdf dia catdoc \
    gimp gimp-plugin-registry \
    fuse gparted quicksynergy tree dfc \
    libnotify-bin libnotify-dev \
    redshift geoclue geoclue-hostip numlockx \
    samba samba-doc samba-tools wireshark wondershaper \
    ubuntuone-client ubuntuone-control-panel-qt \
    vlc ffmpeg ffmpeg-doc mplayer mencoder \
    p7zip-full rar zip unzip gzip \
    virtualbox-qt wine \
    ttf-xfree86-nonfree"""

KEYRINGS = """ \
    debian-keyring debian-archive-keyring gnome-keyring \
    debian-ports-archive-keyring python-gnomekeyring python-keyring \
    ubuntu-keyring"""

PROGRAMMING = """ \
    build-essential debianutils ubuntu-dev-tools mesa-utils \
    automake automake1.9-doc ant ant-doc checkinstall checkbox \
    dkms docbook make-doc lynx kdiff3 kdiff3-doc patch rpm2cpio rpm \
    codeblocks kdevelop qt-sdk \
    colormake colordiff colorgcc jq \
    vim vim-doc vim-gtk vim-rails vim-syntax-go vim-syntax-gtk vim-doc \
    flex flex-doc bison bison-doc graphviz exuberant-ctags \
    clisp clisp-doc clisp-dev clisp-module-gdbm \
    erlang erlang-eunit \
    gcc gcc-doc gcc-4.7-source libcunit1 gdb gdb-doc cgdb xxgdb ccache \
    libboost-all-dev libglm-dev libglew-dev libglfw-dev \
    libncursesw5-dev libpcre3-dev zlib1g-dev liblzma-dev \
    openmpi-bin openmpi-checkpoint openmpi-common \
    gfortran \
    ghc ghc-doc ghc-haddock ghc-prof haskell-debian-utils \
    haskell-devscripts haskell-doc cabal-install \
    junit junit-doc maven openjdk-7-doc openjdk-7-jdk openjdk-7-dbg \
    openjdk-7-source openjdk-7-demo icedtea-7-plugin \
    lua5.2 lua5.2-doc luadoc \
    perl perl-doc perl-modules libpadwalker-perl libfile-next-perl \
    php5 php5-mysql phpunit php5-dev \
    nodejs nodejs-dev nodejs-legacy npm \
    python python-doc python3-doc python-pip python3-pip jython jython-doc \
    pychecker pylint pep8 python-autopep8 ruby1.9.1-full shunit2 \
    bzr bzr-builddeb bzr-doc python-bzrlib bzrtools git git-gui git-doc \
    mercurial subversion cvs"""

BABUN = """ \
    clisp lua perl python python3 ruby make cmake colordiff colorgcc \
    p7zip unzip zip gzip ncurses pcre mercurial bzr subversion \
    libboost-devel libboost_python-devel cppunit
    """

CABAL = "buildwrapper scion-browser hoogle terminfo happy hlint"

PY_PACKS = "argcomplete trash-cli"

class NotSudo(Exception):
    """ Throw this if we aren't sudo but need to be. """
    pass

# Functions

def get_code(command, target):
    """ Wrapper function to clone repos.
    Protects against overwriting if target exists.
    command: The command that would run in bash.
    target: Where to clone to.
    """
    cmd = command.split()
    cmd.append(target)
    if not os.path.exists(target):
        subprocess.call(cmd)

def make_cmd(src, dst):
    """ Generator for helper. """
    def cmd_when_dst_empty(files, command, opts=None):
        """ Execute a command if destination empty. """
        if opts == None:
            opts = []
        for fil in files:
            sfile = src + fil
            dfile = dst + fil
            if not os.path.exists(dfile):
                command(sfile, dfile, *opts)
    return cmd_when_dst_empty

def home_config():
    """ Setup the dev environment, stuff goes in the user's home folder. """
    script_path = os.path.realpath(__file__)
    script_dir = script_path[0:script_path.rindex(os.sep)]

    # Leaving trailing sep for ease later.
    src = script_dir + os.sep + 'dot_files' + os.sep
    dst = os.path.expanduser('~') + os.sep

    # Helper function
    helper = make_cmd(src, dst)

    # Copy files that get user details in plain text
    helper(['.bazaar'], shutil.copytree, [True])
    helper(['.gitconfig', '.hgrc'], shutil.copy)

    # Link to config files, and vim folder
    files = ['.bash_aliases', '.gitignore_global', '.hgignore_global',
             '.inputrc', '.vim', '.vimrc', '.ycm_extra_conf.py']
    helper(files, os.symlink)

    # Init vundle for vim plugin install.
    ddir = dst + '.vim' + os.sep + 'bundle' + os.sep
    if not os.path.exists(ddir):
        os.mkdir(ddir)
    get_code('git clone https://github.com/gmarik/Vundle.vim.git',
            ddir + 'Vundle.vim')

    # Setup git/hg prompt.
    get_code('hg clone http://bitbucket.org/sjl/hg-prompt/',
            dst + '.hg-prompt')
    get_code('git clone https://github.com/magicmonty/bash-git-prompt.git',
            dst + '.bash-git-prompt')

    # Highlighter to replace grepping a pipe
    get_code('git clone https://github.com/starcraftman/hhighlighter.git',
            dst + '.hhighlighter')

    # Custom bins go here, ensure it exists
    ddir = dst + '.optSoftware' + os.sep + 'bin'
    if not os.path.exists(ddir):
        os.makedirs(ddir)

    # Ag silver, repo package is old
    ddir = dst + '.ag'
    if os.name == 'posix' and not os.path.exists(ddir):
        get_code('git clone https://github.com/ggreer/the_silver_searcher.git',
                ddir)
        cmd = (ddir + os.sep + 'build.sh').split()
        subprocess.call(cmd)
        sfile = ddir + os.sep + 'ag'
        dfile = dst + '.optSoftware' + os.sep + 'bin'
        shutil.copy(sfile, dfile)

    # Ack, may sometimes be preferred over ag
    ddir = dst + '.ack'
    if not os.path.exists(ddir):
        get_code('git clone https://github.com/petdance/ack2.git', ddir)
        os.chdir(ddir)
        cmd = 'perl Makefile.PL'.split()
        subprocess.call(cmd)
        cmd = 'make ack-standalone'.split()
        subprocess.call(cmd)
        os.chdir(dst)
        sfile = ddir + os.sep + 'ack-standalone'
        dfile = dst + '.optSoftware' + os.sep + 'bin' + os.sep + 'ack'
        shutil.copy(sfile, dfile)

    # Setup powerline fonts if not done.
    ddir = dst + '.fonts'
    dpow = ddir + os.sep + 'powerline-fonts'
    if not os.path.exists(ddir):
        os.mkdir(ddir)
        get_code('git clone https://github.com/Lokaltog/powerline-fonts', dpow)
        cmd = ('fc-cache -vf' + ddir).split()
        subprocess.call(cmd)

    # Create dir for ccache
    ddir = dst + '.ccache'
    if not os.path.exists(ddir):
        os.mkdir(ddir)

def packs_babun():
    """ Setup a fresh babun install. """
    # Install packages
    cmd = ('pact install ' + BABUN).split()
    subprocess.call(cmd)

    # Now prepare then invoke regular setup link common files
    dst = os.path.expanduser('~') + os.sep

    # Make empty directories to ignore parts of linux setup
    for name in ['.ag', '.ack', '.fonts']:
        ddir = dst + name
        if not os.path.exists(ddir):
            os.mkdir(ddir)

    # Backup defaults and allow for new ones
    for name in ['.gitconfig', '.vim']:
        dfile = dst + name
        dfile_bak = dfile + '_bak'
        if os.path.exists(dfile) and not os.path.exists(dfile_bak):
            os.rename(dfile, dfile_bak)

    home_config()

def packs_cabal():
    """ Installs haskell packages for Eclipse Haskell plugin. """
    cmd = 'cabal update'.split()
    subprocess.call(cmd)

    cmd = ('cabal install ' + CABAL).split()
    subprocess.call(cmd)

def packs_linux():
    """ Install packages on the current system. """
    if os.getuid() != 0:
        raise NotSudo

    packages = (PROGRAMS + PROGRAMMING + KEYRINGS).split()

    cache = apt.Cache()
    print("One moment while we update cache.")
    cache.update()
    cache.open(None)
    print("Update done.")

    cmd = 'sudo apt-get install'.split()
    for pack in packages:
        try:
            package = cache[pack]
            if not package.is_installed:
                cmd.append(pack)
        except KeyError:
            print("Package couldn't be selected: %s" % pack)

    print("Please wait, running: " + " ".join(cmd))
    subprocess.call(cmd)

def packs_py():
    """ Installs python packages using pip. """
    # Use python package manager.
    cmd = ('pip install' + PY_PACKS).split()
    subprocess.call(cmd)

    # Install python completion to system bash_completion.d.
    cmd = 'activate-global-python-argcomplete'.split()
    subprocess.call(cmd)

def install_jshint():
    """ Setup jshint for progrmaming javascript with vim. """
    cmd = 'npm install jshint -g'.split()
    subprocess.call(cmd)

def install_pipelight():
    """ Silverlight plugin for firefox/chrome on linux.
    http://www.webupd8.org/2013/08/pipelight-use-silverlight-in-your-linux.html
    """
    if os.getuid() != 0:
        raise NotSudo

    cmds = ['sudo apt-get remove flashplugin-installer',
            'sudo apt-add-repository ppa:pipelight/stable',
            'sudo apt-get update',
            'sudo apt-get install pipelight-multi',
            'pipelight-plugin --enable silverlight',
            'pipelight-plugin --enable flash',]
    cmds = [x.split() for x in cmds]
    [subprocess.call(x) for x in cmds]
    print("Installation over, remember to use a useragent switcher.")

def main():
    """ Main function. """
    mesg = """This script sets up a dev environment.

    choice      effect
    ------------------------------------------------------
    linux       Install debian packages.
    babun       Install babun packages.
    python      Install python libraries via pip.
    cabal       Install haskell packages for eclipse.
    jshint      Install jshint via npm for javascript vim.
    pipelight   Install pipelight flash & silverlight.\
    """
    parser = argparse.ArgumentParser(description=mesg,
            formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('choice', nargs='+', action='append', help='the stages')

    args = parser.parse_args()  # Default parses argv[1:]
    choices = args.choice[0]

    # Use a dict of funcs instead of a case switch
    actions = {'linux': packs_linux,
                'babun': packs_babun,
                'home': home_config,
                'python': packs_py,
                'cabal': packs_cabal,
                'jshint': install_jshint,
                'pipelight': install_pipelight
                }

    try:
        [actions[x]() for x in choices]
    except IOError as exc:
        print('Failed to install: {}'.format(exc))
    except NotSudo:
        print("Rerun this part of script with sudo.")

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#  Created by Riley Labrecque for Shorebound Studios
#  This script probably requires Python 3.3+
#  This script is licensed under the MIT License
#  See the included LICENSE.txt or the following for more info
#  http://www.tldrlegal.com/license/mit-license

from __future__ import print_function

import os

CPP_HEADER = """
// This file is automatically generated!

#include "CSteamworks.h"
"""[1:]

g_files = [f for f in os.listdir('steam') if os.path.isfile(os.path.join('steam', f))]

# We don't currently support the following interfaces because they don't provide a factory of their own.
# You are expected to call GetISteamGeneric to get them. That's a little too much for this script at this point.
# They are extremely small and rarely used interfaces, It might just be better to do it manually for them.
if 'isteamappticket.h' in g_files:
    g_files.remove('isteamappticket.h')
if 'isteamgamecoordinator.h' in g_files:
    g_files.remove('isteamgamecoordinator.h')
if 'isteamps3overlayrenderer.h' in g_files:
    g_files.remove('isteamps3overlayrenderer.h')

# We don't currently support SteamVR
if 'steamvr.h' in g_files:
    g_files.remove('steamvr.h')

g_files.extend(['isteamgameserverutils.h', 'isteamgameservernetworking.h', 'isteamgameserverhttp.h'])

g_GameServerFilenameDict = {
    'isteamgameserverutils.h': 'isteamutils.h',
    'isteamgameservernetworking.h': 'isteamnetworking.h',
    'isteamgameserverhttp.h': 'isteamhttp.h',
}

g_GameServerIFaceDict = {
    'isteamgameserverutils.h': 'ISteamGameServerUtils',
    'isteamgameservernetworking.h': 'ISteamGameServerNetworking',
    'isteamgameserverhttp.h': 'ISteamGameServerHTTP',
}

try:
    os.makedirs('wrapper/')
except OSError:
    pass

g_methodnames = []

for filename in g_files:
    try:
        emulatedfilename = g_GameServerFilenameDict[filename]
    except KeyError:
        emulatedfilename = filename

    print('Opening: "' + filename + '"')
    with open('steam/' + emulatedfilename, 'r') as f:
        output = []
        depth = 0
        iface = None
        ifacedepth = 0
        bInMultiLineCommentDepth = False
        bDisableCode = False
        for linenum, line in enumerate(f):
            linenum += 1
            bMultiLineCommentCodeOnThisLine = False

            line = line.split('//', 1)[0].strip()
            if len(line) == 0:
                continue

            pos = line.find('/*')
            if pos != -1:
                bInMultiLineCommentDepth = True
                endpos = line.find('*/')
                if endpos != -1:
                    bInMultiLineCommentDepth = False
                else:
                    line = line.split('/*', 1)[0].strip()
                    if len(line) == 0:
                        continue
                    else:
                        bMultiLineCommentCodeOnThisLine = True

            pos = line.find('*/')
            if pos != -1:
                bInMultiLineCommentDepth = False

                line = line[pos + len('*/'):].strip()
                if len(line) == 0:
                    continue

            if bInMultiLineCommentDepth and not bMultiLineCommentCodeOnThisLine:
                continue

            pos = line.find('class ISteam')
            if pos != -1:
                if ';' in line:
                    continue
                elif 'Response' in line:  # We don't have a proper way to call responses yet
                    continue

                iface = line[pos + len('class '):].split()[0]
                ifacedepth = depth
                try:
                    iface = g_GameServerIFaceDict[filename]
                except KeyError:
                    pass
                print(iface)

            if iface:
                if line.startswith('#'):
                    output.append(line.strip() + '')
                elif 'virtual' in line and line.endswith('0;'):
                    splitline = line[len('virtual '):].split()
                    state = 0
                    returnvalue = ''
                    methodname = ''
                    realmethodname = ''
                    args = ''
                    for token in splitline:
                        if not token:
                            continue

                        if state == 0:  # Return Value
                            if token.startswith('*'):
                                returnvalue += '*'
                                state = 1
                            elif token.find('(') == -1:
                                returnvalue += token + ' '
                            else:
                                state = 1

                        if state == 1:  # Method Name
                            if token.startswith('*'):
                                token = token[1:]
                            realmethodname = token.split('(', 1)[0]
                            methodname = iface + '_' + realmethodname

                            if methodname in g_methodnames:
                                methodname += '_'
                            g_methodnames.append(methodname)

                            if token[-1] == ')':
                                state = 3
                            elif token[-1] != '(':  # Edge case like f(void arg ) - shouldn't be triggered
                                print("[WARNING] A function doesn't have whitespace between the backets. " + str(linenum) + " - " + methodname)
                                token = token.split('(')[1]
                                state = 2
                            else:
                                state = 2
                                continue

                        if state == 2:  # Args
                            if token.startswith(')'):
                                state = 3
                            elif token.endswith(')'):  # Edge case like f( void arg) - shouldn't be triggered
                                print("[WARNING] A function doesn't have whitespace between the backets. " + str(linenum) + " - " + methodname)
                                args += token[:-1]
                                state = 3
                            else:
                                args += token + ' '

                        if state == 3:  # ) = 0;
                            continue

                    args = args.rstrip()
                    typelessargs = ''
                    if args != '':
                        argssplitted = args.strip().split(' ')
                        for i, token in enumerate(argssplitted):
                            if token == '=' or token == '""':  # Handle defaulted arguments
                                continue
                            if token == '0':  # Like f( int nChannel = 0 )
                                token = argssplitted[i - 2]

                            if token.startswith('**'):
                                typelessargs += token[2:] + ' '
                            elif token.startswith('*'):
                                typelessargs += token[1:] + ' '
                            elif token[-1] == ',':
                                typelessargs += token + ' '
                            elif i == len(argssplitted) - 1:
                                typelessargs += token
                    typelessargs = typelessargs.rstrip()

                    bReturnsCSteamID = False
                    if returnvalue.strip() == 'CSteamID':  # Can not return a class with C ABI
                        bReturnsCSteamID = True
                        returnvalue = 'SteamID_t '  # See CPP_HEADER for more details

                    output.append('SB_API ' + returnvalue + 'S_CALLTYPE ' + methodname + '(' + args + ') {')
                    if bReturnsCSteamID:
                        output.append('\treturn ' + iface[1:] + '()->' + realmethodname + '(' + typelessargs + ').ConvertToUint64();')
                    else:
                        output.append('\treturn ' + iface[1:] + '()->' + realmethodname + '(' + typelessargs + ');')
                    output.append('}')
                    output.append('')

            if '{' in line:
                depth += 1
            if '}' in line:
                depth -= 1
                if iface and depth == ifacedepth:
                    iface = None
                if bDisableCode:
                    output.append('#endif')
                    bDisableCode = False

        if output:
            with open('wrapper/' + os.path.splitext(filename)[0] + '.cpp', 'w') as out:
                print(CPP_HEADER, file=out)
                for line in output:
                    print(line, file=out)
<commit_msg>Added a comment that explains why we skip 'class ISteam' lines with ';'<commit_after>#!/usr/bin/env python
#  Created by Riley Labrecque for Shorebound Studios
#  This script probably requires Python 3.3+
#  This script is licensed under the MIT License
#  See the included LICENSE.txt or the following for more info
#  http://www.tldrlegal.com/license/mit-license

from __future__ import print_function

import os

CPP_HEADER = """
// This file is automatically generated!

#include "CSteamworks.h"
"""[1:]

g_files = [f for f in os.listdir('steam') if os.path.isfile(os.path.join('steam', f))]

# We don't currently support the following interfaces because they don't provide a factory of their own.
# You are expected to call GetISteamGeneric to get them. That's a little too much for this script at this point.
# They are extremely small and rarely used interfaces, It might just be better to do it manually for them.
if 'isteamappticket.h' in g_files:
    g_files.remove('isteamappticket.h')
if 'isteamgamecoordinator.h' in g_files:
    g_files.remove('isteamgamecoordinator.h')
if 'isteamps3overlayrenderer.h' in g_files:
    g_files.remove('isteamps3overlayrenderer.h')

# We don't currently support SteamVR
if 'steamvr.h' in g_files:
    g_files.remove('steamvr.h')

g_files.extend(['isteamgameserverutils.h', 'isteamgameservernetworking.h', 'isteamgameserverhttp.h'])

g_GameServerFilenameDict = {
    'isteamgameserverutils.h': 'isteamutils.h',
    'isteamgameservernetworking.h': 'isteamnetworking.h',
    'isteamgameserverhttp.h': 'isteamhttp.h',
}

g_GameServerIFaceDict = {
    'isteamgameserverutils.h': 'ISteamGameServerUtils',
    'isteamgameservernetworking.h': 'ISteamGameServerNetworking',
    'isteamgameserverhttp.h': 'ISteamGameServerHTTP',
}

try:
    os.makedirs('wrapper/')
except OSError:
    pass

g_methodnames = []

for filename in g_files:
    try:
        emulatedfilename = g_GameServerFilenameDict[filename]
    except KeyError:
        emulatedfilename = filename

    print('Opening: "' + filename + '"')
    with open('steam/' + emulatedfilename, 'r') as f:
        output = []
        depth = 0
        iface = None
        ifacedepth = 0
        bInMultiLineCommentDepth = False
        bDisableCode = False
        for linenum, line in enumerate(f):
            linenum += 1
            bMultiLineCommentCodeOnThisLine = False

            line = line.split('//', 1)[0].strip()
            if len(line) == 0:
                continue

            pos = line.find('/*')
            if pos != -1:
                bInMultiLineCommentDepth = True
                endpos = line.find('*/')
                if endpos != -1:
                    bInMultiLineCommentDepth = False
                else:
                    line = line.split('/*', 1)[0].strip()
                    if len(line) == 0:
                        continue
                    else:
                        bMultiLineCommentCodeOnThisLine = True

            pos = line.find('*/')
            if pos != -1:
                bInMultiLineCommentDepth = False

                line = line[pos + len('*/'):].strip()
                if len(line) == 0:
                    continue

            if bInMultiLineCommentDepth and not bMultiLineCommentCodeOnThisLine:
                continue

            pos = line.find('class ISteam')
            if pos != -1:
                if ';' in line:  # Gets rid of the forward declares in isteamclient.h
                    continue
                elif 'Response' in line:  # We don't have a proper way to call responses yet
                    continue

                iface = line[pos + len('class '):].split()[0]
                ifacedepth = depth
                try:
                    iface = g_GameServerIFaceDict[filename]
                except KeyError:
                    pass
                print(iface)

            if iface:
                if line.startswith('#'):
                    output.append(line.strip() + '')
                elif 'virtual' in line and line.endswith('0;'):
                    splitline = line[len('virtual '):].split()
                    state = 0
                    returnvalue = ''
                    methodname = ''
                    realmethodname = ''
                    args = ''
                    for token in splitline:
                        if not token:
                            continue

                        if state == 0:  # Return Value
                            if token.startswith('*'):
                                returnvalue += '*'
                                state = 1
                            elif token.find('(') == -1:
                                returnvalue += token + ' '
                            else:
                                state = 1

                        if state == 1:  # Method Name
                            if token.startswith('*'):
                                token = token[1:]
                            realmethodname = token.split('(', 1)[0]
                            methodname = iface + '_' + realmethodname

                            if methodname in g_methodnames:
                                methodname += '_'
                            g_methodnames.append(methodname)

                            if token[-1] == ')':
                                state = 3
                            elif token[-1] != '(':  # Edge case like f(void arg ) - shouldn't be triggered
                                print("[WARNING] A function doesn't have whitespace between the backets. " + str(linenum) + " - " + methodname)
                                token = token.split('(')[1]
                                state = 2
                            else:
                                state = 2
                                continue

                        if state == 2:  # Args
                            if token.startswith(')'):
                                state = 3
                            elif token.endswith(')'):  # Edge case like f( void arg) - shouldn't be triggered
                                print("[WARNING] A function doesn't have whitespace between the backets. " + str(linenum) + " - " + methodname)
                                args += token[:-1]
                                state = 3
                            else:
                                args += token + ' '

                        if state == 3:  # ) = 0;
                            continue

                    args = args.rstrip()
                    typelessargs = ''
                    if args != '':
                        argssplitted = args.strip().split(' ')
                        for i, token in enumerate(argssplitted):
                            if token == '=' or token == '""':  # Handle defaulted arguments
                                continue
                            if token == '0':  # Like f( int nChannel = 0 )
                                token = argssplitted[i - 2]

                            if token.startswith('**'):
                                typelessargs += token[2:] + ' '
                            elif token.startswith('*'):
                                typelessargs += token[1:] + ' '
                            elif token[-1] == ',':
                                typelessargs += token + ' '
                            elif i == len(argssplitted) - 1:
                                typelessargs += token
                    typelessargs = typelessargs.rstrip()

                    bReturnsCSteamID = False
                    if returnvalue.strip() == 'CSteamID':  # Can not return a class with C ABI
                        bReturnsCSteamID = True
                        returnvalue = 'SteamID_t '  # See CPP_HEADER for more details

                    output.append('SB_API ' + returnvalue + 'S_CALLTYPE ' + methodname + '(' + args + ') {')
                    if bReturnsCSteamID:
                        output.append('\treturn ' + iface[1:] + '()->' + realmethodname + '(' + typelessargs + ').ConvertToUint64();')
                    else:
                        output.append('\treturn ' + iface[1:] + '()->' + realmethodname + '(' + typelessargs + ');')
                    output.append('}')
                    output.append('')

            if '{' in line:
                depth += 1
            if '}' in line:
                depth -= 1
                if iface and depth == ifacedepth:
                    iface = None
                if bDisableCode:
                    output.append('#endif')
                    bDisableCode = False

        if output:
            with open('wrapper/' + os.path.splitext(filename)[0] + '.cpp', 'w') as out:
                print(CPP_HEADER, file=out)
                for line in output:
                    print(line, file=out)
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-

#
# Camera calibration module
#

#
# External dependencies
#
import cv2

#
# Find the chessboard quickly and draw it
#
def PreviewChessboard( image, pattern_size = ( 9, 6 ) ) :

	# Find the chessboard corners on the image
	found_all, corners = cv2.findChessboardCorners( image, pattern_size, flags = cv2.CALIB_CB_FAST_CHECK )

	# Chessboard found
	if found_all :

		# Convert grayscale image in color
		image = cv2.cvtColor( image, cv2.COLOR_GRAY2BGR )

		# Draw the chessboard corners on the image
		cv2.drawChessboardCorners( image, pattern_size, corners, found_all )

		cv2.imshow( "Calibration", image )
		cv2.waitKey(1)

#
# Camera calibration
#
def TestCalibration() :

	# External dependencies
	import numpy as np
	import glob
	import sys

	# Get image file names
	image_files = glob.glob( sys.argv[1] )

	# Chessboard pattern
	pattern_size = ( 9, 6 )
#	pattern_points = np.zeros( (np.prod(pattern_size), 3), np.float32 )
#	pattern_points[:,:2] = np.indices(pattern_size).T.reshape(-1, 2)

	# Termination criteria
	term = ( cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_COUNT, 30, 0.1 )

	# Scale factor for corner detection
	scale = 0.5

#	obj_points = []
#	img_points = []
#	h, w = 0, 0

	# For each image
	for filename in image_files :

		# Load the image
		image = cv2.imread( filename, cv2.CV_LOAD_IMAGE_GRAYSCALE )

		# image sharpening
#		image_blur = cv2.GaussianBlur( image, (0,0), 105)
#		image = cv2.addWeighted( image, 1.8, image_blur, -0.8, 0 )

#		h, w = image.shape[:2]

		# Preview chessboard on image
		preview = PreviewChessboard( image, pattern_size, scale )
		cv2.imshow( "Chessboard", preview )
		cv2.waitKey()

		# Resize image
		image_small = cv2.resize( image, None, fx=scale, fy=scale )

		# Find the chessboard corners on the image
		found_all, corners = cv2.findChessboardCorners( image_small, pattern_size, None , cv2.cv.CV_CALIB_CB_ADAPTIVE_THRESH + cv2.CALIB_CB_FAST_CHECK + cv2.CALIB_CB_FILTER_QUADS )

		# Chessboard found
		if found_all :

			# Store image and corner informations
#			img_points.append( corners.reshape(-1, 2) )
#			obj_points.append( pattern_points )

			corners /= scale

			# Refine the corner position
#			cv2.cornerSubPix( image, corners, (11, 11), (-1, -1), term )

			# Draw the chessboard corners on the image
			image_color = cv2.cvtColor( image, cv2.COLOR_GRAY2BGR )
			cv2.drawChessboardCorners( image_color, pattern_size, corners, found_all )

			# Resize image for display
			image_color = cv2.resize( image_color, None, fx=0.3, fy=0.3 )

			# Display the image with the found corners
			cv2.imshow( "Chessboard", image_color )
			cv2.waitKey()

		# No chessboard found
		else : print( "Chessboard no found..." )

	# Camera calibration
#	rms, camera_matrix, dist_coefs, rvecs, tvecs = cv2.calibrateCamera( obj_points, img_points, (w, h), None, None )
#	print "RMS:", rms
#	print "Camera matrix:\n", camera_matrix
#	print "Distortion coefficients:\n", dist_coefs.ravel()

	# Cleanup OpenCV windows
	cv2.destroyAllWindows()
<commit_msg>Add resize option to PreviewChessboard.<commit_after># -*- coding:utf-8 -*-

#
# Camera calibration module
#

#
# External dependencies
#
import cv2

#
# Find the chessboard quickly and draw it
#
def PreviewChessboard( image, pattern_size = ( 9, 6 ), resize = False ) :

	# Resize the image
	if resize : small_image = cv2.resize( image, None, fx=0.3, fy=0.3 )
	else : small_image = image

	# Find the chessboard corners on the image
	found_all, corners = cv2.findChessboardCorners( small_image, pattern_size, flags = cv2.CALIB_CB_FAST_CHECK )

	# Chessboard found
	if found_all :

		# Convert grayscale image in color
		small_image = cv2.cvtColor( small_image, cv2.COLOR_GRAY2BGR )

		# Draw the chessboard corners on the image
		cv2.drawChessboardCorners( small_image, pattern_size, corners, found_all )

		cv2.imshow( 'Calibration', small_image )
		cv2.waitKey(1)

#
# Camera calibration
#
def TestCalibration() :

	# External dependencies
	import numpy as np
	import glob
	import sys

	# Get image file names
	image_files = glob.glob( sys.argv[1] )

	# Chessboard pattern
	pattern_size = ( 9, 6 )
#	pattern_points = np.zeros( (np.prod(pattern_size), 3), np.float32 )
#	pattern_points[:,:2] = np.indices(pattern_size).T.reshape(-1, 2)

	# Termination criteria
	term = ( cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_COUNT, 30, 0.1 )

	# Scale factor for corner detection
	scale = 0.5

#	obj_points = []
#	img_points = []
#	h, w = 0, 0

	# For each image
	for filename in image_files :

		# Load the image
		image = cv2.imread( filename, cv2.CV_LOAD_IMAGE_GRAYSCALE )

		# image sharpening
#		image_blur = cv2.GaussianBlur( image, (0,0), 105)
#		image = cv2.addWeighted( image, 1.8, image_blur, -0.8, 0 )

#		h, w = image.shape[:2]

		# Preview chessboard on image
		preview = PreviewChessboard( image, pattern_size, scale )
		cv2.imshow( "Chessboard", preview )
		cv2.waitKey()

		# Resize image
		image_small = cv2.resize( image, None, fx=scale, fy=scale )

		# Find the chessboard corners on the image
		found_all, corners = cv2.findChessboardCorners( image_small, pattern_size, None , cv2.cv.CV_CALIB_CB_ADAPTIVE_THRESH + cv2.CALIB_CB_FAST_CHECK + cv2.CALIB_CB_FILTER_QUADS )

		# Chessboard found
		if found_all :

			# Store image and corner informations
#			img_points.append( corners.reshape(-1, 2) )
#			obj_points.append( pattern_points )

			corners /= scale

			# Refine the corner position
#			cv2.cornerSubPix( image, corners, (11, 11), (-1, -1), term )

			# Draw the chessboard corners on the image
			image_color = cv2.cvtColor( image, cv2.COLOR_GRAY2BGR )
			cv2.drawChessboardCorners( image_color, pattern_size, corners, found_all )

			# Resize image for display
			image_color = cv2.resize( image_color, None, fx=0.3, fy=0.3 )

			# Display the image with the found corners
			cv2.imshow( "Chessboard", image_color )
			cv2.waitKey()

		# No chessboard found
		else : print( "Chessboard no found..." )

	# Camera calibration
#	rms, camera_matrix, dist_coefs, rvecs, tvecs = cv2.calibrateCamera( obj_points, img_points, (w, h), None, None )
#	print "RMS:", rms
#	print "Camera matrix:\n", camera_matrix
#	print "Distortion coefficients:\n", dist_coefs.ravel()

	# Cleanup OpenCV windows
	cv2.destroyAllWindows()
<|endoftext|>


<commit_before>#!/usr/bin/python env
#-*- coding: utf-8 -*-
import MySQLdb
import MySQLdb.cursors
import numpy as np
__author__ = 'huangyi'

divide_before_sql = '''INSERT INTO train_user_before
(user_id,item_id,behavior_type,user_geohash,item_catagory, daydiff)
SELECT user_id,item_id,behavior_type,user_geohash,item_catagory,DATEDIFF(time, '2014-11-18') AS daydiff
FROM train_user WHERE time<= '%s' '''

divide_after_sql = '''INSERT INTO train_user_after
(user_id,item_id,behavior_type,user_geohash,item_catagory, daydiff)
SELECT user_id,item_id,behavior_type,user_geohash,item_catagory,DATEDIFF(time, '2014-11-18') AS daydiff
FROM train_user WHERE time> '%s' '''

merge_look_sql = '''INSERT INTO user_look
(user_id, item_id, look)
SELECT user_id, item_id, group_concat(daydiff) as look
FROM train_user_before WHERE behavior_type=1 GROUP BY user_id, item_id '''

merge_store_sql = '''INSERT INTO user_store
(user_id, item_id, store)
SELECT user_id, item_id, group_concat(daydiff) as store
FROM train_user_before WHERE behavior_type=2 GROUP BY user_id, item_id '''

merge_cart_sql = '''INSERT INTO user_cart
(user_id, item_id, cart)
SELECT user_id, item_id, group_concat(daydiff) as cart
FROM train_user_before WHERE behavior_type=3 GROUP BY user_id, item_id '''

merge_buy_sql = '''INSERT INTO user_buy
(user_id, item_id, buy)
SELECT user_id, item_id, group_concat(daydiff) as buy
FROM train_user_before WHERE behavior_type=4 GROUP BY user_id, item_id '''

#四个表合并太慢，先建立索引。这里应该用outer join，但mysql没有。后面再补充。
merge_all_behavoir = '''INSERT INTO user_features
(user_id, item_id, look, store, cart, buy)
SELECT user_look.user_id, user_look.item_id,
user_look.look, user_store.store, user_cart.cart, user_buy.buy
FROM user_look
LEFT join  user_store ON user_look.user_id = user_store.user_id AND
                        user_look.item_id = user_store.item_id
LEFT join  user_cart  ON user_look.user_id = user_cart.user_id AND
                        user_look.item_id = user_cart.item_id
LEFT join  user_buy   ON user_look.user_id = user_buy.user_id AND
                        user_look.item_id = user_buy.item_id'''

unique_user_sql = '''INSERT INTO unique_user
(user_id, item_id) SELECT user_id, item_id
FROM train_user_before GROUP BY user_id, item_id '''

DELETE_INDEX = 0

class TrainModel(object):
    def __init__(self):
        self.db = MySQLdb.connect("localhost", "root", "199194", "tianchi",
                                  cursorclass = MySQLdb.cursors.DictCursor)
        self.cursor = self.db.cursor()

    def DivideByTime(self, breaktime):
        sql1 = divide_before_sql % breaktime
        sql2 = divide_after_sql % breaktime
        try:
            self.cursor.execute(sql1)
            self.db.commit()
            self.cursor.execute(sql2)
            self.db.commit()
        except MySQLdb.Error,e:
            print "Mysql Error %d: %s" % (e.args[0], e.args[1])
            self.db.rollback()
        self.db.close

    def MergeData(self):
        try:
            '''
            self.cursor.execute(merge_look_sql)
            self.db.commit()
            self.cursor.execute(merge_store_sql)
            self.db.commit()
            self.cursor.execute(merge_cart_sql)
            self.db.commit()
            self.cursor.execute(merge_buy_sql)
            self.db.commit()
            self.cursor.execute(merge_all_behavoir)
            self.db.commit()
            self.cursor.execute(unique_user_sql)
            self.db.commit()
            '''
        except MySQLdb.Error,e:
            print "Mysql Error %d: %s" % (e.args[0], e.args[1])
            self.db.rollback()
        self.db.close

    #当用原始数据统计的四种行为（在表user_look,user_store中）符合某个标准，则输出到文本并插入到表pure_data中
    def SimplifyTrainUser(self):
        #一次处理100万左右数据，多了内存占用过高
        self.cursor.execute('SELECT user_id, item_id, look, store, cart, buy FROM user_features')
        fop = open('feature.txt', 'a')
        while True:
            str_all=self.cursor.fetchone()
            if str_all:
                look_times = len(str_all['look'].split(',')) if str_all['look'] else 0
                store_times = len(str_all['store'].split(',')) if str_all['store'] else 0
                cart_times = len(str_all['cart'].split(',')) if str_all['cart'] else 0
                buy_times = len(str_all['buy'].split(',')) if str_all['buy'] else 0
                lru = max(str_all['look'].split(',') + str_all['store'].split(',') + str_all['cart'].split(',') + str_all['buy'].split(','))
                user_id = str_all['user_id']
                item_id = str_all['item_id']

                #如果不是 浏览次数小于4 && 收藏次数小于1 && 最近15天没有记录
                if not (look_times + store_times + cart_times <= 7 and buy_times==0 and int(lru) <= 21 ):
                    fop.write('%d,%d,\"%s\",\"%s\",\"%s\",\"%s\"' % (user_id,item_id,str_all['look'],str_all['store'],str_all['cart'],str_all['buy']) )
                    fop.write('\n')
                    global DELETE_INDEX
                    DELETE_INDEX += 1

            else:
                fop.close()
                return 0

    #特性：一个内四种行为次数 + 总和
    def ExtractMonthlyBehavior(self, user_id, item_id):
        self.cursor.execute('SELECT look, store, cart, buy FROM pure_data WHERE user_id=%s AND item_id=%s', (user_id, item_id))
        str = self.cursor.fetchall()
        if not str:
            return 0
        look_times = len(str[0]['look'].split(',')) if str[0]['look'] else 0
        store_times = len(str[0]['store'].split(',')) if str[0]['store'] else 0
        cart_times = len(str[0]['cart'].split(',')) if str[0]['cart'] else 0
        buy_times = len(str[0]['buy'].split(',')) if str[0]['buy'] else 0
        all_times = look_times + store_times + cart_times + buy_times

        feature = (look_times, store_times, cart_times, buy_times, all_times )
        return feature

    #特性：最后四天四种行为次数 + 总和
    def ExtractLastdaysBehavior(self, user_id, item_id):
        self.cursor.execute('SELECT look, store, cart, buy FROM pure_data WHERE user_id=%s AND item_id=%s', (user_id, item_id))
        str = self.cursor.fetchall()
        if not str:
            return 0

        look_times = store_times = cart_times = buy_times = 0
        look_date = str[0]['look'].split(',') if str[0]['look'] else 0
        store_date = str[0]['store'].split(',') if str[0]['store'] else 0
        cart_date = str[0]['cart'].split(',') if str[0]['cart'] else 0
        buy_date = str[0]['buy'].split(',') if str[0]['buy'] else 0

        if look_date:
            for l in look_date:
                if l and int(l) >= 26:
                    look_times += 1
        if store_date:
            for l in store_date:
                if l and int(l) >= 26:
                    store_times += 1
        if cart_date:
            for l in cart_date:
                if l and int(l) >= 26:
                    cart_times += 1
        if buy_date:
            for l in buy_date:
                if l and int(l) >= 26:
                    buy_times += 1

        all_times = look_times + store_times + cart_times + buy_times
        feature = (look_times, store_times, cart_times, buy_times, all_times )
        return feature

    #特征：最后一次行为离分割点的间隔
    def ExtractLRUsed(self, user_id, item_id):
        self.cursor.execute(' SELECT max(daydiff) FROM train_user_before WHERE user_id=%s AND item_id=%s', (user_id, item_id))
        LRU = self.cursor.fetchall()
        LRUnumber = LRU[0]['max(daydiff)'] if LRU else 0
        feature = int(LRUnumber)
        return feature

    def MergeFeatures(self):
        fop = open('data_features.txt', 'a')
        self.cursor.execute('SELECT user_id, item_id FROM pure_data')
        record = self.cursor.fetchall()
        for rec in record:
            userid =  rec['user_id']
            itemid =  rec['item_id']

            #融合所有特征
            feature1 = self.ExtractMonthlyBehavior(userid, itemid)
            feature2 = self.ExtractLastdaysBehavior(userid, itemid)
            feature3 = self.ExtractLRUsed(userid, itemid)
            features = feature1, feature2, feature3
            line = features[0] + features[1] + (features[2],)

            fop.write('%d %d %d %d %d %d %d %d %d %d %d' % (line[0],line[1],line[2],line[3],line[4],line[5],line[6],line[7],line[8],line[9],line[10]))
            fop.write('\n')
        fop.close()

    def GenLabels(self):
        fop = open('data_labels.txt', 'a')
        self.cursor.execute('SELECT user_id, item_id FROM pure_data')
        record = self.cursor.fetchall()
        for rec in record:
            userid =  rec['user_id']
            itemid =  rec['item_id']
            purchase_flag = False
            self.cursor.execute('SELECT behavior_type FROM train_user_after WHERE user_id=%s AND item_id=%s', (userid, itemid))
            lines = self.cursor.fetchall()
            if lines:
                for line in lines:
                    if line['behavior_type'] == 4:
                        purchase_flag = True
            if purchase_flag:
                fop.write('1')
                fop.write('\n')
            else:
                fop.write('0')
                fop.write('\n')
        fop.close()

if __name__ == '__main__':
    model = TrainModel()
    #model.DivideByTime("2014-12-18 00:00:00")
    #model.MergeData()
    #model.SimplifyTrainUser()
    #print DELETE_INDEX
    #print model.ExtractMonthlyBehavior(19095, 14148321)
    #print model.ExtractLastdaysBehavior(19095, 15088134)
    #print model.ExtractLRUsed(19095, 15088134)
    #model.MergeFeatures()
    model.GenLabels()<commit_msg>Delete TrainModel.py<commit_after><|endoftext|>