<commit_before># -*- coding: utf-8 -*-

"""Provides personal data."""

import hashlib
import re
import uuid
from string import ascii_letters, digits, punctuation
from typing import Optional, Union

from mimesis.data import (
    BLOOD_GROUPS,
    CALLING_CODES,
    EMAIL_DOMAINS,
    GENDER_SYMBOLS,
    SEXUALITY_SYMBOLS,
    SOCIAL_NETWORKS,
    USERNAMES,
)
from mimesis.enums import Gender, SocialNetwork, TitleType
from mimesis.exceptions import NonEnumerableError
from mimesis.providers.base import BaseDataProvider
from mimesis.random import get_random_item

__all__ = ['Person']

class Person(BaseDataProvider):
    """Class for generating personal data."""

    def __init__(self, *args, **kwargs) -> None:
        """Initialize attributes.

        :param locale: Current locale.
        :param seed: Seed.
        """
        super().__init__(*args, **kwargs)
        self._datafile = 'person.json'
        self.pull(self._datafile)
        self._store = {
            'age': 0,
        }

    class Meta:
        """Class for metadata."""

        name = 'person'

    def age(self, minimum: int = 16, maximum: int = 66) -> int:
        """Get a random integer value.

        :param maximum: Maximum value of age.
        :param minimum: Minimum value of age.
        :return: Random integer.

        :Example:
            23.
        """
        age = self.random.randint(minimum, maximum)
        self._store['age'] = age
        return age

    def work_experience(self, working_start_age: int = 22) -> int:
        """Get a work experience.

        :param working_start_age: Age then person start to work.
        :return: Depend on previous generated age.
        """
        age = self._store['age']
        if age == 0:
            age = self.age()

        return max(age - working_start_age, 0)

    def name(self, gender: Optional[Gender] = None) -> str:
        """Generate a random name.

        :param gender: Gender's enum object.
        :return: Name.

        :Example:
            John.
        """
        key = self._validate_enum(gender, Gender)
        names = self._data['names'].get(key)
        return self.random.choice(names)

    def first_name(self, gender: Optional[Gender] = None):
        """Generate a random first name.

        ..note: An alias for self.name().

        :param gender: Gender's enum object.
        :return: First name.
        """
        return self.name(gender)

    def surname(self, gender: Optional[Gender] = None) -> str:
        """Generate a random surname.

        :param gender: Gender's enum object.
        :return: Surname.

        :Example:
            Smith.
        """
        surnames = self._data['surnames']

        # Surnames separated by gender.
        if isinstance(surnames, dict):
            key = self._validate_enum(gender, Gender)
            surnames = surnames[key]

        return self.random.choice(surnames)

    def last_name(self, gender: Optional[Gender] = None) -> str:
        """Generate a random last name.

        ..note: An alias for self.surname().

        :param gender: Gender's enum object.
        :return: Last name.
        """
        return self.surname(gender)

    def title(self, gender: Optional[Gender] = None,
              title_type: Optional[TitleType] = None) -> str:
        """Generate a random title for name.

        You can generate random prefix or suffix
        for name using this method.

        :param gender: The gender.
        :param title_type: TitleType enum object.
        :return: The title.
        :raises NonEnumerableError: if gender or title_type in incorrect format.

        :Example:
            PhD.
        """
        gender_key = self._validate_enum(gender, Gender)
        title_key = self._validate_enum(title_type, TitleType)

        titles = self._data['title'][gender_key][title_key]
        return self.random.choice(titles)

    def full_name(self, gender: Optional[Gender] = None,
                  reverse: bool = False) -> str:
        """Generate a random full name.

        :param reverse: Return reversed full name.
        :param gender: Gender's enum object.
        :return: Full name.

        :Example:
            Johann Wolfgang.
        """
        if gender is None:
            gender = get_random_item(Gender, rnd=self.random)

        if gender and isinstance(gender, Gender):
            gender = gender
        else:
            raise NonEnumerableError(Gender)

        fmt = '{1} {0}' if reverse else '{0} {1}'
        return fmt.format(
            self.name(gender),
            self.surname(gender),
        )

    def username(self, template: Optional[str] = None) -> str:
        """Generate username by template.

        Supported template placeholders: (U, l, d)

        Supported separators: (-, ., _)

        Template must contain at least one "U" or "l" placeholder.

        If template is None one of the following templates is used:
        ('U_d', 'U.d', 'U-d', 'UU-d', 'UU.d', 'UU_d',
        'ld', 'l-d', 'Ud', 'l.d', 'l_d', 'default')

        :param template: Template.
        :return: Username.
        :raises ValueError: If template is not supported.

        :Example:
            Celloid1873
        """
        min_date = 1800
        max_date = 2070
        default_template = 'l.d'

        templates = ('U_d', 'U.d', 'U-d', 'UU-d', 'UU.d', 'UU_d',
                     'ld', 'l-d', 'Ud', 'l.d', 'l_d', 'default')

        if template is None:
            template = self.random.choice(templates)

        if template == 'default':
            template = default_template

        if not re.fullmatch(r'[Ul\.\-\_d]*[Ul]+[Ul\.\-\_d]*', template):
            raise ValueError(
                "Template '{}' is not supported.".format(template))

        tags = re.findall(r'[Uld\.\-\_]', template)

        username = ''
        for tag in tags:
            if tag == 'U':
                username += self.random.choice(USERNAMES).capitalize()
            elif tag == 'l':
                username += self.random.choice(USERNAMES)
            elif tag == 'd':
                username += str(self.random.randint(min_date, max_date))
            elif tag in '-_.':
                username += tag

        return username

    def password(self, length: int = 8, hashed: bool = False) -> str:
        """Generate a password or hash of password.

        :param length: Length of password.
        :param hashed: MD5 hash.
        :return: Password or hash of password.

        :Example:
            k6dv2odff9#4h
        """
        text = ascii_letters + digits + punctuation
        password = ''.join([self.random.choice(text) for _ in range(length)])

        if hashed:
            md5 = hashlib.md5()
            md5.update(password.encode())
            return md5.hexdigest()

        return password

    def email(self, domains: Union[tuple, list] = None,
              unique: bool = False) -> str:
        """Generate a random email.

        :param domains: List of custom domains for emails.
        :param unique: Make emails unique.
        :type domains: list or tuple
        :return: Email address.
        :raises ValueError: if «unique» is True and the provider was seeded.

        :Example:
            foretime10@live.com
        """
        if unique and self.seed is not None:
            raise ValueError('You cannot use «unique» '
                             'parameter with a seeded provider')

        if not domains:
            domains = EMAIL_DOMAINS

        domain = self.random.choice(domains)

        if not domain.startswith('@'):
            domain = '@{}'.format(domain)

        if unique:
            name = uuid.uuid4().hex
        else:
            name = self.username(template='ld')

        return '{name}{domain}'.format(
            name=name,
            domain=domain,
        )

    def social_media_profile(self,
                             site: Optional[SocialNetwork] = None) -> str:
        """Generate profile for random social network.

        :return: Profile in some network.

        :Example:
            http://facebook.com/some_user
        """
        key = self._validate_enum(site, SocialNetwork)
        website = SOCIAL_NETWORKS[key]
        url = 'https://www.' + website
        return url.format(self.username())

    def gender(self, iso5218: bool = False,
               symbol: bool = False) -> Union[str, int]:
        """Get a random gender.

        Get a random title of gender, code for the representation
        of human sexes is an international standard that defines a
        representation of human sexes through a language-neutral single-digit
        code or symbol of gender.

        :param iso5218:
            Codes for the representation of human sexes is an international
            standard (0 - not known, 1 - male, 2 - female, 9 - not applicable).
        :param symbol: Symbol of gender.
        :return: Title of gender.

        :Example:
            Male
        """
        if iso5218:
            return self.random.choice([0, 1, 2, 9])

        if symbol:
            return self.random.choice(GENDER_SYMBOLS)

        return self.random.choice(self._data['gender'])

    def sex(self, *args, **kwargs):
        """An alias for method self.gender().

        See docstrings of method self.gender() for details.

        :param args: Positional arguments.
        :param kwargs: Keyword arguments.
        :return: Sex
        """
        return self.gender(*args, **kwargs)

    def height(self, minimum: float = 1.5, maximum: float = 2.0) -> str:
        """Generate a random height in meters.

        :param minimum: Minimum value.
        :param float maximum: Maximum value.
        :return: Height.

        :Example:
            1.85.
        """
        h = self.random.uniform(minimum, maximum)
        return '{:0.2f}'.format(h)

    def weight(self, minimum: int = 38, maximum: int = 90) -> int:
        """Generate a random weight in Kg.

        :param minimum: min value
        :param maximum: max value
        :return: Weight.

        :Example:
            48.
        """
        weight = self.random.randint(minimum, maximum)
        return weight

    def blood_type(self) -> str:
        """Get a random blood type.

        :return: Blood type (blood group).

        :Example:
            A+
        """
        return self.random.choice(BLOOD_GROUPS)

    def sexual_orientation(self, symbol: bool = False) -> str:
        """Get a random sexual orientation.

        Obviously, this option will be useful for various
        dating sites and so on.

        :param symbol: Unicode symbol.
        :return: Sexual orientation.

        :Example:
            Heterosexuality.
        """
        if symbol:
            return self.random.choice(SEXUALITY_SYMBOLS)

        sexuality = self._data['sexuality']
        return self.random.choice(sexuality)

    def occupation(self) -> str:
        """Get a random job.

        :return: The name of job.

        :Example:
            Programmer.
        """
        jobs = self._data['occupation']
        return self.random.choice(jobs)

    def political_views(self) -> str:
        """Get a random political views.

        :return: Political views.

        :Example:
            Liberal.
        """
        views = self._data['political_views']
        return self.random.choice(views)

    def worldview(self) -> str:
        """Get a random worldview.

        :return: Worldview.

        :Example:
            Pantheism.
        """
        views = self._data['worldview']
        return self.random.choice(views)

    def views_on(self) -> str:
        """Get a random views on.

        :return: Views on.

        :Example:
            Negative.
        """
        views = self._data['views_on']
        return self.random.choice(views)

    def nationality(self, gender: Optional[Gender] = None) -> str:
        """Get a random nationality.

        :param gender: Gender.
        :return: Nationality.

        :Example:
            Russian
        """
        nationalities = self._data['nationality']

        # Separated by gender
        if isinstance(nationalities, dict):
            key = self._validate_enum(gender, Gender)
            nationalities = nationalities[key]

        return self.random.choice(nationalities)

    def university(self) -> str:
        """Get a random university.

        :return: University name.

        :Example:
            MIT.
        """
        universities = self._data['university']
        return self.random.choice(universities)

    def academic_degree(self) -> str:
        """Get a random academic degree.

        :return: Degree.

        :Example:
            Bachelor.
        """
        degrees = self._data['academic_degree']
        return self.random.choice(degrees)

    def language(self) -> str:
        """Get a random language.

        :return: Random language.

        :Example:
            Irish.
        """
        languages = self._data['language']
        return self.random.choice(languages)

    def telephone(self, mask: str = '', placeholder: str = '#') -> str:
        """Generate a random phone number.

        :param mask: Mask for formatting number.
        :param placeholder: A placeholder for a mask (default is #).
        :return: Phone number.

        :Example:
            +7-(963)-409-11-22.
        """
        if not mask:
            code = self.random.choice(CALLING_CODES)
            default = '{}-(###)-###-####'.format(code)
            masks = self._data.get('telephone_fmt', [default])
            mask = self.random.choice(masks)

        return self.random.custom_code(mask=mask, digit=placeholder)

    def avatar(self, size: int = 256) -> str:
        """Generate a random avatar..

        :param size: Size of avatar.
        :return: Link to avatar.
        """
        url = 'https://api.adorable.io/avatars/{0}/{1}.png'
        return url.format(size, self.password(hashed=True))

    def identifier(self, mask: str = '##-##/##') -> str:
        """Generate a random identifier by mask.

        With this method you can generate any identifiers that
        you need. Simply select the mask that you need.

        :param mask:
            The mask. Here ``@`` is a placeholder for characters and ``#`` is
            placeholder for digits.
        :return: An identifier.

        :Example:
            07-97/04
        """
        return self.random.custom_code(mask=mask)
<commit_msg>Fix typo in docstrings of Person.email<commit_after># -*- coding: utf-8 -*-

"""Provides personal data."""

import hashlib
import re
import uuid
from string import ascii_letters, digits, punctuation
from typing import Optional, Union

from mimesis.data import (
    BLOOD_GROUPS,
    CALLING_CODES,
    EMAIL_DOMAINS,
    GENDER_SYMBOLS,
    SEXUALITY_SYMBOLS,
    SOCIAL_NETWORKS,
    USERNAMES,
)
from mimesis.enums import Gender, SocialNetwork, TitleType
from mimesis.exceptions import NonEnumerableError
from mimesis.providers.base import BaseDataProvider
from mimesis.random import get_random_item

__all__ = ['Person']

class Person(BaseDataProvider):
    """Class for generating personal data."""

    def __init__(self, *args, **kwargs) -> None:
        """Initialize attributes.

        :param locale: Current locale.
        :param seed: Seed.
        """
        super().__init__(*args, **kwargs)
        self._datafile = 'person.json'
        self.pull(self._datafile)
        self._store = {
            'age': 0,
        }

    class Meta:
        """Class for metadata."""

        name = 'person'

    def age(self, minimum: int = 16, maximum: int = 66) -> int:
        """Get a random integer value.

        :param maximum: Maximum value of age.
        :param minimum: Minimum value of age.
        :return: Random integer.

        :Example:
            23.
        """
        age = self.random.randint(minimum, maximum)
        self._store['age'] = age
        return age

    def work_experience(self, working_start_age: int = 22) -> int:
        """Get a work experience.

        :param working_start_age: Age then person start to work.
        :return: Depend on previous generated age.
        """
        age = self._store['age']
        if age == 0:
            age = self.age()

        return max(age - working_start_age, 0)

    def name(self, gender: Optional[Gender] = None) -> str:
        """Generate a random name.

        :param gender: Gender's enum object.
        :return: Name.

        :Example:
            John.
        """
        key = self._validate_enum(gender, Gender)
        names = self._data['names'].get(key)
        return self.random.choice(names)

    def first_name(self, gender: Optional[Gender] = None):
        """Generate a random first name.

        ..note: An alias for self.name().

        :param gender: Gender's enum object.
        :return: First name.
        """
        return self.name(gender)

    def surname(self, gender: Optional[Gender] = None) -> str:
        """Generate a random surname.

        :param gender: Gender's enum object.
        :return: Surname.

        :Example:
            Smith.
        """
        surnames = self._data['surnames']

        # Surnames separated by gender.
        if isinstance(surnames, dict):
            key = self._validate_enum(gender, Gender)
            surnames = surnames[key]

        return self.random.choice(surnames)

    def last_name(self, gender: Optional[Gender] = None) -> str:
        """Generate a random last name.

        ..note: An alias for self.surname().

        :param gender: Gender's enum object.
        :return: Last name.
        """
        return self.surname(gender)

    def title(self, gender: Optional[Gender] = None,
              title_type: Optional[TitleType] = None) -> str:
        """Generate a random title for name.

        You can generate random prefix or suffix
        for name using this method.

        :param gender: The gender.
        :param title_type: TitleType enum object.
        :return: The title.
        :raises NonEnumerableError: if gender or title_type in incorrect format.

        :Example:
            PhD.
        """
        gender_key = self._validate_enum(gender, Gender)
        title_key = self._validate_enum(title_type, TitleType)

        titles = self._data['title'][gender_key][title_key]
        return self.random.choice(titles)

    def full_name(self, gender: Optional[Gender] = None,
                  reverse: bool = False) -> str:
        """Generate a random full name.

        :param reverse: Return reversed full name.
        :param gender: Gender's enum object.
        :return: Full name.

        :Example:
            Johann Wolfgang.
        """
        if gender is None:
            gender = get_random_item(Gender, rnd=self.random)

        if gender and isinstance(gender, Gender):
            gender = gender
        else:
            raise NonEnumerableError(Gender)

        fmt = '{1} {0}' if reverse else '{0} {1}'
        return fmt.format(
            self.name(gender),
            self.surname(gender),
        )

    def username(self, template: Optional[str] = None) -> str:
        """Generate username by template.

        Supported template placeholders: (U, l, d)

        Supported separators: (-, ., _)

        Template must contain at least one "U" or "l" placeholder.

        If template is None one of the following templates is used:
        ('U_d', 'U.d', 'U-d', 'UU-d', 'UU.d', 'UU_d',
        'ld', 'l-d', 'Ud', 'l.d', 'l_d', 'default')

        :param template: Template.
        :return: Username.
        :raises ValueError: If template is not supported.

        :Example:
            Celloid1873
        """
        min_date = 1800
        max_date = 2070
        default_template = 'l.d'

        templates = ('U_d', 'U.d', 'U-d', 'UU-d', 'UU.d', 'UU_d',
                     'ld', 'l-d', 'Ud', 'l.d', 'l_d', 'default')

        if template is None:
            template = self.random.choice(templates)

        if template == 'default':
            template = default_template

        if not re.fullmatch(r'[Ul\.\-\_d]*[Ul]+[Ul\.\-\_d]*', template):
            raise ValueError(
                "Template '{}' is not supported.".format(template))

        tags = re.findall(r'[Uld\.\-\_]', template)

        username = ''
        for tag in tags:
            if tag == 'U':
                username += self.random.choice(USERNAMES).capitalize()
            elif tag == 'l':
                username += self.random.choice(USERNAMES)
            elif tag == 'd':
                username += str(self.random.randint(min_date, max_date))
            elif tag in '-_.':
                username += tag

        return username

    def password(self, length: int = 8, hashed: bool = False) -> str:
        """Generate a password or hash of password.

        :param length: Length of password.
        :param hashed: MD5 hash.
        :return: Password or hash of password.

        :Example:
            k6dv2odff9#4h
        """
        text = ascii_letters + digits + punctuation
        password = ''.join([self.random.choice(text) for _ in range(length)])

        if hashed:
            md5 = hashlib.md5()
            md5.update(password.encode())
            return md5.hexdigest()

        return password

    def email(self, domains: Union[tuple, list] = None,
              unique: bool = False) -> str:
        """Generate a random email.

        :param domains: List of custom domains for emails.
        :param unique: Makes email addresses unique.
        :return: Email address.
        :raises ValueError: if «unique» is True and the provider was seeded.

        :Example:
            foretime10@live.com
        """
        if unique and self.seed is not None:
            raise ValueError('You cannot use «unique» '
                             'parameter with a seeded provider')

        if not domains:
            domains = EMAIL_DOMAINS

        domain = self.random.choice(domains)

        if not domain.startswith('@'):
            domain = '@{}'.format(domain)

        if unique:
            name = uuid.uuid4().hex
        else:
            name = self.username(template='ld')

        return '{name}{domain}'.format(
            name=name,
            domain=domain,
        )

    def social_media_profile(self,
                             site: Optional[SocialNetwork] = None) -> str:
        """Generate profile for random social network.

        :return: Profile in some network.

        :Example:
            http://facebook.com/some_user
        """
        key = self._validate_enum(site, SocialNetwork)
        website = SOCIAL_NETWORKS[key]
        url = 'https://www.' + website
        return url.format(self.username())

    def gender(self, iso5218: bool = False,
               symbol: bool = False) -> Union[str, int]:
        """Get a random gender.

        Get a random title of gender, code for the representation
        of human sexes is an international standard that defines a
        representation of human sexes through a language-neutral single-digit
        code or symbol of gender.

        :param iso5218:
            Codes for the representation of human sexes is an international
            standard (0 - not known, 1 - male, 2 - female, 9 - not applicable).
        :param symbol: Symbol of gender.
        :return: Title of gender.

        :Example:
            Male
        """
        if iso5218:
            return self.random.choice([0, 1, 2, 9])

        if symbol:
            return self.random.choice(GENDER_SYMBOLS)

        return self.random.choice(self._data['gender'])

    def sex(self, *args, **kwargs):
        """An alias for method self.gender().

        See docstrings of method self.gender() for details.

        :param args: Positional arguments.
        :param kwargs: Keyword arguments.
        :return: Sex
        """
        return self.gender(*args, **kwargs)

    def height(self, minimum: float = 1.5, maximum: float = 2.0) -> str:
        """Generate a random height in meters.

        :param minimum: Minimum value.
        :param float maximum: Maximum value.
        :return: Height.

        :Example:
            1.85.
        """
        h = self.random.uniform(minimum, maximum)
        return '{:0.2f}'.format(h)

    def weight(self, minimum: int = 38, maximum: int = 90) -> int:
        """Generate a random weight in Kg.

        :param minimum: min value
        :param maximum: max value
        :return: Weight.

        :Example:
            48.
        """
        weight = self.random.randint(minimum, maximum)
        return weight

    def blood_type(self) -> str:
        """Get a random blood type.

        :return: Blood type (blood group).

        :Example:
            A+
        """
        return self.random.choice(BLOOD_GROUPS)

    def sexual_orientation(self, symbol: bool = False) -> str:
        """Get a random sexual orientation.

        Obviously, this option will be useful for various
        dating sites and so on.

        :param symbol: Unicode symbol.
        :return: Sexual orientation.

        :Example:
            Heterosexuality.
        """
        if symbol:
            return self.random.choice(SEXUALITY_SYMBOLS)

        sexuality = self._data['sexuality']
        return self.random.choice(sexuality)

    def occupation(self) -> str:
        """Get a random job.

        :return: The name of job.

        :Example:
            Programmer.
        """
        jobs = self._data['occupation']
        return self.random.choice(jobs)

    def political_views(self) -> str:
        """Get a random political views.

        :return: Political views.

        :Example:
            Liberal.
        """
        views = self._data['political_views']
        return self.random.choice(views)

    def worldview(self) -> str:
        """Get a random worldview.

        :return: Worldview.

        :Example:
            Pantheism.
        """
        views = self._data['worldview']
        return self.random.choice(views)

    def views_on(self) -> str:
        """Get a random views on.

        :return: Views on.

        :Example:
            Negative.
        """
        views = self._data['views_on']
        return self.random.choice(views)

    def nationality(self, gender: Optional[Gender] = None) -> str:
        """Get a random nationality.

        :param gender: Gender.
        :return: Nationality.

        :Example:
            Russian
        """
        nationalities = self._data['nationality']

        # Separated by gender
        if isinstance(nationalities, dict):
            key = self._validate_enum(gender, Gender)
            nationalities = nationalities[key]

        return self.random.choice(nationalities)

    def university(self) -> str:
        """Get a random university.

        :return: University name.

        :Example:
            MIT.
        """
        universities = self._data['university']
        return self.random.choice(universities)

    def academic_degree(self) -> str:
        """Get a random academic degree.

        :return: Degree.

        :Example:
            Bachelor.
        """
        degrees = self._data['academic_degree']
        return self.random.choice(degrees)

    def language(self) -> str:
        """Get a random language.

        :return: Random language.

        :Example:
            Irish.
        """
        languages = self._data['language']
        return self.random.choice(languages)

    def telephone(self, mask: str = '', placeholder: str = '#') -> str:
        """Generate a random phone number.

        :param mask: Mask for formatting number.
        :param placeholder: A placeholder for a mask (default is #).
        :return: Phone number.

        :Example:
            +7-(963)-409-11-22.
        """
        if not mask:
            code = self.random.choice(CALLING_CODES)
            default = '{}-(###)-###-####'.format(code)
            masks = self._data.get('telephone_fmt', [default])
            mask = self.random.choice(masks)

        return self.random.custom_code(mask=mask, digit=placeholder)

    def avatar(self, size: int = 256) -> str:
        """Generate a random avatar..

        :param size: Size of avatar.
        :return: Link to avatar.
        """
        url = 'https://api.adorable.io/avatars/{0}/{1}.png'
        return url.format(size, self.password(hashed=True))

    def identifier(self, mask: str = '##-##/##') -> str:
        """Generate a random identifier by mask.

        With this method you can generate any identifiers that
        you need. Simply select the mask that you need.

        :param mask:
            The mask. Here ``@`` is a placeholder for characters and ``#`` is
            placeholder for digits.
        :return: An identifier.

        :Example:
            07-97/04
        """
        return self.random.custom_code(mask=mask)
<|endoftext|>


<commit_before>import logging

import sublime
from . import util
from jsonschema.validators import validate
from jsonschema.exceptions import ValidationError

logger = logging.getLogger(__name__)

class Settings:
    """This class provides global access to and management of plugin settings."""

    def __init__(self):
        self._storage = {}

    def load(self):
        """Load the plugin settings."""
        self.observe()
        self.on_update()

    @property
    def settings(self):
        return sublime.load_settings("SublimeLinter.sublime-settings")

    def has(self, name):
        """Return whether the given setting exists."""
        return self.settings.has(name)

    def get(self, name, default=None):
        """Return a plugin setting, defaulting to default if not found."""
        return self.settings.get(name, default)

    def has_changed(self, name):
        current_value = self.get(name)
        try:
            old_value = self._storage[name]
        except KeyError:
            return False
        else:
            return (old_value != current_value)
        finally:
            self._storage[name] = current_value

    def observe(self):
        """Observe changes."""
        settings = sublime.load_settings("SublimeLinter.sublime-settings")
        settings.clear_on_change('sublimelinter-persist-settings')
        settings.add_on_change('sublimelinter-persist-settings', self.on_update)

    def unobserve(self):
        settings = sublime.load_settings("SublimeLinter.sublime-settings")
        settings.clear_on_change('sublimelinter-persist-settings')

    def on_update(self):
        """
        Update state when the user settings change.

        The settings before the change are compared with the new settings.
        Depending on what changes, views will either be redrawn or relinted.

        """
        if not validate_settings():
            return

        from . import style

        # Reparse settings for style rules
        # Linter specific settings can include style rules too
        if self.has_changed('styles') or self.has_changed('linters'):
            style.StyleParser()()

        if self.has_changed('gutter_theme'):
            style.read_gutter_theme()

        sublime.run_command('sublime_linter_config_changed')

def get_settings_objects():
    for name in sublime.find_resources("SublimeLinter.sublime-settings"):
        try:
            yield name, util.load_json(name, from_sl_dir=False)
        except IOError:
            pass
        except ValueError:
            logger.error("Settings file corrupt: {}".format(name))

def validate_settings():
    status_msg = "SublimeLinter - Settings invalid!"
    schema_file = "resources/settings-schema.json"
    schema = util.load_json(schema_file, from_sl_dir=True)
    window = sublime.active_window()
    util.clear_message()
    good = True

    for name, settings in get_settings_objects():
        try:
            validate(settings, schema)
        except ValidationError as error:
            good = False
            error_msg = error.message.split("\n")[0]  # reduce verbosity
            full_msg = "Invalid settings in '{}':\n{}".format(name, error_msg)

            logger.error(full_msg)
            window.status_message(status_msg)
    return good
<commit_msg>fix #1387 don't validate empty settings file<commit_after>import logging

import sublime
from . import util
from jsonschema.validators import validate
from jsonschema.exceptions import ValidationError

logger = logging.getLogger(__name__)

class Settings:
    """This class provides global access to and management of plugin settings."""

    def __init__(self):
        self._storage = {}

    def load(self):
        """Load the plugin settings."""
        self.observe()
        self.on_update()

    @property
    def settings(self):
        return sublime.load_settings("SublimeLinter.sublime-settings")

    def has(self, name):
        """Return whether the given setting exists."""
        return self.settings.has(name)

    def get(self, name, default=None):
        """Return a plugin setting, defaulting to default if not found."""
        return self.settings.get(name, default)

    def has_changed(self, name):
        current_value = self.get(name)
        try:
            old_value = self._storage[name]
        except KeyError:
            return False
        else:
            return (old_value != current_value)
        finally:
            self._storage[name] = current_value

    def observe(self):
        """Observe changes."""
        settings = sublime.load_settings("SublimeLinter.sublime-settings")
        settings.clear_on_change('sublimelinter-persist-settings')
        settings.add_on_change('sublimelinter-persist-settings', self.on_update)

    def unobserve(self):
        settings = sublime.load_settings("SublimeLinter.sublime-settings")
        settings.clear_on_change('sublimelinter-persist-settings')

    def on_update(self):
        """
        Update state when the user settings change.

        The settings before the change are compared with the new settings.
        Depending on what changes, views will either be redrawn or relinted.

        """
        if not validate_settings():
            return

        from . import style

        # Reparse settings for style rules
        # Linter specific settings can include style rules too
        if self.has_changed('styles') or self.has_changed('linters'):
            style.StyleParser()()

        if self.has_changed('gutter_theme'):
            style.read_gutter_theme()

        sublime.run_command('sublime_linter_config_changed')

def get_settings_objects():
    for name in sublime.find_resources("SublimeLinter.sublime-settings"):
        try:
            yield name, util.load_json(name, from_sl_dir=False)
        except IOError:
            pass
        except ValueError:
            logger.error("Settings file corrupt: {}".format(name))

def validate_settings():
    status_msg = "SublimeLinter - Settings invalid!"
    schema_file = "resources/settings-schema.json"
    schema = util.load_json(schema_file, from_sl_dir=True)
    window = sublime.active_window()
    util.clear_message()
    good = True

    for name, settings in get_settings_objects():
        if settings:
            try:
                validate(settings, schema)
            except ValidationError as error:
                good = False
                error_msg = error.message.split("\n")[0]  # reduce verbosity
                full_msg = "Invalid settings in '{}':\n{}".format(name, error_msg)

                logger.error(full_msg)
                window.status_message(status_msg)
    return good
<|endoftext|>


<commit_before>#!/usr/env/python
"""
    See LICENSE file for copyright and license details.
"""

from datetime import datetime

import sys
import getopt

from setup.setup import Setup

# general properties of the app
pprog = 'lisa.py'
pversion = '01.00'
prelease = 'Mass Convenience'
pdate = '2012-04-10'

def main(option):
    """ Main driver. """
    ### Run the application ###
    #NOTE: the import statement loads the views and tables,
    #but when doing an install, they are not created yet.
    #So we skip loading this until we are sure we can start.
    from main.main import MainWrapper
    wrapper = MainWrapper()
    if (option == 'import'):
        wrapper.file_import()
        wrapper.exitstate = 1
    if (option == 'export'):
        wrapper.file_export()
        wrapper.exitstate = 1
    wrapper.run() #run the main method for the program

def install():
    """ install """
    setup = Setup()
    setup.install()
    setup = None

def uninstall():
    """ uninstall """
    setup = Setup()
    setup.uninstall()
    setup = None

def usage():
        """ Print usage info and exit """
        print('''{0} : Less Interaction Saves Action
Options:
 -h : displays this help message
 --install : creates tables and views needed
 --uninstall : deletes all relevant tables and views in the database,
            all data will be destroyed...
 --version : displays version
 --python : displays Python version
All arguments are optional.'''.format(self.pprog))

if __name__ == "__main__":
    # Gonna switch this to optparse later
    try:
        options, xarguments = getopt.getopt(
            sys.argv[1:], 'h', ['import', 'export', 'install', 'uninstall', 'version', 'python'])
    except getopt.error as err:
        print('Error: ' + str(err))
        sys.exit(1)

    option = ''
    for opt in options[:]:
        if opt[0] == '-h':
            usage()
            sys.exit(0)
            break
    for opt in options[:]:
        if opt[0] == '--import':
            option = 'import'
            sys.exit(0)
            break
    for opt in options[:]:
        if opt[0] == '--export':
            option = 'export'
            sys.exit(0)
            break
    for opt in options[:]:
        if opt[0] == '--install':
            install()
            sys.exit(0)
            break
    for opt in options[:]:
        if opt[0] == '--uninstall':
            uninstall()
            sys.exit(0)
            break
    for opt in options[:]:
        if opt[0] == '--version':
            str_list = [
                pprog,
                ' version ',
                pversion,
                ' (',
                pdate,
                '), \'',
                prelease,
                '\' release.']
            print(''.join(str_list))
            sys.exit(0)
            break
    for opt in options[:]:
        if opt[0] == '--python':
            print('Python ' + sys.version)
            sys.exit(0)
            break
    main(option)
<commit_msg>Comment added.<commit_after>#!/usr/env/python
"""
    See LICENSE file for copyright and license details.
"""

from datetime import datetime

import sys
import getopt

from setup.setup import Setup

# general properties of the app
pprog = 'lisa.py'
pversion = '01.00'
prelease = 'Mass Convenience'
pdate = '2012-04-10'

def main(option):
    """ Main driver. """
    ### Run the application ###
    #NOTE: the import statement loads the views and tables,
    #but when doing an install, they are not created yet.
    #So we skip loading this until we are sure we can start.
    from main.main import MainWrapper
    wrapper = MainWrapper()
    if (option == 'import'):
        wrapper.file_import()
        wrapper.exitstate = 1
    if (option == 'export'):
        wrapper.file_export()
        wrapper.exitstate = 1
    #TODO: add stuff for export to ledger format?
    wrapper.run() #run the main method for the program

def install():
    """ install """
    setup = Setup()
    setup.install()
    setup = None

def uninstall():
    """ uninstall """
    setup = Setup()
    setup.uninstall()
    setup = None

def usage():
        """ Print usage info and exit """
        print('''{0} : Less Interaction Saves Action
Options:
 -h : displays this help message
 --install : creates tables and views needed
 --uninstall : deletes all relevant tables and views in the database,
            all data will be destroyed...
 --version : displays version
 --python : displays Python version
All arguments are optional.'''.format(self.pprog))

if __name__ == "__main__":
    # Gonna switch this to optparse later
    try:
        options, xarguments = getopt.getopt(
            sys.argv[1:], 'h', ['import', 'export', 'install', 'uninstall', 'version', 'python'])
    except getopt.error as err:
        print('Error: ' + str(err))
        sys.exit(1)

    option = ''
    for opt in options[:]:
        if opt[0] == '-h':
            usage()
            sys.exit(0)
            break
    for opt in options[:]:
        if opt[0] == '--import':
            option = 'import'
            sys.exit(0)
            break
    for opt in options[:]:
        if opt[0] == '--export':
            option = 'export'
            sys.exit(0)
            break
    for opt in options[:]:
        if opt[0] == '--install':
            install()
            sys.exit(0)
            break
    for opt in options[:]:
        if opt[0] == '--uninstall':
            uninstall()
            sys.exit(0)
            break
    for opt in options[:]:
        if opt[0] == '--version':
            str_list = [
                pprog,
                ' version ',
                pversion,
                ' (',
                pdate,
                '), \'',
                prelease,
                '\' release.']
            print(''.join(str_list))
            sys.exit(0)
            break
    for opt in options[:]:
        if opt[0] == '--python':
            print('Python ' + sys.version)
            sys.exit(0)
            break
    main(option)
<|endoftext|>


<commit_before>import gzip
from optparse import make_option
import os
import shutil
from StringIO import StringIO

from django.conf import settings
from django.core.management.base import BaseCommand

from lxml import etree

NS = {
    'atom': 'http://www.w3.org/2005/Atom',
    'activity': 'http://activitystrea.ms/spec/1.0/',
    }

class Command(BaseCommand):
    help = 'Process files into the database'
    option_list = BaseCommand.option_list + (
        make_option('--move', action='store_true', default=False,
            dest='move', help='move processed files aside'),
        make_option('--dir', action='store', type='string',
            default=settings.DATA_DIR, dest='dir',
            help='directory containing unprocessed data (default=DATA_DIR)'),
        )

    def handle(self, *args, **options):
        unprocessed_files = [f for f in os.listdir(options['dir']) \
            if f.endswith('xml.gz')]
        for unprocessed_file in unprocessed_files:
            print 'processing:', unprocessed_file
            filename = '%s/%s' % (options['dir'], unprocessed_file)
            try:
                self._process_file(filename)
            except IOError:
                print '*******'
                print 'IOError:', filename
                print '*******'
                continue
            if options.get('move', False):
                print 'moving file aside'
                target_filename = '%s/processed/%s' % (options['dir'],
                    unprocessed_file)
                print 'shutil.move("%s", "%s")' % (filename, target_filename)
                shutil.move(filename, target_filename)

    def _process_file(self, filename):
        fp = gzip.open(filename, 'rb')
        entry = []
        for line in fp:
            entry.append(line)
            if '</entry>' in line:
                self._parse_entry(''.join(entry))
                entry = []
        fp.close()

    def _parse_entry(self, entry):
        r = etree.parse(StringIO(entry))
        # published
        e_published = r.find('//{%s}published' % NS['atom'])
        #date_published = time.strptime(e_published.text,
        #    '%Y-%m-%dT%H:%M:%SZ')
        # FIXME: don't be punting on timezone differences now
        date_published = e_published.text.replace('T', ' ')
        date_published = date_published.replace('Z', '')
        # userid
        #e_author = r.find('//{%s}author' % NS['activity'])
        #author_link = e_author.find('.//{%s}id' % NS['atom']).text
        # avatar url
        #author_avatar = e_author.find('.//{%s}link[@rel="avatar"]' % NS['atom']).attrib['href']
        # status id
        #status_link = r.find('./{%s}link' % NS['atom']).attrib['href']
        # summary / content?
        #summary = r.find('./{%s}summary' % NS['atom']).text
        #content = r.find('.//{%s}content' % NS['atom']).text
        return

# NOTE: json url pattern for a single tweet is:
# http://api.twitter.com/1/statuses/show/176547251324329984.json
<commit_msg>removed processfeed; dates back to atom feeds from api v1<commit_after><|endoftext|>


<commit_before>from __future__ import print_function
#  ----------------------------------------------------------------
# Copyright 2016 Cisco Systems
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ------------------------------------------------------------------

#  ----------------------------------------------------------------
#  oc-interfaces.py Sample program illustrating use of generated api
#  ydk.models.openconfig_interfaces.py which inturn is derived from the
#  open-config-interfaces yang module.
#

from ydk.types import Empty, DELETE, Decimal64
from ydk.services import CRUDService
import logging

from samples.session_mgr import establish_session, init_logging
from ydk.models.openconfig.openconfig_interfaces import Interfaces
from ydk.errors import YPYError

def print_interface(interface):
    print('*' * 28)
    print('Interface %s'%interface.name)

    if interface.config is not None:
        print ('  config')
        print('     name:-%s'% interface.config.name)
        if interface.config.type is not None:
            print('     type:-%s'%interface.config.type.__class__)
        print('    enabled:-%s'%interface.config.enabled)

    if interface.state is not None:
        print ('  state')
        print('     name:-%s'% interface.state.name)
        if interface.state.type is not None:
            print('     type:-%s'%interface.state.type.__class__)
        print('     enabled:-%s'%interface.state.enabled)
        if interface.state.admin_status is not None:
            enum_str = 'DOWN'
            if interface.state.admin_status == interface.state.AdminStatus_Enum.UP:
                enum_str = 'UP'
            print('     admin_status:-%s'%enum_str)
        if interface.state.oper_status is not None:
            oper_status_map = { interface.state.OperStatus_Enum.UP : 'UP',
                                interface.state.OperStatus_Enum.DOWN : 'DOWN',
                                interface.state.OperStatus_Enum.TESTING : 'TESTING',
                                interface.state.OperStatus_Enum.UNKNOWN: 'UNKNOWN',
                                interface.state.OperStatus_Enum.DORMANT : 'DORMANT',
                                interface.state.OperStatus_Enum.NOT_PRESENT : 'NOT_PRESENT',
                                }
            print('     oper_status:-%s'%oper_status_map[interface.state.oper_status])

        if interface.state.mtu is not None:
            print('      mtu:-%s'%interface.state.mtu)
        if interface.state.last_change is not None:
            print('      last_change:-%s'%interface.state.last_change)

        if interface.state.counters is not None:
            print('     counters')
            print('        in_unicast_pkts:-%s'%interface.state.counters.in_unicast_pkts)
            print('        in_octets:-%s'%interface.state.counters.in_octets)
            print('        out_unicast_pkts:-%s'%interface.state.counters.out_unicast_pkts)
            print('        out_octets:-%s'%interface.state.counters.out_octets)
            print('        in_multicast_pkts:-%s'%interface.state.counters.in_multicast_pkts)
            print('        in_broadcast_pkts:-%s'%interface.state.counters.in_broadcast_pkts)
            print('        out_multicast_pkts:-%s'%interface.state.counters.out_multicast_pkts)
            print('        out_broadcast_pkts:-%s'%interface.state.counters.out_broadcast_pkts)
            print('        out_discards:-%s'%interface.state.counters.out_discards)
            print('        in_discards:-%s'%interface.state.counters.in_discards)
            print('        in_unknown_protos:-%s'%interface.state.counters.in_unknown_protos)
            print('        in_errors:-%s'%interface.state.counters.in_errors)
            print('        out_errors:-%s'%interface.state.counters.out_errors)
            print('        last_clear:-%s'%interface.state.counters.last_clear)

        if interface.state.ifindex is not None:
            print('     ifindex:-%s'%interface.state.ifindex)

    print('*' * 28)

def read_interfaces(crud_service, provider):

    interfaces_filter = Interfaces()

    try:
        interfaces = crud_service.read(provider, interfaces_filter)
        for interface in interfaces.interface:
            print_interface(interface)
    except YPYError:
        print('An error occurred reading interfaces.')

def create_interfaces_config(crud_service, provider):

    interface = Interfaces.Interface()
    interface.config.name = 'LoopbackYDK'
    interface.name = interface.config.name

    try:
        crud_service.create(provider, interface)
    except YPYError:
        print('An error occurred creating the interface.')

if __name__ == "__main__":
    init_logging()
    provider = establish_session()
    crud_service = CRUDService()
    read_interfaces(crud_service, provider)

    provider.close()
    exit()
<commit_msg>Fix oc-if sample app (#246)<commit_after>#!/usr/bin/env python
#  ----------------------------------------------------------------
# Copyright 2016 Cisco Systems
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ------------------------------------------------------------------

#  ----------------------------------------------------------------
#  oc-interfaces.py Sample program illustrating use of generated api
#  ydk.models.openconfig_interfaces.py which inturn is derived from the
#  open-config-interfaces yang module.
#

from __future__ import print_function
from ydk.types import Empty, DELETE, Decimal64
from ydk.services import CRUDService
import logging

from session_mgr import establish_session, init_logging
from ydk.models.openconfig.openconfig_interfaces import Interfaces
from ydk.errors import YPYError

def print_interface(interface):
    print('*' * 28)
    print('Interface %s'%interface.name)

    if interface.config is not None:
        print ('  config')
        print('     name:-%s'% interface.config.name)
        if interface.config.type is not None:
            print('     type:-%s'%interface.config.type.__class__)
        print('    enabled:-%s'%interface.config.enabled)

    if interface.state is not None:
        print ('  state')
        print('     name:-%s'% interface.state.name)
        if interface.state.type is not None:
            print('     type:-%s'%interface.state.type.__class__)
        print('     enabled:-%s'%interface.state.enabled)
        if interface.state.admin_status is not None:
            enum_str = 'DOWN'
            if interface.state.admin_status == interface.state.AdminStatusEnum.UP:
                enum_str = 'UP'
            print('     admin_status:-%s'%enum_str)
        if interface.state.oper_status is not None:
            oper_status_map = { interface.state.OperStatusEnum.UP : 'UP',
                                interface.state.OperStatusEnum.DOWN : 'DOWN',
                                interface.state.OperStatusEnum.TESTING : 'TESTING',
                                interface.state.OperStatusEnum.UNKNOWN: 'UNKNOWN',
                                interface.state.OperStatusEnum.DORMANT : 'DORMANT',
                                interface.state.OperStatusEnum.NOT_PRESENT : 'NOT_PRESENT',
                                }
            print('     oper_status:-%s'%oper_status_map[interface.state.oper_status])

        if interface.state.mtu is not None:
            print('      mtu:-%s'%interface.state.mtu)
        if interface.state.last_change is not None:
            print('      last_change:-%s'%interface.state.last_change)

        if interface.state.counters is not None:
            print('     counters')
            print('        in_unicast_pkts:-%s'%interface.state.counters.in_unicast_pkts)
            print('        in_octets:-%s'%interface.state.counters.in_octets)
            print('        out_unicast_pkts:-%s'%interface.state.counters.out_unicast_pkts)
            print('        out_octets:-%s'%interface.state.counters.out_octets)
            print('        in_multicast_pkts:-%s'%interface.state.counters.in_multicast_pkts)
            print('        in_broadcast_pkts:-%s'%interface.state.counters.in_broadcast_pkts)
            print('        out_multicast_pkts:-%s'%interface.state.counters.out_multicast_pkts)
            print('        out_broadcast_pkts:-%s'%interface.state.counters.out_broadcast_pkts)
            print('        out_discards:-%s'%interface.state.counters.out_discards)
            print('        in_discards:-%s'%interface.state.counters.in_discards)
            print('        in_unknown_protos:-%s'%interface.state.counters.in_unknown_protos)
            print('        in_errors:-%s'%interface.state.counters.in_errors)
            print('        out_errors:-%s'%interface.state.counters.out_errors)
            print('        last_clear:-%s'%interface.state.counters.last_clear)

        if interface.state.ifindex is not None:
            print('     ifindex:-%s'%interface.state.ifindex)

    print('*' * 28)

def read_interfaces(crud_service, provider):

    interfaces_filter = Interfaces()

    try:
        interfaces = crud_service.read(provider, interfaces_filter)
        for interface in interfaces.interface:
            print_interface(interface)
    except YPYError:
        print('An error occurred reading interfaces.')

def create_interfaces_config(crud_service, provider):

    interface = Interfaces.Interface()
    interface.config.name = 'LoopbackYDK'
    interface.name = interface.config.name

    try:
        crud_service.create(provider, interface)
    except YPYError:
        print('An error occurred creating the interface.')

if __name__ == "__main__":
    init_logging()
    provider = establish_session()
    crud_service = CRUDService()
    read_interfaces(crud_service, provider)

    provider.close()
    exit()
<|endoftext|>


<commit_before># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import OrderedDict
from ._structures import ConcatPlaceholder, IfPlaceholder, InputValuePlaceholder, InputPathPlaceholder, IsPresentPlaceholder, OutputPathPlaceholder, TaskSpec
from ._components import _generate_output_file_name, _default_component_name

def create_container_op_from_task(task_spec: TaskSpec):
    argument_values = task_spec.arguments
    component_spec = task_spec.component_ref._component_spec

    inputs_dict = {input_spec.name: input_spec for input_spec in component_spec.inputs or []}
    container_spec = component_spec.implementation.container

    output_paths = OrderedDict() #Preserving the order to make the kubernetes output names deterministic
    unconfigurable_output_paths = container_spec.file_outputs or {}
    for output in component_spec.outputs or []:
        if output.name in unconfigurable_output_paths:
            output_paths[output.name] = unconfigurable_output_paths[output.name]

    def expand_command_part(arg): #input values with original names
        #(Union[str,Mapping[str, Any]]) -> Union[str,List[str]]
        if arg is None:
            return None
        if isinstance(arg, (str, int, float, bool)):
            return str(arg)

        if isinstance(arg, InputValuePlaceholder):
            input_name = arg.input_name
            input_value = argument_values.get(input_name, None)
            if input_value is not None:
                return str(input_value)
            else:
                input_spec = inputs_dict[input_name]
                if input_spec.optional:
                    return None
                else:
                    raise ValueError('No value provided for input {}'.format(input_name))

        if isinstance(arg, InputPathPlaceholder):
            input_name = arg.input_name
            input_value = argument_values.get(input_name, None)
            if input_value is not None:
                raise ValueError('ContainerOp does not support input artifacts - input {}'.format(input_name))
                #return input_value
            else:
                input_spec = inputs_dict[input_name]
                if input_spec.optional:
                    #Even when we support default values there is no need to check for a default here.
                    #In current execution flow (called by python task factory), the missing argument would be replaced with the default value by python itself.
                    return None
                else:
                    raise ValueError('No value provided for input {}'.format(input_name))

        elif isinstance(arg, OutputPathPlaceholder):
            output_name = arg.output_name
            output_filename = _generate_output_file_name(output_name)
            if arg.output_name in output_paths:
                if output_paths[output_name] != output_filename:
                    raise ValueError('Conflicting output files specified for port {}: {} and {}'.format(output_name, output_paths[output_name], output_filename))
            else:
                output_paths[output_name] = output_filename

            return output_filename

        elif isinstance(arg, ConcatPlaceholder):
            expanded_argument_strings = expand_argument_list(arg.items)
            return ''.join(expanded_argument_strings)

        elif isinstance(arg, IfPlaceholder):
            arg = arg.if_structure
            condition_result = expand_command_part(arg.condition)
            from distutils.util import strtobool
            condition_result_bool = condition_result and strtobool(condition_result) #Python gotcha: bool('False') == True; Need to use strtobool; Also need to handle None and []
            result_node = arg.then_value if condition_result_bool else arg.else_value
            if result_node is None:
                return []
            if isinstance(result_node, list):
                expanded_result = expand_argument_list(result_node)
            else:
                expanded_result = expand_command_part(result_node)
            return expanded_result

        elif isinstance(arg, IsPresentPlaceholder):
            argument_is_present = argument_values.get(arg.input_name, None) is not None
            return str(argument_is_present)
        else:
            raise TypeError('Unrecognized argument type: {}'.format(arg))

    def expand_argument_list(argument_list):
        expanded_list = []
        if argument_list is not None:
            for part in argument_list:
                expanded_part = expand_command_part(part)
                if expanded_part is not None:
                    if isinstance(expanded_part, list):
                        expanded_list.extend(expanded_part)
                    else:
                        expanded_list.append(str(expanded_part))
        return expanded_list

    expanded_command = expand_argument_list(container_spec.command)
    expanded_args = expand_argument_list(container_spec.args)

    return _task_object_factory(
        name=component_spec.name or _default_component_name,
        container_image=container_spec.image,
        command=expanded_command,
        arguments=expanded_args,
        output_paths=output_paths,
    )

_dummy_pipeline=None

def _create_container_op_from_resolved_task(name:str, container_image:str, command=None, arguments=None, output_paths=None):
    from .. import dsl
    global _dummy_pipeline
    need_dummy = dsl.Pipeline._default_pipeline is None
    if need_dummy:
        if _dummy_pipeline == None:
            _dummy_pipeline = dsl.Pipeline('dummy pipeline')
        _dummy_pipeline.__enter__()

    from ._naming import _sanitize_kubernetes_resource_name, _make_name_unique_by_adding_index
    output_name_to_kubernetes = {}
    kubernetes_name_to_output_name = {}
    for output_name in (output_paths or {}).keys():
        kubernetes_name = _sanitize_kubernetes_resource_name(output_name)
        kubernetes_name = _make_name_unique_by_adding_index(kubernetes_name, kubernetes_name_to_output_name, '-')
        output_name_to_kubernetes[output_name] = kubernetes_name
        kubernetes_name_to_output_name[kubernetes_name] = output_name

    output_paths_for_container_op = {output_name_to_kubernetes[name]: path for name, path in output_paths.items()}

    task = dsl.ContainerOp(
        name=name,
        image=container_image,
        command=command,
        arguments=arguments,
        file_outputs=output_paths_for_container_op,
    )

    if need_dummy:
        _dummy_pipeline.__exit__()

    return task

_task_object_factory=_create_container_op_from_resolved_task
<commit_msg>SDK/Components - More meaningful error when trying to convert graph component to ContainerOp (#710)<commit_after># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import OrderedDict
from ._structures import ConcatPlaceholder, IfPlaceholder, InputValuePlaceholder, InputPathPlaceholder, IsPresentPlaceholder, OutputPathPlaceholder, TaskSpec
from ._components import _generate_output_file_name, _default_component_name

def create_container_op_from_task(task_spec: TaskSpec):
    argument_values = task_spec.arguments
    component_spec = task_spec.component_ref._component_spec

    if hasattr(component_spec.implementation, 'graph'):
        raise TypeError('Cannot convert graph component to ContainerOp')

    inputs_dict = {input_spec.name: input_spec for input_spec in component_spec.inputs or []}
    container_spec = component_spec.implementation.container

    output_paths = OrderedDict() #Preserving the order to make the kubernetes output names deterministic
    unconfigurable_output_paths = container_spec.file_outputs or {}
    for output in component_spec.outputs or []:
        if output.name in unconfigurable_output_paths:
            output_paths[output.name] = unconfigurable_output_paths[output.name]

    def expand_command_part(arg): #input values with original names
        #(Union[str,Mapping[str, Any]]) -> Union[str,List[str]]
        if arg is None:
            return None
        if isinstance(arg, (str, int, float, bool)):
            return str(arg)

        if isinstance(arg, InputValuePlaceholder):
            input_name = arg.input_name
            input_value = argument_values.get(input_name, None)
            if input_value is not None:
                return str(input_value)
            else:
                input_spec = inputs_dict[input_name]
                if input_spec.optional:
                    return None
                else:
                    raise ValueError('No value provided for input {}'.format(input_name))

        if isinstance(arg, InputPathPlaceholder):
            input_name = arg.input_name
            input_value = argument_values.get(input_name, None)
            if input_value is not None:
                raise ValueError('ContainerOp does not support input artifacts - input {}'.format(input_name))
                #return input_value
            else:
                input_spec = inputs_dict[input_name]
                if input_spec.optional:
                    #Even when we support default values there is no need to check for a default here.
                    #In current execution flow (called by python task factory), the missing argument would be replaced with the default value by python itself.
                    return None
                else:
                    raise ValueError('No value provided for input {}'.format(input_name))

        elif isinstance(arg, OutputPathPlaceholder):
            output_name = arg.output_name
            output_filename = _generate_output_file_name(output_name)
            if arg.output_name in output_paths:
                if output_paths[output_name] != output_filename:
                    raise ValueError('Conflicting output files specified for port {}: {} and {}'.format(output_name, output_paths[output_name], output_filename))
            else:
                output_paths[output_name] = output_filename

            return output_filename

        elif isinstance(arg, ConcatPlaceholder):
            expanded_argument_strings = expand_argument_list(arg.items)
            return ''.join(expanded_argument_strings)

        elif isinstance(arg, IfPlaceholder):
            arg = arg.if_structure
            condition_result = expand_command_part(arg.condition)
            from distutils.util import strtobool
            condition_result_bool = condition_result and strtobool(condition_result) #Python gotcha: bool('False') == True; Need to use strtobool; Also need to handle None and []
            result_node = arg.then_value if condition_result_bool else arg.else_value
            if result_node is None:
                return []
            if isinstance(result_node, list):
                expanded_result = expand_argument_list(result_node)
            else:
                expanded_result = expand_command_part(result_node)
            return expanded_result

        elif isinstance(arg, IsPresentPlaceholder):
            argument_is_present = argument_values.get(arg.input_name, None) is not None
            return str(argument_is_present)
        else:
            raise TypeError('Unrecognized argument type: {}'.format(arg))

    def expand_argument_list(argument_list):
        expanded_list = []
        if argument_list is not None:
            for part in argument_list:
                expanded_part = expand_command_part(part)
                if expanded_part is not None:
                    if isinstance(expanded_part, list):
                        expanded_list.extend(expanded_part)
                    else:
                        expanded_list.append(str(expanded_part))
        return expanded_list

    expanded_command = expand_argument_list(container_spec.command)
    expanded_args = expand_argument_list(container_spec.args)

    return _task_object_factory(
        name=component_spec.name or _default_component_name,
        container_image=container_spec.image,
        command=expanded_command,
        arguments=expanded_args,
        output_paths=output_paths,
    )

_dummy_pipeline=None

def _create_container_op_from_resolved_task(name:str, container_image:str, command=None, arguments=None, output_paths=None):
    from .. import dsl
    global _dummy_pipeline
    need_dummy = dsl.Pipeline._default_pipeline is None
    if need_dummy:
        if _dummy_pipeline == None:
            _dummy_pipeline = dsl.Pipeline('dummy pipeline')
        _dummy_pipeline.__enter__()

    from ._naming import _sanitize_kubernetes_resource_name, _make_name_unique_by_adding_index
    output_name_to_kubernetes = {}
    kubernetes_name_to_output_name = {}
    for output_name in (output_paths or {}).keys():
        kubernetes_name = _sanitize_kubernetes_resource_name(output_name)
        kubernetes_name = _make_name_unique_by_adding_index(kubernetes_name, kubernetes_name_to_output_name, '-')
        output_name_to_kubernetes[output_name] = kubernetes_name
        kubernetes_name_to_output_name[kubernetes_name] = output_name

    output_paths_for_container_op = {output_name_to_kubernetes[name]: path for name, path in output_paths.items()}

    task = dsl.ContainerOp(
        name=name,
        image=container_image,
        command=command,
        arguments=arguments,
        file_outputs=output_paths_for_container_op,
    )

    if need_dummy:
        _dummy_pipeline.__exit__()

    return task

_task_object_factory=_create_container_op_from_resolved_task
<|endoftext|>


<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Unit tests for the Pipeline class."""

import logging
import platform
import unittest

from apache_beam.pipeline import Pipeline
from apache_beam.pipeline import PipelineOptions
from apache_beam.pipeline import PipelineVisitor
from apache_beam.pvalue import AsSingleton
from apache_beam.runners.dataflow.native_io.iobase import NativeSource
from apache_beam.test_pipeline import TestPipeline
from apache_beam.transforms import CombineGlobally
from apache_beam.transforms import Create
from apache_beam.transforms import FlatMap
from apache_beam.transforms import Map
from apache_beam.transforms import NewDoFn
from apache_beam.transforms import ParDo
from apache_beam.transforms import PTransform
from apache_beam.transforms import Read
from apache_beam.transforms import WindowInto
from apache_beam.transforms.util import assert_that
from apache_beam.transforms.util import equal_to
from apache_beam.transforms.window import IntervalWindow
from apache_beam.transforms.window import WindowFn
from apache_beam.utils.timestamp import MIN_TIMESTAMP

class FakeSource(NativeSource):
  """Fake source returning a fixed list of values."""

  class _Reader(object):

    def __init__(self, vals):
      self._vals = vals

    def __enter__(self):
      return self

    def __exit__(self, exception_type, exception_value, traceback):
      pass

    def __iter__(self):
      for v in self._vals:
        yield v

  def __init__(self, vals):
    self._vals = vals

  def reader(self):
    return FakeSource._Reader(self._vals)

class PipelineTest(unittest.TestCase):

  def setUp(self):
    self.runner_name = 'DirectRunner'

  @staticmethod
  def custom_callable(pcoll):
    return pcoll | '+1' >> FlatMap(lambda x: [x + 1])

  # Some of these tests designate a runner by name, others supply a runner.
  # This variation is just to verify that both means of runner specification
  # work and is not related to other aspects of the tests.

  class CustomTransform(PTransform):

    def expand(self, pcoll):
      return pcoll | '+1' >> FlatMap(lambda x: [x + 1])

  class Visitor(PipelineVisitor):

    def __init__(self, visited):
      self.visited = visited
      self.enter_composite = []
      self.leave_composite = []

    def visit_value(self, value, _):
      self.visited.append(value)

    def enter_composite_transform(self, transform_node):
      self.enter_composite.append(transform_node)

    def leave_composite_transform(self, transform_node):
      self.leave_composite.append(transform_node)

  def test_create(self):
    pipeline = TestPipeline(runner=self.runner_name)
    pcoll = pipeline | 'label1' >> Create([1, 2, 3])
    assert_that(pcoll, equal_to([1, 2, 3]))

    # Test if initial value is an iterator object.
    pcoll2 = pipeline | 'label2' >> Create(iter((4, 5, 6)))
    pcoll3 = pcoll2 | 'do' >> FlatMap(lambda x: [x + 10])
    assert_that(pcoll3, equal_to([14, 15, 16]), label='pcoll3')
    pipeline.run()

  def test_create_singleton_pcollection(self):
    pipeline = TestPipeline(runner=self.runner_name)
    pcoll = pipeline | 'label' >> Create([[1, 2, 3]])
    assert_that(pcoll, equal_to([[1, 2, 3]]))
    pipeline.run()

  def test_read(self):
    pipeline = TestPipeline(runner=self.runner_name)
    pcoll = pipeline | 'read' >> Read(FakeSource([1, 2, 3]))
    assert_that(pcoll, equal_to([1, 2, 3]))
    pipeline.run()

  def test_visit_entire_graph(self):
    pipeline = Pipeline(self.runner_name)
    pcoll1 = pipeline | 'pcoll' >> Create([1, 2, 3])
    pcoll2 = pcoll1 | 'do1' >> FlatMap(lambda x: [x + 1])
    pcoll3 = pcoll2 | 'do2' >> FlatMap(lambda x: [x + 1])
    pcoll4 = pcoll2 | 'do3' >> FlatMap(lambda x: [x + 1])
    transform = PipelineTest.CustomTransform()
    pcoll5 = pcoll4 | transform

    visitor = PipelineTest.Visitor(visited=[])
    pipeline.visit(visitor)
    self.assertEqual(set([pcoll1, pcoll2, pcoll3, pcoll4, pcoll5]),
                     set(visitor.visited))
    self.assertEqual(set(visitor.enter_composite),
                     set(visitor.leave_composite))
    self.assertEqual(2, len(visitor.enter_composite))
    self.assertEqual(visitor.enter_composite[1].transform, transform)
    self.assertEqual(visitor.leave_composite[0].transform, transform)

  def test_apply_custom_transform(self):
    pipeline = TestPipeline(runner=self.runner_name)
    pcoll = pipeline | 'pcoll' >> Create([1, 2, 3])
    result = pcoll | PipelineTest.CustomTransform()
    assert_that(result, equal_to([2, 3, 4]))
    pipeline.run()

  def test_reuse_custom_transform_instance(self):
    pipeline = Pipeline(self.runner_name)
    pcoll1 = pipeline | 'pcoll1' >> Create([1, 2, 3])
    pcoll2 = pipeline | 'pcoll2' >> Create([4, 5, 6])
    transform = PipelineTest.CustomTransform()
    pcoll1 | transform
    with self.assertRaises(RuntimeError) as cm:
      pipeline.apply(transform, pcoll2)
    self.assertEqual(
        cm.exception.message,
        'Transform "CustomTransform" does not have a stable unique label. '
        'This will prevent updating of pipelines. '
        'To apply a transform with a specified label write '
        'pvalue | "label" >> transform')

  def test_reuse_cloned_custom_transform_instance(self):
    pipeline = TestPipeline(runner=self.runner_name)
    pcoll1 = pipeline | 'pc1' >> Create([1, 2, 3])
    pcoll2 = pipeline | 'pc2' >> Create([4, 5, 6])
    transform = PipelineTest.CustomTransform()
    result1 = pcoll1 | transform
    result2 = pcoll2 | 'new_label' >> transform
    assert_that(result1, equal_to([2, 3, 4]), label='r1')
    assert_that(result2, equal_to([5, 6, 7]), label='r2')
    pipeline.run()

  def test_transform_no_super_init(self):
    class AddSuffix(PTransform):

      def __init__(self, suffix):
        # No call to super(...).__init__
        self.suffix = suffix

      def expand(self, pcoll):
        return pcoll | Map(lambda x: x + self.suffix)

    self.assertEqual(
        ['a-x', 'b-x', 'c-x'],
        sorted(['a', 'b', 'c'] | 'AddSuffix' >> AddSuffix('-x')))

  def test_memory_usage(self):
    try:
      import resource
    except ImportError:
      # Skip the test if resource module is not available (e.g. non-Unix os).
      self.skipTest('resource module not available.')
    if platform.mac_ver()[0]:
      # Skip the test on macos, depending on version it returns ru_maxrss in
      # different units.
      self.skipTest('ru_maxrss is not in standard units.')

    def get_memory_usage_in_bytes():
      return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * (2 ** 10)

    def check_memory(value, memory_threshold):
      memory_usage = get_memory_usage_in_bytes()
      if memory_usage > memory_threshold:
        raise RuntimeError(
            'High memory usage: %d > %d' % (memory_usage, memory_threshold))
      return value

    len_elements = 1000000
    num_elements = 10
    num_maps = 100

    pipeline = TestPipeline(runner='DirectRunner')

    # Consumed memory should not be proportional to the number of maps.
    memory_threshold = (
        get_memory_usage_in_bytes() + (3 * len_elements * num_elements))

    biglist = pipeline | 'oom:create' >> Create(
        ['x' * len_elements] * num_elements)
    for i in range(num_maps):
      biglist = biglist | ('oom:addone-%d' % i) >> Map(lambda x: x + 'y')
    result = biglist | 'oom:check' >> Map(check_memory, memory_threshold)
    assert_that(result, equal_to(
        ['x' * len_elements + 'y' * num_maps] * num_elements))

    pipeline.run()

  def test_aggregator_empty_input(self):
    actual = [] | CombineGlobally(max).without_defaults()
    self.assertEqual(actual, [])

  def test_pipeline_as_context(self):
    def raise_exception(exn):
      raise exn
    with self.assertRaises(ValueError):
      with Pipeline(self.runner_name) as p:
        # pylint: disable=expression-not-assigned
        p | Create([ValueError]) | Map(raise_exception)

  def test_eager_pipeline(self):
    p = Pipeline('EagerRunner')
    self.assertEqual([1, 4, 9], p | Create([1, 2, 3]) | Map(lambda x: x*x))

class NewDoFnTest(unittest.TestCase):

  def setUp(self):
    self.runner_name = 'DirectRunner'

  def test_element(self):
    class TestDoFn(NewDoFn):
      def process(self, element):
        yield element + 10

    pipeline = TestPipeline(runner=self.runner_name)
    pcoll = pipeline | 'Create' >> Create([1, 2]) | 'Do' >> ParDo(TestDoFn())
    assert_that(pcoll, equal_to([11, 12]))
    pipeline.run()

  def test_context_param(self):
    class TestDoFn(NewDoFn):
      def process(self, element, context=NewDoFn.ContextParam):
        yield context.element + 10

    pipeline = TestPipeline(runner=self.runner_name)
    pcoll = pipeline | 'Create' >> Create([1, 2])| 'Do' >> ParDo(TestDoFn())
    assert_that(pcoll, equal_to([11, 12]))
    pipeline.run()

  def test_side_input_no_tag(self):
    class TestDoFn(NewDoFn):
      def process(self, element, prefix, suffix):
        return ['%s-%s-%s' % (prefix, element, suffix)]

    pipeline = TestPipeline()
    words_list = ['aa', 'bb', 'cc']
    words = pipeline | 'SomeWords' >> Create(words_list)
    prefix = 'zyx'
    suffix = pipeline | 'SomeString' >> Create(['xyz'])  # side in
    result = words | 'DecorateWordsDoFnNoTag' >> ParDo(
        TestDoFn(), prefix, suffix=AsSingleton(suffix))
    assert_that(result, equal_to(['zyx-%s-xyz' % x for x in words_list]))
    pipeline.run()

  def test_side_input_tagged(self):
    class TestDoFn(NewDoFn):
      def process(self, element, prefix, suffix=NewDoFn.SideInputParam):
        return ['%s-%s-%s' % (prefix, element, suffix)]

    pipeline = TestPipeline()
    words_list = ['aa', 'bb', 'cc']
    words = pipeline | 'SomeWords' >> Create(words_list)
    prefix = 'zyx'
    suffix = pipeline | 'SomeString' >> Create(['xyz'])  # side in
    result = words | 'DecorateWordsDoFnNoTag' >> ParDo(
        TestDoFn(), prefix, suffix=AsSingleton(suffix))
    assert_that(result, equal_to(['zyx-%s-xyz' % x for x in words_list]))
    pipeline.run()

  def test_window_param(self):
    class TestDoFn(NewDoFn):
      def process(self, element, window=NewDoFn.WindowParam):
        yield (float(window.start), float(window.end))

    class TestWindowFn(WindowFn):
      """Windowing function adding two disjoint windows to each element."""

      def assign(self, assign_context):
        _ = assign_context
        return [IntervalWindow(10, 20), IntervalWindow(20, 30)]

      def merge(self, existing_windows):
        return existing_windows

    pipeline = TestPipeline(runner=self.runner_name)
    pcoll = (pipeline
             | 'KVs' >> Create([(1, 10), (2, 20)])
             | 'W' >> WindowInto(windowfn=TestWindowFn())
             | 'Do' >> ParDo(TestDoFn()))
    assert_that(pcoll, equal_to([(10.0, 20.0), (10.0, 20.0),
                                 (20.0, 30.0), (20.0, 30.0)]))
    pipeline.run()

  def test_timestamp_param(self):
    class TestDoFn(NewDoFn):
      def process(self, element, timestamp=NewDoFn.TimestampParam):
        yield timestamp

    pipeline = TestPipeline(runner=self.runner_name)
    pcoll = pipeline | 'Create' >> Create([1, 2]) | 'Do' >> ParDo(TestDoFn())
    assert_that(pcoll, equal_to([MIN_TIMESTAMP, MIN_TIMESTAMP]))
    pipeline.run()

class Bacon(PipelineOptions):

  @classmethod
  def _add_argparse_args(cls, parser):
    parser.add_argument('--slices', type=int)

class Eggs(PipelineOptions):

  @classmethod
  def _add_argparse_args(cls, parser):
    parser.add_argument('--style', default='scrambled')

class Breakfast(Bacon, Eggs):
  pass

class PipelineOptionsTest(unittest.TestCase):

  def test_flag_parsing(self):
    options = Breakfast(['--slices=3', '--style=sunny side up', '--ignored'])
    self.assertEquals(3, options.slices)
    self.assertEquals('sunny side up', options.style)

  def test_keyword_parsing(self):
    options = Breakfast(
        ['--slices=3', '--style=sunny side up', '--ignored'],
        slices=10)
    self.assertEquals(10, options.slices)
    self.assertEquals('sunny side up', options.style)

  def test_attribute_setting(self):
    options = Breakfast(slices=10)
    self.assertEquals(10, options.slices)
    options.slices = 20
    self.assertEquals(20, options.slices)

  def test_view_as(self):
    generic_options = PipelineOptions(['--slices=3'])
    self.assertEquals(3, generic_options.view_as(Bacon).slices)
    self.assertEquals(3, generic_options.view_as(Breakfast).slices)

    generic_options.view_as(Breakfast).slices = 10
    self.assertEquals(10, generic_options.view_as(Bacon).slices)

    with self.assertRaises(AttributeError):
      generic_options.slices  # pylint: disable=pointless-statement

    with self.assertRaises(AttributeError):
      generic_options.view_as(Eggs).slices  # pylint: disable=expression-not-assigned

  def test_defaults(self):
    options = Breakfast(['--slices=3'])
    self.assertEquals(3, options.slices)
    self.assertEquals('scrambled', options.style)

  def test_dir(self):
    options = Breakfast()
    self.assertEquals(
        set(['from_dictionary', 'get_all_options', 'slices', 'style',
             'view_as', 'display_data']),
        set([attr for attr in dir(options) if not attr.startswith('_')]))
    self.assertEquals(
        set(['from_dictionary', 'get_all_options', 'style', 'view_as',
             'display_data']),
        set([attr for attr in dir(options.view_as(Eggs))
             if not attr.startswith('_')]))

if __name__ == '__main__':
  logging.getLogger().setLevel(logging.DEBUG)
  unittest.main()
<commit_msg>Cleanup tests in pipeline_test.<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Unit tests for the Pipeline class."""

import logging
import platform
import unittest

from apache_beam.pipeline import Pipeline
from apache_beam.pipeline import PipelineOptions
from apache_beam.pipeline import PipelineVisitor
from apache_beam.pvalue import AsSingleton
from apache_beam.runners.dataflow.native_io.iobase import NativeSource
from apache_beam.test_pipeline import TestPipeline
from apache_beam.transforms import CombineGlobally
from apache_beam.transforms import Create
from apache_beam.transforms import FlatMap
from apache_beam.transforms import Map
from apache_beam.transforms import NewDoFn
from apache_beam.transforms import ParDo
from apache_beam.transforms import PTransform
from apache_beam.transforms import Read
from apache_beam.transforms import WindowInto
from apache_beam.transforms.util import assert_that
from apache_beam.transforms.util import equal_to
from apache_beam.transforms.window import SlidingWindows
from apache_beam.transforms.window import TimestampedValue
from apache_beam.utils.timestamp import MIN_TIMESTAMP

class FakeSource(NativeSource):
  """Fake source returning a fixed list of values."""

  class _Reader(object):

    def __init__(self, vals):
      self._vals = vals

    def __enter__(self):
      return self

    def __exit__(self, exception_type, exception_value, traceback):
      pass

    def __iter__(self):
      for v in self._vals:
        yield v

  def __init__(self, vals):
    self._vals = vals

  def reader(self):
    return FakeSource._Reader(self._vals)

class PipelineTest(unittest.TestCase):

  @staticmethod
  def custom_callable(pcoll):
    return pcoll | '+1' >> FlatMap(lambda x: [x + 1])

  # Some of these tests designate a runner by name, others supply a runner.
  # This variation is just to verify that both means of runner specification
  # work and is not related to other aspects of the tests.

  class CustomTransform(PTransform):

    def expand(self, pcoll):
      return pcoll | '+1' >> FlatMap(lambda x: [x + 1])

  class Visitor(PipelineVisitor):

    def __init__(self, visited):
      self.visited = visited
      self.enter_composite = []
      self.leave_composite = []

    def visit_value(self, value, _):
      self.visited.append(value)

    def enter_composite_transform(self, transform_node):
      self.enter_composite.append(transform_node)

    def leave_composite_transform(self, transform_node):
      self.leave_composite.append(transform_node)

  def test_create(self):
    pipeline = TestPipeline()
    pcoll = pipeline | 'label1' >> Create([1, 2, 3])
    assert_that(pcoll, equal_to([1, 2, 3]))

    # Test if initial value is an iterator object.
    pcoll2 = pipeline | 'label2' >> Create(iter((4, 5, 6)))
    pcoll3 = pcoll2 | 'do' >> FlatMap(lambda x: [x + 10])
    assert_that(pcoll3, equal_to([14, 15, 16]), label='pcoll3')
    pipeline.run()

  def test_create_singleton_pcollection(self):
    pipeline = TestPipeline()
    pcoll = pipeline | 'label' >> Create([[1, 2, 3]])
    assert_that(pcoll, equal_to([[1, 2, 3]]))
    pipeline.run()

  def test_read(self):
    pipeline = TestPipeline()
    pcoll = pipeline | 'read' >> Read(FakeSource([1, 2, 3]))
    assert_that(pcoll, equal_to([1, 2, 3]))
    pipeline.run()

  def test_visit_entire_graph(self):
    pipeline = Pipeline()
    pcoll1 = pipeline | 'pcoll' >> Create([1, 2, 3])
    pcoll2 = pcoll1 | 'do1' >> FlatMap(lambda x: [x + 1])
    pcoll3 = pcoll2 | 'do2' >> FlatMap(lambda x: [x + 1])
    pcoll4 = pcoll2 | 'do3' >> FlatMap(lambda x: [x + 1])
    transform = PipelineTest.CustomTransform()
    pcoll5 = pcoll4 | transform

    visitor = PipelineTest.Visitor(visited=[])
    pipeline.visit(visitor)
    self.assertEqual(set([pcoll1, pcoll2, pcoll3, pcoll4, pcoll5]),
                     set(visitor.visited))
    self.assertEqual(set(visitor.enter_composite),
                     set(visitor.leave_composite))
    self.assertEqual(2, len(visitor.enter_composite))
    self.assertEqual(visitor.enter_composite[1].transform, transform)
    self.assertEqual(visitor.leave_composite[0].transform, transform)

  def test_apply_custom_transform(self):
    pipeline = TestPipeline()
    pcoll = pipeline | 'pcoll' >> Create([1, 2, 3])
    result = pcoll | PipelineTest.CustomTransform()
    assert_that(result, equal_to([2, 3, 4]))
    pipeline.run()

  def test_reuse_custom_transform_instance(self):
    pipeline = Pipeline()
    pcoll1 = pipeline | 'pcoll1' >> Create([1, 2, 3])
    pcoll2 = pipeline | 'pcoll2' >> Create([4, 5, 6])
    transform = PipelineTest.CustomTransform()
    pcoll1 | transform
    with self.assertRaises(RuntimeError) as cm:
      pipeline.apply(transform, pcoll2)
    self.assertEqual(
        cm.exception.message,
        'Transform "CustomTransform" does not have a stable unique label. '
        'This will prevent updating of pipelines. '
        'To apply a transform with a specified label write '
        'pvalue | "label" >> transform')

  def test_reuse_cloned_custom_transform_instance(self):
    pipeline = TestPipeline()
    pcoll1 = pipeline | 'pc1' >> Create([1, 2, 3])
    pcoll2 = pipeline | 'pc2' >> Create([4, 5, 6])
    transform = PipelineTest.CustomTransform()
    result1 = pcoll1 | transform
    result2 = pcoll2 | 'new_label' >> transform
    assert_that(result1, equal_to([2, 3, 4]), label='r1')
    assert_that(result2, equal_to([5, 6, 7]), label='r2')
    pipeline.run()

  def test_transform_no_super_init(self):
    class AddSuffix(PTransform):

      def __init__(self, suffix):
        # No call to super(...).__init__
        self.suffix = suffix

      def expand(self, pcoll):
        return pcoll | Map(lambda x: x + self.suffix)

    self.assertEqual(
        ['a-x', 'b-x', 'c-x'],
        sorted(['a', 'b', 'c'] | 'AddSuffix' >> AddSuffix('-x')))

  def test_memory_usage(self):
    try:
      import resource
    except ImportError:
      # Skip the test if resource module is not available (e.g. non-Unix os).
      self.skipTest('resource module not available.')
    if platform.mac_ver()[0]:
      # Skip the test on macos, depending on version it returns ru_maxrss in
      # different units.
      self.skipTest('ru_maxrss is not in standard units.')

    def get_memory_usage_in_bytes():
      return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * (2 ** 10)

    def check_memory(value, memory_threshold):
      memory_usage = get_memory_usage_in_bytes()
      if memory_usage > memory_threshold:
        raise RuntimeError(
            'High memory usage: %d > %d' % (memory_usage, memory_threshold))
      return value

    len_elements = 1000000
    num_elements = 10
    num_maps = 100

    pipeline = TestPipeline(runner='DirectRunner')

    # Consumed memory should not be proportional to the number of maps.
    memory_threshold = (
        get_memory_usage_in_bytes() + (3 * len_elements * num_elements))

    biglist = pipeline | 'oom:create' >> Create(
        ['x' * len_elements] * num_elements)
    for i in range(num_maps):
      biglist = biglist | ('oom:addone-%d' % i) >> Map(lambda x: x + 'y')
    result = biglist | 'oom:check' >> Map(check_memory, memory_threshold)
    assert_that(result, equal_to(
        ['x' * len_elements + 'y' * num_maps] * num_elements))

    pipeline.run()

  def test_aggregator_empty_input(self):
    actual = [] | CombineGlobally(max).without_defaults()
    self.assertEqual(actual, [])

  def test_pipeline_as_context(self):
    def raise_exception(exn):
      raise exn
    with self.assertRaises(ValueError):
      with Pipeline() as p:
        # pylint: disable=expression-not-assigned
        p | Create([ValueError]) | Map(raise_exception)

  def test_eager_pipeline(self):
    p = Pipeline('EagerRunner')
    self.assertEqual([1, 4, 9], p | Create([1, 2, 3]) | Map(lambda x: x*x))

class NewDoFnTest(unittest.TestCase):

  def test_element(self):
    class TestDoFn(NewDoFn):
      def process(self, element):
        yield element + 10

    pipeline = TestPipeline()
    pcoll = pipeline | 'Create' >> Create([1, 2]) | 'Do' >> ParDo(TestDoFn())
    assert_that(pcoll, equal_to([11, 12]))
    pipeline.run()

  def test_context_param(self):
    class TestDoFn(NewDoFn):
      def process(self, element, context=NewDoFn.ContextParam):
        yield context.element + 10

    pipeline = TestPipeline()
    pcoll = pipeline | 'Create' >> Create([1, 2])| 'Do' >> ParDo(TestDoFn())
    assert_that(pcoll, equal_to([11, 12]))
    pipeline.run()

  def test_side_input_no_tag(self):
    class TestDoFn(NewDoFn):
      def process(self, element, prefix, suffix):
        return ['%s-%s-%s' % (prefix, element, suffix)]

    pipeline = TestPipeline()
    words_list = ['aa', 'bb', 'cc']
    words = pipeline | 'SomeWords' >> Create(words_list)
    prefix = 'zyx'
    suffix = pipeline | 'SomeString' >> Create(['xyz'])  # side in
    result = words | 'DecorateWordsDoFnNoTag' >> ParDo(
        TestDoFn(), prefix, suffix=AsSingleton(suffix))
    assert_that(result, equal_to(['zyx-%s-xyz' % x for x in words_list]))
    pipeline.run()

  def test_side_input_tagged(self):
    class TestDoFn(NewDoFn):
      def process(self, element, prefix, suffix=NewDoFn.SideInputParam):
        return ['%s-%s-%s' % (prefix, element, suffix)]

    pipeline = TestPipeline()
    words_list = ['aa', 'bb', 'cc']
    words = pipeline | 'SomeWords' >> Create(words_list)
    prefix = 'zyx'
    suffix = pipeline | 'SomeString' >> Create(['xyz'])  # side in
    result = words | 'DecorateWordsDoFnNoTag' >> ParDo(
        TestDoFn(), prefix, suffix=AsSingleton(suffix))
    assert_that(result, equal_to(['zyx-%s-xyz' % x for x in words_list]))
    pipeline.run()

  def test_window_param(self):
    class TestDoFn(NewDoFn):
      def process(self, element, window=NewDoFn.WindowParam):
        yield (element, (float(window.start), float(window.end)))

    pipeline = TestPipeline()
    pcoll = (pipeline
             | Create([1, 7])
             | Map(lambda x: TimestampedValue(x, x))
             | WindowInto(windowfn=SlidingWindows(10, 5))
             | ParDo(TestDoFn()))
    assert_that(pcoll, equal_to([(1, (-5, 5)), (1, (0, 10)),
                                 (7, (0, 10)), (7, (5, 15))]))
    pipeline.run()

  def test_timestamp_param(self):
    class TestDoFn(NewDoFn):
      def process(self, element, timestamp=NewDoFn.TimestampParam):
        yield timestamp

    pipeline = TestPipeline()
    pcoll = pipeline | 'Create' >> Create([1, 2]) | 'Do' >> ParDo(TestDoFn())
    assert_that(pcoll, equal_to([MIN_TIMESTAMP, MIN_TIMESTAMP]))
    pipeline.run()

class Bacon(PipelineOptions):

  @classmethod
  def _add_argparse_args(cls, parser):
    parser.add_argument('--slices', type=int)

class Eggs(PipelineOptions):

  @classmethod
  def _add_argparse_args(cls, parser):
    parser.add_argument('--style', default='scrambled')

class Breakfast(Bacon, Eggs):
  pass

class PipelineOptionsTest(unittest.TestCase):

  def test_flag_parsing(self):
    options = Breakfast(['--slices=3', '--style=sunny side up', '--ignored'])
    self.assertEquals(3, options.slices)
    self.assertEquals('sunny side up', options.style)

  def test_keyword_parsing(self):
    options = Breakfast(
        ['--slices=3', '--style=sunny side up', '--ignored'],
        slices=10)
    self.assertEquals(10, options.slices)
    self.assertEquals('sunny side up', options.style)

  def test_attribute_setting(self):
    options = Breakfast(slices=10)
    self.assertEquals(10, options.slices)
    options.slices = 20
    self.assertEquals(20, options.slices)

  def test_view_as(self):
    generic_options = PipelineOptions(['--slices=3'])
    self.assertEquals(3, generic_options.view_as(Bacon).slices)
    self.assertEquals(3, generic_options.view_as(Breakfast).slices)

    generic_options.view_as(Breakfast).slices = 10
    self.assertEquals(10, generic_options.view_as(Bacon).slices)

    with self.assertRaises(AttributeError):
      generic_options.slices  # pylint: disable=pointless-statement

    with self.assertRaises(AttributeError):
      generic_options.view_as(Eggs).slices  # pylint: disable=expression-not-assigned

  def test_defaults(self):
    options = Breakfast(['--slices=3'])
    self.assertEquals(3, options.slices)
    self.assertEquals('scrambled', options.style)

  def test_dir(self):
    options = Breakfast()
    self.assertEquals(
        set(['from_dictionary', 'get_all_options', 'slices', 'style',
             'view_as', 'display_data']),
        set([attr for attr in dir(options) if not attr.startswith('_')]))
    self.assertEquals(
        set(['from_dictionary', 'get_all_options', 'style', 'view_as',
             'display_data']),
        set([attr for attr in dir(options.view_as(Eggs))
             if not attr.startswith('_')]))

if __name__ == '__main__':
  logging.getLogger().setLevel(logging.DEBUG)
  unittest.main()
<|endoftext|>