<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__all__ = ['MoQuerySet', 'MoManager']

import copy
import logging
from django.db import connections
from django.db.models import Model, Manager, get_model
from django.db.models.query import RawQuerySet
from django.db.utils import DEFAULT_DB_ALIAS
from mosql.query import select, join
from mosql.util import raw, identifier, star
try:
    basestring
except NameError:   # If basestring is not a thing, just alias it to str
    basestring = str

logger = logging.getLogger(__name__)

def _as(src, dest):
    return raw('%s AS %s' % (identifier(src), identifier(dest)))

class MoQuerySet(object):
    """Django query set wrapper to bridge with MoSQL"""

    def __init__(self, model, fields, using):
        self.model = model
        self.fields = fields
        self._db = using
        self._rawqueryset = None
        self._where = {}
        self._joins = []

    def __iter__(self):
        """Iterate through the queryset using the backed RawQuerySet"""
        return self._get_rawqueryset().__iter__()

    def __getitem__(self, k):
        return list(self)[k]

    def __getattr__(self, name):
        return getattr(self._get_rawqueryset(), name)

    def _clone(self):
        clone = MoQuerySet(
            model=self.model, fields=self.fields, using=self._db
        )
        clone._where = self._where.copy()
        clone._joins = copy.copy(self._joins)
        return clone

    def _get_rawqueryset(self):
        if self._rawqueryset is None:
            kwargs = {'select': self.fields}
            if self._where:
                kwargs['where'] = self._where
            if self._joins:
                kwargs['joins'] = [join(**j) for j in self._joins]

            # Import MoSQL's detabase specific fixes
            conn = connections.get(self._db, connections[DEFAULT_DB_ALIAS])
            vendor = conn.vendor
            if vendor == 'postgresql':
                pass
            elif vendor == 'mysql':
                import mosql.mysql as _
                del _   # Get around PyFlakes "imported but not used" warning
            else:
                msg = (
                    'Current database ({vendor}) not supported by MoSQL. '
                    'Will generate standard SQL instead.'
                ).format(vendor=vendor)
                logger.warning(msg)

            raw_query = select(self.model._meta.db_table, **kwargs)
            self._rawqueryset = RawQuerySet(
                raw_query=raw_query, model=self.model, using=self._db
            )
        return self._rawqueryset

    def count(self):
        return len(list(self))

    def where(self, mapping):
        clone = self._clone()
        clone._where.update(mapping)
        return clone

    def join(self, model, alias, on=None, using=None, join_type=None):
        if isinstance(model, basestring):   # Try to lazy-load the model
            parts = model.split('.')
            if len(parts) == 2 and parts[0] and parts[1]:
                model = get_model(*parts)
        if issubclass(model, Model):
            table = model._meta.db_table
        else:
            table = model
        clone = self._clone()
        join_info = {'table': _as(table, alias), 'on': on, 'using': using}
        if join_type is not None:
            join_info['type'] = join_type
        clone._joins.append(join_info)
        return clone

class MoManager(Manager):
    """Django model manager subclass with MoSQL bridging"""

    def select(self, *extra_field_as):
        """Generate a MoQuerySet to kick-off MoSQL query syntax

        :param extra_field_as: Each item should be a 2-tuple indicating the
            field name and the attribute name it will be injected as.
        """
        fields = [star] + [_as(*f) for f in extra_field_as]
        return MoQuerySet(model=self.model, fields=fields, using=self._db)
<commit_msg>Implement repr and better iter<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__all__ = ['MoQuerySet', 'MoManager']

import copy
import logging
from django.db import connections
from django.db.models import Model, Manager, get_model
from django.db.models.query import RawQuerySet
from django.db.utils import DEFAULT_DB_ALIAS
from mosql.query import select, join
from mosql.util import raw, identifier, star
try:
    basestring
except NameError:   # If basestring is not a thing, just alias it to str
    basestring = str

logger = logging.getLogger(__name__)

def _as(src, dest):
    return raw('%s AS %s' % (identifier(src), identifier(dest)))

class MoQuerySet(object):
    """Django query set wrapper to bridge with MoSQL"""

    def __init__(self, model, fields, using):
        self.model = model
        self.fields = fields
        self._db = using
        self._rawqueryset = None
        self._where = {}
        self._joins = []

    def __repr__(self):
        rqs = self._get_rawqueryset()
        text = repr(rqs)
        return '<MoQuerySet: ' + text[14:]  # Replace "<RawQuerySet: "

    def __iter__(self):
        """Iterate through the queryset using the backed RawQuerySet"""
        return iter(self._get_rawqueryset())

    def __getitem__(self, k):
        return list(self)[k]

    def __getattr__(self, name):
        return getattr(self._get_rawqueryset(), name)

    def _clone(self):
        clone = MoQuerySet(
            model=self.model, fields=self.fields, using=self._db
        )
        clone._where = self._where.copy()
        clone._joins = copy.copy(self._joins)
        return clone

    def _get_rawqueryset(self):
        if self._rawqueryset is None:
            kwargs = {'select': self.fields}
            if self._where:
                kwargs['where'] = self._where
            if self._joins:
                kwargs['joins'] = [join(**j) for j in self._joins]

            # Import MoSQL's detabase specific fixes
            conn = connections.get(self._db, connections[DEFAULT_DB_ALIAS])
            vendor = conn.vendor
            if vendor == 'postgresql':
                pass
            elif vendor == 'mysql':
                import mosql.mysql as _
                del _   # Get around PyFlakes "imported but not used" warning
            else:
                msg = (
                    'Current database ({vendor}) not supported by MoSQL. '
                    'Will generate standard SQL instead.'
                ).format(vendor=vendor)
                logger.warning(msg)

            raw_query = select(self.model._meta.db_table, **kwargs)
            self._rawqueryset = RawQuerySet(
                raw_query=raw_query, model=self.model, using=self._db
            )
        return self._rawqueryset

    def count(self):
        return len(list(self))

    def where(self, mapping):
        clone = self._clone()
        clone._where.update(mapping)
        return clone

    def join(self, model, alias, on=None, using=None, join_type=None):
        if isinstance(model, basestring):   # Try to lazy-load the model
            parts = model.split('.')
            if len(parts) == 2 and parts[0] and parts[1]:
                model = get_model(*parts)
        if issubclass(model, Model):
            table = model._meta.db_table
        else:
            table = model
        clone = self._clone()
        join_info = {'table': _as(table, alias), 'on': on, 'using': using}
        if join_type is not None:
            join_info['type'] = join_type
        clone._joins.append(join_info)
        return clone

class MoManager(Manager):
    """Django model manager subclass with MoSQL bridging"""

    def select(self, *extra_field_as):
        """Generate a MoQuerySet to kick-off MoSQL query syntax

        :param extra_field_as: Each item should be a 2-tuple indicating the
            field name and the attribute name it will be injected as.
        """
        fields = [star] + [_as(*f) for f in extra_field_as]
        return MoQuerySet(model=self.model, fields=fields, using=self._db)
<|endoftext|>


<commit_before>import datetime
import threading

def parse_version_string():
    import os
    import os.path
    import subprocess

    path = os.path.abspath(__file__)
    while os.path.islink(path):
        path = os.path.join(os.path.dirname(path), os.readlink(path))
    path = os.path.dirname(path) # go up one level, from repo/loops.py to repo, where README.md is located
    while os.path.islink(path):
        path = os.path.join(os.path.dirname(path), os.readlink(path))
    try:
        version = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], cwd=path).decode('utf-8').strip('\n')
        if version == 'master':
            try:
                with open(os.path.join(path, 'README.md')) as readme:
                    for line in readme.read().splitlines():
                        if line.startswith('This is `python-xdg-basedir` version'):
                            return line.split(' ')[4]
            except:
                pass
        return subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=path).decode('utf-8').strip('\n')
    except:
        pass

__version__ = parse_version_string()

class IterThread(threading.Thread):
    """Helper class used in loops."""
    def __init__(self, iterator):
        super().__init__()
        self.daemon = True
        self.iterator = iterator
        self.stopped = False

    def run(self):
        try:
            self.value = next(self.iterator)
        except StopIteration:
            self.stopped = True

class Loop(threading.Thread):
    """Generic loop thread that periodically checks if it should stop while waiting for the iterable to yield.

    Keyword-only arguments:
    iterable -- The iterable to be looped over. By default, self.get_iterable is called.
    process_value -- A function which will be called with each yielded value as argument. Defaults to self.process_value.
    sleep_length -- A datetime.timedelta representing how long to sleep between each check for the next value or the stop signal. Defaults to half a second.
    """
    def __init__(self, *, iterable=None, process_value=None, sleep_length=datetime.timedelta(seconds=0.5)):
        super().__init__()
        if iterable is not None:
            self.iterable = iterable
        if process_value is not None:
            self.process_value = process_value
        self.stopped = False
        self.sleep_length = sleep_length

    @staticmethod
    def iterable():
        """The iterable to be looped over. Must be overridden in a subclass, or by passing the `iterable' keyword argument to the constructor."""
        raise NotImplementedError('iterable must be overwritten in subclasses, or set explicitly')

    def run(self):
        iterator = iter(self.iterable)
        iter_thread = IterThread(iterator)
        iter_thread.start() # get the first value
        while not self.stopped:
            if not iter_thread.is_alive():
                if iter_thread.stopped: # iterator exhausted
                    return
                else: # iterator has yielded a value
                    self.process_value(value)
                    iter_thread = IterThread(iterator)
                    iter_thread.start() # get the next value
            time.sleep(self.sleep_length.total_seconds())

    @staticmethod
    def process_value(value):
        """Will be called with each yielded value as argument. Must be overridden in a subclass, or by passing the `process_value' keyword argument to the constructor."""
        raise NotImplementedError('process_value must be overwritten in subclasses, or set explicitly')

    def start(self):
        self.stopped = False
        super().start()

    def stop(self):
        self.stopped = True
<commit_msg>Fix process_value call in Loop<commit_after>import datetime
import threading

def parse_version_string():
    import os
    import os.path
    import subprocess

    path = os.path.abspath(__file__)
    while os.path.islink(path):
        path = os.path.join(os.path.dirname(path), os.readlink(path))
    path = os.path.dirname(path) # go up one level, from repo/loops.py to repo, where README.md is located
    while os.path.islink(path):
        path = os.path.join(os.path.dirname(path), os.readlink(path))
    try:
        version = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], cwd=path).decode('utf-8').strip('\n')
        if version == 'master':
            try:
                with open(os.path.join(path, 'README.md')) as readme:
                    for line in readme.read().splitlines():
                        if line.startswith('This is `python-xdg-basedir` version'):
                            return line.split(' ')[4]
            except:
                pass
        return subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=path).decode('utf-8').strip('\n')
    except:
        pass

__version__ = parse_version_string()

class IterThread(threading.Thread):
    """Helper class used in loops."""
    def __init__(self, iterator):
        super().__init__()
        self.daemon = True
        self.iterator = iterator
        self.stopped = False

    def run(self):
        try:
            self.value = next(self.iterator)
        except StopIteration:
            self.stopped = True

class Loop(threading.Thread):
    """Generic loop thread that periodically checks if it should stop while waiting for the iterable to yield.

    Keyword-only arguments:
    iterable -- The iterable to be looped over. By default, self.get_iterable is called.
    process_value -- A function which will be called with each yielded value as argument. Defaults to self.process_value.
    sleep_length -- A datetime.timedelta representing how long to sleep between each check for the next value or the stop signal. Defaults to half a second.
    """
    def __init__(self, *, iterable=None, process_value=None, sleep_length=datetime.timedelta(seconds=0.5)):
        super().__init__()
        if iterable is not None:
            self.iterable = iterable
        if process_value is not None:
            self.process_value = process_value
        self.stopped = False
        self.sleep_length = sleep_length

    @staticmethod
    def iterable():
        """The iterable to be looped over. Must be overridden in a subclass, or by passing the `iterable' keyword argument to the constructor."""
        raise NotImplementedError('iterable must be overwritten in subclasses, or set explicitly')

    def run(self):
        iterator = iter(self.iterable)
        iter_thread = IterThread(iterator)
        iter_thread.start() # get the first value
        while not self.stopped:
            if not iter_thread.is_alive():
                if iter_thread.stopped: # iterator exhausted
                    return
                else: # iterator has yielded a value
                    self.process_value(iter_thread.value)
                    iter_thread = IterThread(iterator)
                    iter_thread.start() # get the next value
            time.sleep(self.sleep_length.total_seconds())

    @staticmethod
    def process_value(value):
        """Will be called with each yielded value as argument. Must be overridden in a subclass, or by passing the `process_value' keyword argument to the constructor."""
        raise NotImplementedError('process_value must be overwritten in subclasses, or set explicitly')

    def start(self):
        self.stopped = False
        super().start()

    def stop(self):
        self.stopped = True
<|endoftext|>


<commit_before>import json
import logging
import sys

from domaintools import API

log = logging.getLogger('domaintools')
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)

misperrors = {'error': 'Error'}
mispattributes = {
    'input': ['domain'],
    'output': ['whois-registrant-email', 'whois-registrant-phone', 'whois-registrant-name',
               'whois-registrar', 'whois-creation-date', 'freetext']
}

moduleinfo = {
    'version': '0.1',
    'author': 'Raphaël Vinot',
    'description': 'DomainTools MISP expansion module.',
    'module-type': ['expansion', 'hover']
}

moduleconfig = ['username', 'api_key']

class DomainTools(object):

    def __init__(self):
        self.reg_mail = set()
        self.reg_phone = set()
        self.reg_name = set()
        self.registrar = set()
        self.creation_date = set()
        self.freetext = ''

    def dump(self):
        to_return = []
        if self.reg_mail:
            to_return.append({'type': ['whois-registrant-email'], 'values': list(self.reg_mail)})
        if self.reg_phone:
            to_return.append({'type': ['whois-registrant-phone'], 'values': list(self.reg_phone)})
        if self.reg_name:
            to_return.append({'type': ['whois-registrant-name'], 'values': list(self.reg_name)})
        if self.registrar:
            to_return.append({'type': ['whois-registrar'], 'values': list(self.registrar)})
        if self.creation_date:
            to_return.append({'type': ['whois-creation-date'], 'values': list(self.creation_date)})
        if self.freetext:
            to_return.append({'type': ['freetext'], 'values': [self.freetext]})
        return to_return

def handler(q=False):
    if not q:
        return q

    request = json.loads(q)
    to_query = None
    for t in mispattributes['input']:
        to_query = request.get(t)
        if to_query:
            break
    if not to_query:
        misperrors['error'] = "Unsupported attributes type"
        return misperrors

    if request.get('config'):
        if (request['config'].get('username') is None) or (request['config'].get('api_key') is None):
            misperrors['error'] = 'DomainTools authentication is incomplete'
            return misperrors
        else:
            domtools = API(request['config'].get('username'), request['config'].get('api_key'))
    else:
        misperrors['error'] = 'DomainTools authentication is missing'
        return misperrors

    whois_entry = domtools.parsed_whois(to_query)
    values = DomainTools()

    if whois_entry.has_key('error'):
        misperrors['error'] = whois_entry['error']['message']
        return misperrors

    if whois_entry.has_key('registrant'):
        values.reg_name.add(whois_entry['registrant'])

    if whois_entry.has_key('registration'):
        values.creation_date.add(whois_entry['registration']['created'])

    if whois_entry.has_key('whois'):
        values.freetext = whois_entry['whois']['record']
    if whois_entry.emails():
        # NOTE: not sure we want to do that (contains registrar emails)
        values.reg_mail |= whois_entry.emails()
    if whois_entry.has_key('parsed_whois'):
        if whois_entry['parsed_whois']['created_date']:
            values.creation_date.add(whois_entry['parsed_whois']['created_date'])
        if whois_entry['parsed_whois']['registrar']['name']:
            values.registrar.add(whois_entry['parsed_whois']['registrar']['name'])
        for key, entry in whois_entry['parsed_whois']['contacts'].items():
            # TODO: pass key as comment
            if entry['email']:
                values.reg_mail.add(entry['email'])
            if entry['phone']:
                values.reg_phone.add(entry['phone'])
            if entry['name']:
                values.reg_name.add(entry['name'])
    return json.dumps({'results': values.dump()})

def introspection():
    return mispattributes

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo
<commit_msg>Add comments to fields when possible<commit_after>import json
import logging
import sys

from domaintools import API

log = logging.getLogger('domaintools')
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)

misperrors = {'error': 'Error'}
mispattributes = {
    'input': ['domain'],
    'output': ['whois-registrant-email', 'whois-registrant-phone', 'whois-registrant-name',
               'whois-registrar', 'whois-creation-date', 'freetext']
}

moduleinfo = {
    'version': '0.1',
    'author': 'Raphaël Vinot',
    'description': 'DomainTools MISP expansion module.',
    'module-type': ['expansion', 'hover']
}

moduleconfig = ['username', 'api_key']

class DomainTools(object):

    def __init__(self):
        self.reg_mail = {}
        self.reg_phone = {}
        self.reg_name = {}
        self.registrar = set()
        self.creation_date = set()
        self.freetext = ''

    def _add_value(self, value_type, value, comment):
        if value_type.get(value):
            if comment:
                value_type[value] += ' - {}'.format(comment)
        else:
            value_type[value] = comment
        return value_type

    def add_mail(self, mail, comment=None):
        self.reg_mail = self._add_value(self.reg_mail, mail, comment)

    def add_phone(self, phone, comment=None):
        self.reg_phone = self._add_value(self.reg_phone, phone, comment)

    def add_name(self, name, comment=None):
        self.reg_name = self._add_value(self.reg_name, name, comment)

    def dump(self):
        to_return = []
        if self.reg_mail:
            for mail, comment in self.reg_mail.items():
                to_return.append({'type': ['whois-registrant-email'], 'values': [mail], 'comment': comment})
        if self.reg_phone:
            for phone, comment in self.reg_phone.items():
                to_return.append({'type': ['whois-registrant-phone'], 'values': [phone], 'comment': comment})
        if self.reg_name:
            for name, comment in self.reg_name.items():
                to_return.append({'type': ['whois-registrant-name'], 'values': [name], 'comment': comment})
        if self.registrar:
            to_return.append({'type': ['whois-registrar'], 'values': list(self.registrar)})
        if self.creation_date:
            to_return.append({'type': ['whois-creation-date'], 'values': list(self.creation_date)})
        if self.freetext:
            to_return.append({'type': ['freetext'], 'values': [self.freetext], 'comment': 'Freetext import'})
        return to_return

def handler(q=False):
    if not q:
        return q

    request = json.loads(q)
    to_query = None
    for t in mispattributes['input']:
        to_query = request.get(t)
        if to_query:
            break
    if not to_query:
        misperrors['error'] = "Unsupported attributes type"
        return misperrors

    if request.get('config'):
        if (request['config'].get('username') is None) or (request['config'].get('api_key') is None):
            misperrors['error'] = 'DomainTools authentication is incomplete'
            return misperrors
        else:
            domtools = API(request['config'].get('username'), request['config'].get('api_key'))
    else:
        misperrors['error'] = 'DomainTools authentication is missing'
        return misperrors

    whois_entry = domtools.parsed_whois(to_query)
    print(whois_entry)
    values = DomainTools()

    if whois_entry.get('error'):
        misperrors['error'] = whois_entry['error']['message']
        return misperrors

    if whois_entry.get('registrant'):
        values.add_name(whois_entry['registrant'], 'Parsed registrant')

    if whois_entry.get('registration'):
        values.creation_date.add(whois_entry['registration']['created'])

    if whois_entry.get('whois'):
        values.freetext = whois_entry['whois']['record']
    if whois_entry.get('parsed_whois'):
        if whois_entry['parsed_whois']['created_date']:
            values.creation_date.add(whois_entry['parsed_whois']['created_date'])
        if whois_entry['parsed_whois']['registrar']['name']:
            values.registrar.add(whois_entry['parsed_whois']['registrar']['name'])
        for key, entry in whois_entry['parsed_whois']['contacts'].items():
            if entry['email']:
                values.add_mail(entry['email'], key)
            if entry['phone']:
                values.add_phone(entry['phone'], key)
            if entry['name']:
                values.add_name(entry['name'], key)
    if whois_entry.emails():
        for mail in whois_entry.emails():
            values.add_mail(mail)
    return json.dumps({'results': values.dump()})

def introspection():
    return mispattributes

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo
<|endoftext|>


<commit_before># Copyright 2021 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A set of MLMD simple types."""

import abc
from typing import Union

from ml_metadata import errors
from ml_metadata.metadata_store.pywrap.metadata_store_extension import metadata_store as metadata_store_serialized
from ml_metadata.proto import metadata_store_pb2
from ml_metadata.simple_types.proto import simple_types_pb2

def _make_exception(msg, error_code):
  """Makes an exception with MLMD error code.

  Args:
    msg: Error message.
    error_code: MLMD error code.

  Returns:
    An exception.
  """

  try:
    exc_type = errors.exception_type_from_error_code(error_code)
    return exc_type(msg)
  except KeyError:
    return errors.UnknownError(msg)

def _get_type_proto(
    type_name: str, types: simple_types_pb2.SimpleTypes
) -> Union[metadata_store_pb2.ArtifactType, metadata_store_pb2.ExecutionType]:
  for artifact_type in types.artifact_types:
    if artifact_type.name == type_name:
      return artifact_type
  for execution_type in types.execution_types:
    if execution_type.name == type_name:
      return execution_type
  raise _make_exception('Type name: {} is not found'.format(type_name),
                        errors.NOT_FOUND)

class _SystemType(abc.ABC):
  """An abstract simple type."""

  def __init__(self, type_name: str):
    """Creates a type instance from system constants.

    Constructs an abstract system type based on 'type_name'. It loads simple
    type proto messages from simple_types_constants.cc and sets '_type'
    attribute based on 'type_name' parameter.

    Args:
      type_name: name of the desired system type.

    Raises:
      NOT_FOUND: if 'type_name' is not found in the pre-loaded simple type list;
      It also raises the corresponding error from wrapped LoadSimpleTypes util
      method.
    """
    [types_str, error_message,
     status_code] = metadata_store_serialized.LoadSimpleTypes()
    if status_code:
      raise _make_exception(error_message.decode('utf-8'), status_code)
    types = simple_types_pb2.SimpleTypes()
    types.ParseFromString(types_str)
    self._type = _get_type_proto(type_name, types)

  @property
  def name(self) -> str:
    return self._type.name

class ArtifactType(_SystemType):
  """An abstract simple artifact type for annotating artifact types."""

  def __init__(
      self, system_type: metadata_store_pb2.ArtifactType.SystemDefinedBaseType):
    extensions = metadata_store_pb2.ArtifactType.SystemDefinedBaseType.DESCRIPTOR.values_by_number[
        system_type].GetOptions().Extensions
    type_name = extensions[metadata_store_pb2.system_type_extension].type_name
    super().__init__(type_name)

class ExecutionType(_SystemType):
  """An abstract simple execution type for annotating execution types."""

  def __init__(
      self,
      system_type: metadata_store_pb2.ExecutionType.SystemDefinedBaseType):
    extensions = metadata_store_pb2.ExecutionType.SystemDefinedBaseType.DESCRIPTOR.values_by_number[
        system_type].GetOptions().Extensions
    type_name = extensions[metadata_store_pb2.system_type_extension].type_name
    super().__init__(type_name)

# A list of system pre-defined artifact types.
class Dataset(ArtifactType):
  """Dataset is a system pre-defined artifact type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ArtifactType.DATASET)

class Model(ArtifactType):
  """Model is a system pre-defined artifact type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ArtifactType.MODEL)

class Statistics(ArtifactType):
  """Statistics is a system pre-defined artifact type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ArtifactType.STATISTICS)

class Metrics(ArtifactType):
  """Metrics is a system pre-defined artifact type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ArtifactType.METRICS)

# A list of system pre-defined execution types.
class Train(ExecutionType):
  """Train is a system pre-defined execution type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ExecutionType.TRAIN)

class Transform(ExecutionType):
  """Transform is a system pre-defined execution type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ExecutionType.TRANSFORM)

class Process(ExecutionType):
  """Process is a system pre-defined execution type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ExecutionType.PROCESS)

class Evaluate(ExecutionType):
  """Evaluate is a system pre-defined execution type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ExecutionType.EVALUATE)

class Deploy(ExecutionType):
  """Deploy is a system pre-defined execution type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ExecutionType.DEPLOY)
<commit_msg>no-op<commit_after># Copyright 2021 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A set of MLMD simple types."""

import abc
from typing import Union

from ml_metadata import errors
from ml_metadata.metadata_store.pywrap.metadata_store_extension import metadata_store as metadata_store_serialized
from ml_metadata.proto import metadata_store_pb2
from ml_metadata.simple_types.proto import simple_types_pb2

def _make_exception(msg, error_code):
  """Makes an exception with MLMD error code.

  Args:
    msg: Error message.
    error_code: MLMD error code.

  Returns:
    An exception.
  """

  try:
    exc_type = errors.exception_type_from_error_code(error_code)
    return exc_type(msg)
  except KeyError:
    return errors.UnknownError(msg)

def _get_type_proto(
    type_name: str, types: simple_types_pb2.SimpleTypes
) -> Union[metadata_store_pb2.ArtifactType, metadata_store_pb2.ExecutionType]:
  for artifact_type in types.artifact_types:
    if artifact_type.name == type_name:
      return artifact_type
  for execution_type in types.execution_types:
    if execution_type.name == type_name:
      return execution_type
  raise _make_exception('Type name: {} is not found'.format(type_name),
                        errors.NOT_FOUND)

class _SystemType(abc.ABC):
  """An abstract simple type."""

  def __init__(self, type_name: str):
    """Creates a type instance from system constants.

    Constructs an abstract system type based on 'type_name'. It loads simple
    type proto messages from simple_types_constants.cc and sets '_type'
    attribute based on 'type_name' parameter.

    Args:
      type_name: name of the desired system type.

    Raises:
      NOT_FOUND: if 'type_name' is not found in the pre-loaded simple type list;
      It also raises the corresponding error from wrapped LoadSimpleTypes util
      method.
    """
    [types_str, error_message,
     status_code] = metadata_store_serialized.LoadSimpleTypes()
    if status_code:
      raise _make_exception(error_message.decode('utf-8'), status_code)
    types = simple_types_pb2.SimpleTypes()
    types.ParseFromString(types_str)
    self._type = _get_type_proto(type_name, types)

  @property
  def name(self) -> str:
    return self._type.name

class ArtifactType(_SystemType):
  """An abstract simple artifact type for annotating artifact types."""

  def __init__(
      self, system_type: metadata_store_pb2.ArtifactType.SystemDefinedBaseType):
    self._system_type = system_type
    extensions = metadata_store_pb2.ArtifactType.SystemDefinedBaseType.DESCRIPTOR.values_by_number[
        system_type].GetOptions().Extensions
    type_name = extensions[metadata_store_pb2.system_type_extension].type_name
    super().__init__(type_name)

  @property
  def system_type(
      self) -> metadata_store_pb2.ArtifactType.SystemDefinedBaseType:
    return self._system_type

class ExecutionType(_SystemType):
  """An abstract simple execution type for annotating execution types."""

  def __init__(
      self,
      system_type: metadata_store_pb2.ExecutionType.SystemDefinedBaseType):
    self._system_type = system_type
    extensions = metadata_store_pb2.ExecutionType.SystemDefinedBaseType.DESCRIPTOR.values_by_number[
        system_type].GetOptions().Extensions
    type_name = extensions[metadata_store_pb2.system_type_extension].type_name
    super().__init__(type_name)

  @property
  def system_type(
      self) -> metadata_store_pb2.ExecutionType.SystemDefinedBaseType:
    return self._system_type

# A list of system pre-defined artifact types.
class Dataset(ArtifactType):
  """Dataset is a system pre-defined artifact type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ArtifactType.DATASET)

class Model(ArtifactType):
  """Model is a system pre-defined artifact type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ArtifactType.MODEL)

class Statistics(ArtifactType):
  """Statistics is a system pre-defined artifact type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ArtifactType.STATISTICS)

class Metrics(ArtifactType):
  """Metrics is a system pre-defined artifact type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ArtifactType.METRICS)

# A list of system pre-defined execution types.
class Train(ExecutionType):
  """Train is a system pre-defined execution type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ExecutionType.TRAIN)

class Transform(ExecutionType):
  """Transform is a system pre-defined execution type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ExecutionType.TRANSFORM)

class Process(ExecutionType):
  """Process is a system pre-defined execution type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ExecutionType.PROCESS)

class Evaluate(ExecutionType):
  """Evaluate is a system pre-defined execution type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ExecutionType.EVALUATE)

class Deploy(ExecutionType):
  """Deploy is a system pre-defined execution type."""

  def __init__(self):
    super().__init__(metadata_store_pb2.ExecutionType.DEPLOY)
<|endoftext|>


<commit_before># (c) 2014, Brian Coca, Josh Drake, et al
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

import os
import time
import errno

try:
    import simplejson as json
except ImportError:
    import json

from ansible import constants as C
from ansible import utils
from ansible.cache.base import BaseCacheModule

class CacheModule(BaseCacheModule):
    """
    A caching module backed by json files.
    """
    def __init__(self, *args, **kwargs):

        self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
        self._cache = {}
        self._cache_dir = C.CACHE_PLUGIN_CONNECTION # expects a dir path
        if not self._cache_dir:
            utils.exit("error, fact_caching_connection is not set, cannot use fact cache")

        if not os.path.exists(self._cache_dir):
            try:
                os.makedirs(self._cache_dir)
            except (OSError,IOError), e:
                utils.warning("error while trying to create cache dir %s : %s" % (self._cache_dir, str(e)))
                return None

    def get(self, key):

        if key in self._cache:
            return self._cache.get(key)

        if self.has_expired(key):
           raise KeyError

        cachefile = "%s/%s" % (self._cache_dir, key)
        try:
            f = open( cachefile, 'r')
        except (OSError,IOError), e:
            utils.warning("error while trying to write to %s : %s" % (cachefile, str(e)))
        else:
            value = json.load(f)
            self._cache[key] = value
            return value
        finally:
            f.close()

    def set(self, key, value):

        self._cache[key] = value

        cachefile = "%s/%s" % (self._cache_dir, key)
        try:
            f = open(cachefile, 'w')
        except (OSError,IOError), e:
            utils.warning("error while trying to read %s : %s" % (cachefile, str(e)))
        else:
            f.write(utils.jsonify(value))
        finally:
            f.close()

    def has_expired(self, key):

        cachefile = "%s/%s" % (self._cache_dir, key)
        try:
            st = os.stat(cachefile)
        except (OSError,IOError), e:
            if e.errno == errno.ENOENT:
                return False
            else:
                utils.warning("error while trying to stat %s : %s" % (cachefile, str(e)))

        if time.time() - st.st_mtime <= self._timeout:
            return False

        if key in self._cache:
            del self._cache[key]
        return True

    def keys(self):
        keys = []
        for k in os.listdir(self._cache_dir):
            if not (k.startswith('.') or self.has_expired(k)):
                keys.append(k)
        return keys

    def contains(self, key):

        if key in self._cache:
            return True

        if self.has_expired(key):
            return False
        try:
            st = os.stat("%s/%s" % (self._cache_dir, key))
            return True
        except (OSError,IOError), e:
            if e.errno == errno.ENOENT:
                return False
            else:
                utils.warning("error while trying to stat %s : %s" % (cachefile, str(e)))

    def delete(self, key):
        del self._cache[key]
        try:
            os.remove("%s/%s" % (self._cache_dir, key))
        except (OSError,IOError), e:
            pass #TODO: only pass on non existing?

    def flush(self):
        self._cache = {}
        for key in self.keys():
            self.delete(key)

    def copy(self):
        ret = dict()
        for key in self.keys():
            ret[key] = self.get(key)
        return ret
<commit_msg>Use codecs module while reading & writing json cache file<commit_after># (c) 2014, Brian Coca, Josh Drake, et al
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

import os
import time
import errno
import codecs

try:
    import simplejson as json
except ImportError:
    import json

from ansible import constants as C
from ansible import utils
from ansible.cache.base import BaseCacheModule

class CacheModule(BaseCacheModule):
    """
    A caching module backed by json files.
    """
    def __init__(self, *args, **kwargs):

        self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
        self._cache = {}
        self._cache_dir = C.CACHE_PLUGIN_CONNECTION # expects a dir path
        if not self._cache_dir:
            utils.exit("error, fact_caching_connection is not set, cannot use fact cache")

        if not os.path.exists(self._cache_dir):
            try:
                os.makedirs(self._cache_dir)
            except (OSError,IOError), e:
                utils.warning("error while trying to create cache dir %s : %s" % (self._cache_dir, str(e)))
                return None

    def get(self, key):

        if key in self._cache:
            return self._cache.get(key)

        if self.has_expired(key):
           raise KeyError

        cachefile = "%s/%s" % (self._cache_dir, key)
        try:
            f = codecs.open(cachefile, 'r', encoding='utf-8')
        except (OSError,IOError), e:
            utils.warning("error while trying to write to %s : %s" % (cachefile, str(e)))
        else:
            value = json.load(f)
            self._cache[key] = value
            return value
        finally:
            f.close()

    def set(self, key, value):

        self._cache[key] = value

        cachefile = "%s/%s" % (self._cache_dir, key)
        try:
            f = codecs.open(cachefile, 'w', encoding='utf-8')
        except (OSError,IOError), e:
            utils.warning("error while trying to read %s : %s" % (cachefile, str(e)))
        else:
            f.write(utils.jsonify(value))
        finally:
            f.close()

    def has_expired(self, key):

        cachefile = "%s/%s" % (self._cache_dir, key)
        try:
            st = os.stat(cachefile)
        except (OSError,IOError), e:
            if e.errno == errno.ENOENT:
                return False
            else:
                utils.warning("error while trying to stat %s : %s" % (cachefile, str(e)))

        if time.time() - st.st_mtime <= self._timeout:
            return False

        if key in self._cache:
            del self._cache[key]
        return True

    def keys(self):
        keys = []
        for k in os.listdir(self._cache_dir):
            if not (k.startswith('.') or self.has_expired(k)):
                keys.append(k)
        return keys

    def contains(self, key):

        if key in self._cache:
            return True

        if self.has_expired(key):
            return False
        try:
            st = os.stat("%s/%s" % (self._cache_dir, key))
            return True
        except (OSError,IOError), e:
            if e.errno == errno.ENOENT:
                return False
            else:
                utils.warning("error while trying to stat %s : %s" % (cachefile, str(e)))

    def delete(self, key):
        del self._cache[key]
        try:
            os.remove("%s/%s" % (self._cache_dir, key))
        except (OSError,IOError), e:
            pass #TODO: only pass on non existing?

    def flush(self):
        self._cache = {}
        for key in self.keys():
            self.delete(key)

    def copy(self):
        ret = dict()
        for key in self.keys():
            ret[key] = self.get(key)
        return ret
<|endoftext|>


<commit_before>from __future__ import print_function

import sys
import os
import re
import argparse
import subprocess
from importlib import import_module
from collections import OrderedDict

def error(msg, exit_status=1):
    print(msg)
    sys.exit(exit_status)

class CMD(object):

    def __init__(self):
        self.registry = OrderedDict()

    def add(self, *aliases):
        def decorator(func):
            cmd_info = {
                'cmd': aliases[0],
                'callable': func,
                'desc': (func.__doc__ or '').rstrip(),
                'aliases': aliases[1:],
            }
            for alias in aliases:
                self.registry[alias] = cmd_info
            return func
        return decorator

    def resolve(self, cmd):
        if cmd in self.registry:
            return self.registry[cmd]['callable']

    def render_help(self):
        help_list = []
        for cmd, cmd_info in self.registry.items():
            if cmd != cmd_info['cmd']:
                continue
            cmd_str = cmd
            if cmd_info['aliases']:
                cmd_str = '{0: <4}(aliases: {1})'.format(
                    cmd_str, ', '.join(cmd_info['aliases']))
            if cmd_info['desc']:
                cmd_str = '{0: <30}{1}'.format(cmd_str, cmd_info['desc'])
            help_list.append(cmd_str)
        return '\n'.join(help_list)

class LuambException(Exception):

    pass

class Luamb(object):

    TYPE_RIO = 1
    TYPE_JIT = 2
    TYPE_ALL = TYPE_RIO | TYPE_JIT

    implementations = {
        TYPE_RIO: 'PUC-Rio Lua',
        TYPE_JIT: 'LuaJIT',
    }

    re_lua = re.compile('Lua(?:JIT)? [0-9.]+')
    re_rocks = re.compile('LuaRocks [0-9.]+')
    re_lua_v = re.compile('(?P<prefix>(?:lua)?(?P<jit>jit)?)(?P<version>.+)')
    re_rocks_s = re.compile('(?:lua)?(?:rocks)?(.+)')
    re_env_name = re.compile('^[^/\*?]+$')

    cmd = CMD()

    def __init__(self, env_dir, lua_default=None, luarocks_default=None,
                 hererocks_module=None):
        self.env_dir = env_dir
        self.lua_default = lua_default
        self.luarocks_default = luarocks_default
        self.hererocks_module = hererocks_module or import_module('hererocks')
        self.supported_versions = {
            self.TYPE_RIO: self._get_supported_versions(
                self.hererocks_module.RioLua),
            self.TYPE_JIT: self._get_supported_versions(
                self.hererocks_module.LuaJIT),
            'rocks': self._get_supported_versions(
                self.hererocks_module.LuaRocks),
        }

    def run(self, argv=None):
        if not argv:
            argv = sys.argv[1:]
        parser = argparse.ArgumentParser(
            prog='luamb',
            add_help=False,
        )
        parser.add_argument(
            'command',
            nargs='?',
            default='',
        )
        parser.add_argument(
            '-h', '--help',
            action='store_true',
        )
        args = parser.parse_args(argv[:1])
        self._show_main_usage = parser.print_usage
        if not args.command or args.help:
            self._show_main_help()
            return
        method = self.cmd.resolve(args.command)
        if not method:
            print("command '{}' not found\n"
                  "try 'luamb --help'".format(args.command))
        else:
            method(self, argv[1:])

    @cmd.add('on', 'enable')
    def cmd_on(self, argv):
        """activate environment
        """

    @cmd.add('off', 'disable')
    def cmd_off(self, argv):
        """deactivate environment
        """

    @cmd.add('mk', 'new', 'create')
    def cmd_mk(self, argv):
        """create new environment
        """
        parser = argparse.ArgumentParser(
            prog='luamb mk',
        )
        parser.add_argument(
            'env_name',
            help="environment name (used as directory name)"
        )
        parser.add_argument(
            '-a', '--associate',
            metavar='PROJECT_DIR',
            help="associate env with project",
        )
        parser.add_argument(
            '-l', '--lua',
            help="version of PUC-Rio Lua",
        )
        parser.add_argument(
            '-j', '--luajit',
            help="version of LuaJIT",
        )
        parser.add_argument(
            '-r', '--luarocks',
            help="version of LuaRocks",
        )
        parser.add_argument(
            '-n', '--no-luarocks',
            action='store_true',
            help="don't install LuaRocks (if default version specified via "
                 "environment variable)",
        )
        args, extra_args = parser.parse_known_args(argv)

        env_name = args.env_name
        if not self.re_env_name.match(env_name):
            raise LuambException("invalid env name: '{}'".format(env_name))

        if args.lua and args.luajit:
            raise LuambException("can't install both PUC-Rio Lua and LuaJIT")

        if args.lua or args.luajit:
            lua_type, lua_version = self._normalize_lua_version(
                args.lua or args.luajit,
                self.TYPE_RIO if args.lua else self.TYPE_JIT
            )
        else:
            if not self.lua_default:
                raise LuambException(
                    "specify Lua version with --lua/--luajit argument "
                    "or set default version via environment variable"
                )

            try:
                lua_type, lua_version = self._normalize_lua_version(
                    self.lua_default)
            except LuambException as exc:
                raise LuambException(
                    "Error parsing default Lua version "
                    "environment variable: {}".format(exc)
                )

        if args.luarocks:
            rocks_version = self._normalize_rocks_version(args.luarocks)
        elif not args.no_luarocks and self.luarocks_default:
            try:
                rocks_version = self._normalize_rocks_version(
                    self.luarocks_default)
            except LuambException as exc:
                raise LuambException(
                    "Error parsing default LuaRocks version "
                    "environment variable: {}".format(exc)
                )
        else:
            rocks_version = None

        env_path = os.path.join(self.env_dir, env_name)

        hererocks_args = [
            sys.executable,
            self.hererocks_module.__file__,
            '--lua' if lua_type == self.TYPE_RIO else '--luajit',
            lua_version,
        ]
        if rocks_version:
            hererocks_args.extend(['--luarocks', rocks_version])
        hererocks_args.extend(extra_args)
        hererocks_args.append(env_path)
        try:
            subprocess.check_call(hererocks_args)
        except subprocess.CalledProcessError:
            raise LuambException("error while running hererocks")

        if args.associate:
            with open(os.path.join(env_path, '.project'), 'w') as f:
                f.write(args.associate)

    @cmd.add('rm', 'remove', 'del')
    def cmd_rm(self, argv):
        """remove environment
        """

    @cmd.add('ls', 'list')
    def cmd_ls(self, argv):
        """list available environments
        """
        envs = next(os.walk(self.env_dir))[1]
        envs.sort()
        for env in envs:
            env_path = os.path.join(self.env_dir, env)
            print(env)
            print('='*len(env))
            print(env_path)
            lua_bin = os.path.join(env_path, 'bin', 'lua')
            try:
                print(self._get_output([lua_bin, '-v'], regex=self.re_lua))
            except LuambException as exc:
                print('Lua:', exc)
            luarocks_bin = os.path.join(env_path, 'bin', 'luarocks')
            try:
                print(self._get_output([luarocks_bin], regex=self.re_rocks))
            except LuambException as exc:
                print('LuaRocks:', exc)
            project_file_path = os.path.join(env_path, '.project')
            if os.path.isfile(project_file_path):
                with open(project_file_path) as f:
                    project = f.read().strip()
                print('project:', project)
            else:
                print('no associated project')
            print()

    def _show_main_help(self):
        self._show_main_usage()
        print("\navailable commands:\n")
        print(self.cmd.render_help())

    def _get_output(self, args, regex=None):
        try:
            output = subprocess.check_output(
                args, stderr=subprocess.STDOUT).decode('utf-8').strip()
        except OSError:
            raise LuambException("executable not found")
        except subprocess.CalledProcessError:
            raise LuambException("error while running executable")
        if not regex:
            return output
        mo = regex.search(output)
        if mo:
            return mo.group(0)
        raise LuambException("error parsing version")

    def _get_supported_versions(self, lua_cls):
        versions = {v: v for v in lua_cls.versions}
        versions.update(lua_cls.translations)
        return versions

    def _normalize_lua_version(self, version_string, lua_type=None):
        version_string = version_string.lower()
        if not lua_type:
            lua_type = self.TYPE_ALL

        groupdict = self.re_lua_v.match(version_string).groupdict()
        prefix = bool(groupdict['prefix'])
        jit = bool(groupdict['jit'])
        version = groupdict['version']

        if lua_type == self.TYPE_ALL and not prefix:
            raise LuambException(
                "specify implementation (PUC-Rio Lua or LuaJIT)")
        if lua_type == self.TYPE_RIO and jit:
            raise LuambException("specify PUC-Rio Lua version, not LuaJIT")
        if lua_type == self.TYPE_JIT and prefix and not jit:
            raise LuambException("specify LuaJIT version, not PUC-Rio")

        if lua_type == self.TYPE_ALL:
            lua_type = self.TYPE_JIT if jit else self.TYPE_RIO

        try:
            norm_version = self.supported_versions[lua_type][version]
        except KeyError:
            raise LuambException("non-supported {} version: {}".format(
                self.implementations[lua_type], version))

        return lua_type, norm_version

    def _normalize_rocks_version(self, version_string):
        version_string = version_string.lower()
        version = self.re_rocks_s.match(version_string).group(1)
        try:
            return self.supported_versions['rocks'][version]
        except KeyError:
            raise LuambException("non-supported LuaRocks version: {}".format(
                version))

if __name__ == '__main__':

    try:
        import hererocks
    except ImportError:
        error("'hererocks' is not installed")

    luamb_dir = os.environ.get('LUAMB_DIR')
    if not luamb_dir:
        error("LUAMB_DIR variable is not set")

    luamb_dir = os.path.abspath(luamb_dir)
    if not os.path.isdir(luamb_dir):
        error("LUAMB_DIR='{}' is not a directory".format(luamb_dir))

    luamb_lua_default = os.environ.get('LUAMB_LUA_DEFAULT')
    luamb_luarocks_default = os.environ.get('LUAMB_LUAROCKS_DEFAULT')

    luamb = Luamb(
        env_dir=luamb_dir,
        lua_default=luamb_lua_default,
        luarocks_default=luamb_luarocks_default,
        hererocks_module=hererocks,
    )

    try:
        luamb.run()
    except LuambException as exc:
        sys.exit(exc)
<commit_msg>output formatting<commit_after>from __future__ import print_function

import sys
import os
import re
import argparse
import subprocess
from importlib import import_module
from collections import OrderedDict

def error(msg, exit_status=1):
    print(msg)
    sys.exit(exit_status)

class CMD(object):

    def __init__(self):
        self.registry = OrderedDict()

    def add(self, *aliases):
        def decorator(func):
            cmd_info = {
                'cmd': aliases[0],
                'callable': func,
                'desc': (func.__doc__ or '').rstrip(),
                'aliases': aliases[1:],
            }
            for alias in aliases:
                self.registry[alias] = cmd_info
            return func
        return decorator

    def resolve(self, cmd):
        if cmd in self.registry:
            return self.registry[cmd]['callable']

    def render_help(self):
        help_list = []
        for cmd, cmd_info in self.registry.items():
            if cmd != cmd_info['cmd']:
                continue
            cmd_str = cmd
            if cmd_info['aliases']:
                cmd_str = '{0: <4}(aliases: {1})'.format(
                    cmd_str, ', '.join(cmd_info['aliases']))
            if cmd_info['desc']:
                cmd_str = '{0: <30}{1}'.format(cmd_str, cmd_info['desc'])
            help_list.append(cmd_str)
        return '\n'.join(help_list)

class LuambException(Exception):

    pass

class Luamb(object):

    TYPE_RIO = 1
    TYPE_JIT = 2
    TYPE_ALL = TYPE_RIO | TYPE_JIT

    implementations = {
        TYPE_RIO: 'PUC-Rio Lua',
        TYPE_JIT: 'LuaJIT',
    }

    re_lua = re.compile('Lua(?:JIT)? [0-9.]+')
    re_rocks = re.compile('LuaRocks [0-9.]+')
    re_lua_v = re.compile('(?P<prefix>(?:lua)?(?P<jit>jit)?)(?P<version>.+)')
    re_rocks_s = re.compile('(?:lua)?(?:rocks)?(.+)')
    re_env_name = re.compile('^[^/\*?]+$')

    cmd = CMD()

    def __init__(self, env_dir, lua_default=None, luarocks_default=None,
                 hererocks_module=None):
        self.env_dir = env_dir
        self.lua_default = lua_default
        self.luarocks_default = luarocks_default
        self.hererocks_module = hererocks_module or import_module('hererocks')
        self.supported_versions = {
            self.TYPE_RIO: self._fetch_supported_versions(
                self.hererocks_module.RioLua),
            self.TYPE_JIT: self._fetch_supported_versions(
                self.hererocks_module.LuaJIT),
            'rocks': self._fetch_supported_versions(
                self.hererocks_module.LuaRocks),
        }

    def run(self, argv=None):
        if not argv:
            argv = sys.argv[1:]
        parser = argparse.ArgumentParser(
            prog='luamb',
            add_help=False,
        )
        parser.add_argument(
            'command',
            nargs='?',
            default='',
        )
        parser.add_argument(
            '-h', '--help',
            action='store_true',
        )
        args = parser.parse_args(argv[:1])
        self._show_main_usage = parser.print_usage
        if not args.command or args.help:
            self._show_main_help()
            return
        method = self.cmd.resolve(args.command)
        if not method:
            print("command '{}' not found\n"
                  "try 'luamb --help'".format(args.command))
        else:
            method(self, argv[1:])

    @cmd.add('on', 'enable')
    def cmd_on(self, argv):
        """activate environment
        """
        print("usage: luamb on ENV_NAME\nthis command is implemented "
              "as shell function and can't be called via luamb.py")

    @cmd.add('off', 'disable')
    def cmd_off(self, argv):
        """deactivate environment
        """
        print("usage: luamb off\nthis command is implemented "
              "as shell function and can't be called via luamb.py")

    @cmd.add('mk', 'new', 'create')
    def cmd_mk(self, argv):
        """create new environment
        """
        parser = argparse.ArgumentParser(
            prog='luamb mk',
        )
        parser.add_argument(
            'env_name',
            metavar='ENV_NAME',
            help="environment name (used as directory name)"
        )
        parser.add_argument(
            '-a', '--associate',
            metavar='PROJECT_DIR',
            help="associate env with project",
        )
        parser.add_argument(
            '-l', '--lua',
            help="version of PUC-Rio Lua",
        )
        parser.add_argument(
            '-j', '--luajit',
            help="version of LuaJIT",
        )
        parser.add_argument(
            '-r', '--luarocks',
            help="version of LuaRocks",
        )
        parser.add_argument(
            '-n', '--no-luarocks',
            action='store_true',
            help="don't install LuaRocks (if default version specified via "
                 "environment variable)",
        )
        args, extra_args = parser.parse_known_args(argv)

        env_name = args.env_name
        if not self.re_env_name.match(env_name):
            raise LuambException("invalid env name: '{}'".format(env_name))

        if args.lua and args.luajit:
            raise LuambException("can't install both PUC-Rio Lua and LuaJIT")

        if args.lua or args.luajit:
            lua_type, lua_version = self._normalize_lua_version(
                args.lua or args.luajit,
                self.TYPE_RIO if args.lua else self.TYPE_JIT
            )
        else:
            if not self.lua_default:
                raise LuambException(
                    "specify Lua version with --lua/--luajit argument "
                    "or set default version via environment variable"
                )

            try:
                lua_type, lua_version = self._normalize_lua_version(
                    self.lua_default)
            except LuambException as exc:
                raise LuambException(
                    "Error parsing default Lua version "
                    "environment variable: {}".format(exc)
                )

        if args.luarocks:
            rocks_version = self._normalize_rocks_version(args.luarocks)
        elif not args.no_luarocks and self.luarocks_default:
            try:
                rocks_version = self._normalize_rocks_version(
                    self.luarocks_default)
            except LuambException as exc:
                raise LuambException(
                    "Error parsing default LuaRocks version "
                    "environment variable: {}".format(exc)
                )
        else:
            rocks_version = None

        env_path = os.path.join(self.env_dir, env_name)

        hererocks_args = [
            sys.executable,
            self.hererocks_module.__file__,
            '--lua' if lua_type == self.TYPE_RIO else '--luajit',
            lua_version,
        ]
        if rocks_version:
            hererocks_args.extend(['--luarocks', rocks_version])
        hererocks_args.extend(extra_args)
        hererocks_args.append(env_path)
        try:
            subprocess.check_call(hererocks_args)
        except subprocess.CalledProcessError:
            raise LuambException("error while running hererocks")

        if args.associate:
            with open(os.path.join(env_path, '.project'), 'w') as f:
                f.write(args.associate)

    @cmd.add('rm', 'remove', 'del')
    def cmd_rm(self, argv):
        """remove environment
        """

    @cmd.add('ls', 'list')
    def cmd_ls(self, argv):
        """list available environments
        """
        envs = next(os.walk(self.env_dir))[1]
        envs.sort()
        for env in envs:
            env_path = os.path.join(self.env_dir, env)
            print(env)
            print('='*len(env))
            print(env_path)
            lua_bin = os.path.join(env_path, 'bin', 'lua')
            try:
                print(self._get_output([lua_bin, '-v'], regex=self.re_lua))
            except LuambException as exc:
                print('Lua:', exc)
            luarocks_bin = os.path.join(env_path, 'bin', 'luarocks')
            try:
                print(self._get_output([luarocks_bin], regex=self.re_rocks))
            except LuambException as exc:
                print('LuaRocks:', exc)
            project_file_path = os.path.join(env_path, '.project')
            if os.path.isfile(project_file_path):
                with open(project_file_path) as f:
                    project = f.read().strip()
                print('project:', project)
            else:
                print('no associated project')
            print()

    def _show_main_help(self):
        self._show_main_usage()
        print("\navailable commands:\n")
        print(self.cmd.render_help())

    def _get_output(self, args, regex=None):
        try:
            output = subprocess.check_output(
                args, stderr=subprocess.STDOUT).decode('utf-8').strip()
        except OSError:
            raise LuambException("executable not found")
        except subprocess.CalledProcessError:
            raise LuambException("error while running executable")
        if not regex:
            return output
        mo = regex.search(output)
        if mo:
            return mo.group(0)
        raise LuambException("error parsing version")

    def _fetch_supported_versions(self, lua_cls):
        versions = {v: v for v in lua_cls.versions}
        versions.update(lua_cls.translations)
        return versions

    def _get_supported_versions(self, product_key, separator=None):
        versions = list(sorted(self.supported_versions[product_key]))
        if not separator:
            return versions
        return separator.join(versions)

    def _normalize_lua_version(self, version_string, lua_type=None):
        version_string = version_string.lower()
        if not lua_type:
            lua_type = self.TYPE_ALL

        groupdict = self.re_lua_v.match(version_string).groupdict()
        prefix = bool(groupdict['prefix'])
        jit = bool(groupdict['jit'])
        version = groupdict['version']

        if lua_type == self.TYPE_ALL and not prefix:
            raise LuambException(
                "specify implementation (PUC-Rio Lua or LuaJIT)")
        if lua_type == self.TYPE_RIO and jit:
            raise LuambException("specify PUC-Rio Lua version, not LuaJIT")
        if lua_type == self.TYPE_JIT and prefix and not jit:
            raise LuambException("specify LuaJIT version, not PUC-Rio")

        if lua_type == self.TYPE_ALL:
            lua_type = self.TYPE_JIT if jit else self.TYPE_RIO

        try:
            norm_version = self.supported_versions[lua_type][version]
        except KeyError:
            supported_versions = self._get_supported_versions(lua_type, '  ')
            raise LuambException(
                "non-supported {} version: {}\n"
                "supported versions are: {}".format(
                    self.implementations[lua_type],
                    version,
                    supported_versions
                )
            )

        return lua_type, norm_version

    def _normalize_rocks_version(self, version_string):
        version_string = version_string.lower()
        version = self.re_rocks_s.match(version_string).group(1)
        try:
            return self.supported_versions['rocks'][version]
        except KeyError:
            supported_versions = self._get_supported_versions('rocks', '  ')
            raise LuambException(
                "non-supported LuaRocks version: {}\n"
                "supported versions are: {}".format(
                    version,
                    supported_versions
                )
            )

if __name__ == '__main__':

    try:
        import hererocks
    except ImportError:
        error("'hererocks' is not installed")

    luamb_dir = os.environ.get('LUAMB_DIR')
    if not luamb_dir:
        error("LUAMB_DIR variable is not set")

    luamb_dir = os.path.abspath(luamb_dir)
    if not os.path.isdir(luamb_dir):
        error("LUAMB_DIR='{}' is not a directory".format(luamb_dir))

    luamb_lua_default = os.environ.get('LUAMB_LUA_DEFAULT')
    luamb_luarocks_default = os.environ.get('LUAMB_LUAROCKS_DEFAULT')

    luamb = Luamb(
        env_dir=luamb_dir,
        lua_default=luamb_lua_default,
        luarocks_default=luamb_luarocks_default,
        hererocks_module=hererocks,
    )

    try:
        luamb.run()
    except LuambException as exc:
        sys.exit(exc)
<|endoftext|>


<commit_before># Copyright 2017 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Rules for manipulation container images."""

load("//container:bundle.bzl", "container_bundle")
load("//container:flatten.bzl", "container_flatten")
load("//container:image.bzl", "container_image", "image")
load("//container:layer.bzl", "container_layer")
load("//container:import.bzl", "container_import")
load("//container:load.bzl", "container_load")
load("//container:pull.bzl", "container_pull")
load("//container:push.bzl", "container_push")

container = struct(
    image = image,
)

# The release of the github.com/google/containerregistry to consume.
CONTAINERREGISTRY_RELEASE = "v0.0.26"

# The release of the container-structure-test repository to use.
# Updated around 1/22/2018.
STRUCTURE_TEST_COMMIT = "b97925142b1a09309537e648ade11b4af47ff7ad"

_local_tool_build_template = """
sh_binary(
    name = "{name}",
    srcs = ["bin/{name}"],
    visibility = ["//visibility:public"],
)
"""

def _local_tool(repository_ctx):
    rctx = repository_ctx
    realpath = rctx.which(rctx.name)
    rctx.symlink(realpath, "bin/%s" % rctx.name)
    rctx.file(
        "WORKSPACE",
        'workspace(name = "{}")\n'.format(rctx.name),
    )
    rctx.file(
        "BUILD",
        _local_tool_build_template.format(name = rctx.name),
    )

local_tool = repository_rule(
    local = True,
    implementation = _local_tool,
)

def repositories():
    """Download dependencies of container rules."""
    excludes = native.existing_rules().keys()

    if "puller" not in excludes:
        native.http_file(
            name = "puller",
            url = ("https://storage.googleapis.com/containerregistry-releases/" +
                   CONTAINERREGISTRY_RELEASE + "/puller.par"),
            sha256 = "42309ba47bb28d1e1b81ef72789dcca396095e191d4f0e49e2e23c297edd27fb",
            executable = True,
        )

    if "importer" not in excludes:
        native.http_file(
            name = "importer",
            url = ("https://storage.googleapis.com/containerregistry-releases/" +
                   CONTAINERREGISTRY_RELEASE + "/importer.par"),
            sha256 = "0a2490584c96bcf961242364d961859b94926182f20a217754730e7097ea6cde",
            executable = True,
        )

    if "containerregistry" not in excludes:
        native.http_archive(
            name = "containerregistry",
            url = ("https://github.com/google/containerregistry/archive/" +
                   CONTAINERREGISTRY_RELEASE + ".tar.gz"),
            strip_prefix = "containerregistry-" + CONTAINERREGISTRY_RELEASE[1:],
        )

    # TODO(mattmoor): Remove all of this (copied from google/containerregistry)
    # once transitive workspace instantiation lands.

    if "httplib2" not in excludes:
        # TODO(mattmoor): Is there a clean way to override?
        native.new_http_archive(
            name = "httplib2",
            url = "https://codeload.github.com/httplib2/httplib2/tar.gz/v0.11.3",
            sha256 = "d9f568c183d1230f271e9c60bd99f3f2b67637c3478c9068fea29f7cca3d911f",
            strip_prefix = "httplib2-0.11.3/python2/httplib2/",
            type = "tar.gz",
            build_file_content = """
py_library(
   name = "httplib2",
   srcs = glob(["**/*.py"]),
   data = ["cacerts.txt"],
   visibility = ["//visibility:public"]
)""",
        )

    # Used by oauth2client
    if "six" not in excludes:
        # TODO(mattmoor): Is there a clean way to override?
        native.new_http_archive(
            name = "six",
            url = "https://pypi.python.org/packages/source/s/six/six-1.9.0.tar.gz",
            sha256 = "e24052411fc4fbd1f672635537c3fc2330d9481b18c0317695b46259512c91d5",
            strip_prefix = "six-1.9.0/",
            type = "tar.gz",
            build_file_content = """
# Rename six.py to __init__.py
genrule(
    name = "rename",
    srcs = ["six.py"],
    outs = ["__init__.py"],
    cmd = "cat $< >$@",
)
py_library(
   name = "six",
   srcs = [":__init__.py"],
   visibility = ["//visibility:public"],
)""",
        )

    # Used for authentication in containerregistry
    if "oauth2client" not in excludes:
        # TODO(mattmoor): Is there a clean way to override?
        native.new_http_archive(
            name = "oauth2client",
            url = "https://codeload.github.com/google/oauth2client/tar.gz/v4.0.0",
            sha256 = "7230f52f7f1d4566a3f9c3aeb5ffe2ed80302843ce5605853bee1f08098ede46",
            strip_prefix = "oauth2client-4.0.0/oauth2client/",
            type = "tar.gz",
            build_file_content = """
py_library(
   name = "oauth2client",
   srcs = glob(["**/*.py"]),
   visibility = ["//visibility:public"],
   deps = [
     "@httplib2//:httplib2",
     "@six//:six",
   ]
)""",
        )

    # Used for parallel execution in containerregistry
    if "concurrent" not in excludes:
        # TODO(mattmoor): Is there a clean way to override?
        native.new_http_archive(
            name = "concurrent",
            url = "https://codeload.github.com/agronholm/pythonfutures/tar.gz/3.0.5",
            sha256 = "a7086ddf3c36203da7816f7e903ce43d042831f41a9705bc6b4206c574fcb765",
            strip_prefix = "pythonfutures-3.0.5/concurrent/",
            type = "tar.gz",
            build_file_content = """
py_library(
   name = "concurrent",
   srcs = glob(["**/*.py"]),
   visibility = ["//visibility:public"]
)""",
        )

    # For packaging python tools.
    if "subpar" not in excludes:
        native.git_repository(
            name = "subpar",
            remote = "https://github.com/google/subpar.git",
            commit = "7e12cc130eb8f09c8cb02c3585a91a4043753c56",
        )

    if "structure_test" not in excludes:
        native.git_repository(
            name = "structure_test",
            remote = "https://github.com/GoogleCloudPlatform/container-structure-test.git",
            commit = STRUCTURE_TEST_COMMIT,
        )

    # For skylark_library.
    if "bazel_skylib" not in excludes:
        native.git_repository(
            name = "bazel_skylib",
            remote = "https://github.com/bazelbuild/bazel-skylib.git",
            tag = "0.2.0",
        )

    if "gzip" not in excludes:
        local_tool(
            name = "gzip",
        )
<commit_msg>Move container structure test pin to fix entrypoint. (#419)<commit_after># Copyright 2017 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Rules for manipulation container images."""

load("//container:bundle.bzl", "container_bundle")
load("//container:flatten.bzl", "container_flatten")
load("//container:image.bzl", "container_image", "image")
load("//container:layer.bzl", "container_layer")
load("//container:import.bzl", "container_import")
load("//container:load.bzl", "container_load")
load("//container:pull.bzl", "container_pull")
load("//container:push.bzl", "container_push")

container = struct(
    image = image,
)

# The release of the github.com/google/containerregistry to consume.
CONTAINERREGISTRY_RELEASE = "v0.0.26"

# The release of the container-structure-test repository to use.
# Pin to commit that overwrites image entrypoint when running tests.
STRUCTURE_TEST_COMMIT = "7411da1d4cb505f1a70ae5d1ca3d961a3fa66cda"

_local_tool_build_template = """
sh_binary(
    name = "{name}",
    srcs = ["bin/{name}"],
    visibility = ["//visibility:public"],
)
"""

def _local_tool(repository_ctx):
    rctx = repository_ctx
    realpath = rctx.which(rctx.name)
    rctx.symlink(realpath, "bin/%s" % rctx.name)
    rctx.file(
        "WORKSPACE",
        'workspace(name = "{}")\n'.format(rctx.name),
    )
    rctx.file(
        "BUILD",
        _local_tool_build_template.format(name = rctx.name),
    )

local_tool = repository_rule(
    local = True,
    implementation = _local_tool,
)

def repositories():
    """Download dependencies of container rules."""
    excludes = native.existing_rules().keys()

    if "puller" not in excludes:
        native.http_file(
            name = "puller",
            url = ("https://storage.googleapis.com/containerregistry-releases/" +
                   CONTAINERREGISTRY_RELEASE + "/puller.par"),
            sha256 = "42309ba47bb28d1e1b81ef72789dcca396095e191d4f0e49e2e23c297edd27fb",
            executable = True,
        )

    if "importer" not in excludes:
        native.http_file(
            name = "importer",
            url = ("https://storage.googleapis.com/containerregistry-releases/" +
                   CONTAINERREGISTRY_RELEASE + "/importer.par"),
            sha256 = "0a2490584c96bcf961242364d961859b94926182f20a217754730e7097ea6cde",
            executable = True,
        )

    if "containerregistry" not in excludes:
        native.http_archive(
            name = "containerregistry",
            url = ("https://github.com/google/containerregistry/archive/" +
                   CONTAINERREGISTRY_RELEASE + ".tar.gz"),
            strip_prefix = "containerregistry-" + CONTAINERREGISTRY_RELEASE[1:],
        )

    # TODO(mattmoor): Remove all of this (copied from google/containerregistry)
    # once transitive workspace instantiation lands.

    if "httplib2" not in excludes:
        # TODO(mattmoor): Is there a clean way to override?
        native.new_http_archive(
            name = "httplib2",
            url = "https://codeload.github.com/httplib2/httplib2/tar.gz/v0.11.3",
            sha256 = "d9f568c183d1230f271e9c60bd99f3f2b67637c3478c9068fea29f7cca3d911f",
            strip_prefix = "httplib2-0.11.3/python2/httplib2/",
            type = "tar.gz",
            build_file_content = """
py_library(
   name = "httplib2",
   srcs = glob(["**/*.py"]),
   data = ["cacerts.txt"],
   visibility = ["//visibility:public"]
)""",
        )

    # Used by oauth2client
    if "six" not in excludes:
        # TODO(mattmoor): Is there a clean way to override?
        native.new_http_archive(
            name = "six",
            url = "https://pypi.python.org/packages/source/s/six/six-1.9.0.tar.gz",
            sha256 = "e24052411fc4fbd1f672635537c3fc2330d9481b18c0317695b46259512c91d5",
            strip_prefix = "six-1.9.0/",
            type = "tar.gz",
            build_file_content = """
# Rename six.py to __init__.py
genrule(
    name = "rename",
    srcs = ["six.py"],
    outs = ["__init__.py"],
    cmd = "cat $< >$@",
)
py_library(
   name = "six",
   srcs = [":__init__.py"],
   visibility = ["//visibility:public"],
)""",
        )

    # Used for authentication in containerregistry
    if "oauth2client" not in excludes:
        # TODO(mattmoor): Is there a clean way to override?
        native.new_http_archive(
            name = "oauth2client",
            url = "https://codeload.github.com/google/oauth2client/tar.gz/v4.0.0",
            sha256 = "7230f52f7f1d4566a3f9c3aeb5ffe2ed80302843ce5605853bee1f08098ede46",
            strip_prefix = "oauth2client-4.0.0/oauth2client/",
            type = "tar.gz",
            build_file_content = """
py_library(
   name = "oauth2client",
   srcs = glob(["**/*.py"]),
   visibility = ["//visibility:public"],
   deps = [
     "@httplib2//:httplib2",
     "@six//:six",
   ]
)""",
        )

    # Used for parallel execution in containerregistry
    if "concurrent" not in excludes:
        # TODO(mattmoor): Is there a clean way to override?
        native.new_http_archive(
            name = "concurrent",
            url = "https://codeload.github.com/agronholm/pythonfutures/tar.gz/3.0.5",
            sha256 = "a7086ddf3c36203da7816f7e903ce43d042831f41a9705bc6b4206c574fcb765",
            strip_prefix = "pythonfutures-3.0.5/concurrent/",
            type = "tar.gz",
            build_file_content = """
py_library(
   name = "concurrent",
   srcs = glob(["**/*.py"]),
   visibility = ["//visibility:public"]
)""",
        )

    # For packaging python tools.
    if "subpar" not in excludes:
        native.git_repository(
            name = "subpar",
            remote = "https://github.com/google/subpar.git",
            commit = "7e12cc130eb8f09c8cb02c3585a91a4043753c56",
        )

    if "structure_test" not in excludes:
        native.git_repository(
            name = "structure_test",
            remote = "https://github.com/GoogleCloudPlatform/container-structure-test.git",
            commit = STRUCTURE_TEST_COMMIT,
        )

    # For skylark_library.
    if "bazel_skylib" not in excludes:
        native.git_repository(
            name = "bazel_skylib",
            remote = "https://github.com/bazelbuild/bazel-skylib.git",
            tag = "0.2.0",
        )

    if "gzip" not in excludes:
        local_tool(
            name = "gzip",
        )
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function, unicode_literals

import os.path
import textwrap

from error import PluginError

_tests = {
    '>': lambda a, b: a > b,
    '<': lambda a, b: a < b,
    '=': lambda a, b: a == b,
}

_multi = {
    'b': 1,
    'k': 1024,
    'm': 1024**2,
    'g': 1024**3,
}

_cache = {}

def _action(value, name, path):
    global _tests, _multi, _cache

    if not value:
        raise PluginError('missing size')

    if value[0] in ('<', '>'):
        test = _tests[value[0]]
        size = value[1:]
    else:
        test = _tests['=']
        size = value

    if size[-1] in 'bkmg':
        size = int(size[:-1]) * _multi[size[-1]]
    else:
        size = int(size)

    path = os.path.realpath(path)
    try:
        return _cache[path]
    except KeyError:
        _cache[path] = test(os.stat(path).st_size, size)
        return _cache[path]

def action(value, name, path):
    try:
        return _action(value, name, path)
    except PluginError:
        raise
    except:
        import sys
        e = sys.exc_info()[1]
        raise PluginError(e.message)

help = '''Filter files by their size. Size must be given as argument, and must follow pattern:

    operator size multiplier

(without spaces).

Operator (can be omitted) is one of:

    > - file size must be bigger then argument
    < - file size must be lower then argument
    = - must be exactly as argument

Multiplier (can be omitted) is one of:
    b - do not multiply
    k - multiply size with 1024
    m - multiply size with 1024 * 1024
    g - multiply size with 1024 * 1024 * 1024'''.strip()
<commit_msg>fixed recognizing condition '='<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function, unicode_literals

import os.path
import textwrap

from error import PluginError

_tests = {
    '>': lambda a, b: a > b,
    '<': lambda a, b: a < b,
    '=': lambda a, b: a == b,
}

_multi = {
    'b': 1,
    'k': 1024,
    'm': 1024**2,
    'g': 1024**3,
}

_cache = {}

def _action(value, name, path):
    global _tests, _multi, _cache

    if not value:
        raise PluginError('missing size')

    if value[0] in ('<', '>', '='):
        test = _tests[value[0]]
        size = value[1:]
    else:
        test = _tests['=']
        size = value

    if size[-1] in 'bkmg':
        size = int(size[:-1]) * _multi[size[-1]]
    else:
        size = int(size)

    path = os.path.realpath(path)
    try:
        return _cache[path]
    except KeyError:
        _cache[path] = test(os.stat(path).st_size, size)
        return _cache[path]

def action(value, name, path):
    try:
        return _action(value, name, path)
    except PluginError:
        raise
    except:
        import sys
        e = sys.exc_info()[1]
        raise PluginError(e.message)

help = '''Filter files by their size. Size must be given as argument, and must follow pattern:

    operator size multiplier

(without spaces).

Operator (can be omitted) is one of:

    > - file size must be bigger then argument
    < - file size must be lower then argument
    = - must be exactly as argument

Multiplier (can be omitted) is one of:
    b - do not multiply
    k - multiply size with 1024
    m - multiply size with 1024 * 1024
    g - multiply size with 1024 * 1024 * 1024'''.strip()
<|endoftext|>


<commit_before>import time

COMMAND_CONNECT_TAG = "CONNECT_TAG"
COMMAND_BEEP_TAG = "BEEP_TAG"

class Message:
    def __init__(self):
        self.queryTemplate = "?action=%s&tag_mac=%s&gateway_mac=%s&gateway_ip=%s&time=%d"

    def toUrlQuery():
        return ""

class ClientMessage(Message):
    def __init__(self, tag):
        Message.__init__(self)

        self.queryTemplate += "&tag_name=%s&tag_rssi=%s"

class DiscoverTagMessage(ClientMessage):
    def __init__(self, tag):
        ClientMessage.__init__(self, tag)

        self.tag = tag
        self.time = time.time()

    def toUrlQuery(self, gateway_mac, gateway_ip):
        return self.queryTemplate % (
            "TAG_DISCOVERED",
            self.tag.mac,
            gateway_mac,
            gateway_ip,
            self.time,
            self.tag.name,
            self.tag.rssi
        )

class TagConnectedMessage(ClientMessage):
    def __init__(self, mac):
        ClientMessage.__init__(self, tag)

        self.mac = mac
        self.time = time.time()

    def toUrlQuery(self, gateway_mac, gateway_ip):
        return self.queryTemplate % (
            "TAG_CONNECTED",
            self.mac,
            gateway_mac,
            gateway_ip,
            self.time
        )

class TagDisconnectedMessage(ClientMessage):
    def __init__(self, tag):
        ClientMessage.__init__(self, tag)

        self.tag = tag
        self.time = time.time()

    def toUrlQuery(self, gateway_mac, gateway_ip):
        return self.queryTemplate % (
            "TAG_DISCONNECTED",
            self.tag.mac,
            gateway_mac,
            gateway_ip,
            self.time,
            self.tag.rssi,
            self.tag.name
        )

class TagNotificationMessage(ClientMessage):
    def __init__(self, tag, type):
        ClientMessage.__init__(self, tag)

        self.queryTemplate += "&notification_type=%s"

        self.tag = tag
        self.type = type
        self.time = time.time()

    def toUrlQuery(self, gateway_mac, gateway_ip):
        return self.queryTemplate % (
            "TAG_NOTIFICATION",
            self.tag.mac,
            gateway_mac,
            gateway_ip,
            self.time,
            self.tag.rssi,
            self.tag.name,
            self.type
        )

class ConnectToTagCommandMessage(Message):
    def __init__(self, mac):
        Message.__init__(self)
        self.mac = mac

    def toUrlQuery(self, gateway_mac, gateway_ip):
        raise "ERROR, CommandMessage can't be converted into url query"

class BeepTagCommandMessage(Message):
    def __init__(self, mac):
        Message.__init__(self)
        self.mac = mac

    def toUrlQuery(self, gateway_mac, gateway_ip):
        raise "ERROR, CommandMessage can't be converted into url query"
<commit_msg>fixed mac issue<commit_after>import time

COMMAND_CONNECT_TAG = "CONNECT_TAG"
COMMAND_BEEP_TAG = "BEEP_TAG"

class Message:
    def __init__(self):
        self.queryTemplate = "?action=%s&tag_mac=%s&gateway_mac=%s&gateway_ip=%s&time=%d"

    def toUrlQuery():
        return ""

class ClientMessage(Message):
    def __init__(self, tag):
        Message.__init__(self)

        self.queryTemplate += "&tag_name=%s&tag_rssi=%s"

class DiscoverTagMessage(ClientMessage):
    def __init__(self, tag):
        ClientMessage.__init__(self, tag)

        self.tag = tag
        self.time = time.time()

    def toUrlQuery(self, gateway_mac, gateway_ip):
        return self.queryTemplate % (
            "TAG_DISCOVERED",
            self.tag.mac,
            gateway_mac,
            gateway_ip,
            self.time,
            self.tag.name,
            self.tag.rssi
        )

class TagConnectedMessage(ClientMessage):
    def __init__(self, tag):
        ClientMessage.__init__(self, tag)

        self.tag = tag
        self.time = time.time()

    def toUrlQuery(self, gateway_mac, gateway_ip):
        return self.queryTemplate % (
            "TAG_CONNECTED",
            self.tag.mac,
            gateway_mac,
            gateway_ip,
            self.time,
            self.tag.name,
            self.tag.rssi
        )

class TagDisconnectedMessage(ClientMessage):
    def __init__(self, tag):
        ClientMessage.__init__(self, tag)

        self.tag = tag
        self.time = time.time()

    def toUrlQuery(self, gateway_mac, gateway_ip):
        return self.queryTemplate % (
            "TAG_DISCONNECTED",
            self.tag.mac,
            gateway_mac,
            gateway_ip,
            self.time,
            self.tag.name,
            self.tag.rssi
        )

class TagNotificationMessage(ClientMessage):
    def __init__(self, tag, type):
        ClientMessage.__init__(self, tag)

        self.queryTemplate += "&notification_type=%s"

        self.tag = tag
        self.type = type
        self.time = time.time()

    def toUrlQuery(self, gateway_mac, gateway_ip):
        return self.queryTemplate % (
            "TAG_NOTIFICATION",
            self.tag.mac,
            gateway_mac,
            gateway_ip,
            self.time,
            self.tag.name,
            self.tag.rssi,
            self.type
        )

class ConnectToTagCommandMessage(Message):
    def __init__(self, mac):
        Message.__init__(self)
        self.mac = mac

    def toUrlQuery(self, gateway_mac, gateway_ip):
        raise "ERROR, CommandMessage can't be converted into url query"

class BeepTagCommandMessage(Message):
    def __init__(self, mac):
        Message.__init__(self)
        self.mac = mac

    def toUrlQuery(self, gateway_mac, gateway_ip):
        raise "ERROR, CommandMessage can't be converted into url query"
<|endoftext|>


<commit_before>from django.conf import settings
from django.core.validators import validate_email
from django.core.exceptions import ValidationError
from django.template import RequestContext, Context
from django.template import loader
from django.http import HttpResponseServerError, HttpResponseNotFound
from django.core.mail import EmailMessage

import django

def validateEmail(email):
    '''
    simple function to verify an email with django's validate_email validator
    '''
    try:
        validate_email(email)
        return True
    except ValidationError:
        return False

def send_mail(
    request, recipients, subject, femail, template, data, bcc=None,
    content='html', attach=False
):

    if not bcc:
        bcc = [settings.MANAGERS[0][1],]

    t = loader.get_template(template)
    if request:
        rendered = t.render({'data':data,}, request)
    else:
        rendered = t.render({'data':data,},)

    headers = {'Reply-To': femail,'From': femail,}

    email = EmailMessage(
        subject, rendered, femail, recipients, bcc, headers=headers
    )

    email.encoding = 'utf-8'

    if content:
        email.content_subtype = content

    if attach:
        try:
            email.attach_file(attach)
        except:
            for field, value in request.FILES.items():
                email.attach(value.name, value.read(), value.content_type)

    status = False
    # try 5 times then quit
    count = 0
    while True:
        try:
            email.send(fail_silently=False)
            status = True
            break
        except:
            count += 1
            if count < 5:
                pass
            else:
                break

    return status
<commit_msg>added logging to send_mail() function in case of error<commit_after>from django.conf import settings
from django.core.validators import validate_email
from django.core.exceptions import ValidationError
from django.template import RequestContext, Context
from django.template import loader
from django.http import HttpResponseServerError, HttpResponseNotFound
from django.core.mail import EmailMessage

import django
import logging
logger = logging.getLogger(__name__)

def validateEmail(email):
    '''
    simple function to verify an email with django's validate_email validator
    '''
    try:
        validate_email(email)
        return True
    except ValidationError:
        return False

def send_mail(
    request, recipients, subject, femail, template, data, bcc=None,
    content='html', attach=False
):

    if not bcc:
        bcc = [settings.MANAGERS[0][1],]

    t = loader.get_template(template)
    if request:
        rendered = t.render({'data':data,}, request)
    else:
        rendered = t.render({'data':data,},)

    headers = {'Reply-To': femail,'From': femail,}

    email = EmailMessage(
        subject, rendered, femail, recipients, bcc, headers=headers
    )

    email.encoding = 'utf-8'

    if content:
        email.content_subtype = content

    if attach:
        try:
            email.attach_file(attach)
        except:
            for field, value in request.FILES.items():
                email.attach(value.name, value.read(), value.content_type)

    status = False
    # try 5 times then quit
    count = 0
    while True:
        try:
            email.send(fail_silently=False)
            status = True
            break
        except Exception as e:
            logger.debug(e)
            count += 1
            if count < 5:
                pass
            else:
                break

    return status
<|endoftext|>