<commit_before>import cybox
import cybox.bindings.cybox_core as core_binding
from cybox.common import ObjectProperties, StructuredText
from cybox.core import Object

class Observable(cybox.Entity):
    """A single Observable.
    """

    def __init__(self, item=None, id_=None):
        """Create an Observable out of 'item'.

        `item` can be any of:
        - an Object
        - an ObservableComposition
        - any subclass of ObjectProperties.

        In the first two cases, the appropriate property of the Observable will
        be set. In the last cases, an Object will be built automatically to
        ensure the correct hierarchy is created.
        """
        if not id_:
            id_ = cybox.utils.create_id(prefix="Observable")

        self.id_ = id_
        self.title = None
        self.description = None

        self.object_ = None
        self.observable_composition = None

        if not item:
            return

        if isinstance(item, Object):
            self.object_ = item
        elif isinstance(item, ObservableComposition):
            self.observable_composition = item
        elif isinstance(item, ObjectProperties):
            self.object_ = Object(item)

    @property
    def object_(self):
        return self._object

    @object_.setter
    def object_(self, value):
        if value:
            if self.observable_composition:
                msg = 'Observable already has an ObservableComposition.'
                raise ValueError(msg)
            if not isinstance(value, Object):
                raise TypeError('value must be a Object')

        self._object = value

    @property
    def observable_composition(self):
        return self._observable_composition

    @observable_composition.setter
    def observable_composition(self, value):
        if value:
            if self.object_:
                raise ValueError('Observable already has an Object.')
            if not isinstance(value, ObservableComposition):
                raise TypeError('value must be an ObservableComposition')

        self._observable_composition = value

    @property
    def description(self):
        return self._description

    @description.setter
    def description(self, value):
        if value is not None and not isinstance(value, StructuredText):
            value = StructuredText(value)
        self._description = value

    def to_obj(self):
        obs_obj = core_binding.ObservableType()

        obs_obj.set_id(self.id_)
        if self.title is not None:
            obs_obj.set_Title(self.title)
        if self.description is not None:
            obs_obj.set_Description(self.description.to_obj())
        if self.object_:
            obs_obj.set_Object(self.object_.to_obj())
        if self.observable_composition:
            obs_obj.set_Observable_Composition(self.observable_composition.to_obj())

        return obs_obj

    def to_dict(self):
        obs_dict = {}

        obs_dict['id'] = self.id_
        if self.title is not None:
            obs_dict['title'] = self.title
        if self.description is not None:
            obs_dict['description'] = self.description.to_dict()
        if self.object_:
            obs_dict['object'] = self.object_.to_dict()
        if self.observable_composition:
            obs_dict['observable_composition'] = self.observable_composition.to_dict()

        return obs_dict

    @staticmethod
    def from_obj(observable_obj):
        if not observable_obj:
            return None

        obs = Observable()

        obs.id_ = observable_obj.get_id()
        obs.title = observable_obj.get_Title()
        obs.description = StructuredText.from_obj(observable_obj.get_Description())
        obs.object_ = Object.from_obj(observable_obj.get_Object())
        obs.observable_composition = ObservableComposition.from_obj(observable_obj.get_Observable_Composition())

        return obs

    @staticmethod
    def from_dict(observable_dict):
        if not observable_dict:
            return None

        obs = Observable()

        obs.id_ = observable_dict.get('id')
        obs.title = observable_dict.get('title')
        obs.description = StructuredText.from_dict(observable_dict.get('description'))
        obs.object_ = Object.from_dict(observable_dict.get('object'))
        obs.observable_composition = ObservableComposition.from_dict(observable_dict.get('observable_composition'))

        return obs

#    @classmethod
#    def dict_from_object(cls, observable_obj):
#        """Parse the observable into a dictionary-esque representation"""
#        observable_dict = {}
#        if observable.get_id() is not None:
#            observable_dict['id'] = observable.get_id()
#        if observable.get_idref() is not None:
#            observable_dict['idref'] = observable.get_idref()
#        if observable.get_Title() is not None:
#            observable_dict['title'] = observable.get_Title()
#        if observable.get_Description() is not None:
#            observable_dict['description'] = structured_text.dict_from_object(observable.get_Description())
#        #TODO - add rest of observable components
#        return observable_dict

class Observables(cybox.Entity):
    """The root CybOX Observables object.

    Observable_Package_Source and Pools are not currently supported.
    """

    def __init__(self, observables=None):
        # Assume major_verion and minor_version are immutable for now
        self._major_version = 2
        self._minor_version = 0
        self.observables = []

        try:
            for obs in observables:
                self.add(obs)
        except TypeError:
            # A single observable
            self.add(observables)

    def add(self, observable):
        if not observable:
            return
        if not isinstance(observable, Observable):
            observable = Observable(observable)
        self.observables.append(observable)

    def to_obj(self):
        observable_list = [x.to_obj() for x in self.observables]
        return core_binding.ObservablesType(
                                cybox_major_version=self._major_version,
                                cybox_minor_version=self._minor_version,
                                Observable=observable_list)

    def to_dict(self):
        return {
                    'major_version': self._major_version,
                    'minor_version': self._minor_version,
                    'observables': [x.to_dict() for x in self.observables]
               }

    @staticmethod
    def from_obj(observables_obj):

        #TODO: look at major_version and minor_version
        obs = Observables()
        # get_Observable() actually returns a list
        for o in observables_obj.get_Observable():
            obs.add(Observable.from_obj(o))

        return obs

    @staticmethod
    def from_dict(observables_dict):
        #TODO: look at major_version and minor_version
        obs = Observables()
        for o in observables_dict.get("observables", []):
            obs.add(Observable.from_dict(o))

        return obs

class ObservableComposition(cybox.Entity):
    '''The ObservableCompositionType entity defines a logical compositions of
    CybOX Observables. The combinatorial behavior is derived from the operator
    property.'''

    OPERATOR_AND = 'AND'
    OPERATOR_OR = 'OR'
    OPERATORS = (OPERATOR_AND, OPERATOR_OR)

    def __init__(self, operator='AND', observables=None):
        self.operator = operator
        self.observables = []

        if observables:
            try:
                for obs in observables:
                    self.add(obs)
            except TypeError as t:
                # A single observable
                self.add(observables)

    @property
    def operator(self):
        return self._operator

    @operator.setter
    def operator(self, value):
        if value not in self.OPERATORS:
            raise ValueError('value must be one of: %s' % ' '.join(self.OPERATORS) )

        self._operator = value

    def add(self, observable):
        if not observable:
            raise ValueError("'observable' must not be None")
        if not isinstance(observable, Observable):
            observable = Observable(observable)
        self.observables.append(observable)

    def to_obj(self):
        observable_list = [x.to_obj() for x in self.observables]
        return core_binding.ObservableCompositionType(
                                operator = self._operator,
                                Observable=observable_list)

    def to_dict(self):
        return {
                    'operator': self._operator,
                    'observables': [x.to_dict() for x in self.observables]
               }

    @staticmethod
    def from_obj(observable_comp_obj):
        if not observable_comp_obj:
            return None

        obs_comp = ObservableComposition()
        # get_Observable() actually returns a list
        for o in observable_comp_obj.get_Observable():
            obs_comp.add(Observable.from_obj(o))

        return obs_comp

    @staticmethod
    def from_dict(observable_comp_dict):
        if not observable_comp_dict:
            return None

        obs_comp = ObservableComposition()
        for o in observable_comp_dict.get("observables", []):
            obs_comp.add(Observable.from_dict(o))

        return obs_comp

<commit_msg>Fix creating obserservable out of ObjectProperties that already have a parent Object<commit_after>import cybox
import cybox.bindings.cybox_core as core_binding
from cybox.common import ObjectProperties, StructuredText
from cybox.core import Object

class Observable(cybox.Entity):
    """A single Observable.
    """

    def __init__(self, item=None, id_=None):
        """Create an Observable out of 'item'.

        `item` can be any of:
        - an Object
        - an ObservableComposition
        - any subclass of ObjectProperties.

        In the first two cases, the appropriate property of the Observable will
        be set. In the last cases, an Object will be built automatically to
        ensure the correct hierarchy is created.
        """
        if not id_:
            id_ = cybox.utils.create_id(prefix="Observable")

        self.id_ = id_
        self.title = None
        self.description = None

        self.object_ = None
        self.observable_composition = None

        if not item:
            return

        if isinstance(item, Object):
            self.object_ = item
        elif isinstance(item, ObservableComposition):
            self.observable_composition = item
        elif isinstance(item, ObjectProperties):
            if item.parent:
                self.object_ = item.parent
            else:
                self.object_ = Object(item)

    @property
    def object_(self):
        return self._object

    @object_.setter
    def object_(self, value):
        if value:
            if self.observable_composition:
                msg = 'Observable already has an ObservableComposition.'
                raise ValueError(msg)
            if not isinstance(value, Object):
                raise TypeError('value must be a Object')

        self._object = value

    @property
    def observable_composition(self):
        return self._observable_composition

    @observable_composition.setter
    def observable_composition(self, value):
        if value:
            if self.object_:
                raise ValueError('Observable already has an Object.')
            if not isinstance(value, ObservableComposition):
                raise TypeError('value must be an ObservableComposition')

        self._observable_composition = value

    @property
    def description(self):
        return self._description

    @description.setter
    def description(self, value):
        if value is not None and not isinstance(value, StructuredText):
            value = StructuredText(value)
        self._description = value

    def to_obj(self):
        obs_obj = core_binding.ObservableType()

        obs_obj.set_id(self.id_)
        if self.title is not None:
            obs_obj.set_Title(self.title)
        if self.description is not None:
            obs_obj.set_Description(self.description.to_obj())
        if self.object_:
            obs_obj.set_Object(self.object_.to_obj())
        if self.observable_composition:
            obs_obj.set_Observable_Composition(self.observable_composition.to_obj())

        return obs_obj

    def to_dict(self):
        obs_dict = {}

        obs_dict['id'] = self.id_
        if self.title is not None:
            obs_dict['title'] = self.title
        if self.description is not None:
            obs_dict['description'] = self.description.to_dict()
        if self.object_:
            obs_dict['object'] = self.object_.to_dict()
        if self.observable_composition:
            obs_dict['observable_composition'] = self.observable_composition.to_dict()

        return obs_dict

    @staticmethod
    def from_obj(observable_obj):
        if not observable_obj:
            return None

        obs = Observable()

        obs.id_ = observable_obj.get_id()
        obs.title = observable_obj.get_Title()
        obs.description = StructuredText.from_obj(observable_obj.get_Description())
        obs.object_ = Object.from_obj(observable_obj.get_Object())
        obs.observable_composition = ObservableComposition.from_obj(observable_obj.get_Observable_Composition())

        return obs

    @staticmethod
    def from_dict(observable_dict):
        if not observable_dict:
            return None

        obs = Observable()

        obs.id_ = observable_dict.get('id')
        obs.title = observable_dict.get('title')
        obs.description = StructuredText.from_dict(observable_dict.get('description'))
        obs.object_ = Object.from_dict(observable_dict.get('object'))
        obs.observable_composition = ObservableComposition.from_dict(observable_dict.get('observable_composition'))

        return obs

#    @classmethod
#    def dict_from_object(cls, observable_obj):
#        """Parse the observable into a dictionary-esque representation"""
#        observable_dict = {}
#        if observable.get_id() is not None:
#            observable_dict['id'] = observable.get_id()
#        if observable.get_idref() is not None:
#            observable_dict['idref'] = observable.get_idref()
#        if observable.get_Title() is not None:
#            observable_dict['title'] = observable.get_Title()
#        if observable.get_Description() is not None:
#            observable_dict['description'] = structured_text.dict_from_object(observable.get_Description())
#        #TODO - add rest of observable components
#        return observable_dict

class Observables(cybox.Entity):
    """The root CybOX Observables object.

    Observable_Package_Source and Pools are not currently supported.
    """

    def __init__(self, observables=None):
        # Assume major_verion and minor_version are immutable for now
        self._major_version = 2
        self._minor_version = 0
        self.observables = []

        try:
            for obs in observables:
                self.add(obs)
        except TypeError:
            # A single observable
            self.add(observables)

    def add(self, observable):
        if not observable:
            return
        if not isinstance(observable, Observable):
            observable = Observable(observable)
        self.observables.append(observable)

    def to_obj(self):
        observable_list = [x.to_obj() for x in self.observables]
        return core_binding.ObservablesType(
                                cybox_major_version=self._major_version,
                                cybox_minor_version=self._minor_version,
                                Observable=observable_list)

    def to_dict(self):
        return {
                    'major_version': self._major_version,
                    'minor_version': self._minor_version,
                    'observables': [x.to_dict() for x in self.observables]
               }

    @staticmethod
    def from_obj(observables_obj):

        #TODO: look at major_version and minor_version
        obs = Observables()
        # get_Observable() actually returns a list
        for o in observables_obj.get_Observable():
            obs.add(Observable.from_obj(o))

        return obs

    @staticmethod
    def from_dict(observables_dict):
        #TODO: look at major_version and minor_version
        obs = Observables()
        for o in observables_dict.get("observables", []):
            obs.add(Observable.from_dict(o))

        return obs

class ObservableComposition(cybox.Entity):
    '''The ObservableCompositionType entity defines a logical compositions of
    CybOX Observables. The combinatorial behavior is derived from the operator
    property.'''

    OPERATOR_AND = 'AND'
    OPERATOR_OR = 'OR'
    OPERATORS = (OPERATOR_AND, OPERATOR_OR)

    def __init__(self, operator='AND', observables=None):
        self.operator = operator
        self.observables = []

        if observables:
            try:
                for obs in observables:
                    self.add(obs)
            except TypeError as t:
                # A single observable
                self.add(observables)

    @property
    def operator(self):
        return self._operator

    @operator.setter
    def operator(self, value):
        if value not in self.OPERATORS:
            raise ValueError('value must be one of: %s' % ' '.join(self.OPERATORS) )

        self._operator = value

    def add(self, observable):
        if not observable:
            raise ValueError("'observable' must not be None")
        if not isinstance(observable, Observable):
            observable = Observable(observable)
        self.observables.append(observable)

    def to_obj(self):
        observable_list = [x.to_obj() for x in self.observables]
        return core_binding.ObservableCompositionType(
                                operator = self._operator,
                                Observable=observable_list)

    def to_dict(self):
        return {
                    'operator': self._operator,
                    'observables': [x.to_dict() for x in self.observables]
               }

    @staticmethod
    def from_obj(observable_comp_obj):
        if not observable_comp_obj:
            return None

        obs_comp = ObservableComposition()
        # get_Observable() actually returns a list
        for o in observable_comp_obj.get_Observable():
            obs_comp.add(Observable.from_obj(o))

        return obs_comp

    @staticmethod
    def from_dict(observable_comp_dict):
        if not observable_comp_dict:
            return None

        obs_comp = ObservableComposition()
        for o in observable_comp_dict.get("observables", []):
            obs_comp.add(Observable.from_dict(o))

        return obs_comp

<|endoftext|>


<commit_before>from django.conf.urls import url

from announcements.views import detail, dismiss
from announcements.views import CreateAnnouncementView, UpdateAnnouncementView
from announcements.views import DeleteAnnouncementView, AnnouncementListView

urlpatterns = [
    url(r"^$", AnnouncementListView.as_view(), name="announcements_list"),
    url(r"^announcement/create/$", CreateAnnouncementView.as_view(), name="announcements_create"),
    url(r"^announcement/(?P<pk>\d+)/$", detail, name="announcements_detail"),
    url(r"^announcement/(?P<pk>\d+)/hide/$", dismiss, name="announcements_dismiss"),
    url(r"^announcement/(?P<pk>\d+)/update/$", UpdateAnnouncementView.as_view(),
        name="announcements_update"),
    url(r"^announcement/(?P<pk>\d+)/delete/$", DeleteAnnouncementView.as_view(),
        name="announcements_delete"),
]
<commit_msg>Lint issue<commit_after>from django.conf.urls import url

from announcements.views import detail, dismiss
from announcements.views import CreateAnnouncementView, UpdateAnnouncementView
from announcements.views import DeleteAnnouncementView, AnnouncementListView

urlpatterns = [
    url(r"^$", AnnouncementListView.as_view(), name="announcements_list"),
    url(r"^announcement/create/$", CreateAnnouncementView.as_view(), name="announcements_create"),
    url(r"^announcement/(?P<pk>\d+)/$", detail, name="announcements_detail"),
    url(r"^announcement/(?P<pk>\d+)/hide/$", dismiss, name="announcements_dismiss"),
    url(r"^announcement/(?P<pk>\d+)/update/$", UpdateAnnouncementView.as_view(),
        name="announcements_update"),
    url(r"^announcement/(?P<pk>\d+)/delete/$", DeleteAnnouncementView.as_view(),
        name="announcements_delete"),
]
<|endoftext|>


<commit_before>#!/usr/bin/python
# arguments are:
#   - cluster scope
#   - cluster role
#   - master connection string

# for the AWS, the folliowing environment variables should be defined:
# - WALE_ENV_DIR: directory where WAL-E environment is kept
# - WAL_S3_BUCKET: a name of the S3 bucket for WAL-E
# - WALE_BACKUP_THRESHOLD_MEGABYTES if WAL amount is above that - use pg_basebackup
# - WALE_BACKUP_THRESHOLD_PERCENTAGE if WAL size exceeds a certain percentage of the
#   latest backup size
from collections import namedtuple
import logging
import os
import psycopg2
import subprocess
import sys

logger = logging.getLogger(__name__)

class Restore(object):

    def __init__(self, scope, role, datadir, connstring):
        self.scope = scope
        self.role = role
        self.master_connection = Restore.parse_connstring(connstring)
        self.data_dir = datadir
        self.env = os.environ.copy()

    @staticmethod
    def parse_connstring(connstring):
        # the connection string is in the form host= port= user=
        # return the dictionary with all components as separare keys
        result = {}
        if connstring:
            for x in connstring.split():
                if x and '=' in x:
                    key, val = x.split('=')
                result[key.strip()] = val.strip()
        return result

    def replica_method(self):
        return self.create_replica_with_pg_basebackup

    def replica_fallback_method(self):
        return None

    def run(self):
        """ creates a new replica using either pg_basebackup or WAL-E """
        method_fn = self.replica_method()
        ret = method_fn()
        if ret != 0 and self.replica_fallback_method() is not None:
            ret = (self.replica_fallback_method())()
        return ret

    def create_replica_with_pg_basebackup(self):
        ret = subprocess.call(['pg_basebackup', '-R', '-D',
                               self.data_dir, '--host=' + self.master_connection['host'],
                               '--port=' + str(self.master_connection['port']),
                               '-U', self.master_connection['user']],
                              env=self.env)
        return ret

class WALERestore(Restore):

    def __init__(self, scope, role, datadir, connstring):
        super(WALERestore, self).__init__(scope, role, datadir, connstring)
        # check the environment variables
        self.init_error = False
        if (self.env.get('WAL_S3_BUCKET') and
            self.env.get('WALE_BACKUP_THRESHOLD_PERCENTAGE') and
           self.env.get('WALE_BACKUP_THRESHOLD_MEGABYTES')) is None:
                self.init_error = True
        else:
            self.wal_e = namedtuple('WALE', 'threshold_megabytes', 'threshold_backup_size_percentage',
                                    's3_bucket', 'cmd', 'dir', 'env_file')

            self.wal_e.dir = self.env.get('WALE_ENV_DIR', '/home/postgres/etc/wal-e.d/env')
            self.wal_e.env_file = os.path.join(self.wal_e.dir, 'WALE_S3_PREFIX')

            self.wal_e.cmd = 'envdir {} wal-e --aws-instance-profile '.\
                format(self.wal_e.dir)
            self.wal_e.s3_bucket = self.env['WAL_S3_BUCKET']
            self.wal_e.threshold_megabytes = self.env['WALE_BACKUP_THRESHOLD_MEGABYTES']
            self.wal_e.threshold_backup_size_percentage = self.env['WALE_BACKUP_THRESHOLD_PERCENTAGE']

            # check that the env file exists, create it otherwise
            try:
                if not os.path.exists(self.wal_e.dir):
                    os.makedirs(self.wal_e.dir)
                # if this is a directory - make sure we have full access there
                elif not (os.path.isdir(self.wal_e.dir) and os.access(self.wal_e.dir, os.R_OK | os.W_OK | os.X_OK)):
                    logger.error("Unable to access {} or not a directory".format(self.wal_e.dir))
                    self.init_error = True
                # if WAL_S3_PREFIX is not there - create it and write the full path to bucket
                if not self.init_error and not os.path.exists(self.wal_e.env_file):
                    with open(self.wal_e.env_file, 'w') as f:
                        f.write("s3://{0}/spilo/{1}/wal/\n".format(self.wal_e.s3_bucket, self.scope))

            except (os.error, IOError) as e:
                logger.error("{0}: WAL-e archiving is disabled".format(e))
                self.init_error = True

    def replica_method(self):
        if self.should_use_s3_to_create_replica(self):
            return self.create_replica_with_s3
        return 1

    def replica_fallback_method(self):
        return self.create_replica_with_pg_basebackup

    def should_use_s3_to_create_replica(self):
        """ determine whether it makes sense to use S3 and not pg_basebackup """
        if self.init_error:
            return False

        threshold_megabytes = self.wal_e.threshold_megabytes
        threshold_backup_size_percentage = self.wal_e.threshold_backup_size_percentage

        try:
            latest_backup = subprocess.check_output(self.wal_e.cmd.split() + ['backup-list', '--detail', 'LATEST'], env=self.env)
            # name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start
            #                                                   wal_segment_backup_stop wal_segment_offset_backup_stop
            # base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z
            # 20310671    00000001000000000000007F    00000040
            # 00000001000000000000007F    00000240
            backup_strings = latest_backup.splitlines() if latest_backup else ()
            if len(backup_strings) != 2:
                return False

            names = backup_strings[0].split()
            vals = backup_strings[1].split()
            if (len(names) != len(vals)) or (len(names) != 7):
                return False

            backup_info = dict(zip(names, vals))
        except subprocess.CalledProcessError as e:
            logger.error("could not query wal-e latest backup: {}".format(e))
            return False

        try:
            backup_size = backup_info['expanded_size_bytes']
            backup_start_segment = backup_info['wal_segment_backup_start']
            backup_start_offset = backup_info['wal_segment_offset_backup_start']
        except Exception as e:
            logger.error("unable to get some of S3 backup parameters: {}".format(e))
            return False

        # WAL filename is XXXXXXXXYYYYYYYY000000ZZ, where X - timeline, Y - LSN logical log file,
        # ZZ - 2 high digits of LSN offset. The rest of the offset is the provided decimal offset,
        # that we have to convert to hex and 'prepend' to the high offset digits.

        lsn_segment = backup_start_segment[8:16]
        # first 2 characters of the result are 0x and the last one is L
        lsn_offset = hex((long(backup_start_segment[16:32], 16) << 24) + long(backup_start_offset))[2:-1]

        # construct the LSN from the segment and offset
        backup_start_lsn = '{}/{}'.format(lsn_segment, lsn_offset)

        conn = None
        cursor = None
        diff_in_bytes = long(backup_size)
        try:
            # get the difference in bytes between the current WAL location and the backup start offset
            conn = psycopg2.connect(**(self.master_connection))
            conn.autocommit = True
            cursor = conn.cursor()
            cursor.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)", (backup_start_lsn,))
            diff_in_bytes = long(cursor.fetchone()[0])
        except psycopg2.Error as e:
            logger.error('could not determine difference with the master location: {}'.format(e))
            return False
        finally:
            cursor and cursor.close()
            conn and conn.close()

        # if the size of the accumulated WAL segments is more than a certan percentage of the backup size
        # or exceeds the pre-determined size - pg_basebackup is chosen instead.
        return (diff_in_bytes < long(threshold_megabytes) * 1048576) and\
               (diff_in_bytes < long(backup_size) * float(threshold_backup_size_percentage) / 100)

    def create_replica_with_s3(self):
        if self.init_error:
            return 1

        ret = subprocess.call(self.wal_e.cmd + ' backup-fetch {} LATEST'.format(self.data_dir), env=self.env)
        self.restore_configuration_files()
        return ret

if __name__ == '__main__':
    if len(sys.argv) == 5:
        # scope, role, datadir, connstring
        restore = WALERestore(*(sys.argv[1:]))
        sys.exit(restore.run())
    sys.exit("Usage: {0} scope role datadir connstring".format(sys.argv[0]))
<commit_msg>fix a typo in the  named tuple definition.<commit_after>#!/usr/bin/python
# arguments are:
#   - cluster scope
#   - cluster role
#   - master connection string

# for the AWS, the folliowing environment variables should be defined:
# - WALE_ENV_DIR: directory where WAL-E environment is kept
# - WAL_S3_BUCKET: a name of the S3 bucket for WAL-E
# - WALE_BACKUP_THRESHOLD_MEGABYTES if WAL amount is above that - use pg_basebackup
# - WALE_BACKUP_THRESHOLD_PERCENTAGE if WAL size exceeds a certain percentage of the
#   latest backup size
from collections import namedtuple
import logging
import os
import psycopg2
import subprocess
import sys

logger = logging.getLogger(__name__)

class Restore(object):

    def __init__(self, scope, role, datadir, connstring):
        self.scope = scope
        self.role = role
        self.master_connection = Restore.parse_connstring(connstring)
        self.data_dir = datadir
        self.env = os.environ.copy()

    @staticmethod
    def parse_connstring(connstring):
        # the connection string is in the form host= port= user=
        # return the dictionary with all components as separare keys
        result = {}
        if connstring:
            for x in connstring.split():
                if x and '=' in x:
                    key, val = x.split('=')
                result[key.strip()] = val.strip()
        return result

    def replica_method(self):
        return self.create_replica_with_pg_basebackup

    def replica_fallback_method(self):
        return None

    def run(self):
        """ creates a new replica using either pg_basebackup or WAL-E """
        method_fn = self.replica_method()
        ret = method_fn()
        if ret != 0 and self.replica_fallback_method() is not None:
            ret = (self.replica_fallback_method())()
        return ret

    def create_replica_with_pg_basebackup(self):
        ret = subprocess.call(['pg_basebackup', '-R', '-D',
                               self.data_dir, '--host=' + self.master_connection['host'],
                               '--port=' + str(self.master_connection['port']),
                               '-U', self.master_connection['user']],
                              env=self.env)
        return ret

class WALERestore(Restore):

    def __init__(self, scope, role, datadir, connstring):
        super(WALERestore, self).__init__(scope, role, datadir, connstring)
        # check the environment variables
        self.init_error = False
        if (self.env.get('WAL_S3_BUCKET') and
            self.env.get('WALE_BACKUP_THRESHOLD_PERCENTAGE') and
           self.env.get('WALE_BACKUP_THRESHOLD_MEGABYTES')) is None:
                self.init_error = True
        else:
            self.wal_e = namedtuple('WALE',
                                    'threshold_megabytes threshold_backup_size_percentage s3_bucket cmd dir env_file')

            self.wal_e.dir = self.env.get('WALE_ENV_DIR', '/home/postgres/etc/wal-e.d/env')
            self.wal_e.env_file = os.path.join(self.wal_e.dir, 'WALE_S3_PREFIX')

            self.wal_e.cmd = 'envdir {} wal-e --aws-instance-profile '.\
                format(self.wal_e.dir)
            self.wal_e.s3_bucket = self.env['WAL_S3_BUCKET']
            self.wal_e.threshold_megabytes = self.env['WALE_BACKUP_THRESHOLD_MEGABYTES']
            self.wal_e.threshold_backup_size_percentage = self.env['WALE_BACKUP_THRESHOLD_PERCENTAGE']

            # check that the env file exists, create it otherwise
            try:
                if not os.path.exists(self.wal_e.dir):
                    os.makedirs(self.wal_e.dir)
                # if this is a directory - make sure we have full access there
                elif not (os.path.isdir(self.wal_e.dir) and os.access(self.wal_e.dir, os.R_OK | os.W_OK | os.X_OK)):
                    logger.error("Unable to access {} or not a directory".format(self.wal_e.dir))
                    self.init_error = True
                # if WAL_S3_PREFIX is not there - create it and write the full path to bucket
                if not self.init_error and not os.path.exists(self.wal_e.env_file):
                    with open(self.wal_e.env_file, 'w') as f:
                        f.write("s3://{0}/spilo/{1}/wal/\n".format(self.wal_e.s3_bucket, self.scope))

            except (os.error, IOError) as e:
                logger.error("{0}: WAL-e archiving is disabled".format(e))
                self.init_error = True

    def replica_method(self):
        if self.should_use_s3_to_create_replica(self):
            return self.create_replica_with_s3
        return 1

    def replica_fallback_method(self):
        return self.create_replica_with_pg_basebackup

    def should_use_s3_to_create_replica(self):
        """ determine whether it makes sense to use S3 and not pg_basebackup """
        if self.init_error:
            return False

        threshold_megabytes = self.wal_e.threshold_megabytes
        threshold_backup_size_percentage = self.wal_e.threshold_backup_size_percentage

        try:
            latest_backup = subprocess.check_output(self.wal_e.cmd.split() + ['backup-list', '--detail', 'LATEST'], env=self.env)
            # name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start
            #                                                   wal_segment_backup_stop wal_segment_offset_backup_stop
            # base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z
            # 20310671    00000001000000000000007F    00000040
            # 00000001000000000000007F    00000240
            backup_strings = latest_backup.splitlines() if latest_backup else ()
            if len(backup_strings) != 2:
                return False

            names = backup_strings[0].split()
            vals = backup_strings[1].split()
            if (len(names) != len(vals)) or (len(names) != 7):
                return False

            backup_info = dict(zip(names, vals))
        except subprocess.CalledProcessError as e:
            logger.error("could not query wal-e latest backup: {}".format(e))
            return False

        try:
            backup_size = backup_info['expanded_size_bytes']
            backup_start_segment = backup_info['wal_segment_backup_start']
            backup_start_offset = backup_info['wal_segment_offset_backup_start']
        except Exception as e:
            logger.error("unable to get some of S3 backup parameters: {}".format(e))
            return False

        # WAL filename is XXXXXXXXYYYYYYYY000000ZZ, where X - timeline, Y - LSN logical log file,
        # ZZ - 2 high digits of LSN offset. The rest of the offset is the provided decimal offset,
        # that we have to convert to hex and 'prepend' to the high offset digits.

        lsn_segment = backup_start_segment[8:16]
        # first 2 characters of the result are 0x and the last one is L
        lsn_offset = hex((long(backup_start_segment[16:32], 16) << 24) + long(backup_start_offset))[2:-1]

        # construct the LSN from the segment and offset
        backup_start_lsn = '{}/{}'.format(lsn_segment, lsn_offset)

        conn = None
        cursor = None
        diff_in_bytes = long(backup_size)
        try:
            # get the difference in bytes between the current WAL location and the backup start offset
            conn = psycopg2.connect(**(self.master_connection))
            conn.autocommit = True
            cursor = conn.cursor()
            cursor.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)", (backup_start_lsn,))
            diff_in_bytes = long(cursor.fetchone()[0])
        except psycopg2.Error as e:
            logger.error('could not determine difference with the master location: {}'.format(e))
            return False
        finally:
            cursor and cursor.close()
            conn and conn.close()

        # if the size of the accumulated WAL segments is more than a certan percentage of the backup size
        # or exceeds the pre-determined size - pg_basebackup is chosen instead.
        return (diff_in_bytes < long(threshold_megabytes) * 1048576) and\
               (diff_in_bytes < long(backup_size) * float(threshold_backup_size_percentage) / 100)

    def create_replica_with_s3(self):
        if self.init_error:
            return 1

        ret = subprocess.call(self.wal_e.cmd + ' backup-fetch {} LATEST'.format(self.data_dir), env=self.env)
        self.restore_configuration_files()
        return ret

if __name__ == '__main__':
    if len(sys.argv) == 5:
        # scope, role, datadir, connstring
        restore = WALERestore(*(sys.argv[1:]))
        sys.exit(restore.run())
    sys.exit("Usage: {0} scope role datadir connstring".format(sys.argv[0]))
<|endoftext|>


<commit_before>"""Helper functions

Consists of functions to typically be used within templates, but also
available to Controllers. This module is available to templates as 'h'.
"""

import re
from routes import util
from webhelpers.pylonslib import Flash as _Flash
flash = _Flash('flash')
error = _Flash('error')

github_re = re.compile(r'^(?:git|http)://github.com/([^/]*)/([^/]*).git$')
repo_or_cz_re = re.compile(r'^(?:git|http)://repo.or.cz/([^/]*)$')

def get_url(obj):
    return util.url_for(controller='query', action='query', id=obj.id)

def _github_handle(repo, obj, match):
    values = {'user' : match.group(1),
              'repo' : match.group(2),
              'sha1' : obj.id,
              'type' : obj.type}
    if obj.type == 'commit':
        return 'http://github.com/%(user)s/%(repo)s/commit/%(sha1)s' % values
    elif obj.type == 'tree' or obj.type == 'blob':
        path_pair = obj.get_path(repo)
        # Could be in the middle of indexing
        if not path_pair:
            return None
        commit, path = path_pair
        values['path'] = path
        values['commit_sha1'] = commit.id
        return 'http://github.com/%(user)s/%(repo)s/%(type)s/%(commit_sha1)s/%(path)s' % values
    elif obj.type == 'tag':
        return get_view_url_for(repo, obj.object)

def _repo_or_cz_handle(repo, obj, match):
    values = {'repo' : match.group(1),
              'type' : obj.type,
              'sha1' : obj.id}
    return 'http://repo.or.cz/w/%(repo)s/%(type)s/%(sha1)s' % values

def get_view_url_for(repo, obj):
    match = github_re.search(repo.url)
    if match:
        return _github_handle(repo, obj, match)
    match = repo_or_cz_re.search(repo.url)
    if match:
        return _repo_or_cz_handle(repo, obj, match)

def pluralize(number, singular, plural=None):
    if plural is None:
        plural = '%ss' % singular
    if number == 1:
        return '1 %s' % singular
    else:
        return '%d %s' % (number, plural)

def liststyled(items, separator, style_left=None, style_right=None):
    if style_left is None:
        style_left=""
    if style_right is None:
        style_right=""
    styled_objects = map((lambda x: style_left + x + style_right), items)
    return separator.join(styled_objects)
<commit_msg>Added view capabilities for git.kernel.org repos<commit_after>"""Helper functions

Consists of functions to typically be used within templates, but also
available to Controllers. This module is available to templates as 'h'.
"""

import re
from routes import util
from webhelpers.pylonslib import Flash as _Flash
flash = _Flash('flash')
error = _Flash('error')

github_com_re = re.compile(r'^(?:git|http)://github\.com/([^/]*)/([^/]*)\.git$')
git_kernel_org_re = re.compile(r'^(?:git|http)://git\.kernel\.org/pub/scm/([^/]*)/(.*\.git$)')
repo_or_cz_re = re.compile(r'^(?:git|http)://repo\.or\.cz/([^/]*)$')

def get_url(obj):
    return util.url_for(controller='query', action='query', id=obj.id)

def _github_com_handle(repo, obj, match):
    values = {'user' : match.group(1),
              'repo' : match.group(2),
              'sha1' : obj.id,
              'type' : obj.type}
    if obj.type == 'commit':
        return 'http://github.com/%(user)s/%(repo)s/commit/%(sha1)s' % values
    elif obj.type == 'tree' or obj.type == 'blob':
        path_pair = obj.get_path(repo)
        # Could be in the middle of indexing
        if not path_pair:
            return None
        commit, path = path_pair
        values['path'] = path
        values['commit_sha1'] = commit.id
        return 'http://github.com/%(user)s/%(repo)s/%(type)s/%(commit_sha1)s/%(path)s' % values
    elif obj.type == 'tag':
        return get_view_url_for(repo, obj.object)

def _git_kernel_org_handle(repo, obj, match):
    values = {'user' : match.group(1),
              'suffix' : match.group(2),
              'type' : obj.type,
              'sha1' : obj.id}
    return 'http://git.kernel.org/?p=%(user)/%(suffix);a=%(type)s;h=%(sha1)s' % values

def _repo_or_cz_handle(repo, obj, match):
    values = {'repo' : match.group(1),
              'type' : obj.type,
              'sha1' : obj.id}
    return 'http://repo.or.cz/w/%(repo)s/%(type)s/%(sha1)s' % values

def get_view_url_for(repo, obj):
    for regex, handler in [(github_com_re, _github_com_handle),
                           (git_kernel_org_re, git_kernel_org_handle),
                           (repo_or_cz_re, repo_or_cz_handle)]:
        match = regex.search(repo.url)
        if match:
            return handler(repo, obj, match)

def pluralize(number, singular, plural=None):
    if plural is None:
        plural = '%ss' % singular
    if number == 1:
        return '1 %s' % singular
    else:
        return '%d %s' % (number, plural)

def liststyled(items, separator, style_left=None, style_right=None):
    if style_left is None:
        style_left=""
    if style_right is None:
        style_right=""
    styled_objects = map((lambda x: style_left + x + style_right), items)
    return separator.join(styled_objects)
<|endoftext|>


<commit_before>#
# Copyright (C) 2015 Satoru SATOH <ssato @ redhat.com>
# License: BSD3
#
from __future__ import absolute_import

import operator

import anytemplate.engines.jinja2
import anytemplate.engines.string

ENGINES = [e for e in
           [anytemplate.engines.string.StringTemplateEngine,
            anytemplate.engines.jinja2.Jjnja2Engine] if e.supports()]

def find_by_filename(filename, engines=ENGINES):
    """
    Find a list of template engine classes to render template `filename`.

    :param filename: Template file name (may be a absolute/relative path)
    :param engines: Template engines

    :return: A list of engines support given template file
    """
    return sorted((e for e in engines if e.supports(filename)),
                  key=operator.methodcaller("priority"))

def find_by_name(name, engines=ENGINES):
    """
    Find a template engine class specified by its name `name`.

    :param name: Template name
    :param engines: Template engines

    :return: A template engine or None if no any template engine of given name
        were found.
    """
    for egn in engines:
        if egn.name() == name:
            return egn

    return None

# vim:sw=4:ts=4:et:
<commit_msg>add anytemplate.engine.list_engines_by_priority to list all supported template engines sorted by each priority<commit_after>#
# Copyright (C) 2015 Satoru SATOH <ssato @ redhat.com>
# License: BSD3
#
from __future__ import absolute_import

import operator

import anytemplate.engines.jinja2
import anytemplate.engines.string

ENGINES = [e for e in
           [anytemplate.engines.string.StringTemplateEngine,
            anytemplate.engines.jinja2.Jjnja2Engine] if e.supports()]

def list_engines_by_priority(engines=ENGINES):
    """
    Return a list of engines supported sorted by each priority.
    """
    return sorted(engines, key=operator.methodcaller("priority"))

def find_by_filename(filename, engines=ENGINES):
    """
    Find a list of template engine classes to render template `filename`.

    :param filename: Template file name (may be a absolute/relative path)
    :param engines: Template engines

    :return: A list of engines support given template file
    """
    return sorted((e for e in engines if e.supports(filename)),
                  key=operator.methodcaller("priority"))

def find_by_name(name, engines=ENGINES):
    """
    Find a template engine class specified by its name `name`.

    :param name: Template name
    :param engines: Template engines

    :return: A template engine or None if no any template engine of given name
        were found.
    """
    for egn in engines:
        if egn.name() == name:
            return egn

    return None

# vim:sw=4:ts=4:et:
<|endoftext|>


<commit_before>import time
import sys
from optparse import OptionParser
import os

from framework import Framework, Colors
from pynessus import Nessus

class Skanner(Framework):

    def __init__(self, configfile):
        super(Skanner, self).__init__(configfile)

    def run(self, options):

        try:
            nessus = Nessus(self.config.get('core', 'server'), self.config.get('core', 'port'))
            user = nessus.User(self.config.get('core', 'user'), self.config.get('core', 'password'))
            if options.scan_uuid is not None:
                found = False
                if nessus.login(user):
                    self.info("Successfully logged in.")
                    nessus.load_scans()
                    for scan in nessus.scans:
                        if scan.uuid == options.scan_uuid:
                            found = True
                            self.info("Found scan %s." % scan.uuid)
                            while scan.status != "completed" and scan.status != "canceled":
                                sys.stdout.write("%s[Status: %s]%s %0.2f%%\r" % (Colors.O, scan.status, Colors.N, scan.progress))
                                sys.stdout.flush()
                                time.sleep(5)
                            if scan.status == "completed":
                                path = scan.download()
                                if path is not None:
                                    self.info("Report downloaded to %s" % path)
                                else:
                                    raise Exception("An error occured while downloading report %s." % r.id)
                            else:
                                raise Exception("Scan has been canceled.")
                    if not found:
                        for scan in nessus.scans:
                            if scan.id == int(options.scan_uuid):
                                found = True
                                self.info("Found report for scan %s" % options.scan_uuid)
                                path = scan.download()
                                if path is not None:
                                    self.info("Report downloaded to %s" % path)
                                else:
                                    raise Exception("An error occured while downloading report %s." % report.id)
                    if not found:
                        raise Exception("Can't find scan identified by %s" % options.scan_uuid)
                else:
                    raise Exception("An error occured while logging you in.")
            else:
                if options.scan_name is None:
                    raise Exception("Scan name not provided. Aborting.")
                if options.policy_name is None:
                    raise Exception("Policy name not provided. Aborting.")

                if options.targets_file is not None:
                    with open(options.targets_file, "rb") as f:
                        targets = f.read().replace("\n", ",")
                elif options.targets is not None:
                    targets = options.targets
                else:
                    raise Exception("No provided targets. Aborting.")

                if nessus.login(user):
                    self.info("Successfully logged in.")
                    nessus.load_policies()
		    nessus.load_templates()
		    nessus.load_folders()
                    scan = nessus.Scan()
                    scan.name = options.scan_name
                    #scan.tag = nessus.folders[0]
                    # does the provided policy exists ?
                    for policy in nessus.policies:
                        if policy.name == options.policy_name:
                            scan.policy = policy
                    if scan.policy:
                        scan.custom_targets = targets

			for folder in nessus.folders:
				if folder.name == "My Scans":
					scan.tag = folder
		        try:
				if scan.launch():
        	                    # scan launched, monitoring progress ...
                	            self.info("Scan %s has been launched, waiting for completion..." % scan.uuid)
                        	    while scan.status != "completed" and scan.status != "canceled":
	                                sys.stdout.write("%s[Status: %s]%s %0.2f%%\r" % (Colors.O, scan.status, Colors.N, scan.progress))
	                                sys.stdout.flush()
	                                time.sleep(5)
	                            if scan.status == "completed":
	                                path = scan.download()
	                                if path is not None:
	                                    self.info("Report downloaded to %s" % path)
	                                else:
        	                            raise Exception("An error occured while downloading report %s." % r.id)
	                            else:
        	                        raise Exception("Scan has been canceled.")
                	        else:
                        	    raise Exception("An error occured when launching the scan.")
			except Exception as e:
				print e.message
                    else:
                        raise Exception("Can't find the policy named %s. Aborting." % options.policy_name)
                    nessus.logout()
                else:
                    raise Exception("An error occured while logging you in.")
        except Exception as e:
            self.error(e.message)

if __name__ == "__main__":

    parser = OptionParser()
    parser.add_option("-i", dest='targets', help="targets ip addresses")
    parser.add_option("--iL", dest='targets_file', help="targets input file")
    parser.add_option("-n", dest='scan_name', help="scan name")
    parser.add_option("-p", dest='policy_name', help="policy_name")
    parser.add_option("-c", dest='configfile', help="configuration file to use")
    parser.add_option("-s", dest='scan_uuid', help="scan uuid to hook")
    (options, args) = parser.parse_args()

    skanner = Skanner(options.configfile)
    skanner.run(options)
<commit_msg>hotfix<commit_after>import time
import sys
from optparse import OptionParser
import os

from framework import Framework, Colors
from pynessus import Nessus

class Skanner(Framework):

    def __init__(self, configfile):
        super(Skanner, self).__init__(configfile)

    def run(self, options):

        try:
            nessus = Nessus(self.config.get('core', 'server'), self.config.get('core', 'port'))
            user = nessus.User(self.config.get('core', 'user'), self.config.get('core', 'password'))
            if options.scan_uuid is not None:
                found = False
                if nessus.login(user):
                    self.info("Successfully logged in.")
                    nessus.load_scans()
                    for scan in nessus.scans:
                        if scan.id == int(options.scan_uuid):
                            found = True
                            self.info("Found scan %s." % scan.uuid)
                            while scan.status != "completed" and scan.status != "canceled":
                                sys.stdout.write("%s[Status: %s]%s %0.2f%%\r" % (Colors.O, scan.status, Colors.N, scan.progress))
                                sys.stdout.flush()
                                time.sleep(5)
                            if scan.status == "completed":
                                path = scan.download()
                                if path is not None:
                                    self.info("Report downloaded to %s" % path)
                                else:
                                    raise Exception("An error occured while downloading report %s." % r.id)
                            else:
                                raise Exception("Scan has been canceled.")
                    if not found:
                        for scan in nessus.scans:
                            if scan.id == int(options.scan_uuid):
                                found = True
                                self.info("Found report for scan %s" % options.scan_uuid)
                                path = scan.download()
                                if path is not None:
                                    self.info("Report downloaded to %s" % path)
                                else:
                                    raise Exception("An error occured while downloading report %s." % report.id)
                    if not found:
                        raise Exception("Can't find scan identified by %s" % options.scan_uuid)
                else:
                    raise Exception("An error occured while logging you in.")
            else:
                if options.scan_name is None:
                    raise Exception("Scan name not provided. Aborting.")
                if options.policy_name is None:
                    raise Exception("Policy name not provided. Aborting.")

                if options.targets_file is not None:
                    with open(options.targets_file, "rb") as f:
                        targets = f.read().replace("\n", ",")
                elif options.targets is not None:
                    targets = options.targets
                else:
                    raise Exception("No provided targets. Aborting.")

                if nessus.login(user):
                    self.info("Successfully logged in.")
                    nessus.load_policies()
		    nessus.load_templates()
		    nessus.load_folders()
                    scan = nessus.Scan()
                    scan.name = options.scan_name
                    #scan.tag = nessus.folders[0]
                    # does the provided policy exists ?
                    for policy in nessus.policies:
                        if policy.name == options.policy_name:
                            scan.policy = policy
                    if scan.policy:
                        scan.custom_targets = targets

			for folder in nessus.folders:
				if folder.name == "My Scans":
					scan.tag = folder
		        try:
				if scan.launch():
        	                    # scan launched, monitoring progress ...
                	            self.info("Scan %s has been launched, waiting for completion..." % scan.uuid)
                        	    while scan.status != "completed" and scan.status != "canceled":
	                                sys.stdout.write("%s[Status: %s]%s %0.2f%%\r" % (Colors.O, scan.status, Colors.N, scan.progress))
	                                sys.stdout.flush()
	                                time.sleep(5)
	                            if scan.status == "completed":
	                                path = scan.download()
	                                if path is not None:
	                                    self.info("Report downloaded to %s" % path)
	                                else:
        	                            raise Exception("An error occured while downloading report %s." % r.id)
	                            else:
        	                        raise Exception("Scan has been canceled.")
                	        else:
                        	    raise Exception("An error occured when launching the scan.")
			except Exception as e:
				print e.message
                    else:
                        raise Exception("Can't find the policy named %s. Aborting." % options.policy_name)
                    nessus.logout()
                else:
                    raise Exception("An error occured while logging you in.")
        except Exception as e:
            self.error(e.message)

if __name__ == "__main__":

    parser = OptionParser()
    parser.add_option("-i", dest='targets', help="targets ip addresses")
    parser.add_option("--iL", dest='targets_file', help="targets input file")
    parser.add_option("-n", dest='scan_name', help="scan name")
    parser.add_option("-p", dest='policy_name', help="policy_name")
    parser.add_option("-c", dest='configfile', help="configuration file to use")
    parser.add_option("-s", dest='scan_uuid', help="scan uuid to hook")
    (options, args) = parser.parse_args()

    skanner = Skanner(options.configfile)
    skanner.run(options)
<|endoftext|>


<commit_before># Copyright (c) 2013 Ian C. Good
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#

from functools import wraps
from socket import getfqdn

from slimta.edge.smtp import SmtpValidators
from slimta.smtp.auth import Auth, CredentialsInvalidError
from slimta.util.dnsbl import check_dnsbl

from slimta.policy.forward import Forward
from slimta.policy.split import RecipientSplit, RecipientDomainSplit
from slimta.policy.spamassassin import SpamAssassin
from slimta.policy.headers import AddDateHeader, AddMessageIdHeader, \
                                  AddReceivedHeader

class RuleHelpers(object):

    @staticmethod
    def _noop_decorator(f):
        @wraps(f)
        def wrapper(*args, **kwargs):
            return f(*args, **kwargs)
        return wrapper

    def __init__(self, options):
        rules = options.get('rules', {})
        self.banner = rules.get('banner')
        if self.banner:
            self.banner = self.banner.format(fqdn=getfqdn())
        self.dnsbl = rules.get('dnsbl')
        self.only_senders = rules.get('only_senders')
        self.only_rcpts = rules.get('only_recipients')
        self.credentials = rules.get('require_credentials')

    def is_sender_ok(self, validators, sender):
        if self.only_senders is not None and sender not in self.only_senders:
            return False
        if self.credentials is not None and not validators.session.auth_result:
            return False
        return True

    def is_recipient_ok(self, recipient):
        if self.only_rcpts is not None and recipient not in self.only_rcpts:
            return False
        return True

    def get_banner_decorator(self):
        if self.dnsbl:
            return check_dnsbl(self.dnsbl, match_code='520')
        return self._noop_decorator

    def set_banner_message(self, reply):
        if self.banner:
            reply.message = self.banner

def build_smtpedge_validators(options):
    rules = RuleHelpers(options)
    class CustomValidators(SmtpValidators):
        @rules.get_banner_decorator()
        def handle_banner(self, reply, address):
            rules.set_banner_message(reply)
        def handle_mail(self, reply, sender):
            if not rules.is_sender_ok(self, sender):
                reply.code = '550'
                reply.message = '5.7.1 Sender <{0}> Not allowed'.format(sender)
        def handle_rcpt(self, reply, rcpt):
            if not rules.is_recipient_ok(rcpt):
                reply.code = '550'
                reply.message = '5.7.1 Recipient <{0}> Not allowed'.format(rcpt)
    return CustomValidators

def build_smtpedge_auth(options):
    rules = RuleHelpers(options)
    if rules.credentials is None:
        return None
    class CustomAuth(Auth):
        def verify_secret(self, username, password, identity=None):
            try:
                assert rules.credentials[username] == password
            except (KeyError, AssertionError):
                raise CredentialsInvalidError()
            return username
        def get_secret(self, username, identity=None):
            try:
                return rules.credentials[username], username
            except KeyError:
                raise CredentialsInvalidError()
    return CustomAuth

def add_queue_policies(queue, policy_options):
    for policy in policy_options:
        if policy.type == 'add_date_header':
            queue.add_policy(AddDateHeader())
        elif policy.type == 'add_messageid_header':
            queue.add_policy(AddMessageIdHeader())
        elif policy.type == 'add_received_header':
            queue.add_policy(AddReceivedHeader())
        elif policy.type == 'recipient_split':
            queue.add_policy(RecipientSplit())
        elif policy.type == 'recipient_domain_split':
            queue.add_policy(RecipientDomainSplit())
        elif policy.type == 'forward':
            forward = Forward()
            for pattern, repl in dict(policy.get('mappings', {})).items():
                forward.add_mapping(pattern, reply)
            queue.add_policy(forward)
        elif policy.type == 'spamassassin':
            host = policy.get('host', 'localhost')
            port = int(policy.get('port', 783))
            queue.add_policy(SpamAssassin((host, port)))

# vim:et:fdm=marker:sts=4:sw=4:ts=4
<commit_msg>allow hostname or fqdn in banner<commit_after># Copyright (c) 2013 Ian C. Good
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#

from functools import wraps
from socket import getfqdn, gethostname

from slimta.edge.smtp import SmtpValidators
from slimta.smtp.auth import Auth, CredentialsInvalidError
from slimta.util.dnsbl import check_dnsbl

from slimta.policy.forward import Forward
from slimta.policy.split import RecipientSplit, RecipientDomainSplit
from slimta.policy.spamassassin import SpamAssassin
from slimta.policy.headers import AddDateHeader, AddMessageIdHeader, \
                                  AddReceivedHeader

class RuleHelpers(object):

    @staticmethod
    def _noop_decorator(f):
        @wraps(f)
        def wrapper(*args, **kwargs):
            return f(*args, **kwargs)
        return wrapper

    def __init__(self, options):
        rules = options.get('rules', {})
        self.banner = rules.get('banner')
        if self.banner:
            self.banner = self.banner.format(fqdn=getfqdn(),
                                             hostname=gethostname())
        self.dnsbl = rules.get('dnsbl')
        self.only_senders = rules.get('only_senders')
        self.only_rcpts = rules.get('only_recipients')
        self.credentials = rules.get('require_credentials')

    def is_sender_ok(self, validators, sender):
        if self.only_senders is not None and sender not in self.only_senders:
            return False
        if self.credentials is not None and not validators.session.auth_result:
            return False
        return True

    def is_recipient_ok(self, recipient):
        if self.only_rcpts is not None and recipient not in self.only_rcpts:
            return False
        return True

    def get_banner_decorator(self):
        if self.dnsbl:
            return check_dnsbl(self.dnsbl, match_code='520')
        return self._noop_decorator

    def set_banner_message(self, reply):
        if self.banner:
            reply.message = self.banner

def build_smtpedge_validators(options):
    rules = RuleHelpers(options)
    class CustomValidators(SmtpValidators):
        @rules.get_banner_decorator()
        def handle_banner(self, reply, address):
            rules.set_banner_message(reply)
        def handle_mail(self, reply, sender):
            if not rules.is_sender_ok(self, sender):
                reply.code = '550'
                reply.message = '5.7.1 Sender <{0}> Not allowed'.format(sender)
        def handle_rcpt(self, reply, rcpt):
            if not rules.is_recipient_ok(rcpt):
                reply.code = '550'
                reply.message = '5.7.1 Recipient <{0}> Not allowed'.format(rcpt)
    return CustomValidators

def build_smtpedge_auth(options):
    rules = RuleHelpers(options)
    if rules.credentials is None:
        return None
    class CustomAuth(Auth):
        def verify_secret(self, username, password, identity=None):
            try:
                assert rules.credentials[username] == password
            except (KeyError, AssertionError):
                raise CredentialsInvalidError()
            return username
        def get_secret(self, username, identity=None):
            try:
                return rules.credentials[username], username
            except KeyError:
                raise CredentialsInvalidError()
    return CustomAuth

def add_queue_policies(queue, policy_options):
    for policy in policy_options:
        if policy.type == 'add_date_header':
            queue.add_policy(AddDateHeader())
        elif policy.type == 'add_messageid_header':
            queue.add_policy(AddMessageIdHeader())
        elif policy.type == 'add_received_header':
            queue.add_policy(AddReceivedHeader())
        elif policy.type == 'recipient_split':
            queue.add_policy(RecipientSplit())
        elif policy.type == 'recipient_domain_split':
            queue.add_policy(RecipientDomainSplit())
        elif policy.type == 'forward':
            forward = Forward()
            for pattern, repl in dict(policy.get('mappings', {})).items():
                forward.add_mapping(pattern, reply)
            queue.add_policy(forward)
        elif policy.type == 'spamassassin':
            host = policy.get('host', 'localhost')
            port = int(policy.get('port', 783))
            queue.add_policy(SpamAssassin((host, port)))

# vim:et:fdm=marker:sts=4:sw=4:ts=4
<|endoftext|>


<commit_before>"""Functions for rounding numbers."""

import functools
from dataclasses import dataclass

import numpy as np

from dapper.dpr_config import rc

@dataclass
class UncertainQtty():
    """Data container associating uncertainty (confidence) to a quantity.

    Includes intelligent rounding and printing functionality.

    Usually, the precision parameter will be set to the (potentially estimated)
    standard deviation of an uncertain quantity.
    However, this class in itself does not define the `prec` attribute
    by anything else than what it does: impact the rounding & printing of `val`.

    Examples:
    >>> for c in [.01, .1, .2, .9, 1]:
    ...    print(UncertainQtty(1.2345, c))
    1.23 ±0.01
    1.2 ±0.1
    1.2 ±0.2
    1.2 ±0.9
    1 ±1

    >>> for c in [.01, 1e-10, 1e-17, 0]:
    ...    print(UncertainQtty(1.2, c))
    1.20 ±0.01
    1.2000000000 ±1e-10
    1.19999999999999996 ±1e-17
    1.2000000000 ±0

    Note that in the case of a confidence of exactly 0,
    it defaults to 10 decimal places.
    Meanwhile, a NaN confidence yields printing using `rc.sigfig`:

    >>> print(UncertainQtty(1.234567, np.nan))
    1.235 ±nan

    Also note the effect of large uncertainty:

    >>> for c in [1, 9, 10, 11, 20, 100, np.inf]:
    ...    print(UncertainQtty(12, c))
    12 ±1
    12 ±9
    10 ±10
    10 ±10
    10 ±20
    0 ±100
    0 ±inf
    """

    val: float
    prec: float

    def __post_init__(self):
        assert self.prec >= 0. or np.isnan(self.prec), "'prec' must be non-negative."

    def round(self=1.0):  # noqa
        """Round intelligently.

        - `prec` to 1 sig.fig.
        - `val` to `round2(val, prec)`.
        """
        if np.isnan(self.prec):
            # Fallback to rc.sigfig
            c = self.prec
            v = round2sigfig(self.val, rc.sigfig)
        else:
            # Normal/general case
            c = round2sigfig(self.prec, 1)
            v = round2(self.val, self.prec)
        return v, c

    def __str__(self):
        """Returns 'val ±prec', using `UncertainQtty.round` and some finesse."""
        v, c = self.round()

        if np.isnan(c):
            # Rounding to fallback (rc.sigfig) already took place
            return f"{v} ±{c}"
        elif c == 0:
            # 0 (i.e. not 1e-300) never arises "naturally" => Treat it "magically"
            # by truncating to a default. Also see https://stackoverflow.com/a/25899600
            n = -10
        else:
            # Normal/general case.
            n = log10int(c)

        frmt = "%.f"
        if n < 0:
            # Ensure we get 1.30 ±0.01, NOT 1.3 ±0.01:
            frmt = "%%0.%df" % -n
        elif np.isfinite(c):
            # if c >= 1.0
            c = int(c)
        v = frmt % v

        return f"{v} ±{c}"

    def __repr__(self):
        """Essentially the same as `__str__`."""
        v, c = str(self).split(" ±")
        return self.__class__.__name__ + f"(val={v}, prec={c})"

    @property
    def prec(self):
        return self._prec

    @prec.setter
    def prec(self, prec):
        assert prec >= 0. or np.isnan(prec), "'prec' must be non-negative."
        self._prec = prec

def np_vectorize(f):
    """Like `np.vectorize`, but with some embellishments.

    - Includes `functools.wraps`
    - Applies `.item()` to output if input was a scalar.

    Parameters
    ----------
    f: callable
        Your function.

    Returns
    -------
    vectorized: callable
        Your function, now element-wise applicable to an iterable.
    """
    vectorized = np.vectorize(f)

    @functools.wraps(f)
    def new(*args, **kwargs):
        output = vectorized(*args, **kwargs)
        if np.isscalar(args[0]) and not isinstance(args[0], np.ndarray):
            output = output.item()
        return output

    return new

@np_vectorize
def _round2prec(num, prec):
    """Don't use (directly)! Suffers from numerical precision.

    This function is left here just for reference. Use `round2` instead.

    The issue is that:
    >>> _round2prec(0.7,.1)
    0.7000000000000001
    """
    return prec * round(num / prec)

@np_vectorize
def log10int(x):
    """Compute decimal order, rounded down.

    Conversion to `int` means that we cannot return nan's or +/- infinity,
    even though this could be meaningful. Instead, we return integers of magnitude
    a little less than IEEE floating point max/min-ima instead.
    This avoids a lot of clauses in the parent/callers to this function.

    Examples
    --------
    >>> log10int([1e-1, 1e-2, 1, 3, 10, np.inf, -np.inf, np.nan])
    array([  -1,   -2,    0,    0,    1,  300, -300, -300])
    """
    # Extreme cases -- https://stackoverflow.com/q/65248379
    if np.isnan(x):
        y = -300
    elif x < 1e-300:
        y = -300
    elif x > 1e+300:
        y = +300
    # Normal case
    else:
        y = int(np.floor(np.log10(np.abs(x))))
    return y

@np_vectorize
def round2(x, prec=1.0):
    r"""Round x to the decimal order appropriate for the precision.

    Parameters
    ----------
    x : array_like
        Value to be rounded.
    prec: float
        Precision, before prettify, which is given by
        $$ \text{prec} = 10^{\text{floor}(-\log_{10}|\text{prec}|)} $$

    Returns
    -------
    Rounded value (always a float).

    See Also
    --------
    `round2sigfig`

    Examples
    --------
    >>> round2(1.65, 0.543)
    1.6
    >>> round2(1.66, 0.543)
    1.7
    >>> round2(1.65, 1.234)
    2.0
    """
    if np.isnan(prec):
        return x
    ndecimal = -log10int(prec)
    return np.round(x, ndecimal)

@np_vectorize
def round2sigfig(x, sigfig=1):
    """Round to significant figures.

    Parameters
    ----------
    x
        Value to be rounded.
    sigfig
        Number of significant figures to include.

    Returns
    -------
    rounded value (always a float).

    See Also
    --------
    np.round : rounds to a given number of *decimals*.
    `round2` : rounds to a given *precision*.

    Examples
    --------
    >>> round2sigfig(1234.5678, 1)
    1000.0
    >>> round2sigfig(1234.5678, 4)
    1235.0
    >>> round2sigfig(1234.5678, 6)
    1234.57
    """
    ndecimal = sigfig - log10int(x) - 1
    return np.round(x, ndecimal)

def is_whole(x, **kwargs):
    """Check if a number is a whole/natural number to precision given by `np.isclose`.

    For actual type checking, use `isinstance(x, (int, np.integer))`.

    Parameters
    ----------
    x: float or ndarray
        Values to be checked

    Returns
    -------
    l: bool
        True if rounded x is close to x, otherwise False
    """
    return np.isclose(x, round(x), **kwargs)
<commit_msg>Enable default value for prec.<commit_after>"""Functions for rounding numbers."""

import functools

import numpy as np

from dapper.dpr_config import rc

class UncertainQtty():
    """Data container associating uncertainty (confidence) to a quantity.

    Includes intelligent rounding and printing functionality.

    Usually, the precision parameter will be set to the (potentially estimated)
    standard deviation of an uncertain quantity.
    However, this class in itself does not define the `prec` attribute
    by anything else than what it does: impact the rounding & printing of `val`.

    Examples:
    >>> for c in [.01, .1, .2, .9, 1]:
    ...    print(UncertainQtty(1.2345, c))
    1.23 ±0.01
    1.2 ±0.1
    1.2 ±0.2
    1.2 ±0.9
    1 ±1

    >>> for c in [.01, 1e-10, 1e-17, 0]:
    ...    print(UncertainQtty(1.2, c))
    1.20 ±0.01
    1.2000000000 ±1e-10
    1.19999999999999996 ±1e-17
    1.2000000000 ±0

    Note that in the case of a confidence of exactly 0,
    it defaults to 10 decimal places.
    Meanwhile, a NaN confidence yields printing using `rc.sigfig`:

    >>> print(UncertainQtty(1.234567, np.nan))
    1.235 ±nan

    Also note the effect of large uncertainty:

    >>> for c in [1, 9, 10, 11, 20, 100, np.inf]:
    ...    print(UncertainQtty(12, c))
    12 ±1
    12 ±9
    10 ±10
    10 ±10
    10 ±20
    0 ±100
    0 ±inf
    """

    def __init__(self, val, prec=np.nan):
        self.val = val
        self.prec = prec

    def round(self=1.0):  # noqa
        """Round intelligently.

        - `prec` to 1 sig.fig.
        - `val` to `round2(val, prec)`.
        """
        if np.isnan(self.prec):
            # Fallback to rc.sigfig
            c = self.prec
            v = round2sigfig(self.val, rc.sigfig)
        else:
            # Normal/general case
            c = round2sigfig(self.prec, 1)
            v = round2(self.val, self.prec)
        return v, c

    def __str__(self):
        """Returns 'val ±prec', using `UncertainQtty.round` and some finesse."""
        v, c = self.round()

        if np.isnan(c):
            # Rounding to fallback (rc.sigfig) already took place
            return f"{v} ±{c}"
        elif c == 0:
            # 0 (i.e. not 1e-300) never arises "naturally" => Treat it "magically"
            # by truncating to a default. Also see https://stackoverflow.com/a/25899600
            n = -10
        else:
            # Normal/general case.
            n = log10int(c)

        frmt = "%.f"
        if n < 0:
            # Ensure we get 1.30 ±0.01, NOT 1.3 ±0.01:
            frmt = "%%0.%df" % -n
        elif np.isfinite(c):
            # if c >= 1.0
            c = int(c)
        v = frmt % v

        return f"{v} ±{c}"

    def __repr__(self):
        """Essentially the same as `__str__`."""
        v, c = str(self).split(" ±")
        return self.__class__.__name__ + f"(val={v}, prec={c})"

    @property
    def prec(self):
        return self._prec

    @prec.setter
    def prec(self, prec):
        assert prec >= 0. or np.isnan(prec), "'prec' must be non-negative."
        self._prec = prec

def np_vectorize(f):
    """Like `np.vectorize`, but with some embellishments.

    - Includes `functools.wraps`
    - Applies `.item()` to output if input was a scalar.

    Parameters
    ----------
    f: callable
        Your function.

    Returns
    -------
    vectorized: callable
        Your function, now element-wise applicable to an iterable.
    """
    vectorized = np.vectorize(f)

    @functools.wraps(f)
    def new(*args, **kwargs):
        output = vectorized(*args, **kwargs)
        if np.isscalar(args[0]) and not isinstance(args[0], np.ndarray):
            output = output.item()
        return output

    return new

@np_vectorize
def _round2prec(num, prec):
    """Don't use (directly)! Suffers from numerical precision.

    This function is left here just for reference. Use `round2` instead.

    The issue is that:
    >>> _round2prec(0.7,.1)
    0.7000000000000001
    """
    return prec * round(num / prec)

@np_vectorize
def log10int(x):
    """Compute decimal order, rounded down.

    Conversion to `int` means that we cannot return nan's or +/- infinity,
    even though this could be meaningful. Instead, we return integers of magnitude
    a little less than IEEE floating point max/min-ima instead.
    This avoids a lot of clauses in the parent/callers to this function.

    Examples
    --------
    >>> log10int([1e-1, 1e-2, 1, 3, 10, np.inf, -np.inf, np.nan])
    array([  -1,   -2,    0,    0,    1,  300, -300, -300])
    """
    # Extreme cases -- https://stackoverflow.com/q/65248379
    if np.isnan(x):
        y = -300
    elif x < 1e-300:
        y = -300
    elif x > 1e+300:
        y = +300
    # Normal case
    else:
        y = int(np.floor(np.log10(np.abs(x))))
    return y

@np_vectorize
def round2(x, prec=1.0):
    r"""Round x to the decimal order appropriate for the precision.

    Parameters
    ----------
    x : array_like
        Value to be rounded.
    prec: float
        Precision, before prettify, which is given by
        $$ \text{prec} = 10^{\text{floor}(-\log_{10}|\text{prec}|)} $$

    Returns
    -------
    Rounded value (always a float).

    See Also
    --------
    `round2sigfig`

    Examples
    --------
    >>> round2(1.65, 0.543)
    1.6
    >>> round2(1.66, 0.543)
    1.7
    >>> round2(1.65, 1.234)
    2.0
    """
    if np.isnan(prec):
        return x
    ndecimal = -log10int(prec)
    return np.round(x, ndecimal)

@np_vectorize
def round2sigfig(x, sigfig=1):
    """Round to significant figures.

    Parameters
    ----------
    x
        Value to be rounded.
    sigfig
        Number of significant figures to include.

    Returns
    -------
    rounded value (always a float).

    See Also
    --------
    np.round : rounds to a given number of *decimals*.
    `round2` : rounds to a given *precision*.

    Examples
    --------
    >>> round2sigfig(1234.5678, 1)
    1000.0
    >>> round2sigfig(1234.5678, 4)
    1235.0
    >>> round2sigfig(1234.5678, 6)
    1234.57
    """
    ndecimal = sigfig - log10int(x) - 1
    return np.round(x, ndecimal)

def is_whole(x, **kwargs):
    """Check if a number is a whole/natural number to precision given by `np.isclose`.

    For actual type checking, use `isinstance(x, (int, np.integer))`.

    Parameters
    ----------
    x: float or ndarray
        Values to be checked

    Returns
    -------
    l: bool
        True if rounded x is close to x, otherwise False
    """
    return np.isclose(x, round(x), **kwargs)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8
# Copyright 2013 The Font Bakery Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# See AUTHORS.txt for the list of Authors and LICENSE.txt for the License.

import sys
# hardcoded for OSX
# sys.path.append('/usr/local/lib/python2.7/site-packages/')
print(sys.argv[1], sys.argv[2])

import fontforge

font = fontforge.open(sys.argv[1])

font.layers["Fore"].is_quadratic = True
font.selection.all()
#   Add Extrema
font.addExtrema()
#   Simplify
font.simplify(1,('setstarttoextremum','removesingletonpoints'))
#   Correct Directions
font.correctDirection()

ttf = font.generate(sys.argv[2])
otf = font.generate(sys.argv[3])
<commit_msg>Making simplify a try: so that it works without exception errors<commit_after>#!/usr/bin/env python
# coding: utf-8
# Copyright 2013 The Font Bakery Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# See AUTHORS.txt for the list of Authors and LICENSE.txt for the License.

import sys
# hardcoded for OSX
# sys.path.append('/usr/local/lib/python2.7/site-packages/')
print(sys.argv[1], sys.argv[2])

import fontforge

font = fontforge.open(sys.argv[1])

font.layers["Fore"].is_quadratic = True
font.selection.all()
#   Add Extrema
font.addExtrema()
#   Simplify
try:
    font.simplify(1,('setstarttoextremum','removesingletonpoints'))
except:
    print "Error: Could not simplify"
    pass
#   Correct Directions
font.correctDirection()

ttf = font.generate(sys.argv[2])
otf = font.generate(sys.argv[3])
<|endoftext|>


<commit_before>"""The entry point for flake8."""

import ast  # noqa
from typing import (  # noqa
    Iterator,
    Tuple,
)

from .docstring.style import DocstringStyle
from .function_description import (
    get_function_descriptions,
)
from .integrity_checker import IntegrityChecker
from .config import (
    Configuration,
    get_config,
)
from .strictness import Strictness

__version__ = '1.8.0'

class DarglintChecker(object):

    name = 'flake8-darglint'
    version = __version__
    config = get_config()

    def __init__(self, tree, filename):
        self.tree = tree
        self.filename = filename
        self.verbosity = 2

    def run(self):
        # type: () -> Iterator[Tuple[int, int, str, type]]
        if '*' in self.config.ignore:
            return

        # Remember the last line number, so that if there is an
        # exception raised by Darglint, we can at least give a decent
        # idea of where it was raised.
        last_line = 1
        try:
            functions = get_function_descriptions(self.tree)
            checker = IntegrityChecker(
                raise_errors=False,
            )
            checker.config = self.config
            for function in functions:
                checker.run_checks(function)

            error_report = checker.get_error_report(
                self.verbosity,
                self.filename
            )
            for line, col, msg in error_report.flake8_report():
                last_line = line
                yield (line, col, msg, type(self))

        except Exception as ex:
            yield (
                last_line,
                0,
                'DAR000: Unexpected exception in darglint: ' + str(ex),
                type(self)
            )

    @classmethod
    def add_options(cls, option_manager):
        defaults = cls.config

        option_manager.add_option(
            '--docstring-style',
            default=defaults.style.name,
            parse_from_config=True,
            help='Docstring style to use for Darglint',
        )

        option_manager.add_option(
            '--strictness',
            default=defaults.strictness.name,
            parse_from_config=True,
            help='Strictness level to use for Darglint',
        )

    @classmethod
    def parse_options(cls, options):
        cls.config.style = DocstringStyle.from_string(options.docstring_style)
        cls.config.strictness = Strictness.from_string(options.strictness)
<commit_msg>Add ignore flag for darglint<commit_after>"""The entry point for flake8."""

import ast  # noqa
from typing import (  # noqa
    Iterator,
    Tuple,
)

from .docstring.style import DocstringStyle
from .function_description import (
    get_function_descriptions,
)
from .integrity_checker import IntegrityChecker
from .config import (
    Configuration,
    get_config,
)
from .strictness import Strictness

__version__ = '1.8.0'

class DarglintChecker(object):

    name = 'flake8-darglint'
    version = __version__
    config = get_config()

    def __init__(self, tree, filename):
        self.tree = tree
        self.filename = filename
        self.verbosity = 2

    def run(self):
        # type: () -> Iterator[Tuple[int, int, str, type]]
        if '*' in self.config.ignore:
            return

        # Remember the last line number, so that if there is an
        # exception raised by Darglint, we can at least give a decent
        # idea of where it was raised.
        last_line = 1
        try:
            functions = get_function_descriptions(self.tree)
            checker = IntegrityChecker(
                raise_errors=False,
            )
            checker.config = self.config
            for function in functions:
                checker.run_checks(function)

            error_report = checker.get_error_report(
                self.verbosity,
                self.filename
            )
            for line, col, msg in error_report.flake8_report():
                last_line = line
                yield (line, col, msg, type(self))

        except Exception as ex:
            yield (
                last_line,
                0,
                'DAR000: Unexpected exception in darglint: ' + str(ex),
                type(self)
            )

    @classmethod
    def add_options(cls, option_manager):
        defaults = cls.config

        option_manager.add_option(
            '--docstring-style',
            default=defaults.style.name,
            parse_from_config=True,
            help='Docstring style to use for Darglint',
        )

        option_manager.add_option(
            '--strictness',
            default=defaults.strictness.name,
            parse_from_config=True,
            help='Strictness level to use for Darglint',
        )

        option_manager.add_option(
            '--darglint-ignore',
            default=defaults.ignore,
            parse_from_config=True,
            nargs='+',
            help='Codes to ignore for Darglint',
        )

    @classmethod
    def parse_options(cls, options):
        cls.config.style = DocstringStyle.from_string(options.docstring_style)
        cls.config.strictness = Strictness.from_string(options.strictness)
        cls.config.ignore = [error.strip() for error in options.darglint_ignore]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import logging
import os.path
from traceback import format_exc

import rarfile
import requests

from aore.config import folders, unrar_config
from aore.miscutils.exceptions import FiasException
from aoxmltableentry import AoXmlTableEntry

class AoRar:
    def __init__(self):
        rarfile.UNRAR_TOOL = unrar_config.path

    def download(self, url):
        logging.info("Downloading %s", url)
        try:
            local_filename = os.path.abspath(folders.temp + "/" + url.split('/')[-1])
            if os.path.isfile(local_filename):
                os.remove(local_filename)
            else:
                os.makedirs(folders.temp)

            request = requests.get(url, stream=True)
            with open(local_filename, 'wb') as f:
                for chunk in request.iter_content(chunk_size=1024):
                    if chunk:
                        f.write(chunk)
        except:
            raise FiasException("Error downloading. Reason : {}".format(format_exc()))

        logging.info("Downloaded %d bytes", int(request.headers['Content-length']))
        return local_filename

    def get_table_entries(self, file_name, allowed_tables):
        if file_name and os.path.isfile(file_name):
            rf = rarfile.RarFile(file_name)

            for arch_entry in rf.infolist():
                xmltable = AoXmlTableEntry.from_rar(arch_entry.filename, rf, arch_entry)
                if xmltable.table_name in allowed_tables:
                    yield xmltable
            else:
                logging.info("All entries processed")
                os.remove(file_name)
        else:
            logging.error("No file specified or not exists")
<commit_msg>Addd check if temp folder exists<commit_after># -*- coding: utf-8 -*-

import logging
import os.path
from traceback import format_exc

import rarfile
import requests

from aore.config import folders, unrar_config
from aore.miscutils.exceptions import FiasException
from aoxmltableentry import AoXmlTableEntry

class AoRar:
    def __init__(self):
        rarfile.UNRAR_TOOL = unrar_config.path

    def download(self, url):
        logging.info("Downloading %s", url)
        try:
            local_filename = os.path.abspath(folders.temp + "/" + url.split('/')[-1])
            if os.path.isfile(local_filename):
                os.remove(local_filename)
            else:
                if not os.path.exists(folders.temp):
                    os.makedirs(folders.temp)

            request = requests.get(url, stream=True)
            with open(local_filename, 'wb') as f:
                for chunk in request.iter_content(chunk_size=1024):
                    if chunk:
                        f.write(chunk)
        except:
            raise FiasException("Error downloading. Reason : {}".format(format_exc()))

        logging.info("Downloaded %d bytes", int(request.headers['Content-length']))
        return local_filename

    def get_table_entries(self, file_name, allowed_tables):
        if file_name and os.path.isfile(file_name):
            rf = rarfile.RarFile(file_name)

            for arch_entry in rf.infolist():
                xmltable = AoXmlTableEntry.from_rar(arch_entry.filename, rf, arch_entry)
                if xmltable.table_name in allowed_tables:
                    yield xmltable
            else:
                logging.info("All entries processed")
                os.remove(file_name)
        else:
            logging.error("No file specified or not exists")
<|endoftext|>