<commit_before># Copyright (c) 2011, Enthought, Ltd.
# Author: Pietro Berkes <pberkes@enthought.com>
# License: Modified BSD license (2-clause)

"""Entry point for pyanno UI application.

At present, the application is based on the wx backend of the traitsui library.
It also supports 2 screen formats:

* for large displays (larger than 1300x850), the main window will be
  1300 x 850 pixels large

* for small displays it will be 1024x768
"""

from traits.etsconfig.api import ETSConfig
ETSConfig.toolkit = 'wx'

import pyanno.ui.appbase.wx_utils as wx_utils
wx_utils.set_background_color_to_white()

from pyanno.ui.pyanno_ui_application import pyanno_application
import numpy

import logging

def main():
    """Create and start the application."""

    # deactivate warnings for operations like log(0.) and log(-inf), which
    # are handled correctly by pyanno
    numpy.seterr(divide='ignore', invalid='ignore')

    with pyanno_application(logging_level=logging.INFO) as app:
        app.open()

if __name__ == '__main__':
    main()
<commit_msg>OTH: temporary workaround to traitsui editing problem<commit_after># Copyright (c) 2011, Enthought, Ltd.
# Author: Pietro Berkes <pberkes@enthought.com>
# License: Modified BSD license (2-clause)

"""Entry point for pyanno UI application.

At present, the application is based on the wx backend of the traitsui library.
It also supports 2 screen formats:

* for large displays (larger than 1300x850), the main window will be
  1300 x 850 pixels large

* for small displays it will be 1024x768
"""

from traits.etsconfig.api import ETSConfig
ETSConfig.toolkit = 'wx'

import pyanno.ui.appbase.wx_utils as wx_utils
wx_utils.set_background_color_to_white()

from pyanno.ui.pyanno_ui_application import pyanno_application
import numpy

import logging

# FIXME remove errors coming from the traitsui editors when pressing
#       a button before the editing is over
import sys
_old_excepthook = sys.excepthook
def _pyanno_excepthook(type, value, traceback):
    if str(value).startswith("'NoneType' object"):
        return
    _old_excepthook(type, value, traceback)

def main():
    """Create and start the application."""

    # deactivate warnings for operations like log(0.) and log(-inf), which
    # are handled correctly by pyanno
    numpy.seterr(divide='ignore', invalid='ignore')

    with pyanno_application(logging_level=logging.INFO) as app:
        try:
            sys.excepthook = _pyanno_excepthook
            app.open()
        finally:
            sys.excepthook = _old_excepthook

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""Classes for creating DNS zones and writing corresponding zone
files.

No validation is done here (e.g. requiring a SOA record) because this
module may be used to create fragments of zone files to be used via an
$INCLUDE directive. Users of this module are encouraged to employ an
external validation process like named-checkzone(8) on zone files.

Host names are passed to dnsrecord.ResourceRecord methods unmodified;
i.e. they must be terminated with a dot ('.') to be interpreted as
fully qualified domain names (FQDNs)--otherwise they will be
interpreted relative to $ORIGIN, so be diligent in minding your dots.

IP addresses may be specified in any format accepted by
ipaddr.IPAddress().

Time values may be specified either as an integer (seconds) or a
string in one of BIND's time formats.

Note that the 'name' keyword argument defaults to '@', so adding an MX
record, for example, adds an MX record for the whole domain unless
otherwise specified. The 'ttl' keyword argument defaults to None, so
records will use the zone's default time-to-live (TTL) unless
otherwise specified.
"""

import time

import ipaddr

import dnsrecord

class Zone(object):

    """Base DNS zone object."""

    # default values for keyword args; values chosen from RFC recommendations
    TTL = '1h'
    REFRESH = '3h'
    RETRY = '1h'
    EXPIRY = '2d'
    NXDOMAIN = '1h'

    def __init__(self, origin, epochserial=False, ttl=TTL):
        """Return a Zone object.

        Args:
            origin: (str) zone's root; '.' will be appended if necessary
              'example.com'
            epochserial: (boolean) whether to use number of seconds since
              epoch as default serial number in SOA record
            ttl: (str or int) default time-to-live for resource records
        """

        self.origin = origin
        if not self.origin.endswith('.'):
            # make sure it looks like FQDN (although it still might be wrong)
            self.origin += '.'
        self.epochserial = epochserial
        self.records = []  # list of dnsrecord objects
        self.ttl = ttl

    def write_file(self, filename):
        """Write zone file.

        Args:
            filename: (str) name of file to be written
              'zonefile.hosts'
        """

        with open(filename, 'w') as fh:
            fh.write('$ORIGIN %s\n' % self.origin)
            fh.write('$TTL %s\n' % self.ttl)
            for record in self.records:
                fh.write('%s\n' % record)
        fh.close()

    def add_record(self, record):
        """Add record to zone.

        This is abstracted from the add_*() methods in case later
        implementations store the records in a different data
        structure. (YagNi?)

        Args:
            record: (dnsrecord.ResourceRecord) record to be added
        """

        self.records.append(record)

    def add_soa(self, mname, rname, serial=None, refresh=REFRESH, retry=RETRY,
                expiry=EXPIRY, nxdomain=NXDOMAIN, name='@', ttl=None):
        """Add Start of Authority record to zone.

        Args:
            mname: (str) host name of name server authoritative for zone
              'ns1.example.com.'
            rname: (str) e-mail address of person responsible for zone
              'hostmaster@example.com'
            serial: (int) serial number
              '1969123100'
            refresh: (str or int) slave's refresh interval
            retry: (str or int) slave's retry interval
            expiry: (str or int) slave's expiry interval
            nxdomain: (str or int) negative caching time (TTL)
            name: (str) name of node to which this record belongs
              'example.com.'
        """

        if serial is None:
            # set default serial number
            if self.epochserial:
                serial = int(time.time())  # number of seconds since epoch
            else:
                serial = int(time.strftime('%Y%m%d00'))  # YYYYMMDD00
        soa = dnsrecord.SOA(name, mname, rname, serial, refresh, retry,
                            expiry, nxdomain, ttl)
        self.add_record(soa)

    def add_ns(self, name_server, name='@', ttl=None):
        """Add Name Server record to zone.

        Args:
            name_server: (str) host name of name server
              'ns1.example.com.'
            name: (str) name of node to which this record belongs
              'example.com.'
            ttl: (str or int) time-to-live
        """

        ns = dnsrecord.NS(name, name_server, ttl)
        self.add_record(ns)

class ForwardZone(Zone):

    """Forward DNS zone."""

    def add_a(self, address, name='@', ttl=None):
        """Add IPv4 Address record to zone.

        Args:
            address: (str) IPv4 address
              '192.168.1.1'
            name: (str) name of node to which this record belongs
              'host.example.com.'
            ttl: (str or int) time-to-live
        """

        a = dnsrecord.A(name, address, ttl)
        self.add_record(a)

    def add_aaaa(self, address, name='@', ttl=None):
        """Add IPv6 Address record to zone.

        Args:
            address: (str) IPv6 address
              '2001:db8::1'
            name: (str) name of node to which this record belongs
              'host.example.com.'
            ttl: (str or int) time-to-live
        """

        aaaa = dnsrecord.AAAA(name, address, ttl)
        self.add_record(aaaa)

    def add_cname(self, canonical_name, name='@', ttl=None):
        """Add Canonical Name record to zone.

        Args:
            canonical: (str) canonical host name of host
              'mail.example.com.'
            name: (str) name of node to which this record belongs
              'mailserver.example.com.'
            ttl: (str or int) time-to-live
        """

        cname = dnsrecord.CNAME(name, canonical_name, ttl)
        self.add_record(cname)

    def add_mx(self, mail_exchanger, preference=10, name='@', ttl=None):
        """Add Mail Exchanger record to zone.

        Args:
            mail_exchanger: (str) host name of mail exchanger
              'mail.example.com.'
            preference: (int) preference value of mail exchanger
            name: (str) name of node to which this record belongs
              'example.com.'
            ttl: (str or int) time-to-live
        """

        mx = dnsrecord.MX(name, preference, mail_exchanger, ttl)
        self.add_record(mx)

    def add_txt(self, text, name='@', ttl=None):
        """Add Text record to zone.

        Args:
            text: (str) textual contents of record
              'This is a text record'
            name: (str) name of node to which this record belongs
              'example.com.'
            ttl: (str or int) time-to-live
        """

        txt = dnsrecord.TXT(name, text, ttl)
        self.add_record(txt)

class ReverseZone(Zone):

    """Reverse DNS zone."""

    def add_ptr(self, address, name='@', ttl=None):
        """Add Pointer record to zone.

        Args:
            address: (str) IPv4 or IPv6 address
              '192.168.1.1'
            name: (str) name of node to which this record belongs
              'ns1.example.com.'
            ttl: (str or int) time-to-live
        """

        ptr = dnsrecord.PTR(address, name, ttl)
        self.add_record(ptr)

def run_tests():
    """Run rudimentary tests of module.

    These are really intended for development and debugging purposes
    rather than a substitute for unit tests.
    """

    # create forward zone and write to file
    z = ForwardZone('example.com')
    z.add_soa('ns1', 'hostmaster')
    z.add_ns('ns1')
    z.add_ns('ns2')
    z.add_mx('mail1')
    z.add_mx('mail2', 20, ttl=600)
    z.add_a('192.168.1.1', 'ns1')
    z.add_aaaa('2001:db8::1', 'ns1')
    z.add_txt('v=spf1 mx ~all')
    z.add_cname('mailserver', 'mail')
    filename = 'fwdzone'
    z.write_file(filename)
    print 'Wrote %s.' % filename

    # create IPv4 reverse zone and write to file
    z = ReverseZone('1.168.192.in-addr.arpa')
    z.add_soa('ns1.example.com.', 'hostmaster@example.com.')
    z.add_ns('ns1.example.com.')
    z.add_ptr('192.168.1.1', 'ns1.example.com.')
    filename = 'revzone4'
    z.write_file(filename)
    print 'Wrote %s.' % filename

    # create IPv6 reverse zone and write to file
    z = ReverseZone('0.0.0.0.0.0.c.f.ip6.arpa', epochserial=True)
    z.add_soa('ns1.example.com.', 'hostmaster@example.com.')
    z.add_ns('ns1.example.com.')
    z.add_ptr('2001:db8::1', 'ns1.example.com.')
    filename = 'revzone6'
    z.write_file(filename)
    print 'Wrote %s.' % filename

if __name__ == '__main__':
    run_tests()
<commit_msg>rename Zone to indicate non-public<commit_after>"""Classes for creating DNS zones and writing corresponding zone
files.

No validation is done here (e.g. requiring a SOA record) because this
module may be used to create fragments of zone files to be used via an
$INCLUDE directive. Users of this module are encouraged to employ an
external validation process like named-checkzone(8) on zone files.

Host names are passed to dnsrecord.ResourceRecord methods unmodified;
i.e. they must be terminated with a dot ('.') to be interpreted as
fully qualified domain names (FQDNs)--otherwise they will be
interpreted relative to $ORIGIN, so be diligent in minding your dots.

IP addresses may be specified in any format accepted by
ipaddr.IPAddress().

Time values may be specified either as an integer (seconds) or a
string in one of BIND's time formats.

Note that the 'name' keyword argument defaults to '@', so adding an MX
record, for example, adds an MX record for the whole domain unless
otherwise specified. The 'ttl' keyword argument defaults to None, so
records will use the zone's default time-to-live (TTL) unless
otherwise specified.
"""

import time

import ipaddr

import dnsrecord

class _Zone(object):

    """Base DNS zone object."""

    # default values for keyword args; values chosen from RFC recommendations
    TTL = '1h'
    REFRESH = '3h'
    RETRY = '1h'
    EXPIRY = '2d'
    NXDOMAIN = '1h'

    def __init__(self, origin, epochserial=False, ttl=TTL):
        """Return a _Zone object.

        Args:
            origin: (str) zone's root; '.' will be appended if necessary
              'example.com'
            epochserial: (boolean) whether to use number of seconds since
              epoch as default serial number in SOA record
            ttl: (str or int) default time-to-live for resource records
        """

        self.origin = origin
        if not self.origin.endswith('.'):
            # make sure it looks like FQDN (although it still might be wrong)
            self.origin += '.'
        self.epochserial = epochserial
        self.records = []  # list of dnsrecord objects
        self.ttl = ttl

    def write_file(self, filename):
        """Write zone file.

        Args:
            filename: (str) name of file to be written
              'zonefile.hosts'
        """

        with open(filename, 'w') as fh:
            fh.write('$ORIGIN %s\n' % self.origin)
            fh.write('$TTL %s\n' % self.ttl)
            for record in self.records:
                fh.write('%s\n' % record)
        fh.close()

    def add_record(self, record):
        """Add record to zone.

        This is abstracted from the add_*() methods in case later
        implementations store the records in a different data
        structure. (YagNi?)

        Args:
            record: (dnsrecord.ResourceRecord) record to be added
        """

        self.records.append(record)

    def add_soa(self, mname, rname, serial=None, refresh=REFRESH, retry=RETRY,
                expiry=EXPIRY, nxdomain=NXDOMAIN, name='@', ttl=None):
        """Add Start of Authority record to zone.

        Args:
            mname: (str) host name of name server authoritative for zone
              'ns1.example.com.'
            rname: (str) e-mail address of person responsible for zone
              'hostmaster@example.com'
            serial: (int) serial number
              '1969123100'
            refresh: (str or int) slave's refresh interval
            retry: (str or int) slave's retry interval
            expiry: (str or int) slave's expiry interval
            nxdomain: (str or int) negative caching time (TTL)
            name: (str) name of node to which this record belongs
              'example.com.'
        """

        if serial is None:
            # set default serial number
            if self.epochserial:
                serial = int(time.time())  # number of seconds since epoch
            else:
                serial = int(time.strftime('%Y%m%d00'))  # YYYYMMDD00
        soa = dnsrecord.SOA(name, mname, rname, serial, refresh, retry,
                            expiry, nxdomain, ttl)
        self.add_record(soa)

    def add_ns(self, name_server, name='@', ttl=None):
        """Add Name Server record to zone.

        Args:
            name_server: (str) host name of name server
              'ns1.example.com.'
            name: (str) name of node to which this record belongs
              'example.com.'
            ttl: (str or int) time-to-live
        """

        ns = dnsrecord.NS(name, name_server, ttl)
        self.add_record(ns)

class ForwardZone(_Zone):

    """Forward DNS zone."""

    def add_a(self, address, name='@', ttl=None):
        """Add IPv4 Address record to zone.

        Args:
            address: (str) IPv4 address
              '192.168.1.1'
            name: (str) name of node to which this record belongs
              'host.example.com.'
            ttl: (str or int) time-to-live
        """

        a = dnsrecord.A(name, address, ttl)
        self.add_record(a)

    def add_aaaa(self, address, name='@', ttl=None):
        """Add IPv6 Address record to zone.

        Args:
            address: (str) IPv6 address
              '2001:db8::1'
            name: (str) name of node to which this record belongs
              'host.example.com.'
            ttl: (str or int) time-to-live
        """

        aaaa = dnsrecord.AAAA(name, address, ttl)
        self.add_record(aaaa)

    def add_cname(self, canonical_name, name='@', ttl=None):
        """Add Canonical Name record to zone.

        Args:
            canonical: (str) canonical host name of host
              'mail.example.com.'
            name: (str) name of node to which this record belongs
              'mailserver.example.com.'
            ttl: (str or int) time-to-live
        """

        cname = dnsrecord.CNAME(name, canonical_name, ttl)
        self.add_record(cname)

    def add_mx(self, mail_exchanger, preference=10, name='@', ttl=None):
        """Add Mail Exchanger record to zone.

        Args:
            mail_exchanger: (str) host name of mail exchanger
              'mail.example.com.'
            preference: (int) preference value of mail exchanger
            name: (str) name of node to which this record belongs
              'example.com.'
            ttl: (str or int) time-to-live
        """

        mx = dnsrecord.MX(name, preference, mail_exchanger, ttl)
        self.add_record(mx)

    def add_txt(self, text, name='@', ttl=None):
        """Add Text record to zone.

        Args:
            text: (str) textual contents of record
              'This is a text record'
            name: (str) name of node to which this record belongs
              'example.com.'
            ttl: (str or int) time-to-live
        """

        txt = dnsrecord.TXT(name, text, ttl)
        self.add_record(txt)

class ReverseZone(_Zone):

    """Reverse DNS zone."""

    def add_ptr(self, address, name='@', ttl=None):
        """Add Pointer record to zone.

        Args:
            address: (str) IPv4 or IPv6 address
              '192.168.1.1'
            name: (str) name of node to which this record belongs
              'ns1.example.com.'
            ttl: (str or int) time-to-live
        """

        ptr = dnsrecord.PTR(address, name, ttl)
        self.add_record(ptr)

def run_tests():
    """Run rudimentary tests of module.

    These are really intended for development and debugging purposes
    rather than a substitute for unit tests.
    """

    # create forward zone and write to file
    z = ForwardZone('example.com')
    z.add_soa('ns1', 'hostmaster')
    z.add_ns('ns1')
    z.add_ns('ns2')
    z.add_mx('mail1')
    z.add_mx('mail2', 20, ttl=600)
    z.add_a('192.168.1.1', 'ns1')
    z.add_aaaa('2001:db8::1', 'ns1')
    z.add_txt('v=spf1 mx ~all')
    z.add_cname('mailserver', 'mail')
    filename = 'fwdzone'
    z.write_file(filename)
    print 'Wrote %s.' % filename

    # create IPv4 reverse zone and write to file
    z = ReverseZone('1.168.192.in-addr.arpa')
    z.add_soa('ns1.example.com.', 'hostmaster@example.com.')
    z.add_ns('ns1.example.com.')
    z.add_ptr('192.168.1.1', 'ns1.example.com.')
    filename = 'revzone4'
    z.write_file(filename)
    print 'Wrote %s.' % filename

    # create IPv6 reverse zone and write to file
    z = ReverseZone('0.0.0.0.0.0.c.f.ip6.arpa', epochserial=True)
    z.add_soa('ns1.example.com.', 'hostmaster@example.com.')
    z.add_ns('ns1.example.com.')
    z.add_ptr('2001:db8::1', 'ns1.example.com.')
    filename = 'revzone6'
    z.write_file(filename)
    print 'Wrote %s.' % filename

if __name__ == '__main__':
    run_tests()
<|endoftext|>


<commit_before>from copy import copy

import numpy as np
import scipy.optimize
from pybotics import kinematics, robot_model, geometry

class Robot:
    def __init__(self, robot_model, name='Pybot'):
        self.robot_model = robot_model
        self.tool = np.eye(4)
        self.world_frame = np.eye(4)
        self.current_joints = [0] * self.num_dof()
        self.joint_stiffness = [0] * self.num_dof()
        self.name = name

    def num_dof(self):
        return len(self.robot_model)

    def fk(self, joint_angles=None, link_limit=None, torques=None, reference_frame=None):

        # validate input
        if torques is not None:
            assert len(torques) == self.num_dof()

        # define output
        transforms = []

        # load current robot joints if none given
        if joint_angles is None:

            # if current joints are empty, assign zero
            if not self.current_joints:
                self.current_joints = [0] * self.num_dof()

            joint_angles = [self.current_joints]

        # make sure joints are contained in a list
        elif not isinstance(joint_angles[0], list):
            joint_angles = [joint_angles]

        # define joint limit, transform up to n-th joint
        tool_transform = np.eye(4)
        if link_limit is None:
            link_limit = self.num_dof()
            tool_transform = self.tool

        # define reference frame
        if reference_frame is None:
            reference_frame = np.eye(4)

        # iterate through input
        for joints in joint_angles:

            # define transform identity matrix to carry multiplications
            transform = np.eye(4)

            # multiply wrt reference frame
            transform = np.dot(reference_frame, transform)

            # multiply through the forward transforms of the joints
            for i in range(link_limit):
                # add the current joint pose to the forward transform
                current_link = self.robot_model[i].copy()
                current_link[2] += joints[i]

                if torques is not None:
                    current_link[2] += torques[i] * self.joint_stiffness[i]

                # get the transform step
                current_link_transform = kinematics.forward_transform(current_link)
                transform = np.dot(transform, current_link_transform)

            # add tool transform
            transform = np.dot(transform, tool_transform)
            transforms.append(transform)

        # return only transform if only one joint config is given
        if len(transforms) == 1:
            transforms = transforms[0]

        return transforms

    def impair_robot_model(self, relative_error=0.05):
        # random error multiplier between [-1,1]
        error_attenuation = 2 * np.random.rand(self.robot_model.shape[0], self.robot_model.shape[1])
        error_attenuation -= 1

        # attenuate the multiplier wrt user input (e.g., limit to 5% of nominal)
        error_attenuation *= relative_error

        # create error delta
        error_delta = np.multiply(self.robot_model, error_attenuation)

        bit_mask = np.isclose(error_delta, np.zeros(error_delta.shape))

        if np.any(bit_mask):
            error_adjustment = 2 * np.random.rand(self.robot_model.shape[0], self.robot_model.shape[1])
            error_adjustment -= 1

            error_adjustment[:, 0] *= abs(error_delta[:, 0]).max()
            error_adjustment[:, 1] *= abs(error_delta[:, 1]).max()
            error_adjustment[:, 2] *= abs(error_delta[:, 2]).max()
            error_adjustment[:, 3] *= abs(error_delta[:, 3]).max()

            error_adjustment = np.multiply(error_adjustment, bit_mask.astype(int))

            error_delta += error_adjustment

        self.robot_model += error_delta

    def set_tool_xyz(self, xyz):
        for i, parameter in enumerate(xyz):
            self.tool[i, -1] = parameter

    def generate_optimization_vector(self, optimization_mask):
        optimization_mask = copy(optimization_mask)
        vector = []

        # get world frame
        world_vector = geometry.pose_2_xyzrpw(self.world_frame)
        for parameter in world_vector:
            truth = optimization_mask.pop(0)
            if truth:
                vector.append(parameter)

        # get MDH parameters
        parameters = np.reshape(self.robot_model, self.robot_model.size)
        for parameter in parameters:
            truth = optimization_mask.pop(0)
            if truth:
                vector.append(parameter)

        # get tool frame
        tool_vector = geometry.pose_2_xyzrpw(self.tool)
        for parameter in tool_vector:
            truth = optimization_mask.pop(0)
            if truth:
                vector.append(parameter)

        # get joint stiffness parameters
        for parameter in self.joint_stiffness:
            truth = optimization_mask.pop(0)
            if truth:
                vector.append(parameter)

        return vector

    def apply_optimization_vector(self, optimization_vector, optimization_mask):
        optimization_mask = copy(optimization_mask)

        # set world frame
        world_vector = geometry.pose_2_xyzrpw(self.world_frame)
        for i in range(len(world_vector)):
            truth = optimization_mask.pop(0)
            if truth:
                parameter = optimization_vector.pop(0)
                world_vector[i] = parameter
        self.world_frame = geometry.xyzrpw_2_pose(world_vector)

        # set MDH parameters
        for i in range(self.robot_model.size):
            truth = optimization_mask.pop(0)
            if truth:
                parameter = optimization_vector.pop(0)
                row = int(i / 4)
                col = int(i % 4)
                self.robot_model[row, col] = parameter

        # set tool frame
        tool_vector = geometry.pose_2_xyzrpw(self.tool)
        for i in range(len(tool_vector)):
            truth = optimization_mask.pop(0)
            if truth:
                parameter = optimization_vector.pop(0)
                tool_vector[i] = parameter
        self.tool = geometry.xyzrpw_2_pose(tool_vector)

        # get joint stiffness parameters
        for i in range(len(self.joint_stiffness)):
            truth = optimization_mask.pop(0)
            if truth:
                parameter = optimization_vector.pop(0)
                self.joint_stiffness[i] = parameter

    def generate_optimization_mask(self, world_mask=False, robot_model_mask=False, tool_mask=False,
                                   joint_stiffness_mask=False):

        if not isinstance(world_mask, list):
            world_mask = [world_mask] * 6

        if not isinstance(robot_model_mask, list):
            robot_model_mask = [robot_model_mask] * 4 * self.num_dof()

        if not isinstance(tool_mask, list):
            tool_mask = [tool_mask] * 6

        if not isinstance(joint_stiffness_mask, list):
            joint_stiffness_mask = [joint_stiffness_mask] * self.num_dof()

        mask = []
        mask.extend(world_mask)
        mask.extend(robot_model_mask)
        mask.extend(tool_mask)
        mask.extend(joint_stiffness_mask)

        return mask

    def generate_parameter_bounds(self, optimization_mask, world_bounds=None, robot_model_bounds=None, tool_bounds=None,
                                  joint_stiffness_bounds=None):

        if world_bounds is None:
            world_bounds = [(None, None)] * 6

        if robot_model_bounds is None:
            robot_model_bounds = [(None, None)] * 4 * self.num_dof()

        if tool_bounds is None:
            tool_bounds = [(None, None)] * 6

        if joint_stiffness_bounds is None:
            joint_stiffness_bounds = [(None, None)] * self.num_dof()

        glob_bounds = []
        glob_bounds.extend(world_bounds)
        glob_bounds.extend(robot_model_bounds)
        glob_bounds.extend(tool_bounds)
        glob_bounds.extend(joint_stiffness_bounds)

        bounds = []
        for i, truth in enumerate(optimization_mask):
            if truth:
                bounds.append(glob_bounds[i])

        return bounds

    def ik(self, pose, joint_angles=None):
        # set initial joints
        if joint_angles is not None:
            assert len(joint_angles) == self.num_dof()
        else:
            joint_angles = self.current_joints

        bounds = [(-np.pi, np.pi)] * self.num_dof()

        is_success = False
        max_iterations = 5
        current_iteration = 0
        result = None
        current_best_result = None
        while not is_success and current_iteration < max_iterations:
            current_iteration += 1
            optimize_result = scipy.optimize.minimize(ik_fit_func,
                                                      joint_angles,
                                                      args=(pose, self),
                                                      method='TNC',
                                                      bounds=bounds,
                                                      options={
                                                          'maxiter': int(1e6),
                                                      }
                                                      )

            if optimize_result.fun < 1e-1:
                is_success = True
                result = optimize_result.x
            else:
                if current_best_result is None or optimize_result.fun < current_best_result[0]:
                    current_best_result = (optimize_result.fun, optimize_result.x)

                joint_angles = np.random.rand(1, self.num_dof())
                joint_angles -= 0.5
                joint_angles *= 2 * np.pi

        if not is_success:
            result = current_best_result[1]

        return result

    def jacobian_world(self, joint_angles=None):
        # set initial joints
        if joint_angles is not None:
            assert len(joint_angles) == self.num_dof()
        else:
            joint_angles = self.current_joints

        jacobian_flange = self.jacobian_flange(joint_angles)
        pose = self.fk(joint_angles)
        rotation = pose[0:3, 0:3]
        jacobian_transform = np.zeros((6, 6))
        jacobian_transform[:3, :3] = rotation
        jacobian_transform[3:, 3:] = rotation
        jacobian_world = np.dot(jacobian_transform, jacobian_flange)

        return jacobian_world

    def jacobian_flange(self, joint_angles=None):
        # set initial joints
        if joint_angles is not None:
            assert len(joint_angles) == self.num_dof()
        else:
            joint_angles = self.current_joints

        # init Cartesian jacobian (6-dof in space)
        jacobian_flange = np.zeros((6, self.num_dof()))
        current_transform = copy(self.tool)

        for i in reversed(range(self.num_dof())):
            d = np.array([
                -current_transform[0, 0] * current_transform[1, 3] + current_transform[1, 0] * current_transform[0, 3],
                - current_transform[0, 1] * current_transform[1, 3] + current_transform[1, 1] * current_transform[0, 3],
                - current_transform[0, 2] * current_transform[1, 3] + current_transform[1, 2] * current_transform[0, 3],
            ])
            delta = current_transform[2, 0:3]
            jacobian_flange[:, i] = np.hstack((d, delta))
            current_link = self.robot_model[i]
            current_link_transform = kinematics.forward_transform(current_link, joint_angle=joint_angles[i])
            current_transform = np.dot(current_link_transform, current_transform)

        return jacobian_flange

def ik_fit_func(joints_angles, pose, robot):
    geometry.wrap_2_pi(joints_angles)
    actual_pose = robot.fk(joints_angles)

    error = actual_pose - pose
    error = np.square(error)
    error = np.sum(error)
    return error
<commit_msg>added joint angle limits, angle validation, and random angle methods<commit_after>from copy import copy

import numpy as np
import scipy.optimize
from pybotics import kinematics, robot_model, geometry

class Robot:
    def __init__(self, robot_model, name='Pybot'):
        self.robot_model = robot_model
        self.tool = np.eye(4)
        self.world_frame = np.eye(4)
        self.current_joints = [0] * self.num_dof()
        self.joint_stiffness = [0] * self.num_dof()
        self.name = name
        self.joint_angle_limits = None

    def validate_joint_angles(self, joint_angles):

        is_success = True
        if self.joint_angle_limits is not None:
            for i, joint_angle in joint_angles:
                if joint_angle > max(self.joint_angle_limits[i]) or joint_angle < min(self.joint_angle_limits[i]):
                    is_success = False
                    break

        return is_success

    def num_dof(self):
        return len(self.robot_model)

    def fk(self, joint_angles=None, link_limit=None, torques=None, reference_frame=None):

        # validate input
        if torques is not None:
            assert len(torques) == self.num_dof()

        # define output
        transforms = []

        # load current robot joints if none given
        if joint_angles is None:

            # if current joints are empty, assign zero
            if not self.current_joints:
                self.current_joints = [0] * self.num_dof()

            joint_angles = [self.current_joints]

        # make sure joints are contained in a list
        elif not isinstance(joint_angles[0], list):
            joint_angles = [joint_angles]

        # define joint limit, transform up to n-th joint
        tool_transform = np.eye(4)
        if link_limit is None:
            link_limit = self.num_dof()
            tool_transform = self.tool

        # define reference frame
        if reference_frame is None:
            reference_frame = np.eye(4)

        # iterate through input
        for joints in joint_angles:

            # define transform identity matrix to carry multiplications
            transform = np.eye(4)

            # multiply wrt reference frame
            transform = np.dot(reference_frame, transform)

            # multiply through the forward transforms of the joints
            for i in range(link_limit):
                # add the current joint pose to the forward transform
                current_link = self.robot_model[i].copy()
                current_link[2] += joints[i]

                if torques is not None:
                    current_link[2] += torques[i] * self.joint_stiffness[i]

                # get the transform step
                current_link_transform = kinematics.forward_transform(current_link)
                transform = np.dot(transform, current_link_transform)

            # add tool transform
            transform = np.dot(transform, tool_transform)
            transforms.append(transform)

        # return only transform if only one joint config is given
        if len(transforms) == 1:
            transforms = transforms[0]

        return transforms

    def impair_robot_model(self, relative_error=0.05):
        # random error multiplier between [-1,1]
        error_attenuation = 2 * np.random.rand(self.robot_model.shape[0], self.robot_model.shape[1])
        error_attenuation -= 1

        # attenuate the multiplier wrt user input (e.g., limit to 5% of nominal)
        error_attenuation *= relative_error

        # create error delta
        error_delta = np.multiply(self.robot_model, error_attenuation)

        bit_mask = np.isclose(error_delta, np.zeros(error_delta.shape))

        if np.any(bit_mask):
            error_adjustment = 2 * np.random.rand(self.robot_model.shape[0], self.robot_model.shape[1])
            error_adjustment -= 1

            error_adjustment[:, 0] *= abs(error_delta[:, 0]).max()
            error_adjustment[:, 1] *= abs(error_delta[:, 1]).max()
            error_adjustment[:, 2] *= abs(error_delta[:, 2]).max()
            error_adjustment[:, 3] *= abs(error_delta[:, 3]).max()

            error_adjustment = np.multiply(error_adjustment, bit_mask.astype(int))

            error_delta += error_adjustment

        self.robot_model += error_delta

    def set_tool_xyz(self, xyz):
        for i, parameter in enumerate(xyz):
            self.tool[i, -1] = parameter

    def generate_optimization_vector(self, optimization_mask):
        optimization_mask = copy(optimization_mask)
        vector = []

        # get world frame
        world_vector = geometry.pose_2_xyzrpw(self.world_frame)
        for parameter in world_vector:
            truth = optimization_mask.pop(0)
            if truth:
                vector.append(parameter)

        # get MDH parameters
        parameters = np.reshape(self.robot_model, self.robot_model.size)
        for parameter in parameters:
            truth = optimization_mask.pop(0)
            if truth:
                vector.append(parameter)

        # get tool frame
        tool_vector = geometry.pose_2_xyzrpw(self.tool)
        for parameter in tool_vector:
            truth = optimization_mask.pop(0)
            if truth:
                vector.append(parameter)

        # get joint stiffness parameters
        for parameter in self.joint_stiffness:
            truth = optimization_mask.pop(0)
            if truth:
                vector.append(parameter)

        return vector

    def apply_optimization_vector(self, optimization_vector, optimization_mask):
        optimization_mask = copy(optimization_mask)

        # set world frame
        world_vector = geometry.pose_2_xyzrpw(self.world_frame)
        for i in range(len(world_vector)):
            truth = optimization_mask.pop(0)
            if truth:
                parameter = optimization_vector.pop(0)
                world_vector[i] = parameter
        self.world_frame = geometry.xyzrpw_2_pose(world_vector)

        # set MDH parameters
        for i in range(self.robot_model.size):
            truth = optimization_mask.pop(0)
            if truth:
                parameter = optimization_vector.pop(0)
                row = int(i / 4)
                col = int(i % 4)
                self.robot_model[row, col] = parameter

        # set tool frame
        tool_vector = geometry.pose_2_xyzrpw(self.tool)
        for i in range(len(tool_vector)):
            truth = optimization_mask.pop(0)
            if truth:
                parameter = optimization_vector.pop(0)
                tool_vector[i] = parameter
        self.tool = geometry.xyzrpw_2_pose(tool_vector)

        # get joint stiffness parameters
        for i in range(len(self.joint_stiffness)):
            truth = optimization_mask.pop(0)
            if truth:
                parameter = optimization_vector.pop(0)
                self.joint_stiffness[i] = parameter

    def generate_optimization_mask(self, world_mask=False, robot_model_mask=False, tool_mask=False,
                                   joint_stiffness_mask=False):

        if not isinstance(world_mask, list):
            world_mask = [world_mask] * 6

        if not isinstance(robot_model_mask, list):
            robot_model_mask = [robot_model_mask] * 4 * self.num_dof()

        if not isinstance(tool_mask, list):
            tool_mask = [tool_mask] * 6

        if not isinstance(joint_stiffness_mask, list):
            joint_stiffness_mask = [joint_stiffness_mask] * self.num_dof()

        mask = []
        mask.extend(world_mask)
        mask.extend(robot_model_mask)
        mask.extend(tool_mask)
        mask.extend(joint_stiffness_mask)

        return mask

    def generate_parameter_bounds(self, optimization_mask, world_bounds=None, robot_model_bounds=None, tool_bounds=None,
                                  joint_stiffness_bounds=None):

        if world_bounds is None:
            world_bounds = [(None, None)] * 6

        if robot_model_bounds is None:
            robot_model_bounds = [(None, None)] * 4 * self.num_dof()

        if tool_bounds is None:
            tool_bounds = [(None, None)] * 6

        if joint_stiffness_bounds is None:
            joint_stiffness_bounds = [(None, None)] * self.num_dof()

        glob_bounds = []
        glob_bounds.extend(world_bounds)
        glob_bounds.extend(robot_model_bounds)
        glob_bounds.extend(tool_bounds)
        glob_bounds.extend(joint_stiffness_bounds)

        bounds = []
        for i, truth in enumerate(optimization_mask):
            if truth:
                bounds.append(glob_bounds[i])

        return bounds

    def ik(self, pose, joint_angles=None):
        # set initial joints
        if joint_angles is not None:
            assert len(joint_angles) == self.num_dof()
        else:
            joint_angles = self.current_joints

        bounds = [(-np.pi, np.pi)] * self.num_dof()

        is_success = False
        max_iterations = 5
        current_iteration = 0
        result = None
        while not is_success and current_iteration < max_iterations:
            current_iteration += 1
            optimize_result = scipy.optimize.minimize(ik_fit_func,
                                                      joint_angles,
                                                      args=(pose, self),
                                                      method='TNC',
                                                      bounds=bounds,
                                                      options={
                                                          'maxiter': int(1e6),
                                                      }
                                                      )

            result = optimize_result.x
            if optimize_result.fun < 1e-1 and self.validate_joint_angles(result):
                is_success = True
            else:
                joint_angles = np.random.rand(1, self.num_dof())
                joint_angles -= 0.5
                joint_angles *= 2 * np.pi

        if not is_success:
            result = None

        return result

    def jacobian_world(self, joint_angles=None):
        # set initial joints
        if joint_angles is not None:
            assert len(joint_angles) == self.num_dof()
        else:
            joint_angles = self.current_joints

        jacobian_flange = self.jacobian_flange(joint_angles)
        pose = self.fk(joint_angles)
        rotation = pose[0:3, 0:3]
        jacobian_transform = np.zeros((6, 6))
        jacobian_transform[:3, :3] = rotation
        jacobian_transform[3:, 3:] = rotation
        jacobian_world = np.dot(jacobian_transform, jacobian_flange)

        return jacobian_world

    def jacobian_flange(self, joint_angles=None):
        # set initial joints
        if joint_angles is not None:
            assert len(joint_angles) == self.num_dof()
        else:
            joint_angles = self.current_joints

        # init Cartesian jacobian (6-dof in space)
        jacobian_flange = np.zeros((6, self.num_dof()))
        current_transform = copy(self.tool)

        for i in reversed(range(self.num_dof())):
            d = np.array([
                -current_transform[0, 0] * current_transform[1, 3] + current_transform[1, 0] * current_transform[0, 3],
                - current_transform[0, 1] * current_transform[1, 3] + current_transform[1, 1] * current_transform[0, 3],
                - current_transform[0, 2] * current_transform[1, 3] + current_transform[1, 2] * current_transform[0, 3],
            ])
            delta = current_transform[2, 0:3]
            jacobian_flange[:, i] = np.hstack((d, delta))
            current_link = self.robot_model[i]
            current_link_transform = kinematics.forward_transform(current_link, joint_angle=joint_angles[i])
            current_transform = np.dot(current_link_transform, current_transform)

        return jacobian_flange

def ik_fit_func(joints_angles, pose, robot):
    geometry.wrap_2_pi(joints_angles)
    actual_pose = robot.fk(joints_angles)

    error = actual_pose - pose
    error = np.square(error)
    error = np.sum(error)
    return error

def random_joints(joint_angle_limits):
    joint_angles = []
    for limits in joint_angle_limits:
        joint_angles.append(np.random.uniform(min(limits), max(limits)))
    return np.array(joint_angles)
<|endoftext|>


<commit_before># coding=utf-8
import os
import shutil

from django.db.models import ImageField
from django.core.files.images import ImageFile
from django.conf import settings
from PIL import Image

from hashing import hexHash

max_image_size = getattr(settings, 'MAX_IMAGE_SIZE', 1200)

def get_parameters(image_path):
    """
    Parses the image path to dictionary
    :param str image_path: image path
    :rtype dict
    """
    image_path = image_path
    image_directory = os.path.dirname(image_path)
    image_filename = os.path.basename(image_path)
    image_name = image_filename.split('.')[0]
    image_extension = image_filename.split('.')[-1]
    return {
        'directory': image_directory,
        'extension': image_extension,
        'name': image_name,
        'filename': image_filename,
        'path': image_path
    }

def fix_image_path(image_path):
    """
    Fixes the image path, transforming it to {md5hash}.{extension}

    :param str image_path: image path
    :rtype str
    """
    image = get_parameters(image_path)

    # early extension fix
    image['extension'] = image['extension'].lower().replace('jpeg', 'jpg')
    # test
    wrong_name = False
    try:                          # check ascii characters in filename
        image['name'].encode('ascii')
    except UnicodeEncodeError:
        wrong_name = True
    except UnicodeDecodeError:
        wrong_name = True

    if len(image['name']) != 32:  # check image name length
        wrong_name = True

    if wrong_name:
        image['name'] = hexHash(image['name'])

    return os.path.join(image['directory'], ''.join([image['name'], '.', image['extension']]))

def normalize_image_size(image_path, max_size):
    """
    Reduces image size to max_size
    :param image_path: path to image
    :param max_size: maximum size
    """
    try:
        image_object = Image.open(image_path)
    except IOError:
        raise IOError("No image found at %s" % image_path)
    image = get_parameters(image_path)
    if max(image_object.size) > max_size:
        resize_rate = max(image_object.size) / float(max_size)
        new_size = (int(image_object.size[0] / resize_rate), int(image_object.size[1] / resize_rate))
        image_object.thumbnail(new_size, Image.ANTIALIAS)
        new_path = os.path.join(image['directory'], ''.join([image['name'], '_thumbnail.jpg']))
        try:
            if image_object.mode != "RGB":
                image_object = image_object.convert("RGB")
            image_object.save(new_path, quality=70)
            shutil.copy(new_path, image['path'])
        except (IOError, KeyError, ValueError):
            try:
                os.remove(new_path)
            except IOError:
                pass
            raise

def save(image_field):
    image_path = image_field.path.encode("utf-8")

    try:
        image_exists = os.path.isfile(image_path)
    except (UnicodeEncodeError, UnicodeDecodeError):
        raise IOError('Incorrect locale settings')

    if not image_exists:
        return

    image_path = image_field.path
    new_image_path = fix_image_path(image_path)
    if new_image_path != image_path:
        image_content = ImageFile(open(image_path))
        try:
            image_field.save(os.path.basename(image_path), image_content, save=True)
        except IOError:
            pass    # TODO: process error
        image_path = new_image_path

    if not os.path.isfile(image_path):
        raise IOError('Failed to create %s' % image_path)
    try:
        normalize_image_size(image_path, max_image_size)
    except IOError:
        raise IOError('Failed to process %s' % image_path)

def upload_to(path):
    """
    Возвращает функцию генерации пути
    """

    def update_filename(instance, filename):
        f = fix_image_path(filename)
        return os.path.join(path, f)

    return update_filename

class ImageFieldHash(ImageField):
    def __init__(self, **kwargs):
        if 'upload_to' in kwargs:
            kwargs['upload_to'] = upload_to(kwargs['upload_to'])
        super(ImageFieldHash, self).__init__(**kwargs)
<commit_msg>added unique hash<commit_after># coding=utf-8
import os
import shutil

from django.db.models import ImageField
from django.core.files.images import ImageFile
from django.conf import settings
from PIL import Image

from hashing import hexHash
import time

max_image_size = getattr(settings, 'MAX_IMAGE_SIZE', 1200)

def get_parameters(image_path):
    """
    Parses the image path to dictionary
    :param str image_path: image path
    :rtype dict
    """
    image_path = image_path
    image_directory = os.path.dirname(image_path)
    image_filename = os.path.basename(image_path)
    image_name = image_filename.split('.')[0]
    image_extension = image_filename.split('.')[-1]
    return {
        'directory': image_directory,
        'extension': image_extension,
        'name': image_name,
        'filename': image_filename,
        'path': image_path
    }

def fix_image_path(image_path):
    """
    Fixes the image path, transforming it to {md5hash}.{extension}

    :param str image_path: image path
    :rtype str
    """
    image = get_parameters(image_path)

    # early extension fix
    image['extension'] = image['extension'].lower().replace('jpeg', 'jpg')
    # test
    wrong_name = False
    try:                          # check ascii characters in filename
        image['name'].encode('ascii')
    except UnicodeError:
        wrong_name = True

    if len(image['name']) != 32:  # check image name length
        wrong_name = True

    if wrong_name:
        image['name'] = hexHash(image['name'] + str(time.time()))

    return os.path.join(image['directory'], ''.join([image['name'], '.', image['extension']]))

def normalize_image_size(image_path, max_size):
    """
    Reduces image size to max_size
    :param image_path: path to image
    :param max_size: maximum size
    """
    try:
        image_object = Image.open(image_path)
    except IOError:
        raise IOError("No image found at %s" % image_path)
    image = get_parameters(image_path)
    if max(image_object.size) > max_size:
        resize_rate = max(image_object.size) / float(max_size)
        new_size = (int(image_object.size[0] / resize_rate), int(image_object.size[1] / resize_rate))
        image_object.thumbnail(new_size, Image.ANTIALIAS)
        new_path = os.path.join(image['directory'], ''.join([image['name'], '_thumbnail.jpg']))
        try:
            if image_object.mode != "RGB":
                image_object = image_object.convert("RGB")
            image_object.save(new_path, quality=70)
            shutil.copy(new_path, image['path'])
        except (IOError, KeyError, ValueError):
            try:
                os.remove(new_path)
            except IOError:
                pass
            raise

def save(image_field):
    image_path = image_field.path.encode("utf-8")

    try:
        image_exists = os.path.isfile(image_path)
    except UnicodeError:
        raise IOError('Incorrect locale settings')

    if not image_exists:
        return

    image_path = image_field.path
    new_image_path = fix_image_path(image_path)
    if new_image_path != image_path:
        image_content = ImageFile(open(image_path))
        try:
            image_field.save(os.path.basename(image_path), image_content, save=True)
        except IOError:
            pass    # TODO: process error
        image_path = new_image_path

    if not os.path.isfile(image_path):
        raise IOError('Failed to create %s' % image_path)
    try:
        normalize_image_size(image_path, max_image_size)
    except IOError:
        raise IOError('Failed to process %s' % image_path)

def upload_to(path):
    """
    Возвращает функцию генерации пути
    """

    def update_filename(instance, filename):
        f = fix_image_path(filename)
        return os.path.join(path, f)

    return update_filename

class ImageFieldHash(ImageField):
    def __init__(self, **kwargs):
        if 'upload_to' in kwargs:
            kwargs['upload_to'] = upload_to(kwargs['upload_to'])
        super(ImageFieldHash, self).__init__(**kwargs)
<|endoftext|>


<commit_before>import os, pkgutil, site

if pkgutil.find_loader("gi"):
    try:
        import gi
        print('Found gi:', os.path.abspath(gi.__file__))
        gi.require_version('Gst', '1.0')
        # from gi.repository import GLib, Gst
    except ValueError:
        print('Couldn\'t find Gst')
        print('Please run \'sudo apt-get install gir1.2-gstreamer-1.0\'')
    print('Environment seems to be ok.')
else:
    print('No gi installed', '\n',
          'Please run \'sudo apt-get install python3-gi\'',
          '\n',
          'A virtual environment might need extra actions like symlinking, ',
          '\n',
          'you might need to do a symlink looking similar to this:',
          '\n',
          'ln -s /usr/lib/python3/dist-packages/gi ',
          '/srv/homeassistant/lib/python3.4/site-packages',
          '\n',
          'run this script inside and outside of the virtual environment to find the paths needed')
    print(site.getsitepackages())<commit_msg>Exit program if exception is raised<commit_after>import os
import pkgutil
import site

if pkgutil.find_loader("gi"):
    try:
        import gi
        print('Found gi:', os.path.abspath(gi.__file__))
        gi.require_version('Gst', '1.0')
        # from gi.repository import GLib, Gst
    except ValueError:
        print('Couldn\'t find Gst')
        print('Please run \'sudo apt-get install gir1.2-gstreamer-1.0\'')
        return False
    print('Environment seems to be ok.')
else:
    print('No gi installed', '\n',
          'Please run \'sudo apt-get install python3-gi\'',
          '\n',
          'A virtual environment might need extra actions like symlinking, ',
          '\n',
          'you might need to do a symlink looking similar to this:',
          '\n',
          'ln -s /usr/lib/python3/dist-packages/gi ',
          '/srv/homeassistant/lib/python3.4/site-packages',
          '\n',
          'run this script inside and outside of the virtual environment to find the paths needed')
    print(site.getsitepackages())<|endoftext|>


<commit_before>import logging
import sys
from os import path

from argparse import ArgumentParser

log = logging.getLogger(__name__)

class ArgumentSuperParser(ArgumentParser):

    opt_config  = ('-c', '--config',  'path to the config file')
    opt_verbose = ('-v', '--verbose', 'output more verbose')
    opt_quiet   = ('-q', '--quiet',   'surpress all output')
    opt_version = ('-V', '--version', '%(prog)s ')

    def __init__(self, prog, version, default_config=None, *args, **kwargs):
        self._prog = prog
        self._version = version
        self._default_config_file = default_config

        self._main = self.get_main_parser(self._prog)
        self._main_argv, self._remaining_argv = self._main.parse_known_args()

        if self._main_argv.quiet:
            self._loglevel = 100
        else:
            self._loglevel = max(30 - self._main_argv.verbose * 10, 10)

        logging.basicConfig(level=self._loglevel, format='%(message)s')

        self._config_file = self._main_argv.config_file

        if self._config_file:
            if path.isfile(self._config_file):
                log.info('Reading default configuration from %s', self._config_file)
                from ConfigParser import ConfigParser
                self._config = self.get_config_parser(self._config_file)
                self._defaults = self.parse_config_defaults(self._config, self._prog)
            else:
                log.debug('Ignoring non-existent config file: %s', self._config_file)
                self._config = None
                self._defaults = {}
        else:
            self._config = None
            self._defaults = {}

        obj = super(ArgumentSuperParser, self).__init__(prog=self._prog, parents=[self._main], *args, **kwargs)

        self.add_opt_version(self._version, self)
        self.set_defaults_for_parser(self, self._defaults)

        return obj

    def get_main_parser(self, prog):
        mp = ArgumentParser(prog=prog, add_help=False)
        self.add_opt_config(mp)
        self.add_opt_verbose(mp)
        self.add_opt_quiet(mp)
        return mp

    def get_config_parser(self, file):
        config = ConfigParser()
        config.read(path.expanduser(file))
        return config

    def parse_config_defaults(self, parser, section):
        if parser.has_section(section):
            return dict(parser.items(section))
        else:
            log.debug('Section %s not found. Ignoring.', section)
            return {}

    def add_opt_config(self, parser):
        o = self.opt_config
        parser.add_argument(o[0], o[1], help=o[2], dest='config_file',
                            default=self._default_config_file)

    def add_opt_verbose(self, parser):
        o = self.opt_verbose
        parser.add_argument(o[0], o[1], default=0, action='count', help=o[2])

    def add_opt_quiet(self, parser):
        o = self.opt_quiet
        parser.add_argument(o[0], o[1], action='store_true', help=o[2])

    def add_opt_version(self, version, parser):
        v = self.opt_version
        parser.add_argument(v[0], v[1], action='version', version=v[2] + version)

    def set_defaults_for_parser(self, parser, defaults):
        parser.set_defaults(verbose=self._main_argv.verbose,
                            quiet=self._main_argv.quiet, **defaults)

    def parse_args(self, *args, **kwargs):
        self._args = super(ArgumentSuperParser, self).parse_args(self._remaining_argv)
        return self._args
<commit_msg>Fixed non working ConfigFile Parser<commit_after>import logging
import sys
from os import path

from argparse import ArgumentParser

log = logging.getLogger(__name__)

class ArgumentSuperParser(ArgumentParser):

    opt_config  = ('-c', '--config',  'path to the config file')
    opt_verbose = ('-v', '--verbose', 'output more verbose')
    opt_quiet   = ('-q', '--quiet',   'surpress all output')
    opt_version = ('-V', '--version', '%(prog)s ')

    def __init__(self, prog, version, default_config=None, *args, **kwargs):
        self._prog = prog
        self._version = version
        self._default_config_file = default_config

        self._main = self.get_main_parser(self._prog)
        self._main_argv, self._remaining_argv = self._main.parse_known_args()

        if self._main_argv.quiet:
            self._loglevel = 100
        else:
            self._loglevel = max(30 - self._main_argv.verbose * 10, 10)

        logging.basicConfig(level=self._loglevel, format='%(message)s')

        self._config_file = path.expanduser(self._main_argv.config_file)

        if self._config_file:
            if path.isfile(self._config_file):
                log.info('Reading default configuration from %s', self._config_file)
                from ConfigParser import ConfigParser
                self._config = ConfigParser()
                self._config.read(self._config_file)
                self._default_config = self.parse_config_defaults(self._config, self._prog)
            else:
                log.debug('Ignoring non-existent config file: %s', self._config_file)
                self._config = None
                self._default_config = {}
        else:
            self._config = None
            self._default_config = {}

        super(ArgumentSuperParser, self).__init__(prog=self._prog, parents=[self._main], *args, **kwargs)
        self.add_opt_version(self._version, self)
        self.set_defaults(verbose=self._main_argv.verbose,
                            quiet=self._main_argv.quiet, **self._default_config)

    def get_main_parser(self, prog):
        mp = ArgumentParser(prog=prog, add_help=False)
        self.add_opt_config(mp)
        self.add_opt_verbose(mp)
        self.add_opt_quiet(mp)
        return mp

    def parse_config_defaults(self, parser, section):
        if parser.has_section(section):
            return dict(parser.items(section))
        else:
            log.debug('Section %s not found. Ignoring.', section)
            return {}

    def add_opt_config(self, parser):
        o = self.opt_config
        parser.add_argument(o[0], o[1], help=o[2], dest='config_file',
                            default=self._default_config_file)

    def add_opt_verbose(self, parser):
        o = self.opt_verbose
        parser.add_argument(o[0], o[1], default=0, action='count', help=o[2])

    def add_opt_quiet(self, parser):
        o = self.opt_quiet
        parser.add_argument(o[0], o[1], action='store_true', help=o[2])

    def add_opt_version(self, version, parser):
        v = self.opt_version
        parser.add_argument(v[0], v[1], action='version', version=v[2] + version)

    def parse_args(self, *args, **kwargs):
        self._args = super(ArgumentSuperParser, self).parse_args(self._remaining_argv)
        return self._args
<|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy
import spectral

class MineralClassification:

    def __init__(self, libraryFilename):
        """
        Construct a new MineralClassification object with the USGS Digital
        Spectral Library 06.

        Args:
            libraryFilename (str):    filename of the spectral library
        """
        self.library = spectral.open_image(libraryFilename)

    def classifyImage(self, imageFilename, classifiedFilename):
        """
        Classify minerals in an AVIRIS image using spectral angle mapper
        classification and save the results to a file.

        Args:
            imageFilename (str):      filename of the image to be classified
            classifiedFilename (str): filename of the classified image

        Returns:
            None
        """

        # open the image
        image = spectral.open_image(imageFilename)
        data = image.asarray()
        M = image.shape[0]
        N = image.shape[1]

        # define a resampler
        # TODO detect and scale units
        # TODO band resampler should do this
        resample = spectral.BandResampler([x/1000 for x in image.bands.centers],
                                      self.library.bands.centers)

        # allocate a zero-initialized MxN array for the classified image
        classified = numpy.zeros(shape=(M,N), dtype=numpy.uint16)

        # for each pixel in the image
        for x in xrange(M):

            for y in xrange(N):

                # read the pixel from the file
                pixel = data[x,y]

                # if it is not a no data pixel
                if not numpy.isclose(pixel[0], -0.005):

                    # resample the pixel ignoring NaNs from target bands that don't overlap
                    # TODO fix spectral library so that bands are in order
                    resampledPixel = numpy.nan_to_num(resample(pixel))

                    # calculate spectral angles
                    angles = spectral.spectral_angles(resampledPixel[numpy.newaxis,
                                                                     numpy.newaxis,
                                                                     ...],
                                                      self.library.spectra)

                    # get classification
                    classified[x,y] = numpy.argmin(angles) + 1

        # save the classified image to a file
        spectral.io.envi.save_classification(classifiedFilename,
                                             classified,
                                             class_names=['No data']+self.library.names,
                                             metadata={
                                                 'data ignore value': 0,
                                                 'description': 'Mineral classified image.',
                                                 'map info': image.metadata.get('map info')
                                             })
<commit_msg>detect AVIRIS-C no data value<commit_after># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy
import spectral

class MineralClassification:

    def __init__(self, libraryFilename):
        """
        Construct a new MineralClassification object with the USGS Digital
        Spectral Library 06.

        Args:
            libraryFilename (str):    filename of the spectral library
        """
        self.library = spectral.open_image(libraryFilename)

    def classifyImage(self, imageFilename, classifiedFilename):
        """
        Classify minerals in an AVIRIS image using spectral angle mapper
        classification and save the results to a file.

        Args:
            imageFilename (str):      filename of the image to be classified
            classifiedFilename (str): filename of the classified image

        Returns:
            None
        """

        # open the image
        image = spectral.open_image(imageFilename)
        data = image.asarray()
        M = image.shape[0]
        N = image.shape[1]

        # define a resampler
        # TODO detect and scale units
        # TODO band resampler should do this
        resample = spectral.BandResampler([x/1000 for x in image.bands.centers],
                                      self.library.bands.centers)

        # allocate a zero-initialized MxN array for the classified image
        classified = numpy.zeros(shape=(M,N), dtype=numpy.uint16)

        # for each pixel in the image
        for x in xrange(M):

            for y in xrange(N):

                # read the pixel from the file
                pixel = data[x,y]

                # if it is not a no data pixel
                if not numpy.isclose(pixel[0], -0.005) and not pixel[0]==-50:

                    # resample the pixel ignoring NaNs from target bands that don't overlap
                    # TODO fix spectral library so that bands are in order
                    resampledPixel = numpy.nan_to_num(resample(pixel))

                    # calculate spectral angles
                    angles = spectral.spectral_angles(resampledPixel[numpy.newaxis,
                                                                     numpy.newaxis,
                                                                     ...],
                                                      self.library.spectra)

                    # get classification
                    classified[x,y] = numpy.argmin(angles) + 1

        # save the classified image to a file
        spectral.io.envi.save_classification(classifiedFilename,
                                             classified,
                                             class_names=['No data']+self.library.names,
                                             metadata={
                                                 'data ignore value': 0,
                                                 'description': 'Mineral classified image.',
                                                 'map info': image.metadata.get('map info')
                                             })
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""xml2json.py  Convert XML to JSON

Relies on ElementTree for the XML parsing.  This is based on
pesterfish.py but uses a different XML->JSON mapping.
The XML->JSON mapping is described at
http://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html

Rewritten to a command line utility by Hay Kranen < github.com/hay > with
contributions from George Hamilton (gmh04) and Dan Brown (jdanbrown)

XML                              JSON
<e/>                             "e": null
<e>text</e>                      "e": "text"
<e name="value" />               "e": { "@name": "value" }
<e name="value">text</e>         "e": { "@name": "value", "#text": "text" }
<e> texttext </e> "e": { "a": "text", "b": "text" }
<e> text text </e> "e": { "a": ["text", "text"] }
<e> text text </e>        "e": { "#text": "text", "a": "text" }

This is very similar to the mapping used for Yahoo Web Services
(http://developer.yahoo.com/common/json.html#xml).

This is a mess in that it is so unpredictable -- it requires lots of testing
(e.g. to see if values are lists or strings or dictionaries).  For use
in Python this could be vastly cleaner.  Think about whether the internal
form can be more self-consistent while maintaining good external
characteristics for the JSON.

Look at the Yahoo version closely to see how it works.  Maybe can adopt
that completely if it makes more sense...

R. White, 2006 November 6
"""
from __future__ import print_function, absolute_import

import sys
import json
import argparse

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET

def strip_tag(tag):
    strip_ns_tag = tag
    split_array = tag.split('}')
    if len(split_array) > 1:
        strip_ns_tag = split_array[1]
        tag = strip_ns_tag
    return tag

def elem_to_internal(elem, strip_ns=1, strip=1):
    """Convert an Element into an internal dictionary (not JSON!)."""

    d = {}
    elem_tag = elem.tag
    if strip_ns:
        elem_tag = strip_tag(elem.tag)
    else:
        for key, value in list(elem.attrib.items()):
            d['@' + key] = value

    # loop over subelements to merge them
    for subelem in elem:
        v = elem_to_internal(subelem, strip_ns=strip_ns, strip=strip)

        tag = subelem.tag
        if strip_ns:
            tag = strip_tag(subelem.tag)

        value = v[tag]

        try:
            # add to existing list for this tag
            d[tag].append(value)
        except AttributeError:
            # turn existing entry into a list
            d[tag] = [d[tag], value]
        except KeyError:
            # add a new non-list entry
            d[tag] = value
    text = elem.text
    tail = elem.tail
    if strip:
        # ignore leading and trailing whitespace
        if text:
            text = text.strip()
        if tail:
            tail = tail.strip()

    if tail:
        d['#tail'] = tail

    if d:
        # use #text element if other attributes exist
        if text:
            d["#text"] = text
    else:
        # text is the value if no attributes
        d = text or None
    return {elem_tag: d}

def internal_to_elem(pfsh, factory=ET.Element):
    """Convert an internal dictionary (not JSON!) into an Element.

    Whatever Element implementation we could import will be
    used by default; if you want to use something else, pass the
    Element class as the factory parameter.
    """

    attribs = {}
    text = None
    tail = None
    sublist = []
    tag = list(pfsh.keys())
    if len(tag) != 1:
        raise ValueError("Illegal structure with multiple tags: %s" % tag)
    tag = tag[0]
    value = pfsh[tag]
    if isinstance(value, dict):
        for k, v in list(value.items()):
            if k[:1] == "@":
                attribs[k[1:]] = v
            elif k == "#text":
                text = v
            elif k == "#tail":
                tail = v
            elif isinstance(v, list):
                for v2 in v:
                    sublist.append(internal_to_elem({k: v2}, factory=factory))
            else:
                sublist.append(internal_to_elem({k: v}, factory=factory))
    else:
        text = value
    e = factory(tag, attribs)
    for sub in sublist:
        e.append(sub)
    e.text = text
    e.tail = tail
    return e

def elem2json(elem, pretty=False, strip_ns=1, strip=1):
    """Convert an ElementTree or Element into a JSON string."""

    if hasattr(elem, 'getroot'):
        elem = elem.getroot()

    if pretty:
        return json.dumps(elem_to_internal(elem, strip_ns=strip_ns, strip=strip), sort_keys=True, indent=4, separators=(',', ': '))
    else:
        return json.dumps(elem_to_internal(elem, strip_ns=strip_ns, strip=strip))

def json2elem(json_data, factory=ET.Element):
    """Convert a JSON string into an Element.

    Whatever Element implementation we could import will be used by
    default; if you want to use something else, pass the Element class
    as the factory parameter.
    """

    return internal_to_elem(json.loads(json_data), factory)

def xml2json(xmlstring, pretty=False, strip_ns=1, strip=1):
    """Convert an XML string into a JSON string."""

    elem = ET.fromstring(xmlstring)
    return elem2json(elem, pretty, strip_ns=strip_ns, strip=strip)

def json2xml(json_data, factory=ET.Element):
    """Convert a JSON string into an XML string.

    Whatever Element implementation we could import will be used by
    default; if you want to use something else, pass the Element class
    as the factory parameter.
    """
    if not isinstance(json_data, dict):
        json_data = json.loads(json_data)

    elem = internal_to_elem(json_data, factory)
    return ET.tostring(elem)

def main():
    p = argparse.ArgumentParser(
        description='Converts XML to JSON or the other way around.  Reads from standard input by default, or from file if given.',
        prog='xml2json',
        usage='%(prog)s -t xml2json -o file.json [file]'
    )
    p.add_argument('--type', '-t', help="'xml2json' or 'json2xml'", default="xml2json")
    p.add_argument('--out', '-o', help="Write to the file 'OUT' instead of stdout")
    p.add_argument('--fin', '-f', help="Read from the file 'IN' instead of stdin")
    p.add_argument(
        '--strip_text', action="store_true",
        dest="strip_text", help="Strip text for xml2json")
    p.add_argument(
        '--pretty', action="store_true",
        dest="pretty", help="Format JSON output so it is easier to read")
    p.add_argument(
        '--strip_namespace', action="store_true",
        dest="strip_ns", help="Strip namespace for xml2json")
    p.add_argument(
        '--strip_newlines', action="store_true",
        dest="strip_nl", help="Strip newlines for xml2json")
    args = p.parse_args()

    if args.fin:
        with open(args.fin) as f:
            data = f.read()
    else:
        #data = input()
        data = sys.stdin.readline()

    strip = 0
    strip_ns = 0
    if args.strip_text:
        strip = 1
    if args.strip_ns:
        strip_ns = 1
    if args.strip_nl:
        data = data.replace('\n', '').replace('\r', '')
    if (args.type == "xml2json"):
        out = xml2json(data, args.pretty, strip_ns, strip)
    else:
        out = json2xml(data)

    if (args.out):
        file = open(args.out, 'w')
        file.write(out)
        file.close()
    else:
        print(out)

if __name__ == "__main__":
    main()
<commit_msg>Add the function to convert between XML and JSON.<commit_after>#!/usr/bin/env python

"""xml2json.py  Convert XML to JSON

Relies on ElementTree for the XML parsing.  This is based on
pesterfish.py but uses a different XML->JSON mapping.
The XML->JSON mapping is described at
http://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html

Rewritten to a command line utility by Hay Kranen < github.com/hay > with
contributions from George Hamilton (gmh04) and Dan Brown (jdanbrown)

XML                              JSON
<e/>                             "e": null
<e>text</e>                      "e": "text"
<e name="value" />               "e": { "@name": "value" }
<e name="value">text</e>         "e": { "@name": "value", "#text": "text" }
<e> texttext </e> "e": { "a": "text", "b": "text" }
<e> text text </e> "e": { "a": ["text", "text"] }
<e> text text </e>        "e": { "#text": "text", "a": "text" }

This is very similar to the mapping used for Yahoo Web Services
(http://developer.yahoo.com/common/json.html#xml).

This is a mess in that it is so unpredictable -- it requires lots of testing
(e.g. to see if values are lists or strings or dictionaries).  For use
in Python this could be vastly cleaner.  Think about whether the internal
form can be more self-consistent while maintaining good external
characteristics for the JSON.

Look at the Yahoo version closely to see how it works.  Maybe can adopt
that completely if it makes more sense...

R. White, 2006 November 6
"""
from __future__ import print_function, absolute_import

import sys
import json
import argparse

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET

def strip_tag(tag):
    strip_ns_tag = tag
    split_array = tag.split('}')
    if len(split_array) > 1:
        strip_ns_tag = split_array[1]
        tag = strip_ns_tag
    return tag

def elem_to_internal(elem, strip_ns=1, strip=1):
    """Convert an Element into an internal dictionary (not JSON!)."""

    d = {}
    elem_tag = elem.tag
    if strip_ns:
        elem_tag = strip_tag(elem.tag)
    else:
        for key, value in list(elem.attrib.items()):
            d['@' + key] = value

    # loop over subelements to merge them
    for subelem in elem:
        v = elem_to_internal(subelem, strip_ns=strip_ns, strip=strip)

        tag = subelem.tag
        if strip_ns:
            tag = strip_tag(subelem.tag)

        value = v[tag]

        try:
            # add to existing list for this tag
            d[tag].append(value)
        except AttributeError:
            # turn existing entry into a list
            d[tag] = [d[tag], value]
        except KeyError:
            # add a new non-list entry
            d[tag] = value
    text = elem.text
    tail = elem.tail
    if strip:
        # ignore leading and trailing whitespace
        if text:
            text = text.strip()
        if tail:
            tail = tail.strip()

    if tail:
        d['#tail'] = tail

    if d:
        # use #text element if other attributes exist
        if text:
            d["#text"] = text
    else:
        # text is the value if no attributes
        d = text or None
    return {elem_tag: d}

def internal_to_elem(pfsh, factory=ET.Element):
    """Convert an internal dictionary (not JSON!) into an Element.

    Whatever Element implementation we could import will be
    used by default; if you want to use something else, pass the
    Element class as the factory parameter.
    """

    attribs = {}
    text = None
    tail = None
    sublist = []
    tag = list(pfsh.keys())
    if len(tag) != 1:
        raise ValueError("Illegal structure with multiple tags: %s" % tag)
    tag = tag[0]
    value = pfsh[tag]
    if isinstance(value, dict):
        for k, v in list(value.items()):
            if k[:1] == "@":
                attribs[k[1:]] = v
            elif k == "#text":
                text = v
            elif k == "#tail":
                tail = v
            elif isinstance(v, list):
                for v2 in v:
                    sublist.append(internal_to_elem({k: v2}, factory=factory))
            else:
                sublist.append(internal_to_elem({k: v}, factory=factory))
    else:
        text = value
    e = factory(tag, attribs)
    for sub in sublist:
        e.append(sub)
    e.text = text
    e.tail = tail
    return e

def elem2json(elem, pretty=False, strip_ns=1, strip=1):
    """Convert an ElementTree or Element into a JSON string."""

    if hasattr(elem, 'getroot'):
        elem = elem.getroot()

    if pretty:
        return json.dumps(elem_to_internal(elem, strip_ns=strip_ns, strip=strip), sort_keys=True, indent=4, separators=(',', ': '))
    else:
        return json.dumps(elem_to_internal(elem, strip_ns=strip_ns, strip=strip))

def json2elem(json_data, factory=ET.Element):
    """Convert a JSON string into an Element.

    Whatever Element implementation we could import will be used by
    default; if you want to use something else, pass the Element class
    as the factory parameter.
    """

    return internal_to_elem(json.loads(json_data), factory)

def xml2json(xmlstring, pretty=False, strip_ns=1, strip=1):
    """Convert an XML string into a JSON string."""

    elem = ET.fromstring(xmlstring)
    return elem2json(elem, pretty, strip_ns=strip_ns, strip=strip)

def json2xml(json_data, factory=ET.Element):
    """Convert a JSON string into an XML string.

    Whatever Element implementation we could import will be used by
    default; if you want to use something else, pass the Element class
    as the factory parameter.
    """
    if not isinstance(json_data, dict):
        json_data = json.loads(json_data)

    elem = internal_to_elem(json_data, factory)
    return ET.tostring(elem)

def xml2dict(xmlstring, strip_ns=1, strip=1):
    elem = ET.fromstring(xmlstring)
    if hasattr(elem, 'getroot'):
        elem = elem.getroot()
    return elem_to_internal(elem, strip_ns=strip_ns, strip=strip)

def dict2xml(dict_data, factory=ET.Element):
    elem = internal_to_elem(dict_data, factory)
    return ET.tostring(elem)

def main():
    p = argparse.ArgumentParser(
        description='Converts XML to JSON or the other way around.  Reads from standard input by default, or from file if given.',
        prog='xml2json',
        usage='%(prog)s -t xml2json -o file.json [file]'
    )
    p.add_argument('--type', '-t', help="'xml2json' or 'json2xml'", default="xml2json")
    p.add_argument('--out', '-o', help="Write to the file 'OUT' instead of stdout")
    p.add_argument('--fin', '-f', help="Read from the file 'IN' instead of stdin")
    p.add_argument(
        '--strip_text', action="store_true",
        dest="strip_text", help="Strip text for xml2json")
    p.add_argument(
        '--pretty', action="store_true",
        dest="pretty", help="Format JSON output so it is easier to read")
    p.add_argument(
        '--strip_namespace', action="store_true",
        dest="strip_ns", help="Strip namespace for xml2json")
    p.add_argument(
        '--strip_newlines', action="store_true",
        dest="strip_nl", help="Strip newlines for xml2json")
    args = p.parse_args()

    if args.fin:
        with open(args.fin) as f:
            data = f.read()
    else:
        #data = input()
        data = sys.stdin.readline()

    strip = 0
    strip_ns = 0
    if args.strip_text:
        strip = 1
    if args.strip_ns:
        strip_ns = 1
    if args.strip_nl:
        data = data.replace('\n', '').replace('\r', '')
    if (args.type == "xml2json"):
        out = xml2json(data, args.pretty, strip_ns, strip)
    else:
        out = json2xml(data)

    if (args.out):
        file = open(args.out, 'w')
        file.write(out)
        file.close()
    else:
        print(out)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import boto3
import botocore

__all__ = [
    'init_adapter', 'get_current_images_on_ecs', 'get_latest_images_from_ecr_registry',
    'get_s3_file', 'get_images_by_repository', 'delete_images_from_repository'
]

def init_adapter(cn):
    pass

def get_current_images_on_ecs(cn, cluster, region='us-east-1'):
    ''' Returns with the images currently running on an ecs cluster.

    :param cluster: name of the ecs cluster
    :type cluster: str
    :param region: aws region
    :type region: str

    :return: dict
    '''

    ecs_client = boto3.client('ecs', region_name=region)
    current_images = {}

    services = _get_all_services_on_ecs(ecs_client, cluster)
    for service in services:
        last_parts = service['image'].split('/', 1)[-1].split(':')
        image_name = last_parts[0]
        if last_parts[1][0] == 'v':
            version = int(last_parts[1][1:])
            current_images.update({image_name: (version, service['name'])})

    return current_images

def get_latest_images_from_ecr_registry(cn, registry_id, region='us-east-1'):
    ''' Returns with the latest images in the ecr (registry).

    :param registry_id: id of the ecr registry
    :type registry_id: str
    :param region: aws region
    :type region: str

    :return: dict
    '''
    ecr_client = boto3.client('ecr', region_name=region)

    latest_images = {}
    repositories = _get_all_resources(ecr_client.describe_repositories, 'repositories', registryId=registry_id)

    for repo in repositories:
        images = get_images_by_repository(cn, repo['repositoryName'], ecr_client=ecr_client, region=region)
        latest_images.update({repo['repositoryName']: (_get_latest_version(images), )})

    return latest_images

def get_s3_file(cn, bucket, key, region='us-east-1'):
    ''' Returns with a content of an s3 file.

    :param bucket: bucket name
    :type bucket: str
    :param key: s3 object key
    :type key: str
    :param region: aws region
    :type region: str

    :return: bytes
    '''

    client = boto3.client('s3', region_name=region)
    s3_object = client.get_object(Bucket=bucket, Key=key)
    return s3_object['Body'].read()

def get_images_by_repository(cn, repository, region='us-east-1', ecr_client=None):
    ''' Returns with all images of the given repository

    :param repository: repository name
    :type repository: str
    :param region: aws region
    :type region: str
    :param ecr_client: boto3 client for ecr
    :type region: obj

    :return: list
    '''
    if not ecr_client:
        ecr_client = boto3.client('ecr', region_name=region)

    try:
        return _get_all_resources(ecr_client.list_images, 'imageIds', repositoryName=repository)
    except botocore.exceptions.ClientError as e:
        return []

    return []

def delete_images_from_repository(cn, repository, image_digests, region='us-east-1'):
    ecr_client = boto3.client('ecr', region_name=region)

    response = ecr_client.batch_delete_image(
        registryId=cn.g_('app_config').get('ecr_registry'),
        repositoryName=repository,
        imageIds=[
            {'imageDigest': digest} for digest in image_digests
        ]
    )

    return response

def _get_latest_version(images):
    latest_version = 0
    for image in images:
        version = image.get('imageTag', '')[1:]

        try:
            if latest_version < int(version):
                latest_version = int(version)
        except ValueError:
            pass

    return latest_version

def _get_all_services_on_ecs(client, cluster):
    services = []
    images = []

    tasks = _get_tasks(client, cluster)
    for task in tasks['tasks']:
        service = client.describe_task_definition(taskDefinition=task['taskDefinitionArn'])['taskDefinition']['containerDefinitions'][0]
        if service['image'] not in images:
            services.append(service)
            images.append(service['image'])
            # print(service['image'])

    return services

def _get_tasks(client, cluster):
    task_arns = _get_all_resources(client.list_tasks, 'taskArns', maxResults=100, cluster=cluster)
    return client.describe_tasks(cluster=cluster, tasks=task_arns)

def _get_all_resources(fn, field_name, **kwargs):
    resources = []
    next_token = ''

    while isinstance(next_token, str):
        if next_token:
            kwargs.update({'nextToken': next_token})

        partial_result = fn(**kwargs)
        resources += partial_result[field_name]
        next_token = partial_result.get('nextToken')

    return resources
<commit_msg>delete more than 100 images at once<commit_after>import boto3
import botocore

__all__ = [
    'init_adapter', 'get_current_images_on_ecs', 'get_latest_images_from_ecr_registry',
    'get_s3_file', 'get_images_by_repository', 'delete_images_from_repository'
]

def init_adapter(cn):
    pass

def get_current_images_on_ecs(cn, cluster, region='us-east-1'):
    ''' Returns with the images currently running on an ecs cluster.

    :param cluster: name of the ecs cluster
    :type cluster: str
    :param region: aws region
    :type region: str

    :return: dict
    '''

    ecs_client = boto3.client('ecs', region_name=region)
    current_images = {}

    services = _get_all_services_on_ecs(ecs_client, cluster)
    for service in services:
        last_parts = service['image'].split('/', 1)[-1].split(':')
        image_name = last_parts[0]
        if last_parts[1][0] == 'v':
            version = int(last_parts[1][1:])
            current_images.update({image_name: (version, service['name'])})

    return current_images

def get_latest_images_from_ecr_registry(cn, registry_id, region='us-east-1'):
    ''' Returns with the latest images in the ecr (registry).

    :param registry_id: id of the ecr registry
    :type registry_id: str
    :param region: aws region
    :type region: str

    :return: dict
    '''
    ecr_client = boto3.client('ecr', region_name=region)

    latest_images = {}
    repositories = _get_all_resources(ecr_client.describe_repositories, 'repositories', registryId=registry_id)

    for repo in repositories:
        images = get_images_by_repository(cn, repo['repositoryName'], ecr_client=ecr_client, region=region)
        latest_images.update({repo['repositoryName']: (_get_latest_version(images), )})

    return latest_images

def get_s3_file(cn, bucket, key, region='us-east-1'):
    ''' Returns with a content of an s3 file.

    :param bucket: bucket name
    :type bucket: str
    :param key: s3 object key
    :type key: str
    :param region: aws region
    :type region: str

    :return: bytes
    '''

    client = boto3.client('s3', region_name=region)
    s3_object = client.get_object(Bucket=bucket, Key=key)
    return s3_object['Body'].read()

def get_images_by_repository(cn, repository, region='us-east-1', ecr_client=None):
    ''' Returns with all images of the given repository

    :param repository: repository name
    :type repository: str
    :param region: aws region
    :type region: str
    :param ecr_client: boto3 client for ecr
    :type region: obj

    :return: list
    '''
    if not ecr_client:
        ecr_client = boto3.client('ecr', region_name=region)

    try:
        return _get_all_resources(ecr_client.list_images, 'imageIds', repositoryName=repository)
    except botocore.exceptions.ClientError as e:
        return []

    return []

def delete_images_from_repository(cn, repository, image_digests, region='us-east-1'):
    ecr_client = boto3.client('ecr', region_name=region)
    responses = []

    start_idx = 0
    while start_idx < len(image_digests):
        response = ecr_client.batch_delete_image(
            registryId=cn.g_('app_config').get('ecr_registry'),
            repositoryName=repository,
            imageIds=[
                {'imageDigest': digest} for digest in image_digests[start_idx:start_idx + 99]
            ]
        )

        responses.append(response)
        start_idx += 99

    return responses

def _get_latest_version(images):
    latest_version = 0
    for image in images:
        version = image.get('imageTag', '')[1:]

        try:
            if latest_version < int(version):
                latest_version = int(version)
        except ValueError:
            pass

    return latest_version

def _get_all_services_on_ecs(client, cluster):
    services = []
    images = []

    tasks = _get_tasks(client, cluster)
    for task in tasks['tasks']:
        service = client.describe_task_definition(taskDefinition=task['taskDefinitionArn'])['taskDefinition']['containerDefinitions'][0]
        if service['image'] not in images:
            services.append(service)
            images.append(service['image'])
            # print(service['image'])

    return services

def _get_tasks(client, cluster):
    task_arns = _get_all_resources(client.list_tasks, 'taskArns', maxResults=100, cluster=cluster)
    return client.describe_tasks(cluster=cluster, tasks=task_arns)

def _get_all_resources(fn, field_name, **kwargs):
    resources = []
    next_token = ''

    while isinstance(next_token, str):
        if next_token:
            kwargs.update({'nextToken': next_token})

        partial_result = fn(**kwargs)
        resources += partial_result[field_name]
        next_token = partial_result.get('nextToken')

    return resources
<|endoftext|>