<commit_before># Copyright 2015 Infoblox Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_log import log as logging

from infoblox_client import exceptions as ib_ex
from infoblox_client import objects as obj
from infoblox_client import utils as ib_utils

LOG = logging.getLogger(__name__)

class InfobloxObjectManager(object):

    def __init__(self, connector):
        self.connector = connector

    def create_network_view(self, network_view, extattrs):
        return obj.NetworkView.create(self.connector,
                                      name=network_view,
                                      extattrs=extattrs)

    def delete_network_view(self, network_view):
        # never delete default network view
        if network_view == 'default':
            return
        nview = obj.NetworkView.search(self.connector,
                                       name=network_view)
        if nview:
            nview.delete()

    def create_dns_view(self, network_view, dns_view):
        return obj.DNSView.create(self.connector,
                                  name=dns_view,
                                  network_view=network_view)

    def delete_dns_view(self, dns_view):
        dns_view = obj.DNSView.search(self.connector,
                                      name=dns_view)
        if dns_view:
            dns_view.delete()

    def create_network(self, net_view_name, cidr, nameservers=None,
                       members=None, gateway_ip=None, dhcp_trel_ip=None,
                       network_extattrs=None):
        """Create NIOS Network."""

        # NIOS does not allow to set Dhcp options for IPv6 over WAPI,
        # so limit options usage with IPv4 only
        ipv4 = ib_utils.determine_ip_version(cidr) == 4

        options = []
        if ipv4 and nameservers:
            options.append(obj.DhcpOption(name='domain-name-servers',
                                          value=",".join(nameservers)))
        if ipv4 and gateway_ip:
            options.append(obj.DhcpOption(name='routers',
                                          value=gateway_ip))
        if ipv4 and dhcp_trel_ip:
            options.append(obj.DhcpOption(name='dhcp-server-identifier',
                                          num=54,
                                          value=dhcp_trel_ip))
        return obj.Network.create(self.connector,
                                  network_view=net_view_name,
                                  cidr=cidr,
                                  members=members,
                                  options=options,
                                  extattrs=network_extattrs,
                                  check_if_exists=False)

    def get_network(self, network_view, cidr):
        return obj.Network.search(self.connector,
                                  network_view=network_view,
                                  cidr=cidr)

    def create_ip_range(self, network_view, start_ip, end_ip, network,
                        disable, range_extattrs):
        """Creates IPRange or fails if already exists."""
        return obj.IPRange.create(self.connector,
                                  network_view=network_view,
                                  start_addr=start_ip,
                                  end_addr=end_ip,
                                  cidr=network,
                                  disable=disable,
                                  extattrs=range_extattrs,
                                  check_if_exists=False)

    def delete_ip_range(self, network_view, start_ip, end_ip):
        range = obj.IPRange.search(self.connector,
                                   network_view=network_view,
                                   start_addr=start_ip,
                                   end_addr=end_ip)
        if range:
            range.delete()

    def has_networks(self, network_view_name):
        networks = obj.Network.search_all(self.connector,
                                          network_view=network_view_name)
        return bool(networks)

    def network_exists(self, network_view, cidr):
        """Deprecated, use get_network() instead."""
        LOG.warning(
            "DEPRECATION WARNING! Using network_exists() is deprecated "
            "and to be removed in next releases. "
            "Use get_network() or objects.Network.search instead")
        network = obj.Network.search(self.connector,
                                     network_view=network_view,
                                     cidr=cidr)
        return network is not None

    def delete_network(self, network_view, cidr):
        network = obj.Network.search(self.connector,
                                     network_view=network_view,
                                     cidr=cidr)
        if network:
            network.delete()

    def create_network_from_template(self, network_view, cidr, template,
                                     extattrs):
        return obj.Network.create(self.connector,
                                  network_view=network_view,
                                  cidr=cidr,
                                  template=template,
                                  extattrs=extattrs,
                                  check_if_exists=False)

    def update_network_options(self, ib_network, extattrs=None):
        if extattrs:
            ib_network.extattrs = extattrs
        return ib_network.update()

    def get_host_record(self, dns_view, ip):
        return obj.HostRecord.search(self.connector,
                                     view=dns_view,
                                     ip=ip)

    def find_hostname(self, dns_view, hostname, ip):
        return obj.HostRecord.search(self.connector,
                                     name=hostname,
                                     view=dns_view,
                                     ip=ip)

    def create_host_record_for_given_ip(self, dns_view, zone_auth,
                                        hostname, mac, ip, extattrs,
                                        use_dhcp):
        name = '.'.join([hostname, zone_auth])
        ip_obj = obj.IP.create(ip=ip, mac=mac, configure_for_dhcp=use_dhcp)
        return obj.HostRecord.create(self.connector,
                                     view=dns_view,
                                     name=name,
                                     ip=ip_obj,
                                     extattrs=extattrs,
                                     check_if_exists=False)

    def create_host_record_from_range(self, dns_view, network_view_name,
                                      zone_auth, hostname, mac, first_ip,
                                      last_ip, extattrs, use_dhcp):
        name = '.'.join([hostname, zone_auth])
        ip_alloc = obj.IPAllocation.next_available_ip_from_range(
            network_view_name, first_ip, last_ip)
        ip_obj = obj.IP.create(ip=ip_alloc, mac=mac,
                               configure_for_dhcp=use_dhcp)
        return obj.HostRecord.create(self.connector,
                                     view=dns_view,
                                     name=name,
                                     ip=ip_obj,
                                     extattrs=extattrs,
                                     check_if_exists=False)

    def delete_host_record(self, dns_view, ip_address):
        host_record = obj.HostRecord.search(self.connector,
                                            view=dns_view, ip=ip_address)
        if host_record:
            host_record.delete()

    def create_fixed_address_for_given_ip(self, network_view, mac, ip,
                                          extattrs):
        return obj.FixedAddress.create(self.connector,
                                       network_view=network_view,
                                       mac=mac,
                                       ip=ip,
                                       extattrs=extattrs,
                                       check_if_exists=False)

    def create_fixed_address_from_range(self, network_view, mac, first_ip,
                                        last_ip, extattrs):
        ip = obj.IPAllocation.next_available_ip_from_range(
            network_view, first_ip, last_ip)
        return obj.FixedAddress.create(self.connector,
                                       ip=ip,
                                       mac=mac,
                                       network_view=network_view,
                                       extattrs=extattrs,
                                       check_if_exists=False)

    def create_fixed_address_from_cidr(self, netview, mac, cidr, extattrs):
        ip = obj.IPAllocation.next_available_ip_from_cidr(netview, cidr)
        return obj.FixedAddress.create(self.connector,
                                       network_view=netview,
                                       ip=ip,
                                       mac=mac,
                                       extattrs=extattrs,
                                       check_if_exists=False)

    def delete_fixed_address(self, network_view, ip_address):
        fixed_address = obj.FixedAddress.search(self.connector,
                                                network_view=network_view,
                                                ip=ip_address)
        if fixed_address:
            fixed_address.delete()

    def add_ip_to_record(self, host_record, ip, mac, use_dhcp=True):
        ip_obj = obj.IP.create(ip=ip, mac=mac, configure_for_dhcp=use_dhcp)
        host_record.ip.append(ip_obj)
        return host_record.update()

    def add_ip_to_host_record_from_range(self, host_record, network_view,
                                         mac, first_ip, last_ip,
                                         use_dhcp=True):
        ip_alloc = obj.IPAllocation.next_available_ip_from_range(
            network_view, first_ip, last_ip)
        ip_obj = obj.IP.create(ip=ip_alloc, mac=mac,
                               configure_for_dhcp=use_dhcp)
        host_record.ip.append(ip_obj)
        return host_record.update()

    def delete_ip_from_host_record(self, host_record, ip):
        host_record.ip.remove(ip)
        return host_record.update()

    def has_dns_zones(self, dns_view):
        zones = obj.DNSZone.search_all(self.connector, view=dns_view)
        return bool(zones)

    def create_dns_zone(self, dns_view, dns_zone,
                        grid_primary=None, grid_secondaries=None,
                        zone_format=None, ns_group=None, prefix=None,
                        extattrs=None):
        return obj.DNSZone.create(self.connector,
                                  fqdn=dns_zone,
                                  view=dns_view,
                                  extattrs=extattrs,
                                  zone_format=zone_format,
                                  ns_group=ns_group,
                                  prefix=prefix,
                                  grid_primary=grid_primary,
                                  grid_secondaries=grid_secondaries)

    def delete_dns_zone(self, dns_view, dns_zone_fqdn):
        dns_zone = obj.DNSZone.search(self.connector,
                                      fqdn=dns_zone_fqdn,
                                      view=dns_view)
        if dns_zone:
            dns_zone.delete()

    def update_host_record_eas(self, dns_view, ip, extattrs):
        host_record = obj.HostRecord.search(self.connector,
                                            view=dns_view,
                                            ip=ip)
        if host_record:
            host_record.extattrs = extattrs
            host_record.update()

    def update_fixed_address_eas(self, network_view, ip, extattrs):
        fixed_address = obj.FixedAddress.search(self.connector,
                                                network_view=network_view,
                                                ip=ip)
        if fixed_address:
            fixed_address.extattrs = extattrs
            fixed_address.update()

    def update_dns_record_eas(self, dns_view, ip, extattrs):
        a_record = obj.ARecordBase.search(self.connector,
                                          ip=ip,
                                          view=dns_view)
        if a_record:
            a_record.extattrs = extattrs
            a_record.update()

        ptr_record = obj.PtrRecord.search(self.connector,
                                          ip=ip,
                                          view=dns_view)
        if ptr_record:
            ptr_record.extattrs = extattrs
            ptr_record.update()

    def bind_name_with_host_record(self, dns_view, ip, name, extattrs):
        host_record = obj.HostRecord.search(self.connector,
                                            view=dns_view,
                                            ip=ip)
        if host_record:
            host_record.name = name
            host_record.extattrs = extattrs
            host_record.update()

    def bind_name_with_record_a(self, dns_view, ip, name, bind_list,
                                extattrs):
        is_ipv4 = ib_utils.determine_ip_version(ip) == 4
        if ((is_ipv4 and 'record:a' in bind_list) or
                (not is_ipv4 and 'record:aaaa' in bind_list)):
            obj.ARecordBase.create(self.connector,
                                   view=dns_view,
                                   ip=ip,
                                   name=name,
                                   extattrs=extattrs,
                                   update_if_exists=True)

        if 'record:ptr' in bind_list:
            obj.PtrRecord.create(self.connector,
                                 view=dns_view,
                                 ip=ip,
                                 ptrdname=name,
                                 extattrs=extattrs,
                                 update_if_exists=True)

    def unbind_name_from_record_a(self, dns_view, ip, name, unbind_list):
        is_ipv4 = ib_utils.determine_ip_version(ip) == 4
        if ((is_ipv4 and 'record:a' in unbind_list) or
                (not is_ipv4 and 'record:aaaa' in unbind_list)):
            a_record = obj.ARecordBase.search(self.connector,
                                              view=dns_view,
                                              ip=ip,
                                              name=name)
            if a_record:
                self.delete_objects_associated_with_a_record(a_record.name,
                                                             a_record.view,
                                                             unbind_list)
                a_record.delete()

        if 'record:ptr' in unbind_list:
            ptr_record = obj.PtrRecord.search(self.connector,
                                              view=dns_view,
                                              ip=ip,
                                              ptrdname=name)
            if ptr_record:
                ptr_record.delete()

    def get_member(self, member):
        member.fetch()
        return member

    def get_all_ea_definitions(self):
        return obj.EADefinition.search_all(self.connector)

    def create_ea_definition(self, ea_def):
        try:
            return obj.EADefinition.create(self.connector,
                                           check_if_exists=False,
                                           **ea_def)
        except ib_ex.InfobloxCannotCreateObject:
            LOG.error('Unable to create Extensible Attribute Definition '
                      '%s' % ea_def)

    def create_required_ea_definitions(self, required_ea_defs):
        existing_ea_defs = self.get_all_ea_definitions()
        missing_ea_defs = filter(lambda x: not next(
            (y for y in existing_ea_defs if x['name'] == y.name), None),
            required_ea_defs)

        for ea_def in missing_ea_defs:
            self.create_ea_definition(ea_def)

    def restart_all_services(self, member):
        if not member._ref:
            member.fetch()
        self.connector.call_func('restartservices', member._ref,
                                 {'restart_option': 'RESTART_IF_NEEDED',
                                  'service_option': 'ALL'})

    def delete_objects_associated_with_a_record(self, name, view, delete_list):
        """Deletes records associated with record:a or record:aaaa."""
        search_objects = {}
        if 'record:cname' in delete_list:
            search_objects['record:cname'] = 'canonical'
        if 'record:txt' in delete_list:
            search_objects['record:txt'] = 'name'

        if not search_objects:
            return

        for obj_type, search_type in search_objects.items():
            payload = {'view': view,
                       search_type: name}
            ib_objs = self.connector.get_object(obj_type, payload)
            if ib_objs:
                for ib_obj in ib_objs:
                    self.delete_object_by_ref(ib_obj['_ref'])

    def delete_all_associated_objects(self, network_view, ip, delete_list):
        LOG.warning(
            "DEPRECATION WARNING! Using delete_all_associated_objects() "
            "is deprecated and to be removed in next releases. "
            "Use unbind_name_from_record_a() instead.")

    def delete_object_by_ref(self, ref):
        try:
            self.connector.delete_object(ref)
        except ib_ex.InfobloxCannotDeleteObject:
            pass
<commit_msg>Allow domain-name-servers for ipv6<commit_after># Copyright 2015 Infoblox Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_log import log as logging

from infoblox_client import exceptions as ib_ex
from infoblox_client import objects as obj
from infoblox_client import utils as ib_utils

LOG = logging.getLogger(__name__)

class InfobloxObjectManager(object):

    def __init__(self, connector):
        self.connector = connector

    def create_network_view(self, network_view, extattrs):
        return obj.NetworkView.create(self.connector,
                                      name=network_view,
                                      extattrs=extattrs)

    def delete_network_view(self, network_view):
        # never delete default network view
        if network_view == 'default':
            return
        nview = obj.NetworkView.search(self.connector,
                                       name=network_view)
        if nview:
            nview.delete()

    def create_dns_view(self, network_view, dns_view):
        return obj.DNSView.create(self.connector,
                                  name=dns_view,
                                  network_view=network_view)

    def delete_dns_view(self, dns_view):
        dns_view = obj.DNSView.search(self.connector,
                                      name=dns_view)
        if dns_view:
            dns_view.delete()

    def create_network(self, net_view_name, cidr, nameservers=None,
                       members=None, gateway_ip=None, dhcp_trel_ip=None,
                       network_extattrs=None):
        """Create NIOS Network."""

        # NIOS does not allow to set Dhcp options for IPv6 over WAPI,
        # so limit options usage with IPv4 only
        ipv4 = ib_utils.determine_ip_version(cidr) == 4

        options = []
        if nameservers:
            options.append(obj.DhcpOption(name='domain-name-servers',
                                          value=",".join(nameservers)))
        if ipv4 and gateway_ip:
            options.append(obj.DhcpOption(name='routers',
                                          value=gateway_ip))
        if ipv4 and dhcp_trel_ip:
            options.append(obj.DhcpOption(name='dhcp-server-identifier',
                                          num=54,
                                          value=dhcp_trel_ip))
        return obj.Network.create(self.connector,
                                  network_view=net_view_name,
                                  cidr=cidr,
                                  members=members,
                                  options=options,
                                  extattrs=network_extattrs,
                                  check_if_exists=False)

    def get_network(self, network_view, cidr):
        return obj.Network.search(self.connector,
                                  network_view=network_view,
                                  cidr=cidr)

    def create_ip_range(self, network_view, start_ip, end_ip, network,
                        disable, range_extattrs):
        """Creates IPRange or fails if already exists."""
        return obj.IPRange.create(self.connector,
                                  network_view=network_view,
                                  start_addr=start_ip,
                                  end_addr=end_ip,
                                  cidr=network,
                                  disable=disable,
                                  extattrs=range_extattrs,
                                  check_if_exists=False)

    def delete_ip_range(self, network_view, start_ip, end_ip):
        range = obj.IPRange.search(self.connector,
                                   network_view=network_view,
                                   start_addr=start_ip,
                                   end_addr=end_ip)
        if range:
            range.delete()

    def has_networks(self, network_view_name):
        networks = obj.Network.search_all(self.connector,
                                          network_view=network_view_name)
        return bool(networks)

    def network_exists(self, network_view, cidr):
        """Deprecated, use get_network() instead."""
        LOG.warning(
            "DEPRECATION WARNING! Using network_exists() is deprecated "
            "and to be removed in next releases. "
            "Use get_network() or objects.Network.search instead")
        network = obj.Network.search(self.connector,
                                     network_view=network_view,
                                     cidr=cidr)
        return network is not None

    def delete_network(self, network_view, cidr):
        network = obj.Network.search(self.connector,
                                     network_view=network_view,
                                     cidr=cidr)
        if network:
            network.delete()

    def create_network_from_template(self, network_view, cidr, template,
                                     extattrs):
        return obj.Network.create(self.connector,
                                  network_view=network_view,
                                  cidr=cidr,
                                  template=template,
                                  extattrs=extattrs,
                                  check_if_exists=False)

    def update_network_options(self, ib_network, extattrs=None):
        if extattrs:
            ib_network.extattrs = extattrs
        return ib_network.update()

    def get_host_record(self, dns_view, ip):
        return obj.HostRecord.search(self.connector,
                                     view=dns_view,
                                     ip=ip)

    def find_hostname(self, dns_view, hostname, ip):
        return obj.HostRecord.search(self.connector,
                                     name=hostname,
                                     view=dns_view,
                                     ip=ip)

    def create_host_record_for_given_ip(self, dns_view, zone_auth,
                                        hostname, mac, ip, extattrs,
                                        use_dhcp):
        name = '.'.join([hostname, zone_auth])
        ip_obj = obj.IP.create(ip=ip, mac=mac, configure_for_dhcp=use_dhcp)
        return obj.HostRecord.create(self.connector,
                                     view=dns_view,
                                     name=name,
                                     ip=ip_obj,
                                     extattrs=extattrs,
                                     check_if_exists=False)

    def create_host_record_from_range(self, dns_view, network_view_name,
                                      zone_auth, hostname, mac, first_ip,
                                      last_ip, extattrs, use_dhcp):
        name = '.'.join([hostname, zone_auth])
        ip_alloc = obj.IPAllocation.next_available_ip_from_range(
            network_view_name, first_ip, last_ip)
        ip_obj = obj.IP.create(ip=ip_alloc, mac=mac,
                               configure_for_dhcp=use_dhcp)
        return obj.HostRecord.create(self.connector,
                                     view=dns_view,
                                     name=name,
                                     ip=ip_obj,
                                     extattrs=extattrs,
                                     check_if_exists=False)

    def delete_host_record(self, dns_view, ip_address):
        host_record = obj.HostRecord.search(self.connector,
                                            view=dns_view, ip=ip_address)
        if host_record:
            host_record.delete()

    def create_fixed_address_for_given_ip(self, network_view, mac, ip,
                                          extattrs):
        return obj.FixedAddress.create(self.connector,
                                       network_view=network_view,
                                       mac=mac,
                                       ip=ip,
                                       extattrs=extattrs,
                                       check_if_exists=False)

    def create_fixed_address_from_range(self, network_view, mac, first_ip,
                                        last_ip, extattrs):
        ip = obj.IPAllocation.next_available_ip_from_range(
            network_view, first_ip, last_ip)
        return obj.FixedAddress.create(self.connector,
                                       ip=ip,
                                       mac=mac,
                                       network_view=network_view,
                                       extattrs=extattrs,
                                       check_if_exists=False)

    def create_fixed_address_from_cidr(self, netview, mac, cidr, extattrs):
        ip = obj.IPAllocation.next_available_ip_from_cidr(netview, cidr)
        return obj.FixedAddress.create(self.connector,
                                       network_view=netview,
                                       ip=ip,
                                       mac=mac,
                                       extattrs=extattrs,
                                       check_if_exists=False)

    def delete_fixed_address(self, network_view, ip_address):
        fixed_address = obj.FixedAddress.search(self.connector,
                                                network_view=network_view,
                                                ip=ip_address)
        if fixed_address:
            fixed_address.delete()

    def add_ip_to_record(self, host_record, ip, mac, use_dhcp=True):
        ip_obj = obj.IP.create(ip=ip, mac=mac, configure_for_dhcp=use_dhcp)
        host_record.ip.append(ip_obj)
        return host_record.update()

    def add_ip_to_host_record_from_range(self, host_record, network_view,
                                         mac, first_ip, last_ip,
                                         use_dhcp=True):
        ip_alloc = obj.IPAllocation.next_available_ip_from_range(
            network_view, first_ip, last_ip)
        ip_obj = obj.IP.create(ip=ip_alloc, mac=mac,
                               configure_for_dhcp=use_dhcp)
        host_record.ip.append(ip_obj)
        return host_record.update()

    def delete_ip_from_host_record(self, host_record, ip):
        host_record.ip.remove(ip)
        return host_record.update()

    def has_dns_zones(self, dns_view):
        zones = obj.DNSZone.search_all(self.connector, view=dns_view)
        return bool(zones)

    def create_dns_zone(self, dns_view, dns_zone,
                        grid_primary=None, grid_secondaries=None,
                        zone_format=None, ns_group=None, prefix=None,
                        extattrs=None):
        return obj.DNSZone.create(self.connector,
                                  fqdn=dns_zone,
                                  view=dns_view,
                                  extattrs=extattrs,
                                  zone_format=zone_format,
                                  ns_group=ns_group,
                                  prefix=prefix,
                                  grid_primary=grid_primary,
                                  grid_secondaries=grid_secondaries)

    def delete_dns_zone(self, dns_view, dns_zone_fqdn):
        dns_zone = obj.DNSZone.search(self.connector,
                                      fqdn=dns_zone_fqdn,
                                      view=dns_view)
        if dns_zone:
            dns_zone.delete()

    def update_host_record_eas(self, dns_view, ip, extattrs):
        host_record = obj.HostRecord.search(self.connector,
                                            view=dns_view,
                                            ip=ip)
        if host_record:
            host_record.extattrs = extattrs
            host_record.update()

    def update_fixed_address_eas(self, network_view, ip, extattrs):
        fixed_address = obj.FixedAddress.search(self.connector,
                                                network_view=network_view,
                                                ip=ip)
        if fixed_address:
            fixed_address.extattrs = extattrs
            fixed_address.update()

    def update_dns_record_eas(self, dns_view, ip, extattrs):
        a_record = obj.ARecordBase.search(self.connector,
                                          ip=ip,
                                          view=dns_view)
        if a_record:
            a_record.extattrs = extattrs
            a_record.update()

        ptr_record = obj.PtrRecord.search(self.connector,
                                          ip=ip,
                                          view=dns_view)
        if ptr_record:
            ptr_record.extattrs = extattrs
            ptr_record.update()

    def bind_name_with_host_record(self, dns_view, ip, name, extattrs):
        host_record = obj.HostRecord.search(self.connector,
                                            view=dns_view,
                                            ip=ip)
        if host_record:
            host_record.name = name
            host_record.extattrs = extattrs
            host_record.update()

    def bind_name_with_record_a(self, dns_view, ip, name, bind_list,
                                extattrs):
        is_ipv4 = ib_utils.determine_ip_version(ip) == 4
        if ((is_ipv4 and 'record:a' in bind_list) or
                (not is_ipv4 and 'record:aaaa' in bind_list)):
            obj.ARecordBase.create(self.connector,
                                   view=dns_view,
                                   ip=ip,
                                   name=name,
                                   extattrs=extattrs,
                                   update_if_exists=True)

        if 'record:ptr' in bind_list:
            obj.PtrRecord.create(self.connector,
                                 view=dns_view,
                                 ip=ip,
                                 ptrdname=name,
                                 extattrs=extattrs,
                                 update_if_exists=True)

    def unbind_name_from_record_a(self, dns_view, ip, name, unbind_list):
        is_ipv4 = ib_utils.determine_ip_version(ip) == 4
        if ((is_ipv4 and 'record:a' in unbind_list) or
                (not is_ipv4 and 'record:aaaa' in unbind_list)):
            a_record = obj.ARecordBase.search(self.connector,
                                              view=dns_view,
                                              ip=ip,
                                              name=name)
            if a_record:
                self.delete_objects_associated_with_a_record(a_record.name,
                                                             a_record.view,
                                                             unbind_list)
                a_record.delete()

        if 'record:ptr' in unbind_list:
            ptr_record = obj.PtrRecord.search(self.connector,
                                              view=dns_view,
                                              ip=ip,
                                              ptrdname=name)
            if ptr_record:
                ptr_record.delete()

    def get_member(self, member):
        member.fetch()
        return member

    def get_all_ea_definitions(self):
        return obj.EADefinition.search_all(self.connector)

    def create_ea_definition(self, ea_def):
        try:
            return obj.EADefinition.create(self.connector,
                                           check_if_exists=False,
                                           **ea_def)
        except ib_ex.InfobloxCannotCreateObject:
            LOG.error('Unable to create Extensible Attribute Definition '
                      '%s' % ea_def)

    def create_required_ea_definitions(self, required_ea_defs):
        existing_ea_defs = self.get_all_ea_definitions()
        missing_ea_defs = filter(lambda x: not next(
            (y for y in existing_ea_defs if x['name'] == y.name), None),
            required_ea_defs)

        for ea_def in missing_ea_defs:
            self.create_ea_definition(ea_def)

    def restart_all_services(self, member):
        if not member._ref:
            member.fetch()
        self.connector.call_func('restartservices', member._ref,
                                 {'restart_option': 'RESTART_IF_NEEDED',
                                  'service_option': 'ALL'})

    def delete_objects_associated_with_a_record(self, name, view, delete_list):
        """Deletes records associated with record:a or record:aaaa."""
        search_objects = {}
        if 'record:cname' in delete_list:
            search_objects['record:cname'] = 'canonical'
        if 'record:txt' in delete_list:
            search_objects['record:txt'] = 'name'

        if not search_objects:
            return

        for obj_type, search_type in search_objects.items():
            payload = {'view': view,
                       search_type: name}
            ib_objs = self.connector.get_object(obj_type, payload)
            if ib_objs:
                for ib_obj in ib_objs:
                    self.delete_object_by_ref(ib_obj['_ref'])

    def delete_all_associated_objects(self, network_view, ip, delete_list):
        LOG.warning(
            "DEPRECATION WARNING! Using delete_all_associated_objects() "
            "is deprecated and to be removed in next releases. "
            "Use unbind_name_from_record_a() instead.")

    def delete_object_by_ref(self, ref):
        try:
            self.connector.delete_object(ref)
        except ib_ex.InfobloxCannotDeleteObject:
            pass
<|endoftext|>


<commit_before>import sys

trainfile, testfile, outfile = sys.argv[1:4]

global_sum = 0
global_count = 0
item_sums = {}
item_counts = {}
with open(trainfile) as f:
    for line in f:
        user, item, rating = line.strip().split(',')[:3]
        rating = float(rating)
        global_sum += rating
        global_count += 1
        if item not in item_sums:
            item_sums[item] = rating
            item_counts[item] = 1
        else:
            item_sums[item] += rating
            item_counts[item] += 1

global_mean = global_sum / global_count
item_means = {}
for item in item_sums.iterkeys():
    n = item_counts[item]
    item_means[item] = item_sums[item] / n - global_mean

with open(outfile, 'w') as outf:
    with open(testfile) as testf:
        for line in testf:
            user, item = line.strip().split(',')[:2]
            pred = global_mean
            if item in item_means:
                pred += item_means[item]
            print >>outf, "%s,%s,%.3f" % (user, item, pred)
<commit_msg>Refactor external algorithm test<commit_after>import sys

class ItemMeanData(object):
    def __init__(self):
        self.global_sum = 0
        self.global_count = 0
        self.item_sums = {}
        self.item_counts = {}

    def train(self, trainfile):
        with open(trainfile) as f:
            for line in f:
                user, item, rating = line.strip().split(',')[:3]
                item = int(item)
                rating = float(rating)
                self.global_sum += rating
                self.global_count += 1
                if item not in self.item_sums:
                    self.item_sums[item] = rating
                    self.item_counts[item] = 1
                else:
                    self.item_sums[item] += rating
                    self.item_counts[item] += 1

    def global_mean(self):
        return self.global_sum / self.global_count

    def item_set(self):
        return set(self.item_counts.iterkeys())

    def item_mean_offsets(self):
        means = {}
        gmean = self.global_mean()
        for item, n in self.item_counts.iteritems():
            means[item] = self.item_sums[item] / n - gmean
        return gmean, means

    def score_items(self, to_score, output):
        global_mean, item_means = self.item_mean_offsets()
        for user, items in to_score.iteritems():
            for item in items:
                pred = global_mean
                if item in item_means:
                    pred += item_means[item]
                print >> output, "%s,%s,%.3f" % (user, item, pred)

def load_test_pairs(testfile):
    to_score = {}
    with open(testfile) as testf:
        for line in testf:
            user, item = line.strip().split(',')[:2]
            user = int(user)
            item = int(item)
            if user in to_score:
                to_score[user].add(item)
            else:
                to_score[user] = set([item])
    return to_score

def load_query_users(userfile, items):
    to_score = {}
    with open(userfile) as userf:
        for line in userf:
            user = int(line.strip())
            to_score[user] = items
    return to_score

if sys.argv[1] == '--for-users':
    userfile, trainfile = sys.argv[2:4]
    testfile = None
    outfile = None
else:
    trainfile, testfile, outfile = sys.argv[1:4]
    userfile = None

model = ItemMeanData()
model.train(trainfile)

if testfile is not None:
    to_score = load_test_pairs(testfile)
elif userfile is not None:
    to_score = load_query_users(userfile, model.item_set())
else:
    print >> sys.stderr, "no user file specified"
    sys.exit(1)

if outfile is None:
    model.score_items(to_score, sys.stdout)
else:
    with open(outfile, 'w') as outf:
        model.score_items(to_score, outf)
<|endoftext|>


<commit_before>import shutil
import os
import subprocess

from ..util import Resources
from .. import set_destination

class MoveFiles:

    def __init__(self, rule, copy, dest_folder=None):
        """
        Move audiofiles to the appropriate destination directories,
        as determined by the 'rule' function passed to rule
        :param rule:
        :return:
        """
        self.rule = rule
        self.dest_folder = set_destination()
        if not self.dest_folder:
            self.dest_folder = dest_folder

        if copy:
            self.command = "COPY"
            self.join = ["&", "COPY"]
        else:
            self.command = "MOVE"
            self.join = ["&", "MOVE"]

    def move_files(self, files, src_dir=None):
        """
        Iterate over the elements of a ProcessDirectory object, and move them to the correct directory,
        using python subprocess

        :param files:
        :return:
        """
        files.first()

        while files.has_next():
            command = [self.command]
            tracks = files.next()
            valid_cd = True
            for i in range(len(tracks)):
                if self.rule.is_valid(tracks[i]):
                    repo_path = self.rule.get_dest(tracks[i])
                    if repo_path is not None:
                        full_path = os.path.join(self.dest_folder, repo_path)
                        if not os.path.exists(full_path):
                            os.makedirs(full_path)

                        if i != 0:
                            command += self.join
                        command.append(tracks[i].file_name)
                        dest = full_path
                        dest_filename = self.rule.get_filename(tracks[i])
                        if dest_filename:
                            dest = os.path.join(dest, dest_filename)
                        command.append(dest)
                else:
                    cd_valid = False

            directory = os.path.split(tracks[0].file_name)[0]
            if cd_valid:
                print("Moving " + ascii(directory))
                subprocess.run(command, shell=True)

                if not Resources.is_release(directory):
                    shutil.rmtree(directory)
            else:
                print("Invalid content found in " + directory + ", not copying.")
<commit_msg>Correct variable name<commit_after>import shutil
import os
import subprocess

from ..util import Resources
from .. import set_destination

class MoveFiles:

    def __init__(self, rule, copy, dest_folder=None):
        """
        Move audiofiles to the appropriate destination directories,
        as determined by the 'rule' function passed to rule
        :param rule:
        :return:
        """
        self.rule = rule
        self.dest_folder = set_destination()
        if not self.dest_folder:
            self.dest_folder = dest_folder

        if copy:
            self.command = "COPY"
            self.join = ["&", "COPY"]
        else:
            self.command = "MOVE"
            self.join = ["&", "MOVE"]

    def move_files(self, files, src_dir=None):
        """
        Iterate over the elements of a ProcessDirectory object, and move them to the correct directory,
        using python subprocess

        :param files:
        :return:
        """
        files.first()

        while files.has_next():
            command = [self.command]
            tracks = files.next()
            cd_valid = True
            for i in range(len(tracks)):
                if self.rule.is_valid(tracks[i]):
                    repo_path = self.rule.get_dest(tracks[i])
                    if repo_path is not None:
                        full_path = os.path.join(self.dest_folder, repo_path)
                        if not os.path.exists(full_path):
                            os.makedirs(full_path)

                        if i != 0:
                            command += self.join
                        command.append(tracks[i].file_name)
                        dest = full_path
                        dest_filename = self.rule.get_filename(tracks[i])
                        if dest_filename:
                            dest = os.path.join(dest, dest_filename)
                        command.append(dest)
                else:
                    cd_valid = False

            directory = os.path.split(tracks[0].file_name)[0]
            if cd_valid:
                print("Moving " + ascii(directory))
                subprocess.run(command, shell=True)

                if not Resources.is_release(directory):
                    shutil.rmtree(directory)
            else:
                print("Invalid content found in " + directory + ", not copying.")
<|endoftext|>


<commit_before>import logging
import socket
import datetime
import time
import os
import logging
import logging.handlers
import pprint
import threading
import anhost
import json
import sys
sys.path.append("..")
import netaddr

FORMAT = "[%(filename)s:%(lineno)s - %(threadName)s %(funcName)20s] %(levelname)10s %(message)s"
logging.basicConfig(format=FORMAT)

logger = logging.getLogger("%s | %s | " % (os.getpid(), __file__) )
logger.setLevel(logging.DEBUG)
socketHandler = logging.handlers.SocketHandler('localhost',
                    logging.handlers.DEFAULT_TCP_LOGGING_PORT)
logger.addHandler(socketHandler)

def str_to_dict(str_dict):
  unicode_dict = json.loads(str_dict)
  string_dict = {}
  for k in unicode_dict:
    string_dict[k.encode('utf-8')] = unicode_dict[k].encode('utf-8')
  return string_dict

def bit_mask(mask):
  x = mask.split(".")
  bc = 0
  for k in x:
    bc += bin(int(k)).count("1")
  return str(bc)

## short simple code to just broadcast unicast message
def send_broadcast(local_ip,msg,port):
  logger.debug("\tSEND_BROADCAST")
  sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
  prefix = '.'.join(local_ip.split(".")[0:3])+'.'
  suff = local_ip.split(".")[-1]
  logger.debug("\t\tbroadcasting to: %s*" % prefix)
  logger.debug("\t\tnot sending to: %s" % prefix+suff)
  for i in range(1,254):
    if i != int(suff):
      sock.sendto(msg, (prefix+str(i),port))
    else:
      logger.debug(i)
  logger.debug("\t\tbroadcast done.")

def write_n_fi(n_fi, n_list):
  logger.debug("\t\tWRITE_N_FI")
  x = open(n_fi,'w')
  for k in n_list:
    logger.debug("\t\t\twriting: %s" % k)
    x.write(json.dumps(k)+'\n')
  x.close()

def read_n_fi(n_fi):
  logger.debug("\t\tREAD_N_FI")
  x = open(n_fi,'r')
  neighbor = []
  for l in x:
    p = str_to_dict(l.strip())
    logger.debug("\t\t\tloaded: %s" % p)
    neighbor.append(p)
  x.close()
  logger.info("%s" % neighbor)
  return neighbor

def send_update(sock,n_fi,rip_port,dev):
  logger.debug("\tSEND_UPDATE")
  ## list of dicts
  n_dict = read_n_fi(n_fi)
  ip_self = sock.getsockname()[0]
  ## before we send out, we should update the gateway to be
  ## this node.
  rip_update = []
  for k in n_dict:
    x = anhost.Route()
    x.set_route(k)
    ## not a class, is dict?
    x.set_gw(anhost.get_ip_address(dev))
    rip_update.append(x.transmit_route())
  data_str = json.dumps(rip_update)

  sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
  #pull neighbors from updates, send to that set
  for rip_neigh in n_dict:
    target = rip_neigh['Gateway']
    sock.sendto(data_str, (target,rip_port))
    logger.debug("\t\tupdate sent to %s" % target)

def send_handler(sock,n_fi,port,dev):
  while True:
    logger.debug("SEND_HANDLER")
    send_update(sock, n_fi,port,dev)
    time.sleep(10)

#FIXME: add routes to linux routing table
def recv_update(neigh_fi,addr, update):
  add_list = []
  up_list = []
  neighbors = read_n_fi(neigh_fi)
  logger.debug("\tRECV_UPDATE")
  logger.debug("\t\tupdate from: %s" % addr)
  current_time = datetime.datetime.now()

  #list of Routes
  logger.debug(pprint.pformat(update))

  for route in update:
    x = anhost.Route()
    x.set_route(route)
    x.met += 1
    pprint.pprint(x)
    if x.met <= 16:
      net_str = convert_mask(x.dst,x.mask)
      bm = bit_mask(x.mask)
      network = netaddr.IPNetwork("%s/%s" % (x.dst,bm))
      there = False
      for neigh in neighbors:
        bm = bit_mask(neigh.mask)
        have_net = netaddr.IPNetwork("%s/%s" % (neigh.dst,bm))
        if network == have_net:
          there = neighbors[neigh]
          break
      if there:
        add_list.append(x)
      else:
        if x.met < there.met:
          up_list.append(x)

  logger.info("neighbors before add: %s" % neighbors)
  logger.debug("added list: %s" % add_list)
  logger.debug("updated list: %s" % up_list)

  ### need to update and return our modified dict
  # create new neighbor list
  update_neighbors = []
  # add new entries
  for k in add_list:
    k.set_ttl(current_time)
    update_neighbors.append(k.get_route())
    ##FIXME: add to linux route table

  #logger.info("dst_list: %s" % dst_list)
  #logger.info("neighbors: %s" % neighbors)
  logger.info("update neigh: %s" % update_neighbors)

  ## add new updates to replace originals in this update
  for k in up_list:
    k.set_ttl(current_time)
    update_neighbors.append(k.get_route())

  ## add original entries not seen in this update
  ## by not adding above, we remove all update entries
  for k in neighbors:
    if k not in update_neighbors:
      if (current_time-k.get_ttl() > datetime.timedelta(minutes=1)):
        logger.info("Route timeout: %s" % k)
        logger.info("current time %s, last update %s"%(current_time,k.get_ttl()))
      else:
        update_neighbors.append(k.get_route())
    ## delete the current linux route table information
    ## FIXME: else:

  logger.debug(pprint.pformat(update_neighbors))
  return update_neighbors

def recv_handler(rip_sock,n_fi):
  logger.debug("\tRECV_HANDLER")
  #try:
  while True:
    logger.debug("\t\taceepting messages...")
    msg, addr = rip_sock.recvfrom(4096)
    logger.debug("\t\tmessage: %s" % msg)
    logger.debug("\t\tsender's addr: (%s,%s)" % (addr[0],addr[1]))
    update = recv_update(n_fi,addr[0], json.loads(msg))
    write_n_fi(n_fi,update)
    logger.debug("\t\tupdate written out to file.")
    time.sleep(15)
    logger.info("UPDATED NEIGHBORS: %s" % pprint.pformat(read_n_fi(n_fi)))
  #except Exception,e:
  #  logger.error("Recver Error: %s" % str(e))

def rip_server(code, serv_port, rip_port,serv_fi, dev):
  neigh = "%s.rip" % rip_port
  logger.debug("RIP SERVER:")
  logger.debug("PID: %s" % os.getpid())
  logger.debug("DEVICES: %s" % dev)
  local_ip = anhost.get_ip_address(dev)
  # dst : via, cost
  x = open(neigh,'w')
  x.close()
  #write_n_fi(neigh,{local_ip:[local_ip,0]})
  routes = anhost.non_default_routes()
  l_route = []
  for route in routes:
    ## FIXME for kvm - eth0 is management
    if route["Iface"] != "eth0":
      ## default routes gets linux, but now we need to add arbitrary ttl for rip
      ##FIXME change get_routes to use Routes()
      route["TTL"] = datetime.datetime.now().strftime("%Y%j%H%M%S%f")
      l_route.append(route)
  write_n_fi(neigh,l_route)

  logger.debug("\tinitial neighbor list: %s" % read_n_fi(neigh))

  #set up rip server socket
  rip_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
  rip_sock.bind((local_ip,rip_port))
  rip_sock.setblocking(1) # blocking recv

  ## start rip on neighbors
  send_broadcast(local_ip, code,serv_port)

  try:
    ## recver thread
    recv_thread = threading.Thread(target=recv_handler, args=(rip_sock,neigh,))
    recv_thread.start()
  except Exception,e:
    logger.error("Receving Thread Error")
    raise Exception(e)

  try:
    ## sender thread
    send_thread = threading.Thread(target=send_handler,\
                  args=(rip_sock,neigh,rip_port,dev,))
    send_thread.start()
  except Exception,e:
    logger.error("Sending Thread Error")
    raise Exception(e)

  try:
    recv_thread.join()
    send_thread.join()
  except KeyboardInterrupt:
    logging.info("\tServer killed by Ctrl-C")
    os.remove(serv_fi)
    os.remove(neigh)
    raise KeyboardInterrupt
  except Exception, e:
    logging.error("\tRIP Server Crash: %s" % e)
    os.remove(serv_fi)
    os.remove(neigh)
    raise Exception(e)
<commit_msg>sending to neighbors<commit_after>import logging
import socket
import datetime
import time
import os
import logging
import logging.handlers
import pprint
import threading
import anhost
import json
import sys
sys.path.append("..")
import netaddr

rip_neighbors = []

FORMAT = "[%(filename)s:%(lineno)s - %(threadName)s %(funcName)20s] %(levelname)10s %(message)s"
logging.basicConfig(format=FORMAT)

logger = logging.getLogger("%s | %s | " % (os.getpid(), __file__) )
logger.setLevel(logging.DEBUG)
socketHandler = logging.handlers.SocketHandler('localhost',
                    logging.handlers.DEFAULT_TCP_LOGGING_PORT)
logger.addHandler(socketHandler)

def str_to_dict(str_dict):
  unicode_dict = json.loads(str_dict)
  string_dict = {}
  for k in unicode_dict:
    string_dict[k.encode('utf-8')] = unicode_dict[k].encode('utf-8')
  return string_dict

def bit_mask(mask):
  x = mask.split(".")
  bc = 0
  for k in x:
    bc += bin(int(k)).count("1")
  return str(bc)

## short simple code to just broadcast unicast message
def send_broadcast(local_ip,msg,port):
  logger.debug("\tSEND_BROADCAST")
  sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
  prefix = '.'.join(local_ip.split(".")[0:3])+'.'
  suff = local_ip.split(".")[-1]
  logger.debug("\t\tbroadcasting to: %s*" % prefix)
  logger.debug("\t\tnot sending to: %s" % prefix+suff)
  for i in range(1,254):
    if i != int(suff):
      sock.sendto(msg, (prefix+str(i),port))
    else:
      logger.debug(i)
  logger.debug("\t\tbroadcast done.")

def write_n_fi(n_fi, n_list):
  logger.debug("\t\tWRITE_N_FI")
  x = open(n_fi,'w')
  for k in n_list:
    logger.debug("\t\t\twriting: %s" % k)
    x.write(json.dumps(k)+'\n')
  x.close()

def read_n_fi(n_fi):
  logger.debug("\t\tREAD_N_FI")
  x = open(n_fi,'r')
  neighbor = []
  for l in x:
    p = str_to_dict(l.strip())
    logger.debug("\t\t\tloaded: %s" % p)
    neighbor.append(p)
  x.close()
  logger.info("%s" % neighbor)
  return neighbor

def send_update(sock,n_fi,rip_port,dev):
  logger.debug("\tSEND_UPDATE")
  ## list of dicts
  n_dict = read_n_fi(n_fi)
  ip_self = sock.getsockname()[0]
  ## before we send out, we should update the gateway to be
  ## this node.
  rip_update = []
  for k in n_dict:
    x = anhost.Route()
    x.set_route(k)
    ## not a class, is dict?
    x.set_gw(anhost.get_ip_address(dev))
    rip_update.append(x.transmit_route())
  data_str = json.dumps(rip_update)

  sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
  #pull neighbors from updates, send to that set
  for rip_neigh in rip_neighbors:
    for route in rip_update:
      sock.sendto(data_str, (rip_neigh,rip_port))
      logger.debug("\t\tupdate sent to %s" % rip_neigh)

def send_handler(sock,n_fi,port,dev):
  while True:
    logger.debug("SEND_HANDLER")
    send_update(sock, n_fi,port,dev)
    time.sleep(10)

#FIXME: add routes to linux routing table
def recv_update(neigh_fi,addr, update):
  add_list = []
  up_list = []
  neighbors = read_n_fi(neigh_fi)
  logger.debug("\tRECV_UPDATE")
  logger.debug("\t\tupdate from: %s" % addr)
  current_time = datetime.datetime.now()

  #list of Routes
  logger.debug(pprint.pformat(update))

  for route in update:
    x = anhost.Route()
    x.set_route(route)
    x.met += 1
    pprint.pprint(x)
    if x.met <= 16:
      net_str = convert_mask(x.dst,x.mask)
      bm = bit_mask(x.mask)
      network = netaddr.IPNetwork("%s/%s" % (x.dst,bm))
      there = False
      for neigh in neighbors:
        bm = bit_mask(neigh.mask)
        have_net = netaddr.IPNetwork("%s/%s" % (neigh.dst,bm))
        if network == have_net:
          there = neighbors[neigh]
          break
      if there:
        add_list.append(x)
      else:
        if x.met < there.met:
          up_list.append(x)

  logger.info("neighbors before add: %s" % neighbors)
  logger.debug("added list: %s" % add_list)
  logger.debug("updated list: %s" % up_list)

  ### need to update and return our modified dict
  # create new neighbor list
  update_neighbors = []
  # add new entries
  for k in add_list:
    k.set_ttl(current_time)
    update_neighbors.append(k.get_route())
    ##FIXME: add to linux route table

  #logger.info("dst_list: %s" % dst_list)
  #logger.info("neighbors: %s" % neighbors)
  logger.info("update neigh: %s" % update_neighbors)

  ## add new updates to replace originals in this update
  for k in up_list:
    k.set_ttl(current_time)
    update_neighbors.append(k.get_route())

  ## add original entries not seen in this update
  ## by not adding above, we remove all update entries
  for k in neighbors:
    if k not in update_neighbors:
      if (current_time-k.get_ttl() > datetime.timedelta(minutes=1)):
        logger.info("Route timeout: %s" % k)
        logger.info("current time %s, last update %s"%(current_time,k.get_ttl()))
      else:
        update_neighbors.append(k.get_route())
    ## delete the current linux route table information
    ## FIXME: else:

  logger.debug(pprint.pformat(update_neighbors))
  return update_neighbors

def recv_handler(rip_sock,n_fi):
  logger.debug("\tRECV_HANDLER")
  #try:
  while True:
    logger.debug("\t\taceepting messages...")
    msg, addr = rip_sock.recvfrom(4096)
    logger.debug("\t\tmessage: %s" % msg)
    logger.debug("\t\tsender's addr: (%s,%s)" % (addr[0],addr[1]))
    ## FIXME need to tie rip_neighbors with a ttl and the route table
    if addr[0] not in rip_neighbors:
      rip_neighbors.append(addr[0])
    update = recv_update(n_fi,addr[0], json.loads(msg))
    write_n_fi(n_fi,update)
    logger.debug("\t\tupdate written out to file.")
    time.sleep(15)
    logger.info("UPDATED NEIGHBORS: %s" % pprint.pformat(read_n_fi(n_fi)))
  #except Exception,e:
  #  logger.error("Recver Error: %s" % str(e))

def rip_server(code, serv_port, rip_port,serv_fi, dev):
  neigh = "%s.rip" % rip_port
  logger.debug("RIP SERVER:")
  logger.debug("PID: %s" % os.getpid())
  logger.debug("DEVICES: %s" % dev)
  local_ip = anhost.get_ip_address(dev)
  # dst : via, cost
  x = open(neigh,'w')
  x.close()
  #write_n_fi(neigh,{local_ip:[local_ip,0]})
  routes = anhost.non_default_routes()
  l_route = []
  for route in routes:
    ## FIXME for kvm - eth0 is management
    if route["Iface"] != "eth0":
      ## default routes gets linux, but now we need to add arbitrary ttl for rip
      ##FIXME change get_routes to use Routes()
      route["TTL"] = datetime.datetime.now().strftime("%Y%j%H%M%S%f")
      l_route.append(route)
  write_n_fi(neigh,l_route)

  logger.debug("\tinitial neighbor list: %s" % read_n_fi(neigh))

  #set up rip server socket
  rip_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
  rip_sock.bind((local_ip,rip_port))
  rip_sock.setblocking(1) # blocking recv

  ## start rip on neighbors
  send_broadcast(local_ip, code,serv_port)

  try:
    ## recver thread
    recv_thread = threading.Thread(target=recv_handler, args=(rip_sock,neigh,))
    recv_thread.start()
  except Exception,e:
    logger.error("Receving Thread Error")
    raise Exception(e)

  try:
    ## sender thread
    send_thread = threading.Thread(target=send_handler,\
                  args=(rip_sock,neigh,rip_port,dev,))
    send_thread.start()
  except Exception,e:
    logger.error("Sending Thread Error")
    raise Exception(e)

  try:
    recv_thread.join()
    send_thread.join()
  except KeyboardInterrupt:
    logging.info("\tServer killed by Ctrl-C")
    os.remove(serv_fi)
    os.remove(neigh)
    raise KeyboardInterrupt
  except Exception, e:
    logging.error("\tRIP Server Crash: %s" % e)
    os.remove(serv_fi)
    os.remove(neigh)
    raise Exception(e)
<|endoftext|>


<commit_before>###

import multiprocessing
import utilities
import tile_statistics
import subprocess
import universal_util
from functools import partial
import sys
sys.path.append('../')
import constants_and_names

# Creates list of tiles to iterate through
# mangrove_biomass_tile_list = utilities.tile_list(constants_and_names.mangrove_biomass_dir)
tile_list = ["00N_000E", "00N_050W"] # test tiles
print tile_list

# # For downloading all tiles in the folders
# download_list = [constants_and_names.mangrove_biomass_dir]
#
# for input in download_list:
#     utilities.s3_folder_download('{}'.format(input), '.')

# For copying individual tiles to spot machine for testing
for tile in tile_list:
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.mangrove_biomass_dir, constants_and_names.pattern_mangrove_biomass, tile), '.')      # mangrove biomass tiles
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.biomass_dir, tile, constants_and_names.pattern_biomass), '.')

# The column names for the tile summary statistics.
# If the statistics calculations are changed in tile_statistics.py, the list here needs to be changed, too.
headers = ['tile_id', 'tile_name', 'pixel_count', 'mean', 'median', 'percentile10', 'percentile25',
           'percentile75', 'percentile90', 'min', 'max']
header_no_brackets = ', '.join(headers)

# Creates the output text file with the column names
with open(constants_and_names.tile_stats, 'w+') as f:
    f.write(header_no_brackets  +'\r\n')
f.close()

# For multiprocessor use
count = multiprocessing.cpu_count()
pool = multiprocessing.Pool(processes=count/3)
pool.map(tile_statistics.create_tile_statistics, tile_list)

# # For single processor use
# for tile in mangrove_biomass_tile_list:
#     tile_statistics.create_tile_statistics(tile)

# Copies the text file to the location on s3 that the tiles are from
cmd = ['aws', 's3', 'cp', constants_and_names.tile_stats, constants_and_names.tile_stats_dir]
subprocess.check_call(cmd)<commit_msg>Working on tile statistics script.<commit_after>###

import multiprocessing
import utilities
import tile_statistics
import subprocess
from functools import partial
import sys
sys.path.append('../')
import constants_and_names

# Creates list of tiles to iterate through
# mangrove_biomass_tile_list = utilities.tile_list(constants_and_names.mangrove_biomass_dir)
tile_list = ["00N_000E", "00N_050W"] # test tiles
print tile_list

# # For downloading all tiles in the folders
# download_list = [constants_and_names.mangrove_biomass_dir]
#
# for input in download_list:
#     utilities.s3_folder_download('{}'.format(input), '.')

# For copying individual tiles to spot machine for testing
for tile in tile_list:
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.mangrove_biomass_dir, constants_and_names.pattern_mangrove_biomass, tile), '.')      # mangrove biomass tiles
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.biomass_dir, tile, constants_and_names.pattern_biomass), '.')

# The column names for the tile summary statistics.
# If the statistics calculations are changed in tile_statistics.py, the list here needs to be changed, too.
headers = ['tile_id', 'tile_name', 'pixel_count', 'mean', 'median', 'percentile10', 'percentile25',
           'percentile75', 'percentile90', 'min', 'max']
header_no_brackets = ', '.join(headers)

# Creates the output text file with the column names
with open(constants_and_names.tile_stats, 'w+') as f:
    f.write(header_no_brackets  +'\r\n')
f.close()

# For multiprocessor use
count = multiprocessing.cpu_count()
pool = multiprocessing.Pool(processes=count/4)
pool.map(tile_statistics.create_tile_statistics, tile_list)

# # For single processor use
# for tile in mangrove_biomass_tile_list:
#     tile_statistics.create_tile_statistics(tile)

# Copies the text file to the location on s3 that the tiles are from
cmd = ['aws', 's3', 'cp', constants_and_names.tile_stats, constants_and_names.tile_stats_dir]
subprocess.check_call(cmd)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Blitz speed reading trainer add-on for Anki
#
# Copyright (C) 2016  Jakub Szypulka <jakub AT szypulka.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from aqt.reviewer import Reviewer
import time

def myShow(self):                  # Customize the show() function by...
    origShow(self)                 # executing the function as usual (1/2),...
    self.start_time = time.time()  # but starting a timer.

origShow = Reviewer.show           # Execute the function as usual (2/2)
Reviewer.show = myShow             # Execute the function as usual (3/3)

def myDefaultEase(self):
    elapsed_time = time.time() - self.start_time  # How much time elapsed?
    if elapsed_time < 1:                          # If less than 1s...
        return 3                                  # suggest "Easy"/"Normal".
    if elapsed_time < 3:                          # If less than 3s...
        return 2                                  # suggest "Hard".
    else:                                         # If more than 3s...
        return 1                                  # suggest "Again".

Reviewer._defaultEase = myDefaultEase             # Replace default ease function
<commit_msg>Use showQuestion hook to get start time<commit_after># -*- coding: utf-8 -*-

# Blitz speed reading trainer add-on for Anki
#
# Copyright (C) 2016  Jakub Szypulka <jakub AT szypulka.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from anki.hooks import addHook
from aqt.reviewer import Reviewer
import time

start_time = None

def onShowQuestion():
    global start_time
    start_time = time.time()

addHook('showQuestion', onShowQuestion)

def myDefaultEase(self):
    elapsed_time = time.time() - start_time       # How much time elapsed?
    if elapsed_time < 1:                          # If less than 1s...
        return 3                                  # suggest "Easy"/"Normal".
    if elapsed_time < 3:                          # If less than 3s...
        return 2                                  # suggest "Hard".
    else:                                         # If more than 3s...
        return 1                                  # suggest "Again".

Reviewer._defaultEase = myDefaultEase             # Replace default ease function
<|endoftext|>


<commit_before># -*- mode: Python ; coding: utf-8 -*-
#
# Copyright (C) 2014 tobynet
#
# License: GNU AGPL, version 3 or later;
# http://www.gnu.org/copyleft/agpl.html

"""Add-on for Anki 2.x that helps using with the touch interface."""

from PyQt4.QtCore import Qt, QEvent

from aqt import mw
from aqt.utils import showInfo, tooltip
from anki.hooks import wrap

pressed_position = None;

def show_answer():
    u"""Change state to answer"""
    if mw.reviewer.state == "question":
        mw.reviewer._showAnswerHack()

def answer_card(correct):
    u"""Answer with correct or incrrect"""
    tooltip("CORRECT!!" if correct else "miss...", 1000)
    mw.reviewer._answerCard(
      mw.reviewer._defaultEase() if correct else 1)

def is_correct_area(pressed_pos,released_pos):
    u"""Pointing position is in a correct area?"""
    return pressed_pos.x() < released_pos.x()

def my_event(event):
    u"""Hook event to detect touch"""
    tooltip("Event: " + `event`)
    if event.type() == QEvent.TouchBegin:
        show_answer()

def my_mouse_press_event(event):
    u"""Hook mouse event to detect pressed button firstly"""
    global pressed_position

    #tooltip("press: {0}".format(event.pos().x()))
    show_answer()
    pressed_position = event.pos()

def my_mouse_release_event(event):
    u"""Hook mouse event to detect swipe"""
    global pressed_position

    #tooltip("release: {0} ({1})".format(event.pos().x(), event.pos().x() - pressed_position.x()))
    if mw.reviewer.state == "answer":
        answer_card(
            is_correct_area(
                pressed_pos = pressed_position,
                released_pos = event.pos()))

# Hook some events
mw.web.mousePressEvent = wrap(mw.web.mousePressEvent, my_mouse_press_event)
mw.web.mouseReleaseEvent = wrap(mw.web.mouseReleaseEvent, my_mouse_release_event)

# I can't hook "mw.web.event", why?????
#mw.web.event = wrap(mw.web.event, my_event)

<commit_msg>Add version number to .py<commit_after># -*- mode: Python ; coding: utf-8 -*-
#
# Copyright (C) 2014 tobynet
#
# License: GNU AGPL, version 3 or later;
# http://www.gnu.org/copyleft/agpl.html

__Version__ = "0.1.0"

"""Add-on for Anki 2.x that helps using with the touch interface."""

from PyQt4.QtCore import Qt, QEvent

from aqt import mw
from aqt.utils import showInfo, tooltip
from anki.hooks import wrap

pressed_position = None;

def show_answer():
    u"""Change state to answer"""
    if mw.reviewer.state == "question":
        mw.reviewer._showAnswerHack()

def answer_card(correct):
    u"""Answer with correct or incrrect"""
    tooltip("CORRECT!!" if correct else "miss...", 1000)
    mw.reviewer._answerCard(
      mw.reviewer._defaultEase() if correct else 1)

def is_correct_area(pressed_pos,released_pos):
    u"""Pointing position is in a correct area?"""
    return pressed_pos.x() < released_pos.x()

def my_event(event):
    u"""Hook event to detect touch"""
    tooltip("Event: " + `event`)
    if event.type() == QEvent.TouchBegin:
        show_answer()

def my_mouse_press_event(event):
    u"""Hook mouse event to detect pressed button firstly"""
    global pressed_position

    #tooltip("press: {0}".format(event.pos().x()))
    show_answer()
    pressed_position = event.pos()

def my_mouse_release_event(event):
    u"""Hook mouse event to detect swipe"""
    global pressed_position

    #tooltip("release: {0} ({1})".format(event.pos().x(), event.pos().x() - pressed_position.x()))
    if mw.reviewer.state == "answer":
        answer_card(
            is_correct_area(
                pressed_pos = pressed_position,
                released_pos = event.pos()))

# Hook some events
mw.web.mousePressEvent = wrap(mw.web.mousePressEvent, my_mouse_press_event)
mw.web.mouseReleaseEvent = wrap(mw.web.mouseReleaseEvent, my_mouse_release_event)

# I can't hook "mw.web.event", why?????
#mw.web.event = wrap(mw.web.event, my_event)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014 Eficent (<http://www.eficent.com/>)
#              <contact@eficent.com>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

{
    "name": "Analytic Plan Mass Create",
    "version": "1.0",
    "author": "Eficent",
    "website": "www.eficent.com",
    "category": "Generic Modules/Projects & Services",
    "depends": ["analytic_plan", "account"],
    "description": """
Analytic Plan Mass Create
=========================
    """,
    "data": [
        "views/analytic_plan_mass_create_template_view.xml",
        "wizard/analytic_plan_mass_create_view.xml",
        "security/ir.model.access.csv"
    ],
    'installable': True,
    'active': False,
}<commit_msg>[IMP] make it work for v9<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014 Eficent (<http://www.eficent.com/>)
#              <contact@eficent.com>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

{
    "name": "Analytic Plan Mass Create",
    "version": "1.0",
    "author": "Eficent",
    "website": "www.eficent.com",
    "category": "Generic Modules/Projects & Services",
    "depends": ["analytic_plan", "account"],
    "description": """
Analytic Plan Mass Create
=========================
    """,
    "data": [
        "views/analytic_plan_mass_create_template_view.xml",
        "wizard/analytic_plan_mass_create_view.xml",
        "security/ir.model.access.csv"
    ],
    'installable': False,
    'active': False,
}
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Entry point for importing common Anode packages. Most files should only need to import from here.
"""

__author__ = 'Adam R. Smith'
__license__ = 'Apache 2.0'

__all__ = []

# Tell the magic import log setup to pass through this file
from anode.util.log import import_paths
import_paths.append(__name__)

from anode.util.log import log
__all__ += ['log']

from anode.core.bootstrap import CFG, obj_registry
__all__ += ['CFG', 'obj_registry']

from anode.util.async import spawn, switch
__all__ += ['spawn', 'switch']

from anode.core.process import AnodeProcessError, GreenProcess, GreenProcessSupervisor, PythonProcess
__all__ += ['AnodeProcessError', 'GreenProcess', 'GreenProcessSupervisor', 'PythonProcess']

from anode.net import messaging, channel
__all__ += ['messaging', 'channel']
<commit_msg>missing import<commit_after>#!/usr/bin/env python

"""
Entry point for importing common Anode packages. Most files should only need to import from here.
"""

__author__ = 'Adam R. Smith'
__license__ = 'Apache 2.0'

__all__ = []

# Tell the magic import log setup to pass through this file
from anode.util.log import import_paths
import_paths.append(__name__)

from anode.util.log import log
__all__ += ['log']

from anode.core.bootstrap import CFG, obj_registry, AnodeObject
__all__ += ['CFG', 'obj_registry', 'AnodeObject']

from anode.util.async import spawn, switch
__all__ += ['spawn', 'switch']

from anode.core.process import AnodeProcessError, GreenProcess, GreenProcessSupervisor, PythonProcess
__all__ += ['AnodeProcessError', 'GreenProcess', 'GreenProcessSupervisor', 'PythonProcess']

from anode.net import messaging, channel
__all__ += ['messaging', 'channel']
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import os
from collections import defaultdict, Counter

from qtpy import QtWidgets, QtGui, QtCore
from qtpy.QtCore import Qt

from glue.core import ComponentID
from glue.core.parse import ParsedComponentLink, ParsedCommand
from glue.utils.qt import load_ui

from glue.dialogs.component_manager.qt.equation_editor import EquationEditorDialog

__all__ = ['ComponentManagerWidget']

class ComponentTreeWidget(QtWidgets.QTreeWidget):

    order_changed = QtCore.Signal()

    def dropEvent(self, event):
        items = self.selectedItems()
        selected = items[0] if len(items) == 1 else None
        super(ComponentTreeWidget, self).dropEvent(event)
        self.selection = self.selectionModel()
        self.selection.select(QtCore.QItemSelection(self.indexFromItem(selected, 0),
                                                    self.indexFromItem(selected, self.columnCount() - 1)),
                              QtCore.QItemSelectionModel.ClearAndSelect)
        self.order_changed.emit()

    def mousePressEvent(self, event):
        self.clearSelection()
        super(ComponentTreeWidget, self).mousePressEvent(event)

class ComponentManagerWidget(QtWidgets.QDialog):

    def __init__(self, data_collection=None, parent=None):

        super(ComponentManagerWidget, self).__init__(parent=parent)

        self.ui = load_ui('component_manager.ui', self,
                          directory=os.path.dirname(__file__))

        self.data_collection = data_collection

        self._components = defaultdict(lambda: defaultdict(list))
        self._state = defaultdict(dict)

        for data in data_collection:

            for cid in data.primary_components:
                comp_state = {}
                comp_state['cid'] = cid
                comp_state['label'] = cid.label
                self._state[data][cid] = comp_state
                self._components[data]['main'].append(cid)

            self._components[data]['derived'] = []

            for cid in data.derived_components:
                comp = data.get_component(cid)
                if isinstance(comp.link, ParsedComponentLink):
                    comp_state = {}
                    comp_state['cid'] = cid
                    comp_state['label'] = cid.label
                    comp_state['equation'] = comp.link._parsed._cmd
                    self._state[data][cid] = comp_state
                    self._components[data]['derived'].append(cid)

        # Populate data combo
        for data in self.data_collection:
            self.ui.combosel_data.addItem(data.label, userData=data)

        self.ui.combosel_data.setCurrentIndex(0)
        self.ui.combosel_data.currentIndexChanged.connect(self._update_component_lists)
        self._update_component_lists()

        self.ui.button_remove_main.clicked.connect(self._remove_main_component)

        self.ui.button_add_derived.clicked.connect(self._add_derived_component)
        self.ui.button_edit_derived.clicked.connect(self._edit_derived_component)
        self.ui.button_remove_derived.clicked.connect(self._remove_derived_component)

        self.ui.list_main_components.itemSelectionChanged.connect(self._update_selection_main)
        self.ui.list_derived_components.itemSelectionChanged.connect(self._update_selection_derived)

        self._update_selection_main()
        self._update_selection_derived()

        self.ui.list_main_components.itemChanged.connect(self._update_state)
        self.ui.list_derived_components.itemChanged.connect(self._update_state)
        self.ui.list_main_components.order_changed.connect(self._update_state)
        self.ui.list_derived_components.order_changed.connect(self._update_state)

        self.ui.button_ok.clicked.connect(self.accept)
        self.ui.button_cancel.clicked.connect(self.reject)

    def _update_selection_main(self):
        enabled = self.selected_main_component is not None
        self.button_remove_main.setEnabled(enabled)

    def _update_selection_derived(self):
        enabled = self.selected_derived_component is not None
        self.button_edit_derived.setEnabled(enabled)
        self.button_remove_derived.setEnabled(enabled)

    @property
    def data(self):
        return self.ui.combosel_data.currentData()

    @property
    def selected_main_component(self):
        items = self.ui.list_main_components.selectedItems()
        if len(items) == 1:
            return items[0].data(0, Qt.UserRole)
        else:
            return None

    @property
    def selected_derived_component(self):
        items = self.ui.list_derived_components.selectedItems()
        if len(items) == 1:
            return items[0].data(0, Qt.UserRole)
        else:
            return None

    def _update_component_lists(self, *args):

        # This gets called when the data is changed and we need to update the
        # components shown in the lists.

        self.ui.list_main_components.blockSignals(True)

        self.ui.list_main_components.clear()
        self.ui.list_derived_components.clear()

        root = self.ui.list_main_components.invisibleRootItem()

        for cid in self._components[self.data]['main']:
            item = QtWidgets.QTreeWidgetItem(root, [self._state[self.data][cid]['label']])
            item.setData(0, Qt.UserRole, cid)
            item.setFlags(item.flags() | Qt.ItemIsEditable)
            item.setFlags(item.flags() ^ Qt.ItemIsDropEnabled)

        self.ui.list_main_components.blockSignals(False)

        self.ui.list_derived_components.blockSignals(True)

        root = self.ui.list_derived_components.invisibleRootItem()

        for cid in self._components[self.data]['derived']:
            item = QtWidgets.QTreeWidgetItem(root, [self._state[self.data][cid]['label']])
            item.setData(0, Qt.UserRole, cid)
            item.setFlags(item.flags() | Qt.ItemIsEditable)
            item.setFlags(item.flags() ^ Qt.ItemIsDropEnabled)

        self.ui.list_derived_components.blockSignals(False)

        self._validate()

    def _validate(self):

        # Figure out a list of all the labels so that we can check which ones
        # are duplicates.
        print(list(self._state[self.data].values()))
        labels = [c['label'] for c in self._state[self.data].values()]
        label_count = Counter(labels)

        brush_red = QtGui.QBrush(Qt.red)
        brush_black = QtGui.QBrush(Qt.black)

        if label_count.most_common(1)[0][1] > 1:
            for component_list in (self.ui.list_main_components,
                                   self.ui.list_derived_components):
                component_list.blockSignals(True)
                root = component_list.invisibleRootItem()
                for idx in range(root.childCount()):
                    item = root.child(idx)
                    if label_count[item.text(0)] > 1:
                        item.setForeground(0, brush_red)
                    else:
                        item.setForeground(0, brush_black)
                component_list.blockSignals(False)

            self.ui.label_status.setStyleSheet('color: red')
            self.ui.label_status.setText('Error: some components have duplicate names')
            self.ui.button_ok.setEnabled(False)
            self.ui.combosel_data.setEnabled(False)
            return

        self.ui.label_status.setStyleSheet('')
        self.ui.label_status.setText('')
        self.ui.button_ok.setEnabled(True)
        self.ui.combosel_data.setEnabled(True)

    def _update_state(self, *args):

        self._components[self.data]['main'] = []
        root = self.ui.list_main_components.invisibleRootItem()
        for idx in range(root.childCount()):
            item = root.child(idx)
            cid = item.data(0, Qt.UserRole)
            self._state[self.data][cid]['label'] = item.text(0)
            self._components[self.data]['main'].append(cid)

        self._components[self.data]['derived'] = []
        root = self.ui.list_derived_components.invisibleRootItem()
        for idx in range(root.childCount()):
            item = root.child(idx)
            cid = item.data(0, Qt.UserRole)
            self._state[self.data][cid]['label'] = item.text(0)
            self._components[self.data]['derived'].append(cid)

        self._update_component_lists()

    def _remove_main_component(self, *args):
        cid = self.selected_main_component
        if cid is None:
            return
        self._components[self.data]['main'].remove(cid)
        self._state[self.data].pop(cid)
        self._update_component_lists()

    def _remove_derived_component(self, *args):
        cid = self.selected_derived_component
        if cid is None:
            return
        self._components[self.data]['derived'].remove(cid)
        self._state[self.data].pop(cid)
        self._update_component_lists()

    def _add_derived_component(self, *args):

        comp_state = {}
        comp_state['cid'] = ComponentID('')
        comp_state['label'] = 'New component'
        comp_state['equation'] = ''

        self._components[self.data]['derived'].append(comp_state['cid'])
        self._state[self.data][comp_state['cid']] = comp_state

        self._update_component_lists()

    def _edit_derived_component(self, *args):

        cid = self.selected_derived_component

        dialog = EquationEditorDialog(self.data, self._state[self.data][cid]['equation'], parent=self)
        dialog.setWindowFlags(self.windowFlags() | Qt.Window)
        dialog.setFocus()
        dialog.raise_()
        dialog.exec_()

        if dialog.final_expression is None:
            return

        self._state[self.data][cid]['equation'] = dialog.final_expression

    def accept(self):

        for data in self._components:

            cids_main = self._components[data]['main']
            cids_derived = self._components[data]['derived']

            cids_all = cids_main + cids_derived

            cids_existing = data.components

            for cid_old in cids_existing:
                if not any(cid_old is cid_new for cid_new in cids_all):
                    data.remove_component(cid_old)

            # TODO: make it so labels in expression take into account renaming
            components = dict((cid.label, cid) for cid in data.components)

            for cid_new in cids_derived:
                if not any(cid_new is cid_old for cid_old in cids_existing):
                    pc = ParsedCommand(self._state[data][cid_new]['equation'], components)
                    link = ParsedComponentLink(cid_new, pc)
                    data.add_component_link(link)

            data.reorder_components(cids_all)

        super(ComponentManagerWidget, self).accept()

def main():

    import numpy as np

    from glue.core.data import Data
    from glue.core.data_collection import DataCollection

    x = np.random.random((5, 5))
    y = x * 3
    dc = DataCollection()
    dc.append(Data(label='test1', x=x, y=y))
    dc.append(Data(label='test2', a=x, b=y))

    widget = ComponentManagerWidget(dc)
    widget.exec_()

    for data in dc:
        print('-' * 72)
        print(data)

if __name__ == "__main__":
    from glue.utils.qt import get_qapp
    app = get_qapp()
    main()
<commit_msg>Simplify code for new component manager<commit_after>from __future__ import absolute_import, division, print_function

import os
from collections import defaultdict, Counter

from qtpy import QtWidgets, QtGui, QtCore
from qtpy.QtCore import Qt

from glue.core import ComponentID
from glue.core.parse import ParsedComponentLink, ParsedCommand
from glue.utils.qt import load_ui

from glue.dialogs.component_manager.qt.equation_editor import EquationEditorDialog

__all__ = ['ComponentManagerWidget']

class ComponentTreeWidget(QtWidgets.QTreeWidget):

    order_changed = QtCore.Signal()

    def select_item(self, item):
        self.selection = self.selectionModel()
        self.selection.select(QtCore.QItemSelection(self.indexFromItem(item, 0),
                                                    self.indexFromItem(item, self.columnCount() - 1)),
                              QtCore.QItemSelectionModel.ClearAndSelect)

    @property
    def selected_item(self):
        items = self.selectedItems()
        return items[0] if len(items) == 1 else None

    @property
    def selected_cid(self):
        selected = self.selected_item
        return None if selected is None else selected.data(0, Qt.UserRole)

    def add_cid_and_label(self, cid, label):
        item = QtWidgets.QTreeWidgetItem(self.invisibleRootItem(), [label])
        item.setData(0, Qt.UserRole, cid)
        item.setFlags(item.flags() | Qt.ItemIsEditable)
        item.setFlags(item.flags() ^ Qt.ItemIsDropEnabled)

    def __iter__(self):
        root = self.invisibleRootItem()
        for idx in range(root.childCount()):
            yield root.child(idx)

    def dropEvent(self, event):
        selected = self.selected_item
        super(ComponentTreeWidget, self).dropEvent(event)
        self.select_item(selected)
        self.order_changed.emit()

    def mousePressEvent(self, event):
        self.clearSelection()
        super(ComponentTreeWidget, self).mousePressEvent(event)

class ComponentManagerWidget(QtWidgets.QDialog):

    def __init__(self, data_collection=None, parent=None):

        super(ComponentManagerWidget, self).__init__(parent=parent)

        self.ui = load_ui('component_manager.ui', self,
                          directory=os.path.dirname(__file__))

        self.list = {}
        self.list['main'] = self.ui.list_main_components
        self.list['derived'] = self.ui.list_derived_components

        self.data_collection = data_collection

        self._components = defaultdict(lambda: defaultdict(list))
        self._state = defaultdict(dict)

        for data in data_collection:

            for cid in data.primary_components:
                comp_state = {}
                comp_state['cid'] = cid
                comp_state['label'] = cid.label
                self._state[data][cid] = comp_state
                self._components[data]['main'].append(cid)

            self._components[data]['derived'] = []

            for cid in data.derived_components:
                comp = data.get_component(cid)
                if isinstance(comp.link, ParsedComponentLink):
                    comp_state = {}
                    comp_state['cid'] = cid
                    comp_state['label'] = cid.label
                    comp_state['equation'] = comp.link._parsed._cmd
                    self._state[data][cid] = comp_state
                    self._components[data]['derived'].append(cid)

        # Populate data combo
        for data in self.data_collection:
            self.ui.combosel_data.addItem(data.label, userData=data)

        self.ui.combosel_data.setCurrentIndex(0)
        self.ui.combosel_data.currentIndexChanged.connect(self._update_component_lists)
        self._update_component_lists()

        self.ui.button_remove_main.clicked.connect(self._remove_main_component)

        self.ui.button_add_derived.clicked.connect(self._add_derived_component)
        self.ui.button_edit_derived.clicked.connect(self._edit_derived_component)
        self.ui.button_remove_derived.clicked.connect(self._remove_derived_component)

        self.ui.list_main_components.itemSelectionChanged.connect(self._update_selection_main)
        self.ui.list_derived_components.itemSelectionChanged.connect(self._update_selection_derived)

        self._update_selection_main()
        self._update_selection_derived()

        self.ui.list_main_components.itemChanged.connect(self._update_state)
        self.ui.list_derived_components.itemChanged.connect(self._update_state)
        self.ui.list_main_components.order_changed.connect(self._update_state)
        self.ui.list_derived_components.order_changed.connect(self._update_state)

        self.ui.button_ok.clicked.connect(self.accept)
        self.ui.button_cancel.clicked.connect(self.reject)

    def _update_selection_main(self):
        enabled = self.list['main'].selected_cid is not None
        self.button_remove_main.setEnabled(enabled)

    def _update_selection_derived(self):
        enabled = self.list['derived'].selected_cid is not None
        self.button_edit_derived.setEnabled(enabled)
        self.button_remove_derived.setEnabled(enabled)

    @property
    def data(self):
        return self.ui.combosel_data.currentData()

    def _update_component_lists(self, *args):

        # This gets called when the data is changed and we need to update the
        # components shown in the lists.

        for component_list in ('main', 'derived'):

            self.list[component_list].blockSignals(True)

            self.list[component_list].clear()
            for cid in self._components[self.data][component_list]:
                self.list[component_list].add_cid_and_label(cid, self._state[self.data][cid]['label'])

            self.list[component_list].blockSignals(False)

        self._validate()

    def _validate(self):

        # Construct a list of all labels for the current dataset so that
        # we can check which ones are duplicates
        labels = [c['label'] for c in self._state[self.data].values()]
        label_count = Counter(labels)

        if label_count.most_common(1)[0][1] > 1:

            # If we are here, there are duplicates somewhere in the list
            # of components.

            brush_red = QtGui.QBrush(Qt.red)
            brush_black = QtGui.QBrush(Qt.black)

            for component_list in ('main', 'derived'):

                self.list[component_list].blockSignals(True)

                for item in self.list[component_list]:
                    label = item.text(0)
                    if label_count[label] > 1:
                        item.setForeground(0, brush_red)
                    else:
                        item.setForeground(0, brush_black)

                self.list[component_list].blockSignals(False)

            self.ui.label_status.setStyleSheet('color: red')
            self.ui.label_status.setText('Error: some components have duplicate names')
            self.ui.button_ok.setEnabled(False)
            self.ui.combosel_data.setEnabled(False)

        else:

            self.ui.label_status.setStyleSheet('')
            self.ui.label_status.setText('')
            self.ui.button_ok.setEnabled(True)
            self.ui.combosel_data.setEnabled(True)

    def _update_state(self, *args):

        for component_list in ('main', 'derived'):

            self._components[self.data][component_list] = []
            for item in self.list[component_list]:
                cid = item.data(0, Qt.UserRole)
                self._state[self.data][cid]['label'] = item.text(0)
                self._components[self.data][component_list].append(cid)

        self._update_component_lists()

    def _remove_main_component(self, *args):
        cid = self.list['main'].selected_cid
        if cid is not None:
            self._components[self.data]['main'].remove(cid)
            self._state[self.data].pop(cid)
            self._update_component_lists()

    def _remove_derived_component(self, *args):
        cid = self.list['derived'].selected_cid
        if cid is not None:
            self._components[self.data]['derived'].remove(cid)
            self._state[self.data].pop(cid)
            self._update_component_lists()

    def _add_derived_component(self, *args):

        comp_state = {}
        comp_state['cid'] = ComponentID('')
        comp_state['label'] = 'New component'
        comp_state['equation'] = ''

        self._components[self.data]['derived'].append(comp_state['cid'])
        self._state[self.data][comp_state['cid']] = comp_state

        self._update_component_lists()

    def _edit_derived_component(self, *args):

        cid = self.list['derived'].selected_cid

        dialog = EquationEditorDialog(self.data, self._state[self.data][cid]['equation'], parent=self)
        dialog.setWindowFlags(self.windowFlags() | Qt.Window)
        dialog.setFocus()
        dialog.raise_()
        dialog.exec_()

        if dialog.final_expression is None:
            return

        self._state[self.data][cid]['equation'] = dialog.final_expression

    def accept(self):

        for data in self._components:

            cids_main = self._components[data]['main']
            cids_derived = self._components[data]['derived']

            cids_all = cids_main + cids_derived

            cids_existing = data.components

            for cid_old in cids_existing:
                if not any(cid_old is cid_new for cid_new in cids_all):
                    data.remove_component(cid_old)

            # TODO: make it so labels in expression take into account renaming
            components = dict((cid.label, cid) for cid in data.components)

            for cid_new in cids_derived:
                if not any(cid_new is cid_old for cid_old in cids_existing):
                    pc = ParsedCommand(self._state[data][cid_new]['equation'], components)
                    link = ParsedComponentLink(cid_new, pc)
                    data.add_component_link(link)

            data.reorder_components(cids_all)

        super(ComponentManagerWidget, self).accept()

def main():

    import numpy as np

    from glue.core.data import Data
    from glue.core.data_collection import DataCollection

    x = np.random.random((5, 5))
    y = x * 3
    dc = DataCollection()
    dc.append(Data(label='test1', x=x, y=y))
    dc.append(Data(label='test2', a=x, b=y))

    widget = ComponentManagerWidget(dc)
    widget.exec_()

    for data in dc:
        print('-' * 72)
        print(data)

if __name__ == "__main__":
    from glue.utils.qt import get_qapp
    app = get_qapp()
    main()
<|endoftext|>