<commit_before># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__all__ = [
    'GoDaddyDNSDriver'
]

try:
    import simplejson as json
except:
    import json

from libcloud.common.base import ConnectionKey, JsonResponse
from libcloud.common.types import LibcloudError
from libcloud.utils.py3 import httplib
from libcloud.dns.types import Provider, RecordType, RecordDoesNotExistError
from libcloud.dns.base import DNSDriver, Zone, Record

API_ROOT = 'https://api.godaddy.com/'
VALID_RECORD_EXTRA_PARAMS = ['prio', 'ttl']

class GoDaddyDNSException(LibcloudError):
    def __init__(self, code, message):
        self.code = code
        self.message = message
        self.args = (code, message)

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        return '<GoDaddyDNSException in %d: %s>' % (self.code, self.message)

class GoDaddyDNSResponse(JsonResponse):
    valid_response_codes = [httplib.OK, httplib.ACCEPTED, httplib.CREATED,
                            httplib.NO_CONTENT]

    def parse_body(self):
        if not self.body:
            return None
        # json.loads doesn't like the regex expressions used in godaddy schema
        self.body = self.body.replace('\\.', '\\\\.')

        data = json.loads(self.body)
        return data

    def parse_error(self):
        data = self.parse_body()
        raise GoDaddyDNSException(
            data['code'], data['message'])

    def success(self):
        return self.status in self.valid_response_codes

class GoDaddyDNSConnection(ConnectionKey):
    responseCls = GoDaddyDNSResponse
    host = API_ROOT

    allow_insecure = False

    def __init__(self, key, secret, shopper_id, secure=True, host=None,
                 port=None, url=None, timeout=None,
                 proxy_url=None, backoff=None, retry_delay=None):
        super(GoDaddyDNSConnection, self).__init__(
            key,
            secure=secure, host=host,
            port=port, url=url,
            timeout=timeout,
            proxy_url=proxy_url,
            backoff=backoff,
            retry_delay=retry_delay)
        self.key = key
        self.secret = secret
        self.shopper_id = shopper_id

    def add_default_headers(self, headers):
        headers['X-Shopper-Id'] = self.shopper_id
        headers['Authorization'] = "sso-key %s:%s" % \
            (self.key, self.secret)
        return headers

class GoDaddyDNSDriver(DNSDriver):
    """
    A driver for GoDaddy DNS.

    This is for customers of GoDaddy
    who wish to purchase, update existing domains
    and manage records for DNS zones owned by GoDaddy NS servers.
    """
    type = Provider.GODADDY
    name = 'GoDaddy DNS'
    website = 'https://www.godaddy.com/'
    connectionCls = GoDaddyDNSConnection

    RECORD_TYPE_MAP = {
        RecordType.A: 'A',
        RecordType.AAAA: 'AAAA',
        RecordType.CNAME: 'CNAME',
        RecordType.MX: 'MX',
        RecordType.NS: 'SPF',
        RecordType.SRV: 'SRV',
        RecordType.TXT: 'TXT',
    }

    def __init__(self, shopper_id, key, secret,
                 secure=True, host=None, port=None):
        """
        Instantiate a new `GoDaddyDNSDriver`

        :param  shopper_id: Your customer ID or shopper ID with GoDaddy
        :type   shopper_id: ``str``

        :param  key: Your access key from developer.godaddy.com
        :type   key: ``str``

        :param  secret: Your access key secret
        :type   secret: ``str``
        """
        super(GoDaddyDNSDriver, self).__init__(key=key, secret=secret,
                                               secure=secure,
                                               host=host, port=port,
                                               shopper_id=str(shopper_id))

    def list_zones(self):
        """
        Return a list of zones (purchased domains)

        :return: ``list`` of :class:`Zone`
        """
        result = self.connection.request(
            '/v1/domains/').object
        zones = self._to_zones(result)
        return zones

    def list_records(self, zone):
        """
        Return a list of records for the provided zone.

        :param zone: Zone to list records for.
        :type zone: :class:`Zone`

        :return: ``list`` of :class:`Record`
        """
        result = self.connection.request(
            '/v1/domains/%s/records' % (zone.domain)).object
        records = self._to_records(items=result, zone=zone)
        return records

    def create_record(self, name, zone, type, data, extra=None):
        """
        Create a new record.

        :param name: Record name without the domain name (e.g. www).
                     Note: If you want to create a record for a base domain
                     name, you should specify empty string ('') for this
                     argument.
        :type  name: ``str``

        :param zone: Zone where the requested record is created.
        :type  zone: :class:`Zone`

        :param type: DNS record type (A, AAAA, ...).
        :type  type: :class:`RecordType`

        :param data: Data for the record (depends on the record type).
        :type  data: ``str``

        :param extra: Extra attributes (driver specific). (optional)
        :type extra: ``dict``

        :rtype: :class:`Record`
        """
        new_record = self._format_record(name, type, data, extra)
        self.connection.request(
            '/v1/domains/%s/records' % (zone.domain), method='PATCH',
            data=[new_record])
        id = self._get_id_of_record(name, type)
        return Record(
            id=id, name=name,
            type=type, data=data,
            zone=zone, driver=self,
            ttl=new_record['ttl'],
            extra=extra)

    def update_record(self, record, name, type, data, extra=None):
        """
        Update an existing record.

        :param record: Record to update.
        :type  record: :class:`Record`

        :param name: Record name without the domain name (e.g. www).
                     Note: If you want to create a record for a base domain
                     name, you should specify empty string ('') for this
                     argument.
        :type  name: ``str``

        :param type: DNS record type (A, AAAA, ...).
        :type  type: :class:`RecordType`

        :param data: Data for the record (depends on the record type).
        :type  data: ``str``

        :param extra: (optional) Extra attributes (driver specific).
        :type  extra: ``dict``

        :rtype: :class:`Record`
        """
        new_record = self._format_record(name, type, data, extra)
        self.connection.request(
            '/v1/domains/%s/records/%s/%s' % (record.zone.domain,
                                              record.type,
                                              record.name),
            method='PUT',
            data=[new_record])
        id = self._get_id_of_record(name, type)
        return Record(
            id=id, name=name,
            type=type, data=data,
            zone=record.zone, driver=self,
            ttl=new_record['ttl'],
            extra=extra)

    def get_record(self, zone_id, record_id):
        """
        Return a Record instance.

        :param zone_id: ID of the required zone
        :type  zone_id: ``str``

        :param record_id: ID of the required record
        :type  record_id: ``str``

        :rtype: :class:`Record`
        """
        parts = record_id.split(':')
        result = self.connection.request(
            '/v1/domains/%s/records/%s/%s' % (
                zone_id,
                parts[1],
                parts[0])).object
        if len(result) == 0:
            raise RecordDoesNotExistError(record_id,
                                          driver=self,
                                          record_id=record_id)
        return self._to_record(result[0],
                               self.get_zone(zone_id))

    def get_zone(self, zone_id):
        """
        Get a zone (by domain)

        :param  zone_id: The domain, not the ID
        :type   zone_id: ``str``

        :rtype:  :class:`Zone`
        """
        result = self.connection.request(
            '/v1/domains/%s/' % zone_id).object
        zone = self._to_zone(result)
        return zone

    def delete_zone(self, zone):
        """
        Delete a zone.

        Note: This will CANCEL a purchased domain

        :param zone: Zone to delete.
        :type  zone: :class:`Zone`

        :rtype: ``bool``
        """
        self.connection.request(
            '/v1/domains/%s' % (zone.domain),
            method='DELETE')
        # no error means ok
        return True

    def ex_check_availability(self, domain, for_transfer=False):
        """
        Check the availability of the domain

        :param   domain: the domain name e.g. wazzlewobbleflooble.com
        :type    domain: ``str``

        :param   for_transfer: Check if domain is available for transfer
        :type    for_transfer: ``bool``

        :rtype: `list` of :class:`GoDaddyAvailability`
        """
        result = self.connection.request(
            '/v1/domains/available',
            method='GET',
            params={
                'domain': domain,
                'forTransfer': str(for_transfer)
            }
        ).object
        return GoDaddyAvailability(
            domain=result['domain'],
            available=result['available'],
            price=result['price'],
            currency=result['currency'],
            period=result['period']
        )

    def ex_list_tlds(self):
        """
        List available TLDs for sale

        :rtype: ``list`` of :class:`GoDaddyTLD`
        """
        result = self.connection.request(
            '/v1/domains/tlds',
            method='GET'
        ).object
        return self._to_tlds(result)

    def ex_get_purchase_schema(self, tld):
        """
        Get the schema that needs completing to purchase a new domain
        Use this in conjunction with ex_purchase_domain

        :param   tld: The top level domain e.g com, eu, uk
        :type    tld: ``str``

        :rtype: `dict` the JSON Schema
        """
        result = self.connection.request(
            '/v1/domains/purchase/schema/%s' % tld,
            method='GET'
        ).object
        return result

    def ex_get_agreements(self, tld, privacy=True):
        """
        Get the legal agreements for a tld
        Use this in conjunction with ex_purchase_domain

        :param   tld: The top level domain e.g com, eu, uk
        :type    tld: ``str``

        :rtype: `dict` the JSON Schema
        """
        result = self.connection.request(
            '/v1/domains/agreements',
            params={
                'tlds': tld,
                'privacy': str(privacy)
            },
            method='GET'
        ).object
        agreements = []
        for item in result:
            agreements.append(
                GoDaddyLegalAgreement(
                    agreement_key=item['agreementKey'],
                    title=item['title'],
                    url=item['url'],
                    content=item['content']))
        return agreements

    def ex_purchase_domain(self, purchase_request):
        """
        Purchase a domain with GoDaddy

        :param  purchase_request: The completed document
            from ex_get_purchase_schema
        :type   purchase_request: ``dict``

        :rtype: :class:`GoDaddyDomainPurchaseResponse` Your order
        """
        result = self.connection.request(
            '/v1/domains/purchase',
            data=purchase_request,
            method='POST'
        ).object
        return GoDaddyDomainPurchaseResponse(
            order_id=result['orderId'],
            item_count=result['itemCount'],
            total=result['total'],
            currency=result['currency']
        )

    def _format_record(self, name, type, data, extra):
        if extra is None:
            extra = {}
        new_record = {}
        if type == RecordType.SRV:
            new_record = {
                'type': type,
                'name': name,
                'data': data,
                'priority': 1,
                'ttl': extra.get('ttl', 5),
                'service': extra.get('service', ''),
                'protocol': extra.get('protocol', ''),
                'port': extra.get('port', ''),
                'weight': extra.get('weight', '1')
            }
        else:
            new_record = {
                'type': type,
                'name': name,
                'data': data,
                'priority': 1,
                'ttl': extra.get('ttl', 5)
            }
        return new_record

    def _to_zones(self, items):
        zones = []
        for item in items:
            zones.append(self._to_zone(item))
        return zones

    def _to_zone(self, item):
        extra = {"expires": item['expires']}
        zone = Zone(id=item['domainId'], domain=item['domain'],
                    type='master', ttl=None,
                    driver=self, extra=extra)
        return zone

    def _to_records(self, items, zone=None):
        records = []

        for item in items:
            records.append(self._to_record(item=item, zone=zone))
        return records

    def _to_record(self, item, zone=None):
        ttl = item['ttl']
        type = self._string_to_record_type(item['type'])
        name = item['name']
        id = self._get_id_of_record(name, type)
        record = Record(id=id, name=name,
                        type=type, data=item['data'],
                        zone=zone, driver=self,
                        ttl=ttl)
        return record

    def _to_tlds(self, items):
        tlds = []
        for item in items:
            tlds.append(self._to_tld(item))
        return tlds

    def _to_tld(self, item):
        return GoDaddyTLD(
            name=item['name'],
            tld_type=item['type']
        )

    def _get_id_of_record(self, name, type):
        return '%s:%s' % (name, type)

class GoDaddyAvailability(object):
    def __init__(self, domain, available, price, currency, period):
        self.domain = domain
        self.available = bool(available)
        # currency comes in micro-units, convert to dollars.
        self.price = float(price) / 1000000
        self.currency = currency
        self.period = int(period)

class GoDaddyTLD(object):
    def __init__(self, name, tld_type):
        self.name = name
        self.type = tld_type

class GoDaddyDomainPurchaseResponse(object):
    def __init__(self, order_id, item_count, total, currency):
        self.order_id = order_id
        self.item_count = item_count
        self.total = total
        self.current = currency

class GoDaddyLegalAgreement(object):
    def __init__(self, agreement_key, title, url, content):
        self.agreement_key = agreement_key
        self.title = title
        self.url = url
        self.content = content
<commit_msg>Fix a bug in the GoDaddy DNS driver - host attribute needs to be a hostname and not a URL.<commit_after># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__all__ = [
    'GoDaddyDNSDriver'
]

try:
    import simplejson as json
except:
    import json

from libcloud.common.base import ConnectionKey, JsonResponse
from libcloud.common.types import LibcloudError
from libcloud.utils.py3 import httplib
from libcloud.dns.types import Provider, RecordType, RecordDoesNotExistError
from libcloud.dns.base import DNSDriver, Zone, Record

API_HOST = 'api.godaddy.com'
VALID_RECORD_EXTRA_PARAMS = ['prio', 'ttl']

class GoDaddyDNSException(LibcloudError):
    def __init__(self, code, message):
        self.code = code
        self.message = message
        self.args = (code, message)

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        return '<GoDaddyDNSException in %d: %s>' % (self.code, self.message)

class GoDaddyDNSResponse(JsonResponse):
    valid_response_codes = [httplib.OK, httplib.ACCEPTED, httplib.CREATED,
                            httplib.NO_CONTENT]

    def parse_body(self):
        if not self.body:
            return None
        # json.loads doesn't like the regex expressions used in godaddy schema
        self.body = self.body.replace('\\.', '\\\\.')

        data = json.loads(self.body)
        return data

    def parse_error(self):
        data = self.parse_body()
        raise GoDaddyDNSException(
            data['code'], data['message'])

    def success(self):
        return self.status in self.valid_response_codes

class GoDaddyDNSConnection(ConnectionKey):
    responseCls = GoDaddyDNSResponse
    host = API_HOST

    allow_insecure = False

    def __init__(self, key, secret, shopper_id, secure=True, host=None,
                 port=None, url=None, timeout=None,
                 proxy_url=None, backoff=None, retry_delay=None):
        super(GoDaddyDNSConnection, self).__init__(
            key,
            secure=secure, host=host,
            port=port, url=url,
            timeout=timeout,
            proxy_url=proxy_url,
            backoff=backoff,
            retry_delay=retry_delay)
        self.key = key
        self.secret = secret
        self.shopper_id = shopper_id

    def add_default_headers(self, headers):
        headers['X-Shopper-Id'] = self.shopper_id
        headers['Authorization'] = "sso-key %s:%s" % \
            (self.key, self.secret)
        return headers

class GoDaddyDNSDriver(DNSDriver):
    """
    A driver for GoDaddy DNS.

    This is for customers of GoDaddy
    who wish to purchase, update existing domains
    and manage records for DNS zones owned by GoDaddy NS servers.
    """
    type = Provider.GODADDY
    name = 'GoDaddy DNS'
    website = 'https://www.godaddy.com/'
    connectionCls = GoDaddyDNSConnection

    RECORD_TYPE_MAP = {
        RecordType.A: 'A',
        RecordType.AAAA: 'AAAA',
        RecordType.CNAME: 'CNAME',
        RecordType.MX: 'MX',
        RecordType.NS: 'SPF',
        RecordType.SRV: 'SRV',
        RecordType.TXT: 'TXT',
    }

    def __init__(self, shopper_id, key, secret,
                 secure=True, host=None, port=None):
        """
        Instantiate a new `GoDaddyDNSDriver`

        :param  shopper_id: Your customer ID or shopper ID with GoDaddy
        :type   shopper_id: ``str``

        :param  key: Your access key from developer.godaddy.com
        :type   key: ``str``

        :param  secret: Your access key secret
        :type   secret: ``str``
        """
        super(GoDaddyDNSDriver, self).__init__(key=key, secret=secret,
                                               secure=secure,
                                               host=host, port=port,
                                               shopper_id=str(shopper_id))

    def list_zones(self):
        """
        Return a list of zones (purchased domains)

        :return: ``list`` of :class:`Zone`
        """
        result = self.connection.request(
            '/v1/domains/').object
        zones = self._to_zones(result)
        return zones

    def list_records(self, zone):
        """
        Return a list of records for the provided zone.

        :param zone: Zone to list records for.
        :type zone: :class:`Zone`

        :return: ``list`` of :class:`Record`
        """
        result = self.connection.request(
            '/v1/domains/%s/records' % (zone.domain)).object
        records = self._to_records(items=result, zone=zone)
        return records

    def create_record(self, name, zone, type, data, extra=None):
        """
        Create a new record.

        :param name: Record name without the domain name (e.g. www).
                     Note: If you want to create a record for a base domain
                     name, you should specify empty string ('') for this
                     argument.
        :type  name: ``str``

        :param zone: Zone where the requested record is created.
        :type  zone: :class:`Zone`

        :param type: DNS record type (A, AAAA, ...).
        :type  type: :class:`RecordType`

        :param data: Data for the record (depends on the record type).
        :type  data: ``str``

        :param extra: Extra attributes (driver specific). (optional)
        :type extra: ``dict``

        :rtype: :class:`Record`
        """
        new_record = self._format_record(name, type, data, extra)
        self.connection.request(
            '/v1/domains/%s/records' % (zone.domain), method='PATCH',
            data=[new_record])
        id = self._get_id_of_record(name, type)
        return Record(
            id=id, name=name,
            type=type, data=data,
            zone=zone, driver=self,
            ttl=new_record['ttl'],
            extra=extra)

    def update_record(self, record, name, type, data, extra=None):
        """
        Update an existing record.

        :param record: Record to update.
        :type  record: :class:`Record`

        :param name: Record name without the domain name (e.g. www).
                     Note: If you want to create a record for a base domain
                     name, you should specify empty string ('') for this
                     argument.
        :type  name: ``str``

        :param type: DNS record type (A, AAAA, ...).
        :type  type: :class:`RecordType`

        :param data: Data for the record (depends on the record type).
        :type  data: ``str``

        :param extra: (optional) Extra attributes (driver specific).
        :type  extra: ``dict``

        :rtype: :class:`Record`
        """
        new_record = self._format_record(name, type, data, extra)
        self.connection.request(
            '/v1/domains/%s/records/%s/%s' % (record.zone.domain,
                                              record.type,
                                              record.name),
            method='PUT',
            data=[new_record])
        id = self._get_id_of_record(name, type)
        return Record(
            id=id, name=name,
            type=type, data=data,
            zone=record.zone, driver=self,
            ttl=new_record['ttl'],
            extra=extra)

    def get_record(self, zone_id, record_id):
        """
        Return a Record instance.

        :param zone_id: ID of the required zone
        :type  zone_id: ``str``

        :param record_id: ID of the required record
        :type  record_id: ``str``

        :rtype: :class:`Record`
        """
        parts = record_id.split(':')
        result = self.connection.request(
            '/v1/domains/%s/records/%s/%s' % (
                zone_id,
                parts[1],
                parts[0])).object
        if len(result) == 0:
            raise RecordDoesNotExistError(record_id,
                                          driver=self,
                                          record_id=record_id)
        return self._to_record(result[0],
                               self.get_zone(zone_id))

    def get_zone(self, zone_id):
        """
        Get a zone (by domain)

        :param  zone_id: The domain, not the ID
        :type   zone_id: ``str``

        :rtype:  :class:`Zone`
        """
        result = self.connection.request(
            '/v1/domains/%s/' % zone_id).object
        zone = self._to_zone(result)
        return zone

    def delete_zone(self, zone):
        """
        Delete a zone.

        Note: This will CANCEL a purchased domain

        :param zone: Zone to delete.
        :type  zone: :class:`Zone`

        :rtype: ``bool``
        """
        self.connection.request(
            '/v1/domains/%s' % (zone.domain),
            method='DELETE')
        # no error means ok
        return True

    def ex_check_availability(self, domain, for_transfer=False):
        """
        Check the availability of the domain

        :param   domain: the domain name e.g. wazzlewobbleflooble.com
        :type    domain: ``str``

        :param   for_transfer: Check if domain is available for transfer
        :type    for_transfer: ``bool``

        :rtype: `list` of :class:`GoDaddyAvailability`
        """
        result = self.connection.request(
            '/v1/domains/available',
            method='GET',
            params={
                'domain': domain,
                'forTransfer': str(for_transfer)
            }
        ).object
        return GoDaddyAvailability(
            domain=result['domain'],
            available=result['available'],
            price=result['price'],
            currency=result['currency'],
            period=result['period']
        )

    def ex_list_tlds(self):
        """
        List available TLDs for sale

        :rtype: ``list`` of :class:`GoDaddyTLD`
        """
        result = self.connection.request(
            '/v1/domains/tlds',
            method='GET'
        ).object
        return self._to_tlds(result)

    def ex_get_purchase_schema(self, tld):
        """
        Get the schema that needs completing to purchase a new domain
        Use this in conjunction with ex_purchase_domain

        :param   tld: The top level domain e.g com, eu, uk
        :type    tld: ``str``

        :rtype: `dict` the JSON Schema
        """
        result = self.connection.request(
            '/v1/domains/purchase/schema/%s' % tld,
            method='GET'
        ).object
        return result

    def ex_get_agreements(self, tld, privacy=True):
        """
        Get the legal agreements for a tld
        Use this in conjunction with ex_purchase_domain

        :param   tld: The top level domain e.g com, eu, uk
        :type    tld: ``str``

        :rtype: `dict` the JSON Schema
        """
        result = self.connection.request(
            '/v1/domains/agreements',
            params={
                'tlds': tld,
                'privacy': str(privacy)
            },
            method='GET'
        ).object
        agreements = []
        for item in result:
            agreements.append(
                GoDaddyLegalAgreement(
                    agreement_key=item['agreementKey'],
                    title=item['title'],
                    url=item['url'],
                    content=item['content']))
        return agreements

    def ex_purchase_domain(self, purchase_request):
        """
        Purchase a domain with GoDaddy

        :param  purchase_request: The completed document
            from ex_get_purchase_schema
        :type   purchase_request: ``dict``

        :rtype: :class:`GoDaddyDomainPurchaseResponse` Your order
        """
        result = self.connection.request(
            '/v1/domains/purchase',
            data=purchase_request,
            method='POST'
        ).object
        return GoDaddyDomainPurchaseResponse(
            order_id=result['orderId'],
            item_count=result['itemCount'],
            total=result['total'],
            currency=result['currency']
        )

    def _format_record(self, name, type, data, extra):
        if extra is None:
            extra = {}
        new_record = {}
        if type == RecordType.SRV:
            new_record = {
                'type': type,
                'name': name,
                'data': data,
                'priority': 1,
                'ttl': extra.get('ttl', 5),
                'service': extra.get('service', ''),
                'protocol': extra.get('protocol', ''),
                'port': extra.get('port', ''),
                'weight': extra.get('weight', '1')
            }
        else:
            new_record = {
                'type': type,
                'name': name,
                'data': data,
                'priority': 1,
                'ttl': extra.get('ttl', 5)
            }
        return new_record

    def _to_zones(self, items):
        zones = []
        for item in items:
            zones.append(self._to_zone(item))
        return zones

    def _to_zone(self, item):
        extra = {"expires": item['expires']}
        zone = Zone(id=item['domainId'], domain=item['domain'],
                    type='master', ttl=None,
                    driver=self, extra=extra)
        return zone

    def _to_records(self, items, zone=None):
        records = []

        for item in items:
            records.append(self._to_record(item=item, zone=zone))
        return records

    def _to_record(self, item, zone=None):
        ttl = item['ttl']
        type = self._string_to_record_type(item['type'])
        name = item['name']
        id = self._get_id_of_record(name, type)
        record = Record(id=id, name=name,
                        type=type, data=item['data'],
                        zone=zone, driver=self,
                        ttl=ttl)
        return record

    def _to_tlds(self, items):
        tlds = []
        for item in items:
            tlds.append(self._to_tld(item))
        return tlds

    def _to_tld(self, item):
        return GoDaddyTLD(
            name=item['name'],
            tld_type=item['type']
        )

    def _get_id_of_record(self, name, type):
        return '%s:%s' % (name, type)

class GoDaddyAvailability(object):
    def __init__(self, domain, available, price, currency, period):
        self.domain = domain
        self.available = bool(available)
        # currency comes in micro-units, convert to dollars.
        self.price = float(price) / 1000000
        self.currency = currency
        self.period = int(period)

class GoDaddyTLD(object):
    def __init__(self, name, tld_type):
        self.name = name
        self.type = tld_type

class GoDaddyDomainPurchaseResponse(object):
    def __init__(self, order_id, item_count, total, currency):
        self.order_id = order_id
        self.item_count = item_count
        self.total = total
        self.current = currency

class GoDaddyLegalAgreement(object):
    def __init__(self, agreement_key, title, url, content):
        self.agreement_key = agreement_key
        self.title = title
        self.url = url
        self.content = content
<|endoftext|>


<commit_before>f=open("standard_data/train.txt","r")

id_dict={}
relation_dict={}

for line in f:
    line=line.strip()
    line_tokens=line.split()
    try:
        a=id_dict[line_tokens[0]]
    except:
        id_dict[line_tokens[0]]=0

    try:
        a=id_dict[line_tokens[2]]
    except:
        id_dict[line_tokens[2]]=0

    try:
        a=relation_dict[line_tokens[1]]
    except:
        relation_dict[line_tokens[1]]=0

f=open("standard_data/test.txt","r")
for line in f:
    line=line.strip()
    line_tokens=line.split()
    try:
        a=id_dict[line_tokens[0]]
    except:
        id_dict[line_tokens[0]]=0

    try:
        a=id_dict[line_tokens[2]]
    except:
        id_dict[line_tokens[2]]=0

    try:
        a=relation_dict[line_tokens[1]]
    except:
        relation_dict[line_tokens[1]]=0

f=open("standard_data/valid.txt","r")
for line in f:
    line=line.strip()
    line_tokens=line.split()
    try:
        a=id_dict[line_tokens[0]]
    except:
        id_dict[line_tokens[0]]=0

    try:
        a=id_dict[line_tokens[2]]
    except:
        id_dict[line_tokens[2]]=0

    try:
        a=relation_dict[line_tokens[1]]
    except:
        relation_dict[line_tokens[1]]=0

id_count=0
fw=open("standard_data/entity2id.txt","w")

for key in id_dict:
    fw.write(key+"\t"+str(id_count)+"\n")
    id_count=id_count+1

fw.close()

id_count=0
fw=open("standard_data/relation2id.txt","w")

for key in relation_dict:
    fw.write(key+"\t"+str(id_count)+"\n")
    id_count=id_count+1

fw.close()

print "Done"
<commit_msg>Update makeid_file.py<commit_after># -*- coding: utf-8 -*-
###Generates unique id for every entity and relation (predicate)
#Author: Nausheen Fatma, 25th June 2017

f=open("standard_data/train.txt","r")

id_dict={}
relation_dict={}

for line in f:
    line=line.strip()
    line_tokens=line.split()
    try:
        a=id_dict[line_tokens[0]]
    except:
        id_dict[line_tokens[0]]=0

    try:
        a=id_dict[line_tokens[2]]
    except:
        id_dict[line_tokens[2]]=0

    try:
        a=relation_dict[line_tokens[1]]
    except:
        relation_dict[line_tokens[1]]=0

f=open("standard_data/test.txt","r")
for line in f:
    line=line.strip()
    line_tokens=line.split()
    try:
        a=id_dict[line_tokens[0]]
    except:
        id_dict[line_tokens[0]]=0

    try:
        a=id_dict[line_tokens[2]]
    except:
        id_dict[line_tokens[2]]=0

    try:
        a=relation_dict[line_tokens[1]]
    except:
        relation_dict[line_tokens[1]]=0

f=open("standard_data/valid.txt","r")
for line in f:
    line=line.strip()
    line_tokens=line.split()
    try:
        a=id_dict[line_tokens[0]]
    except:
        id_dict[line_tokens[0]]=0

    try:
        a=id_dict[line_tokens[2]]
    except:
        id_dict[line_tokens[2]]=0

    try:
        a=relation_dict[line_tokens[1]]
    except:
        relation_dict[line_tokens[1]]=0

id_count=0
fw=open("standard_data/entity2id.txt","w")

for key in id_dict:
    fw.write(key+"\t"+str(id_count)+"\n")
    id_count=id_count+1

fw.close()

id_count=0
fw=open("standard_data/relation2id.txt","w")

for key in relation_dict:
    fw.write(key+"\t"+str(id_count)+"\n")
    id_count=id_count+1

fw.close()

print "Done"
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import ConfigParser
from subprocess import call

def run_docker():
  config = ConfigParser.ConfigParser()
  config.readfp(open('cluster_setup.ini'))
  fdqn_base = config.get('main', 'domain')
  controller_string = ""
  for item in config.items('controllers'):
    controller_string += item[0] + ":" + item[1] + ","
  controller_string = "ETCD_INITIAL_CLUSTER=" + controller_string[:-1]
  volume_option = os.getcwd() + ":/mnt"
  os_option = "OS=" + config.get('main', 'os')
  version_option = "VERSION=" + config.get('main', 'version')
  container_runtime_option = "CONTAINER_RUNTIME=" + config.get('main', 'runtime')
  cni_provider_option = "CNI_PROVIDER=" + config.get('main', 'cniprovider')
  command_line = ['docker', 'run', '--rm',
                 '-v', volume_option,
                 '-e', os_option,
                 '-e', version_option,
                 '-e', container_runtime_option,
                 '-e', cni_provider_option,
                 '-e', controller_string,
                 '-e', 'ETCD_IP="%{::ipaddress_eth0}"',
                 '-e', 'KUBE_API_ADVERTISE_ADDRESS="%{::ipaddress_eth0}"',
                 '-e', 'INSTALL_DASHBOARD=true', 'puppet/kubetool:3.0.1']
  call(command_line)
  return config

def make_yaml_files(config):
  class_block = """
classes:
  - kubernetes
"""
  for controller in config.items('controllers'):
    f_name = controller[0] + "." + config.get('main', 'domain')
    outfile = open(f_name, 'w')
    outfile.writelines(class_block)
    with open('Rhel.yaml') as infile:
      outfile.write(infile.read())
    with open(controller[0] + ".yaml") as infile:
      outfile.write(infile.read())

  for worker in config.items('workers'):
    f_name = worker[0] + "." + config.get('main', 'domain')
    outfile = open(f_name, 'w')
    outfile.writelines(class_block)
    with open('Rhel.yaml') as infile:
      outfile.write(infile.read())1

def main():
  config = run_docker()
  make_yaml_files(config)

if __name__ == '__main__':
  main()
<commit_msg>stuff<commit_after>#!/usr/bin/env python
import os
import ConfigParser
from subprocess import call

def run_docker():
  config = ConfigParser.ConfigParser()
  config.readfp(open('cluster_setup.ini'))
  fdqn_base = config.get('main', 'domain')
  controller_string = ""
  for item in config.items('controllers'):
    controller_string += item[0] + ":" + item[1] + ","
  controller_string = "ETCD_INITIAL_CLUSTER=" + controller_string[:-1]
  volume_option = os.getcwd() + ":/mnt"
  os_option = "OS=" + config.get('main', 'os')
  version_option = "VERSION=" + config.get('main', 'version')
  container_runtime_option = "CONTAINER_RUNTIME=" + config.get('main', 'runtime')
  cni_provider_option = "CNI_PROVIDER=" + config.get('main', 'cniprovider')
  command_line = ['docker', 'run', '--rm',
                 '-v', volume_option,
                 '-e', os_option,
                 '-e', version_option,
                 '-e', container_runtime_option,
                 '-e', cni_provider_option,
                 '-e', controller_string,
                 '-e', 'ETCD_IP="%{::ipaddress_eth0}"',
                 '-e', 'KUBE_API_ADVERTISE_ADDRESS="%{::ipaddress_eth0}"',
                 '-e', 'INSTALL_DASHBOARD=true', 'puppet/kubetool:3.0.1']
  call(command_line)
  return config

def make_yaml_files(config):
  class_block = """
classes:
  - kubernetes
"""
  for controller in config.items('controllers'):
    f_name = controller[0] + "." + config.get('main', 'domain') + ".yaml"
    outfile = open(f_name, 'w')
    outfile.writelines(class_block)
    with open('Rhel.yaml') as infile:
      outfile.write(infile.read())
    with open(controller[0] + ".yaml") as infile:
      outfile.write(infile.read())

  for worker in config.items('workers'):
    f_name = worker[0] + "." + config.get('main', 'domain') + ".yaml"
    outfile = open(f_name, 'w')
    outfile.writelines(class_block)
    with open('Rhel.yaml') as infile:
      outfile.write(infile.read())1

def main():
  config = run_docker()
  make_yaml_files(config)

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before>import itertools
from collections import OrderedDict, namedtuple

import numpy as np
from sympy import Indexed

from devito.ir.support import Stencil
from devito.exceptions import DSEException
from devito.symbolics import retrieve_indexed, q_indirect

__all__ = ['collect']

def collect(exprs):
    """
    Determine groups of aliasing expressions in ``exprs``.

    An expression A aliases an expression B if both A and B apply the same
    operations to the same input operands, with the possibility for indexed objects
    to index into locations at a fixed constant offset in each dimension.

    For example: ::

        exprs = (a[i+1] + b[i+1], a[i+1] + b[j+1], a[i] + c[i],
                 a[i+2] - b[i+2], a[i+2] + b[i], a[i-1] + b[i-1])

    The following expressions in ``exprs`` alias to ``a[i] + b[i]``: ::

        a[i+1] + b[i+1] : same operands and operations, distance along i = 1
        a[i-1] + b[i-1] : same operands and operations, distance along i = -1

    Whereas the following do not: ::

        a[i+1] + b[j+1] : because at least one index differs
        a[i] + c[i] : because at least one of the operands differs
        a[i+2] - b[i+2] : because at least one operation differs
        a[i+2] + b[i] : because distance along ``i`` differ (+2 and +0)
    """
    ExprData = namedtuple('ExprData', 'dimensions offsets')

    # Discard expressions:
    # - that surely won't alias to anything
    # - that are non-scalar
    candidates = OrderedDict()
    for expr in exprs:
        if expr.lhs.is_Indexed:
            continue
        indexeds = retrieve_indexed(expr.rhs, mode='all')
        if indexeds and not any(q_indirect(i) for i in indexeds):
            handle = calculate_offsets(indexeds)
            if handle:
                candidates[expr.rhs] = ExprData(*handle)

    aliases = OrderedDict()
    mapper = OrderedDict()
    unseen = list(candidates)
    while unseen:
        # Find aliasing expressions
        handle = unseen.pop(0)
        group = [handle]
        for e in list(unseen):
            if compare(handle, e) and\
                    is_translated(candidates[handle].offsets, candidates[e].offsets):
                group.append(e)
                unseen.remove(e)
        mapper.update([(i, group) for i in group])

        # Try creating a basis for the aliasing expressions' offsets
        offsets = [tuple(candidates[e].offsets) for e in group]
        try:
            COM, distances = calculate_COM(offsets)
        except DSEException:
            # Ignore these potential aliases and move on
            continue

        alias = create_alias(handle, COM)
        # In circumstances in which an expression has repeated coefficients, e.g.
        # ... + 0.025*a[...] + 0.025*b[...],
        # We may have found a common basis (i.e., same COM, same alias) at this point
        v = aliases.setdefault(alias, Alias(alias, candidates[handle].dimensions))
        v.extend(group, distances)

    # Heuristically attempt to relax the aliases offsets
    # to maximize the likelyhood of loop fusion
    grouped = OrderedDict()
    for i in aliases.values():
        grouped.setdefault(i.dimensions, []).append(i)
    for dimensions, group in grouped.items():
        ideal_anti_stencil = Stencil.union(*[i.anti_stencil for i in group])
        for i in group:
            if i.anti_stencil.subtract(ideal_anti_stencil).empty:
                aliases[i.alias] = i.relax(ideal_anti_stencil)

    return mapper, aliases

# Helpers

def create_alias(expr, offsets):
    """
    Create an aliasing expression of ``expr`` by replacing the offsets of each
    indexed object in ``expr`` with the new values in ``offsets``. ``offsets``
    is an ordered sequence of tuples with as many elements as the number of
    indexed objects in ``expr``.
    """
    indexeds = retrieve_indexed(expr, mode='all')
    assert len(indexeds) == len(offsets)

    mapper = {}
    for indexed, ofs in zip(indexeds, offsets):
        base = indexed.base
        dimensions = base.function.indices
        assert len(dimensions) == len(ofs)
        mapper[indexed] = indexed.func(base, *[sum(i) for i in zip(dimensions, ofs)])

    return expr.xreplace(mapper)

def calculate_COM(offsets):
    """
    Determine the centre of mass (COM) in a collection of offsets.

    The COM is a basis to span the vectors in ``offsets``.

    Also return the distance of each element E in ``offsets`` from the COM (i.e.,
    the coefficients that when multiplied by the COM give exactly E).
    """
    COM = []
    for ofs in zip(*offsets):
        handle = []
        for i in zip(*ofs):
            strides = sorted(set(i))
            # Heuristic:
            # - middle point if odd number of values, or
            # - strides average otherwise
            index = int((len(strides) - 1) / 2)
            if (len(strides) - 1) % 2 == 0:
                handle.append(strides[index])
            else:
                handle.append(int(np.mean(strides, dtype=int)))
        COM.append(tuple(handle))

    distances = []
    for ofs in offsets:
        handle = distance(COM, ofs)
        if len(handle) != 1:
            raise DSEException("%s cannot be represented by the COM %s" %
                               (str(ofs), str(COM)))
        distances.append(handle.pop())

    return COM, distances

def calculate_offsets(indexeds):
    """
    Return a list of tuples, with one tuple for each indexed object appearing
    in ``indexeds``. A tuple represents the offsets from the origin (0, 0, ..., 0)
    along each dimension. All objects must use the same dimensions, in the same
    order; otherwise, ``None`` is returned.

    For example, given: ::

        indexeds = [A[i,j,k], B[i,j+2,k+3]]

    Return: ::

        [(0, 0, 0), (0, 2, 3)]
    """
    processed = []
    reference = indexeds[0].base.function.indices
    for indexed in indexeds:
        dimensions = indexed.base.function.indices
        if dimensions != reference:
            return None
        handle = []
        for d, i in zip(dimensions, indexed.indices):
            offset = i - d
            if offset.is_Number:
                handle.append(int(offset))
            else:
                return None
        processed.append(tuple(handle))
    return tuple(reference), processed

def distance(ofs1, ofs2):
    """
    Determine the distance of ``ofs2`` from ``ofs1``.
    """
    assert len(ofs1) == len(ofs2)
    handle = set()
    for o1, o2 in zip(ofs1, ofs2):
        assert len(o1) == len(o2)
        handle.add(tuple(i2 - i1 for i1, i2 in zip(o1, o2)))
    return handle

def is_translated(ofs1, ofs2):
    """
    Return True if ``ofs2`` is translated w.r.t. to ``ofs1``, False otherwise.

    For example: ::

        e1 = A[i,j] + A[i,j+1]
        e2 = A[i+1,j] + A[i+1,j+1]

    ``ofs1`` would be [(0, 0), (0, 1)], while ``ofs2`` would be [(1, 0), (1,1)], so
    ``e2`` is translated w.r.t. ``e1`` by ``(1, 0)``, and True is returned.
    """
    return len(distance(ofs1, ofs2)) == 1

def compare(e1, e2):
    """
    Return True if the two expressions e1 and e2 alias each other, False otherwise.
    """
    if type(e1) == type(e2) and len(e1.args) == len(e2.args):
        if e1.is_Atom:
            return True if e1 == e2 else False
        elif isinstance(e1, Indexed) and isinstance(e2, Indexed):
            return True if e1.base == e2.base else False
        else:
            for a1, a2 in zip(e1.args, e2.args):
                if not compare(a1, a2):
                    return False
            return True
    else:
        return False

class Alias(object):

    """
    Map an expression (the so called "alias") to a set of aliasing expressions.
    For each aliasing expression, the distance from the alias along each dimension
    is tracked.
    """

    def __init__(self, alias, dimensions, aliased=None, distances=None,
                 ghost_offsets=None):
        self.alias = alias
        self.dimensions = dimensions

        self.aliased = aliased or []
        self.distances = distances or []
        self._ghost_offsets = ghost_offsets or []

        assert len(self.aliased) == len(self.distances)
        assert all(len(i) == len(dimensions) for i in self.distances)

    @property
    def anti_stencil(self):
        handle = Stencil()
        for d, i in zip(self.dimensions, zip(*self.distances)):
            handle[d].update(set(i))
        for d, i in zip(self.dimensions, zip(*self._ghost_offsets)):
            handle[d].update(set(i))
        return handle

    @property
    def with_distance(self):
        return [(i, OrderedDict(zip(self.dimensions, j)))
                for i, j in zip(self.aliased, self.distances)]

    def extend(self, aliased, distances):
        assert len(aliased) == len(distances)
        assert all(len(i) == len(self.dimensions) for i in distances)
        self.aliased.extend(aliased)
        self.distances.extend(distances)

    def relax(self, distances):
        return Alias(self.alias, self.dimensions, self.aliased, self.distances,
                     self._ghost_offsets + list(itertools.product(*distances.values())))
<commit_msg>dse: Update aliases mapper at the right place<commit_after>import itertools
from collections import OrderedDict, namedtuple

import numpy as np
from sympy import Indexed

from devito.ir.support import Stencil
from devito.exceptions import DSEException
from devito.symbolics import retrieve_indexed, q_indirect

__all__ = ['collect']

def collect(exprs):
    """
    Determine groups of aliasing expressions in ``exprs``.

    An expression A aliases an expression B if both A and B apply the same
    operations to the same input operands, with the possibility for indexed objects
    to index into locations at a fixed constant offset in each dimension.

    For example: ::

        exprs = (a[i+1] + b[i+1], a[i+1] + b[j+1], a[i] + c[i],
                 a[i+2] - b[i+2], a[i+2] + b[i], a[i-1] + b[i-1])

    The following expressions in ``exprs`` alias to ``a[i] + b[i]``: ::

        a[i+1] + b[i+1] : same operands and operations, distance along i = 1
        a[i-1] + b[i-1] : same operands and operations, distance along i = -1

    Whereas the following do not: ::

        a[i+1] + b[j+1] : because at least one index differs
        a[i] + c[i] : because at least one of the operands differs
        a[i+2] - b[i+2] : because at least one operation differs
        a[i+2] + b[i] : because distance along ``i`` differ (+2 and +0)
    """
    ExprData = namedtuple('ExprData', 'dimensions offsets')

    # Discard expressions:
    # - that surely won't alias to anything
    # - that are non-scalar
    candidates = OrderedDict()
    for expr in exprs:
        if expr.lhs.is_Indexed:
            continue
        indexeds = retrieve_indexed(expr.rhs, mode='all')
        if indexeds and not any(q_indirect(i) for i in indexeds):
            handle = calculate_offsets(indexeds)
            if handle:
                candidates[expr.rhs] = ExprData(*handle)

    aliases = OrderedDict()
    mapper = OrderedDict()
    unseen = list(candidates)
    while unseen:
        # Find aliasing expressions
        handle = unseen.pop(0)
        group = [handle]
        for e in list(unseen):
            if compare(handle, e) and\
                    is_translated(candidates[handle].offsets, candidates[e].offsets):
                group.append(e)
                unseen.remove(e)

        # Try creating a basis for the aliasing expressions' offsets
        offsets = [tuple(candidates[e].offsets) for e in group]
        try:
            COM, distances = calculate_COM(offsets)
        except DSEException:
            # Ignore these potential aliases and move on
            continue

        alias = create_alias(handle, COM)

        # An alias has been created, so I can now update the expression mapper
        mapper.update([(i, group) for i in group])

        # In circumstances in which an expression has repeated coefficients, e.g.
        # ... + 0.025*a[...] + 0.025*b[...],
        # We may have found a common basis (i.e., same COM, same alias) at this point
        v = aliases.setdefault(alias, Alias(alias, candidates[handle].dimensions))
        v.extend(group, distances)

    # Heuristically attempt to relax the aliases offsets
    # to maximize the likelyhood of loop fusion
    grouped = OrderedDict()
    for i in aliases.values():
        grouped.setdefault(i.dimensions, []).append(i)
    for dimensions, group in grouped.items():
        ideal_anti_stencil = Stencil.union(*[i.anti_stencil for i in group])
        for i in group:
            if i.anti_stencil.subtract(ideal_anti_stencil).empty:
                aliases[i.alias] = i.relax(ideal_anti_stencil)

    return mapper, aliases

# Helpers

def create_alias(expr, offsets):
    """
    Create an aliasing expression of ``expr`` by replacing the offsets of each
    indexed object in ``expr`` with the new values in ``offsets``. ``offsets``
    is an ordered sequence of tuples with as many elements as the number of
    indexed objects in ``expr``.
    """
    indexeds = retrieve_indexed(expr, mode='all')
    assert len(indexeds) == len(offsets)

    mapper = {}
    for indexed, ofs in zip(indexeds, offsets):
        base = indexed.base
        dimensions = base.function.indices
        assert len(dimensions) == len(ofs)
        mapper[indexed] = indexed.func(base, *[sum(i) for i in zip(dimensions, ofs)])

    return expr.xreplace(mapper)

def calculate_COM(offsets):
    """
    Determine the centre of mass (COM) in a collection of offsets.

    The COM is a basis to span the vectors in ``offsets``.

    Also return the distance of each element E in ``offsets`` from the COM (i.e.,
    the coefficients that when multiplied by the COM give exactly E).
    """
    COM = []
    for ofs in zip(*offsets):
        handle = []
        for i in zip(*ofs):
            strides = sorted(set(i))
            # Heuristic:
            # - middle point if odd number of values, or
            # - strides average otherwise
            index = int((len(strides) - 1) / 2)
            if (len(strides) - 1) % 2 == 0:
                handle.append(strides[index])
            else:
                handle.append(int(np.mean(strides, dtype=int)))
        COM.append(tuple(handle))

    distances = []
    for ofs in offsets:
        handle = distance(COM, ofs)
        if len(handle) != 1:
            raise DSEException("%s cannot be represented by the COM %s" %
                               (str(ofs), str(COM)))
        distances.append(handle.pop())

    return COM, distances

def calculate_offsets(indexeds):
    """
    Return a list of tuples, with one tuple for each indexed object appearing
    in ``indexeds``. A tuple represents the offsets from the origin (0, 0, ..., 0)
    along each dimension. All objects must use the same dimensions, in the same
    order; otherwise, ``None`` is returned.

    For example, given: ::

        indexeds = [A[i,j,k], B[i,j+2,k+3]]

    Return: ::

        [(0, 0, 0), (0, 2, 3)]
    """
    processed = []
    reference = indexeds[0].base.function.indices
    for indexed in indexeds:
        dimensions = indexed.base.function.indices
        if dimensions != reference:
            return None
        handle = []
        for d, i in zip(dimensions, indexed.indices):
            offset = i - d
            if offset.is_Number:
                handle.append(int(offset))
            else:
                return None
        processed.append(tuple(handle))
    return tuple(reference), processed

def distance(ofs1, ofs2):
    """
    Determine the distance of ``ofs2`` from ``ofs1``.
    """
    assert len(ofs1) == len(ofs2)
    handle = set()
    for o1, o2 in zip(ofs1, ofs2):
        assert len(o1) == len(o2)
        handle.add(tuple(i2 - i1 for i1, i2 in zip(o1, o2)))
    return handle

def is_translated(ofs1, ofs2):
    """
    Return True if ``ofs2`` is translated w.r.t. to ``ofs1``, False otherwise.

    For example: ::

        e1 = A[i,j] + A[i,j+1]
        e2 = A[i+1,j] + A[i+1,j+1]

    ``ofs1`` would be [(0, 0), (0, 1)], while ``ofs2`` would be [(1, 0), (1,1)], so
    ``e2`` is translated w.r.t. ``e1`` by ``(1, 0)``, and True is returned.
    """
    return len(distance(ofs1, ofs2)) == 1

def compare(e1, e2):
    """
    Return True if the two expressions e1 and e2 alias each other, False otherwise.
    """
    if type(e1) == type(e2) and len(e1.args) == len(e2.args):
        if e1.is_Atom:
            return True if e1 == e2 else False
        elif isinstance(e1, Indexed) and isinstance(e2, Indexed):
            return True if e1.base == e2.base else False
        else:
            for a1, a2 in zip(e1.args, e2.args):
                if not compare(a1, a2):
                    return False
            return True
    else:
        return False

class Alias(object):

    """
    Map an expression (the so called "alias") to a set of aliasing expressions.
    For each aliasing expression, the distance from the alias along each dimension
    is tracked.
    """

    def __init__(self, alias, dimensions, aliased=None, distances=None,
                 ghost_offsets=None):
        self.alias = alias
        self.dimensions = dimensions

        self.aliased = aliased or []
        self.distances = distances or []
        self._ghost_offsets = ghost_offsets or []

        assert len(self.aliased) == len(self.distances)
        assert all(len(i) == len(dimensions) for i in self.distances)

    @property
    def anti_stencil(self):
        handle = Stencil()
        for d, i in zip(self.dimensions, zip(*self.distances)):
            handle[d].update(set(i))
        for d, i in zip(self.dimensions, zip(*self._ghost_offsets)):
            handle[d].update(set(i))
        return handle

    @property
    def with_distance(self):
        return [(i, OrderedDict(zip(self.dimensions, j)))
                for i, j in zip(self.aliased, self.distances)]

    def extend(self, aliased, distances):
        assert len(aliased) == len(distances)
        assert all(len(i) == len(self.dimensions) for i in distances)
        self.aliased.extend(aliased)
        self.distances.extend(distances)

    def relax(self, distances):
        return Alias(self.alias, self.dimensions, self.aliased, self.distances,
                     self._ghost_offsets + list(itertools.product(*distances.values())))
<|endoftext|>


<commit_before>#!/usr/bin/env python

import rethinkdb as r
from optparse import OptionParser
import sys
import os

def usesidRemap(x):
    r = e['datafile_id']
    if (e['usesid']):
        r = e['usesid']
    print r
    return r

if __name__ == "__main__":
    parser = OptionParser()
    parser.add_option("-i", "--id", dest="id",
                      help="dataset id", type="string")
    parser.add_option("-P", "--port", dest="port", type="int",
                      help="rethinkdb port", default=30815)

    (options, args) = parser.parse_args()

    (options, args) = parser.parse_args()
    if options.id is None:
        print "You must specify a dataset id"
        sys.exit(1)

    conn = r.connect('localhost', options.port, db='materialscommons')

    dataset = r.table('datasets').get(options.id).run(conn)

    if dataset is None:
        print "No such dataset: " + options.id
        sys.exit(1)

    print "getting all files for the dataset with the name '" + dataset["title"] + "'"

    entries = r.table('dataset2datafile')\
        .eq_join('datafile_id',r.table("datafiles")).zip()\
        .filter({"dataset_id":options.id})\
        .pluck("usesid","datafile_id")\
        .run(conn)

    for e in entries:
        print e['datafile_id']
        if (e['usesid']):
            print "  ==>>" + e['usesid']

    print "-----------------------"

    mapping = map(None,entries)

    print "-----------------------"

    for x in mapping:
        print x<commit_msg>Script for generating tar of files for dataset, now generating all paths of data files from database information<commit_after>#!/usr/bin/env python

import rethinkdb as r
from optparse import OptionParser
import sys
import os

def usesidRemap(x):
    r = x['datafile_id']
    if (x['usesid']):
        r = x['usesid']
    return r

def makePath(x):
    s = x.split('-')[1]
    return s[:2] + "/" + s[2:] + "/" + x

def getOptions():
    parser = OptionParser()
    parser.add_option("-i", "--id", dest="id",
                      help="dataset id", type="string")
    parser.add_option("-P", "--port",dest="port", type="int",
                      help="rethinkdb port", default=30815)
    parser.add_option("-B", "--base",dest="base", type='string');

    (options, args) = parser.parse_args()

    if options.id is None:
        print "You must specify a dataset id"
        sys.exit(1)

    if options.base is None:
        print "You must specify a base directory"
        sys.exit(1)

    return options

if __name__ == "__main__":

    options = getOptions()
    dataset_id = options.id
    base_dir = options.base
    db_port = options.port

    conn = r.connect('localhost', db_port, db='materialscommons')

    dataset = r.table('datasets').get(dataset_id).run(conn)

    if dataset is None:
        print "No such dataset: " + dataset_id
        sys.exit(1)

    print "getting all files for the dataset with the name '" + dataset["title"] + "'"

    entries = list(r.table('dataset2datafile')\
        .get_all(dataset_id, index='dataset_id')\
        .eq_join('datafile_id',r.table('datafiles'))
        .zip().pluck('usesid', 'datafile_id').run(conn))

    mapping = map(makePath,map(usesidRemap,entries))

    for x in mapping:
        print base_dir + "/" + x<|endoftext|>


<commit_before>from .codes import RESPONSE_MSG
from json.decoder import JSONDecodeError
from flask_executor import Executor

from syft.core.common.message import SignedImmediateSyftMessageWithReply
from syft.core.common.message import SignedImmediateSyftMessageWithoutReply

from .node import node
from .exceptions import (
    PyGridError,
    UserNotFoundError,
    RoleNotFoundError,
    GroupNotFoundError,
    AuthorizationError,
    MissingRequestKeyError,
    InvalidCredentialsError,
)

executor = Executor()

def process_as_syft_message(message_class, message_content, sign_key):
    message = message_class(**message_content)
    signed_message = message.sign(signing_key=sign_key)

    response = {}
    if isinstance(signed_message, SignedImmediateSyftMessageWithReply):
        response = node.recv_immediate_msg_with_reply(msg=signed_message)
        response = response.message
    elif isinstance(signed_message, SignedImmediateSyftMessageWithoutReply):
        node.recv_immediate_msg_without_reply(msg=signed_message)
    else:
        node.recv_eventual_msg_without_reply(msg=signed_message)

    return response

def task_handler(route_function, data, mandatory, optional=[]):
    args = {}
    response_body = {}

    if not data:
        data = {}

    # Fill mandatory args
    for (arg, error) in mandatory.items():
        value = data.get(arg)

        # If not found
        if not value:
            raise error  # Specific Error
        else:
            args[arg] = value  # Add in args dict

    for opt in optional:
        value = data.get(opt)

        # If found
        if value:
            args[opt] = value  # Add in args dict

    # Execute task
    response_body = route_function(**args)

    return response_body
<commit_msg>Move generic route_logic function to task_handler module<commit_after>from .codes import RESPONSE_MSG
from json.decoder import JSONDecodeError
from flask_executor import Executor
from flask import request

from syft.core.common.message import SignedImmediateSyftMessageWithReply
from syft.core.common.message import SignedImmediateSyftMessageWithoutReply

# third party
from nacl.encoding import HexEncoder
from nacl.signing import SigningKey

from .node import node
from .exceptions import (
    PyGridError,
    UserNotFoundError,
    RoleNotFoundError,
    GroupNotFoundError,
    AuthorizationError,
    MissingRequestKeyError,
    InvalidCredentialsError,
)

executor = Executor()

def process_as_syft_message(message_class, message_content, sign_key):
    message = message_class(**message_content)
    signed_message = message.sign(signing_key=sign_key)

    response = {}
    if isinstance(signed_message, SignedImmediateSyftMessageWithReply):
        response = node.recv_immediate_msg_with_reply(msg=signed_message)
        response = response.message
    elif isinstance(signed_message, SignedImmediateSyftMessageWithoutReply):
        node.recv_immediate_msg_without_reply(msg=signed_message)
    else:
        node.recv_eventual_msg_without_reply(msg=signed_message)

    return response

def task_handler(route_function, data, mandatory, optional=[]):
    args = {}
    response_body = {}

    if not data:
        data = {}

    # Fill mandatory args
    for (arg, error) in mandatory.items():
        value = data.get(arg)

        # If not found
        if not value:
            raise error  # Specific Error
        else:
            args[arg] = value  # Add in args dict

    for opt in optional:
        value = data.get(opt)

        # If found
        if value:
            args[opt] = value  # Add in args dict

    # Execute task
    response_body = route_function(**args)

    return response_body

def route_logic(message_class, current_user, content):
    user_key = SigningKey(current_user.private_key.encode("utf-8"), encoder=HexEncoder)

    content = {
        "address": node.address,
        "content": content,
        "reply_to": node.address,
    }

    syft_message = {}
    syft_message["message_class"] = message_class
    syft_message["message_content"] = content
    syft_message["sign_key"] = user_key

    # Execute task
    response_msg = task_handler(
        route_function=process_as_syft_message,
        data=syft_message,
        mandatory={
            "message_class": MissingRequestKeyError,
            "message_content": MissingRequestKeyError,
            "sign_key": MissingRequestKeyError,
        },
    )
    return response_msg
<|endoftext|>


<commit_before># Python Security Project (PySec) and its related class files.
#
# PySec is a set of tools for secure application development under Linux
#
# Copyright 2014 PySec development team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# -*- coding: ascii -*-
"""Module with statistics utilities."""

__all__ = 'avg',

def avg():
    """Generator to calculate arithmetic mean"""
    avg = el = (yield .0)
    if el is None:
        yield .0
        raise StopIteration
    n = .0
    while 1:
        n += 1
        el = (yield avg)
        if el is None:
            yield avg
            raise StopIteration
        el = float(el)
        avg = (avg + el / n) / ((n + 1) / n)
<commit_msg>add intial values for avg()<commit_after># Python Security Project (PySec) and its related class files.
#
# PySec is a set of tools for secure application development under Linux
#
# Copyright 2014 PySec development team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# -*- coding: ascii -*-
"""Module with statistics utilities."""

__all__ = 'avg',

def avg(*values):
    """Generator to calculate arithmetic mean"""
    n = -1.
    tot = .0
    for n, el in enumerate(values):
        tot += el
    n += 1.
    avg = 0 if n == 0. else tot/n
    while 1:
        el = (yield avg)
        if el is None:
            continue
        el = float(el)
        avg = (avg + el / n) / ((n + 1) / n)
        n += 1

<|endoftext|>


<commit_before>from collections import OrderedDict

from graphql_relay.utils import base64

from ...types import ObjectType, Schema, String
from ..connection import ConnectionField, PageInfo
from ..node import Node

letter_chars = ['A', 'B', 'C', 'D', 'E']

class Letter(ObjectType):

    class Meta:
        interfaces = (Node, )

    letter = String()

class Query(ObjectType):
    letters = ConnectionField(Letter)
    connection_letters = ConnectionField(Letter)

    node = Node.Field()

    def resolve_letters(self, args, context, info):
        return list(letters.values())

    def resolve_connection_letters(self, args, context, info):
        return Letter.Connection(
            page_info=PageInfo(
                has_next_page=True,
                has_previous_page=False
            ),
            edges=[
                Letter.Connection.Edge(
                    node=Letter(id=0, letter='A'),
                    cursor='a-cursor'
                ),
            ]
        )

schema = Schema(Query)

letters = OrderedDict()
for i, letter in enumerate(letter_chars):
    l = Letter(id=i, letter=letter)
    letters[letter] = l

def edges(selected_letters):
    return [
        {
            'node': {
                'id': base64('Letter:%s' % l.id),
                'letter': l.letter
            },
            'cursor': base64('arrayconnection:%s' % l.id)
        }
        for l in [letters[i] for i in selected_letters]
    ]

def cursor_for(ltr):
    l = letters[ltr]
    return base64('arrayconnection:%s' % l.id)

def execute(args=''):
    if args:
        args = '(' + args + ')'

    return schema.execute('''
    {
        letters%s {
            edges {
                node {
                    id
                    letter
                }
                cursor
            }
            pageInfo {
                hasPreviousPage
                hasNextPage
                startCursor
                endCursor
            }
        }
    }
    ''' % args)

def check(args, letters, has_previous_page=False, has_next_page=False):
    result = execute(args)
    expected_edges = edges(letters)
    expected_page_info = {
        'hasPreviousPage': has_previous_page,
        'hasNextPage': has_next_page,
        'endCursor': expected_edges[-1]['cursor'] if expected_edges else None,
        'startCursor': expected_edges[0]['cursor'] if expected_edges else None
    }

    assert not result.errors
    assert result.data == {
        'letters': {
            'edges': expected_edges,
            'pageInfo': expected_page_info
        }
    }

def test_returns_all_elements_without_filters():
    check('', 'ABCDE')

def test_respects_a_smaller_first():
    check('first: 2', 'AB', has_next_page=True)

def test_respects_an_overly_large_first():
    check('first: 10', 'ABCDE')

def test_respects_a_smaller_last():
    check('last: 2', 'DE', has_previous_page=True)

def test_respects_an_overly_large_last():
    check('last: 10', 'ABCDE')

def test_respects_first_and_after():
    check('first: 2, after: "{}"'.format(cursor_for('B')), 'CD', has_next_page=True)

def test_respects_first_and_after_with_long_first():
    check('first: 10, after: "{}"'.format(cursor_for('B')), 'CDE')

def test_respects_last_and_before():
    check('last: 2, before: "{}"'.format(cursor_for('D')), 'BC', has_previous_page=True)

def test_respects_last_and_before_with_long_last():
    check('last: 10, before: "{}"'.format(cursor_for('D')), 'ABC')

def test_respects_first_and_after_and_before_too_few():
    check('first: 2, after: "{}", before: "{}"'.format(cursor_for('A'), cursor_for('E')), 'BC', has_next_page=True)

def test_respects_first_and_after_and_before_too_many():
    check('first: 4, after: "{}", before: "{}"'.format(cursor_for('A'), cursor_for('E')), 'BCD')

def test_respects_first_and_after_and_before_exactly_right():
    check('first: 3, after: "{}", before: "{}"'.format(cursor_for('A'), cursor_for('E')), "BCD")

def test_respects_last_and_after_and_before_too_few():
    check('last: 2, after: "{}", before: "{}"'.format(cursor_for('A'), cursor_for('E')), 'CD', has_previous_page=True)

def test_respects_last_and_after_and_before_too_many():
    check('last: 4, after: "{}", before: "{}"'.format(cursor_for('A'), cursor_for('E')), 'BCD')

def test_respects_last_and_after_and_before_exactly_right():
    check('last: 3, after: "{}", before: "{}"'.format(cursor_for('A'), cursor_for('E')), 'BCD')

def test_returns_no_elements_if_first_is_0():
    check('first: 0', '', has_next_page=True)

def test_returns_all_elements_if_cursors_are_invalid():
    check('before: "invalid" after: "invalid"', 'ABCDE')

def test_returns_all_elements_if_cursors_are_on_the_outside():
    check(
        'before: "{}" after: "{}"'.format(
            base64(
                'arrayconnection:%s' % 6),
            base64(
                'arrayconnection:%s' % -1)),
        'ABCDE')

def test_returns_no_elements_if_cursors_cross():
    check('before: "{}" after: "{}"'.format(base64('arrayconnection:%s' % 2), base64('arrayconnection:%s' % 4)), '')

def test_connection_type_nodes():
    result = schema.execute('''
    {
        connectionLetters {
            edges {
                node {
                    id
                    letter
                }
                cursor
            }
            pageInfo {
                hasPreviousPage
                hasNextPage
            }
        }
    }
    ''')

    assert not result.errors
    assert result.data == {
        'connectionLetters': {
            'edges': [{
                'node': {
                    'id': 'TGV0dGVyOjA=',
                    'letter': 'A',
                },
                'cursor': 'a-cursor',
            }],
            'pageInfo': {
                'hasPreviousPage': False,
                'hasNextPage': True,
            }
        }
    }
<commit_msg>Added connection promise tests<commit_after>from collections import OrderedDict

from graphql_relay.utils import base64
from promise import Promise

from ...types import ObjectType, Schema, String
from ..connection import ConnectionField, PageInfo
from ..node import Node

letter_chars = ['A', 'B', 'C', 'D', 'E']

class Letter(ObjectType):

    class Meta:
        interfaces = (Node, )

    letter = String()

class Query(ObjectType):
    letters = ConnectionField(Letter)
    connection_letters = ConnectionField(Letter)
    promise_letters = ConnectionField(Letter)

    node = Node.Field()

    def resolve_letters(self, args, context, info):
        return list(letters.values())

    def resolve_promise_letters(self, args, context, info):
        return Promise.resolve(list(letters.values()))

    def resolve_connection_letters(self, args, context, info):
        return Letter.Connection(
            page_info=PageInfo(
                has_next_page=True,
                has_previous_page=False
            ),
            edges=[
                Letter.Connection.Edge(
                    node=Letter(id=0, letter='A'),
                    cursor='a-cursor'
                ),
            ]
        )

schema = Schema(Query)

letters = OrderedDict()
for i, letter in enumerate(letter_chars):
    l = Letter(id=i, letter=letter)
    letters[letter] = l

def edges(selected_letters):
    return [
        {
            'node': {
                'id': base64('Letter:%s' % l.id),
                'letter': l.letter
            },
            'cursor': base64('arrayconnection:%s' % l.id)
        }
        for l in [letters[i] for i in selected_letters]
    ]

def cursor_for(ltr):
    l = letters[ltr]
    return base64('arrayconnection:%s' % l.id)

def execute(args=''):
    if args:
        args = '(' + args + ')'

    return schema.execute('''
    {
        letters%s {
            edges {
                node {
                    id
                    letter
                }
                cursor
            }
            pageInfo {
                hasPreviousPage
                hasNextPage
                startCursor
                endCursor
            }
        }
    }
    ''' % args)

def check(args, letters, has_previous_page=False, has_next_page=False):
    result = execute(args)
    expected_edges = edges(letters)
    expected_page_info = {
        'hasPreviousPage': has_previous_page,
        'hasNextPage': has_next_page,
        'endCursor': expected_edges[-1]['cursor'] if expected_edges else None,
        'startCursor': expected_edges[0]['cursor'] if expected_edges else None
    }

    assert not result.errors
    assert result.data == {
        'letters': {
            'edges': expected_edges,
            'pageInfo': expected_page_info
        }
    }

def test_returns_all_elements_without_filters():
    check('', 'ABCDE')

def test_respects_a_smaller_first():
    check('first: 2', 'AB', has_next_page=True)

def test_respects_an_overly_large_first():
    check('first: 10', 'ABCDE')

def test_respects_a_smaller_last():
    check('last: 2', 'DE', has_previous_page=True)

def test_respects_an_overly_large_last():
    check('last: 10', 'ABCDE')

def test_respects_first_and_after():
    check('first: 2, after: "{}"'.format(cursor_for('B')), 'CD', has_next_page=True)

def test_respects_first_and_after_with_long_first():
    check('first: 10, after: "{}"'.format(cursor_for('B')), 'CDE')

def test_respects_last_and_before():
    check('last: 2, before: "{}"'.format(cursor_for('D')), 'BC', has_previous_page=True)

def test_respects_last_and_before_with_long_last():
    check('last: 10, before: "{}"'.format(cursor_for('D')), 'ABC')

def test_respects_first_and_after_and_before_too_few():
    check('first: 2, after: "{}", before: "{}"'.format(cursor_for('A'), cursor_for('E')), 'BC', has_next_page=True)

def test_respects_first_and_after_and_before_too_many():
    check('first: 4, after: "{}", before: "{}"'.format(cursor_for('A'), cursor_for('E')), 'BCD')

def test_respects_first_and_after_and_before_exactly_right():
    check('first: 3, after: "{}", before: "{}"'.format(cursor_for('A'), cursor_for('E')), "BCD")

def test_respects_last_and_after_and_before_too_few():
    check('last: 2, after: "{}", before: "{}"'.format(cursor_for('A'), cursor_for('E')), 'CD', has_previous_page=True)

def test_respects_last_and_after_and_before_too_many():
    check('last: 4, after: "{}", before: "{}"'.format(cursor_for('A'), cursor_for('E')), 'BCD')

def test_respects_last_and_after_and_before_exactly_right():
    check('last: 3, after: "{}", before: "{}"'.format(cursor_for('A'), cursor_for('E')), 'BCD')

def test_returns_no_elements_if_first_is_0():
    check('first: 0', '', has_next_page=True)

def test_returns_all_elements_if_cursors_are_invalid():
    check('before: "invalid" after: "invalid"', 'ABCDE')

def test_returns_all_elements_if_cursors_are_on_the_outside():
    check(
        'before: "{}" after: "{}"'.format(
            base64(
                'arrayconnection:%s' % 6),
            base64(
                'arrayconnection:%s' % -1)),
        'ABCDE')

def test_returns_no_elements_if_cursors_cross():
    check('before: "{}" after: "{}"'.format(base64('arrayconnection:%s' % 2), base64('arrayconnection:%s' % 4)), '')

def test_connection_type_nodes():
    result = schema.execute('''
    {
        connectionLetters {
            edges {
                node {
                    id
                    letter
                }
                cursor
            }
            pageInfo {
                hasPreviousPage
                hasNextPage
            }
        }
    }
    ''')

    assert not result.errors
    assert result.data == {
        'connectionLetters': {
            'edges': [{
                'node': {
                    'id': 'TGV0dGVyOjA=',
                    'letter': 'A',
                },
                'cursor': 'a-cursor',
            }],
            'pageInfo': {
                'hasPreviousPage': False,
                'hasNextPage': True,
            }
        }
    }

def test_connection_promise():
    result = schema.execute('''
    {
        promiseLetters(first:1) {
            edges {
                node {
                    id
                    letter
                }
            }
            pageInfo {
                hasPreviousPage
                hasNextPage
            }
        }
    }
    ''')

    assert not result.errors
    assert result.data == {
        'promiseLetters': {
            'edges': [{
                'node': {
                    'id': 'TGV0dGVyOjA=',
                    'letter': 'A',
                },
            }],
            'pageInfo': {
                'hasPreviousPage': False,
                'hasNextPage': True,
            }
        }
    }
<|endoftext|>


<commit_before>"""Defines version to be imported in the module and obtained from setup.py
"""
# This file has been copied and modified from the DataLad codebase
# https://github.com/datalad/datalad/blob/master/datalad/version.py
# www.datalad.org
# Many thanks to the DataLad developers

import sys
from os.path import lexists, dirname, join as opj, curdir

# Hard coded version, to be done by release process,
# it is also "parsed" (not imported) by setup.py, that is why assigned as
# __hardcoded_version__ later and not vise versa
#
# NOTE this should have the format of
# NEW_RELEASE.dev0
# since the __version__ will be later automatically parsed by adding the git
# commit hash
# so valid versions will be 1.2.1.dev0, 1.4.dev0, etc
__version__ = '1.2.5'
__hardcoded_version__ = __version__
__full_version__ = __version__

# NOTE: might cause problems with "python setup.py develop" deployments
#  so I have even changed buildbot to use  pip install -e .
moddir = dirname(__file__)
projdir = curdir if moddir == 'cortex' else dirname(moddir)
if lexists(opj(projdir, '.git')):
    # If under git -- attempt to deduce a better "dynamic" version following git
    try:
        from subprocess import Popen, PIPE
        # Note: Popen does not support `with` way correctly in 2.7
        #
        git = Popen(
            ['git', 'describe', '--abbrev=4', '--dirty', '--match', r'[0-9]*\.*'],
            stdout=PIPE, stderr=PIPE,
            cwd=projdir
        )
        if git.wait() != 0:
            raise OSError("Could not run git describe")
        line = git.stdout.readlines()[0]
        _ = git.stderr.readlines()
        # Just take describe and replace initial '-' with .dev to be more "pythonish"
        # Encoding simply because distutils' LooseVersion compares only StringType
        # and thus misses in __cmp__ necessary wrapping for unicode strings
        #
        # remove the version from describe and stick the current version
        # hardcoded in __version__
        dev_suffix = line.strip().decode('ascii').split('-')
        # in case we are at release, we shouldn't have any dev
        if len(dev_suffix) > 1:
            dev_suffix = dev_suffix[1:]
            dev_suffix = ".dev{}".format('-'.join(dev_suffix))
        else:
            dev_suffix = ""
        # remove dev suffix from hardcoded version
        if 'dev' in __version__:
            __version__ = '.'.join(__version__.split('.')[:-1])
        # stick the automatically generated dev version
        __full_version__ = __version__ + dev_suffix
        # To follow PEP440 we can't have all the git fanciness
        __version__ = __full_version__.split('-')[0]
    except (SyntaxError, AttributeError, IndexError):
        raise
    except:
        # just stick to the hard-coded
        pass
<commit_msg>MNT back to dev version [skip ci]<commit_after>"""Defines version to be imported in the module and obtained from setup.py
"""
# This file has been copied and modified from the DataLad codebase
# https://github.com/datalad/datalad/blob/master/datalad/version.py
# www.datalad.org
# Many thanks to the DataLad developers

import sys
from os.path import lexists, dirname, join as opj, curdir

# Hard coded version, to be done by release process,
# it is also "parsed" (not imported) by setup.py, that is why assigned as
# __hardcoded_version__ later and not vise versa
#
# NOTE this should have the format of
# NEW_RELEASE.dev0
# since the __version__ will be later automatically parsed by adding the git
# commit hash
# so valid versions will be 1.2.1.dev0, 1.4.dev0, etc
__version__ = '1.3.0.dev0'
__hardcoded_version__ = __version__
__full_version__ = __version__

# NOTE: might cause problems with "python setup.py develop" deployments
#  so I have even changed buildbot to use  pip install -e .
moddir = dirname(__file__)
projdir = curdir if moddir == 'cortex' else dirname(moddir)
if lexists(opj(projdir, '.git')):
    # If under git -- attempt to deduce a better "dynamic" version following git
    try:
        from subprocess import Popen, PIPE
        # Note: Popen does not support `with` way correctly in 2.7
        #
        git = Popen(
            ['git', 'describe', '--abbrev=4', '--dirty', '--match', r'[0-9]*\.*'],
            stdout=PIPE, stderr=PIPE,
            cwd=projdir
        )
        if git.wait() != 0:
            raise OSError("Could not run git describe")
        line = git.stdout.readlines()[0]
        _ = git.stderr.readlines()
        # Just take describe and replace initial '-' with .dev to be more "pythonish"
        # Encoding simply because distutils' LooseVersion compares only StringType
        # and thus misses in __cmp__ necessary wrapping for unicode strings
        #
        # remove the version from describe and stick the current version
        # hardcoded in __version__
        dev_suffix = line.strip().decode('ascii').split('-')
        # in case we are at release, we shouldn't have any dev
        if len(dev_suffix) > 1:
            dev_suffix = dev_suffix[1:]
            dev_suffix = ".dev{}".format('-'.join(dev_suffix))
        else:
            dev_suffix = ""
        # remove dev suffix from hardcoded version
        if 'dev' in __version__:
            __version__ = '.'.join(__version__.split('.')[:-1])
        # stick the automatically generated dev version
        __full_version__ = __version__ + dev_suffix
        # To follow PEP440 we can't have all the git fanciness
        __version__ = __full_version__.split('-')[0]
    except (SyntaxError, AttributeError, IndexError):
        raise
    except:
        # just stick to the hard-coded
        pass
<|endoftext|>


<commit_before>import os
import subprocess
import logging
import openpnm as op

rootdir = os.path.split(os.path.split(op.__file__)[0])[0]
examples_dir = os.path.join(rootdir, 'examples')
simulations_dir = os.path.join(examples_dir, 'simulations')

class ExamplesTest():

    def setup_class(self):
        pass

    def _run_shell_command(self, command_line_args):
        try:
            proc = subprocess.run(command_line_args, timeout=600)
        except (OSError, subprocess.CalledProcessError) as exception:
            logging.info('Exception occured: ' + str(exception))
            logging.info('Subprocess failed')
            return False
        else:
            # no exception was raised
            logging.info('Subprocess finished')
        return proc.returncode == 0

    def _notebook_run(self, path):
        """Execute a notebook via nbconvert and collect output.
           :returns (parsed nb object, execution errors)
        """
        dirname, __ = os.path.split(path)
        args = ["jupyter", "nbconvert", "--to", "notebook", "--execute",
                "--ExecutePreprocessor.timeout=360",
                "--output", "temp_output.ipynb", path]
        rc = self._run_shell_command(args)
        print(path, rc)
        print('-'*30)
        if rc:
            os.remove(os.path.join(dirname, "temp_output.ipynb"))
        return rc

    def test_advection_diffusion(self):
        nbook = os.path.join(simulations_dir, 'Advection-Diffusion.ipynb')
        rc = self._notebook_run(nbook)
        assert rc

    def test_fickian_diffusion(self):
        nbook = os.path.join(simulations_dir, 'Fickian Diffusion.ipynb')
        rc = self._notebook_run(nbook)
        assert rc

    def test_transient_fickian_diffusion(self):
        nbook = os.path.join(simulations_dir, 'Transient Fickian Diffusion.ipynb')
        rc = self._notebook_run(nbook)
        assert rc

    def test_relative_diffusion(self):
        nbook = os.path.join(simulations_dir, 'Relative Diffusivity.ipynb')
        rc = self._notebook_run(nbook)
        assert rc

    def test_rel_perm_2d(self):
        nbook = os.path.join(simulations_dir, 'Relative Permeability in 2D.ipynb')
        rc = self._notebook_run(nbook)
        assert rc

if __name__ == '__main__':
    t = ExamplesTest()
    self = t
    t.setup_class()
    for item in t.__dir__():
        if item.startswith('test'):
            print('running test: '+item)
            t.__getattribute__(item)()
<commit_msg>adding tutorial tests<commit_after>import os
import subprocess
import logging
import openpnm as op

rootdir = os.path.split(os.path.split(op.__file__)[0])[0]
examples_dir = os.path.join(rootdir, 'examples')
simulations_dir = os.path.join(examples_dir, 'simulations')

class ExamplesTest():

    def setup_class(self):
        pass

    def _run_shell_command(self, command_line_args):
        try:
            proc = subprocess.run(command_line_args, timeout=600)
        except (OSError, subprocess.CalledProcessError) as exception:
            logging.info('Exception occured: ' + str(exception))
            logging.info('Subprocess failed')
            return False
        else:
            # no exception was raised
            logging.info('Subprocess finished')
        return proc.returncode == 0

    def _notebook_run(self, path):
        """Execute a notebook via nbconvert and collect output.
           :returns (parsed nb object, execution errors)
        """
        dirname, __ = os.path.split(path)
        args = ["jupyter", "nbconvert", "--to", "notebook", "--execute",
                "--ExecutePreprocessor.timeout=360",
                "--output", "temp_output.ipynb", path]
        rc = self._run_shell_command(args)
        print(path, rc)
        print('-'*30)
        if rc:
            os.remove(os.path.join(dirname, "temp_output.ipynb"))
        return rc

    def test_tutorial_01(self):
        nbook = os.path.join(simulations_dir, '01 - Intro to OpenPNM - Basics.ipynb')
        rc = self._notebook_run(nbook)
        assert rc

    def test_tutorial_02(self):
        nbook = os.path.join(simulations_dir, '02 - Intro to OpenPNM - Intermediate.ipynb')
        rc = self._notebook_run(nbook)
        assert rc

    def test_tutorial_03(self):
        nbook = os.path.join(simulations_dir, '03 - Intro to OpenPNM - Advanced.ipynb')
        rc = self._notebook_run(nbook)
        assert rc

    def test_tutorial_workspaces_and_projects(self):
        nbook = os.path.join(simulations_dir, 'Using Workspace and Projects.ipynb')
        rc = self._notebook_run(nbook)
        assert rc

if __name__ == '__main__':
    t = ExamplesTest()
    self = t
    t.setup_class()
    for item in t.__dir__():
        if item.startswith('test'):
            print('running test: '+item)
            t.__getattribute__(item)()
<|endoftext|>