<commit_before>"""
switchboard.tests.test_models
~~~~~~~~~~~~~~~

:copyright: (c) 2012 SourceForge.
:license: Apache License 2.0, see LICENSE for more details.
"""

from datetime import datetime

from nose.tools import assert_equals, assert_true, assert_false
from mock import Mock, patch

from ..manager import SwitchManager
from ..models import MongoModel, VersioningMongoModel, Switch

class TestMongoModel(object):
    def setup(self):
        self.m = MongoModel()

    def teardown(self):
        MongoModel.c.drop()

    @patch('switchboard.models.MongoModel.update')
    def test_get_or_create_get(self, update):
        self.m.create(key=0, foo='bar')
        defaults = dict(foo='bar')
        instance, created = self.m.get_or_create(defaults=defaults, key=0)
        assert_false(created)
        assert_false(update.called)
        assert_equals(instance.foo, 'bar')

class TestVersioningMongoModel(object):
    def setup(self):
        self.m = VersioningMongoModel(_id='0')

    def teardown(self):
        VersioningMongoModel._versioned_collection().drop()

    def test_diff_fields_added(self):
        self.m.previous_version = lambda: VersioningMongoModel(a=1, b=2)
        self.m.c.find_one = lambda x: dict(a=1, b=2, c=3)
        delta = self.m._diff()
        assert_equals(delta['added'], dict(c=3))

    def test_diff_fields_deleted(self):
        self.m.previous_version = lambda: VersioningMongoModel(a=1, b=2)
        self.m.c.find_one = lambda x: dict(a=1)
        delta = self.m._diff()
        assert_equals(delta['deleted'], dict(b=2))

    def test_diff_fields_changed(self):
        self.m.previous_version = lambda: VersioningMongoModel(a=1, b=2)
        self.m.c.find_one = lambda x: dict(a=1, b=3)
        delta = self.m._diff()
        assert_equals(delta['changed'], dict(b=(2, 3)))

    def test_diff_fields_same(self):
        self.m.previous_version = lambda: VersioningMongoModel(a=1, b=2)
        self.m.c.find_one = lambda x: dict(a=1, b=2)
        delta = self.m._diff()
        assert_equals(delta['changed'], dict())
        assert_equals(delta['added'], dict())
        assert_equals(delta['deleted'], dict())

    def test_diff_created(self):
        self.m.previous_version = lambda: None
        self.m.c.find_one = lambda x: dict(a=1, b=2)
        delta = self.m._diff()
        assert_equals(delta['changed'], dict())
        assert_equals(delta['added'], dict(a=1, b=2))
        assert_equals(delta['deleted'], dict())

    def test_diff_removed(self):
        self.m.previous_version = lambda: VersioningMongoModel(a=1, b=2)
        self.m.c.find_one = lambda x: None
        delta = self.m._diff()
        assert_equals(delta['changed'], dict())
        assert_equals(delta['added'], dict())
        assert_equals(delta['deleted'], dict(a=1, b=2))

    def test_diff_noop(self):
        self.m.previous_version = lambda: None
        self.m.c.find_one = lambda x: None
        delta = self.m._diff()
        assert_equals(delta, dict(added={}, deleted={}, changed={}))

    def test_previous_version_new(self):
        c = Mock()
        c.find.return_value = None
        self.m._versioned_collection = lambda: c
        prev = self.m.previous_version()
        assert_false(hasattr(prev, '_id'))

    def test_previous_version_singlediff(self):
        delta = dict(
            added=dict(a=1, b=2)
        )
        c = Mock()
        c.find.return_value = [dict(timestamp=datetime.utcnow(),
                                    delta=delta)]
        self.m._versioned_collection = lambda: c
        prev = self.m.previous_version()
        assert_equals(prev.a, 1)
        assert_equals(prev.b, 2)

    def test_previous_version_multidiff(self):
        v1 = dict(
            timestamp=datetime.utcnow(),
            delta=dict(added=dict(a=1, b=2))
        )
        v2 = dict(
            timestamp=datetime.utcnow(),
            delta=dict(changed=dict(b=(2, 3)))
        )
        v3 = dict(
            timestamp=datetime.utcnow(),
            delta=dict(added=dict(c=4))
        )
        v4 = dict(
            timestamp=datetime.utcnow(),
            delta=dict(deleted=dict(a=1))
        )
        c = Mock()
        c.find.return_value = [v1, v2, v3, v4]
        self.m._versioned_collection = lambda: c
        prev = self.m.previous_version()
        assert_equals(prev.b, 3)
        assert_equals(prev.c, 4)
        assert_false(hasattr(prev, 'a'))

class TestConstant(object):
    def setup(self):
        self.operator = SwitchManager()

    def test_disabled(self):
        assert_true(hasattr(self.operator, 'DISABLED'))
        assert_equals(self.operator.DISABLED, 1)

    def test_selective(self):
        assert_true(hasattr(self.operator, 'SELECTIVE'))
        assert_equals(self.operator.SELECTIVE, 2)

    def test_global(self):
        assert_true(hasattr(self.operator, 'GLOBAL'))
        assert_equals(self.operator.GLOBAL, 3)

    def test_include(self):
        assert_true(hasattr(self.operator, 'INCLUDE'))
        assert_equals(self.operator.INCLUDE, 'i')

    def test_exclude(self):
        assert_true(hasattr(self.operator, 'EXCLUDE'))
        assert_equals(self.operator.EXCLUDE, 'e')

class TestSwitch(object):
    def setup(self):
        self.switch = Switch.create(key='test')

    def teardown(self):
        Switch.c.drop()
        Switch._versioned_collection().drop()

    def test_save_version_changed(self):
        self.switch.key = 'test2'
        self.switch.save()
        _id = self.switch._id
        assert_equals(self.switch.to_bson(),
                      self.switch.c.find_one(dict(_id=_id)))
        vc = self.switch._versioned_collection()
        versions = list(vc.find(dict(switch_id=_id)))
        assert_true(versions)
        versions.sort(key=lambda x:x['timestamp'])
        version = versions[-1]
        assert_true(version)
        assert_equals(version['delta']['changed']['key'], ('test', 'test2'))
<commit_msg>Test the other logic branch in get_or_create.<commit_after>"""
switchboard.tests.test_models
~~~~~~~~~~~~~~~

:copyright: (c) 2012 SourceForge.
:license: Apache License 2.0, see LICENSE for more details.
"""

from datetime import datetime

from nose.tools import assert_equals, assert_true, assert_false
from mock import Mock, patch

from ..manager import SwitchManager
from ..models import MongoModel, VersioningMongoModel, Switch

class TestMongoModel(object):
    def setup(self):
        self.m = MongoModel()

    def teardown(self):
        MongoModel.c.drop()

    @patch('switchboard.models.MongoModel.update')
    def test_get_or_create_get(self, update):
        self.m.create(key=0, foo='bar')
        defaults = dict(foo='bar')
        instance, created = self.m.get_or_create(defaults=defaults, key=0)
        assert_false(created)
        assert_false(update.called)
        assert_equals(instance.foo, 'bar')

    @patch('switchboard.models.MongoModel.update')
    @patch('switchboard.helpers.MockCollection.find_one')
    def test_get_or_create_create(self, find_one, update):
        find_one.side_effect = [None, dict(foo='bar', key=0)]
        defaults = dict(foo='bar')
        instance, created = self.m.get_or_create(defaults=defaults, key=0)
        assert_true(created)
        assert_true(update.called)
        assert_equals(instance.foo, 'bar')

class TestVersioningMongoModel(object):
    def setup(self):
        self.m = VersioningMongoModel(_id='0')

    def teardown(self):
        VersioningMongoModel._versioned_collection().drop()

    def test_diff_fields_added(self):
        self.m.previous_version = lambda: VersioningMongoModel(a=1, b=2)
        self.m.c.find_one = lambda x: dict(a=1, b=2, c=3)
        delta = self.m._diff()
        assert_equals(delta['added'], dict(c=3))

    def test_diff_fields_deleted(self):
        self.m.previous_version = lambda: VersioningMongoModel(a=1, b=2)
        self.m.c.find_one = lambda x: dict(a=1)
        delta = self.m._diff()
        assert_equals(delta['deleted'], dict(b=2))

    def test_diff_fields_changed(self):
        self.m.previous_version = lambda: VersioningMongoModel(a=1, b=2)
        self.m.c.find_one = lambda x: dict(a=1, b=3)
        delta = self.m._diff()
        assert_equals(delta['changed'], dict(b=(2, 3)))

    def test_diff_fields_same(self):
        self.m.previous_version = lambda: VersioningMongoModel(a=1, b=2)
        self.m.c.find_one = lambda x: dict(a=1, b=2)
        delta = self.m._diff()
        assert_equals(delta['changed'], dict())
        assert_equals(delta['added'], dict())
        assert_equals(delta['deleted'], dict())

    def test_diff_created(self):
        self.m.previous_version = lambda: None
        self.m.c.find_one = lambda x: dict(a=1, b=2)
        delta = self.m._diff()
        assert_equals(delta['changed'], dict())
        assert_equals(delta['added'], dict(a=1, b=2))
        assert_equals(delta['deleted'], dict())

    def test_diff_removed(self):
        self.m.previous_version = lambda: VersioningMongoModel(a=1, b=2)
        self.m.c.find_one = lambda x: None
        delta = self.m._diff()
        assert_equals(delta['changed'], dict())
        assert_equals(delta['added'], dict())
        assert_equals(delta['deleted'], dict(a=1, b=2))

    def test_diff_noop(self):
        self.m.previous_version = lambda: None
        self.m.c.find_one = lambda x: None
        delta = self.m._diff()
        assert_equals(delta, dict(added={}, deleted={}, changed={}))

    def test_previous_version_new(self):
        c = Mock()
        c.find.return_value = None
        self.m._versioned_collection = lambda: c
        prev = self.m.previous_version()
        assert_false(hasattr(prev, '_id'))

    def test_previous_version_singlediff(self):
        delta = dict(
            added=dict(a=1, b=2)
        )
        c = Mock()
        c.find.return_value = [dict(timestamp=datetime.utcnow(),
                                    delta=delta)]
        self.m._versioned_collection = lambda: c
        prev = self.m.previous_version()
        assert_equals(prev.a, 1)
        assert_equals(prev.b, 2)

    def test_previous_version_multidiff(self):
        v1 = dict(
            timestamp=datetime.utcnow(),
            delta=dict(added=dict(a=1, b=2))
        )
        v2 = dict(
            timestamp=datetime.utcnow(),
            delta=dict(changed=dict(b=(2, 3)))
        )
        v3 = dict(
            timestamp=datetime.utcnow(),
            delta=dict(added=dict(c=4))
        )
        v4 = dict(
            timestamp=datetime.utcnow(),
            delta=dict(deleted=dict(a=1))
        )
        c = Mock()
        c.find.return_value = [v1, v2, v3, v4]
        self.m._versioned_collection = lambda: c
        prev = self.m.previous_version()
        assert_equals(prev.b, 3)
        assert_equals(prev.c, 4)
        assert_false(hasattr(prev, 'a'))

class TestConstant(object):
    def setup(self):
        self.operator = SwitchManager()

    def test_disabled(self):
        assert_true(hasattr(self.operator, 'DISABLED'))
        assert_equals(self.operator.DISABLED, 1)

    def test_selective(self):
        assert_true(hasattr(self.operator, 'SELECTIVE'))
        assert_equals(self.operator.SELECTIVE, 2)

    def test_global(self):
        assert_true(hasattr(self.operator, 'GLOBAL'))
        assert_equals(self.operator.GLOBAL, 3)

    def test_include(self):
        assert_true(hasattr(self.operator, 'INCLUDE'))
        assert_equals(self.operator.INCLUDE, 'i')

    def test_exclude(self):
        assert_true(hasattr(self.operator, 'EXCLUDE'))
        assert_equals(self.operator.EXCLUDE, 'e')

class TestSwitch(object):
    def setup(self):
        self.switch = Switch.create(key='test')

    def teardown(self):
        Switch.c.drop()
        Switch._versioned_collection().drop()

    def test_save_version_changed(self):
        self.switch.key = 'test2'
        self.switch.save()
        _id = self.switch._id
        assert_equals(self.switch.to_bson(),
                      self.switch.c.find_one(dict(_id=_id)))
        vc = self.switch._versioned_collection()
        versions = list(vc.find(dict(switch_id=_id)))
        assert_true(versions)
        versions.sort(key=lambda x:x['timestamp'])
        version = versions[-1]
        assert_true(version)
        assert_equals(version['delta']['changed']['key'], ('test', 'test2'))
<|endoftext|>


<commit_before># Copyright (c) 2017, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt
import unittest, os, base64
from frappe import safe_decode
from frappe.email.receive import Email
from frappe.email.email_body import (replace_filename_with_cid,
					get_email, inline_style_in_html, get_header)
from frappe.email.doctype.email_queue.email_queue import SendMailContext, QueueBuilder

class TestEmailBody(unittest.TestCase):
	def setUp(self):
		email_html = '''

Hey John Doe!

This is embedded image you asked for

'''
		email_text = '''
Hey John Doe!
This is the text version of this email
'''

		img_path = os.path.abspath('assets/frappe/images/frappe-favicon.svg')
		with open(img_path, 'rb') as f:
			img_content = f.read()
			img_base64 = base64.b64encode(img_content).decode()

		# email body keeps 76 characters on one line
		self.img_base64 = fixed_column_width(img_base64, 76)

		self.email_string = get_email(
			recipients=['test@example.com'],
			sender='me@example.com',
			subject='Test Subject',
			content=email_html,
			text_content=email_text
		).as_string().replace("\r\n", "\n")

	def test_prepare_message_returns_already_encoded_string(self):
		uni_chr1 = chr(40960)
		uni_chr2 = chr(1972)

		queue_doc = QueueBuilder(
			recipients=['test@example.com'],
			sender='me@example.com',
			subject='Test Subject',
			message='
' + uni_chr1 + 'abcd' + uni_chr2 + '
',
			text_content='whatever').process()[0]
		mail_ctx = SendMailContext(queue_doc = queue_doc)
		result = mail_ctx.build_message(recipient_email = 'test@test.com')
		self.assertTrue(b"
=EA=80=80abcd=DE=B4
" in result)

	def test_prepare_message_returns_cr_lf(self):
		queue_doc = QueueBuilder(
			recipients=['test@example.com'],
			sender='me@example.com',
			subject='Test Subject',
			message='
\n this is a test of newlines\n' + '
',
			text_content='whatever').process()[0]

		mail_ctx = SendMailContext(queue_doc = queue_doc)
		result = safe_decode(mail_ctx.build_message(recipient_email='test@test.com'))

		self.assertTrue(result.count('\n') == result.count("\r"))

	def test_image(self):
		img_signature = '''
Content-Type: image/svg+xml
MIME-Version: 1.0
Content-Transfer-Encoding: base64
Content-Disposition: inline; filename="frappe-favicon.svg"
'''
		self.assertTrue(img_signature in self.email_string)
		self.assertTrue(self.img_base64 in self.email_string)

	def test_text_content(self):
		text_content = '''
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: quoted-printable

Hey John Doe!
This is the text version of this email
'''
		self.assertTrue(text_content in self.email_string)

	def test_email_content(self):
		html_head = '''
Content-Type: text/html; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: quoted-printable

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.=
w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

'''

		html = '''
Hey John Doe!
'''

		self.assertTrue(html_head in self.email_string)
		self.assertTrue(html in self.email_string)

	def test_replace_filename_with_cid(self):
		original_message = '''

		'''
		message, inline_images = replace_filename_with_cid(original_message)

		processed_message = '''

		'''.format(inline_images[0].get('content_id'))
		self.assertEqual(message, processed_message)

	def test_inline_styling(self):
		html = '''

Hi John

This is a test email

'''
		transformed_html = '''

Hi John

This is a test email

'''
		self.assertTrue(transformed_html in inline_style_in_html(html))

	def test_email_header(self):
		email_html = '''

Hey John Doe!

This is embedded image you asked for

'''
		email_string = get_email(
			recipients=['test@example.com'],
			sender='me@example.com',
			subject='Test Subject',
			content=email_html,
			header=['Email Title', 'green']
		).as_string().replace("\r\n", "\n")
		# REDESIGN-TODO: Add style for indicators in email
		self.assertTrue('''''' in email_string)
		self.assertTrue('Email Title' in email_string)

	def test_get_email_header(self):
		html = get_header(['This is test', 'orange'])
		self.assertTrue('' in html)
		self.assertTrue('This is test' in html)

		html = get_header(['This is another test'])
		self.assertTrue('This is another test' in html)

		html = get_header('This is string')
		self.assertTrue('This is string' in html)

	def test_8bit_utf_8_decoding(self):
		text_content_bytes = b"\xed\x95\x9c\xea\xb8\x80\xe1\xa5\xa1\xe2\x95\xa5\xe0\xba\xaa\xe0\xa4\x8f"
		text_content = text_content_bytes.decode('utf-8')

		content_bytes = b"""MIME-Version: 1.0
Content-Type: text/plain; charset=utf-8
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
From: test1_@erpnext.com
Reply-To: test2_@erpnext.com
""" + text_content_bytes

		mail = Email(content_bytes)
		self.assertEqual(mail.text_content, text_content)

def fixed_column_width(string, chunk_size):
	parts = [string[0 + i:chunk_size + i] for i in range(0, len(string), chunk_size)]
	return '\n'.join(parts)
<commit_msg>test: Update test<commit_after># Copyright (c) 2017, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt
import unittest, os, base64
from frappe import safe_decode
from frappe.email.receive import Email
from frappe.email.email_body import (replace_filename_with_cid,
					get_email, inline_style_in_html, get_header)
from frappe.email.doctype.email_queue.email_queue import SendMailContext, QueueBuilder

class TestEmailBody(unittest.TestCase):
	def setUp(self):
		email_html = '''

Hey John Doe!

This is embedded image you asked for

'''
		email_text = '''
Hey John Doe!
This is the text version of this email
'''

		img_path = os.path.abspath('assets/frappe/images/frappe-favicon.svg')
		with open(img_path, 'rb') as f:
			img_content = f.read()
			img_base64 = base64.b64encode(img_content).decode()

		# email body keeps 76 characters on one line
		self.img_base64 = fixed_column_width(img_base64, 76)

		self.email_string = get_email(
			recipients=['test@example.com'],
			sender='me@example.com',
			subject='Test Subject',
			content=email_html,
			text_content=email_text
		).as_string().replace("\r\n", "\n")

	def test_prepare_message_returns_already_encoded_string(self):
		uni_chr1 = chr(40960)
		uni_chr2 = chr(1972)

		queue_doc = QueueBuilder(
			recipients=['test@example.com'],
			sender='me@example.com',
			subject='Test Subject',
			message='
' + uni_chr1 + 'abcd' + uni_chr2 + '
',
			text_content='whatever').process()[0]
		mail_ctx = SendMailContext(queue_doc = queue_doc)
		result = mail_ctx.build_message(recipient_email = 'test@test.com')
		self.assertTrue(b"
=EA=80=80abcd=DE=B4
" in result)

	def test_prepare_message_returns_cr_lf(self):
		queue_doc = QueueBuilder(
			recipients=['test@example.com'],
			sender='me@example.com',
			subject='Test Subject',
			message='
\n this is a test of newlines\n' + '
',
			text_content='whatever').process()[0]

		mail_ctx = SendMailContext(queue_doc = queue_doc)
		result = safe_decode(mail_ctx.build_message(recipient_email='test@test.com'))

		self.assertTrue(result.count('\n') == result.count("\r"))

	def test_image(self):
		img_signature = '''
Content-Type: image/svg+xml
MIME-Version: 1.0
Content-Transfer-Encoding: base64
Content-Disposition: inline; filename="frappe-favicon.svg"
'''
		self.assertTrue(img_signature in self.email_string)
		self.assertTrue(self.img_base64 in self.email_string)

	def test_text_content(self):
		text_content = '''
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: quoted-printable

Hey John Doe!
This is the text version of this email
'''
		self.assertTrue(text_content in self.email_string)

	def test_email_content(self):
		html_head = '''
Content-Type: text/html; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: quoted-printable

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.=
w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

'''

		html = '''
Hey John Doe!
'''

		self.assertTrue(html_head in self.email_string)
		self.assertTrue(html in self.email_string)

	def test_replace_filename_with_cid(self):
		original_message = '''

		'''
		message, inline_images = replace_filename_with_cid(original_message)

		processed_message = '''

		'''.format(inline_images[0].get('content_id'))
		self.assertEqual(message, processed_message)

	def test_inline_styling(self):
		html = '''

Hi John

This is a test email

'''
		transformed_html = '''

Hi John

This is a test email

'''
		self.assertTrue(transformed_html in inline_style_in_html(html))

	def test_email_header(self):
		email_html = '''

Hey John Doe!

This is embedded image you asked for

'''
		email_string = get_email(
			recipients=['test@example.com'],
			sender='me@example.com',
			subject='Test Subject',
			content=email_html,
			header=['Email Title', 'green']
		).as_string().replace("\r\n", "\n")
		# REDESIGN-TODO: Add style for indicators in email
		self.assertTrue('''''' in email_string)
		self.assertTrue('Email Title' in email_string)

	def test_get_email_header(self):
		html = get_header(['This is test', 'orange'])
		self.assertTrue('' in html)
		self.assertTrue('This is test' in html)

		html = get_header(['This is another test'])
		self.assertTrue('This is another test' in html)

		html = get_header('This is string')
		self.assertTrue('This is string' in html)

	def test_8bit_utf_8_decoding(self):
		text_content_bytes = b"\xed\x95\x9c\xea\xb8\x80\xe1\xa5\xa1\xe2\x95\xa5\xe0\xba\xaa\xe0\xa4\x8f"
		text_content = text_content_bytes.decode('utf-8')

		content_bytes = b"""MIME-Version: 1.0
Content-Type: text/plain; charset=utf-8
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
From: test1_@erpnext.com
Reply-To: test2_@erpnext.com
""" + text_content_bytes

		mail = Email(content_bytes)
		self.assertEqual(mail.text_content, text_content)

def fixed_column_width(string, chunk_size):
	parts = [string[0 + i:chunk_size + i] for i in range(0, len(string), chunk_size)]
	return '\n'.join(parts)
<|endoftext|>


<commit_before># Author: Leland McInnes <leland.mcinnes@gmail.com>
#
# License: BSD 3 clause
import numpy as np
import numba

_mock_identity = np.eye(2, dtype=np.float64)
_mock_ones = np.ones(2, dtype=np.float64)

@numba.njit(fastmath=True)
def euclidean(x, y):
    """Standard euclidean distance.

    ..math::
        D(x, y) = \sqrt{\sum_i (x_i - y_i)^2}
    """
    result = 0.0
    for i in range(x.shape[0]):
        result += (x[i] - y[i]) ** 2
    return np.sqrt(result)

@numba.njit()
def standardised_euclidean(x, y, sigma=_mock_ones):
    """Euclidean distance standardised against a vector of standard
    deviations per coordinate.

    ..math::
        D(x, y) = \sqrt{\sum_i \frac{(x_i - y_i)**2}{v_i}}
    """
    result = 0.0
    for i in range(x.shape[0]):
        result += ((x[i] - y[i]) ** 2) / sigma[i]

    return np.sqrt(result)

@numba.njit()
def manhattan(x, y):
    """Manhatten, taxicab, or l1 distance.

    ..math::
        D(x, y) = \sum_i |x_i - y_i|
    """
    result = 0.0
    for i in range(x.shape[0]):
        result += np.abs(x[i] - y[i])

    return result

@numba.njit()
def chebyshev(x, y):
    """Chebyshev or l-infinity distance.

    ..math::
        D(x, y) = \max_i |x_i - y_i|
    """
    result = 0.0
    for i in range(x.shape[0]):
        result = max(result, np.abs(x[i] - y[i]))

    return result

@numba.njit()
def minkowski(x, y, p=2):
    """Minkowski distance.

    ..math::
        D(x, y) = \left(\sum_i |x_i - y_i|^p\right)^{\frac{1}{p}}

    This is a general distance. For p=1 it is equivalent to
    manhattan distance, for p=2 it is Euclidean distance, and
    for p=infinity it is Chebyshev distance. In general it is better
    to use the more specialised functions for those distances.
    """
    result = 0.0
    for i in range(x.shape[0]):
        result += (np.abs(x[i] - y[i])) ** p

    return result ** (1.0 / p)

@numba.njit()
def weighted_minkowski(x, y, w=_mock_identity, p=2):
    """A weighted version of Minkowski distance.

    ..math::
        D(x, y) = \left(\sum_i w_i |x_i - y_i|^p\right)^{\frac{1}{p}}

    If weights w_i are inverse standard deviations of data in each dimension
    then this represented a standardised Minkowski distance (and is
    equivalent to standardised Euclidean distance for p=1).
    """
    result = 0.0
    for i in range(x.shape[0]):
        result += (w[i] * np.abs(x[i] - y[i])) ** p

    return result ** (1.0 / p)

@numba.njit()
def mahalanobis(x, y, vinv=_mock_identity):
    result = 0.0

    diff = np.empty(x.shape[0], dtype=np.float64)

    for i in range(x.shape[0]):
        diff[i] = x[i] - y[i]

    for i in range(x.shape[0]):
        tmp = 0.0
        for j in range(x.shape[0]):
            tmp += vinv[i, j] * diff[j]
        result += tmp * diff[i]

    return np.sqrt(result)

@numba.njit()
def hamming(x, y):
    result = 0.0
    for i in range(x.shape[0]):
        if x[i] != y[i]:
            result += 1.0

    return float(result) / x.shape[0]

@numba.njit()
def canberra(x, y):
    result = 0.0
    for i in range(x.shape[0]):
        denominator = np.abs(x[i]) + np.abs(y[i])
        if denominator > 0:
            result += np.abs(x[i] - y[i]) / denominator

    return result

@numba.njit()
def bray_curtis(x, y):
    numerator = 0.0
    denominator = 0.0
    for i in range(x.shape[0]):
        numerator += np.abs(x[i] - y[i])
        denominator += np.abs(x[i] + y[i])

    if denominator > 0.0:
        return float(numerator) / denominator
    else:
        return 0.0

@numba.njit()
def jaccard(x, y):
    num_non_zero = 0.0
    num_equal = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_non_zero += (x_true or y_true)
        num_equal += (x_true and y_true)

    if num_non_zero == 0.0:
        return 0.0
    else:
        return float(num_non_zero - num_equal) / num_non_zero

@numba.njit()
def matching(x, y):
    num_not_equal = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_not_equal += (x_true != y_true)

    return float(num_not_equal) / x.shape[0]

@numba.njit()
def dice(x, y):
    num_true_true = 0.0
    num_not_equal = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_true_true += (x_true and y_true)
        num_not_equal += (x_true != y_true)

    return num_not_equal / (2.0 * num_true_true + num_not_equal)

@numba.njit()
def kulsinski(x, y):
    num_true_true = 0.0
    num_not_equal = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_true_true += (x_true and y_true)
        num_not_equal += (x_true != y_true)

    if num_not_equal == 0:
        return 0.0
    else:
        return float(num_not_equal - num_true_true + x.shape[0]) / \
                (num_not_equal + x.shape[0])

@numba.njit()
def rogers_tanimoto(x, y):
    num_not_equal = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_not_equal += (x_true != y_true)

    return (2.0 * num_not_equal) / (x.shape[0] + num_not_equal)

@numba.njit()
def russellrao(x, y):
    num_true_true = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_true_true += (x_true and y_true)

    if (num_true_true == np.sum(x != 0) and
        num_true_true == np.sum(y != 0)):
        return 0.0
    else:
        return float(x.shape[0] - num_true_true) / (x.shape[0])

@numba.njit()
def sokal_michener(x, y):
    num_not_equal = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_not_equal += (x_true != y_true)

    return (2.0 * num_not_equal) / (x.shape[0] + num_not_equal)

@numba.njit()
def sokal_sneath(x, y):
    num_true_true = 0.0
    num_not_equal = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_true_true += (x_true and y_true)
        num_not_equal += (x_true != y_true)

    return num_not_equal / (0.5 * num_true_true + num_not_equal)

@numba.njit()
def haversine(x, y):
    if x.shape[0] != 2:
        raise ValueError('haversine is only defined for 2 dimensional data')
    sin_lat = np.sin(0.5 * (x[0] - y[0]))
    sin_long = np.sin(0.5 * (x[1] - y[1]))
    result = np.sqrt(sin_lat ** 2 + np.cos(x[0]) * np.cos(y[0]) * sin_long ** 2)
    return 2.0 * np.arcsin(result)

@numba.njit()
def yule(x, y):
    num_true_true = 0.0
    num_true_false = 0.0
    num_false_true = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_true_true += (x_true and y_true)
        num_true_false += (x_true and (not y_true))
        num_false_true += ((not x_true) and y_true)

    num_false_false = x.shape[0] - num_true_true - num_true_false - \
                      num_false_true

    return (2.0 * num_true_false * num_false_true) / \
           (num_true_true * num_false_false + num_true_false * num_false_true)

@numba.njit()
def cosine(x, y):
    result = 0.0
    norm_x = 0.0
    norm_y = 0.0
    for i in range(x.shape[0]):
        result += x[i] * y[i]
        norm_x += x[i]**2
        norm_y += y[i]**2

    if norm_x == 0.0 or norm_y == 0.0:
        return 1.0
    else:
        return 1.0 - (result / np.sqrt(norm_x * norm_y))

@numba.njit()
def correlation(x, y):
    mu_x = 0.0
    mu_y = 0.0
    norm_x = 0.0
    norm_y = 0.0
    dot_product = 0.0

    for i in range(x.shape[0]):
        mu_x += x[i]
        mu_y += y[i]

    mu_x /= x.shape[0]
    mu_y /= x.shape[0]

    for i in range(x.shape[0]):
        shifted_x = x[i] - mu_x
        shifted_y = y[i] - mu_y
        norm_x += shifted_x ** 2
        norm_y += shifted_y ** 2
        dot_product += shifted_x * shifted_y

    if dot_product == 0.0:
        return 1.0
    else:
        return (1.0 - (dot_product / np.sqrt(norm_x * norm_y)))

named_distances = {
    # general minkowski distances
    'euclidean': euclidean,
    'l2': euclidean,
    'manhattan': manhattan,
    'taxicab': manhattan,
    'l1': manhattan,
    'chebyshev': chebyshev,
    'linfinity': chebyshev,
    'linfty': chebyshev,
    'linf': chebyshev,
    'minkowski': minkowski,
    # Standardised/weighted distances
    'seuclidean': standardised_euclidean,
    'standardised_euclidean': standardised_euclidean,
    'wminkowski': weighted_minkowski,
    'weighted_minkowski': weighted_minkowski,
    'mahalanobis': mahalanobis,
    # Other distances
    'canberra': canberra,
    'cosine': cosine,
    'correlation': correlation,
    'haversine': haversine,
    'braycurtis': bray_curtis,
    # Binary distances
    'hamming': hamming,
    'jaccard': jaccard,
    'dice': dice,
    'matching': matching,
    'kulsinski': kulsinski,
    'rogerstanimoto': rogers_tanimoto,
    'russellrao': russellrao,
    'sokalsneath': sokal_sneath,
    'sokalmichener': sokal_michener,
    'yule': yule,
}
<commit_msg>Match bray curtis to scipy -- ill defined for bad cases<commit_after># Author: Leland McInnes <leland.mcinnes@gmail.com>
#
# License: BSD 3 clause
import numpy as np
import numba

_mock_identity = np.eye(2, dtype=np.float64)
_mock_ones = np.ones(2, dtype=np.float64)

@numba.njit(fastmath=True)
def euclidean(x, y):
    """Standard euclidean distance.

    ..math::
        D(x, y) = \sqrt{\sum_i (x_i - y_i)^2}
    """
    result = 0.0
    for i in range(x.shape[0]):
        result += (x[i] - y[i]) ** 2
    return np.sqrt(result)

@numba.njit()
def standardised_euclidean(x, y, sigma=_mock_ones):
    """Euclidean distance standardised against a vector of standard
    deviations per coordinate.

    ..math::
        D(x, y) = \sqrt{\sum_i \frac{(x_i - y_i)**2}{v_i}}
    """
    result = 0.0
    for i in range(x.shape[0]):
        result += ((x[i] - y[i]) ** 2) / sigma[i]

    return np.sqrt(result)

@numba.njit()
def manhattan(x, y):
    """Manhatten, taxicab, or l1 distance.

    ..math::
        D(x, y) = \sum_i |x_i - y_i|
    """
    result = 0.0
    for i in range(x.shape[0]):
        result += np.abs(x[i] - y[i])

    return result

@numba.njit()
def chebyshev(x, y):
    """Chebyshev or l-infinity distance.

    ..math::
        D(x, y) = \max_i |x_i - y_i|
    """
    result = 0.0
    for i in range(x.shape[0]):
        result = max(result, np.abs(x[i] - y[i]))

    return result

@numba.njit()
def minkowski(x, y, p=2):
    """Minkowski distance.

    ..math::
        D(x, y) = \left(\sum_i |x_i - y_i|^p\right)^{\frac{1}{p}}

    This is a general distance. For p=1 it is equivalent to
    manhattan distance, for p=2 it is Euclidean distance, and
    for p=infinity it is Chebyshev distance. In general it is better
    to use the more specialised functions for those distances.
    """
    result = 0.0
    for i in range(x.shape[0]):
        result += (np.abs(x[i] - y[i])) ** p

    return result ** (1.0 / p)

@numba.njit()
def weighted_minkowski(x, y, w=_mock_identity, p=2):
    """A weighted version of Minkowski distance.

    ..math::
        D(x, y) = \left(\sum_i w_i |x_i - y_i|^p\right)^{\frac{1}{p}}

    If weights w_i are inverse standard deviations of data in each dimension
    then this represented a standardised Minkowski distance (and is
    equivalent to standardised Euclidean distance for p=1).
    """
    result = 0.0
    for i in range(x.shape[0]):
        result += (w[i] * np.abs(x[i] - y[i])) ** p

    return result ** (1.0 / p)

@numba.njit()
def mahalanobis(x, y, vinv=_mock_identity):
    result = 0.0

    diff = np.empty(x.shape[0], dtype=np.float64)

    for i in range(x.shape[0]):
        diff[i] = x[i] - y[i]

    for i in range(x.shape[0]):
        tmp = 0.0
        for j in range(x.shape[0]):
            tmp += vinv[i, j] * diff[j]
        result += tmp * diff[i]

    return np.sqrt(result)

@numba.njit()
def hamming(x, y):
    result = 0.0
    for i in range(x.shape[0]):
        if x[i] != y[i]:
            result += 1.0

    return float(result) / x.shape[0]

@numba.njit()
def canberra(x, y):
    result = 0.0
    for i in range(x.shape[0]):
        denominator = np.abs(x[i]) + np.abs(y[i])
        if denominator > 0:
            result += np.abs(x[i] - y[i]) / denominator

    return result

@numba.njit()
def bray_curtis(x, y):
    numerator = 0.0
    denominator = 0.0
    for i in range(x.shape[0]):
        numerator += np.abs(x[i] - y[i])
        denominator += np.abs(x[i] + y[i])

    if denominator > 0.0:
        return float(numerator) / denominator
    else:
        return np.nan

@numba.njit()
def jaccard(x, y):
    num_non_zero = 0.0
    num_equal = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_non_zero += (x_true or y_true)
        num_equal += (x_true and y_true)

    if num_non_zero == 0.0:
        return 0.0
    else:
        return float(num_non_zero - num_equal) / num_non_zero

@numba.njit()
def matching(x, y):
    num_not_equal = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_not_equal += (x_true != y_true)

    return float(num_not_equal) / x.shape[0]

@numba.njit()
def dice(x, y):
    num_true_true = 0.0
    num_not_equal = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_true_true += (x_true and y_true)
        num_not_equal += (x_true != y_true)

    return num_not_equal / (2.0 * num_true_true + num_not_equal)

@numba.njit()
def kulsinski(x, y):
    num_true_true = 0.0
    num_not_equal = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_true_true += (x_true and y_true)
        num_not_equal += (x_true != y_true)

    if num_not_equal == 0:
        return 0.0
    else:
        return float(num_not_equal - num_true_true + x.shape[0]) / \
                (num_not_equal + x.shape[0])

@numba.njit()
def rogers_tanimoto(x, y):
    num_not_equal = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_not_equal += (x_true != y_true)

    return (2.0 * num_not_equal) / (x.shape[0] + num_not_equal)

@numba.njit()
def russellrao(x, y):
    num_true_true = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_true_true += (x_true and y_true)

    if (num_true_true == np.sum(x != 0) and
        num_true_true == np.sum(y != 0)):
        return 0.0
    else:
        return float(x.shape[0] - num_true_true) / (x.shape[0])

@numba.njit()
def sokal_michener(x, y):
    num_not_equal = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_not_equal += (x_true != y_true)

    return (2.0 * num_not_equal) / (x.shape[0] + num_not_equal)

@numba.njit()
def sokal_sneath(x, y):
    num_true_true = 0.0
    num_not_equal = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_true_true += (x_true and y_true)
        num_not_equal += (x_true != y_true)

    return num_not_equal / (0.5 * num_true_true + num_not_equal)

@numba.njit()
def haversine(x, y):
    if x.shape[0] != 2:
        raise ValueError('haversine is only defined for 2 dimensional data')
    sin_lat = np.sin(0.5 * (x[0] - y[0]))
    sin_long = np.sin(0.5 * (x[1] - y[1]))
    result = np.sqrt(sin_lat ** 2 + np.cos(x[0]) * np.cos(y[0]) * sin_long ** 2)
    return 2.0 * np.arcsin(result)

@numba.njit()
def yule(x, y):
    num_true_true = 0.0
    num_true_false = 0.0
    num_false_true = 0.0
    for i in range(x.shape[0]):
        x_true = x[i] != 0
        y_true = y[i] != 0
        num_true_true += (x_true and y_true)
        num_true_false += (x_true and (not y_true))
        num_false_true += ((not x_true) and y_true)

    num_false_false = x.shape[0] - num_true_true - num_true_false - \
                      num_false_true

    return (2.0 * num_true_false * num_false_true) / \
           (num_true_true * num_false_false + num_true_false * num_false_true)

@numba.njit()
def cosine(x, y):
    result = 0.0
    norm_x = 0.0
    norm_y = 0.0
    for i in range(x.shape[0]):
        result += x[i] * y[i]
        norm_x += x[i]**2
        norm_y += y[i]**2

    if norm_x == 0.0 or norm_y == 0.0:
        return 1.0
    else:
        return 1.0 - (result / np.sqrt(norm_x * norm_y))

@numba.njit()
def correlation(x, y):
    mu_x = 0.0
    mu_y = 0.0
    norm_x = 0.0
    norm_y = 0.0
    dot_product = 0.0

    for i in range(x.shape[0]):
        mu_x += x[i]
        mu_y += y[i]

    mu_x /= x.shape[0]
    mu_y /= x.shape[0]

    for i in range(x.shape[0]):
        shifted_x = x[i] - mu_x
        shifted_y = y[i] - mu_y
        norm_x += shifted_x ** 2
        norm_y += shifted_y ** 2
        dot_product += shifted_x * shifted_y

    if dot_product == 0.0:
        return 1.0
    else:
        return (1.0 - (dot_product / np.sqrt(norm_x * norm_y)))

named_distances = {
    # general minkowski distances
    'euclidean': euclidean,
    'l2': euclidean,
    'manhattan': manhattan,
    'taxicab': manhattan,
    'l1': manhattan,
    'chebyshev': chebyshev,
    'linfinity': chebyshev,
    'linfty': chebyshev,
    'linf': chebyshev,
    'minkowski': minkowski,
    # Standardised/weighted distances
    'seuclidean': standardised_euclidean,
    'standardised_euclidean': standardised_euclidean,
    'wminkowski': weighted_minkowski,
    'weighted_minkowski': weighted_minkowski,
    'mahalanobis': mahalanobis,
    # Other distances
    'canberra': canberra,
    'cosine': cosine,
    'correlation': correlation,
    'haversine': haversine,
    'braycurtis': bray_curtis,
    # Binary distances
    'hamming': hamming,
    'jaccard': jaccard,
    'dice': dice,
    'matching': matching,
    'kulsinski': kulsinski,
    'rogerstanimoto': rogers_tanimoto,
    'russellrao': russellrao,
    'sokalsneath': sokal_sneath,
    'sokalmichener': sokal_michener,
    'yule': yule,
}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
__version__ = '1.4.1'
__author__ = 'Nico Schlömer'
__author_email__ = 'nico.schloemer@gmail.com'
__website__ = 'https://github.com/nschloe/meshio'
<commit_msg>version bump<commit_after># -*- coding: utf-8 -*-
#
__version__ = '1.4.2'
__author__ = 'Nico Schlömer'
__author_email__ = 'nico.schloemer@gmail.com'
__website__ = 'https://github.com/nschloe/meshio'
<|endoftext|>


<commit_before>'''
Installation of Python packages using pip.
==========================================

A state module to manage system installed python packages

.. code-block:: yaml

    virtualenvwrapper:
      pip.installed
'''

# Import salt libs
from salt.exceptions import CommandExecutionError, CommandNotFoundError

def installed(name,
              pip_bin=None,
              requirements=None,
              env=None,
              bin_env=None,
              log=None,
              proxy=None,
              timeout=None,
              repo=None,
              editable=None,
              find_links=None,
              index_url=None,
              extra_index_url=None,
              no_index=False,
              mirrors=None,
              build=None,
              target=None,
              download=None,
              download_cache=None,
              source=None,
              upgrade=False,
              force_reinstall=False,
              ignore_installed=False,
              no_deps=False,
              no_install=False,
              no_download=False,
              install_options=None,
              user=None,
              cwd=None,
              __env__='base'):
    '''
    Make sure the package is installed

    name
        The name of the python package to install
    pip_bin :  None
        Deprecated, use bin_env
    env : None
        Deprecated, use bin_env
    bin_env : None
        the pip executable or virtualenv to use

    '''
    if pip_bin and not bin_env:
        bin_env = pip_bin
    elif env and not bin_env:
        bin_env = env

    ret = {'name': name, 'result': None, 'comment': '', 'changes': {}}
    try:
        pip_list = __salt__['pip.list'](name, bin_env, runas=user, cwd=cwd)
    except (CommandNotFoundError, CommandExecutionError) as err:
        ret['result'] = False
        ret['comment'] = 'Error installing \'{0}\': {1}'.format(name, err)
        return ret

    if ignore_installed == False and name.lower() in (p.lower() for p in pip_list):
        if force_reinstall == False and upgrade == False:
            ret['result'] = True
            ret['comment'] = 'Package already installed'
            return ret

    if __opts__['test']:
        ret['result'] = None
        ret['comment'] = 'Python package {0} is set to be installed'.format(
                name)
        return ret

    if repo:
        name = repo

    pip_install_call = __salt__['pip.install'](
        pkgs=name,
        requirements=requirements,
        bin_env=bin_env,
        log=log,
        proxy=proxy,
        timeout=timeout,
        editable=editable,
        find_links=find_links,
        index_url=index_url,
        extra_index_url=extra_index_url,
        no_index=no_index,
        mirrors=mirrors,
        build=build,
        target=target,
        download=download,
        download_cache=download_cache,
        source=source,
        upgrade=upgrade,
        force_reinstall=force_reinstall,
        ignore_installed=ignore_installed,
        no_deps=no_deps,
        no_install=no_install,
        no_download=no_download,
        install_options=install_options,
        runas=user,
        cwd=cwd,
        __env__=__env__
    )

    if pip_install_call and (pip_install_call['retcode'] == 0):
        ret['result'] = True

        pkg_list = __salt__['pip.list'](name, bin_env, runas=user, cwd=cwd)
        if not pkg_list:
            ret['comment'] = (
                'There was no error installing package \'{0}\' although '
                'it does not show when calling \'pip.freeze\'.'.format(name)
            )
            ret['changes']["{0}==???".format(name)] = 'Installed'
            return ret

        version = list(pkg_list.values())[0]
        pkg_name = next(iter(pkg_list))
        ret['changes']["{0}=={1}".format(pkg_name, version)] = 'Installed'
        ret['comment'] = 'Package was successfully installed'
    elif pip_install_call:
        ret['result'] = False
        ret['comment'] = ('Failed to install package {0}. '
                          'Error: {1} {2}').format(name,
                                                   pip_install_call['stdout'],
                                                   pip_install_call['stderr'])
    else:
        ret['result'] = False
        ret['comment'] = 'Could not install package'

    return ret

def removed(name,
            packages=None,
            requirements=None,
            bin_env=None,
            log=None,
            proxy=None,
            timeout=None,
            user=None,
            cwd=None,
            __env__='base'):
    """
    Make sure that a package is not installed.

    name
        The name of the package to uninstall
    bin_env : None
        the pip executable or virtualenenv to use
    """

    ret = {'name': name, 'result': None, 'comment': '', 'changes': {}}

    try:
        pip_list = __salt__["pip.list"](bin_env=bin_env, runas=user, cwd=cwd)
    except (CommandExecutionError, CommandNotFoundError) as err:
        ret['result'] = False
        ret['comment'] = 'Error uninstalling \'{0}\': {1}'.format(name, err)
        return ret

    if name not in pip_list:
        ret["result"] = True
        ret["comment"] = "Package is not installed."
        return ret

    if __opts__['test']:
        ret['result'] = None
        ret['comment'] = 'Package {0} is set to be removed'.format(name)
        return ret

    if __salt__["pip.uninstall"](pkgs=name,
                                 requirements=requirements,
                                 bin_env=bin_env,
                                 log=log,
                                 proxy=proxy,
                                 timeout=timeout,
                                 runas=user,
                                 cwd=cwd,
                                 __env__='base'):
        ret["result"] = True
        ret["changes"][name] = "Removed"
        ret["comment"] = "Package was successfully removed."
    else:
        ret["result"] = False
        ret["comment"] = "Could not remove package."
    return ret
<commit_msg>Sub `is False` for `== False`<commit_after>'''
Installation of Python packages using pip.
==========================================

A state module to manage system installed python packages

.. code-block:: yaml

    virtualenvwrapper:
      pip.installed
'''

# Import salt libs
from salt.exceptions import CommandExecutionError, CommandNotFoundError

def installed(name,
              pip_bin=None,
              requirements=None,
              env=None,
              bin_env=None,
              log=None,
              proxy=None,
              timeout=None,
              repo=None,
              editable=None,
              find_links=None,
              index_url=None,
              extra_index_url=None,
              no_index=False,
              mirrors=None,
              build=None,
              target=None,
              download=None,
              download_cache=None,
              source=None,
              upgrade=False,
              force_reinstall=False,
              ignore_installed=False,
              no_deps=False,
              no_install=False,
              no_download=False,
              install_options=None,
              user=None,
              cwd=None,
              __env__='base'):
    '''
    Make sure the package is installed

    name
        The name of the python package to install
    pip_bin :  None
        Deprecated, use bin_env
    env : None
        Deprecated, use bin_env
    bin_env : None
        the pip executable or virtualenv to use

    '''
    if pip_bin and not bin_env:
        bin_env = pip_bin
    elif env and not bin_env:
        bin_env = env

    ret = {'name': name, 'result': None, 'comment': '', 'changes': {}}
    try:
        pip_list = __salt__['pip.list'](name, bin_env, runas=user, cwd=cwd)
    except (CommandNotFoundError, CommandExecutionError) as err:
        ret['result'] = False
        ret['comment'] = 'Error installing \'{0}\': {1}'.format(name, err)
        return ret

    if ignore_installed is False and name.lower() in (p.lower() for p in pip_list):
        if force_reinstall is False and upgrade is False:
            ret['result'] = True
            ret['comment'] = 'Package already installed'
            return ret

    if __opts__['test']:
        ret['result'] = None
        ret['comment'] = 'Python package {0} is set to be installed'.format(
                name)
        return ret

    if repo:
        name = repo

    pip_install_call = __salt__['pip.install'](
        pkgs=name,
        requirements=requirements,
        bin_env=bin_env,
        log=log,
        proxy=proxy,
        timeout=timeout,
        editable=editable,
        find_links=find_links,
        index_url=index_url,
        extra_index_url=extra_index_url,
        no_index=no_index,
        mirrors=mirrors,
        build=build,
        target=target,
        download=download,
        download_cache=download_cache,
        source=source,
        upgrade=upgrade,
        force_reinstall=force_reinstall,
        ignore_installed=ignore_installed,
        no_deps=no_deps,
        no_install=no_install,
        no_download=no_download,
        install_options=install_options,
        runas=user,
        cwd=cwd,
        __env__=__env__
    )

    if pip_install_call and (pip_install_call['retcode'] == 0):
        ret['result'] = True

        pkg_list = __salt__['pip.list'](name, bin_env, runas=user, cwd=cwd)
        if not pkg_list:
            ret['comment'] = (
                'There was no error installing package \'{0}\' although '
                'it does not show when calling \'pip.freeze\'.'.format(name)
            )
            ret['changes']["{0}==???".format(name)] = 'Installed'
            return ret

        version = list(pkg_list.values())[0]
        pkg_name = next(iter(pkg_list))
        ret['changes']["{0}=={1}".format(pkg_name, version)] = 'Installed'
        ret['comment'] = 'Package was successfully installed'
    elif pip_install_call:
        ret['result'] = False
        ret['comment'] = ('Failed to install package {0}. '
                          'Error: {1} {2}').format(name,
                                                   pip_install_call['stdout'],
                                                   pip_install_call['stderr'])
    else:
        ret['result'] = False
        ret['comment'] = 'Could not install package'

    return ret

def removed(name,
            packages=None,
            requirements=None,
            bin_env=None,
            log=None,
            proxy=None,
            timeout=None,
            user=None,
            cwd=None,
            __env__='base'):
    """
    Make sure that a package is not installed.

    name
        The name of the package to uninstall
    bin_env : None
        the pip executable or virtualenenv to use
    """

    ret = {'name': name, 'result': None, 'comment': '', 'changes': {}}

    try:
        pip_list = __salt__["pip.list"](bin_env=bin_env, runas=user, cwd=cwd)
    except (CommandExecutionError, CommandNotFoundError) as err:
        ret['result'] = False
        ret['comment'] = 'Error uninstalling \'{0}\': {1}'.format(name, err)
        return ret

    if name not in pip_list:
        ret["result"] = True
        ret["comment"] = "Package is not installed."
        return ret

    if __opts__['test']:
        ret['result'] = None
        ret['comment'] = 'Package {0} is set to be removed'.format(name)
        return ret

    if __salt__["pip.uninstall"](pkgs=name,
                                 requirements=requirements,
                                 bin_env=bin_env,
                                 log=log,
                                 proxy=proxy,
                                 timeout=timeout,
                                 runas=user,
                                 cwd=cwd,
                                 __env__='base'):
        ret["result"] = True
        ret["changes"][name] = "Removed"
        ret["comment"] = "Package was successfully removed."
    else:
        ret["result"] = False
        ret["comment"] = "Could not remove package."
    return ret
<|endoftext|>


<commit_before>import os
import sys
import readline
import glob
import re

import random
import itertools
import time
import queue
import multiprocessing as mp

import argparse
import logging

import pandas as pd
from edisgo.grid.network import EDisGo
from edisgo.grid.network import Results
from edisgo.flex_opt.exceptions import MaximumIterationError

def setup_logging(logfilename=None,
                  logfile_loglevel='debug',
                  console_loglevel='info',
                  **logging_kwargs):
    # a dict to help with log level definition
    loglevel_dict = {'info': logging.INFO,
                     'debug': logging.DEBUG,
                     'warn': logging.WARNING,
                     'warning': logging.WARNING,
                     'error': logging.ERROR,
                     'critical': logging.CRITICAL}

    if not (logfilename):
        logfilename = 'edisgo_run.log'

    logging.basicConfig(filename=logfilename,
                        format='%(asctime)s - %(name)s -' +
                               ' %(levelname)s - %(message)s',
                        datefmt='%m/%d/%Y %H:%M:%S',
                        level=loglevel_dict[logfile_loglevel])

    root_logger = logging.getLogger()

    console_stream = logging.StreamHandler()
    console_stream.setLevel(loglevel_dict[console_loglevel])
    console_formatter = logging.Formatter(fmt='%(asctime)s - %(name)s -' +
                                              ' %(levelname)s - %(message)s',
                                          datefmt='%m/%d/%Y %H:%M:%S')
    console_stream.setFormatter(console_formatter)

    # add stream handler to root logger
    root_logger.addHandler(console_stream)

    return root_logger

def _get_griddistrict(ding0_filepath):
    """
    Just get the grid district number from ding0 data file path

    Parameters
    ----------
    ding0_filepath : str
        Path to ding0 data ending typically
        `/path/to/ding0_data/"ding0_grids__" + str(``grid_district``) + ".xxx"`
    Returns
    -------
    int
        grid_district number
    """
    grid_district = os.path.basename(ding0_filepath)
    grid_district_search = re.search('[_]+\d+', grid_district)
    if grid_district_search:
        grid_district = int(grid_district_search.group(0)[2:])
        return grid_district
    else:
        raise (KeyError('Grid District not found in '.format(grid_district)))

def run_edisgo_basic(ding0_filepath,
                     generator_scenario=None,
                     analysis='worst-case',
                     *edisgo_grid):
    """
    Analyze edisgo grid extension cost as reference scenario

    Parameters
    ----------
    ding0_filepath : str
        Path to ding0 data ending typically
        `/path/to/ding0_data/"ding0_grids__" + str(``grid_district``) + ".xxx"`

    analysis : str
        Either 'worst-case' or 'timeseries'

    generator_scenario : None or :obj:`str`
        If provided defines which scenario of future generator park to use
        and invokes import of these generators. Possible options are 'nep2035'
        and 'ego100'.

    Returns
    -------
    edisgo_grid : :class:`~.grid.network.EDisGo`
        eDisGo network container
    costs : :pandas:`pandas.Dataframe<dataframe>`
        Cost of grid extension
    grid_issues : dict
        Grids resulting in an error including error message

    """

    grid_district = _get_griddistrict(ding0_filepath)

    grid_issues = {'grid': [],
                   'msg': []}

    logging.info('Grid expansion for MV grid district {}'.format(grid_district))

    if edisgo_grid: # if an edisgo_grid is passed in arg then ignore everything else
        edisgo_grid = edisgo_grid[0]
    else:
        if 'worst-case' in analysis:
            edisgo_grid = EDisGo(ding0_grid=ding0_filepath,
                                 worst_case_analysis=analysis)
        elif 'timeseries' in analysis:
            edisgo_grid = EDisGo(ding0_grid=ding0_filepath,
                                 timeseries_generation_fluctuating='oedb',
                                 timeseries_load='demandlib')
    # Import generators
    if generator_scenario:
        logging.info('Grid expansion for scenario \'{}\'.'.format(generator_scenario))
        edisgo_grid.import_generators(generator_scenario=generator_scenario)
    else:
        logging.info('Grid expansion with no generator imports based on scenario')

    try:
        # Do non-linear power flow analysis with PyPSA
        edisgo_grid.analyze()

        # Do grid reinforcement
        edisgo_grid.reinforce()

        # Get costs
        costs_grouped = \
            edisgo_grid.network.results.grid_expansion_costs.groupby(
                ['type']).sum()
        costs = pd.DataFrame(costs_grouped.values,
                             columns=costs_grouped.columns,
                             index=[[edisgo_grid.network.id] * len(costs_grouped),
                                    costs_grouped.index])
        costs.reset_index(inplace=True)

        logging.info('SUCCESS!')
    except MaximumIterationError:
        grid_issues['grid'].append(edisgo_grid.network.id)
        grid_issues['msg'].append(str(edisgo_grid.network.results.unresolved_issues))
        costs = pd.DataFrame()
        logging.info('Unresolved issues left after grid expansion.')
    except Exception as e:
        grid_issues['grid'].append(edisgo_grid.network.id)
        grid_issues['msg'].append(repr(e))
        costs = pd.DataFrame()
        logging.info('Inexplicable Error, Please Check error messages and logs.')

    return edisgo_grid, costs, grid_issues

def run_edisgo_twice(run_args):
    """

    Returns
    -------
    all_costs_before_geno_import : list
        Grid extension cost before grid connection of new generators
    all_grid_issues_before_geno_import : list
        Remaining overloading or over-voltage issues in grid
    all_costs : list
        Grid extension cost due to grid connection of new generators
    all_grid_issues : list
        Remaining overloading or over-voltage issues in grid
    """
    # base case with no generator import

    edisgo_grid, \
    costs_before_geno_import, \
    grid_issues_before_geno_import = run_func(*run_args)

    # clear the pypsa object and results from edisgo_grid
    edisgo_grid.network.results = Results()
    edisgo_grid.network.pypsa = None

    all_costs_before_geno_import.append(costs_before_geno_import)

    all_grid_issues_before_geno_import['grid'].extend(
        grid_issues_before_geno_import['grid'])

    all_grid_issues_before_geno_import['msg'].extend(
        grid_issues_before_geno_import['msg'])

    # case after generator import
    run_args = [ding0_filename]
    run_args.extend(run_args_opt)
    run_args.append(edisgo_grid)

    _, costs, \
    grid_issues = run_func(*run_args)

    all_costs.append(costs)
    all_grid_issues['grid'].extend(grid_issues['grid'])
    all_grid_issues['msg'].extend(grid_issues['msg'])

    return all_costs_before_geno_import, all_grid_issues_before_geno_import, \
           all_costs, all_grid_issues

def run_pool(number_of_processes, worker_lifetime, edisgo_args):

    def collect_pool_results(result):
        results.append(result)

    results = []

    pool = mp.Pool(number_of_processes,
                   maxtasksperchild=worker_lifetime)

    if analysis_mode == 'worst-case':
        for grid_id in grid_ids:
            pool.apply_async(func=run_edisgo_worst_case,
                             args=(grid_id, data_dir),
                             kwds=tech_dict,
                             callback=collect_pool_results)
    elif analysis_mode == 'timeseries':
        for grid_id in grid_ids:
            pool.apply_async(func=run_edisgo_timeseries_worst_case,
                             args=(grid_id, data_dir),
                             kwds=tech_dict,
                             callback=collect_pool_results)
    else:
        logger.error("Unknown analysis mode {]".format(analysis_mode))

    pool.close()
    pool.join()

    # process results data
    costs_dfs = [r[0] for r in results]
    faulty_grids_dfs = [r[1] for r in results]
    costs_before_generators_dfs = [r[2] for r in results]
    faulty_grids_before_generators_dfs = [r[3] for r in results]

    if costs_dfs:
        costs = pd.concat(costs_dfs, axis=0)
    else:
        costs = pd.DataFrame()

    if faulty_grids_dfs:
        faulty_grids = pd.concat(faulty_grids_dfs, axis=0)
    else:
        faulty_grids = pd.DataFrame()

    if costs_before_generators_dfs:
        costs_before_generators = pd.concat(costs_before_generators_dfs, axis=0)
    else:

        costs_before_generators = pd.DataFrame()

    if faulty_grids_before_generators_dfs:
        faulty_grids_before_generators = pd.concat(
            faulty_grids_before_generators_dfs, axis=0)
    else:
        faulty_grids_before_generators = pd.DataFrame()

    return costs, faulty_grids, costs_before_generators, faulty_grids_before_generators

if __name__ == '__main__':
    # create the argument parser
    parser = argparse.ArgumentParser(description="Commandline running" + \
                                                 "of eDisGo")

    # add the verbosity arguments

    ding0_files_parsegroup = parser.add_mutually_exclusive_group(required=True)

    ding0_files_parsegroup.add_argument('-f', '--ding0-file-path', type=str,
                                        action='store',
                                        dest='ding0_filename',
                                        help='Path to a single ding0 file.')
    ding0_files_parsegroup.add_argument('-d', '--ding0-files-directory', type=str,
                                        action='store',
                                        dest='ding0_dirglob',
                                        help='Path to a directory of ding0 files ' + \
                                             'along with  a file name pattern for glob input.')
    ding0_files_parsegroup.add_argument('-ds', '--ding0-files-directory-selection', type=str,
                                        nargs=3,
                                        action='store',
                                        dest='ding0_dir_select',
                                        help='Path to a directory of ding0 files, ' + \
                                             'Path to file with list of grid district numbers ' + \
                                             '(one number per line), ' + \
                                             'and file name template using {} where number ' + \
                                             'is to be inserted . Convention is to use ' + \
                                             'a double underscore before grid district number ' + \
                                             ' like so \'__{}\'.')

    analysis_parsegroup = parser.add_mutually_exclusive_group()

    analysis_parsegroup.add_argument('-wc', '--worst-case',
                                     action='store_true',
                                     help='Performs a worst-case simulation with ' + \
                                          'a single snapshot')

    analysis_parsegroup.add_argument('-ts', '--timeseries',
                                     action='store_true',
                                     help='Performs a worst-case simulation with ' + \
                                          'a time-series')

    parser.add_argument('-s', '--scenario',
                        type=str,
                        default=None,
                        choices=[None, 'nep2035', 'ego100'],
                        help="\'None\' or \'string\'\n" + \
                             "If provided defines which scenario " + \
                             "of future generator park to use " + \
                             "and invokes import of these generators.\n" + \
                             "Possible options are \'nep2035\'and \'ego100\'.")

    parser.add_argument('-o', '--output-dir',
                        nargs='?',
                        metavar='/path/to/output/',
                        dest="out_dir",
                        type=str,
                        default=os.path.join(sys.path[0]),
                        help='Absolute path to results data location.')

    parser.add_argument('--steps',
                        nargs='?',
                        metavar='4',
                        dest="steps",
                        type=int,
                        default=1,
                        help='Number of grid district that are analyzed in one '
                             'bunch. Hence, that are saved into one CSV file.')

    parser.add_argument('-lw', '--lifetime-workers',
                        nargs='?',
                        metavar='1..inf',
                        dest="worker_lifetime",
                        type=int,
                        default=None,
                        help='Lifetime of a worker of the cluster doing the '
                             'work. The lifetime is given is number of jobs a'
                             ' worker does before it is replaced by a freshly '
                             'new one.'
                             'The default sets the lifetime to the pools '
                             'lifetime. This can cause memory issues!')

    analysis_parsegroup.add_argument('-p', '--parallel',
                                     action='store_true',
                                     help='Parallel execution of multiple '
                                          'grids. Parallelization is provided '
                                          'by multiprocessing.')

    args = parser.parse_args(sys.argv[1:])

    # get current time for output file names
    exec_time = pd.datetime.now().strftime('%Y-%m-%d_%H%M')

    logger = setup_logging(logfilename='test.log',
                           logfile_loglevel='debug',
                           console_loglevel='info')

    # get the list of files to run on
    if args.ding0_filename:
        ding0_file_list = [args.ding0_filename]

    elif args.ding0_dirglob:
        ding0_file_list = glob.glob(args.ding0_dirglob)

    elif args.ding0_dirselect:
        with open(args.ding0_dirselect[1], 'r') as file_handle:
            ding0_file_list_grid_district_numbers = list(file_handle)

        ding0_file_list = map(lambda x: args.ding0_dirselect[0] +
                                        args.ding0_dirselect[2].format(x),
                              ding0_file_list_grid_district_numbers)
    else:
        raise FileNotFoundError('Some of the Arguments for input files are missing.')

    # this is the serial version of the run system
    run_func = run_edisgo_basic

    run_args_opt_no_scenario = [None]
    run_args_opt = [args.scenario]
    if args.worst_case:
        run_args_opt_no_scenario.append('worst-case')
        run_args_opt.append('worst-case')
    elif args.timeseries:
        run_args_opt_no_scenario.append('timeseries')
        run_args_opt.append('timeseries')

    all_costs_before_geno_import = []
    all_grid_issues_before_geno_import = {'grid': [], 'msg': []}
    all_costs = []
    all_grid_issues = {'grid': [], 'msg': []}

    if not args.parallel:
        for ding0_filename in ding0_file_list:
            grid_district = _get_griddistrict(ding0_filename)

            run_args = [ding0_filename]
            run_args.extend(run_args_opt_no_scenario)

            all_costs_before_geno_import, \
            all_grid_issues_before_geno_import, \
                all_costs, all_grid_issues = run_edisgo_twice(run_args)
    else:
        raise NotImplementedError("Parallelization will come soon (:")

    # consolidate costs for all the networks
    all_costs_before_geno_import = pd.concat(all_costs_before_geno_import,
                                             ignore_index=True)
    all_costs = pd.concat(all_costs, ignore_index=True)
    # write costs and error messages to csv files

    pd.DataFrame(all_grid_issues_before_geno_import).to_csv(
        args.out_dir +
        exec_time + '_' +
        'grid_issues_before_geno_import.csv', index=False)

    with open(args.out_dir +
              exec_time + '_' + 'costs_before_geno_import.csv', 'a') as f:
        f.write(',,,# units: length in km,, total_costs in kEUR\n')
        all_costs_before_geno_import.to_csv(f)

    pd.DataFrame(all_grid_issues).to_csv(args.out_dir + exec_time + '_' + \
                                         'grid_issues.csv', index=False)
    with open(args.out_dir +
              exec_time + '_' + 'costs.csv', 'a') as f:
        f.write(',,,# units: length in km,, total_costs in kEUR\n')
        all_costs.to_csv(f)<commit_msg>Fix typo<commit_after>import os
import sys
import readline
import glob
import re

import random
import itertools
import time
import queue
import multiprocessing as mp

import argparse
import logging

import pandas as pd
from edisgo.grid.network import EDisGo
from edisgo.grid.network import Results
from edisgo.flex_opt.exceptions import MaximumIterationError

def setup_logging(logfilename=None,
                  logfile_loglevel='debug',
                  console_loglevel='info',
                  **logging_kwargs):
    # a dict to help with log level definition
    loglevel_dict = {'info': logging.INFO,
                     'debug': logging.DEBUG,
                     'warn': logging.WARNING,
                     'warning': logging.WARNING,
                     'error': logging.ERROR,
                     'critical': logging.CRITICAL}

    if not (logfilename):
        logfilename = 'edisgo_run.log'

    logging.basicConfig(filename=logfilename,
                        format='%(asctime)s - %(name)s -' +
                               ' %(levelname)s - %(message)s',
                        datefmt='%m/%d/%Y %H:%M:%S',
                        level=loglevel_dict[logfile_loglevel])

    root_logger = logging.getLogger()

    console_stream = logging.StreamHandler()
    console_stream.setLevel(loglevel_dict[console_loglevel])
    console_formatter = logging.Formatter(fmt='%(asctime)s - %(name)s -' +
                                              ' %(levelname)s - %(message)s',
                                          datefmt='%m/%d/%Y %H:%M:%S')
    console_stream.setFormatter(console_formatter)

    # add stream handler to root logger
    root_logger.addHandler(console_stream)

    return root_logger

def _get_griddistrict(ding0_filepath):
    """
    Just get the grid district number from ding0 data file path

    Parameters
    ----------
    ding0_filepath : str
        Path to ding0 data ending typically
        `/path/to/ding0_data/"ding0_grids__" + str(``grid_district``) + ".xxx"`
    Returns
    -------
    int
        grid_district number
    """
    grid_district = os.path.basename(ding0_filepath)
    grid_district_search = re.search('[_]+\d+', grid_district)
    if grid_district_search:
        grid_district = int(grid_district_search.group(0)[2:])
        return grid_district
    else:
        raise (KeyError('Grid District not found in '.format(grid_district)))

def run_edisgo_basic(ding0_filepath,
                     generator_scenario=None,
                     analysis='worst-case',
                     *edisgo_grid):
    """
    Analyze edisgo grid extension cost as reference scenario

    Parameters
    ----------
    ding0_filepath : str
        Path to ding0 data ending typically
        `/path/to/ding0_data/"ding0_grids__" + str(``grid_district``) + ".xxx"`

    analysis : str
        Either 'worst-case' or 'timeseries'

    generator_scenario : None or :obj:`str`
        If provided defines which scenario of future generator park to use
        and invokes import of these generators. Possible options are 'nep2035'
        and 'ego100'.

    Returns
    -------
    edisgo_grid : :class:`~.grid.network.EDisGo`
        eDisGo network container
    costs : :pandas:`pandas.Dataframe<dataframe>`
        Cost of grid extension
    grid_issues : dict
        Grids resulting in an error including error message

    """

    grid_district = _get_griddistrict(ding0_filepath)

    grid_issues = {'grid': [],
                   'msg': []}

    logging.info('Grid expansion for MV grid district {}'.format(grid_district))

    if edisgo_grid: # if an edisgo_grid is passed in arg then ignore everything else
        edisgo_grid = edisgo_grid[0]
    else:
        if 'worst-case' in analysis:
            edisgo_grid = EDisGo(ding0_grid=ding0_filepath,
                                 worst_case_analysis=analysis)
        elif 'timeseries' in analysis:
            edisgo_grid = EDisGo(ding0_grid=ding0_filepath,
                                 timeseries_generation_fluctuating='oedb',
                                 timeseries_load='demandlib')
    # Import generators
    if generator_scenario:
        logging.info('Grid expansion for scenario \'{}\'.'.format(generator_scenario))
        edisgo_grid.import_generators(generator_scenario=generator_scenario)
    else:
        logging.info('Grid expansion with no generator imports based on scenario')

    try:
        # Do non-linear power flow analysis with PyPSA
        edisgo_grid.analyze()

        # Do grid reinforcement
        edisgo_grid.reinforce()

        # Get costs
        costs_grouped = \
            edisgo_grid.network.results.grid_expansion_costs.groupby(
                ['type']).sum()
        costs = pd.DataFrame(costs_grouped.values,
                             columns=costs_grouped.columns,
                             index=[[edisgo_grid.network.id] * len(costs_grouped),
                                    costs_grouped.index])
        costs.reset_index(inplace=True)

        logging.info('SUCCESS!')
    except MaximumIterationError:
        grid_issues['grid'].append(edisgo_grid.network.id)
        grid_issues['msg'].append(str(edisgo_grid.network.results.unresolved_issues))
        costs = pd.DataFrame()
        logging.info('Unresolved issues left after grid expansion.')
    except Exception as e:
        grid_issues['grid'].append(edisgo_grid.network.id)
        grid_issues['msg'].append(repr(e))
        costs = pd.DataFrame()
        logging.info('Inexplicable Error, Please Check error messages and logs.')

    return edisgo_grid, costs, grid_issues

def run_edisgo_twice(run_args):
    """

    Returns
    -------
    all_costs_before_geno_import : list
        Grid extension cost before grid connection of new generators
    all_grid_issues_before_geno_import : list
        Remaining overloading or over-voltage issues in grid
    all_costs : list
        Grid extension cost due to grid connection of new generators
    all_grid_issues : list
        Remaining overloading or over-voltage issues in grid
    """
    # base case with no generator import

    edisgo_grid, \
    costs_before_geno_import, \
    grid_issues_before_geno_import = run_func(*run_args)

    # clear the pypsa object and results from edisgo_grid
    edisgo_grid.network.results = Results()
    edisgo_grid.network.pypsa = None

    all_costs_before_geno_import.append(costs_before_geno_import)

    all_grid_issues_before_geno_import['grid'].extend(
        grid_issues_before_geno_import['grid'])

    all_grid_issues_before_geno_import['msg'].extend(
        grid_issues_before_geno_import['msg'])

    # case after generator import
    run_args = [ding0_filename]
    run_args.extend(run_args_opt)
    run_args.append(edisgo_grid)

    _, costs, \
    grid_issues = run_func(*run_args)

    all_costs.append(costs)
    all_grid_issues['grid'].extend(grid_issues['grid'])
    all_grid_issues['msg'].extend(grid_issues['msg'])

    return all_costs_before_geno_import, all_grid_issues_before_geno_import, \
           all_costs, all_grid_issues

def run_pool(number_of_processes, worker_lifetime, edisgo_args):

    def collect_pool_results(result):
        results.append(result)

    results = []

    pool = mp.Pool(number_of_processes,
                   maxtasksperchild=worker_lifetime)

    if analysis_mode == 'worst-case':
        for grid_id in grid_ids:
            pool.apply_async(func=run_edisgo_worst_case,
                             args=(grid_id, data_dir),
                             kwds=tech_dict,
                             callback=collect_pool_results)
    elif analysis_mode == 'timeseries':
        for grid_id in grid_ids:
            pool.apply_async(func=run_edisgo_timeseries_worst_case,
                             args=(grid_id, data_dir),
                             kwds=tech_dict,
                             callback=collect_pool_results)
    else:
        logger.error("Unknown analysis mode {]".format(analysis_mode))

    pool.close()
    pool.join()

    # process results data
    costs_dfs = [r[0] for r in results]
    faulty_grids_dfs = [r[1] for r in results]
    costs_before_generators_dfs = [r[2] for r in results]
    faulty_grids_before_generators_dfs = [r[3] for r in results]

    if costs_dfs:
        costs = pd.concat(costs_dfs, axis=0)
    else:
        costs = pd.DataFrame()

    if faulty_grids_dfs:
        faulty_grids = pd.concat(faulty_grids_dfs, axis=0)
    else:
        faulty_grids = pd.DataFrame()

    if costs_before_generators_dfs:
        costs_before_generators = pd.concat(costs_before_generators_dfs, axis=0)
    else:

        costs_before_generators = pd.DataFrame()

    if faulty_grids_before_generators_dfs:
        faulty_grids_before_generators = pd.concat(
            faulty_grids_before_generators_dfs, axis=0)
    else:
        faulty_grids_before_generators = pd.DataFrame()

    return costs, faulty_grids, costs_before_generators, faulty_grids_before_generators

if __name__ == '__main__':
    # create the argument parser
    parser = argparse.ArgumentParser(description="Commandline running" + \
                                                 "of eDisGo")

    # add the verbosity arguments

    ding0_files_parsegroup = parser.add_mutually_exclusive_group(required=True)

    ding0_files_parsegroup.add_argument('-f', '--ding0-file-path', type=str,
                                        action='store',
                                        dest='ding0_filename',
                                        help='Path to a single ding0 file.')
    ding0_files_parsegroup.add_argument('-d', '--ding0-files-directory', type=str,
                                        action='store',
                                        dest='ding0_dirglob',
                                        help='Path to a directory of ding0 files ' + \
                                             'along with  a file name pattern for glob input.')
    ding0_files_parsegroup.add_argument('-ds', '--ding0-files-directory-selection', type=str,
                                        nargs=3,
                                        action='store',
                                        dest='ding0_dir_select',
                                        help='Path to a directory of ding0 files, ' + \
                                             'Path to file with list of grid district numbers ' + \
                                             '(one number per line), ' + \
                                             'and file name template using {} where number ' + \
                                             'is to be inserted . Convention is to use ' + \
                                             'a double underscore before grid district number ' + \
                                             ' like so \'__{}\'.')

    analysis_parsegroup = parser.add_mutually_exclusive_group()

    analysis_parsegroup.add_argument('-wc', '--worst-case',
                                     action='store_true',
                                     help='Performs a worst-case simulation with ' + \
                                          'a single snapshot')

    analysis_parsegroup.add_argument('-ts', '--timeseries',
                                     action='store_true',
                                     help='Performs a worst-case simulation with ' + \
                                          'a time-series')

    parser.add_argument('-s', '--scenario',
                        type=str,
                        default=None,
                        choices=[None, 'nep2035', 'ego100'],
                        help="\'None\' or \'string\'\n" + \
                             "If provided defines which scenario " + \
                             "of future generator park to use " + \
                             "and invokes import of these generators.\n" + \
                             "Possible options are \'nep2035\'and \'ego100\'.")

    parser.add_argument('-o', '--output-dir',
                        nargs='?',
                        metavar='/path/to/output/',
                        dest="out_dir",
                        type=str,
                        default=os.path.join(sys.path[0]),
                        help='Absolute path to results data location.')

    parser.add_argument('--steps',
                        nargs='?',
                        metavar='4',
                        dest="steps",
                        type=int,
                        default=1,
                        help='Number of grid district that are analyzed in one '
                             'bunch. Hence, that are saved into one CSV file.')

    parser.add_argument('-lw', '--lifetime-workers',
                        nargs='?',
                        metavar='1..inf',
                        dest="worker_lifetime",
                        type=int,
                        default=None,
                        help='Lifetime of a worker of the cluster doing the '
                             'work. The lifetime is given is number of jobs a'
                             ' worker does before it is replaced by a freshly '
                             'new one.'
                             'The default sets the lifetime to the pools '
                             'lifetime. This can cause memory issues!')

    analysis_parsegroup.add_argument('-p', '--parallel',
                                     action='store_true',
                                     help='Parallel execution of multiple '
                                          'grids. Parallelization is provided '
                                          'by multiprocessing.')

    args = parser.parse_args(sys.argv[1:])

    # get current time for output file names
    exec_time = pd.datetime.now().strftime('%Y-%m-%d_%H%M')

    logger = setup_logging(logfilename='test.log',
                           logfile_loglevel='debug',
                           console_loglevel='info')

    # get the list of files to run on
    if args.ding0_filename:
        ding0_file_list = [args.ding0_filename]

    elif args.ding0_dirglob:
        ding0_file_list = glob.glob(args.ding0_dirglob)

    elif args.ding0_dir_select:
        with open(args.ding0_dir_select[1], 'r') as file_handle:
            ding0_file_list_grid_district_numbers = list(file_handle)

        ding0_file_list = map(lambda x: args.ding0_dir_select[0] +
                                        args.ding0_dir_select[2].format(x),
                              ding0_file_list_grid_district_numbers)
    else:
        raise FileNotFoundError('Some of the Arguments for input files are missing.')

    # this is the serial version of the run system
    run_func = run_edisgo_basic

    run_args_opt_no_scenario = [None]
    run_args_opt = [args.scenario]
    if args.worst_case:
        run_args_opt_no_scenario.append('worst-case')
        run_args_opt.append('worst-case')
    elif args.timeseries:
        run_args_opt_no_scenario.append('timeseries')
        run_args_opt.append('timeseries')

    all_costs_before_geno_import = []
    all_grid_issues_before_geno_import = {'grid': [], 'msg': []}
    all_costs = []
    all_grid_issues = {'grid': [], 'msg': []}

    if not args.parallel:
        for ding0_filename in ding0_file_list:
            grid_district = _get_griddistrict(ding0_filename)

            run_args = [ding0_filename]
            run_args.extend(run_args_opt_no_scenario)

            all_costs_before_geno_import, \
            all_grid_issues_before_geno_import, \
                all_costs, all_grid_issues = run_edisgo_twice(run_args)
    else:
        raise NotImplementedError("Parallelization will come soon (:")

    # consolidate costs for all the networks
    all_costs_before_geno_import = pd.concat(all_costs_before_geno_import,
                                             ignore_index=True)
    all_costs = pd.concat(all_costs, ignore_index=True)
    # write costs and error messages to csv files

    pd.DataFrame(all_grid_issues_before_geno_import).to_csv(
        args.out_dir +
        exec_time + '_' +
        'grid_issues_before_geno_import.csv', index=False)

    with open(args.out_dir +
              exec_time + '_' + 'costs_before_geno_import.csv', 'a') as f:
        f.write(',,,# units: length in km,, total_costs in kEUR\n')
        all_costs_before_geno_import.to_csv(f)

    pd.DataFrame(all_grid_issues).to_csv(args.out_dir + exec_time + '_' + \
                                         'grid_issues.csv', index=False)
    with open(args.out_dir +
              exec_time + '_' + 'costs.csv', 'a') as f:
        f.write(',,,# units: length in km,, total_costs in kEUR\n')
        all_costs.to_csv(f)<|endoftext|>


<commit_before>"""
picture.py
Author: Avery Wallis
Credit: None so far

Assignment:
Use the ggame library to "paint" a graphical picture of something (e.g. a house, a face or landscape).
Use at least:
1. Three different Color objects.
2. Ten different Sprite objects.
3. One (or more) RectangleAsset objects.
4. One (or more) CircleAsset objects.
5. One (or more) EllipseAsset objects.
6. One (or more) PolygonAsset objects.
See:
https://github.com/HHS-IntroProgramming/Standards-and-Syllabus/wiki/Displaying-Graphics
for general information on how to use ggame.
See:
http://brythonserver.github.io/ggame/
for detailed information on ggame.
"""

from ggame import App, Color, LineStyle, Sprite, RectangleAsset, CircleAsset, EllipseAsset, PolygonAsset

red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)
orange = Color(0xffa500, 1.0)
skin =Color(0xFCD15B, 1.0)
wall=Color(0xC4CACC, 1.0)

thinline= LineStyle(1, black)
thickline= LineStyle(5, black)
thickishline= LineStyle(2.5, black)
noline=LineStyle(0, black)

linedivide=LineAsset ([(250,20), (250,750)], thinline, black)
wall=RectangleAsset(500,500, noline, wall)
blueportal=ElllipseAsset(20,60,thinline,blue))

Sprite(wall, (400,20))
Sprite(wall, (100,20))
Sprite(linedivide, (0,0))
Sprite(blueportal, (200,200))

myapp = App()
myapp.run()
<commit_msg>Updated from Brython Server: 1/27/2016 1:32:00 PM<commit_after>"""
picture.py
Author: Avery Wallis
Credit: None so far

Assignment:
Use the ggame library to "paint" a graphical picture of something (e.g. a house, a face or landscape).
Use at least:
1. Three different Color objects.
2. Ten different Sprite objects.
3. One (or more) RectangleAsset objects.
4. One (or more) CircleAsset objects.
5. One (or more) EllipseAsset objects.
6. One (or more) PolygonAsset objects.
See:
https://github.com/HHS-IntroProgramming/Standards-and-Syllabus/wiki/Displaying-Graphics
for general information on how to use ggame.
See:
http://brythonserver.github.io/ggame/
for detailed information on ggame.
"""

from ggame import App, Color, LineStyle, Sprite, RectangleAsset, CircleAsset, EllipseAsset, PolygonAsset

red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)
orange = Color(0xffa500, 1.0)
skin =Color(0xFCD15B, 1.0)
wall=Color(0xC4CACC, 1.0)

thinline= LineStyle(1, black)
thickline= LineStyle(5, black)
thickishline= LineStyle(2.5, black)
noline=LineStyle(0, black)

linedivide=LineAsset ([(250,20), (250,750)], thinline, black)
wall=RectangleAsset(500,500, noline, wall)
blueportal=ElllipseAsset(20,60,thinline,blue))

Sprite(wall, (400,20))
Sprite(wall, (100,20))
Sprite(blueportal, (200,200))

myapp = App()
myapp.run()
<|endoftext|>