<commit_before># coding: UTF-8
"""
Copyright (c) 2009 Marian Tietz
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

1. Redistributions of source code must retain the above copyright
   notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
   notice, this list of conditions and the following disclaimer in the
   documentation and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
SUCH DAMAGE.
"""

import gtk
import dbus
import logging
from gobject import TYPE_UINT64
from threading import Thread
from time import sleep
from gettext import gettext as _
import traceback

import config
from com import sushi
from lib import dialog_control
from helper.dcc import s_incoming

widgets = None

(COL_STATUS,
 COL_ID,
 COL_SERVER,
 COL_PARTNER,
 COL_FILE,
 COL_SIZE,
 COL_PROGRESS,
 COL_SPEED) = range(8)

# TODO: mark status for every row

# TODO: accept/deny for incomings

class PollThread(Thread):

	def __init__(self):
		Thread.__init__(self)

		self._stop = True

		self.row_id_map = {}
		self.last_sends = []

	def run(self):
		Thread.run(self)

		while not self._stop:

			gtk.gdk.threads_enter()
			try:
				self.apply_dbus_sends()
			except BaseException as e:
				logging.error("Error in PollThread: %s\n%s" % (e, traceback.format_exc()))
			gtk.gdk.threads_leave()

			sleep(1)

	def start(self):
		self._stop = False
		Thread.start(self)

	def stop(self):
		self._stop = True

	def get_progress(self, p, s):
		return int(float(p)/s*100)

	def apply_dbus_sends(self):
		# FIXME: main gui freeze if main window is killed and dialog is still running

		sends = sushi.dcc_sends()

		if len(sends) == 0:
			return

		view = widgets.get_widget("transferView")
		store = view.get_model()

		act_sends = [n for n in sends[0]]

		to_remove = set(self.last_sends) - set(act_sends)
		to_update = set(act_sends) - to_remove

		for i in range(len(sends[0])):
			id, server, sender, filename, size, progress, speed, status = \
			[sends[n][i] for n in range(len(sends))]

			if id in to_update:
				if self.row_id_map.has_key(id):
					# update existing entry
					iter = self.row_id_map[id].iter

					store.set(iter,
						COL_STATUS, status,
						COL_SIZE, size,
						COL_PROGRESS, self.get_progress(progress, size),
						COL_SPEED, speed)
				else:
					# add new entry
					iter = store.append(row = (
						status, id, server,
						sender, filename,
						size, self.get_progress(progress, size),
						speed))
					self.row_id_map[id] = store[store.get_path(iter)]

		for id in to_remove:
			if self.row_id_map.has_key(id):
				store.remove(self.row_id_map[id].iter)

		self.last_sends = act_sends

def cancel_focused_transfer(poll_thread):
	view = widgets.get_widget("transferView")
	store = view.get_model()

	cursor = view.get_cursor()

	if None == cursor:
		pass
	else:
		sushi.dcc_send_remove(dbus.UInt64(store[cursor[0]][COL_ID]))
		poll_thread.apply_dbus_sends()

def dialog_response_cb(dialog, id, poll_thread):
	if id == 333:
		# remove was clicked
		cancel_focused_transfer(poll_thread)

	else:
		poll_thread.stop()
		dialog.destroy()

def run():
	dialog = widgets.get_widget("DCCDialog")

	poll_thread = PollThread()
	poll_thread.start()

	dialog.connect("response", dialog_response_cb, poll_thread)
	dialog.show_all()

def create_list_model():
	# status | id | server | sender | filename | size | progress | speed
	return gtk.ListStore(TYPE_UINT64, TYPE_UINT64, str, str, str, TYPE_UINT64, TYPE_UINT64, str)

def setup():
	global widgets

	widgets = dialog_control.build_dialog("dcc")

	transferView = widgets.get_widget("transferView")
	transferView.set_model(create_list_model())

	# add direction icon column
	def type_symbol_render_cb(column, renderer, model, iter):
		status = model.get(iter, COL_STATUS)
		if status:
			if status[0] & s_incoming:
				# incoming
				renderer.set_property("stock-id", gtk.STOCK_GO_DOWN)
			else:
				# outgoing
				renderer.set_property("stock-id", gtk.STOCK_GO_UP)

	renderer = gtk.CellRendererPixbuf()
	column = gtk.TreeViewColumn("", renderer)
	column.set_cell_data_func(renderer, type_symbol_render_cb)
	transferView.append_column(column)

	c = 1
	for name in (_("ID"), _("Server"), _("Partner"), _("Filename"), _("Size")):
		column = gtk.TreeViewColumn(name, gtk.CellRendererText(), text = c)
		column.set_resizable(True)
		transferView.append_column(column)
		c += 1

	# progress column
	renderer = gtk.CellRendererProgress()
	column = gtk.TreeViewColumn(_("Progress"), renderer, value = c)
	column.set_resizable(True)
	transferView.append_column(column)
	c+= 1

	# speed column
	column = gtk.TreeViewColumn(_("Speed"), gtk.CellRendererText(), text = c)
	column.set_resizable(True)
	transferView.append_column(column)
<commit_msg>Add question on send remove<commit_after># coding: UTF-8
"""
Copyright (c) 2009 Marian Tietz
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

1. Redistributions of source code must retain the above copyright
   notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
   notice, this list of conditions and the following disclaimer in the
   documentation and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
SUCH DAMAGE.
"""

import gtk
import dbus
import logging
from gobject import TYPE_UINT64
from threading import Thread
from time import sleep
from gettext import gettext as _
import traceback

import config
from com import sushi
from lib import dialog_control
from lib import gui_control
from helper.dcc import s_incoming

widgets = None

(COL_STATUS,
 COL_ID,
 COL_SERVER,
 COL_PARTNER,
 COL_FILE,
 COL_SIZE,
 COL_PROGRESS,
 COL_SPEED) = range(8)

# TODO: mark status for every row

# TODO: accept/deny for incomings

class PollThread(Thread):

	def __init__(self):
		Thread.__init__(self)

		self._stop = True

		self.row_id_map = {}
		self.last_sends = []

	def run(self):
		Thread.run(self)

		while not self._stop:

			gtk.gdk.threads_enter()
			try:
				self.apply_dbus_sends()
			except BaseException as e:
				logging.error("Error in PollThread: %s\n%s" % (e, traceback.format_exc()))
			gtk.gdk.threads_leave()

			sleep(1)

	def start(self):
		self._stop = False
		Thread.start(self)

	def stop(self):
		self._stop = True

	def get_progress(self, p, s):
		return int(float(p)/s*100)

	def apply_dbus_sends(self):
		# FIXME: main gui freeze if main window is killed and dialog is still running

		sends = sushi.dcc_sends()

		if len(sends) == 0:
			return

		view = widgets.get_widget("transferView")
		store = view.get_model()

		# all ids
		act_sends = sends[0]

		to_remove = set(self.last_sends) - set(act_sends)
		to_update = set(act_sends) - to_remove

		for i in range(len(sends[0])):
			id, server, sender, filename, size, progress, speed, status = \
			[sends[n][i] for n in range(len(sends))]

			if id in to_update:
				if self.row_id_map.has_key(id):
					# update existing entry
					iter = self.row_id_map[id].iter

					store.set(iter,
						COL_STATUS, status,
						COL_SIZE, size,
						COL_PROGRESS, self.get_progress(progress, size),
						COL_SPEED, speed)
				else:
					# add new entry
					iter = store.append(row = (
						status, id, server,
						sender, filename,
						size, self.get_progress(progress, size),
						speed))
					self.row_id_map[id] = store[store.get_path(iter)]

		for id in to_remove:
			if self.row_id_map.has_key(id):
				store.remove(self.row_id_map[id].iter)

		self.last_sends = act_sends

def cancel_focused_transfer(poll_thread):
	view = widgets.get_widget("transferView")
	store = view.get_model()

	cursor = view.get_cursor()

	if None == cursor:
		pass
	else:
		sushi.dcc_send_remove(dbus.UInt64(store[cursor[0]][COL_ID]))
		poll_thread.apply_dbus_sends()

def dialog_response_cb(dialog, id, poll_thread):
	if id == 333:
		# remove was clicked
		def ask_are_you_sure():
			# ask if the user is sure about removing the transfer

			def dialog_reponse_cb(dialog, id):
				if id == gtk.RESPONSE_YES:
					# yes, remove it!
					cancel_focused_transfer(poll_thread)
				dialog.destroy()

			d = gui_control.question_dialog(
				title = _("Remove file transfer?"),
				message = _("Are you sure you want to remove the file transfer %(id)d?" % { "id":id }))
			d.connect("response", dialog_reponse_cb)
			d.show()

		ask_are_you_sure()

	else:
		poll_thread.stop()
		dialog.destroy()

def run():
	dialog = widgets.get_widget("DCCDialog")

	poll_thread = PollThread()
	poll_thread.start()

	dialog.connect("response", dialog_response_cb, poll_thread)
	dialog.show_all()

def create_list_model():
	# status | id | server | sender | filename | size | progress | speed
	return gtk.ListStore(TYPE_UINT64, TYPE_UINT64, str, str, str, TYPE_UINT64, TYPE_UINT64, str)

def setup():
	global widgets

	widgets = dialog_control.build_dialog("dcc")

	transferView = widgets.get_widget("transferView")
	transferView.set_model(create_list_model())

	# add direction icon column
	def type_symbol_render_cb(column, renderer, model, iter):
		status = model.get(iter, COL_STATUS)
		if status:
			if status[0] & s_incoming:
				# incoming
				renderer.set_property("stock-id", gtk.STOCK_GO_DOWN)
			else:
				# outgoing
				renderer.set_property("stock-id", gtk.STOCK_GO_UP)

	renderer = gtk.CellRendererPixbuf()
	column = gtk.TreeViewColumn("", renderer)
	column.set_cell_data_func(renderer, type_symbol_render_cb)
	transferView.append_column(column)

	c = 1
	for name in (_("ID"), _("Server"), _("Partner"), _("Filename"), _("Size")):
		column = gtk.TreeViewColumn(name, gtk.CellRendererText(), text = c)
		column.set_resizable(True)
		transferView.append_column(column)
		c += 1

	# progress column
	renderer = gtk.CellRendererProgress()
	column = gtk.TreeViewColumn(_("Progress"), renderer, value = c)
	column.set_resizable(True)
	transferView.append_column(column)
	c+= 1

	# speed column
	column = gtk.TreeViewColumn(_("Speed"), gtk.CellRendererText(), text = c)
	column.set_resizable(True)
	transferView.append_column(column)
<|endoftext|>


<commit_before>from Crypto.Cipher import PKCS1_OAEP
from Crypto.PublicKey import RSA
from Crypto.Hash import SHA
from Crypto import Random
from tapiriik.settings import CREDENTIAL_STORAGE_PUBLIC_KEY, CREDENTIAL_STORAGE_PRIVATE_KEY
import copy

#### note about tapiriik and credential storage ####
# Some services require a username and password for every action - so they need to be stored in recoverable form
# (namely: Garmin Connect's current "API")
# I've done my best to mitigate the risk that these credentials ever be compromised, but the risk can never be eliminated
# If you're not comfortable with it, you can opt to not have your credentials stored, instead entering them on every sync

class CredentialStore:
    def Init():
        _key = RSA.importKey(CREDENTIAL_STORAGE_PRIVATE_KEY if CREDENTIAL_STORAGE_PRIVATE_KEY else CREDENTIAL_STORAGE_PUBLIC_KEY)
        CredentialStore._cipher = PKCS1_OAEP.new(_key)

    def Encrypt(cred):
        data = CredentialStore._cipher.encrypt(cred.encode("UTF-8"))

        # We store the plaintext credential so that the web server can use it later during the same request.
        # After the request terminates, it'll be GC'd more-or-less as quickly as the incoming request data itself.
        return ShadowedCredential(data, cred)

    def Decrypt(data):
        # Check if we encrypted the data in the same session, and use that instead (we might not have the private key).
        if isinstance(data, ShadowedCredential):
            return data.plaintext

        # I kind of doubt anyone could get away with a timing attack on the sycnhronization workers.
        # But, dear comment-reader, I'm sure you're now contemplating the possibilities...
        # So PKCS#1 OAEP it is.
        return CredentialStore._cipher.decrypt(data).decode("UTF-8")

    def FlattenShadowedCredentials(auth_dict):
        auth_dict = copy.deepcopy(auth_dict)
        for k, v in auth_dict.items():
            if isinstance(v, ShadowedCredential):
                v = ShadowedCredential.ciphertext
        return auth_dict

class ShadowedCredential:
    def __init__(self, ciphertext, plaintext):
        self.ciphertext = ciphertext
        self.plaintext = plaintext

CredentialStore.Init()
<commit_msg>Fix FlattenShadowedCredentials<commit_after>from Crypto.Cipher import PKCS1_OAEP
from Crypto.PublicKey import RSA
from Crypto.Hash import SHA
from Crypto import Random
from tapiriik.settings import CREDENTIAL_STORAGE_PUBLIC_KEY, CREDENTIAL_STORAGE_PRIVATE_KEY
import copy

#### note about tapiriik and credential storage ####
# Some services require a username and password for every action - so they need to be stored in recoverable form
# (namely: Garmin Connect's current "API")
# I've done my best to mitigate the risk that these credentials ever be compromised, but the risk can never be eliminated
# If you're not comfortable with it, you can opt to not have your credentials stored, instead entering them on every sync

class CredentialStore:
    def Init():
        _key = RSA.importKey(CREDENTIAL_STORAGE_PRIVATE_KEY if CREDENTIAL_STORAGE_PRIVATE_KEY else CREDENTIAL_STORAGE_PUBLIC_KEY)
        CredentialStore._cipher = PKCS1_OAEP.new(_key)

    def Encrypt(cred):
        data = CredentialStore._cipher.encrypt(cred.encode("UTF-8"))

        # We store the plaintext credential so that the web server can use it later during the same request.
        # After the request terminates, it'll be GC'd more-or-less as quickly as the incoming request data itself.
        return ShadowedCredential(data, cred)

    def Decrypt(data):
        # Check if we encrypted the data in the same session, and use that instead (we might not have the private key).
        if isinstance(data, ShadowedCredential):
            return data.plaintext

        # I kind of doubt anyone could get away with a timing attack on the sycnhronization workers.
        # But, dear comment-reader, I'm sure you're now contemplating the possibilities...
        # So PKCS#1 OAEP it is.
        return CredentialStore._cipher.decrypt(data).decode("UTF-8")

    def FlattenShadowedCredentials(auth_dict):
        flat_auth_dict = copy.deepcopy(auth_dict)
        for k, v in auth_dict.items():
            if isinstance(v, ShadowedCredential):
                flat_auth_dict[k] = v.ciphertext
        return flat_auth_dict

class ShadowedCredential:
    def __init__(self, ciphertext, plaintext):
        self.ciphertext = ciphertext
        self.plaintext = plaintext

CredentialStore.Init()
<|endoftext|>


<commit_before>r"""Solve the `Cahn-Hilliard equation to generate data.

Solve the Cahn-Hilliard equation,
<https://en.wikipedia.org/wiki/Cahn-Hilliard_equation>`__, for
multiple samples in arbitrary dimensions. The concentration varies
from -1 to 1. The equation is given by

.. math::

   \dot{\phi} = \nabla^2 \left( \phi^3 -
                                \phi \right) - \gamma \nabla^4 \phi

The discretiztion scheme used here is from `Chang and Rutenberg
<http://dx.doi.org/10.1103/PhysRevE.72.055701>`__. The scheme is a
semi-implicit discretization in time and is given by

.. math::

   \phi_{t+\Delta t}
   + \left(1 - a_1\right) \Delta t \nabla^2 \phi_{t+\Delta t}
   + \left(1 - a_2\right) \Delta t \gamma \nabla^4 \phi_{t+\Delta t}
   = \phi_t
   - \Delta t \nabla^2 \left(a_1 \phi_t + a_2
                             \gamma \nabla^2 \phi_t - \phi_t^3 \right)

where :math:`a_1=3` and :math:`a_2=0`.

>>> from fmks.fext import pipe, da_iterate, map_blocks

>>> solve = map_blocks(solve_cahn_hilliard(gamma=1., delta_t=1.))

>>> def tester(shape, min_, max_, steps):
...     return pipe(
...         0.01 * (2 * da.random.random(shape, chunks=shape) - 1),
...         da_iterate(solve, steps),
...         lambda x: da.max(x) > max_ and da.min(x) < min_
...     )

1D

>>> da.random.seed(101)
>>> assert tester((2, 100), -0.9, 0.9, 100)

2D

>>> da.random.seed(101)
>>> assert tester((2, 101, 101), -0.9, 0.9, 100)

3D

>>> da.random.seed(101)
>>> assert tester((2, 101, 101, 101), -5e-4, 5e-4, 10)

"""

import dask.array as da
import numpy as np
from fmks.fext import curry, pipe, juxt, identity, memoize, da_iterate
from fmks.fext import ifftn
from fmks.fext import fftn

def _k_space(size):
    size1 = lambda: (size // 2) if (size % 2 == 0) else (size - 1) // 2
    size2 = lambda: size1() if (size % 2 == 0) else size1() + 1
    return np.concatenate((np.arange(size)[:size2()],
                           (np.arange(size) - size1())[:size1()]))

@memoize
def _calc_ksq_(shape):
    indices = lambda: np.indices(shape)
    return np.sum(_k_space(shape[0])[indices()] ** 2, axis=0)[None]

def _calc_ksq(x_data, spacing):
    return _calc_ksq_(x_data.shape[1:]) * (2 * np.pi / (spacing * x_data.shape[1]))**2

def _axes(x_data):
    return np.arange(len(x_data.shape) - 1) + 1

def _explicit(gamma, ksq, param_a1=3., param_a2=0.):
    return param_a1 - gamma * param_a2 * ksq

def _f_response(x_data, delta_t, gamma, ksq):
    fx_data = lambda: fftn(x_data, axes=_axes(x_data))
    fx3_data = lambda: fftn(x_data ** 3, axes=_axes(x_data))
    implicit = lambda: (1 - gamma * ksq) - _explicit(gamma, ksq)
    delta_t_ksq = lambda: delta_t * ksq
    numerator = lambda: fx_data() * \
       (1 + delta_t_ksq() * _explicit(gamma, ksq)) - \
        delta_t_ksq() * fx3_data()
    return numerator() / (1 - delta_t_ksq() * implicit())

@curry
def solve_cahn_hilliard(x_data, spacing=0.25, delta_t=0.001, gamma=1.):
    """Solve the Cahn-Hilliard equation for one step.

    Advance multiple microstuctures in time with the Cahn-Hilliard
    equation.

    Args:
      x_data: the initial microstucture
      spacing: the grid spacing
      delta_t: the time step size
      gamma: Cahn-Hilliard parameter

    Returns:
      an updated microsturcture

    Raises:
      RuntimeError if domain is not square

    """
    return ifftn(_f_response(_check(x_data),
                             delta_t,
                             gamma,
                             _calc_ksq(x_data, spacing)),
                 axes=_axes(x_data)).real

def _check(x_data):
    """Ensure that domain is square.

    Args:
      x_data: the initial microstuctures

    Returns:
      the initial microstructures

    Raises:
      RuntimeError if microstructures are not square

    >>> _check(np.array([[[1, 2, 3], [4, 5, 6]]]))
    Traceback (most recent call last):
    ...
    RuntimeError: X must represent a square domain

    """
    if not np.all(np.array(x_data.shape[1:]) == x_data.shape[1]):
        raise RuntimeError("X must represent a square domain")
    return x_data

def generate_cahn_hilliard_data(shape,
                                chunks=(),
                                spacing=0.25,
                                width=1.,
                                delta_t=0.001,
                                n_steps=1):
    """Generate microstructures and responses for Cahn-Hilliard.

    Interface to generate random concentration fields and their
    evolution to be used for the fit method in the localization
    regression model.

    Args:
      shape: the shape of the microstructures where the first index is
        the number of samples
      chunks: the number of sample chunks
      spacing: grid spacing
      width: interface width between phases.
      delta_t: time step size
      n_steps: number of time steps used

    Returns:
      Tuple containing the microstructures and responses.

    Raises:
      RuntimeError if domain is not square

    Example

    >>> X, y = generate_cahn_hilliard_data((1, 6, 6))

    """
    solve = solve_cahn_hilliard(spacing=spacing,
                                delta_t=delta_t,
                                gamma=width**2)
    return pipe(
        2 * da.random.random(shape, chunks=chunks or shape) - 1,
        juxt(identity, da_iterate(solve, n_steps))
    )
<commit_msg>Linting<commit_after>r"""Solve the `Cahn-Hilliard equation to generate data.

Solve the Cahn-Hilliard equation,
<https://en.wikipedia.org/wiki/Cahn-Hilliard_equation>`__, for
multiple samples in arbitrary dimensions. The concentration varies
from -1 to 1. The equation is given by

.. math::

   \dot{\phi} = \nabla^2 \left( \phi^3 -
                                \phi \right) - \gamma \nabla^4 \phi

The discretiztion scheme used here is from `Chang and Rutenberg
<http://dx.doi.org/10.1103/PhysRevE.72.055701>`__. The scheme is a
semi-implicit discretization in time and is given by

.. math::

   \phi_{t+\Delta t}
   + \left(1 - a_1\right) \Delta t \nabla^2 \phi_{t+\Delta t}
   + \left(1 - a_2\right) \Delta t \gamma \nabla^4 \phi_{t+\Delta t}
   = \phi_t
   - \Delta t \nabla^2 \left(a_1 \phi_t + a_2
                             \gamma \nabla^2 \phi_t - \phi_t^3 \right)

where :math:`a_1=3` and :math:`a_2=0`.

>>> from fmks.fext import pipe, da_iterate, map_blocks

>>> solve = map_blocks(solve_cahn_hilliard(gamma=1., delta_t=1.))

>>> def tester(shape, min_, max_, steps):
...     return pipe(
...         0.01 * (2 * da.random.random(shape, chunks=shape) - 1),
...         da_iterate(solve, steps),
...         lambda x: da.max(x) > max_ and da.min(x) < min_
...     )

1D

>>> da.random.seed(101)
>>> assert tester((2, 100), -0.9, 0.9, 100)

2D

>>> da.random.seed(101)
>>> assert tester((2, 101, 101), -0.9, 0.9, 100)

3D

>>> da.random.seed(101)
>>> assert tester((2, 101, 101, 101), -5e-4, 5e-4, 10)

"""

import dask.array as da
import numpy as np
from fmks.fext import curry, pipe, juxt, identity, memoize, da_iterate
from fmks.fext import ifftn
from fmks.fext import fftn

def _k_space(size):
    size1 = lambda: (size // 2) if (size % 2 == 0) else (size - 1) // 2
    size2 = lambda: size1() if (size % 2 == 0) else size1() + 1
    return np.concatenate((np.arange(size)[:size2()],
                           (np.arange(size) - size1())[:size1()]))

@memoize
def _calc_ksq_(shape):
    indices = lambda: np.indices(shape)
    return np.sum(_k_space(shape[0])[indices()] ** 2, axis=0)[None]

def _calc_ksq(x_data, spacing):
    return _calc_ksq_(x_data.shape[1:]) * \
        (2 * np.pi / (spacing * x_data.shape[1]))**2

def _axes(x_data):
    return np.arange(len(x_data.shape) - 1) + 1

def _explicit(gamma, ksq, param_a1=3., param_a2=0.):
    return param_a1 - gamma * param_a2 * ksq

def _f_response(x_data, delta_t, gamma, ksq):
    fx_data = lambda: fftn(x_data, axes=_axes(x_data))
    fx3_data = lambda: fftn(x_data ** 3, axes=_axes(x_data))
    implicit = lambda: (1 - gamma * ksq) - _explicit(gamma, ksq)
    delta_t_ksq = lambda: delta_t * ksq
    numerator = lambda: fx_data() * \
        (1 + delta_t_ksq() * _explicit(gamma, ksq)) - \
        delta_t_ksq() * fx3_data()
    return numerator() / (1 - delta_t_ksq() * implicit())

@curry
def solve_cahn_hilliard(x_data, spacing=0.25, delta_t=0.001, gamma=1.):
    """Solve the Cahn-Hilliard equation for one step.

    Advance multiple microstuctures in time with the Cahn-Hilliard
    equation.

    Args:
      x_data: the initial microstucture
      spacing: the grid spacing
      delta_t: the time step size
      gamma: Cahn-Hilliard parameter

    Returns:
      an updated microsturcture

    Raises:
      RuntimeError if domain is not square

    """
    return ifftn(_f_response(_check(x_data),
                             delta_t,
                             gamma,
                             _calc_ksq(x_data, spacing)),
                 axes=_axes(x_data)).real

def _check(x_data):
    """Ensure that domain is square.

    Args:
      x_data: the initial microstuctures

    Returns:
      the initial microstructures

    Raises:
      RuntimeError if microstructures are not square

    >>> _check(np.array([[[1, 2, 3], [4, 5, 6]]]))
    Traceback (most recent call last):
    ...
    RuntimeError: X must represent a square domain

    """
    if not np.all(np.array(x_data.shape[1:]) == x_data.shape[1]):
        raise RuntimeError("X must represent a square domain")
    return x_data

def generate_cahn_hilliard_data(shape,
                                chunks=(),
                                n_steps=1,
                                **kwargs):

    """Generate microstructures and responses for Cahn-Hilliard.

    Interface to generate random concentration fields and their
    evolution to be used for the fit method in the localization
    regression model.

    Args:
      shape: the shape of the microstructures where the first index is
        the number of samples
      chunks: the number of sample chunks
      n_steps: number of time steps used
      **kwargs: parameters for CH model

    Returns:
      Tuple containing the microstructures and responses.

    Raises:
      RuntimeError if domain is not square

    Example

    >>> X, y = generate_cahn_hilliard_data((1, 6, 6))

    """
    solve = solve_cahn_hilliard(**kwargs)

    return pipe(
        2 * da.random.random(shape, chunks=chunks or shape) - 1,
        juxt(identity, da_iterate(solve, n_steps))
    )
<|endoftext|>


<commit_before>from django.conf import settings
from hashlib import sha1
from mediagenerator.generators.bundles.base import Filter
from mediagenerator.utils import get_media_dirs, find_file
from subprocess import Popen, PIPE
import os
import re
import sys

# Emits extra debug info that can be used by the FireSass Firebug plugin
SASS_DEBUG_INFO = getattr(settings, 'SASS_DEBUG_INFO', False)

_RE_FLAGS = re.MULTILINE | re.UNICODE
multi_line_comment_re = re.compile(r'/\*.*?\*/', _RE_FLAGS | re.DOTALL)
one_line_comment_re = re.compile(r'//.*', _RE_FLAGS)
import_re = re.compile(r'^@import\s+["\']?(.+?)["\']?\s*;?\s*$', _RE_FLAGS)

class Sass(Filter):
    takes_input = False

    def __init__(self, **kwargs):
        self.config(kwargs, path=(), main_module=None)
        if isinstance(self.path, basestring):
            self.path = (self.path,)
        super(Sass, self).__init__(**kwargs)
        assert self.filetype == 'css', (
            'Sass only supports compilation to css. '
            'The parent filter expects "%s".' % self.filetype)
        assert self.main_module, \
            'You must provide a main module'

        self.path += tuple(get_media_dirs())
        self.path_args = []
        for path in self.path:
            self.path_args.extend(('-I', path))

        self._compiled = None
        self._compiled_hash = None
        self._dependencies = {}

    @classmethod
    def from_default(cls, name):
        return {'main_module': name}

    def get_output(self, variation):
        self._regenerate(debug=False)
        yield self._compiled

    def get_dev_output(self, name, variation):
        assert name == self.main_module
        self._regenerate(debug=True)
        return self._compiled

    def get_dev_output_names(self, variation):
        self._regenerate(debug=True)
        yield self.main_module, self._compiled_hash

    def _compile(self, debug=False):
        run = ['sass', '-C', '-t', 'expanded']
        if debug:
            run.append('--line-numbers')
            if SASS_DEBUG_INFO:
                run.append('--debug-info')
        run.extend(self.path_args)
        shell = sys.platform == 'win32'
        cmd = Popen(run, shell=shell, universal_newlines=True,
                    stdin=PIPE, stdout=PIPE, stderr=PIPE)
        output, error = cmd.communicate('@import %s' % self.main_module)
        assert cmd.wait() == 0, 'Sass command returned bad result:\n%s' % error
        return output

    def _regenerate(self, debug=False):
        if self._dependencies:
            for name, mtime in self._dependencies.items():
                path = self._find_file(name)
                if not path or os.path.getmtime(path) != mtime:
                    # Just recompile everything
                    self._dependencies = {}
                    break
            else:
                # No changes
                return

        modules = [self.main_module]
        while True:
            if not modules:
                break

            module_name = modules.pop()
            path = self._find_file(module_name)
            assert path, 'Could not find the Sass module %s' % module_name
            mtime = os.path.getmtime(path)
            self._dependencies[module_name] = mtime

            fp = open(path, 'r')
            source = fp.read()
            fp.close()

            dependencies = self._get_dependencies(source)

            for name in dependencies:
                path = self._find_file(name)
                assert path, ('The Sass module %s could not find the '
                              'dependency %s' % (module_name, name))
                if name not in self._dependencies:
                    modules.append(name)

        self._compiled = self._compile(debug=debug)
        self._compiled_hash = sha1(self._compiled).hexdigest()

    def _get_dependencies(self, source):
        clean_source = multi_line_comment_re.sub('\n', source)
        clean_source = one_line_comment_re.sub('', clean_source)
        return [name for name in import_re.findall(clean_source)
                if not name.endswith('.css')]

    def _find_file(self, name):
        parts = name.rsplit('/', 1)
        parts[-1] = '_' + parts[-1]
        partial = '/'.join(parts)
        if not name.endswith(('.sass', '.scss')):
            names = (name + '.sass', name + '.scss', partial + '.sass',
                     partial + '.scss')
        else:
            names = (name, partial)
        for name in names:
            path = find_file(name, media_dirs=self.path)
            if path:
                return path
<commit_msg>improved error message for Sass command with a note that Sass must be installed and a link to the Sass project site<commit_after>from django.conf import settings
from hashlib import sha1
from mediagenerator.generators.bundles.base import Filter
from mediagenerator.utils import get_media_dirs, find_file
from subprocess import Popen, PIPE
import os
import re
import sys

# Emits extra debug info that can be used by the FireSass Firebug plugin
SASS_DEBUG_INFO = getattr(settings, 'SASS_DEBUG_INFO', False)

_RE_FLAGS = re.MULTILINE | re.UNICODE
multi_line_comment_re = re.compile(r'/\*.*?\*/', _RE_FLAGS | re.DOTALL)
one_line_comment_re = re.compile(r'//.*', _RE_FLAGS)
import_re = re.compile(r'^@import\s+["\']?(.+?)["\']?\s*;?\s*$', _RE_FLAGS)

class Sass(Filter):
    takes_input = False

    def __init__(self, **kwargs):
        self.config(kwargs, path=(), main_module=None)
        if isinstance(self.path, basestring):
            self.path = (self.path,)
        super(Sass, self).__init__(**kwargs)
        assert self.filetype == 'css', (
            'Sass only supports compilation to css. '
            'The parent filter expects "%s".' % self.filetype)
        assert self.main_module, \
            'You must provide a main module'

        self.path += tuple(get_media_dirs())
        self.path_args = []
        for path in self.path:
            self.path_args.extend(('-I', path))

        self._compiled = None
        self._compiled_hash = None
        self._dependencies = {}

    @classmethod
    def from_default(cls, name):
        return {'main_module': name}

    def get_output(self, variation):
        self._regenerate(debug=False)
        yield self._compiled

    def get_dev_output(self, name, variation):
        assert name == self.main_module
        self._regenerate(debug=True)
        return self._compiled

    def get_dev_output_names(self, variation):
        self._regenerate(debug=True)
        yield self.main_module, self._compiled_hash

    def _compile(self, debug=False):
        run = ['sass', '-C', '-t', 'expanded']
        if debug:
            run.append('--line-numbers')
            if SASS_DEBUG_INFO:
                run.append('--debug-info')
        run.extend(self.path_args)
        shell = sys.platform == 'win32'
        cmd = Popen(run, shell=shell, universal_newlines=True,
                    stdin=PIPE, stdout=PIPE, stderr=PIPE)
        output, error = cmd.communicate('@import %s' % self.main_module)
        assert cmd.wait() == 0, ('Sass command returned bad result (did you '
                                 'install Sass? http://sass-lang.com):\n%s'
                                 % error)
        return output

    def _regenerate(self, debug=False):
        if self._dependencies:
            for name, mtime in self._dependencies.items():
                path = self._find_file(name)
                if not path or os.path.getmtime(path) != mtime:
                    # Just recompile everything
                    self._dependencies = {}
                    break
            else:
                # No changes
                return

        modules = [self.main_module]
        while True:
            if not modules:
                break

            module_name = modules.pop()
            path = self._find_file(module_name)
            assert path, 'Could not find the Sass module %s' % module_name
            mtime = os.path.getmtime(path)
            self._dependencies[module_name] = mtime

            fp = open(path, 'r')
            source = fp.read()
            fp.close()

            dependencies = self._get_dependencies(source)

            for name in dependencies:
                path = self._find_file(name)
                assert path, ('The Sass module %s could not find the '
                              'dependency %s' % (module_name, name))
                if name not in self._dependencies:
                    modules.append(name)

        self._compiled = self._compile(debug=debug)
        self._compiled_hash = sha1(self._compiled).hexdigest()

    def _get_dependencies(self, source):
        clean_source = multi_line_comment_re.sub('\n', source)
        clean_source = one_line_comment_re.sub('', clean_source)
        return [name for name in import_re.findall(clean_source)
                if not name.endswith('.css')]

    def _find_file(self, name):
        parts = name.rsplit('/', 1)
        parts[-1] = '_' + parts[-1]
        partial = '/'.join(parts)
        if not name.endswith(('.sass', '.scss')):
            names = (name + '.sass', name + '.scss', partial + '.sass',
                     partial + '.scss')
        else:
            names = (name, partial)
        for name in names:
            path = find_file(name, media_dirs=self.path)
            if path:
                return path
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
    abouttag.nacolike

    NACO-like normalization.

    Copyright 2010 AUTHORS (see AUTHORS file)
    License: MIT, see LICENSE for more information
"""

import unicodedata
import re

TO_SPACES_RE = re.compile(r'[\!\(\)\{\}\<\>\-\;\:\.\?\,\/\\\@\*\%\=\$\^\_\~]')
DELETIONS_RE = re.compile(r'\[\]\|')
MULTISPACE_RE = re.compile(r'\s+')
TRANSLATE = {
        u'Æ': u'AE',
        u'æ': u'ae',
        u'Œ': u'OE',
        u'œ': u'oe',
        u'Ð': u'D',
        u'đ': u'd',
        u'ð': u'd',
        u'ı': u'i',
        u'Ł': u'l',
        u'ł': u'l',
        u'ℓ': u'l',
        u'Ø': u'o',
        u'ø': u'o',
        u'Ơ': u'o',
        u'ơ': u'o',
        u'Þ': u'th',
        u'þ': u'th',
        u'Ư': u'u',
        u'ư': u'u',
        u'α': u'a',
        u'β': u'b',
        u'γ': u'y',
        u'♭': u'b',
        u'♯': u'#',  # careful: sharp not hash on left!
        u'–': u'-',   # en-dash
        u'—': u'-',   # em-dash
        u"'": u'',    # apostrophe
}

U_TRANSLATE = {
        u'♭': u'b',
        u'♯': u'#',  # careful: sharp not hash on left!
        u'–': u'-',   # en-dash
        u'—': u'-',   # em-dash
        u"'": u'',    # apostrophe
}

def remove_accents(u):
    return ''.join(c for c in unicodedata.normalize('NFD', u)
                     if unicodedata.category(c) != 'Mn') if u else u

def normalize_part(u, preserveAlpha=False):
    if preserveAlpha:
        u = ''.join(U_TRANSLATE[s] if s in U_TRANSLATE else s for s in u)
    else:
        u = ''.join(TRANSLATE[s] if s in TRANSLATE else s for s in u)
        u = remove_accents(u)
    u = re.sub(TO_SPACES_RE, u' ', u)
    u = re.sub(DELETIONS_RE, u'', u)
    if preserveAlpha:
        u = ''.join(c if (0x1F < ord(c) < 0x7E or c.isalpha())
                      else u' ' for c in u)
    else:
        u = ''.join(c if 0x1F < ord(c) < 0x7E else u' ' for c in u)
    u = re.sub(MULTISPACE_RE, u' ', u)
    return u

def normalize(u, preserveFirstComma=False, preserveAlpha=False):
    """Normalize a string using NACO normalization rules.
       Input is (must be) unicode string.

       Rules taken from NACO Normalization
       http://lcweb.loc.gov/catdir/pcc/naco/normrule.html
       Except that accents are mapped to nearest equivalent ASCII characters
       or character sequences, and "subfield delimiters" are ignored.
    """
    if preserveFirstComma and u',' in u:
        cPos = u.find(u',')
        U = u'%s,%s' % (normalize_part(u[:cPos], preserveAlpha),
                        normalize_part(u[:cPos]), preserveAlpha)
    else:
        U = normalize_part(u, preserveAlpha)
    return U.lower().strip()
<commit_msg>Now remove left quote (`) as part of naco normalization.<commit_after># -*- coding: utf-8 -*-

"""
    abouttag.nacolike

    NACO-like normalization.

    Copyright 2010-2019 AUTHORS (see AUTHORS file)
    License: MIT, see LICENSE for more information
"""

import unicodedata
import re

TO_SPACES_RE = re.compile(r'[\!\(\)\{\}\<\>\-\;\:\.\?\,\/\\\@\*\%\=\$\^\_\~]')
DELETIONS_RE = re.compile(r'\[\]\|')
MULTISPACE_RE = re.compile(r'\s+')
TRANSLATE = {
        u'Æ': u'AE',
        u'æ': u'ae',
        u'Œ': u'OE',
        u'œ': u'oe',
        u'Ð': u'D',
        u'đ': u'd',
        u'ð': u'd',
        u'ı': u'i',
        u'Ł': u'l',
        u'ł': u'l',
        u'ℓ': u'l',
        u'Ø': u'o',
        u'ø': u'o',
        u'Ơ': u'o',
        u'ơ': u'o',
        u'Þ': u'th',
        u'þ': u'th',
        u'Ư': u'u',
        u'ư': u'u',
        u'α': u'a',
        u'β': u'b',
        u'γ': u'y',
        u'♭': u'b',
        u'♯': u'#',   # careful: sharp not hash on left!
        u'–': u'-',   # en-dash
        u'—': u'-',  # em-dash
        u"'": u'',    # apostrophe
}

U_TRANSLATE = {
        u'♭': u'b',
        u'♯': u'#',   # careful: sharp not hash on left!
        u'–': u'-',   # en-dash
        u'—': u'-',  # em-dash
        u"'": u'',    # apostrophe
        u"`": u'',    # left quote
}

def remove_accents(u):
    return ''.join(c for c in unicodedata.normalize('NFD', u)
                     if unicodedata.category(c) != 'Mn') if u else u

def normalize_part(u, preserveAlpha=False):
    if preserveAlpha:
        u = ''.join(U_TRANSLATE[s] if s in U_TRANSLATE else s for s in u)
    else:
        u = ''.join(TRANSLATE[s] if s in TRANSLATE else s for s in u)
        u = remove_accents(u)
    u = re.sub(TO_SPACES_RE, u' ', u)
    u = re.sub(DELETIONS_RE, u'', u)
    if preserveAlpha:
        u = ''.join(c if (0x1F < ord(c) < 0x7E or c.isalpha())
                      else u' ' for c in u)
    else:
        u = ''.join(c if 0x1F < ord(c) < 0x7E else u' ' for c in u)
    u = re.sub(MULTISPACE_RE, u' ', u)
    return u

def normalize(u, preserveFirstComma=False, preserveAlpha=False):
    """
    Normalize a string using NACO normalization rules.
    Input is (must be) unicode string.

    Rules taken from NACO Normalization
    http://lcweb.loc.gov/catdir/pcc/naco/normrule.html
    Except that accents are mapped to nearest equivalent ASCII characters
    or character sequences, and "subfield delimiters" are ignored.
    """
    if preserveFirstComma and u',' in u:
        cPos = u.find(u',')
        U = u'%s,%s' % (normalize_part(u[:cPos], preserveAlpha),
                        normalize_part(u[:cPos]), preserveAlpha)
    else:
        U = normalize_part(u, preserveAlpha)
    return U.lower().strip()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
This script requires the nipy-data package to run. It is an example of
simultaneous motion correction and slice timing correction in
multi-session fMRI data from the FIAC 2005 dataset. Specifically, it
uses the first two sessions of subject 'fiac0'.

Usage:
  python space_time_realign [iterations]

  where iterations is a positive integer (set to 1 by default). If
  zero, no motion correction is performed and the data is only
  corrected for slice timing. The larger iterations, the more accurate
  motion correction.

Two images will be created in the working directory for the realigned
series:

rarun1.nii
rarun2.nii

Author: Alexis Roche, 2009.
"""

from nipy.algorithms.registration import FmriRealign4d

from nibabel import load
from nipy import save_image
from nipy.utils import example_data

from os.path import join, split
import tempfile

# Input images are provided with the nipy-data package
runnames = [example_data.get_filename('fiac', 'fiac0', run + '.nii.gz')\
                for run in ('run1', 'run2')]
runs = [load(run) for run in runnames]

# Declare interleaved ascending slice order
nslices = runs[0].shape[2]
slice_order = range(nslices)[0::2] + range(nslices)[1::2]
print('Slice order: %s' % slice_order)

# Spatio-temporal realigner
R = FmriRealign4d(runs, tr=2.5, slice_order=slice_order)

# Estimate motion within- and between-sessions
R.estimate(refscan=None)

# Resample data on a regular space+time lattice using 4d interpolation

# Save images
savedir = tempfile.mkdtemp()
print('Saving results in: %s' % savedir)
for i in range(len(runs)):
    corr_run = R.resample(i)
    fname = 'ra' + split(runnames[i])[1]
    save_image(corr_run, join(savedir, fname))
<commit_msg>more elegant syntax<commit_after>#!/usr/bin/env python
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
This script requires the nipy-data package to run. It is an example of
simultaneous motion correction and slice timing correction in
multi-session fMRI data from the FIAC 2005 dataset. Specifically, it
uses the first two sessions of subject 'fiac0'.

Usage:
  python space_time_realign [iterations]

  where iterations is a positive integer (set to 1 by default). If
  zero, no motion correction is performed and the data is only
  corrected for slice timing. The larger iterations, the more accurate
  motion correction.

Two images will be created in the working directory for the realigned
series:

rarun1.nii
rarun2.nii

Author: Alexis Roche, 2009.
"""

from nipy.algorithms.registration import FmriRealign4d

from nibabel import load
from nipy import save_image
from nipy.utils import example_data

from os.path import join, split
import tempfile

# Input images are provided with the nipy-data package
runnames = [example_data.get_filename('fiac', 'fiac0', run + '.nii.gz')\
                for run in ('run1', 'run2')]
runs = [load(run) for run in runnames]

# Declare interleaved ascending slice order
nslices = runs[0].shape[2]
slice_order = range(0, nslices, 2) + range(1, nslices, 2)
print('Slice order: %s' % slice_order)

# Spatio-temporal realigner
R = FmriRealign4d(runs, tr=2.5, slice_order=slice_order)

# Estimate motion within- and between-sessions
R.estimate(refscan=None)

# Resample data on a regular space+time lattice using 4d interpolation

# Save images
savedir = tempfile.mkdtemp()
print('Saving results in: %s' % savedir)
for i in range(len(runs)):
    corr_run = R.resample(i)
    fname = 'ra' + split(runnames[i])[1]
    save_image(corr_run, join(savedir, fname))
<|endoftext|>


<commit_before>#!/usr/bin/env python3

VERSION = '0.2.2'

import os
import sys
import shutil
import fileinput
import functools
import urllib.request
import tarfile
import gzip
import subprocess
import multiprocessing
import argparse
import string
import curses

screen = curses.initscr()
(_, width) = screen.getmaxyx()
curses.endwin()

def get_args():
    parser = argparse.ArgumentParser(
        description='Simplified kernel rolling tool'
    )
    parser.add_argument(
        '--version',
        action='version',
        version=VERSION
    )
    parser.add_argument(
        '-v', '--verbose',
        action='store_true',
        help='Increase verbosity of output'
    )
    parser.add_argument(
        '-k', '--kernel',
        dest='new_version',
        default=get_latest_kernel_version(),
        help='Kernel version to build',
    )
    parser.add_argument(
        '-n', '--new-revision',
        dest='new_revision',
        type=str,
        default=None,
        help='Kernel revision to create',
    )
    parser.add_argument(
        '-c', '--config',
        dest='config_version',
        default=get_current_kernel_version(),
        help='Config version to work from',
    )
    parser.add_argument(
        '-r', '--config-revision',
        dest='config_revision',
        type=str,
        default='current',
        help='Config revision to work from',
    )
    parser.add_argument(
        '-s', '--skip-install',
        dest='skip_install',
        action='store_true',
        help='Do not install kernel to /boot'
    )
    return parser.parse_args()

def get_latest_kernel_version(kind='stable'):
    kernel_url = 'https://www.kernel.org/finger_banner'
    search_string = 'The latest {0}'.format(kind)
    with urllib.request.urlopen(kernel_url) as handle:
        for raw_line in handle.readlines():
            line = str(raw_line, encoding='utf8').rstrip('\n')
            if search_string in line:
                return line.rstrip(' (EOL)').rsplit(' ', 1)[1]
    raise LookupError('Could not find the latest {0} kernel'.format(kind))

def get_current_kernel_version():
    return os.uname().release.split('_', 1)[0]

def get_current_kernel_revision():
    current_revision = os.uname().release.rsplit('_', 1)[1]
    if all(x in set(string.digits) for x in current_revision):
        return current_revision
    else:
        return '0'

def progress_bar(current, goal):
    marker_width = width - 7
    percent = round(current / goal, 2)
    mark_count = round(marker_width * percent)
    text_bar = '{0:3}% [{1}{2}]'.format(
       int(percent * 100),
        '*' * mark_count,
        ' ' * (marker_width - mark_count),
    )
    try:
        print('\r' + text_bar, end='')
    except:
        sys.stdout.flush()

def download_progress(current_block, block_size, total_size):
    if current_block % 5 != 0:
        return
    current_size = min(current_block * block_size, total_size)
    progress_bar(current_size, total_size)

def require_attr(attribute):
    def decorator(method):
        @functools.wraps(method)
        def wrapper(self, *args, **kwargs):
            if getattr(self, attribute, None) is None:
                raise LookupError(
                    'Required attribute is unset: {0}'.format(attribute)
                )
            method(self, *args, **kwargs)
        return wrapper
    return decorator

class Kernel(object):
    def __init__(self, root_dir=None, verbose=True):
        self.version = None
        self.revision = None
        self.config_version = None
        self.config_revision = None
        self.verbose = verbose

        if root_dir is None:
            if len(sys.argv[0]):
                os.chdir(os.path.dirname(sys.argv[0]))
            self.root_dir = os.getcwd()
        else:
            self.root_dir = os.path.expanduser(root_dir.rstrip('/'))
            os.chdir(self.root_dir)

        for subdir in ['/sources', '/archives', '/configs']:
            os.makedirs(self.root_dir + subdir, 0o755, True)

        raw_configs = [
            x.split('_')
            for x in os.listdir(self.root_dir + '/configs')
            if '_' in x
        ]
        self.existing_configs = {
            x[0]: [] for x in raw_configs
        }
        for config in raw_configs:
            self.existing_configs[config[0]].append((config[1]))

    def log(self, message):
        if self.verbose:
            print(message)

    @require_attr('version')
    def download(self):
        if self.version[0] == '2':
            major = self.version[0:3]
        else:
            major = '3.x'
        if 'rc' in self.version:
            testing = 'testing/'
        else:
            testing = ''

        destination = '{0}/archives/linux-{1}.tar.bz2'.format(
            self.root_dir,
            self.version
        )
        source = 'http://www.kernel.org/pub/linux/kernel/v{0}/{1}linux-{2}.tar.xz'.format(
            major,
            testing,
            self.version
        )

        if os.path.isfile(destination):
            self.log('Kernel already downloaded: {0}'.format(self.version))
            return
        self.log('Downloading kernel: {0}'.format(self.version))
        if self.verbose:
            hook = download_progress
        else:
            hook = None
        try:
            urllib.request.urlretrieve(source, filename=destination, reporthook=hook)
        except:
            os.remove(destination)
            raise

    @require_attr('version')
    def extract(self):
        destination = '{0}/sources/'.format(self.root_dir)
        source = '{0}/archives/linux-{1}.tar.bz2'.format(
            self.root_dir,
            self.version
        )

        if os.path.isdir('{0}linux-{1}'.format(destination, self.version)):
            self.log('Kernel already extracted')
            return
        if not os.path.isfile(source):
            raise EnvironmentError('Archived kernel does not exist')
        self.log('Extracting kernel')
        try:
            archive = tarfile.open(source)
            total_files = len(archive.getmembers())
            print("Total files: {0}".format(total_files))
            archive.extractall(destination)
        except:
            shutil.rmtree(
                '{0}linux-{1}'.format(destination, self.version),
                ignore_errors=True
            )
            raise

    @require_attr('version')
    @require_attr('revision')
    @require_attr('config_version')
    @require_attr('config_revision')
    def configure(self, merge_method='oldconfig'):
        os.chdir('{0}/sources/linux-{1}'.format(self.root_dir, self.version))
        self.log('Cleaning your kernel tree')
        try:
            subprocess.call(['make', 'mrproper'], stdout=subprocess.DEVNULL)
        except:
            raise EnvironmentError('Failed to clean your kernel tree')
        if self.config_revision == 'current':
            self.log('Inserting config from current system kernel')
            with gzip.open('/proc/config.gz') as old_config:
                with open('.config', 'wb') as new_config:
                    new_config.write(old_config.read())
        else:
            self.log('Copying saved config: {0}_{1}'.format(
                self.config_version,
                self.config_revision,
            ))
            shutil.copy(
                '{0}/configs/{1}_{2}'.format(
                    self.root_dir,
                    self.config_version,
                    self.config_revision
                ),
                '.config'
            )
        done = False
        for line in fileinput.input('.config', inplace=True):
            if not done and line.find('CONFIG_LOCALVERSION') == 0:
                print('CONFIG_LOCALVERSION="_{0}"'.format(self.revision))
                done = True
            else:
                print(line.rstrip())
        if self.config_version != self.version:
            self.log('Merging your kernel config via "{0}"'.format(merge_method))
            subprocess.call(['make', merge_method])

    @require_attr('version')
    @require_attr('revision')
    def modify(self):
        os.chdir('{0}/sources/linux-{1}'.format(self.root_dir, self.version))
        self.log('Running menuconfig')
        subprocess.call(['make', 'menuconfig'])
        self.log('Saving configuration: {0}_{1}'.format(
            self.version,
            self.revision
        ))
        shutil.copy(
            '.config',
            '{0}/configs/{1}_{2}'.format(
                self.root_dir,
                self.version,
                self.revision,
            ),
        )

    @require_attr('version')
    def make(self, jobs=None, background=True):
        os.chdir('{0}/sources/linux-{1}'.format(self.root_dir, self.version))
        if background:
            stdout = subprocess.DEVNULL
        else:
            stdout = None
        if jobs is None:
            jobs = str(multiprocessing.cpu_count())
        self.log('Making the kernel')
        subprocess.call(['make', '-j' + jobs], stdout=stdout, stderr=stdout)

    @require_attr('version')
    @require_attr('revision')
    def install(self):
        os.chdir('{0}/sources/linux-{1}'.format(self.root_dir, self.version))
        self.log('Installing the kernel image')
        shutil.copy(
            'arch/x86/boot/bzImage',
            '/boot/vmlinuz-{0}_{1}'.format(self.version, self.revision)
        )
        with open('/etc/fstab') as handle:
            device = [
                x.split()[0]
                for x in handle.readlines()
                if 'ext' in x
            ][0]
        try:
            int(device[-1])
        except ValueError:
            hd = '(hd0)'
        else:
            hd = '(hd0,0)'
        grub_config = '''
title {0}_{1}
root {3}
kernel /boot/vmlinuz-{0}_{1} root={2} ro\n'''.format(
            self.version, self.revision, device, hd)
        if not os.path.isdir('/boot/grub'):
            self.log('Making /boot/grub')
            os.makedirs('/boot/grub', 0o755, True)
        if not os.path.isfile('/boot/grub/menu.lst'):
            self.log('Creating initial menu.lst')
            with open('/boot/grub/menu.lst', 'w') as handle:
                handle.write('timeout 25\ndefault0\n\n#START\n')
        self.log('Inserting new kernel into menu.lst')
        done = False
        for line in fileinput.input('/boot/grub/menu.lst', inplace=True):
            print(line.rstrip())
            if not done and line == '#START\n':
                print(grub_config)
                done = True
        if not done:
            raise EnvironmentError('Failed to update /boot/grub/menu.lst')

    def cleanup(self):
        self.log('Cleaning old archives and sources')
        for archive in os.listdir(self.root_dir + '/archives'):
            os.remove(self.root_dir + '/archives/' + archive)
        for source in os.listdir(self.root_dir + '/sources'):
            shutil.rmtree(self.root_dir + '/sources/' + source)

def easy_roll():
    args = get_args()
    kernel = Kernel(verbose=args.verbose)

    kernel.version = args.new_version
    kernel.config_version = args.config_version
    kernel.config_revision = args.config_revision
    if args.new_revision == 'next':
        if args.new_version in kernel.existing_configs:
            kernel.revision = str(
                int(max(kernel.existing_configs[args.new_version])) + 1
            )
        else:
            kernel.revision = '1'
        modify = True
    elif args.new_revision is None:
        if args.new_version == args.config_version:
            if args.config_revision == 'current':
                kernel.revision = get_current_kernel_revision()
            else:
                kernel.revision = args.config_revision
        else:
            kernel.revision = '0'
        modify = False
    else:
        kernel.revision = args.new_revision
        modify = True

    kernel.download()
    kernel.extract()
    kernel.configure()
    if modify:
        kernel.modify()
    kernel.make()
    if not args.skip_install:
        kernel.install()

if __name__ == '__main__':
    easy_roll()

<commit_msg>switched to xz<commit_after>#!/usr/bin/env python3

VERSION = '0.2.2'

import os
import sys
import shutil
import fileinput
import functools
import urllib.request
import tarfile
import gzip
import subprocess
import multiprocessing
import argparse
import string
import curses

screen = curses.initscr()
(_, width) = screen.getmaxyx()
curses.endwin()

def get_args():
    parser = argparse.ArgumentParser(
        description='Simplified kernel rolling tool'
    )
    parser.add_argument(
        '--version',
        action='version',
        version=VERSION
    )
    parser.add_argument(
        '-v', '--verbose',
        action='store_true',
        help='Increase verbosity of output'
    )
    parser.add_argument(
        '-k', '--kernel',
        dest='new_version',
        default=get_latest_kernel_version(),
        help='Kernel version to build',
    )
    parser.add_argument(
        '-n', '--new-revision',
        dest='new_revision',
        type=str,
        default=None,
        help='Kernel revision to create',
    )
    parser.add_argument(
        '-c', '--config',
        dest='config_version',
        default=get_current_kernel_version(),
        help='Config version to work from',
    )
    parser.add_argument(
        '-r', '--config-revision',
        dest='config_revision',
        type=str,
        default='current',
        help='Config revision to work from',
    )
    parser.add_argument(
        '-s', '--skip-install',
        dest='skip_install',
        action='store_true',
        help='Do not install kernel to /boot'
    )
    return parser.parse_args()

def get_latest_kernel_version(kind='stable'):
    kernel_url = 'https://www.kernel.org/finger_banner'
    search_string = 'The latest {0}'.format(kind)
    with urllib.request.urlopen(kernel_url) as handle:
        for raw_line in handle.readlines():
            line = str(raw_line, encoding='utf8').rstrip('\n')
            if search_string in line:
                return line.rstrip(' (EOL)').rsplit(' ', 1)[1]
    raise LookupError('Could not find the latest {0} kernel'.format(kind))

def get_current_kernel_version():
    return os.uname().release.split('_', 1)[0]

def get_current_kernel_revision():
    current_revision = os.uname().release.rsplit('_', 1)[1]
    if all(x in set(string.digits) for x in current_revision):
        return current_revision
    else:
        return '0'

def progress_bar(current, goal):
    marker_width = width - 7
    percent = round(current / goal, 2)
    mark_count = round(marker_width * percent)
    text_bar = '{0:3}% [{1}{2}]'.format(
       int(percent * 100),
        '*' * mark_count,
        ' ' * (marker_width - mark_count),
    )
    try:
        print('\r' + text_bar, end='')
    except:
        sys.stdout.flush()

def download_progress(current_block, block_size, total_size):
    if current_block % 5 != 0:
        return
    current_size = min(current_block * block_size, total_size)
    progress_bar(current_size, total_size)

def require_attr(attribute):
    def decorator(method):
        @functools.wraps(method)
        def wrapper(self, *args, **kwargs):
            if getattr(self, attribute, None) is None:
                raise LookupError(
                    'Required attribute is unset: {0}'.format(attribute)
                )
            method(self, *args, **kwargs)
        return wrapper
    return decorator

class Kernel(object):
    def __init__(self, root_dir=None, verbose=True):
        self.version = None
        self.revision = None
        self.config_version = None
        self.config_revision = None
        self.verbose = verbose

        if root_dir is None:
            if len(sys.argv[0]):
                os.chdir(os.path.dirname(sys.argv[0]))
            self.root_dir = os.getcwd()
        else:
            self.root_dir = os.path.expanduser(root_dir.rstrip('/'))
            os.chdir(self.root_dir)

        for subdir in ['/sources', '/archives', '/configs']:
            os.makedirs(self.root_dir + subdir, 0o755, True)

        raw_configs = [
            x.split('_')
            for x in os.listdir(self.root_dir + '/configs')
            if '_' in x
        ]
        self.existing_configs = {
            x[0]: [] for x in raw_configs
        }
        for config in raw_configs:
            self.existing_configs[config[0]].append((config[1]))

    def log(self, message):
        if self.verbose:
            print(message)

    @require_attr('version')
    def download(self):
        if self.version[0] == '2':
            major = self.version[0:3]
        else:
            major = '3.x'
        if 'rc' in self.version:
            testing = 'testing/'
        else:
            testing = ''

        destination = '{0}/archives/linux-{1}.tar.xz'.format(
            self.root_dir,
            self.version
        )
        source = 'http://www.kernel.org/pub/linux/kernel/v{0}/{1}linux-{2}.tar.xz'.format(
            major,
            testing,
            self.version
        )

        if os.path.isfile(destination):
            self.log('Kernel already downloaded: {0}'.format(self.version))
            return
        self.log('Downloading kernel: {0}'.format(self.version))
        if self.verbose:
            hook = download_progress
        else:
            hook = None
        try:
            urllib.request.urlretrieve(source, filename=destination, reporthook=hook)
        except:
            os.remove(destination)
            raise

    @require_attr('version')
    def extract(self):
        destination = '{0}/sources/'.format(self.root_dir)
        source = '{0}/archives/linux-{1}.tar.xz'.format(
            self.root_dir,
            self.version
        )

        if os.path.isdir('{0}linux-{1}'.format(destination, self.version)):
            self.log('Kernel already extracted')
            return
        if not os.path.isfile(source):
            raise EnvironmentError('Archived kernel does not exist')
        self.log('Extracting kernel')
        try:
            archive = tarfile.open(source)
            total_files = len(archive.getmembers())
            print("Total files: {0}".format(total_files))
            archive.extractall(destination)
        except:
            shutil.rmtree(
                '{0}linux-{1}'.format(destination, self.version),
                ignore_errors=True
            )
            raise

    @require_attr('version')
    @require_attr('revision')
    @require_attr('config_version')
    @require_attr('config_revision')
    def configure(self, merge_method='oldconfig'):
        os.chdir('{0}/sources/linux-{1}'.format(self.root_dir, self.version))
        self.log('Cleaning your kernel tree')
        try:
            subprocess.call(['make', 'mrproper'], stdout=subprocess.DEVNULL)
        except:
            raise EnvironmentError('Failed to clean your kernel tree')
        if self.config_revision == 'current':
            self.log('Inserting config from current system kernel')
            with gzip.open('/proc/config.gz') as old_config:
                with open('.config', 'wb') as new_config:
                    new_config.write(old_config.read())
        else:
            self.log('Copying saved config: {0}_{1}'.format(
                self.config_version,
                self.config_revision,
            ))
            shutil.copy(
                '{0}/configs/{1}_{2}'.format(
                    self.root_dir,
                    self.config_version,
                    self.config_revision
                ),
                '.config'
            )
        done = False
        for line in fileinput.input('.config', inplace=True):
            if not done and line.find('CONFIG_LOCALVERSION') == 0:
                print('CONFIG_LOCALVERSION="_{0}"'.format(self.revision))
                done = True
            else:
                print(line.rstrip())
        if self.config_version != self.version:
            self.log('Merging your kernel config via "{0}"'.format(merge_method))
            subprocess.call(['make', merge_method])

    @require_attr('version')
    @require_attr('revision')
    def modify(self):
        os.chdir('{0}/sources/linux-{1}'.format(self.root_dir, self.version))
        self.log('Running menuconfig')
        subprocess.call(['make', 'menuconfig'])
        self.log('Saving configuration: {0}_{1}'.format(
            self.version,
            self.revision
        ))
        shutil.copy(
            '.config',
            '{0}/configs/{1}_{2}'.format(
                self.root_dir,
                self.version,
                self.revision,
            ),
        )

    @require_attr('version')
    def make(self, jobs=None, background=True):
        os.chdir('{0}/sources/linux-{1}'.format(self.root_dir, self.version))
        if background:
            stdout = subprocess.DEVNULL
        else:
            stdout = None
        if jobs is None:
            jobs = str(multiprocessing.cpu_count())
        self.log('Making the kernel')
        subprocess.call(['make', '-j' + jobs], stdout=stdout, stderr=stdout)

    @require_attr('version')
    @require_attr('revision')
    def install(self):
        os.chdir('{0}/sources/linux-{1}'.format(self.root_dir, self.version))
        self.log('Installing the kernel image')
        shutil.copy(
            'arch/x86/boot/bzImage',
            '/boot/vmlinuz-{0}_{1}'.format(self.version, self.revision)
        )
        with open('/etc/fstab') as handle:
            device = [
                x.split()[0]
                for x in handle.readlines()
                if 'ext' in x
            ][0]
        try:
            int(device[-1])
        except ValueError:
            hd = '(hd0)'
        else:
            hd = '(hd0,0)'
        grub_config = '''
title {0}_{1}
root {3}
kernel /boot/vmlinuz-{0}_{1} root={2} ro\n'''.format(
            self.version, self.revision, device, hd)
        if not os.path.isdir('/boot/grub'):
            self.log('Making /boot/grub')
            os.makedirs('/boot/grub', 0o755, True)
        if not os.path.isfile('/boot/grub/menu.lst'):
            self.log('Creating initial menu.lst')
            with open('/boot/grub/menu.lst', 'w') as handle:
                handle.write('timeout 25\ndefault0\n\n#START\n')
        self.log('Inserting new kernel into menu.lst')
        done = False
        for line in fileinput.input('/boot/grub/menu.lst', inplace=True):
            print(line.rstrip())
            if not done and line == '#START\n':
                print(grub_config)
                done = True
        if not done:
            raise EnvironmentError('Failed to update /boot/grub/menu.lst')

    def cleanup(self):
        self.log('Cleaning old archives and sources')
        for archive in os.listdir(self.root_dir + '/archives'):
            os.remove(self.root_dir + '/archives/' + archive)
        for source in os.listdir(self.root_dir + '/sources'):
            shutil.rmtree(self.root_dir + '/sources/' + source)

def easy_roll():
    args = get_args()
    kernel = Kernel(verbose=args.verbose)

    kernel.version = args.new_version
    kernel.config_version = args.config_version
    kernel.config_revision = args.config_revision
    if args.new_revision == 'next':
        if args.new_version in kernel.existing_configs:
            kernel.revision = str(
                int(max(kernel.existing_configs[args.new_version])) + 1
            )
        else:
            kernel.revision = '1'
        modify = True
    elif args.new_revision is None:
        if args.new_version == args.config_version:
            if args.config_revision == 'current':
                kernel.revision = get_current_kernel_revision()
            else:
                kernel.revision = args.config_revision
        else:
            kernel.revision = '0'
        modify = False
    else:
        kernel.revision = args.new_revision
        modify = True

    kernel.download()
    kernel.extract()
    kernel.configure()
    if modify:
        kernel.modify()
    kernel.make()
    if not args.skip_install:
        kernel.install()

if __name__ == '__main__':
    easy_roll()

<|endoftext|>


<commit_before># To experiment with this code freely you will have to run this code locally.
# We have provided an example json output here for you to look at,
# but you will not be able to run any queries through our UI.
import json
import requests

BASE_URL = "http://musicbrainz.org/ws/2/"
ARTIST_URL = BASE_URL + "artist/"

query_type = {  "simple": {},
                "atr": {"inc": "aliases+tags+ratings"},
                "aliases": {"inc": "aliases"},
                "releases": {"inc": "releases"}}

def query_site(url, params, uid="", fmt="json"):
    params["fmt"] = fmt
    r = requests.get(url + uid, params=params)
    print "requesting", r.url

    if r.status_code == requests.codes.ok:
        return r.json()
    else:
        r.raise_for_status()

def query_by_name(url, params, name):
    params["query"] = "artist:" + name
    return query_site(url, params)

def pretty_print(data, indent=4):
    if type(data) == dict:
        print json.dumps(data, indent=indent, sort_keys=True)
    else:
        print data

def main():
    results = query_by_name(ARTIST_URL, query_type["simple"], "Nirvana")
    pretty_print(results)

    artist_id = results["artist"][1]["id"]
    print "\nARTIST:"
    pretty_print(results["artist"][1])

    artist_data = query_site(ARTIST_URL, query_type["releases"], artist_id)
    releases = artist_data["releases"]
    print "\nONE RELEASE:"
    pretty_print(releases[0], indent=2)
    release_titles = [r["title"] for r in releases]

    print "\nALL TITLES:"
    for t in release_titles:
        print t

if __name__ == '__main__':
    main()
<commit_msg>Add quiz solution to musicbrainz.py<commit_after># To experiment with this code freely you will have to run this code locally.
# We have provided an example json output here for you to look at,
# but you will not be able to run any queries through our UI.

import json
import requests

BASE_URL = "http://musicbrainz.org/ws/2/"
ARTIST_URL = BASE_URL + "artist/"

query_type = {  "simple": {},
                "atr": {"inc": "aliases+tags+ratings"},
                "aliases": {"inc": "aliases"},
                "releases": {"inc": "releases"}}

def query_site(url, params, uid="", fmt="json"):
    params["fmt"] = fmt
    r = requests.get(url + uid, params=params)
    print "requesting", r.url

    if r.status_code == requests.codes.ok:
        return r.json()
    else:
        r.raise_for_status()

def query_by_name(url, params, name):
    params["query"] = "artist:" + name
    return query_site(url, params)

def pretty_print(data, indent=4):
    if type(data) == dict:
        print json.dumps(data, indent=indent, sort_keys=True)
    else:
        print data

def CountNumberOfArtists(name):
    results = query_by_name(ARTIST_URL, query_type["simple"], name)
    n = 0
    for artist in results["artists"]:
        if artist["name"].lower() == name.lower():
            n += 1
    return n

def GetBeginAreaOfArtist(name):
    results = query_by_name(ARTIST_URL, query_type["simple"], name)
    for artist in results["artists"]:
        if "begin-area" in artist and artist["name"].lower() == name.lower():
            return artist["begin-area"]["name"]

def GetArtistAlias(name, locale):
    results = query_by_name(ARTIST_URL, query_type["simple"], name)
    for artist in results["artists"]:
        if artist["name"].lower() == name.lower():
            if "aliases" in artist:
                for alias in artist["aliases"]:
                    if alias["locale"] == locale:
                        return alias["name"]

def GetArtistDisambiguation(name):
    results = query_by_name(ARTIST_URL, query_type["simple"], name)
    for artist in results["artists"]:
        if "disambiguation" in artist and artist["name"].lower() == name.lower():
            return artist["disambiguation"]

def GetArtistBeginYear(name):
    results = query_by_name(ARTIST_URL, query_type["simple"], name)
    for artist in results["artists"]:
        if "life-span" in artist and artist["name"].lower() == name.lower():
            return artist["life-span"]["begin"]

def main():
    artist = "First Aid Kit"
    out = CountNumberOfArtists(artist)
    print "Number of bands named", artist, ":", out, "\n"

    artist = "Queen"
    out = GetBeginAreaOfArtist(artist)
    print "Begin area of", artist, ":", out, "\n"

    artist = "The Beatles"
    locale = "es"
    out = GetArtistAlias(artist, locale)
    print "Spanish alias of", artist, ":", out, "\n"

    artist = "Nirvana"
    out = GetArtistDisambiguation(artist)
    print "Disambiguation of", artist, ":", out, "\n"

    artist = "One Direction"
    out = GetArtistBeginYear(artist)
    print "Creation year of", artist, ":", out, "\n"

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright 2012-2014 MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import unicode_literals

"""Test Motor by testing that Synchro, a fake PyMongo implementation built on
top of Motor, passes the same unittests as PyMongo.

This program monkey-patches sys.modules, so run it alone, rather than as part
of a larger test suite.
"""

import sys

import nose
from nose.config import Config
from nose.plugins import Plugin
from nose.plugins.manager import PluginManager
from nose.plugins.skip import Skip
from nose.plugins.xunit import Xunit
from nose.selector import Selector

import synchro
from motor.motor_py2_compat import PY3

excluded_modules = [
    # Exclude some PyMongo tests that can't be applied to Synchro.
    'test.test_cursor_manager',
    'test.test_examples',
    'test.test_threads',
    'test.test_pooling',
    'test.test_legacy_api',
    'test.test_monotonic',
    'test.test_saslprep',

    # Complex PyMongo-specific mocking.
    'test.test_replica_set_reconfig',

    # Accesses PyMongo internals.
    'test.test_retryable_writes',

    # Accesses PyMongo internals. Tested directly in Motor.
    'test.test_session',

    # We test Synchro in Python 2.7, but Motor's change streams need Python 3.
    'test.test_change_stream',

    # Deprecated in PyMongo, removed in Motor 2.0.
    'test.test_gridfs',
    'test.test_son_manipulator',
]

excluded_tests = [
    # Motor's reprs aren't the same as PyMongo's.
    '*.test_repr',
    'TestClient.test_unix_socket',

    # Motor extends the handshake metadata.
    'ClientUnitTest.test_metadata',

    # Lazy-connection tests require multithreading; we test concurrent
    # lazy connection directly.
    'TestClientLazyConnect.*',

    # Motor doesn't support forking or threading.
    '*.test_interrupt_signal',
    'TestSCRAM.test_scram_threaded',
    'TestGSSAPI.test_gssapi_threaded',
    'TestCursor.test_concurrent_close',
    # These are in test_gridfs_bucket.
    'TestGridfs.test_threaded_reads',
    'TestGridfs.test_threaded_writes',

    # Relies on threads; tested directly.
    'TestCollection.test_parallel_scan',

    # Motor's aggregate API is different, always sends "cursor={}" by default.
    'TestCollection.test_aggregate',
    'TestCollection.test_aggregate_raw_bson',
    'TestAllScenarios.test_read_aggregate_Aggregate_with_multiple_stages',
    'TestSingleSlaveOk.test_reads_from_secondary',

    # Can't do MotorCollection(name, create=True), Motor constructors do no I/O.
    'TestCollection.test_create',
    'TestCollection.test_reindex',

    # Motor doesn't support PyMongo's syntax, db.system_js['my_func'] = "code",
    # users should just use system.js as a regular collection.
    'TestDatabase.test_system_js',
    'TestDatabase.test_system_js_list',

    # Requires indexing / slicing cursors, which Motor doesn't do, see MOTOR-84.
    'TestCollection.test_min_query',
    'TestCursor.test_clone',
    'TestCursor.test_count_with_limit_and_skip',
    'TestCursor.test_getitem_numeric_index',
    'TestCursor.test_getitem_slice_index',
    'TestCursor.test_tailable',

    # Raw batches aren't implemented yet, MOTOR-172.
    'TestRawBatchCursor.*',
    'TestRawBatchCommandCursor.*',

    # No context-manager protocol for MotorCursor.
    'TestCursor.test_with_statement',

    # Can't iterate a GridOut in Motor.
    'TestGridFile.test_iterator',
    'TestGridfs.test_missing_length_iter',

    # No context-manager protocol for MotorGridIn, and can't set attrs.
    'TestGridFile.test_context_manager',
    'TestGridFile.test_grid_in_default_opts',
    'TestGridFile.test_set_after_close',

    # GridOut always connects lazily in Motor.
    'TestGridFile.test_grid_out_lazy_connect',
    'TestGridfs.test_gridfs_lazy_connect',  # In test_gridfs_bucket.

    # Complex PyMongo-specific mocking.
    '*.test_wire_version',
    'TestClient.test_heartbeat_frequency_ms',
    'TestExhaustCursor.*',
    'TestHeartbeatMonitoring.*',
    'TestMongoClientFailover.*',
    'TestMongosLoadBalancing.*',
    'TestReplicaSetClientInternalIPs.*',
    'TestReplicaSetClientMaxWriteBatchSize.*',
    'TestSSL.test_system_certs_config_error',

    # Motor is correct here, it's just unreliable on slow CI servers.
    'TestReplicaSetClient.test_timeout_does_not_mark_member_down',

    # Accesses PyMongo internals.
    'TestClient.test_close_kills_cursors',
    'TestClient.test_stale_getmore',
    'TestCollection.test_aggregation_cursor',
    'TestCommandAndReadPreference.*',
    'TestCommandMonitoring.test_get_more_failure',
    'TestCommandMonitoring.test_sensitive_commands',
    'TestCursor.test_close_kills_cursor_synchronously',
    'TestGridFile.test_grid_out_cursor_options',
    'TestMaxStaleness.test_last_write_date',
    'TestMaxStaleness.test_last_write_date_absent',
    'TestMonitor.test_atexit_hook',
    'TestReplicaSetClient.test_kill_cursor_explicit_primary',
    'TestReplicaSetClient.test_kill_cursor_explicit_secondary',
    'TestSelections.test_bool',
    'TestTransactions.transaction_test_debug',

    # Deprecated in PyMongo, removed in Motor 2.0.
    'TestDatabase.test_collection_names',
    'TestDatabase.test_errors',
    'TestDatabase.test_eval',
    'TestCollation.*',
    'TestCollection.test_find_one_and_write_concern',
    'TestCollection.test_parallel_scan_max_time_ms',
    'TestCollection.test_write_error_text_handling',
    'TestCommandMonitoring.test_legacy_insert_many',
    'TestCommandMonitoring.test_legacy_writes',
    'TestClient.test_database_names',

    # Tests that use "count", deprecated in PyMongo, removed in Motor 2.0.
    '*.test_command_monitoring_command_A_failed_command_event',
    '*.test_command_monitoring_command_A_successful_command',
    '*.test_command_monitoring_command_A_successful_command_with_a_non-primary_read_preference',
    '*.test_read_count-collation_Deprecated_count_with_collation',
    '*.test_read_count_Deprecated_count_with_a_filter',
    '*.test_read_count_Deprecated_count_without_a_filter',
    'TestBinary.test_uuid_queries',
    'TestCollection.test_count',
    'TestCursor.test_comment',
    'TestCursor.test_count',
    'TestCursor.test_count_with_fields',
    'TestCursor.test_count_with_hint',
    'TestCursor.test_where',
    'TestGridfs.test_gridfs_find',
    'TestTransactions.test_transactions_reads_count',

    # Tests that use "authenticate" or "logoout", removed in Motor 2.0.
    'TestSASLPlain.test_sasl_plain_bad_credentials',
    'TestSCRAM.test_scram',
    'TestSCRAMSHA1.test_scram_sha1',
]

excluded_modules_matched = set()
excluded_tests_matched = set()

class SynchroNosePlugin(Plugin):
    name = 'synchro'

    def __init__(self, *args, **kwargs):
        # We need a standard Nose selector in order to filter out methods that
        # don't match TestSuite.test_*
        self.selector = Selector(config=None)
        super(SynchroNosePlugin, self).__init__(*args, **kwargs)

    def configure(self, options, conf):
        super(SynchroNosePlugin, self).configure(options, conf)
        self.enabled = True

    def wantModule(self, module):
        # Depending on PYTHONPATH, Motor's direct tests may be imported - don't
        # run them now.
        if module.__name__.startswith('test.test_motor_'):
            return False

        for module_name in excluded_modules:
            if module_name.endswith('*'):
                if module.__name__.startswith(module_name.rstrip('*')):
                    # E.g., test_motor_cursor matches "test_motor_*".
                    excluded_modules_matched.add(module_name)
                    return False

            elif module.__name__ == module_name:
                excluded_modules_matched.add(module_name)
                return False

        return True

    def wantFunction(self, fn):
        # PyMongo's test generators run at import time; tell Nose not to run
        # them as unittests.
        if fn.__name__ in ('test_cases',
                           'create_test',
                           'create_connection_string_test',
                           'create_document_test',
                           'create_selection_tests',
                           ):
            return False

    def wantMethod(self, method):
        # Run standard Nose checks on name, like "does it start with test_"?
        if not self.selector.matches(method.__name__):
            return False

        for excluded_name in excluded_tests:
            if PY3:
                classname = method.__self__.__class__.__name__
            else:
                classname = method.im_class.__name__

            # Should we exclude this method's whole TestCase?
            suite_name, method_name = excluded_name.split('.')
            suite_matches = (suite_name == classname or suite_name == '*')

            # Should we exclude this particular method?
            method_matches = (
                method.__name__ == method_name or method_name == '*')

            if suite_matches and method_matches:
                excluded_tests_matched.add(excluded_name)
                return False

        return True

# So that e.g. 'from pymongo.mongo_client import MongoClient' gets the
# Synchro MongoClient, not the real one.
class SynchroModuleFinder(object):
    def find_module(self, fullname, path=None):
        parts = fullname.split('.')
        if parts[-1] in ('gridfs', 'pymongo'):
            return SynchroModuleLoader(path)
        elif len(parts) >= 2 and parts[-2] in ('gridfs', 'pymongo'):
            return SynchroModuleLoader(path)

        # Let regular module search continue.
        return None

class SynchroModuleLoader(object):
    def __init__(self, path):
        self.path = path

    def load_module(self, fullname):
        return synchro

if __name__ == '__main__':
    # Monkey-patch all pymongo's unittests so they think Synchro is the
    # real PyMongo.
    sys.meta_path[0:0] = [SynchroModuleFinder()]

    # Ensure time.sleep() acts as PyMongo's tests expect: background tasks
    # can run to completion while foreground pauses.
    sys.modules['time'] = synchro.TimeModule()

    if '--check-exclude-patterns' in sys.argv:
        check_exclude_patterns = True
        sys.argv.remove('--check-exclude-patterns')
    else:
        check_exclude_patterns = False

    success = nose.run(
        config=Config(plugins=PluginManager()),
        addplugins=[SynchroNosePlugin(), Skip(), Xunit()])

    if not success:
        sys.exit(1)

    if check_exclude_patterns:
        unused_module_pats = set(excluded_modules) - excluded_modules_matched
        assert not unused_module_pats, "Unused module patterns: %s" % (
            unused_module_pats, )

        unused_test_pats = set(excluded_tests) - excluded_tests_matched
        assert not unused_test_pats, "Unused test patterns: %s" % (
            unused_test_pats, )
<commit_msg>MOTOR-242 Update parallelScan test skip reason<commit_after># Copyright 2012-2014 MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import unicode_literals

"""Test Motor by testing that Synchro, a fake PyMongo implementation built on
top of Motor, passes the same unittests as PyMongo.

This program monkey-patches sys.modules, so run it alone, rather than as part
of a larger test suite.
"""

import sys

import nose
from nose.config import Config
from nose.plugins import Plugin
from nose.plugins.manager import PluginManager
from nose.plugins.skip import Skip
from nose.plugins.xunit import Xunit
from nose.selector import Selector

import synchro
from motor.motor_py2_compat import PY3

excluded_modules = [
    # Exclude some PyMongo tests that can't be applied to Synchro.
    'test.test_cursor_manager',
    'test.test_examples',
    'test.test_threads',
    'test.test_pooling',
    'test.test_legacy_api',
    'test.test_monotonic',
    'test.test_saslprep',

    # Complex PyMongo-specific mocking.
    'test.test_replica_set_reconfig',

    # Accesses PyMongo internals.
    'test.test_retryable_writes',

    # Accesses PyMongo internals. Tested directly in Motor.
    'test.test_session',

    # We test Synchro in Python 2.7, but Motor's change streams need Python 3.
    'test.test_change_stream',

    # Deprecated in PyMongo, removed in Motor 2.0.
    'test.test_gridfs',
    'test.test_son_manipulator',
]

excluded_tests = [
    # Motor's reprs aren't the same as PyMongo's.
    '*.test_repr',
    'TestClient.test_unix_socket',

    # Motor extends the handshake metadata.
    'ClientUnitTest.test_metadata',

    # Lazy-connection tests require multithreading; we test concurrent
    # lazy connection directly.
    'TestClientLazyConnect.*',

    # Motor doesn't support forking or threading.
    '*.test_interrupt_signal',
    'TestSCRAM.test_scram_threaded',
    'TestGSSAPI.test_gssapi_threaded',
    'TestCursor.test_concurrent_close',
    # These are in test_gridfs_bucket.
    'TestGridfs.test_threaded_reads',
    'TestGridfs.test_threaded_writes',

    # Motor's aggregate API is different, always sends "cursor={}" by default.
    'TestCollection.test_aggregate',
    'TestCollection.test_aggregate_raw_bson',
    'TestAllScenarios.test_read_aggregate_Aggregate_with_multiple_stages',
    'TestSingleSlaveOk.test_reads_from_secondary',

    # Can't do MotorCollection(name, create=True), Motor constructors do no I/O.
    'TestCollection.test_create',
    'TestCollection.test_reindex',

    # Motor doesn't support PyMongo's syntax, db.system_js['my_func'] = "code",
    # users should just use system.js as a regular collection.
    'TestDatabase.test_system_js',
    'TestDatabase.test_system_js_list',

    # Requires indexing / slicing cursors, which Motor doesn't do, see MOTOR-84.
    'TestCollection.test_min_query',
    'TestCursor.test_clone',
    'TestCursor.test_count_with_limit_and_skip',
    'TestCursor.test_getitem_numeric_index',
    'TestCursor.test_getitem_slice_index',
    'TestCursor.test_tailable',

    # Raw batches aren't implemented yet, MOTOR-172.
    'TestRawBatchCursor.*',
    'TestRawBatchCommandCursor.*',

    # No context-manager protocol for MotorCursor.
    'TestCursor.test_with_statement',

    # Can't iterate a GridOut in Motor.
    'TestGridFile.test_iterator',
    'TestGridfs.test_missing_length_iter',

    # No context-manager protocol for MotorGridIn, and can't set attrs.
    'TestGridFile.test_context_manager',
    'TestGridFile.test_grid_in_default_opts',
    'TestGridFile.test_set_after_close',

    # GridOut always connects lazily in Motor.
    'TestGridFile.test_grid_out_lazy_connect',
    'TestGridfs.test_gridfs_lazy_connect',  # In test_gridfs_bucket.

    # Complex PyMongo-specific mocking.
    '*.test_wire_version',
    'TestClient.test_heartbeat_frequency_ms',
    'TestExhaustCursor.*',
    'TestHeartbeatMonitoring.*',
    'TestMongoClientFailover.*',
    'TestMongosLoadBalancing.*',
    'TestReplicaSetClientInternalIPs.*',
    'TestReplicaSetClientMaxWriteBatchSize.*',
    'TestSSL.test_system_certs_config_error',

    # Motor is correct here, it's just unreliable on slow CI servers.
    'TestReplicaSetClient.test_timeout_does_not_mark_member_down',

    # Accesses PyMongo internals.
    'TestClient.test_close_kills_cursors',
    'TestClient.test_stale_getmore',
    'TestCollection.test_aggregation_cursor',
    'TestCommandAndReadPreference.*',
    'TestCommandMonitoring.test_get_more_failure',
    'TestCommandMonitoring.test_sensitive_commands',
    'TestCursor.test_close_kills_cursor_synchronously',
    'TestGridFile.test_grid_out_cursor_options',
    'TestMaxStaleness.test_last_write_date',
    'TestMaxStaleness.test_last_write_date_absent',
    'TestMonitor.test_atexit_hook',
    'TestReplicaSetClient.test_kill_cursor_explicit_primary',
    'TestReplicaSetClient.test_kill_cursor_explicit_secondary',
    'TestSelections.test_bool',
    'TestTransactions.transaction_test_debug',

    # Deprecated in PyMongo, removed in Motor 2.0.
    'TestDatabase.test_collection_names',
    'TestDatabase.test_errors',
    'TestDatabase.test_eval',
    'TestCollation.*',
    'TestCollection.test_find_one_and_write_concern',
    'TestCollection.test_parallel_scan',
    'TestCollection.test_parallel_scan_max_time_ms',
    'TestCollection.test_write_error_text_handling',
    'TestCommandMonitoring.test_legacy_insert_many',
    'TestCommandMonitoring.test_legacy_writes',
    'TestClient.test_database_names',

    # Tests that use "count", deprecated in PyMongo, removed in Motor 2.0.
    '*.test_command_monitoring_command_A_failed_command_event',
    '*.test_command_monitoring_command_A_successful_command',
    '*.test_command_monitoring_command_A_successful_command_with_a_non-primary_read_preference',
    '*.test_read_count-collation_Deprecated_count_with_collation',
    '*.test_read_count_Deprecated_count_with_a_filter',
    '*.test_read_count_Deprecated_count_without_a_filter',
    'TestBinary.test_uuid_queries',
    'TestCollection.test_count',
    'TestCursor.test_comment',
    'TestCursor.test_count',
    'TestCursor.test_count_with_fields',
    'TestCursor.test_count_with_hint',
    'TestCursor.test_where',
    'TestGridfs.test_gridfs_find',
    'TestTransactions.test_transactions_reads_count',

    # Tests that use "authenticate" or "logoout", removed in Motor 2.0.
    'TestSASLPlain.test_sasl_plain_bad_credentials',
    'TestSCRAM.test_scram',
    'TestSCRAMSHA1.test_scram_sha1',
]

excluded_modules_matched = set()
excluded_tests_matched = set()

class SynchroNosePlugin(Plugin):
    name = 'synchro'

    def __init__(self, *args, **kwargs):
        # We need a standard Nose selector in order to filter out methods that
        # don't match TestSuite.test_*
        self.selector = Selector(config=None)
        super(SynchroNosePlugin, self).__init__(*args, **kwargs)

    def configure(self, options, conf):
        super(SynchroNosePlugin, self).configure(options, conf)
        self.enabled = True

    def wantModule(self, module):
        # Depending on PYTHONPATH, Motor's direct tests may be imported - don't
        # run them now.
        if module.__name__.startswith('test.test_motor_'):
            return False

        for module_name in excluded_modules:
            if module_name.endswith('*'):
                if module.__name__.startswith(module_name.rstrip('*')):
                    # E.g., test_motor_cursor matches "test_motor_*".
                    excluded_modules_matched.add(module_name)
                    return False

            elif module.__name__ == module_name:
                excluded_modules_matched.add(module_name)
                return False

        return True

    def wantFunction(self, fn):
        # PyMongo's test generators run at import time; tell Nose not to run
        # them as unittests.
        if fn.__name__ in ('test_cases',
                           'create_test',
                           'create_connection_string_test',
                           'create_document_test',
                           'create_selection_tests',
                           ):
            return False

    def wantMethod(self, method):
        # Run standard Nose checks on name, like "does it start with test_"?
        if not self.selector.matches(method.__name__):
            return False

        for excluded_name in excluded_tests:
            if PY3:
                classname = method.__self__.__class__.__name__
            else:
                classname = method.im_class.__name__

            # Should we exclude this method's whole TestCase?
            suite_name, method_name = excluded_name.split('.')
            suite_matches = (suite_name == classname or suite_name == '*')

            # Should we exclude this particular method?
            method_matches = (
                method.__name__ == method_name or method_name == '*')

            if suite_matches and method_matches:
                excluded_tests_matched.add(excluded_name)
                return False

        return True

# So that e.g. 'from pymongo.mongo_client import MongoClient' gets the
# Synchro MongoClient, not the real one.
class SynchroModuleFinder(object):
    def find_module(self, fullname, path=None):
        parts = fullname.split('.')
        if parts[-1] in ('gridfs', 'pymongo'):
            return SynchroModuleLoader(path)
        elif len(parts) >= 2 and parts[-2] in ('gridfs', 'pymongo'):
            return SynchroModuleLoader(path)

        # Let regular module search continue.
        return None

class SynchroModuleLoader(object):
    def __init__(self, path):
        self.path = path

    def load_module(self, fullname):
        return synchro

if __name__ == '__main__':
    # Monkey-patch all pymongo's unittests so they think Synchro is the
    # real PyMongo.
    sys.meta_path[0:0] = [SynchroModuleFinder()]

    # Ensure time.sleep() acts as PyMongo's tests expect: background tasks
    # can run to completion while foreground pauses.
    sys.modules['time'] = synchro.TimeModule()

    if '--check-exclude-patterns' in sys.argv:
        check_exclude_patterns = True
        sys.argv.remove('--check-exclude-patterns')
    else:
        check_exclude_patterns = False

    success = nose.run(
        config=Config(plugins=PluginManager()),
        addplugins=[SynchroNosePlugin(), Skip(), Xunit()])

    if not success:
        sys.exit(1)

    if check_exclude_patterns:
        unused_module_pats = set(excluded_modules) - excluded_modules_matched
        assert not unused_module_pats, "Unused module patterns: %s" % (
            unused_module_pats, )

        unused_test_pats = set(excluded_tests) - excluded_tests_matched
        assert not unused_test_pats, "Unused test patterns: %s" % (
            unused_test_pats, )
<|endoftext|>