<commit_before>###############################################################################
# Copyright (c) 2018, National Research Foundation (Square Kilometre Array)
#
# Licensed under the BSD 3-Clause License (the "License"); you may not use
# this file except in compliance with the License. You may obtain a copy
# of the License at
#
#   https://opensource.org/licenses/BSD-3-Clause
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################

"""Tests for :py:mod:`katdal.applycal`."""
from __future__ import print_function, division, absolute_import
from builtins import object, range

import numpy as np
from numpy.testing import assert_array_equal
from nose.tools import assert_raises, assert_equal
import dask.array as da

from katdal.sensordata import SensorCache
from katdal.categorical import CategoricalData
from katdal.lazy_indexer import DaskLazyIndexer
from katdal.applycal import (calc_delay_correction, add_applycal_sensors,
                             calc_correction_per_corrprod,
                             apply_vis_correction, add_applycal_transform)

POLS = ['v', 'h']
ANTS = ['m000', 'm001', 'm002', 'm003']
CENTRE_FREQ = 1284.0
BANDWIDTH = 856.0
N_CHANS = 128
N_DUMPS = 100
SAMPLE_RATE = 1712.0

INPUTS = [ant + pol for ant in ANTS for pol in POLS]
FREQS = CENTRE_FREQ + BANDWIDTH / N_CHANS * (np.arange(N_CHANS) - N_CHANS // 2)
INDEX1, INDEX2 = np.triu_indices(len(INPUTS))
CORRPRODS = [(INPUTS[i1], INPUTS[i2]) for i1, i2 in zip(INDEX1, INDEX2)]
N_CORRPRODS = len(CORRPRODS)

def create_sensor_cache():
    """Create a SensorCache for testing applycal sensors."""
    cache = {}
    delays = np.arange(len(ANTS))
    delays = np.array([delays, 100 * delays]) / SAMPLE_RATE
    cache['cal_product_K'] = CategoricalData([np.zeros_like(delays), delays],
                                             events=[0, 10, N_DUMPS])
    return SensorCache(cache, timestamps=np.arange(N_DUMPS, dtype=float),
                       dump_period=1.)

def delay_gains(pol, ant):
    delay = 100 ** pol * ant
    return np.exp(2j * np.pi * delay / SAMPLE_RATE * FREQS).astype('complex64')

def gains_per_corrprod(dumps, channels, corrprods=()):
    input_map = {ant + pol: (pol_idx, ant_idx)
                 for (pol_idx, pol) in enumerate(POLS)
                 for (ant_idx, ant) in enumerate(ANTS)}
    gains_per_input = np.array([delay_gains(*input_map[inp])
                                for inp in INPUTS])
    gains_per_input = gains_per_input[:, channels]
    gain1 = gains_per_input[INDEX1[corrprods]].T
    gain2 = gains_per_input[INDEX2[corrprods]].T
    return gain1 * gain2.conj()

class TestCorrectionPerInput(object):
    """Test the :func:`~katdal.applycal.calc_*_correction` functions."""
    def setup(self):
        self.cache = create_sensor_cache()

    def test_calc_delay_correction(self):
        for n in range(len(ANTS)):
            for m in range(len(POLS)):
                sensor = calc_delay_correction(self.cache, 'K', (m, n), FREQS)
                assert_array_equal(sensor[n],
                                   np.ones(N_CHANS, dtype='complex64'))
                assert_array_equal(sensor[10 + n], delay_gains(m, n))

class TestVirtualCorrectionSensors(object):
    """Test :func:`~katdal.applycal.add_applycal_sensors` function."""
    def setup(self):
        self.cache = create_sensor_cache()
        add_applycal_sensors(self.cache, ANTS, POLS, FREQS)

    def test_add_sensors_does_nothing_if_no_ants_or_pols(self):
        cache = create_sensor_cache()
        n_virtuals_before = len(cache.virtual)
        add_applycal_sensors(cache, [], [], [])
        n_virtuals_after = len(cache.virtual)
        assert_equal(n_virtuals_after, n_virtuals_before)

    def test_delay_sensors(self):
        for n, ant in enumerate(ANTS):
            for m, pol in enumerate(POLS):
                sensor_name = 'Calibration/{}{}_correction_K'.format(ant, pol)
                sensor = self.cache.get(sensor_name)
                assert_array_equal(sensor[10 + n], delay_gains(m, n))

    def test_unknown_inputs_and_products(self):
        known_input = 'Calibration/{}{}'.format(ANTS[0], POLS[0])
        with assert_raises(KeyError):
            self.cache.get('Calibration/unknown_correction_K')
        with assert_raises(KeyError):
            self.cache.get(known_input + '_correction_unknown')
        with assert_raises(KeyError):
            self.cache.get(known_input + '_correction_K_unknown')

class TestCorrectionPerCorrprod(object):
    """Test :func:`~katdal.applycal.calc_correction_per_corrprod` function."""
    def setup(self):
        self.cache = create_sensor_cache()
        add_applycal_sensors(self.cache, ANTS, POLS, FREQS)

    def test_correction_per_corrprod(self):
        dump = 15
        channels = list(range(22, 38))
        cal_products = ['K']
        gains = calc_correction_per_corrprod(dump, channels, self.cache,
                                             INPUTS, INDEX1, INDEX2,
                                             cal_products)
        expected_gains = gains_per_corrprod([dump], channels)
        assert_array_equal(gains, expected_gains)

class TestApplyCal(object):
    """Test :func:`~katdal.applycal.add_applycal_transform` function."""
    def setup(self):
        self.cache = create_sensor_cache()
        add_applycal_sensors(self.cache, ANTS, POLS, FREQS)
        time_keep = np.full(N_DUMPS, False, dtype=np.bool_)
        time_keep[10:20] = True
        freq_keep = np.full(N_CHANS, False, dtype=np.bool_)
        freq_keep[22:38] = True
        corrprod_keep = np.full(N_CORRPRODS, True, dtype=np.bool_)
        # Throw out the first antenna
        for n, inp in enumerate(INPUTS):
            if inp.startswith(ANTS[0]):
                corrprod_keep[INDEX1 == n] = False
                corrprod_keep[INDEX2 == n] = False
        self.stage1 = (time_keep, freq_keep, corrprod_keep)
        # List of selected correlation products
        self.corrprods = [cp for n, cp in enumerate(CORRPRODS)
                          if corrprod_keep[n]]

    def test_applycal_vis(self):
        vis_real = np.random.randn(N_DUMPS, N_CHANS, N_CORRPRODS)
        vis_imag = np.random.randn(N_DUMPS, N_CHANS, N_CORRPRODS)
        vis = np.asarray(vis_real + 1j * vis_imag, dtype='complex64')
        vis_dask = da.from_array(vis, chunks=(10, 4, 6))
        indexer = DaskLazyIndexer(vis_dask, self.stage1)
        cal_products = ['K']
        add_applycal_transform(indexer, self.cache, self.corrprods,
                               cal_products, apply_vis_correction)
        # Apply stage 2 selection on top of stage 1
        stage2 = np.s_[5, 2:5, :]
        stage1_indices = tuple(k.nonzero()[0] for k in self.stage1)
        final_indices = tuple(i[s] for s, i in zip(stage2, stage1_indices))
        gains = gains_per_corrprod(*final_indices)
        # Quick and dirty oindex of vis (yet another way doing axes in reverse)
        selected_vis = vis
        dims = reversed(range(vis.ndim))
        for dim, indices in zip(dims, reversed(final_indices)):
            selected_vis = np.take(selected_vis, indices, axis=dim)
        expected_vis = selected_vis * gains
        calibrated_vis = indexer[stage2]
        assert_array_equal(calibrated_vis, expected_vis)
<commit_msg>Test the handling of delays that are NaN<commit_after>###############################################################################
# Copyright (c) 2018, National Research Foundation (Square Kilometre Array)
#
# Licensed under the BSD 3-Clause License (the "License"); you may not use
# this file except in compliance with the License. You may obtain a copy
# of the License at
#
#   https://opensource.org/licenses/BSD-3-Clause
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################

"""Tests for :py:mod:`katdal.applycal`."""
from __future__ import print_function, division, absolute_import
from builtins import object, range

import numpy as np
from numpy.testing import assert_array_equal
from nose.tools import assert_raises, assert_equal
import dask.array as da

from katdal.sensordata import SensorCache
from katdal.categorical import CategoricalData
from katdal.lazy_indexer import DaskLazyIndexer
from katdal.applycal import (calc_delay_correction, add_applycal_sensors,
                             calc_correction_per_corrprod,
                             apply_vis_correction, add_applycal_transform)

POLS = ['v', 'h']
ANTS = ['m000', 'm001', 'm002', 'm003']
CENTRE_FREQ = 1284.0
BANDWIDTH = 856.0
N_CHANS = 128
N_DUMPS = 100
SAMPLE_RATE = 1712.0

INPUTS = [ant + pol for ant in ANTS for pol in POLS]
FREQS = CENTRE_FREQ + BANDWIDTH / N_CHANS * (np.arange(N_CHANS) - N_CHANS // 2)
INDEX1, INDEX2 = np.triu_indices(len(INPUTS))
CORRPRODS = [(INPUTS[i1], INPUTS[i2]) for i1, i2 in zip(INDEX1, INDEX2)]
N_CORRPRODS = len(CORRPRODS)

SKIP_ANT = 0
BAD_DELAY_ANT = 1

def create_delay(pol, ant):
    """Synthesise a delay in seconds from `pol` and `ant` indices."""
    return (100 ** pol * ant / SAMPLE_RATE) if ant != BAD_DELAY_ANT else np.nan

def create_sensor_cache():
    """Create a SensorCache for testing applycal sensors."""
    cache = {}
    pols = range(len(POLS))
    ants = range(len(ANTS))
    delays = [create_delay(pol, ant) for pol in pols for ant in ants]
    delays = np.array(delays).reshape(len(pols), len(ants))
    cache['cal_product_K'] = CategoricalData([np.zeros_like(delays), delays],
                                             events=[0, 10, N_DUMPS])
    return SensorCache(cache, timestamps=np.arange(N_DUMPS, dtype=float),
                       dump_period=1.)

def delay_gains(pol, ant):
    """Figure out gain correction for delay given `pol` and `ant` indices."""
    # Zero out missing delays (indicated by NaN)
    delay = np.nan_to_num(create_delay(pol, ant))
    return np.exp(2j * np.pi * delay * FREQS).astype('complex64')

def gains_per_corrprod(dumps, channels, corrprods=()):
    """Predict corrprod correction for a time-frequency-baseline selection."""
    input_map = {ant + pol: (pol_idx, ant_idx)
                 for (pol_idx, pol) in enumerate(POLS)
                 for (ant_idx, ant) in enumerate(ANTS)}
    gains_per_input = np.array([delay_gains(*input_map[inp])
                                for inp in INPUTS])
    gains_per_input = gains_per_input[:, channels]
    gain1 = gains_per_input[INDEX1[corrprods]].T
    gain2 = gains_per_input[INDEX2[corrprods]].T
    return gain1 * gain2.conj()

class TestCorrectionPerInput(object):
    """Test the :func:`~katdal.applycal.calc_*_correction` functions."""
    def setup(self):
        self.cache = create_sensor_cache()

    def test_calc_delay_correction(self):
        for n in range(len(ANTS)):
            for m in range(len(POLS)):
                sensor = calc_delay_correction(self.cache, 'K', (m, n), FREQS)
                assert_array_equal(sensor[n],
                                   np.ones(N_CHANS, dtype='complex64'))
                assert_array_equal(sensor[10 + n], delay_gains(m, n))

class TestVirtualCorrectionSensors(object):
    """Test :func:`~katdal.applycal.add_applycal_sensors` function."""
    def setup(self):
        self.cache = create_sensor_cache()
        add_applycal_sensors(self.cache, ANTS, POLS, FREQS)

    def test_add_sensors_does_nothing_if_no_ants_or_pols(self):
        cache = create_sensor_cache()
        n_virtuals_before = len(cache.virtual)
        add_applycal_sensors(cache, [], [], [])
        n_virtuals_after = len(cache.virtual)
        assert_equal(n_virtuals_after, n_virtuals_before)

    def test_delay_sensors(self):
        for n, ant in enumerate(ANTS):
            for m, pol in enumerate(POLS):
                sensor_name = 'Calibration/{}{}_correction_K'.format(ant, pol)
                sensor = self.cache.get(sensor_name)
                assert_array_equal(sensor[10 + n], delay_gains(m, n))

    def test_unknown_inputs_and_products(self):
        known_input = 'Calibration/{}{}'.format(ANTS[0], POLS[0])
        with assert_raises(KeyError):
            self.cache.get('Calibration/unknown_correction_K')
        with assert_raises(KeyError):
            self.cache.get(known_input + '_correction_unknown')
        with assert_raises(KeyError):
            self.cache.get(known_input + '_correction_K_unknown')

class TestCorrectionPerCorrprod(object):
    """Test :func:`~katdal.applycal.calc_correction_per_corrprod` function."""
    def setup(self):
        self.cache = create_sensor_cache()
        add_applycal_sensors(self.cache, ANTS, POLS, FREQS)

    def test_correction_per_corrprod(self):
        dump = 15
        channels = list(range(22, 38))
        cal_products = ['K']
        gains = calc_correction_per_corrprod(dump, channels, self.cache,
                                             INPUTS, INDEX1, INDEX2,
                                             cal_products)
        expected_gains = gains_per_corrprod([dump], channels)
        assert_array_equal(gains, expected_gains)

class TestApplyCal(object):
    """Test :func:`~katdal.applycal.add_applycal_transform` function."""
    def setup(self):
        self.cache = create_sensor_cache()
        add_applycal_sensors(self.cache, ANTS, POLS, FREQS)
        time_keep = np.full(N_DUMPS, False, dtype=np.bool_)
        time_keep[10:20] = True
        freq_keep = np.full(N_CHANS, False, dtype=np.bool_)
        freq_keep[22:38] = True
        corrprod_keep = np.full(N_CORRPRODS, True, dtype=np.bool_)
        # Throw out one antenna
        for n, inp in enumerate(INPUTS):
            if inp.startswith(ANTS[SKIP_ANT]):
                corrprod_keep[INDEX1 == n] = False
                corrprod_keep[INDEX2 == n] = False
        self.stage1 = (time_keep, freq_keep, corrprod_keep)
        # List of selected correlation products
        self.corrprods = [cp for n, cp in enumerate(CORRPRODS)
                          if corrprod_keep[n]]

    def test_applycal_vis(self):
        vis_real = np.random.randn(N_DUMPS, N_CHANS, N_CORRPRODS)
        vis_imag = np.random.randn(N_DUMPS, N_CHANS, N_CORRPRODS)
        vis = np.asarray(vis_real + 1j * vis_imag, dtype='complex64')
        vis_dask = da.from_array(vis, chunks=(10, 4, 6))
        indexer = DaskLazyIndexer(vis_dask, self.stage1)
        cal_products = ['K']
        add_applycal_transform(indexer, self.cache, self.corrprods,
                               cal_products, apply_vis_correction)
        # Apply stage 2 selection on top of stage 1
        stage2 = np.s_[5, 2:5, :]
        stage1_indices = tuple(k.nonzero()[0] for k in self.stage1)
        final_indices = tuple(i[s] for s, i in zip(stage2, stage1_indices))
        gains = gains_per_corrprod(*final_indices)
        # Quick and dirty oindex of vis (yet another way doing axes in reverse)
        selected_vis = vis
        dims = reversed(range(vis.ndim))
        for dim, indices in zip(dims, reversed(final_indices)):
            selected_vis = np.take(selected_vis, indices, axis=dim)
        expected_vis = selected_vis * gains
        calibrated_vis = indexer[stage2]
        assert_array_equal(calibrated_vis, expected_vis)
<|endoftext|>


<commit_before># Calculate your odds of surviving the Murder Trivia Party chalices game
from collections import defaultdict
import sys

try:
	CHALICES = int(sys.argv[1])
except (IndexError, ValueError):
	CHALICES = 8

chances = {0: 1.0} # Initially, you have 100% chance of having no placed pellets.
total_chances = []
for poison in range(CHALICES): # though you won't normally see the full set
	# Add another poison pellet.
	newchance = defaultdict(float)
	for poisoned, prob in chances.items():
		# There are 'poisoned' chalices already poisoned when this pellet
		# gets added. Thus there are that many chances out of CHALICES to
		# re-poison a poisoned chalice.
		if poisoned:
			newchance[poisoned] += prob * poisoned / CHALICES
		# The remaining probability is that you poisoned a new one.
		true_brew = CHALICES - poisoned
		if true_brew:
			newchance[poisoned + 1] += prob * true_brew / CHALICES
		# This will always add up to the original probability.
		# The total probability in the dictionary will always equal 1.0.
	chances = newchance
	print("With %d poison pellets, your chances are:" % (poison + 1))
	tot = 0.0
	for poisoned, prob in sorted(chances.items()):
		survive = 100 - poisoned * 100.0 / CHALICES
		print("%5.2f%% chance of %d poisoned for %5.2f%% survival chance ==> %5.2f%%"
			% (prob*100, poisoned, survive, survive * prob))
		tot += survive * prob
	print("-- total probability of survival %5.2f" % tot)
	total_chances.append(tot)

print("Summary:")
print(" ", "  ".join(" %2s   " % (n+1) for n in range(CHALICES)))
print(" ", "  ".join("%5.2f%%" % c for c in total_chances))

# Summary:
#     1       2       3       4       5       6       7       8
#   87.50%  76.56%  66.99%  58.62%  51.29%  44.88%  39.27%  34.36%
<commit_msg>chalices: Add info about "all survive" chance<commit_after># Calculate your odds of surviving the Murder Trivia Party chalices game
from collections import defaultdict
import sys

try:
	CHALICES = int(sys.argv[1])
except (IndexError, ValueError):
	CHALICES = 8

chances = {0: 1.0} # Initially, you have 100% chance of having no placed pellets.
total_chances = []
for poison in range(CHALICES): # though you won't normally see the full set
	# Add another poison pellet.
	newchance = defaultdict(float)
	for poisoned, prob in chances.items():
		# There are 'poisoned' chalices already poisoned when this pellet
		# gets added. Thus there are that many chances out of CHALICES to
		# re-poison a poisoned chalice.
		if poisoned:
			newchance[poisoned] += prob * poisoned / CHALICES
		# The remaining probability is that you poisoned a new one.
		true_brew = CHALICES - poisoned
		if true_brew:
			newchance[poisoned + 1] += prob * true_brew / CHALICES
		# This will always add up to the original probability.
		# The total probability in the dictionary will always equal 1.0.
	chances = newchance
	print("With %d poison pellets, your chances are:" % (poison + 1))
	tot = 0.0
	for poisoned, prob in sorted(chances.items()):
		survive = 100 - poisoned * 100.0 / CHALICES
		print("%5.2f%% chance of %d poisoned for %5.2f%% survival chance ==> %5.2f%%"
			% (prob*100, poisoned, survive, survive * prob))
		tot += survive * prob
	print("-- total probability of survival %5.2f" % tot)
	total_chances.append(tot)

print("Summary:")
print(" ", "  ".join(" %2s   " % (n+1) for n in range(CHALICES)), "", "<-- pellets")
print(" ", "  ".join("%5.2f%%" % c for c in total_chances), " <-- you live")
print(" ", "  ".join("%5.2f%%" % (100-c) for c in total_chances), " <-- you die")
print(" ", "  ".join("%5.2f%%" % (((c/100)**(CHALICES-n))*100) for n, c in enumerate(total_chances, 1)), "<-- all live")

# Summary:
#     1       2       3       4       5       6       7       8
#   87.50%  76.56%  66.99%  58.62%  51.29%  44.88%  39.27%  34.36%
<|endoftext|>


<commit_before>__time = __import__('time')
from ..patcher import slurp_properties

__patched__ = ['sleep']
slurp_properties(__time, globals(), ignore=__patched__, srckeys=dir(__time))
from ..greenthread import sleep

sleep  # silence pyflakes
<commit_msg>Declare sleep as a global instead of relying on import<commit_after>"""Greenified :mod:`time` module

The only thing that needs to be patched from :mod:`time` is :func:`time.sleep` to yield instead
of block the thread.
"""
__time = __import__('time')
from ..patcher import slurp_properties

__patched__ = ['sleep']
slurp_properties(__time, globals(), ignore=__patched__, srckeys=dir(__time))
from .. import greenthread

sleep = greenthread.sleep
<|endoftext|>


<commit_before>"""
Copyright 2017-2019 Fizyr (https://fizyr.com)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import tensorflow as tf

from .tf_version import tf_version_ok

def setup_gpu(gpu_id):
    if tf_version_ok((2, 0, 0)):
        if gpu_id == 'cpu' or gpu_id == -1:
            tf.config.experimental.set_visible_devices([], 'GPU')
            return

        gpus = tf.config.experimental.list_physical_devices('GPU')
        if gpus:
            # Restrict TensorFlow to only use the first GPU.
            try:
                # Currently, memory growth needs to be the same across GPUs.
                for gpu in gpus:
                    tf.config.experimental.set_memory_growth(gpu, True)

                # Use only the selcted gpu.
                tf.config.experimental.set_visible_devices(gpus[int(gpu_id)], 'GPU')
            except RuntimeError as e:
                # Visible devices must be set before GPUs have been initialized.
                print(e)

            logical_gpus = tf.config.experimental.list_logical_devices('GPU')
            print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
    else:
        import os
        if gpu_id == 'cpu' or gpu_id == -1:
            os.environ['CUDA_VISIBLE_DEVICES'] = ""
            return

        os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_id)
        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        tf.keras.backend.set_session(tf.Session(config=config))
<commit_msg>Remove compatibility codes from setup_gpu<commit_after>"""
Copyright 2017-2019 Fizyr (https://fizyr.com)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import tensorflow as tf

from .tf_version import tf_version_ok

def setup_gpu(gpu_id):
    if gpu_id == 'cpu' or gpu_id == -1:
        tf.config.set_visible_devices([], 'GPU')
        return

    gpus = tf.config.list_physical_devices('GPU')
    if gpus:
        # Restrict TensorFlow to only use the first GPU.
        try:
            # Currently, memory growth needs to be the same across GPUs.
            for gpu in gpus:
                tf.config.experimental.set_memory_growth(gpu, True)

            # Use only the selcted gpu.
            tf.config.set_visible_devices(gpus[int(gpu_id)], 'GPU')
        except RuntimeError as e:
            # Visible devices must be set before GPUs have been initialized.
            print(e)

        logical_gpus = tf.config.list_logical_devices('GPU')
        print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
<|endoftext|>


<commit_before>try:
    import simplejson as json
except ImportError:
    try:
        import json
    except ImportError:
        raise EnvironmentError("You must have a JSON module installed such as simplejson")
import urllib
import urllib2
import base64

# Define all exceptions
# Corresponds to HTTP response codes specified here:
# http://support.chargify.com/faqs/api/api-user-guide
class ChargifyError(Exception):
    def __init__(self, error_data=None, *a, **kw):
        self.error_data = error_data or {}
        super(ChargifyError, self).__init__(*a, **kw)

class ChargifyConnectionError(ChargifyError):pass
class ChargifyUnauthorizedError(ChargifyError):pass
class ChargifyForbiddenError(ChargifyError):pass
class ChargifyNotFoundError(ChargifyError):pass
class ChargifyUnprocessableEntityError(ChargifyError):pass
class ChargifyServerError(ChargifyError):pass

ERROR_CODES = {
    201: False,
    401: ChargifyUnauthorizedError,
    403: ChargifyForbiddenError,
    404: ChargifyNotFoundError,
    422: ChargifyUnprocessableEntityError,
    500: ChargifyServerError,
}

# Maps certain function names to HTTP verbs
VERBS = {
    'create':'POST',
    'read':'GET',
    'update':'PUT',
    'delete':'DELETE'
}

# A list of identifiers that should be extracted and placed into the url string if they are
# passed into the kwargs.
IDENTIFIERS = {
    'customer_id':'customers',
    'product_id':'products',
    'subscription_id':'subscriptions',
    'component_id':'components',
    'handle':'handle'
}

class ChargifyHttpClient(object):
    """
    Extracted from the main Chargify class so it can be stubbed out during testing.
    """

    def make_request(self, url, method, data, api_key):
        """
        Actually responsible for making the HTTP request.
        :param url: The URL to load.
        :param method: The HTTP method to use.
        :param data: Any POST data that should be included with the request.
        """
        opener = urllib2.build_opener(urllib2.HTTPHandler)
        request = urllib2.Request(url=url, data=data)

        # Build header
        request.get_method = lambda: method
        request.add_header('Authorization', 'Basic %s' % base64.encodestring('%s:%s' % (api_key, 'x'))[:-1])
        request.add_header('User-Agent', 'Chargify Python Client')
        request.add_header('Accept', 'application/json')
        request.add_header('Content-Type', 'application/json')
        if data is None:
            request.add_header('Content-Length','0')

        # Make request and trap for HTTP errors
        try:
            response = opener.open(request)
        except urllib2.HTTPError, e:
            response = e
        except urllib2.URLError, e:
            raise ChargifyConnectionError(e)

        result = response.read()

        try:
            data = json.loads(result)
        except ValueError:
            data = {'body': result} #Is not JSON

        if response.code in ERROR_CODES and ERROR_CODES[response.code] is not False:
            error_class = ERROR_CODES[e.code]
            raise error_class(data)

        return data

class Chargify(object):
    """
    A client for the Chargify API.
    """
    api_key = ''
    sub_domain = ''
    path = []
    domain = 'https://%s.chargify.com/'
    client = None

    def __init__(self, api_key, sub_domain, path=None, client=None):
        """
        :param api_key: The API key for your Chargify account.
        :param sub_domain: The sub domain of your Chargify account.
        :param path: The current path constructed for this request.
        :param client: The HTTP client to use to make the request.
        """
        self.api_key = api_key
        self.sub_domain = sub_domain
        self.path = path or []
        self.client = client or ChargifyHttpClient()

    def __getattr__(self, attr):
        """
        Uses attribute chaining to help construct the url path of the request.
        """
        try:
            return object.__getattr__(self, attr)
        except AttributeError:
            return Chargify(self.api_key, self.sub_domain, self.path + [attr], self.client)

    def construct_request(self, **kwargs):
        """
        :param kwargs: The arguments passed into the request. Valid values are:
            'customer_id', 'product_id', 'subscription_id', 'component_id', 'handle' will be extracted
            and placed into the url. 'data' will be serialized into a JSON string and POSTed with
            the request.
        """
        path = self.path[:]

        # Find the HTTP method if we were called with create(), update(), read(), or delete()
        if path[-1] in VERBS.keys():
            action = path.pop()
            method = VERBS[action]
        else:
            method = 'GET'

        # Extract certain kwargs and place them in the url instead
        for identifier, name in IDENTIFIERS.items():
            value = kwargs.pop(identifier, None)
            if value:
                path.insert(path.index(name)+1, str(value))

        # Convert the data to a JSON string
        data = kwargs.pop('data', None)
        if data:
            data = json.dumps(data)

        # Build query string
        if method == 'GET' and kwargs:
            args = "?%s" % (urllib.urlencode(kwargs.items()))
        else:
            args = ''

        # Build url
        url = self.domain % self.sub_domain
        url = url + '/'.join(path) + '.json' + args

        return url, method, data

    def __call__(self, **kwargs):
        url, method, data = self.construct_request(**kwargs)
        return self.client.make_request(url, method, data, self.api_key)

<commit_msg>Add support for statements, product families and coupons.<commit_after>try:
    import simplejson as json
except ImportError:
    try:
        import json
    except ImportError:
        raise EnvironmentError("You must have a JSON module installed such as simplejson")
import urllib
import urllib2
import base64

# Define all exceptions
# Corresponds to HTTP response codes specified here:
# http://support.chargify.com/faqs/api/api-user-guide
class ChargifyError(Exception):
    def __init__(self, error_data=None, *a, **kw):
        self.error_data = error_data or {}
        super(ChargifyError, self).__init__(*a, **kw)

class ChargifyConnectionError(ChargifyError):pass
class ChargifyUnauthorizedError(ChargifyError):pass
class ChargifyForbiddenError(ChargifyError):pass
class ChargifyNotFoundError(ChargifyError):pass
class ChargifyUnprocessableEntityError(ChargifyError):pass
class ChargifyServerError(ChargifyError):pass

ERROR_CODES = {
    201: False,
    401: ChargifyUnauthorizedError,
    403: ChargifyForbiddenError,
    404: ChargifyNotFoundError,
    422: ChargifyUnprocessableEntityError,
    500: ChargifyServerError,
}

# Maps certain function names to HTTP verbs
VERBS = {
    'create':'POST',
    'read':'GET',
    'update':'PUT',
    'delete':'DELETE'
}

# A list of identifiers that should be extracted and placed into the url string if they are
# passed into the kwargs.
IDENTIFIERS = {
    'customer_id':'customers',
    'product_id':'products',
    'subscription_id':'subscriptions',
    'component_id':'components',
    'handle':'handle',
    'statement_id':'statements',
    'product_family_id':'product_families',
    'coupon_id':'coupons'
}

class ChargifyHttpClient(object):
    """
    Extracted from the main Chargify class so it can be stubbed out during testing.
    """

    def make_request(self, url, method, data, api_key):
        """
        Actually responsible for making the HTTP request.
        :param url: The URL to load.
        :param method: The HTTP method to use.
        :param data: Any POST data that should be included with the request.
        """
        opener = urllib2.build_opener(urllib2.HTTPHandler)
        request = urllib2.Request(url=url, data=data)

        # Build header
        request.get_method = lambda: method
        request.add_header('Authorization', 'Basic %s' % base64.encodestring('%s:%s' % (api_key, 'x'))[:-1])
        request.add_header('User-Agent', 'Chargify Python Client')
        request.add_header('Accept', 'application/json')
        request.add_header('Content-Type', 'application/json')
        if data is None:
            request.add_header('Content-Length','0')

        # Make request and trap for HTTP errors
        try:
            response = opener.open(request)
        except urllib2.HTTPError, e:
            response = e
        except urllib2.URLError, e:
            raise ChargifyConnectionError(e)

        result = response.read()

        try:
            data = json.loads(result)
        except ValueError:
            data = {'body': result} #Is not JSON

        if response.code in ERROR_CODES and ERROR_CODES[response.code] is not False:
            error_class = ERROR_CODES[e.code]
            raise error_class(data)

        return data

class Chargify(object):
    """
    A client for the Chargify API.
    """
    api_key = ''
    sub_domain = ''
    path = []
    domain = 'https://%s.chargify.com/'
    client = None

    def __init__(self, api_key, sub_domain, path=None, client=None):
        """
        :param api_key: The API key for your Chargify account.
        :param sub_domain: The sub domain of your Chargify account.
        :param path: The current path constructed for this request.
        :param client: The HTTP client to use to make the request.
        """
        self.api_key = api_key
        self.sub_domain = sub_domain
        self.path = path or []
        self.client = client or ChargifyHttpClient()

    def __getattr__(self, attr):
        """
        Uses attribute chaining to help construct the url path of the request.
        """
        try:
            return object.__getattr__(self, attr)
        except AttributeError:
            return Chargify(self.api_key, self.sub_domain, self.path + [attr], self.client)

    def construct_request(self, **kwargs):
        """
        :param kwargs: The arguments passed into the request. Valid values are:
            'customer_id', 'product_id', 'subscription_id', 'component_id', 'handle' will be extracted
            and placed into the url. 'data' will be serialized into a JSON string and POSTed with
            the request.
        """
        path = self.path[:]

        # Find the HTTP method if we were called with create(), update(), read(), or delete()
        if path[-1] in VERBS.keys():
            action = path.pop()
            method = VERBS[action]
        else:
            method = 'GET'

        # Extract certain kwargs and place them in the url instead
        for identifier, name in IDENTIFIERS.items():
            value = kwargs.pop(identifier, None)
            if value:
                path.insert(path.index(name)+1, str(value))

        # Convert the data to a JSON string
        data = kwargs.pop('data', None)
        if data:
            data = json.dumps(data)

        # Build query string
        if method == 'GET' and kwargs:
            args = "?%s" % (urllib.urlencode(kwargs.items()))
        else:
            args = ''

        # Build url
        url = self.domain % self.sub_domain
        url = url + '/'.join(path) + '.json' + args

        return url, method, data

    def __call__(self, **kwargs):
        url, method, data = self.construct_request(**kwargs)
        return self.client.make_request(url, method, data, self.api_key)

<|endoftext|>


<commit_before># crappy AI; routinely loses
# v2 - rewinds instead of simulating boards

from copy import deepcopy
from chess_engine import *
from numpy import array
debug = False
#from random import randint

# piece square tables- are these symmetrical? I don't think the queen is

pawnmatrix = [
[ 0,  0,  0,  0,  0,  0,  0,  0],
[50, 50, 50, 50, 50, 50, 50, 50],
[10, 10, 20, 30, 30, 20, 10, 10],
[ 5,  5, 10, 25, 25, 10,  5,  5],
[ 0,  0,  0, 20, 20,  0,  0,  0],
[ 5, -5,-10,  0,  0,-10, -5,  5],
[ 5, 10, 10,-20,-20, 10, 10,  5],
[ 0,  0,  0,  0,  0,  0,  0,  0]]

knightmatrix = [
[-50,-40,-30,-30,-30,-30,-40,-50],
[-40,-20,  0,  0,  0,  0,-20,-40],
[-30,  0, 10, 15, 15, 10,  0,-30],
[-30,  5, 15, 20, 20, 15,  5,-30],
[-30,  0, 15, 20, 20, 15,  0,-30],
[-30,  5, 10, 15, 15, 10,  5,-30],
[-40,-20,  0,  5,  5,  0,-20,-40],
[-50,-40,-30,-30,-30,-30,-40,-50]]

bishopmatrix = [
[-20,-10,-10,-10,-10,-10,-10,-20],
[-10,  0,  0,  0,  0,  0,  0,-10],
[-10,  0,  5, 10, 10,  5,  0,-10],
[-10,  5,  5, 10, 10,  5,  5,-10],
[-10,  0, 10, 10, 10, 10,  0,-10],
[-10, 10, 10, 10, 10, 10, 10,-10],
[-10,  5,  0,  0,  0,  0,  5,-10],
[-20,-10,-10,-10,-10,-10,-10,-20]]

rookmatrix = [
[ 0,  0,  0,  0,  0,  0,  0,  0],
[ 5, 10, 10, 10, 10, 10, 10,  5],
[-5,  0,  0,  0,  0,  0,  0, -5],
[-5,  0,  0,  0,  0,  0,  0, -5],
[-5,  0,  0,  0,  0,  0,  0, -5],
[-5,  0,  0,  0,  0,  0,  0, -5],
[-5,  0,  0,  0,  0,  0,  0, -5],
[ 0,  0,  0,  5,  5,  0,  0,  0]]

queenmatrix = [
[-20,-10,-10, -5, -5,-10,-10,-20],
[-10,  0,  0,  0,  0,  0,  0,-10],
[-10,  0,  5,  5,  5,  5,  0,-10],
[ -5,  0,  5,  5,  5,  5,  0, -5],
[  0,  0,  5,  5,  5,  5,  0, -5],
[-10,  5,  5,  5,  5,  5,  0,-10],
[-10,  0,  5,  0,  0,  0,  0,-10],
[-20,-10,-10, -5, -5,-10,-10,-20]]

kingmatrix_early = [
[-30,-40,-40,-50,-50,-40,-40,-30],
[-30,-40,-40,-50,-50,-40,-40,-30],
[-30,-40,-40,-50,-50,-40,-40,-30],
[-30,-40,-40,-50,-50,-40,-40,-30],
[-20,-30,-30,-40,-40,-30,-30,-20],
[-10,-20,-20,-20,-20,-20,-20,-10],
[ 20, 20,  0,  0,  0,  0, 20, 20],
[ 20, 30, 10,  0,  0, 10, 30, 20]]

kingmatrix_late = [
[-50,-40,-30,-20,-20,-30,-40,-50],
[-30,-20,-10,  0,  0,-10,-20,-30],
[-30,-10, 20, 30, 30, 20,-10,-30],
[-30,-10, 30, 40, 40, 30,-10,-30],
[-30,-10, 30, 40, 40, 30,-10,-30],
[-30,-10, 20, 30, 30, 20,-10,-30],
[-30,-30,  0,  0,  0,  0,-30,-30],
[-50,-30,-30,-30,-30,-30,-30,-50]]

def reverse(matrix):
    newmatrix = []
    for i in range(1,9):
        newmatrix.append(matrix[-i])
    return newmatrix

posvalmatrix = {
'p': pawnmatrix,
'R': rookmatrix,
'Q': queenmatrix,
'N': knightmatrix,
'B': bishopmatrix,
'K': kingmatrix_early
}

piecevals = {
'p': 100,
'R': 500,
'Q': 900,
'N': 295,
'B': 305,
'K': 5000
}

weights = {'piece_weights': piecevals, 'square_weights': posvalmatrix}

def piece_pos_value(piece):
    global posvalmatrix
    piecename = piece.display()[1]
    piecematrix = posvalmatrix[piecename]
    if piece.display()[0] == 'w':
        piecematrix = reverse(piecematrix)
    if piece.pos is None:
        return 0
    else:
        x = piece.pos[0]-1
        y = piece.pos[1]-1
    return piecematrix[y][x]

def positional_advantage(board, col):
    adv = 0
    for piece in board.pieces.values():
        if piece.col == col:
            colval = 1
        else:
            colval = -1
        adv += (piece_pos_value(piece) * colval)
        #print "%s positional bonus: %s" % (piece, (piece_pos_value(piece) * colval))
    return adv

def count_material(board, col):
    global piecevals
    val_total = 0
    for p in board.pieces.values():
        if p.col == col:
            val_total += piecevals[p.display()[1]]
    return val_total

def count_moves(board, col):
    move_total = 0
    if col == board.whose_turn:
        movelist = board.available_moves
    else:
        movelist = board.gen_all_moves(col)[0]
    for p in movelist.values():
        for m in p:
            move_total += 1
    return move_total

def count_threats(board, col): # higher means you have more threats on the enemy
    threat_total = 0
    if col != board.whose_turn:
        movelist = board.threatened_moves
    else:
        movelist = board.gen_all_moves(col, attack_map = True)[0]
    for p in movelist.values():
        for m in p:
            threat_total += 1
    return threat_total

def check_pawns(board, col):
    pawn_files = []
    dp = 0 # doubled pawns (two pawns on one file)
    bp = 0 # blocked pawns (pawns that cannot advance directly forward)
    ip = 0 # isolated pawns (pawns without a pawn on an adjacent file)
    for p in board.pieces.values():
        if isinstance(p, Pawn) and p.col == col and p.pos is not None:
            if p.pos[0] in pawn_files:
                dp += 1
            pawn_files.append(p.pos[0])
            if p.col == 'white':
                forward_dir = array((0,1))
            else:
                forward_dir = array((0,-1))
            infront = board.squares[tuple(array(p.pos) + forward_dir)]
            if isinstance(infront, Piece):
                bp += 1
    for f in pawn_files:
        if f-1 not in pawn_files and f+1 not in pawn_files:
            ip += 1
    return dp, bp, ip

def evaluate(board, verbose = False):
    material_weight = 1
    position_weight = 5
    #threat_weight = 1
    bp_weight, ip_weight, dp_weight = 25, 75, 50
    material_advantage = (count_material(board, 'white') - count_material(board, 'black')) * material_weight
    pos_advantage = (positional_advantage(board, 'white') - positional_advantage(board, 'black')) * position_weight
    #mobility = (count_moves(board, 'white') - count_moves(board, 'black')) * mobility_weight
    #threats = (count_threats(board, 'white') - count_threats(board, 'black')) * threat_weight
    dp, bp, ip =  tuple(-array(check_pawns(board, 'white')) + array(check_pawns(board, 'black')))
    problem_pawns = sum([bp*bp_weight, ip*ip_weight, dp*dp_weight])

    if verbose:
        print "Material advantage: %s" % material_advantage
        print "Positional advantage: %s" % pos_advantage
        #print "Threat advantage: %s" % threats
        print "Doubled, blocked, isolated pawn advantage: %s, %s, %s" % (dp*dp_weight, bp*bp_weight, ip*ip_weight)
        print "Total favourability for white: %s" % sum([material_advantage, pos_advantage, problem_pawns])
    return sum([material_advantage, pos_advantage, problem_pawns])

def lookahead(board, depth=2, max_depth=2): # simulates all possible moves for boards at an arbitrary depth
    new_movelist = {} # here we store the moves at this depth, and the evaluation of their best outcome for us
    movelist = get_movelist(board)
    if depth > 0:
        for move in movelist:
            #print "%smoving %s from %s to %s" % (' '*depth, board.squares[move[0]], parse(move[0]), parse(move[1]))
            board.move(move[0], move[1])
            #print "move history: " + str(board.movehistory)
            if board.winner == None: # return the value of the best move up the tree
                newboard_val = lookahead(board, depth-1, depth) # here is the recursion
            else: # except if the game is over, in which case return a speial value
                win_vals = {'white': 9999, 'black': -9999, 'nobody': 0}
                newboard_val = win_vals[board.winner]
            new_movelist[move] = newboard_val
            #print "%s-this position: %s" % (' '*depth, newboard_val)
            #print "%srewinding from %s to %s" % (' '*depth, parse(move[1]), parse(move[0]))
            board.rewind()

        # get best move:
        vals = [n for n in new_movelist.values()]
        print "vals: " + str(vals)
        if board.whose_turn == 'white':
            m = max(vals)
        else:
            m = min(vals)
        bestmoves = [i for i, j in enumerate(vals) if j == m]
        bestmove = new_movelist.keys()[bestmoves[0]]
        print (" "*depth) + "best move after %s at %s to %s is: %s at %s to %s" % (board.squares[board.movehistory[-1][0][1]], parse(board.movehistory[-1][0][0]), parse(board.movehistory[-1][0][1]), board.squares[bestmove[0]], parse(bestmove[0]), parse(bestmove[1]))

        if depth < max_depth:
            return evaluate(sim_move(board, (bestmove[0], bestmove[1])))

        else:
            return bestmove

    elif depth == 0:
        return evaluate(board)

def get_movelist(board):
    movelist = []
    for piece, moves in board.available_moves.iteritems():
        for movepos in moves:
            movelist.append((piece.pos, movepos))
    return movelist

def evaluate_moves(board):
    move_dict = {}
    for piece, movelist in board.available_moves.iteritems():
        for movepos in movelist:
            simboard = simulate(board)
            simboard.squares[piece.pos].move(movepos, simboard)
            move_dict[(piece.pos, movepos)] = evaluate(simboard)
    return move_dict

def best_move(board):
    col = board.whose_turn
    movedict = evaluate_moves(board)
    vals = [n for n in movedict.values()]
    if col == 'white':
        m = max(vals)
    else:
        m = min(vals)
    bestmoves = [i for i, j in enumerate(vals) if j == m]
    if len(bestmoves) == 1:
        bestmove = movedict.keys()[bestmoves[0]]
        return (bestmove[0], bestmove[1])
    else:
        bestmove = movedict.keys()[bestmoves[randint(0, len(bestmoves))]]
        return (bestmove[0], bestmove[1])

def make_best_move(board):
    #movedict = evaluate_moves(board)
    bestmove = lookahead(board)
    print "Moving %s from %s to %s" % (board.squares[bestmove[0]], parse(bestmove[0]), parse(bestmove[1]))
    board.move(bestmove[0], bestmove[1])

def simulate(board):
    simboard = deepcopy(board)
    simboard.real_game = False
    return simboard

def sim_move(board, move): # move should be a tuple of tuples
    simboard = simulate(board)
    simboard.move(move[0], move[1])
    return simboard

def play_game():
    mainboard = chess12.Board()
    mainboard.newgame()
    while mainboard.winner == None:
        make_best_move(mainboard)
<commit_msg>Even more pyflakes warnings<commit_after># crappy AI; routinely loses
# v2 - rewinds instead of simulating boards

from copy import deepcopy
from chess_engine import parse, randint, chess12, Piece, Pawn
from numpy import array
debug = False
# from random import randint

# piece square tables- are these symmetrical? I don't think the queen is

pawnmatrix = [
    [ 0,  0,  0,  0,  0,  0,  0,  0],
    [50, 50, 50, 50, 50, 50, 50, 50],
    [10, 10, 20, 30, 30, 20, 10, 10],
    [ 5,  5, 10, 25, 25, 10,  5,  5],
    [ 0,  0,  0, 20, 20,  0,  0,  0],
    [ 5, -5,-10,  0,  0,-10, -5,  5],
    [ 5, 10, 10,-20,-20, 10, 10,  5],
    [ 0,  0,  0,  0,  0,  0,  0,  0]]

knightmatrix = [
    [-50,-40,-30,-30,-30,-30,-40,-50],
    [-40,-20,  0,  0,  0,  0,-20,-40],
    [-30,  0, 10, 15, 15, 10,  0,-30],
    [-30,  5, 15, 20, 20, 15,  5,-30],
    [-30,  0, 15, 20, 20, 15,  0,-30],
    [-30,  5, 10, 15, 15, 10,  5,-30],
    [-40,-20,  0,  5,  5,  0,-20,-40],
    [-50,-40,-30,-30,-30,-30,-40,-50]]

bishopmatrix = [
    [-20,-10,-10,-10,-10,-10,-10,-20],
    [-10,  0,  0,  0,  0,  0,  0,-10],
    [-10,  0,  5, 10, 10,  5,  0,-10],
    [-10,  5,  5, 10, 10,  5,  5,-10],
    [-10,  0, 10, 10, 10, 10,  0,-10],
    [-10, 10, 10, 10, 10, 10, 10,-10],
    [-10,  5,  0,  0,  0,  0,  5,-10],
    [-20,-10,-10,-10,-10,-10,-10,-20]]

rookmatrix = [
    [ 0,  0,  0,  0,  0,  0,  0,  0],
    [ 5, 10, 10, 10, 10, 10, 10,  5],
    [-5,  0,  0,  0,  0,  0,  0, -5],
    [-5,  0,  0,  0,  0,  0,  0, -5],
    [-5,  0,  0,  0,  0,  0,  0, -5],
    [-5,  0,  0,  0,  0,  0,  0, -5],
    [-5,  0,  0,  0,  0,  0,  0, -5],
    [ 0,  0,  0,  5,  5,  0,  0,  0]]

queenmatrix = [
    [-20,-10,-10, -5, -5,-10,-10,-20],
    [-10,  0,  0,  0,  0,  0,  0,-10],
    [-10,  0,  5,  5,  5,  5,  0,-10],
    [ -5,  0,  5,  5,  5,  5,  0, -5],
    [  0,  0,  5,  5,  5,  5,  0, -5],
    [-10,  5,  5,  5,  5,  5,  0,-10],
    [-10,  0,  5,  0,  0,  0,  0,-10],
    [-20,-10,-10, -5, -5,-10,-10,-20]]

kingmatrix_early = [
    [-30,-40,-40,-50,-50,-40,-40,-30],
    [-30,-40,-40,-50,-50,-40,-40,-30],
    [-30,-40,-40,-50,-50,-40,-40,-30],
    [-30,-40,-40,-50,-50,-40,-40,-30],
    [-20,-30,-30,-40,-40,-30,-30,-20],
    [-10,-20,-20,-20,-20,-20,-20,-10],
    [ 20, 20,  0,  0,  0,  0, 20, 20],
    [ 20, 30, 10,  0,  0, 10, 30, 20]]

kingmatrix_late = [
    [-50,-40,-30,-20,-20,-30,-40,-50],
    [-30,-20,-10,  0,  0,-10,-20,-30],
    [-30,-10, 20, 30, 30, 20,-10,-30],
    [-30,-10, 30, 40, 40, 30,-10,-30],
    [-30,-10, 30, 40, 40, 30,-10,-30],
    [-30,-10, 20, 30, 30, 20,-10,-30],
    [-30,-30,  0,  0,  0,  0,-30,-30],
    [-50,-30,-30,-30,-30,-30,-30,-50]]

def reverse(matrix):
    newmatrix = []
    for i in range(1, 9):
        newmatrix.append(matrix[-i])
    return newmatrix

posvalmatrix = {
    'p': pawnmatrix,
    'R': rookmatrix,
    'Q': queenmatrix,
    'N': knightmatrix,
    'B': bishopmatrix,
    'K': kingmatrix_early
}

piecevals = {
    'p': 100,
    'R': 500,
    'Q': 900,
    'N': 295,
    'B': 305,
    'K': 5000
}

weights = {'piece_weights': piecevals, 'square_weights': posvalmatrix}

def piece_pos_value(piece):
    global posvalmatrix
    piecename = piece.display()[1]
    piecematrix = posvalmatrix[piecename]
    if piece.display()[0] == 'w':
        piecematrix = reverse(piecematrix)
    if piece.pos is None:
        return 0
    else:
        x = piece.pos[0]-1
        y = piece.pos[1]-1
    return piecematrix[y][x]

def positional_advantage(board, col):
    adv = 0
    for piece in board.pieces.values():
        if piece.col == col:
            colval = 1
        else:
            colval = -1
        adv += (piece_pos_value(piece) * colval)
        # print "%s positional bonus: %s" % (piece, (piece_pos_value(piece) * colval))
    return adv

def count_material(board, col):
    global piecevals
    val_total = 0
    for p in board.pieces.values():
        if p.col == col:
            val_total += piecevals[p.display()[1]]
    return val_total

def count_moves(board, col):
    move_total = 0
    if col == board.whose_turn:
        movelist = board.available_moves
    else:
        movelist = board.gen_all_moves(col)[0]
    for p in movelist.values():
        for m in p:
            move_total += 1
    return move_total

def count_threats(board, col):  # higher means you have more threats on the enemy
    threat_total = 0
    if col != board.whose_turn:
        movelist = board.threatened_moves
    else:
        movelist = board.gen_all_moves(col, attack_map=True)[0]
    for p in movelist.values():
        for m in p:
            threat_total += 1
    return threat_total

def check_pawns(board, col):
    pawn_files = []
    dp = 0  # doubled pawns (two pawns on one file)
    bp = 0  # blocked pawns (pawns that cannot advance directly forward)
    ip = 0  # isolated pawns (pawns without a pawn on an adjacent file)
    for p in board.pieces.values():
        if isinstance(p, Pawn) and p.col == col and p.pos is not None:
            if p.pos[0] in pawn_files:
                dp += 1
            pawn_files.append(p.pos[0])
            if p.col == 'white':
                forward_dir = array((0, 1))
            else:
                forward_dir = array((0, -1))
            infront = board.squares[tuple(array(p.pos) + forward_dir)]
            if isinstance(infront, Piece):
                bp += 1
    for f in pawn_files:
        if f-1 not in pawn_files and f+1 not in pawn_files:
            ip += 1
    return dp, bp, ip

def evaluate(board, verbose=False):
    material_weight = 1
    position_weight = 5
    # threat_weight = 1
    bp_weight, ip_weight, dp_weight = 25, 75, 50
    material_advantage = (count_material(board, 'white') - count_material(board, 'black')) * material_weight
    pos_advantage = (positional_advantage(board, 'white') - positional_advantage(board, 'black')) * position_weight
    # mobility = (count_moves(board, 'white') - count_moves(board, 'black')) * mobility_weight
    # threats = (count_threats(board, 'white') - count_threats(board, 'black')) * threat_weight
    dp, bp, ip = tuple(-array(check_pawns(board, 'white')) + array(check_pawns(board, 'black')))
    problem_pawns = sum([bp*bp_weight, ip*ip_weight, dp*dp_weight])

    if verbose:
        print "Material advantage: %s" % material_advantage
        print "Positional advantage: %s" % pos_advantage
        # print "Threat advantage: %s" % threats
        print "Doubled, blocked, isolated pawn advantage: %s, %s, %s" % (dp*dp_weight, bp*bp_weight, ip*ip_weight)
        print "Total favourability for white: %s" % sum([material_advantage, pos_advantage, problem_pawns])
    return sum([material_advantage, pos_advantage, problem_pawns])

def lookahead(board, depth=2, max_depth=2):  # simulates all possible moves for boards at an arbitrary depth
    new_movelist = {}  # here we store the moves at this depth, and the evaluation of their best outcome for us
    movelist = get_movelist(board)
    if depth > 0:
        for move in movelist:
            # print "%smoving %s from %s to %s" % (' '*depth, board.squares[move[0]], parse(move[0]), parse(move[1]))
            board.move(move[0], move[1])
            # print "move history: " + str(board.movehistory)
            if board.winner is None:  # return the value of the best move up the tree
                newboard_val = lookahead(board, depth-1, depth)  # here is the recursion
            else:  # except if the game is over, in which case return a speial value
                win_vals = {'white': 9999, 'black': -9999, 'nobody': 0}
                newboard_val = win_vals[board.winner]
            new_movelist[move] = newboard_val
            # print "%s-this position: %s" % (' '*depth, newboard_val)
            # print "%srewinding from %s to %s" % (' '*depth, parse(move[1]), parse(move[0]))
            board.rewind()

        #  get best move:
        vals = [n for n in new_movelist.values()]
        print "vals: " + str(vals)
        if board.whose_turn == 'white':
            m = max(vals)
        else:
            m = min(vals)
        bestmoves = [i for i, j in enumerate(vals) if j == m]
        bestmove = new_movelist.keys()[bestmoves[0]]
        print (" "*depth) + "best move after %s at %s to %s is: %s at %s to %s" % \
            (board.squares[board.movehistory[-1][0][1]], parse(board.movehistory[-1][0][0]),
             parse(board.movehistory[-1][0][1]), board.squares[bestmove[0]],
             parse(bestmove[0]), parse(bestmove[1]))

        if depth < max_depth:
            return evaluate(sim_move(board, (bestmove[0], bestmove[1])))

        else:
            return bestmove

    elif depth == 0:
        return evaluate(board)

def get_movelist(board):
    movelist = []
    for piece, moves in board.available_moves.iteritems():
        for movepos in moves:
            movelist.append((piece.pos, movepos))
    return movelist

def evaluate_moves(board):
    move_dict = {}
    for piece, movelist in board.available_moves.iteritems():
        for movepos in movelist:
            simboard = simulate(board)
            simboard.squares[piece.pos].move(movepos, simboard)
            move_dict[(piece.pos, movepos)] = evaluate(simboard)
    return move_dict

def best_move(board):
    col = board.whose_turn
    movedict = evaluate_moves(board)
    vals = [n for n in movedict.values()]
    if col == 'white':
        m = max(vals)
    else:
        m = min(vals)
    bestmoves = [i for i, j in enumerate(vals) if j == m]
    if len(bestmoves) == 1:
        bestmove = movedict.keys()[bestmoves[0]]
        return (bestmove[0], bestmove[1])
    else:
        bestmove = movedict.keys()[bestmoves[randint(0, len(bestmoves))]]
        return (bestmove[0], bestmove[1])

def make_best_move(board):
    # movedict = evaluate_moves(board)
    bestmove = lookahead(board)
    print "Moving %s from %s to %s" % (board.squares[bestmove[0]], parse(bestmove[0]), parse(bestmove[1]))
    board.move(bestmove[0], bestmove[1])

def simulate(board):
    simboard = deepcopy(board)
    simboard.real_game = False
    return simboard

def sim_move(board, move):  # move should be a tuple of tuples
    simboard = simulate(board)
    simboard.move(move[0], move[1])
    return simboard

def play_game():
    mainboard = chess12.Board()
    mainboard.newgame()
    while mainboard.winner is None:
        make_best_move(mainboard)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
Run arbitrary pprof experiments on the Infosun Chimaira cluster.

This module prepares slurm scripts to:
    * Build
    * Run
    * Collect results
on Infosun's chimaira cluster. All the hard work is done after instancing the
pprof study on the local disks of the assigned cluster node. The results
are copied to a configurable shared directory. After all partial jobs are
completed the results are collected with a separate job.

The resource management system used on chimaira is SLURM.
"""

from plumbum import cli, local, FG
from plumbum.cmd import awk, sbatch, chmod, pprof
from pprof.settings import config
import os

def dispatch_collect_job(exp, deps):
    """
    Dispatch a collect job that waits for all jobs to finish.

    :exp:
        Which experiment do you want to collect results for
    """
    cmd = sbatch["--job-name=collect-" + exp]
    dep_string = ""
    if len(deps) > 0:
        dep_string = "--dependency=afterany"
    print deps
    for job in deps:
        dep_string = dep_string + ":" + job.rstrip()
    cmd = cmd["-A", config["account"],
              "-p", config["partition"]]
    cmd = cmd[dep_string]
    script = prepare_collect_results_script(exp)
    cmd[script] & FG

def dump_slurm_script(script_name, log_name, commands, uuid=None):
    """Write all commands into a file with the given script name and
        configure slurm to use the given log_name with it

    :script_name: Where do you want the slurm script
    :log_name: Where should we put the log output of the SLURM run
    :commands: What commands sould the script execute
    """
    from pprof.settings import config as ppcfg
    from os import path

    with open(script_name, 'w') as slurm:
        slurm.write("#!/bin/sh\n")
        slurm.write("#SBATCH -o {}\n".format(log_name))
        slurm.write("#SBATCH -t \"12:00:00\"\n")
        slurm.write("#SBATCH --hint=nomultithread")
        slurm.write("export LD_LIBRARY_PATH=\"{}:$LD_LIBRARY_PATH\"\n".format(
            path.join(config["papi"], "lib")))
        slurm.write(
            "export PPROF_TESTINPUTS=\"{}\"\n".format(ppcfg["testdir"]))
        slurm.write("export PPROF_TMP_DIR=\"{}\"\n".format(ppcfg["tmpdir"]))
        slurm.write("export PPROF_DB_HOST=\"{}\"\n".format(ppcfg["db_host"]))
        slurm.write("export PPROF_DB_PORT=\"{}\"\n".format(ppcfg["db_port"]))
        slurm.write("export PPROF_DB_NAME=\"{}\"\n".format(ppcfg["db_name"]))
        slurm.write("export PPROF_DB_USER=\"{}\"\n".format(ppcfg["db_user"]))
        slurm.write("export PPROF_DB_PASS=\"{}\"\n".format(ppcfg["db_pass"]))
        if uuid:
            slurm.write("export PPROF_EXPERIMENT_ID=\"{}\"\n".format(uuid))
        for c in commands:
            slurm.write("{}\n".format(str(c)))
    chmod("+x", script_name)

def prepare_collect_results_script(experiment):
    """Prepare a SLURM script that collects all project results

    :experiment: Which experiment we want to collect the results for
    :returns: A list of SLURM job ids

    """
    commands = []
    if not config["llvm"]:
        config["llvm"] = os.path.join(config["nodedir"], "install")
    script_name = os.path.join(os.getcwd(), config["slurm_script"])
    slurm_script = script_name + ".collect"
    log_file = os.path.join(config["resultsdir"], experiment + ".log")
    commands.append(pprof["run",
                          "-E", experiment,
                          "-B", config["resultsdir"]])
    dump_slurm_script(slurm_script, log_file, commands)
    return slurm_script

def prepare_slurm_script(experiment, project, experiment_id):
    """
    Prepare a slurm script that executes the pprof experiment for a given project.

    :experiment: The experiment we want to execute
    :project: Filter all but the given project
    """
    commands = []
    if not config["llvm"]:
        config["llvm"] = os.path.join(config["nodedir"], "install")
    pprof = local["pprof"]

    slurm_script = os.path.join(os.getcwd(), config["slurm_script"])
    log_file = os.path.join(config["resultsdir"],
                            experiment + "-" + project + ".log")

    if config["local_build"]:
        commands.append(pprof["build", "-j", config["cpus-per-task"],
                              "-B", config["nodedir"], "-I", config["isl"],
                              "-L", config["likwidir"], "-P", config["papi"]])
    commands.append(pprof["run",
                          "-P", project,
                          "-E", experiment,
                          "-B", config["nodedir"],
                          "--likwid-prefix", config["likwiddir"],
                          "-L", config["llvm"]])
    # commands.append(
    #    cp["-ar", node_results, os.path.join(config["resultsdir"],
    #                                          experiment)])
    # commands.append(mv[node_error_log, error_log])
    dump_slurm_script(slurm_script, log_file, commands, experiment_id)
    return slurm_script

def dispatch_jobs(exp, projects):
    """Dispatch sbatch scripts to slurm for all projects given

    projects: List of projects that need to be dispatched to SLURM
    :returns: a list of SLURM job ids

    """
    jobs = []
    from uuid import uuid4

    experiment_id = uuid4()
    for project in projects:
        if len(project) == 0:
            continue
        slurm_script = prepare_slurm_script(exp, project, experiment_id)
        sbatch_cmd = sbatch[
            "--job-name=" + exp + "-" + project,
            "-A", config["account"],
            "-p", config["partition"],
            "--ntasks", "1",
            "--exclusive", slurm_script]

        if "cpus-per-task" in config:
            sbatch_cmd = sbatch_cmd["--cpus-per-task", config["cpus-per-task"]]

        job_id = (sbatch_cmd | awk['{ print $4 }'])()
        jobs.append(job_id)
    return jobs

class Chimaira(cli.Application):

    """Execute the polyprof study on the chimaira cluster via slurm"""

    @cli.switch(["-E", "--experiment"], str, list=True, mandatory=True,
                help="Which experiments should be built")
    def experiments(self, exps):
        config["experiments"] = exps

    @cli.switch(["--nodedir"], str, help="Where to build on the node")
    def nodedir(self, dirname):
        config["nodedir"] = dirname

    @cli.switch(["--resultsdir"], str, mandatory=True,
                help="Where should the results be placed")
    def resultsdir(self, dirname):
        config["resultsdir"] = dirname

    @cli.switch(["--scriptname"], str,
                help="How should the sbatch script be called?")
    def scriptname(self, name):
        config["slurm_script"] = name

    @cli.switch(["--cpus-per-task"], int,
                help="How many cpu should we request per task?")
    def cpus_per_task(self, cpt):
        config["cpus-per-task"] = cpt

    @cli.switch(["-I"], str, help="ISL prefix")
    def isl(self, isl_prefix):
        config["isl"] = isl_prefix

    @cli.switch(["-L"], str, help="Likwid prefix")
    def likwid(self, likwid_prefix):
        config["likwiddir"] = likwid_prefix

    @cli.switch(["-P"], str, help="PAPI prefix")
    def papi(self, papi_prefix):
        config["papi"] = papi_prefix

    @cli.switch(["-p"], str, help="SLURM partition to use")
    def partition(self, partition):
        config["partition"] = partition

    @cli.switch(["-A"], str, help="SLURM account to use")
    def account(self, account):
        config["account"] = account

    @cli.switch(["--llvm-prefix"], str, help="LLVM prefix")
    def llvm(self, llvm_prefix):
        config["llvm"] = llvm_prefix

    @cli.switch(["--build"], help="Build LLVM/Polly/Polli/Clang on the node")
    def local_build(self):
        config["local_build"] = True

    def main(self):
        from itertools import chain

        # That is an awful lot of filtering required for parsing that
        # output... grmpf
        for exp in config["experiments"]:
            print "Experiment: {}".format(exp)
            pprof_list = pprof["run", "-l", "-E", exp]
            prj_list = pprof_list().split(',')
            prj_list = map(unicode.strip, prj_list)
            prj_list = map(lambda x: x.split('\n'), prj_list)
            prj_list = filter(None, prj_list)
            prj_list = list(chain.from_iterable(prj_list))
            prj_list = filter(lambda x: not '>>' in x, prj_list)
            prj_list = filter(None, prj_list)
            print "  {} projects".format(len(prj_list))
            print
            print ", ".join(prj_list)
            jobs = dispatch_jobs(exp, prj_list)
        #    dispatch_collect_job(exp, jobs)

if __name__ == '__main__':
    Chimaira.run()
<commit_msg>chimaira: missing newline<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
Run arbitrary pprof experiments on the Infosun Chimaira cluster.

This module prepares slurm scripts to:
    * Build
    * Run
    * Collect results
on Infosun's chimaira cluster. All the hard work is done after instancing the
pprof study on the local disks of the assigned cluster node. The results
are copied to a configurable shared directory. After all partial jobs are
completed the results are collected with a separate job.

The resource management system used on chimaira is SLURM.
"""

from plumbum import cli, local, FG
from plumbum.cmd import awk, sbatch, chmod, pprof
from pprof.settings import config
import os

def dispatch_collect_job(exp, deps):
    """
    Dispatch a collect job that waits for all jobs to finish.

    :exp:
        Which experiment do you want to collect results for
    """
    cmd = sbatch["--job-name=collect-" + exp]
    dep_string = ""
    if len(deps) > 0:
        dep_string = "--dependency=afterany"
    print deps
    for job in deps:
        dep_string = dep_string + ":" + job.rstrip()
    cmd = cmd["-A", config["account"],
              "-p", config["partition"]]
    cmd = cmd[dep_string]
    script = prepare_collect_results_script(exp)
    cmd[script] & FG

def dump_slurm_script(script_name, log_name, commands, uuid=None):
    """Write all commands into a file with the given script name and
        configure slurm to use the given log_name with it

    :script_name: Where do you want the slurm script
    :log_name: Where should we put the log output of the SLURM run
    :commands: What commands sould the script execute
    """
    from pprof.settings import config as ppcfg
    from os import path

    with open(script_name, 'w') as slurm:
        slurm.write("#!/bin/sh\n")
        slurm.write("#SBATCH -o {}\n".format(log_name))
        slurm.write("#SBATCH -t \"12:00:00\"\n")
        slurm.write("#SBATCH --hint=nomultithread\n")
        slurm.write("export LD_LIBRARY_PATH=\"{}:$LD_LIBRARY_PATH\"\n".format(
            path.join(config["papi"], "lib")))
        slurm.write(
            "export PPROF_TESTINPUTS=\"{}\"\n".format(ppcfg["testdir"]))
        slurm.write("export PPROF_TMP_DIR=\"{}\"\n".format(ppcfg["tmpdir"]))
        slurm.write("export PPROF_DB_HOST=\"{}\"\n".format(ppcfg["db_host"]))
        slurm.write("export PPROF_DB_PORT=\"{}\"\n".format(ppcfg["db_port"]))
        slurm.write("export PPROF_DB_NAME=\"{}\"\n".format(ppcfg["db_name"]))
        slurm.write("export PPROF_DB_USER=\"{}\"\n".format(ppcfg["db_user"]))
        slurm.write("export PPROF_DB_PASS=\"{}\"\n".format(ppcfg["db_pass"]))
        if uuid:
            slurm.write("export PPROF_EXPERIMENT_ID=\"{}\"\n".format(uuid))
        for c in commands:
            slurm.write("{}\n".format(str(c)))
    chmod("+x", script_name)

def prepare_collect_results_script(experiment):
    """Prepare a SLURM script that collects all project results

    :experiment: Which experiment we want to collect the results for
    :returns: A list of SLURM job ids

    """
    commands = []
    if not config["llvm"]:
        config["llvm"] = os.path.join(config["nodedir"], "install")
    script_name = os.path.join(os.getcwd(), config["slurm_script"])
    slurm_script = script_name + ".collect"
    log_file = os.path.join(config["resultsdir"], experiment + ".log")
    commands.append(pprof["run",
                          "-E", experiment,
                          "-B", config["resultsdir"]])
    dump_slurm_script(slurm_script, log_file, commands)
    return slurm_script

def prepare_slurm_script(experiment, project, experiment_id):
    """
    Prepare a slurm script that executes the pprof experiment for a given project.

    :experiment: The experiment we want to execute
    :project: Filter all but the given project
    """
    commands = []
    if not config["llvm"]:
        config["llvm"] = os.path.join(config["nodedir"], "install")
    pprof = local["pprof"]

    slurm_script = os.path.join(os.getcwd(), config["slurm_script"])
    log_file = os.path.join(config["resultsdir"],
                            experiment + "-" + project + ".log")

    if config["local_build"]:
        commands.append(pprof["build", "-j", config["cpus-per-task"],
                              "-B", config["nodedir"], "-I", config["isl"],
                              "-L", config["likwidir"], "-P", config["papi"]])
    commands.append(pprof["run",
                          "-P", project,
                          "-E", experiment,
                          "-B", config["nodedir"],
                          "--likwid-prefix", config["likwiddir"],
                          "-L", config["llvm"]])
    # commands.append(
    #    cp["-ar", node_results, os.path.join(config["resultsdir"],
    #                                          experiment)])
    # commands.append(mv[node_error_log, error_log])
    dump_slurm_script(slurm_script, log_file, commands, experiment_id)
    return slurm_script

def dispatch_jobs(exp, projects):
    """Dispatch sbatch scripts to slurm for all projects given

    projects: List of projects that need to be dispatched to SLURM
    :returns: a list of SLURM job ids

    """
    jobs = []
    from uuid import uuid4

    experiment_id = uuid4()
    for project in projects:
        if len(project) == 0:
            continue
        slurm_script = prepare_slurm_script(exp, project, experiment_id)
        sbatch_cmd = sbatch[
            "--job-name=" + exp + "-" + project,
            "-A", config["account"],
            "-p", config["partition"],
            "--ntasks", "1",
            "--exclusive", slurm_script]

        if "cpus-per-task" in config:
            sbatch_cmd = sbatch_cmd["--cpus-per-task", config["cpus-per-task"]]

        job_id = (sbatch_cmd | awk['{ print $4 }'])()
        jobs.append(job_id)
    return jobs

class Chimaira(cli.Application):

    """Execute the polyprof study on the chimaira cluster via slurm"""

    @cli.switch(["-E", "--experiment"], str, list=True, mandatory=True,
                help="Which experiments should be built")
    def experiments(self, exps):
        config["experiments"] = exps

    @cli.switch(["--nodedir"], str, help="Where to build on the node")
    def nodedir(self, dirname):
        config["nodedir"] = dirname

    @cli.switch(["--resultsdir"], str, mandatory=True,
                help="Where should the results be placed")
    def resultsdir(self, dirname):
        config["resultsdir"] = dirname

    @cli.switch(["--scriptname"], str,
                help="How should the sbatch script be called?")
    def scriptname(self, name):
        config["slurm_script"] = name

    @cli.switch(["--cpus-per-task"], int,
                help="How many cpu should we request per task?")
    def cpus_per_task(self, cpt):
        config["cpus-per-task"] = cpt

    @cli.switch(["-I"], str, help="ISL prefix")
    def isl(self, isl_prefix):
        config["isl"] = isl_prefix

    @cli.switch(["-L"], str, help="Likwid prefix")
    def likwid(self, likwid_prefix):
        config["likwiddir"] = likwid_prefix

    @cli.switch(["-P"], str, help="PAPI prefix")
    def papi(self, papi_prefix):
        config["papi"] = papi_prefix

    @cli.switch(["-p"], str, help="SLURM partition to use")
    def partition(self, partition):
        config["partition"] = partition

    @cli.switch(["-A"], str, help="SLURM account to use")
    def account(self, account):
        config["account"] = account

    @cli.switch(["--llvm-prefix"], str, help="LLVM prefix")
    def llvm(self, llvm_prefix):
        config["llvm"] = llvm_prefix

    @cli.switch(["--build"], help="Build LLVM/Polly/Polli/Clang on the node")
    def local_build(self):
        config["local_build"] = True

    def main(self):
        from itertools import chain

        # That is an awful lot of filtering required for parsing that
        # output... grmpf
        for exp in config["experiments"]:
            print "Experiment: {}".format(exp)
            pprof_list = pprof["run", "-l", "-E", exp]
            prj_list = pprof_list().split(',')
            prj_list = map(unicode.strip, prj_list)
            prj_list = map(lambda x: x.split('\n'), prj_list)
            prj_list = filter(None, prj_list)
            prj_list = list(chain.from_iterable(prj_list))
            prj_list = filter(lambda x: not '>>' in x, prj_list)
            prj_list = filter(None, prj_list)
            print "  {} projects".format(len(prj_list))
            print
            print ", ".join(prj_list)
            jobs = dispatch_jobs(exp, prj_list)
        #    dispatch_collect_job(exp, jobs)

if __name__ == '__main__':
    Chimaira.run()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding:utf-8 -*-
# Powered By KK Studio
# Domain Page

from BaseHandler import BaseHandler
from tornado.web import authenticated as Auth
from model.models import Domain, Groups, Record

class IndexHandler(BaseHandler):
    @Auth
    def get(self):
        page = int(self.get_argument('page', 1))
        line = int(self.get_argument('line', 20))
        offset = (page - 1) * line
        data = self.db.query(Domain).order_by(Domain.id.desc()).offset(offset).limit(line).all()
        grps = self.db.query(Groups).all()
        group = {}
        for i in grps:
            group[i.id] = i.name
        status = {1:u'已启用',2:u'暂停解析'}
        self.nav_active['/domain'] = 'active'
        self.render('domain/index.html',data=data,group=group,status=status)

# 新增域名
class CreateDomainHandler(BaseHandler):
    @Auth
    def post(self):
        domain = self.get_argument('domain',None)
        gid = self.get_argument('gid',None)
        comment = self.get_argument('comment',None)
        if not domain and not gid:
            return self.jsonReturn({'code': -1, 'msg': u'参数错误'})
        chk = self.db.query(Domain).filter_by(zone=domain).first()
        if chk:
            return self.jsonReturn({'code': -2, 'msg': u'域名重复'})
        # Check Group ID
        grp = self.db.query(Groups).filter_by(id=gid).first()
        if not grp:
            return self.jsonReturn({'code': -3, 'msg': u'分组不存在'})
        d = Domain(zone=domain,gid=gid,comment=comment,create_time=self.time,update_time=self.time)
        self.db.add(d)
        self.db.commit()
        if d.id:
            self.db.query(Groups).filter_by(id=gid).update({'domain_count': Groups.domain_count+1})
            self.db.commit()
            code = 0
            msg = u'成功添加域名'
        else:
            self.db.rollback()
            gid = 0
            code = -4
            msg = u'保存域名失败'
        return self.jsonReturn({'code': code, 'msg': msg, 'gid': gid})

# 编辑域名
class UpdateDomainHandler(BaseHandler):
    @Auth
    def post(self):
        domain_id = self.get_argument('id',None)
        gid = self.get_argument('gid',None)
        comment = self.get_argument('comment',None)
        if not domain_id and not gid:
            return self.jsonReturn({'code': -1, 'msg': u'参数错误'})
        data = self.db.query(Domain).filter_by(id=domain_id).first()
        if not data:
            return self.jsonReturn({'code': -2, 'msg': u'域名不存在'})
        # Check Group ID
        group = self.db.query(Groups).filter_by(id=gid).first()
        if not group:
            return self.jsonReturn({'code': -3, 'msg': u'分组不存在'})
        self.db.query(Domain).filter_by(id=domain_id).update({'gid': gid, 'comment': comment, 'update_time': self.time})
        self.db.query(Groups).filter_by(id=data.gid).update({'domain_count': Groups.domain_count-1})
        self.db.query(Groups).filter_by(id=gid).update({'domain_count': Groups.domain_count+1})
        self.db.commit()
        return self.jsonReturn({'code': 0, 'msg': 'Success'})

# 域名状态管理
class StatusDomainHandler(BaseHandler):
    @Auth
    def post(self):
        status = self.get_argument('status',None) # 1 or 2
        id = self.get_argument('id',None)
        if not id and status not in ['1','2']:
            return self.jsonReturn({'code': -1, 'msg': u'参数错误'})
        data = self.db.query(Domain).filter_by(id=id).first()
        if not data:
            return self.jsonReturn({'code': -2, 'msg': u'域名不存在'})
        self.db.query(Domain).filter_by(id=id).update({'status': status, 'update_time': self.time})
        self.db.commit()
        return self.jsonReturn({'code': 0, 'msg': 'Success'})

# 删除域名：这里需要添加权限控制，风险操作！！！
class DeleteDomainHandler(BaseHandler):
    @Auth
    def post(self):
        id = self.get_argument('id',None)
        if not id:
            return self.jsonReturn({'code': -1, 'msg': u'参数错误'})
        data = self.db.query(Domain).filter_by(id=id).first()
        if not data:
            return self.jsonReturn({'code': -2, 'msg': u'域名不存在'})
        self.db.query(Domain).filter_by(id=id).delete() # 删除域名
        self.db.query(Record).filter_by(zone=data.zone).delete() # 删除解析记录
        self.db.query(Groups).filter_by(id=data.gid).update({'domain_count': Groups.domain_count - 1}) # 更新分组数据统计
        self.db.commit()
        return self.jsonReturn({'code': 0, 'msg': 'Success'})

class GroupHandler(BaseHandler):
    @Auth
    def get(self):
        self.nav_active['/domain/group'] = 'active'
        data = self.db.query(Groups).order_by(Groups.id.desc()).all()
        self.render('domain/group.html',data=data)

class RecordHandler(BaseHandler):
    @Auth
    def get(self):
        zone = self.get_argument('zone',None)
        if zone:
            data = self.db.query(Record).filter_by(zone=zone).order_by(Record.host,Record.type,Record.data).all()
        else:
            zone = ''
            data = []
        status = {1: u'已启用', 2: u'暂停解析'}
        self.nav_active['/domain/record'] = 'active'
        self.render('domain/record.html',data=data,status=status,zone=zone)

<commit_msg>Add create record interface<commit_after>#!/usr/bin/python
# -*- coding:utf-8 -*-
# Powered By KK Studio
# Domain Page

from BaseHandler import BaseHandler
from tornado.web import authenticated as Auth
from model.models import Domain, Groups, Record

class IndexHandler(BaseHandler):
    @Auth
    def get(self):
        page = int(self.get_argument('page', 1))
        line = int(self.get_argument('line', 20))
        offset = (page - 1) * line
        data = self.db.query(Domain).order_by(Domain.id.desc()).offset(offset).limit(line).all()
        grps = self.db.query(Groups).all()
        group = {}
        for i in grps:
            group[i.id] = i.name
        status = {1:u'已启用',2:u'暂停解析'}
        self.nav_active['/domain'] = 'active'
        self.render('domain/index.html',data=data,group=group,status=status)

# 新增域名
class CreateDomainHandler(BaseHandler):
    @Auth
    def post(self):
        domain = self.get_argument('domain',None)
        gid = self.get_argument('gid',None)
        comment = self.get_argument('comment',None)
        if not domain and not gid:
            return self.jsonReturn({'code': -1, 'msg': u'参数错误'})
        chk = self.db.query(Domain).filter_by(zone=domain).first()
        if chk:
            return self.jsonReturn({'code': -2, 'msg': u'域名重复'})
        # Check Group ID
        grp = self.db.query(Groups).filter_by(id=gid).first()
        if not grp:
            return self.jsonReturn({'code': -3, 'msg': u'分组不存在'})
        d = Domain(zone=domain,gid=gid,comment=comment,create_time=self.time,update_time=self.time)
        self.db.add(d)
        self.db.commit()
        if d.id:
            self.db.query(Groups).filter_by(id=gid).update({'domain_count': Groups.domain_count+1})
            self.db.commit()
            code = 0
            msg = u'成功添加域名'
        else:
            self.db.rollback()
            gid = 0
            code = -4
            msg = u'保存域名失败'
        return self.jsonReturn({'code': code, 'msg': msg, 'gid': gid})

# 编辑域名
class UpdateDomainHandler(BaseHandler):
    @Auth
    def post(self):
        domain_id = self.get_argument('id',None)
        gid = self.get_argument('gid',None)
        comment = self.get_argument('comment',None)
        if not domain_id and not gid:
            return self.jsonReturn({'code': -1, 'msg': u'参数错误'})
        data = self.db.query(Domain).filter_by(id=domain_id).first()
        if not data:
            return self.jsonReturn({'code': -2, 'msg': u'域名不存在'})
        # Check Group ID
        group = self.db.query(Groups).filter_by(id=gid).first()
        if not group:
            return self.jsonReturn({'code': -3, 'msg': u'分组不存在'})
        self.db.query(Domain).filter_by(id=domain_id).update({'gid': gid, 'comment': comment, 'update_time': self.time})
        self.db.query(Groups).filter_by(id=data.gid).update({'domain_count': Groups.domain_count-1})
        self.db.query(Groups).filter_by(id=gid).update({'domain_count': Groups.domain_count+1})
        self.db.commit()
        return self.jsonReturn({'code': 0, 'msg': 'Success'})

# 域名状态管理
class StatusDomainHandler(BaseHandler):
    @Auth
    def post(self):
        status = self.get_argument('status',None) # 1 or 2
        id = self.get_argument('id',None)
        if not id and status not in ['1','2']:
            return self.jsonReturn({'code': -1, 'msg': u'参数错误'})
        data = self.db.query(Domain).filter_by(id=id).first()
        if not data:
            return self.jsonReturn({'code': -2, 'msg': u'域名不存在'})
        self.db.query(Domain).filter_by(id=id).update({'status': status, 'update_time': self.time})
        self.db.commit()
        return self.jsonReturn({'code': 0, 'msg': 'Success'})

# 删除域名：这里需要添加权限控制，风险操作！！！
class DeleteDomainHandler(BaseHandler):
    @Auth
    def post(self):
        id = self.get_argument('id',None)
        if not id:
            return self.jsonReturn({'code': -1, 'msg': u'参数错误'})
        data = self.db.query(Domain).filter_by(id=id).first()
        if not data:
            return self.jsonReturn({'code': -2, 'msg': u'域名不存在'})
        self.db.query(Domain).filter_by(id=id).delete() # 删除域名
        self.db.query(Record).filter_by(zone=data.zone).delete() # 删除解析记录
        self.db.query(Groups).filter_by(id=data.gid).update({'domain_count': Groups.domain_count - 1}) # 更新分组数据统计
        self.db.commit()
        return self.jsonReturn({'code': 0, 'msg': 'Success'})

class GroupHandler(BaseHandler):
    @Auth
    def get(self):
        self.nav_active['/domain/group'] = 'active'
        data = self.db.query(Groups).order_by(Groups.id.desc()).all()
        self.render('domain/group.html',data=data)

class RecordHandler(BaseHandler):
    @Auth
    def get(self):
        zone = self.get_argument('zone',None)
        if zone:
            data = self.db.query(Record).filter_by(zone=zone).order_by(Record.host,Record.type,Record.data).all()
        else:
            zone = ''
            data = []
        status = {1: u'已启用', 2: u'暂停解析'}
        self.nav_active['/domain/record'] = 'active'
        self.render('domain/record.html',data=data,status=status,zone=zone)

# 新增记录
class CreateRecordHandler(BaseHandler):
    @Auth
    def post(self):
        host = self.get_argument('host', None)
        zone = self.get_argument('zone', None)
        type = self.get_argument('type', None)
        data = self.get_argument('data', None)
        ttl = self.get_argument('ttl', 600)
        mx_priority = self.get_argument('mx_priority', None)
        comment = self.get_argument('comment', None)
        if not host or not zone or not type or not data:
            return self.jsonReturn({'code': -1, 'msg': u'参数错误'})
        type = type.upper()
        if type not in ['A','AAAA','MX','NS','CNAME','TXT','PTR']:
            return self.jsonReturn({'code': -1, 'msg': u'解析类型错误'})
        if host == '@' and type in ['NS','SOA']:
            return self.jsonReturn({'code': -2, 'msg': u'禁止添加该类型的解析'})
        domain = self.db.query(Domain).filter_by(zone=zone).first()
        if not domain:
            return self.jsonReturn({'code': -3, 'msg': u'域名不存在'})
        if type in ['A','AAAA','NS','CNAME','TXT','PTR']:
            r = Record(host=host,zone=zone,type=type,data=data,ttl=ttl, comment=comment, create_time=self.time, update_time=self.time)
        else: # type = 'MX'
            if not mx_priority:
                mx_priority = 10
            r = Record(host=host, zone=zone, type=type, data=data, ttl=ttl, mx_priority=mx_priority, comment=comment, create_time=self.time,update_time=self.time)
        self.db.add(r)
        self.db.query(Domain).filter_by(id=domain.id).update({'record_count': Domain.record_count + 1}) # 更新记录统计
        self.db.commit()
        return self.jsonReturn({'code': 0, 'msg': 'Success'})

<|endoftext|>


<commit_before>import numpy as np
from scipy.ndimage import filters
import scipy.signal
from kid_readout.utils.fftfilt import fftfilt
import random

def lpf256(ts):
    return fftfilt(scipy.signal.firwin(256,1/256.),ts)

def medmadmask(ts,thresh=8,axis=0):
    med = np.median(ts,axis=axis)
    deviations = np.abs(ts-med)
    mad = np.median(deviations,axis=axis)
    mask = deviations > thresh*mad
    return mask

def deglitch_block(ts,thresh=5):
    tsl = np.roll(np.abs(fftfilt(scipy.signal.firwin(16,1/16.),ts)),-8)
    mask = medmadmask(tsl,thresh=thresh)
    mask[:-50] = mask[:-50] | mask[50:]
    mask[50:] = mask[50:] | mask[:-50]
    nmask = mask.sum()
#    print "rejecting",nmask/float(ts.shape[0])
    out = ts.copy()
    try:
        out[mask] = np.array(random.sample(ts[~mask],nmask))
    except ValueError:
        print "more masked values than samples to draw from!"
    return out

def deglitch_window(data, window_length, thresh=6):
    out = np.empty_like(data)
    step = window_length//2
    nstep = data.shape[0]//step
    for k in xrange(nstep):
        start = k-1
        if start < 0:
            start = 0
#        print start*step,(k+1)*step,
        chunk = data[start*step:(k+1)*step]
        res = deglitch_block(chunk,thresh=thresh)
        out[start*step:((start+1)*step)] = res[:step]
    out[start*step:start*step + len(res)] = res
    return out

#the following takes up way too much memory and time, no good.
def despike_full(data, window_length, rejection_threshold=7, preprocess_function=lpf256):
    preproc = np.abs(lpf256(data))
    medians = filters.median_filter(preproc, window_length)
    abs_deviations = np.abs(preproc - medians)
    mads = filters.median_filter(abs_deviations, window_length)
    spike_flags = abs_deviations > (mads*rejection_threshold)
    nspikes = spike_flags.sum()
    data[spike_flags] = np.array(random.sample(data[~spike_flags],nspikes))
    return data<commit_msg>Add improved deglitching code. Not yet used in pipeline<commit_after>import numpy as np
from scipy.ndimage import filters
import scipy.signal
from kid_readout.utils.fftfilt import fftfilt
import random

def lpf256(ts):
    return fftfilt(scipy.signal.firwin(256,1/256.),ts)

def medmadmask(ts,thresh=8,axis=0):
    med = np.median(ts,axis=axis)
    deviations = np.abs(ts-med)
    mad = np.median(deviations,axis=axis)
    mask = deviations > thresh*mad
    return mask

def deglitch_block(ts,thresh=5):
    tsl = np.roll(np.abs(fftfilt(scipy.signal.firwin(16,1/16.),ts)),-8)
    mask = medmadmask(tsl,thresh=thresh)
    mask[:-50] = mask[:-50] | mask[50:]
    mask[50:] = mask[50:] | mask[:-50]
    nmask = mask.sum()
#    print "rejecting",nmask/float(ts.shape[0])
    out = ts.copy()
    try:
        out[mask] = np.array(random.sample(ts[~mask],nmask))
    except ValueError:
        print "more masked values than samples to draw from!"
    return out

def deglitch_window(data, window_length, thresh=6):
    out = np.empty_like(data)
    step = window_length//2
    nstep = data.shape[0]//step
    for k in xrange(nstep):
        start = k-1
        if start < 0:
            start = 0
#        print start*step,(k+1)*step,
        chunk = data[start*step:(k+1)*step]
        res = deglitch_block(chunk,thresh=thresh)
        out[start*step:((start+1)*step)] = res[:step]
    out[start*step:start*step + len(res)] = res
    return out

#the following takes up way too much memory and time, no good.
def despike_full(data, window_length, rejection_threshold=7, preprocess_function=lpf256):
    preproc = np.abs(lpf256(data))
    medians = filters.median_filter(preproc, window_length)
    abs_deviations = np.abs(preproc - medians)
    mads = filters.median_filter(abs_deviations, window_length)
    spike_flags = abs_deviations > (mads*rejection_threshold)
    nspikes = spike_flags.sum()
    data[spike_flags] = np.array(random.sample(data[~spike_flags],nspikes))
    return data

def deglitch_mask_block_mad(ts,thresh=5,mask_extend=50,debug=False):
    median = np.median(ts)
    deviations = np.abs(ts-median)
    mad = np.median(deviations)
    mask = deviations > (mad*thresh)
    if debug:
        from matplotlib import pyplot as plt
        plt.plot(deviations)
        plt.plot(mask*deviations.max(),'o')
    new_mask = mask.copy()
    for offset in range(1,mask_extend):
        new_mask[:-offset] |= mask[offset:]
        new_mask[offset:] |= mask[:-offset]
    mask = new_mask
#    mask[:-mask_extend] = (~mask[:-mask_extend] | ~mask[mask_extend:])
#    mask[mask_extend:] = mask[mask_extend:] | mask[:-mask_extend]
    if debug:
        plt.plot(mask*deviations.max(),'x',mew=2)
    return mask
def deglitch_mask_mad(ts,thresh=5,mask_extend=50,window_length=2**8):
    full_mask = np.zeros(ts.shape,dtype='bool')
    step = window_length//2
    nstep = ts.shape[0]//step
    for k in xrange(nstep):
        start = k-1
        if start < 0:
            start = 0
        chunk = ts[start*step:(k+1)*step]
        mask = deglitch_mask_block_mad(chunk,thresh=thresh,mask_extend=mask_extend)
        full_mask[start*step:((start+1)*step)] |= mask[:step]
    full_mask[start*step:start*step+len(mask)] |= mask
    return full_mask

<|endoftext|>


<commit_before>from kolibri.plugins.app.kolibri_plugin import App
from kolibri.plugins.registry import registered_plugins

SHARE_FILE = "share_file"

CAPABILITES = (SHARE_FILE,)

class AppInterface(object):
    __slot__ = "_capabilities"

    def __init__(self):
        self._capabilities = {}

    def __contains__(self, capability):
        return capability in self._capabilities

    def register(self, **kwargs):
        for capability in CAPABILITES:
            if capability in kwargs:
                self._capabilities[capability] = kwargs[capability]

    @property
    def enabled(self):
        return App in registered_plugins

    @property
    def capabilities(self):
        if self.enabled:
            return {key: (key in self._capabilities) for key in CAPABILITES}
        return {key: False for key in CAPABILITES}

    def share_file(self, filename, message):
        if SHARE_FILE not in self._capabilities:
            raise NotImplementedError("Sharing files is not supported on this platform")
        return self._capabilities[SHARE_FILE](filename=filename, message=message)

interface = AppInterface()
<commit_msg>Add method to get initialize url.<commit_after>from django.core.urlresolvers import reverse

from kolibri.plugins.app.kolibri_plugin import App
from kolibri.plugins.registry import registered_plugins

SHARE_FILE = "share_file"

CAPABILITES = (SHARE_FILE,)

class AppInterface(object):
    __slot__ = "_capabilities"

    def __init__(self):
        self._capabilities = {}

    def __contains__(self, capability):
        return capability in self._capabilities

    def register(self, **kwargs):
        for capability in CAPABILITES:
            if capability in kwargs:
                self._capabilities[capability] = kwargs[capability]

    def get_initialize_url(self, next_url=None):
        if not self.enabled:
            raise RuntimeError("App plugin is not enabled")
        # Import here to prevent a circular import
        from kolibri.core.device.models import DeviceAppKey

        url = reverse(
            "kolibri:kolibri.plugins.app:initialize", args=(DeviceAppKey.get_app_key(),)
        )
        if next_url is None:
            return url
        return url + "?next={}".format(next_url)

    @property
    def enabled(self):
        return App in registered_plugins

    @property
    def capabilities(self):
        if self.enabled:
            return {key: (key in self._capabilities) for key in CAPABILITES}
        return {key: False for key in CAPABILITES}

    def share_file(self, filename, message):
        if SHARE_FILE not in self._capabilities:
            raise NotImplementedError("Sharing files is not supported on this platform")
        return self._capabilities[SHARE_FILE](filename=filename, message=message)

interface = AppInterface()
<|endoftext|>


<commit_before>import re

from collections import OrderedDict
from bs4 import BeautifulSoup

from wikicurses import formats

class UrwidMarkupHandler:
    def __init__(self):
        self._list = []

    def add(self, text, attribute):
        if self and self[-1][0] == attribute:
            self[-1][1] += text
        else:
            self._list.append([attribute, text])

    def __iter__(self):
        return map(tuple, self._list)

    def __len__(self):
        return len(self._list)

    def __getitem__(self, key):
        return self._list[key]

def _processExtractSection(section):
    items = UrwidMarkupHandler()
    for i in section:
        if isinstance(i, str):
            strings = (i,)
        else:
            if i.name == 'h2' or i.find('h2'):
                break
            strings = i.strings
        for item in strings:
            if [i for i in item.parents if i.name == 'table'\
                    and {'wiki-sidebar', 'infobox'}.intersection(i.get('class', ()))]:
                continue
            partags = {i.name for i in item.parents}
            format = sum(formats[i] for i in set(i.name for i in formats).intersection(partags))
            if [i for i in partags if re.fullmatch('h[3-6]', i)]:
                format = 'h'
                item = re.sub('\[edit\]$', '', item)
                item = re.sub('Edit$', '', item)
            items.add(item, format)
    if items:
        items[0][1] = items[0][1].lstrip()
        items[-1][1] = items[-1][1].rstrip() + '\n'
    return items

def parseExtract(html):
    html = re.sub('\n+', '\n', html).replace('\t', ' ')
    sections = OrderedDict()
    soup = BeautifulSoup(html)
    for i in soup.find_all(['p', 'br']):
        i.insert_after(soup.new_string('\n'))
    for i in soup.find_all('li'):
        i.insert_before(soup.new_string('- '))
    sections[''] = _processExtractSection(soup.body or soup)
    for i in soup.find_all('h2'):
        title = re.sub('\[edit\]$', '', i.text)
        title = re.sub('Edit$', '', title)
        if title not in ('Contents', 'External links', 'References'):
            sections[title.strip()] = _processExtractSection(i.next_siblings)
    for i in sections:
        if not sections[i]:
            del sections[i]
    return sections

def parseFeature(html):
    return BeautifulSoup(html).text

def _processDisambigSection(section):
    items = []
    for i in section:
        if isinstance(i, str):
            continue
        if i.name == 'h2' or i.find('h2'):
            break
        for item in i.find_all('li'):
            items.append((item.a.text if item.a else '', item.text.split('\n')[0]))
    return items

def parseDisambig(html):
    sections = OrderedDict()
    soup = BeautifulSoup(html)
    sections[''] = _processDisambigSection(soup)
    for i in soup.find_all('h2'):
        title = re.sub('\[edit\]$', '', i.text)
        title = re.sub('Edit$', '', title)
        if title not in ('Contents', 'See also'):
            sections[title] = _processDisambigSection(i.next_siblings)
    return sections
<commit_msg>Two blank lines between sections instead of one<commit_after>import re

from collections import OrderedDict
from bs4 import BeautifulSoup

from wikicurses import formats

class UrwidMarkupHandler:
    def __init__(self):
        self._list = []

    def add(self, text, attribute):
        if self and self[-1][0] == attribute:
            self[-1][1] += text
        else:
            self._list.append([attribute, text])

    def __iter__(self):
        return map(tuple, self._list)

    def __len__(self):
        return len(self._list)

    def __getitem__(self, key):
        return self._list[key]

def _processExtractSection(section):
    items = UrwidMarkupHandler()
    for i in section:
        if isinstance(i, str):
            strings = (i,)
        else:
            if i.name == 'h2' or i.find('h2'):
                break
            strings = i.strings
        for item in strings:
            if [i for i in item.parents if i.name == 'table'\
                    and {'wiki-sidebar', 'infobox'}.intersection(i.get('class', ()))]:
                continue
            partags = {i.name for i in item.parents}
            format = sum(formats[i] for i in set(i.name for i in formats).intersection(partags))
            if [i for i in partags if re.fullmatch('h[3-6]', i)]:
                format = 'h'
                item = re.sub('\[edit\]$', '', item)
                item = re.sub('Edit$', '', item)
            items.add(item, format)
    if items:
        items[0][1] = items[0][1].lstrip()
        items[-1][1] = items[-1][1].rstrip() + '\n\n'
    return items

def parseExtract(html):
    html = re.sub('\n+', '\n', html).replace('\t', ' ')
    sections = OrderedDict()
    soup = BeautifulSoup(html)
    for i in soup.find_all(['p', 'br']):
        i.insert_after(soup.new_string('\n'))
    for i in soup.find_all('li'):
        i.insert_before(soup.new_string('- '))
    sections[''] = _processExtractSection(soup.body or soup)
    for i in soup.find_all('h2'):
        title = re.sub('\[edit\]$', '', i.text)
        title = re.sub('Edit$', '', title)
        if title not in ('Contents', 'External links', 'References'):
            sections[title.strip()] = _processExtractSection(i.next_siblings)
    for i in sections:
        if not sections[i]:
            del sections[i]
    return sections

def parseFeature(html):
    return BeautifulSoup(html).text

def _processDisambigSection(section):
    items = []
    for i in section:
        if isinstance(i, str):
            continue
        if i.name == 'h2' or i.find('h2'):
            break
        for item in i.find_all('li'):
            items.append((item.a.text if item.a else '', item.text.split('\n')[0]))
    return items

def parseDisambig(html):
    sections = OrderedDict()
    soup = BeautifulSoup(html)
    sections[''] = _processDisambigSection(soup)
    for i in soup.find_all('h2'):
        title = re.sub('\[edit\]$', '', i.text)
        title = re.sub('Edit$', '', title)
        if title not in ('Contents', 'See also'):
            sections[title] = _processDisambigSection(i.next_siblings)
    return sections
<|endoftext|>