<commit_before># Copyright 2019-present MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Run the SRV support tests."""

import sys

from time import sleep

sys.path[0:0] = [""]

import pymongo

from pymongo import common
from pymongo.errors import ConfigurationError
from pymongo.srv_resolver import _HAVE_DNSPYTHON
from pymongo.mongo_client import MongoClient
from test import client_knobs, unittest
from test.utils import wait_until, FunctionCallRecorder

WAIT_TIME = 0.1

class SrvPollingKnobs(object):
    def __init__(self, ttl_time=None, min_srv_rescan_interval=None,
                 nodelist_callback=None,
                 count_resolver_calls=False):
        self.ttl_time = ttl_time
        self.min_srv_rescan_interval = min_srv_rescan_interval
        self.nodelist_callback = nodelist_callback
        self.count_resolver_calls = count_resolver_calls

        self.old_min_srv_rescan_interval = None
        self.old_dns_resolver_response = None

    def enable(self):
        self.old_min_srv_rescan_interval = common.MIN_SRV_RESCAN_INTERVAL
        self.old_dns_resolver_response = \
            pymongo.srv_resolver._SrvResolver.get_hosts_and_min_ttl

        if self.min_srv_rescan_interval is not None:
            common.MIN_SRV_RESCAN_INTERVAL = self.min_srv_rescan_interval

        def mock_get_hosts_and_min_ttl(resolver, *args):
            nodes, ttl = self.old_dns_resolver_response(resolver)
            if self.nodelist_callback is not None:
                nodes = self.nodelist_callback()
            if self.ttl_time is not None:
                ttl = self.ttl_time
            return nodes, ttl

        if self.count_resolver_calls:
            patch_func = FunctionCallRecorder(mock_get_hosts_and_min_ttl)
        else:
            patch_func = mock_get_hosts_and_min_ttl

        pymongo.srv_resolver._SrvResolver.get_hosts_and_min_ttl = patch_func

    def __enter__(self):
        self.enable()

    def disable(self):
        common.MIN_SRV_RESCAN_INTERVAL = self.old_min_srv_rescan_interval
        pymongo.srv_resolver._SrvResolver.get_hosts_and_min_ttl = \
            self.old_dns_resolver_response

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.disable()

class TestSrvPolling(unittest.TestCase):

    BASE_SRV_RESPONSE = [
        ("localhost.test.build.10gen.cc", 27017),
        ("localhost.test.build.10gen.cc", 27018)]

    CONNECTION_STRING = "mongodb+srv://test1.test.build.10gen.cc"

    def setUp(self):
        if not _HAVE_DNSPYTHON:
            raise unittest.SkipTest("SRV polling tests require the dnspython "
                                    "module")
        # Patch timeouts to ensure short rescan SRV interval.
        self.client_knobs = client_knobs(
            heartbeat_frequency=WAIT_TIME, min_heartbeat_interval=WAIT_TIME,
            events_queue_frequency=WAIT_TIME)
        self.client_knobs.enable()

    def tearDown(self):
        self.client_knobs.disable()

    def get_nodelist(self, client):
        return client._topology.description.server_descriptions().keys()

    def assert_nodelist_change(self, expected_nodelist, client):
        """Check if the client._topology eventually sees all nodes in the
        expected_nodelist.
        """
        def predicate():
            nodelist = self.get_nodelist(client)
            if set(expected_nodelist) == set(nodelist):
                return True
            return False
        wait_until(predicate, "see expected nodelist",
                   timeout=100*WAIT_TIME)

    def assert_nodelist_nochange(self, expected_nodelist, client):
        """Check if the client._topology ever deviates from seeing all nodes
        in the expected_nodelist. Consistency is checked after sleeping for
        (WAIT_TIME * 10) seconds. Also check that the resolver is called at
        least once.
        """
        sleep(WAIT_TIME*10)
        nodelist = self.get_nodelist(client)
        if set(expected_nodelist) != set(nodelist):
            msg = "Client nodelist %s changed unexpectedly (expected %s)"
            raise self.fail(msg % (nodelist, expected_nodelist))
        self.assertGreaterEqual(
            pymongo.srv_resolver._SrvResolver.get_hosts_and_min_ttl.call_count,
            1, "resolver was never called")
        return True

    def run_scenario(self, dns_response, expect_change):
        if callable(dns_response):
            dns_resolver_response = dns_response
        else:
            def dns_resolver_response():
                return dns_response

        if expect_change:
            assertion_method = self.assert_nodelist_change
            count_resolver_calls = False
            expected_response = dns_response
        else:
            assertion_method = self.assert_nodelist_nochange
            count_resolver_calls = True
            expected_response = self.BASE_SRV_RESPONSE

        # Patch timeouts to ensure short test running times.
        with SrvPollingKnobs(
                ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME):
            client = MongoClient(self.CONNECTION_STRING)
            self.assert_nodelist_change(self.BASE_SRV_RESPONSE, client)
            # Patch list of hosts returned by DNS query.
            with SrvPollingKnobs(
                    nodelist_callback=dns_resolver_response,
                    count_resolver_calls=count_resolver_calls):
                assertion_method(expected_response, client)

    def test_addition(self):
        response = self.BASE_SRV_RESPONSE[:]
        response.append(
            ("localhost.test.build.10gen.cc", 27019))
        self.run_scenario(response, True)

    def test_removal(self):
        response = self.BASE_SRV_RESPONSE[:]
        response.remove(
            ("localhost.test.build.10gen.cc", 27018))
        self.run_scenario(response, True)

    def test_replace_one(self):
        response = self.BASE_SRV_RESPONSE[:]
        response.remove(
            ("localhost.test.build.10gen.cc", 27018))
        response.append(
            ("localhost.test.build.10gen.cc", 27019))
        self.run_scenario(response, True)

    def test_replace_both_with_one(self):
        response = [("localhost.test.build.10gen.cc", 27019)]
        self.run_scenario(response, True)

    def test_replace_both_with_two(self):
        response = [("localhost.test.build.10gen.cc", 27019),
                    ("localhost.test.build.10gen.cc", 27020)]
        self.run_scenario(response, True)

    def test_dns_failures(self):
        from dns import exception
        for exc in (exception.FormError, exception.TooBig, exception.Timeout):
            def response_callback(*args):
                raise exc("DNS Failure!")
            self.run_scenario(response_callback, False)

    def test_dns_record_lookup_empty(self):
        response = []
        self.run_scenario(response, False)

    def _test_recover_from_initial(self, initial_callback):
        # Construct a valid final response callback distinct from base.
        response_final = self.BASE_SRV_RESPONSE[:]
        response_final.pop()
        def final_callback():
            return response_final

        with SrvPollingKnobs(
                ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME,
                nodelist_callback=initial_callback,
                count_resolver_calls=True):
            # Client uses unpatched method to get initial nodelist
            client = MongoClient(self.CONNECTION_STRING)
            # Invalid DNS resolver response should not change nodelist.
            self.assert_nodelist_nochange(self.BASE_SRV_RESPONSE, client)

        with SrvPollingKnobs(
                ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME,
                nodelist_callback=final_callback):
            # Nodelist should reflect new valid DNS resolver response.
            self.assert_nodelist_change(response_final, client)

    def test_recover_from_initially_empty_seedlist(self):
        def empty_seedlist():
            return []
        self._test_recover_from_initial(empty_seedlist)

    def test_recover_from_initially_erroring_seedlist(self):
        def erroring_seedlist():
            raise ConfigurationError
        self._test_recover_from_initial(erroring_seedlist)

    def test_10_all_dns_selected(self):
        response = [("localhost.test.build.10gen.cc", 27017),
                    ("localhost.test.build.10gen.cc", 27019),
                    ("localhost.test.build.10gen.cc", 27020)]

        def nodelist_callback():
            return response
        with SrvPollingKnobs(ttl_time=WAIT_TIME,
                             min_srv_rescan_interval=WAIT_TIME):
            client = MongoClient(self.CONNECTION_STRING, srvMaxHosts=0)
            self.addCleanup(client.close)
            with SrvPollingKnobs(nodelist_callback=nodelist_callback):
                self.assert_nodelist_change(response, client)

    def test_11_all_dns_selected(self):
        response = [("localhost.test.build.10gen.cc", 27019),
                    ("localhost.test.build.10gen.cc", 27020)]

        def nodelist_callback():
            return response

        with SrvPollingKnobs(
                ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME):
            client = MongoClient(self.CONNECTION_STRING, srvMaxHosts=2)
            self.addCleanup(client.close)
            with SrvPollingKnobs(nodelist_callback=nodelist_callback):
                self.assert_nodelist_change(response, client)

    def test_12_new_dns_randomly_selected(self):
        response = [("localhost.test.build.10gen.cc", 27020),
                    ("localhost.test.build.10gen.cc", 27019),
                    ("localhost.test.build.10gen.cc", 27017)]

        def nodelist_callback():
            return response

        with SrvPollingKnobs(
                ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME):
            client = MongoClient(self.CONNECTION_STRING, srvMaxHosts=2)
            self.addCleanup(client.close)
            with SrvPollingKnobs(nodelist_callback=nodelist_callback):
                sleep(2*common.MIN_SRV_RESCAN_INTERVAL)
                final_topology = set(
                    client.topology_description.server_descriptions())
                self.assertIn(("localhost.test.build.10gen.cc", 27017),
                              final_topology)
                self.assertEqual(len(final_topology), 2)

    def test_does_not_flipflop(self):
        with SrvPollingKnobs(
                ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME):
            client = MongoClient(self.CONNECTION_STRING, srvMaxHosts=1)
            self.addCleanup(client.close)
            old = set(client.topology_description.server_descriptions())
            sleep(4*WAIT_TIME)
            new = set(client.topology_description.server_descriptions())
            self.assertSetEqual(old, new)

if __name__ == '__main__':
    unittest.main()
<commit_msg>PYTHON-2948 Add prose test for SRV polling with a custom service name (#764)<commit_after># Copyright 2019-present MongoDB, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Run the SRV support tests."""

import sys

from time import sleep

sys.path[0:0] = [""]

import pymongo

from pymongo import common
from pymongo.errors import ConfigurationError
from pymongo.srv_resolver import _HAVE_DNSPYTHON
from pymongo.mongo_client import MongoClient
from test import client_knobs, unittest
from test.utils import wait_until, FunctionCallRecorder

WAIT_TIME = 0.1

class SrvPollingKnobs(object):
    def __init__(self, ttl_time=None, min_srv_rescan_interval=None,
                 nodelist_callback=None,
                 count_resolver_calls=False):
        self.ttl_time = ttl_time
        self.min_srv_rescan_interval = min_srv_rescan_interval
        self.nodelist_callback = nodelist_callback
        self.count_resolver_calls = count_resolver_calls

        self.old_min_srv_rescan_interval = None
        self.old_dns_resolver_response = None

    def enable(self):
        self.old_min_srv_rescan_interval = common.MIN_SRV_RESCAN_INTERVAL
        self.old_dns_resolver_response = \
            pymongo.srv_resolver._SrvResolver.get_hosts_and_min_ttl

        if self.min_srv_rescan_interval is not None:
            common.MIN_SRV_RESCAN_INTERVAL = self.min_srv_rescan_interval

        def mock_get_hosts_and_min_ttl(resolver, *args):
            nodes, ttl = self.old_dns_resolver_response(resolver)
            if self.nodelist_callback is not None:
                nodes = self.nodelist_callback()
            if self.ttl_time is not None:
                ttl = self.ttl_time
            return nodes, ttl

        if self.count_resolver_calls:
            patch_func = FunctionCallRecorder(mock_get_hosts_and_min_ttl)
        else:
            patch_func = mock_get_hosts_and_min_ttl

        pymongo.srv_resolver._SrvResolver.get_hosts_and_min_ttl = patch_func

    def __enter__(self):
        self.enable()

    def disable(self):
        common.MIN_SRV_RESCAN_INTERVAL = self.old_min_srv_rescan_interval
        pymongo.srv_resolver._SrvResolver.get_hosts_and_min_ttl = \
            self.old_dns_resolver_response

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.disable()

class TestSrvPolling(unittest.TestCase):

    BASE_SRV_RESPONSE = [
        ("localhost.test.build.10gen.cc", 27017),
        ("localhost.test.build.10gen.cc", 27018)]

    CONNECTION_STRING = "mongodb+srv://test1.test.build.10gen.cc"

    def setUp(self):
        if not _HAVE_DNSPYTHON:
            raise unittest.SkipTest("SRV polling tests require the dnspython "
                                    "module")
        # Patch timeouts to ensure short rescan SRV interval.
        self.client_knobs = client_knobs(
            heartbeat_frequency=WAIT_TIME, min_heartbeat_interval=WAIT_TIME,
            events_queue_frequency=WAIT_TIME)
        self.client_knobs.enable()

    def tearDown(self):
        self.client_knobs.disable()

    def get_nodelist(self, client):
        return client._topology.description.server_descriptions().keys()

    def assert_nodelist_change(self, expected_nodelist, client):
        """Check if the client._topology eventually sees all nodes in the
        expected_nodelist.
        """
        def predicate():
            nodelist = self.get_nodelist(client)
            if set(expected_nodelist) == set(nodelist):
                return True
            return False
        wait_until(predicate, "see expected nodelist",
                   timeout=100*WAIT_TIME)

    def assert_nodelist_nochange(self, expected_nodelist, client):
        """Check if the client._topology ever deviates from seeing all nodes
        in the expected_nodelist. Consistency is checked after sleeping for
        (WAIT_TIME * 10) seconds. Also check that the resolver is called at
        least once.
        """
        sleep(WAIT_TIME*10)
        nodelist = self.get_nodelist(client)
        if set(expected_nodelist) != set(nodelist):
            msg = "Client nodelist %s changed unexpectedly (expected %s)"
            raise self.fail(msg % (nodelist, expected_nodelist))
        self.assertGreaterEqual(
            pymongo.srv_resolver._SrvResolver.get_hosts_and_min_ttl.call_count,
            1, "resolver was never called")
        return True

    def run_scenario(self, dns_response, expect_change):
        if callable(dns_response):
            dns_resolver_response = dns_response
        else:
            def dns_resolver_response():
                return dns_response

        if expect_change:
            assertion_method = self.assert_nodelist_change
            count_resolver_calls = False
            expected_response = dns_response
        else:
            assertion_method = self.assert_nodelist_nochange
            count_resolver_calls = True
            expected_response = self.BASE_SRV_RESPONSE

        # Patch timeouts to ensure short test running times.
        with SrvPollingKnobs(
                ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME):
            client = MongoClient(self.CONNECTION_STRING)
            self.assert_nodelist_change(self.BASE_SRV_RESPONSE, client)
            # Patch list of hosts returned by DNS query.
            with SrvPollingKnobs(
                    nodelist_callback=dns_resolver_response,
                    count_resolver_calls=count_resolver_calls):
                assertion_method(expected_response, client)

    def test_addition(self):
        response = self.BASE_SRV_RESPONSE[:]
        response.append(
            ("localhost.test.build.10gen.cc", 27019))
        self.run_scenario(response, True)

    def test_removal(self):
        response = self.BASE_SRV_RESPONSE[:]
        response.remove(
            ("localhost.test.build.10gen.cc", 27018))
        self.run_scenario(response, True)

    def test_replace_one(self):
        response = self.BASE_SRV_RESPONSE[:]
        response.remove(
            ("localhost.test.build.10gen.cc", 27018))
        response.append(
            ("localhost.test.build.10gen.cc", 27019))
        self.run_scenario(response, True)

    def test_replace_both_with_one(self):
        response = [("localhost.test.build.10gen.cc", 27019)]
        self.run_scenario(response, True)

    def test_replace_both_with_two(self):
        response = [("localhost.test.build.10gen.cc", 27019),
                    ("localhost.test.build.10gen.cc", 27020)]
        self.run_scenario(response, True)

    def test_dns_failures(self):
        from dns import exception
        for exc in (exception.FormError, exception.TooBig, exception.Timeout):
            def response_callback(*args):
                raise exc("DNS Failure!")
            self.run_scenario(response_callback, False)

    def test_dns_record_lookup_empty(self):
        response = []
        self.run_scenario(response, False)

    def _test_recover_from_initial(self, initial_callback):
        # Construct a valid final response callback distinct from base.
        response_final = self.BASE_SRV_RESPONSE[:]
        response_final.pop()
        def final_callback():
            return response_final

        with SrvPollingKnobs(
                ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME,
                nodelist_callback=initial_callback,
                count_resolver_calls=True):
            # Client uses unpatched method to get initial nodelist
            client = MongoClient(self.CONNECTION_STRING)
            # Invalid DNS resolver response should not change nodelist.
            self.assert_nodelist_nochange(self.BASE_SRV_RESPONSE, client)

        with SrvPollingKnobs(
                ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME,
                nodelist_callback=final_callback):
            # Nodelist should reflect new valid DNS resolver response.
            self.assert_nodelist_change(response_final, client)

    def test_recover_from_initially_empty_seedlist(self):
        def empty_seedlist():
            return []
        self._test_recover_from_initial(empty_seedlist)

    def test_recover_from_initially_erroring_seedlist(self):
        def erroring_seedlist():
            raise ConfigurationError
        self._test_recover_from_initial(erroring_seedlist)

    def test_10_all_dns_selected(self):
        response = [("localhost.test.build.10gen.cc", 27017),
                    ("localhost.test.build.10gen.cc", 27019),
                    ("localhost.test.build.10gen.cc", 27020)]

        def nodelist_callback():
            return response
        with SrvPollingKnobs(ttl_time=WAIT_TIME,
                             min_srv_rescan_interval=WAIT_TIME):
            client = MongoClient(self.CONNECTION_STRING, srvMaxHosts=0)
            self.addCleanup(client.close)
            with SrvPollingKnobs(nodelist_callback=nodelist_callback):
                self.assert_nodelist_change(response, client)

    def test_11_all_dns_selected(self):
        response = [("localhost.test.build.10gen.cc", 27019),
                    ("localhost.test.build.10gen.cc", 27020)]

        def nodelist_callback():
            return response

        with SrvPollingKnobs(
                ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME):
            client = MongoClient(self.CONNECTION_STRING, srvMaxHosts=2)
            self.addCleanup(client.close)
            with SrvPollingKnobs(nodelist_callback=nodelist_callback):
                self.assert_nodelist_change(response, client)

    def test_12_new_dns_randomly_selected(self):
        response = [("localhost.test.build.10gen.cc", 27020),
                    ("localhost.test.build.10gen.cc", 27019),
                    ("localhost.test.build.10gen.cc", 27017)]

        def nodelist_callback():
            return response

        with SrvPollingKnobs(
                ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME):
            client = MongoClient(self.CONNECTION_STRING, srvMaxHosts=2)
            self.addCleanup(client.close)
            with SrvPollingKnobs(nodelist_callback=nodelist_callback):
                sleep(2*common.MIN_SRV_RESCAN_INTERVAL)
                final_topology = set(
                    client.topology_description.server_descriptions())
                self.assertIn(("localhost.test.build.10gen.cc", 27017),
                              final_topology)
                self.assertEqual(len(final_topology), 2)

    def test_does_not_flipflop(self):
        with SrvPollingKnobs(
                ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME):
            client = MongoClient(self.CONNECTION_STRING, srvMaxHosts=1)
            self.addCleanup(client.close)
            old = set(client.topology_description.server_descriptions())
            sleep(4*WAIT_TIME)
            new = set(client.topology_description.server_descriptions())
            self.assertSetEqual(old, new)

    def test_srv_service_name(self):
        # Construct a valid final response callback distinct from base.
        response = [
            ("localhost.test.build.10gen.cc.", 27019),
            ("localhost.test.build.10gen.cc.", 27020)
        ]

        def nodelist_callback():
            return response

        with SrvPollingKnobs(
                ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME):
            client = MongoClient(
                "mongodb+srv://test22.test.build.10gen.cc/?srvServiceName"
                "=customname")
            with SrvPollingKnobs(nodelist_callback=nodelist_callback):
                self.assert_nodelist_change(response, client)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>""" ntsne.py
    numpy wrapper for bh_tsne (https://github.com/lvdmaaten/bhtsne)
    Brian DeCost bdecost@andrew.cmu.edu
"""
import os
import re
import shutil
import struct
import tempfile
import subprocess
import numpy as np

# vdM's bh_tsne reads and writes from/to these hardcoded paths
# in the directory from which bh_tsne is run
DATAFILE = 'data.dat'
RESULTFILE = 'result.dat'
TSNESOURCE = 'https://github.com/lvdmaaten/bhtsne'

# check to see if bh_tsne in on system PATH or in the CWD;
# if not, clone and build it in ~/.ntsne if necessary
augmented_path = os.getenv('PATH') + ':{}'.format(os.getcwd())
TSNE = shutil.which('bh_tsne', path=augmented_path)
if TSNE is None:
    TSNEDIR = os.path.expanduser('~/.ntsne')
    TSNE = os.path.join(TSNEDIR, 'bh_tsne')

# set default bh_tsne parameters
THETA = 0.5
PERPLEXITY = 30
MAP_DIMS = 2
MAX_ITER = 1000
SEED = None

def build_bhtsne():
    """ clone and build lvdmaaten's bhtsne """
    subprocess.call(['git', 'clone', TSNESOURCE, TSNEDIR])
    subprocess.call(['g++', 'sptree.cpp', 'tsne.cpp', '-o', 'bh_tsne', '-O2'], cwd=TSNEDIR)
    return

def write_tsne_input(X, theta=THETA, perplexity=PERPLEXITY, map_dims=MAP_DIMS, max_iter=MAX_ITER, seed=SEED, cwd=''):
    """ serialize 2D data matrix (numpy array) with t-SNE options to vdM's binary input format """

    with open(os.path.join(cwd, DATAFILE), 'wb') as f:
        n, d = X.shape
        f.write(struct.pack('=i', n))   # number of instances
        f.write(struct.pack('=i', d))   # initial dimensionality
        f.write(struct.pack('=d', theta))
        f.write(struct.pack('=d', perplexity))
        f.write(struct.pack('=i', map_dims))
        f.write(struct.pack('=i', max_iter))
        f.write(X.tobytes())

        if seed is not None:
            f.write(struct.pack('=i', map_dims))

def read_tsne_results(cwd=''):
    """ load t-SNE results from vdM's binary results file format """
    with open(os.path.join(cwd, RESULTFILE), 'rb') as f:
        n, = struct.unpack('=i', f.read(4))  # number of instances
        md, = struct.unpack('=i', f.read(4)) # map dimensionality
        sz = struct.calcsize('=d')
        buf = f.read()
        x_tsne = [struct.unpack_from('=d', buf, sz*offset)
                  for offset in range(n*md)]

    return np.array(x_tsne).reshape((n,md))

def tsne(X, theta=THETA, perplexity=PERPLEXITY, map_dims=MAP_DIMS, max_iter=MAX_ITER, seed=SEED):
    """ simple wrapper function for applying bh_tsne to the data matrix X """
    with tempfile.TemporaryDirectory() as tmpdir:
        write_tsne_input(X, theta=theta, perplexity=perplexity,
                         map_dims=map_dims, max_iter=max_iter, seed=seed, cwd=tmpdir)

        subprocess.call(TSNE, cwd=tmpdir)
        x_tsne = read_tsne_results(cwd=tmpdir)

    return x_tsne

def tsne_error(results):
    """ find the error string for each iteration; get the min (likely the last iteration...) """
    errorstrings = re.findall('error is \d+\.\d+', results.decode())
    # lexicographic ordering is equivalent to splitting each sting and converting to float...
    error = min(errorstrings).split()[-1]
    return float(error)

def best_tsne(X, theta=THETA, perplexity=PERPLEXITY, map_dims=MAP_DIMS, max_iter=MAX_ITER, seed=SEED, n_repeats=10):
    """ run bh_tsne {n_iter} times and return results with lowest KL divergence """
    lowest_error = 1e9
    x_tsne = None

    with tempfile.TemporaryDirectory() as tmpdir:
        write_tsne_input(X, theta=theta, perplexity=perplexity,
                         map_dims=map_dims, max_iter=max_iter, seed=seed, cwd=tmpdir)

        for iteration in range(n_repeats):
            results = subprocess.check_output(TSNE, cwd=tmpdir)
            error = tsne_error(results)
            if error < lowest_error:
                lowest_error = error
                x_tsne = read_tsne_results(cwd=tmpdir)

    return x_tsne

if not os.path.isfile(TSNE):
    print('bh_tsne not found; cloning from {}'.format(TSNESOURCE))
    build_bhtsne()
<commit_msg>update parameter name in docstring for ntsne.best_tsne<commit_after>""" ntsne.py
    numpy wrapper for bh_tsne (https://github.com/lvdmaaten/bhtsne)
    Brian DeCost bdecost@andrew.cmu.edu
"""
import os
import re
import shutil
import struct
import tempfile
import subprocess
import numpy as np

# vdM's bh_tsne reads and writes from/to these hardcoded paths
# in the directory from which bh_tsne is run
DATAFILE = 'data.dat'
RESULTFILE = 'result.dat'
TSNESOURCE = 'https://github.com/lvdmaaten/bhtsne'

# check to see if bh_tsne in on system PATH or in the CWD;
# if not, clone and build it in ~/.ntsne if necessary
augmented_path = os.getenv('PATH') + ':{}'.format(os.getcwd())
TSNE = shutil.which('bh_tsne', path=augmented_path)
if TSNE is None:
    TSNEDIR = os.path.expanduser('~/.ntsne')
    TSNE = os.path.join(TSNEDIR, 'bh_tsne')

# set default bh_tsne parameters
THETA = 0.5
PERPLEXITY = 30
MAP_DIMS = 2
MAX_ITER = 1000
SEED = None

def build_bhtsne():
    """ clone and build lvdmaaten's bhtsne """
    subprocess.call(['git', 'clone', TSNESOURCE, TSNEDIR])
    subprocess.call(['g++', 'sptree.cpp', 'tsne.cpp', '-o', 'bh_tsne', '-O2'], cwd=TSNEDIR)
    return

def write_tsne_input(X, theta=THETA, perplexity=PERPLEXITY, map_dims=MAP_DIMS, max_iter=MAX_ITER, seed=SEED, cwd=''):
    """ serialize 2D data matrix (numpy array) with t-SNE options to vdM's binary input format """

    with open(os.path.join(cwd, DATAFILE), 'wb') as f:
        n, d = X.shape
        f.write(struct.pack('=i', n))   # number of instances
        f.write(struct.pack('=i', d))   # initial dimensionality
        f.write(struct.pack('=d', theta))
        f.write(struct.pack('=d', perplexity))
        f.write(struct.pack('=i', map_dims))
        f.write(struct.pack('=i', max_iter))
        f.write(X.tobytes())

        if seed is not None:
            f.write(struct.pack('=i', map_dims))

def read_tsne_results(cwd=''):
    """ load t-SNE results from vdM's binary results file format """
    with open(os.path.join(cwd, RESULTFILE), 'rb') as f:
        n, = struct.unpack('=i', f.read(4))  # number of instances
        md, = struct.unpack('=i', f.read(4)) # map dimensionality
        sz = struct.calcsize('=d')
        buf = f.read()
        x_tsne = [struct.unpack_from('=d', buf, sz*offset)
                  for offset in range(n*md)]

    return np.array(x_tsne).reshape((n,md))

def tsne(X, theta=THETA, perplexity=PERPLEXITY, map_dims=MAP_DIMS, max_iter=MAX_ITER, seed=SEED):
    """ simple wrapper function for applying bh_tsne to the data matrix X """
    with tempfile.TemporaryDirectory() as tmpdir:
        write_tsne_input(X, theta=theta, perplexity=perplexity,
                         map_dims=map_dims, max_iter=max_iter, seed=seed, cwd=tmpdir)

        subprocess.call(TSNE, cwd=tmpdir)
        x_tsne = read_tsne_results(cwd=tmpdir)

    return x_tsne

def tsne_error(results):
    """ find the error string for each iteration; get the min (likely the last iteration...) """
    errorstrings = re.findall('error is \d+\.\d+', results.decode())
    # lexicographic ordering is equivalent to splitting each sting and converting to float...
    error = min(errorstrings).split()[-1]
    return float(error)

def best_tsne(X, theta=THETA, perplexity=PERPLEXITY, map_dims=MAP_DIMS, max_iter=MAX_ITER, seed=SEED, n_repeats=10):
    """ run bh_tsne {n_repeats} times and return results with lowest KL divergence """
    lowest_error = 1e9
    x_tsne = None

    with tempfile.TemporaryDirectory() as tmpdir:
        write_tsne_input(X, theta=theta, perplexity=perplexity,
                         map_dims=map_dims, max_iter=max_iter, seed=seed, cwd=tmpdir)

        for __ in range(n_repeats):
            results = subprocess.check_output(TSNE, cwd=tmpdir)
            error = tsne_error(results)
            if error < lowest_error:
                lowest_error = error
                x_tsne = read_tsne_results(cwd=tmpdir)

    return x_tsne

if not os.path.isfile(TSNE):
    print('bh_tsne not found; cloning from {}'.format(TSNESOURCE))
    build_bhtsne()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# This is the python source code for a little app that lets you
# test various aspects of texturing in OpenGL.
#
# It is part of the Vision Egg package.
#
# Copyright (c) 2002 Andrew Straw.  Distributed under the terms of the
# GNU General Public License (GPL).

import string
__version__ = string.split('$Revision$')[1]
__date__ = string.join(string.split('$Date$')[1:3], ' ')
__author__ = 'Andrew Straw <astraw@users.sourceforge.net>'

import time

import pygame
from pygame.locals import *

from OpenGL.GL import * # PyOpenGL packages
from Numeric import *
import Tkinter

import VisionEgg
import VisionEgg.GUI

class TextureTestFrame(Tkinter.Frame):
    def __init__(self,master=None,**cnf):
        apply(Tkinter.Frame.__init__,(self,master),cnf)
        self.winfo_toplevel().title('Texture Test')
        self.pack()

        self.vid_info_frame = VisionEgg.GUI.VideoInfoFrame(self)
        self.vid_info_frame.pack()

        self.tex_compression = Tkinter.BooleanVar()
        self.tex_compression.set(VisionEgg.video_info.tex_compress)
        Tkinter.Checkbutton(self,
                            text="Texture compression",
                            variable=self.tex_compression,
                            command=self.set_tex_compression).pack()

        Tkinter.Label(self,text="Texture width:").pack()
        self.tex_width = Tkinter.StringVar()
        self.tex_width.set("512")
        Tkinter.OptionMenu(self,self.tex_width,"1","2","4","8","16","32","64","128","256","512","1024","2048","4096").pack()

        Tkinter.Label(self,text="Texture height:").pack()
        self.tex_height = Tkinter.StringVar()
        self.tex_height.set("512")
        Tkinter.OptionMenu(self,self.tex_height,"1","2","4","8","16","32","64","128","256","512","1024","2048","4096").pack()

        Tkinter.Label(self,text="image file to use as texture:").pack()
        self.image_file = Tkinter.StringVar()
        self.image_file.set("(none - generate own)")
        Tkinter.Entry(self,textvariable=self.image_file).pack()

        Tkinter.Button(self,text="do glTexSubImage2D ('blit') speed test",command=self.do_blit_speed).pack()
        Tkinter.Button(self,text="get maximum number of resident textures",command=self.do_resident_textures).pack()

    def do_blit_speed(self):
        glEnable( GL_TEXTURE_2D )
        print "Using texture from file: %s (Not really, yet)"%self.image_file.get()
        VisionEgg.video_info.tex_compress = self.tex_compression.get()

        tex1 = VisionEgg.Texture(size=(int(self.tex_width.get()),int(self.tex_height.get())))
        tex_id = tex1.load() # initialize the original texture
        tex_buf = tex1.get_texture_buffer()

        # show the texture first
        VisionEgg.OrthographicProjection().set_GL_projection_matrix()
        glClearColor(0.0, 0.0, 0.0, 0.0)
        glClear(GL_COLOR_BUFFER_BIT)
        glBegin(GL_QUADS)
        glTexCoord2f(0.0, tex1.buf_bf)
        glVertex3f(-1.0,-1.0,-6.0)
        glTexCoord2f(1.0, tex1.buf_bf)
        glVertex3f(1.0,-1.0,-6.0)
        glTexCoord2f(1.0, tex1.buf_tf)
        glVertex3f(1.0,1.0,-6.0)
        glTexCoord2f(0.0, tex1.buf_tf)
        glVertex3f(-1.0,1.0,-6.0)
        glEnd()
        VisionEgg.swap_buffers()

        texture_dest = tex1.get_texture_buffer()
        texture_source = tex1.get_pil_image()

        num = 10
        start = time.time()
        for i in range(num):
            print "BROKEN!!!"
#            texture_dest.put_sub_image(texture_source,(0,0),(texture_source.size[0],texture_source.size[1]))
        stop = time.time()

        print "Did %d calls to glTexSubImage2D in %f seconds."%(num,stop-start)

        print "blit speed!"
        tex_buf.free()

    def do_resident_textures(self):
        print "Using texture from file: %s (Not really, yet)"%self.image_file.get()
        print "Not implemented yet!"

    def set_tex_compression(self):
        """Callback for tick button"""
        VisionEgg.video_info.tex_compress = self.tex_compression.get()
        self.vid_info_frame.update()

app_window = []
def start_texture_test_menu():
    app_window.texture_test_frame = TextureTestFrame(app_window)
    app_window.texture_test_frame.pack()

if __name__ == '__main__':
    app_window = VisionEgg.GUI.StartGraphicsFrame(callback = start_texture_test_menu)
    app_window.mainloop()

<commit_msg>Got glAreTexturesResident code working (hopefully).<commit_after>#!/usr/bin/env python
#
# This is the python source code for a little app that lets you
# test various aspects of texturing in OpenGL.
#
# It is part of the Vision Egg package.
#
# Copyright (c) 2002 Andrew Straw.  Distributed under the terms of the
# GNU General Public License (GPL).

import string
__version__ = string.split('$Revision$')[1]
__date__ = string.join(string.split('$Date$')[1:3], ' ')
__author__ = 'Andrew Straw <astraw@users.sourceforge.net>'

import time

import pygame
from pygame.locals import *

from OpenGL.GL import * # PyOpenGL packages
from Numeric import *
import Tkinter

import VisionEgg
import VisionEgg.GUI

class TextureTestFrame(Tkinter.Frame):
    def __init__(self,master=None,**cnf):
        apply(Tkinter.Frame.__init__,(self,master),cnf)
        self.winfo_toplevel().title('Texture Test')
        self.pack()

        self.vid_info_frame = VisionEgg.GUI.VideoInfoFrame(self)
        self.vid_info_frame.pack()

        self.tex_compression = Tkinter.BooleanVar()
        self.tex_compression.set(VisionEgg.video_info.tex_compress)
        Tkinter.Checkbutton(self,
                            text="Texture compression",
                            variable=self.tex_compression,
                            command=self.set_tex_compression).pack()

        Tkinter.Label(self,text="Texture width:").pack()
        self.tex_width = Tkinter.StringVar()
        self.tex_width.set("512")
        Tkinter.OptionMenu(self,self.tex_width,"1","2","4","8","16","32","64","128","256","512","1024","2048","4096").pack()

        Tkinter.Label(self,text="Texture height:").pack()
        self.tex_height = Tkinter.StringVar()
        self.tex_height.set("512")
        Tkinter.OptionMenu(self,self.tex_height,"1","2","4","8","16","32","64","128","256","512","1024","2048","4096").pack()

        Tkinter.Label(self,text="image file to use as texture:").pack()
        self.image_file = Tkinter.StringVar()
        self.image_file.set("(none - generate own)")
        Tkinter.Entry(self,textvariable=self.image_file).pack()

        Tkinter.Button(self,text="do glTexSubImage2D ('blit') speed test",command=self.do_blit_speed).pack()
        Tkinter.Button(self,text="get maximum number of resident textures",command=self.do_resident_textures).pack()

    def do_blit_speed(self):
        glEnable( GL_TEXTURE_2D )
        print "Using texture from file: %s (Not really, yet)"%self.image_file.get()
        VisionEgg.video_info.tex_compress = self.tex_compression.get()

        tex1 = VisionEgg.Texture(size=(int(self.tex_width.get()),int(self.tex_height.get())))
        tex_id = tex1.load() # initialize the original texture
        tex_buf = tex1.get_texture_buffer()

        # show the texture first
        VisionEgg.OrthographicProjection().set_GL_projection_matrix()
        glClearColor(0.0, 0.0, 0.0, 0.0)
        glClear(GL_COLOR_BUFFER_BIT)
        glBegin(GL_QUADS)
        glTexCoord2f(0.0, tex1.buf_bf)
        glVertex3f(-1.0,-1.0,-6.0)
        glTexCoord2f(1.0, tex1.buf_bf)
        glVertex3f(1.0,-1.0,-6.0)
        glTexCoord2f(1.0, tex1.buf_tf)
        glVertex3f(1.0,1.0,-6.0)
        glTexCoord2f(0.0, tex1.buf_tf)
        glVertex3f(-1.0,1.0,-6.0)
        glEnd()
        VisionEgg.swap_buffers()

        texture_dest = tex1.get_texture_buffer()
        texture_source = tex1.get_pil_image()

        num = 10
        start = time.time()
        for i in range(num):
            print "BROKEN!!!"
#            texture_dest.put_sub_image(texture_source,(0,0),(texture_source.size[0],texture_source.size[1]))
        stop = time.time()

        print "Did %d calls to glTexSubImage2D in %f seconds."%(num,stop-start)

        print "blit speed!"
        tex_buf.free()

    def do_resident_textures(self):
        print "Not implemented yet!"
        glEnable( GL_TEXTURE_2D )
        print "Using texture from file: %s (Not really, yet)"%self.image_file.get()
        VisionEgg.video_info.tex_compress = self.tex_compression.get()
        orig =  VisionEgg.Texture(size=(int(self.tex_width.get()),int(self.tex_height.get())))

        texs = []
        tex_ids = []
        tex_bufs = []

        done = 0
        counter = 0
        while not done:
            counter = counter + 1
            tex_ids.append( orig.load() )
            tex_bufs.append( orig.get_texture_buffer() )

            answers = glAreTexturesResident( tex_ids )
            if type(answers) != type([1,2]):
                answers = list([answers])
            for answer in answers:
                if answer == 0:
                    done = 1 # loop until one of the textuers isn't resident
            max_within_reason = 50
            if counter > max_within_reason:
                print "Stopping glAreTexturesResident() test -- Over %d textures are reported resident!"%max_within_reason
                done = 1
        print "%d textures were reported resident, but %d were not."%(counter-1,counter)
        for buf in tex_bufs:
            buf.free()

    def set_tex_compression(self):
        """Callback for tick button"""
        VisionEgg.video_info.tex_compress = self.tex_compression.get()
        self.vid_info_frame.update()

app_window = []
def start_texture_test_menu():
    app_window.texture_test_frame = TextureTestFrame(app_window)
    app_window.texture_test_frame.pack()

if __name__ == '__main__':
    app_window = VisionEgg.GUI.StartGraphicsFrame(callback = start_texture_test_menu)
    app_window.mainloop()

<|endoftext|>


<commit_before>from unittest import TestCase
from importlib import import_module
from grab.tools.hook import CustomImporter

MODULES = """
content
feed
http
logs
ping
russian
text
work
control
files
__init__
lxml_tools
progress
selenium_tools
user_agent
yandex
debug
google
internal
metric
pwork
structured
w3lib_encoding
encoding
html
lock
parser
rex
system
watch
""".strip().splitlines()

class ToolsDeprecatedTestCase(TestCase):
    def setUp(self):
        self.hook = CustomImporter()

    def test_import(self):
        for mod_name in MODULES:
            path = 'grab.tools.%s' % mod_name
            self.hook.find_module(path)
            self.hook.load_module(path)

    def test_hook(self):
        self.assertIsInstance(self.hook.find_module('grab.tools.yandex'),
                              CustomImporter)
        self.assertEqual(self.hook.find_module('grab.spider'),
                         None)

        self.hook.find_module('grab.tools.lxml_tools')
        self.assertEqual(self.hook.name, '.etree')

        self.hook.find_module('grab.tools.module')
        self.assertRaises(ImportError, self.hook.load_module,
                          'grab.tools.module')
<commit_msg>Update tools tests<commit_after>from unittest import TestCase
from importlib import import_module
from grab.tools.hook import CustomImporter

MODULES = """
content
feed
http
logs
ping
russian
text
work
control
files
__init__
lxml_tools
progress
selenium_tools
user_agent
yandex
debug
google
internal
metric
pwork
structured
w3lib_encoding
encoding
html
lock
parser
rex
system
watch
""".strip().splitlines()

class ToolsDeprecatedTestCase(TestCase):
    def setUp(self):
        self.hook = CustomImporter()

    def test_import_traditional_way(self):
        for mod_name in MODULES:
            path = 'grab.tools.%s' % mod_name
            import_module(path)

    def test_hook(self):
        for mod_name in MODULES:
            path = 'grab.tools.%s' % mod_name
            self.hook.find_module(path)
            self.hook.load_module(path)

    def test_hook2(self):
        self.assertIsInstance(self.hook.find_module('grab.tools.yandex'),
                              CustomImporter)
        self.assertEqual(self.hook.find_module('grab.spider'),
                         None)

        self.hook.find_module('grab.tools.lxml_tools')
        self.assertEqual(self.hook.name, '.etree')

        self.hook.find_module('grab.tools.module')
        self.assertRaises(ImportError, self.hook.load_module,
                          'grab.tools.module')

<|endoftext|>


<commit_before>import ok
import time
import numpy

SERIALNUM = "1328000677"
BITFILENAME = 'fpga.bit'

byteControlInAddr = 0x00
bitReset = 0
mascbitReset = 0x0001<<bitReset

bitTriggerFrec = 1
mascbitTriggerFrec = 0x0001<<bitTriggerFrec

byteDataInAddr = 0x03

# 0x077D -> 1KS/s  (2.8MHz)
# 0x0719 -> 5KS/s  (14MHz)
# 0x0E19 -> 10KS/s (28MHz)
# 0x1519 -> 15KS/s (42MHz)
# 0x1C19 -> 20KS/s (56MHz)
# 0x2319 -> 25Ks/s (70MHz)     // Se llena la FIFO de tramas a partir de esta frecuencia
# 0x2A19 -> 30KS/s (84MHz)
frecValueDic = {'1000':0x0000,'2500':0x0001<<2,'5000':0x0002<<2,'10000':0x0003<<2,'15000':0x0004<<2,'20000':0x0005<<2,'25000':0x006<<2,'30000':0x0007<<2}

# 000  1k
# 001  2.5k
# 010  5k
# 011  10k
# 100  15k
# 101  20k
# 110  25k
# 111  30k

byteControlOutAddr = 0x21
bitDatoDisponible = 0
mascbitDatoDisponible = 0x0001<<bitDatoDisponible
bit4k = 0
bit16k = 1
bit32k = 2
bit128k = 3
bit1M = 4
bit5M = 5
bit10M = 6
bit60M = 7

byteDataAvAddr = 0x22

byteDataOutAddr = 0xA0

class OpalKelly():

  def __init__(self):
    self._xem = ok.FrontPanel()
    self._xem.OpenBySerial(SERIALNUM)
    self._xem.LoadDefaultPLLConfiguration()
    self._xem.ConfigureFPGA(BITFILENAME)
    self._byteControlIn = 0
    self._frecValue = 0
    msg = ('Device idVendor = ' + str(SERIALNUM) + ' not found')
    # was it found?
    if self._xem.IsFrontPanelEnabled() is not True :
        raise ValueError(msg)

  def reset(self):
    self._byteControlIn =  mascbitReset
    self._xem.SetWireInValue(byteControlInAddr, self._byteControlIn)
    self._xem.UpdateWireIns()

  def start(self,frec=30000):
    self.reset()
    time.sleep(.1)

    self._byteControlIn =  self._byteControlIn & ~mascbitReset
    self._xem.SetWireInValue(byteControlInAddr, self._byteControlIn)
    self._xem.UpdateWireIns()
    time.sleep(.1)

    if not (str(frec) in frecValueDic) :
       raise 'frec not valid'
    self._byteControlIn = frecValueDic[str(frec)]
    self._byteControlIn =  self._byteControlIn | mascbitTriggerFrec
    self._xem.SetWireInValue(byteControlInAddr, self._byteControlIn)
    self._xem.UpdateWireIns()
    time.sleep(.1)

    self._byteControlIn =  self._byteControlIn & ~mascbitTriggerFrec
    self._xem.SetWireInValue(byteControlInAddr, self._byteControlIn)
    self._xem.UpdateWireIns()
    time.sleep(.1)

  def is_data_ready(self):
    self._xem.UpdateWireOuts()
    a = self._xem.GetWireOutValue(byteControlOutAddr) & mascbitDatoDisponible
    return (a==mascbitDatoDisponible)

  def data_available(self):
    self._xem.UpdateWireOuts()
    a = self._xem.GetWireOutValue(byteControlOutAddr)
    if a & (0x0001<<bit60M) :
      if (a & 0x00FF) is not 0x00FF :
        print "error0"
      return 60000000
    if a & (0x0001<<bit10M) :
      if (a & 0x00FF) is not 0x007F :
        print "error1"
      return 10000000
    if a & (0x0001<<bit5M) :
      if (a & 0x00FF) is not 0x003F :
        print "error2"
      return 5000000
    if a & (0x0001<<bit1M) :
      if (a & 0x00FF) is not 0x001F :
        print "error3"
      return 1000000
    if a & (0x0001<<bit128k) :
      if (a & 0x00FF) is not 0x000F :
        print "error4"
      return 128000
    if a & (0x0001<<bit32k) :
      if (a & 0x00FF) is not 0x0007 :
        print "error5"
      return 32000
    if a & (0x0001<<bit16k) :
      if (a & 0x00FF) is not 0x0003 :
        print "error6"
      return 16000
    if a & (0x0001<<bit4k) :
      if (a & 0x00FF) is not 0x0001 :
        print "error7"
      return 4000
    else:
      print "error8"
      return 0

  def data_count_available(self):
    self._xem.UpdateWireOuts()
    a = self._xem.GetWireOutValue(byteDataAvAddr)
    return a

  def read_data(self,datos):
    n = self._xem.ReadFromPipeOut(byteDataOutAddr, datos)
    return n

  def read_block_data(self,datos):
    n = self._xem.ReadFromBlockPipeOut(byteDataOutAddr, 1024,datos)
    return n

  def close(self):
    self.reset()
    self._xem.ResetFPGA()

if __name__ == '__main__':
  import sys
  try:
    l = numpy.ndarray(100000,numpy.int16)
    f = open('salida.txt','w')
    a = OpalKelly()
    a.reset()
    time.sleep(.1)
    a.start(10000)
    n = 0
    largo = 40
    while n<10:

      while (a.data_available() < 100000):
        time.sleep(.01)

      n += 1

      b = a.read_data(l)
      f.write(l)
    f.close()
    a.close()
  except:
    print sys.exc_info()[0]
    print sys.exc_info()[1]
    print sys.exc_info()[2]
    f.close()
    a.close()
<commit_msg>Actualizacion de la api con la fpga<commit_after>import ok
import time
import numpy
import logging
logger = logging.getLogger('okapi')

SERIALNUM = "1328000677"
BITFILENAME = 'fpga.bit'
AMPCOUNT = 1

byteControlInAddr = 0x00
bitReset = 0
maskbitReset = 0x0001<<bitReset
bitAmpCount = 8
maskbitAmpCount = (AMPCOUNT-1)<<bitAmpCount
bitTriggerFrec = 1
maskbitTriggerFrec = 0x0001<<bitTriggerFrec

byteDataInAddr = 0x03

# 0x077D -> 1KS/s  (2.8MHz)
# 0x0719 -> 5KS/s  (14MHz)
# 0x0E19 -> 10KS/s (28MHz)
# 0x1519 -> 15KS/s (42MHz)
# 0x1C19 -> 20KS/s (56MHz)
# 0x2319 -> 25Ks/s (70MHz)     // Se llena la FIFO de tramas a partir de esta frecuencia
# 0x2A19 -> 30KS/s (84MHz)
frecValueDic = {'1000':0x0000,'2500':0x0001<<2,'5000':0x0002<<2,'10000':0x0003<<2,'15000':0x0004<<2,'20000':0x0005<<2,'25000':0x006<<2,'30000':0x0007<<2}

# 000  1k
# 001  2.5k
# 010  5k
# 011  10k
# 100  15k
# 101  20k
# 110  25k
# 111  30k

byteControlOutAddr = 0x21
bitDatoDisponible = 0
maskbitDatoDisponible = 0x0001<<bitDatoDisponible
bit4k = 0
bit16k = 1
bit32k = 2
bit128k = 3
bit1M = 4
bit5M = 5
bit10M = 6
bit60M = 7

byteDataAvAddr = 0x22

byteDataOutAddr = 0xA0

class OpalKelly():

  def __init__(self):
    self._xem = ok.FrontPanel()
    self._xem.OpenBySerial(SERIALNUM)
    self._xem.LoadDefaultPLLConfiguration()
    self._xem.ConfigureFPGA(BITFILENAME)
    self._byteControlIn = maskbitAmpCount
    self._frecValue = 0
    msg = ('Device idVendor = ' + str(SERIALNUM) + ' not found')
    # was it found?
    if self._xem.IsFrontPanelEnabled() is not True :
        raise ValueError(msg)

  def reset(self):
    self._byteControlIn = self._byteControlIn | maskbitReset
    logger.debug('reset ' + bin(self._byteControlIn) )
    self._xem.SetWireInValue(byteControlInAddr, self._byteControlIn)
    self._xem.UpdateWireIns()

  def start(self,frec=30000):
    self.reset()
    time.sleep(.1)

    self._byteControlIn = self._byteControlIn & ~maskbitReset
    logger.debug('start1 ' + bin(self._byteControlIn) )
    self._xem.SetWireInValue(byteControlInAddr, self._byteControlIn)
    self._xem.UpdateWireIns()
    time.sleep(.1)

    if not (str(frec) in frecValueDic) :
       raise 'frec not valid'
    self._byteControlIn = self._byteControlIn | frecValueDic[str(frec)]
    self._byteControlIn = self._byteControlIn | maskbitTriggerFrec
    self._xem.SetWireInValue(byteControlInAddr, self._byteControlIn)
    self._xem.UpdateWireIns()
    time.sleep(.1)

    self._byteControlIn =  self._byteControlIn & ~maskbitTriggerFrec
    logger.debug('start3 ' + bin(self._byteControlIn) )
    self._xem.SetWireInValue(byteControlInAddr, self._byteControlIn)
    self._xem.UpdateWireIns()
    time.sleep(.1)

  def is_data_ready(self):
    self._xem.UpdateWireOuts()
    a = self._xem.GetWireOutValue(byteControlOutAddr) & maskbitDatoDisponible
    return (a==maskbitDatoDisponible)

  def data_available(self):
    self._xem.UpdateWireOuts()
    a = self._xem.GetWireOutValue(byteControlOutAddr)
    if a & (0x0001<<bit60M) :
      if (a & 0x00FF) is not 0x00FF :
        print "error0"
      return 60000000
    if a & (0x0001<<bit10M) :
      if (a & 0x00FF) is not 0x007F :
        print "error1"
      return 10000000
    if a & (0x0001<<bit5M) :
      if (a & 0x00FF) is not 0x003F :
        print "error2"
      return 5000000
    if a & (0x0001<<bit1M) :
      if (a & 0x00FF) is not 0x001F :
        print "error3"
      return 1000000
    if a & (0x0001<<bit128k) :
      if (a & 0x00FF) is not 0x000F :
        print "error4"
      return 128000
    if a & (0x0001<<bit32k) :
      if (a & 0x00FF) is not 0x0007 :
        print "error5"
      return 32000
    if a & (0x0001<<bit16k) :
      if (a & 0x00FF) is not 0x0003 :
        print "error6"
      return 16000
    if a & (0x0001<<bit4k) :
      if (a & 0x00FF) is not 0x0001 :
        print "error7"
      return 4000
    else:
      print "error8"
      return 0

  def data_count_available(self):
    self._xem.UpdateWireOuts()
    a = self._xem.GetWireOutValue(byteDataAvAddr)
    return a

  def read_data(self,datos):
    n = self._xem.ReadFromPipeOut(byteDataOutAddr, datos)
    return n

  def read_block_data(self,datos):
    n = self._xem.ReadFromBlockPipeOut(byteDataOutAddr, 1024,datos)
    return n

  def close(self):
    self.reset()
    self._xem.ResetFPGA()

if __name__ == '__main__':
  import sys
  formatter = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
  dateformat = '%Y/%m/%d %I:%M:%S %p'

  logging.basicConfig(filename='logs/okapi.log', filemode='w',
      level=logging.DEBUG, format=formatter, datefmt = dateformat)
  logger.info('okapi test started')

  try:
    l = numpy.ndarray(100000,numpy.int16)
    f = open('salida.txt','w')
    a = OpalKelly()
    a.reset()
    time.sleep(.1)
    a.start(10000)
    n = 0
    while n<10:

      while (a.data_available() < 100000):
        time.sleep(.01)

      n += 1

      b = a.read_data(l)
      f.write(l)
    f.close()
    a.close()
    logger.info('okapi test ended')
  except:
    logger.error(str(sys.exc_info()[1]))
    logger.info('Ended with errors')
    f.close()
    a.close()
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import uuid

import numpy as np

from glue.utils import nonpartial
from glue.core.exceptions import IncompatibleAttribute

from .multi_scatter import MultiColorScatter
from .layer_state import ScatterLayerState
from ..common.layer_artist import VispyLayerArtist

class ScatterLayerArtist(VispyLayerArtist):
    """
    A layer artist to render 3d scatter plots.
    """

    def __init__(self, vispy_viewer, layer=None, layer_state=None):

        super(ScatterLayerArtist, self).__init__(layer)

        self._clip_limits = None

        self.layer = layer or layer_state.layer
        self.vispy_viewer = vispy_viewer
        self.vispy_widget = vispy_viewer._vispy_widget

        # TODO: need to remove layers when layer artist is removed
        self._viewer_state = vispy_viewer.state
        self.state = layer_state or ScatterLayerState(layer=self.layer)
        if self.state not in self._viewer_state.layers:
            self._viewer_state.layers.append(self.state)

        # We create a unique ID for this layer artist, that will be used to
        # refer to the layer artist in the MultiColorScatter. We have to do this
        # rather than use self.id because we can't guarantee the latter is
        # unique.
        self.id = str(uuid.uuid4())

        # We need to use MultiColorScatter instance to store scatter plots, but
        # we should only have one per canvas. Therefore, we store the
        # MultiColorScatter instance in the vispy viewer instance.
        if not hasattr(self.vispy_widget, '_multiscat'):
            multiscat = MultiColorScatter()
            multiscat.set_gl_state(depth_test=False,
                                   blend=True,
                                   blend_func=('src_alpha', 'one_minus_src_alpha'))

            self.vispy_widget.add_data_visual(multiscat)
            self.vispy_widget._multiscat = multiscat

        self._multiscat = self.vispy_widget._multiscat
        self._multiscat.allocate(self.id)
        self._multiscat.set_zorder(self.id, self.get_zorder)

        try:
            self.state.add_callback('*', self._update_from_state, as_kwargs=True)
        except TypeError:  # glue-core >= 0.11
            self.state.add_global_callback(self._update_from_state)

        self._update_from_state(**self.state.as_dict())

        self._viewer_state.add_callback('x_att', nonpartial(self._update_data))
        self._viewer_state.add_callback('y_att', nonpartial(self._update_data))
        self._viewer_state.add_callback('z_att', nonpartial(self._update_data))

        # Set data caches
        self._marker_keep = Ellipsis
        self._marker_data = None
        self._color_data = None
        self._size_data = None

        self._update_data()

        self.visible = True

    @property
    def visual(self):
        return self._multiscat

    def _update_visibility(self):
        self._multiscat.set_visible(self.id, self.visible)
        self.redraw()

    def get_zorder(self):
        return self.zorder

    def redraw(self):
        """
        Redraw the Vispy canvas
        """
        self._multiscat._update()
        self.vispy_widget.canvas.update()

    def clear(self):
        """
        Remove the layer artist from the visualization
        """
        self._multiscat.set_data_values(self.id, [], [], [])

    def update(self):
        """
        Update the visualization to reflect the underlying data
        """
        self.redraw()
        self._changed = False

    def _update_from_state(self, **props):
        if any('size' in prop for prop in props):
            self._update_sizes()
        if any('color' in prop or 'cmap' in prop for prop in props):
            self._update_colors()
        if 'alpha' in props:
            self._update_alpha()
        self.redraw()

    def _update_sizes(self):
        if self.state.size_mode is None:
            pass
        elif self.state.size_mode == 'Fixed':
            self._multiscat.set_size(self.id, self.state.size * self.state.size_scaling)
        else:
            data = self.layer[self.state.size_attribute].ravel()
            data = data[self._marker_keep]
            if self.state.size_vmax == self.state.size_vmin:
                size = np.ones(data.shape) * 10
            else:
                size = (20 * (data - self.state.size_vmin) /
                        (self.state.size_vmax - self.state.size_vmin))
            size_data = size * self.state.size_scaling
            size_data[np.isnan(data)] = 0.
            self._multiscat.set_size(self.id, size_data)

    def _update_colors(self):
        if self.state.color_mode is None:
            pass
        elif self.state.color_mode == 'Fixed':
            self._multiscat.set_color(self.id, self.state.color)
        else:
            data = self.layer[self.state.cmap_attribute].ravel()
            data = data[self._marker_keep]
            if self.state.cmap_vmax == self.state.cmap_vmin:
                cmap_data = np.ones(data.shape) * 0.5
            else:
                cmap_data = ((data - self.state.cmap_vmin) /
                             (self.state.cmap_vmax - self.state.cmap_vmin))
            cmap_data = self.state.cmap(cmap_data)
            cmap_data[:, 3][np.isnan(data)] = 0.
            self._multiscat.set_color(self.id, cmap_data)

    def _update_alpha(self):
        self._multiscat.set_alpha(self.id, self.state.alpha)

    def _update_data(self):

        try:
            x = self.layer[self._viewer_state.x_att].ravel()
            y = self.layer[self._viewer_state.y_att].ravel()
            z = self.layer[self._viewer_state.z_att].ravel()
        except AttributeError:
            return
        except (IncompatibleAttribute, IndexError):
            # The following includes a call to self.clear()
            self.disable_invalid_attributes(self._viewer_state.x_att,
                                            self._viewer_state.y_att,
                                            self._viewer_state.z_att)
            return
        else:
            self._enabled = True

        if self._clip_limits is None:
            keep = Ellipsis
        else:
            xmin, xmax, ymin, ymax, zmin, zmax = self._clip_limits
            keep = (x >= xmin) & (x <= xmax) & (y >= ymin) & (y <= ymax) & (z >= zmin) & (z <= zmax)
            x, y, z = x[keep], y[keep], z[keep]

        self._marker_data = np.array([x, y, z]).transpose()
        self._marker_keep = keep

        # We need to make sure we update the sizes and colors in case
        # these were set as arrays, since the size of the data might have
        # changed (in the case of subsets)

        with self._multiscat.delay_update():
            self._multiscat.set_data_values(self.id, x, y, z)
            self._update_sizes()
            self._update_colors()

        self.redraw()

    @property
    def default_limits(self):
        if self._marker_data is None:
            raise ValueError("Data not yet set")
        dmin = np.nanmin(self._marker_data, axis=0)
        dmax = np.nanmax(self._marker_data, axis=0)
        # TODO: the following can be optimized
        return tuple(np.array([dmin, dmax]).transpose().ravel())

    def set_clip(self, limits):
        self._clip_limits = limits
        self._update_data()
<commit_msg>Define _marker_keep earlier<commit_after>from __future__ import absolute_import, division, print_function

import uuid

import numpy as np

from glue.utils import nonpartial
from glue.core.exceptions import IncompatibleAttribute

from .multi_scatter import MultiColorScatter
from .layer_state import ScatterLayerState
from ..common.layer_artist import VispyLayerArtist

class ScatterLayerArtist(VispyLayerArtist):
    """
    A layer artist to render 3d scatter plots.
    """

    def __init__(self, vispy_viewer, layer=None, layer_state=None):

        super(ScatterLayerArtist, self).__init__(layer)

        self._clip_limits = None

        self._marker_keep = Ellipsis

        # Set data caches
        self._marker_data = None
        self._color_data = None
        self._size_data = None

        self.layer = layer or layer_state.layer
        self.vispy_viewer = vispy_viewer
        self.vispy_widget = vispy_viewer._vispy_widget

        # TODO: need to remove layers when layer artist is removed
        self._viewer_state = vispy_viewer.state
        self.state = layer_state or ScatterLayerState(layer=self.layer)
        if self.state not in self._viewer_state.layers:
            self._viewer_state.layers.append(self.state)

        # We create a unique ID for this layer artist, that will be used to
        # refer to the layer artist in the MultiColorScatter. We have to do this
        # rather than use self.id because we can't guarantee the latter is
        # unique.
        self.id = str(uuid.uuid4())

        # We need to use MultiColorScatter instance to store scatter plots, but
        # we should only have one per canvas. Therefore, we store the
        # MultiColorScatter instance in the vispy viewer instance.
        if not hasattr(self.vispy_widget, '_multiscat'):
            multiscat = MultiColorScatter()
            multiscat.set_gl_state(depth_test=False,
                                   blend=True,
                                   blend_func=('src_alpha', 'one_minus_src_alpha'))

            self.vispy_widget.add_data_visual(multiscat)
            self.vispy_widget._multiscat = multiscat

        self._multiscat = self.vispy_widget._multiscat
        self._multiscat.allocate(self.id)
        self._multiscat.set_zorder(self.id, self.get_zorder)

        try:
            self.state.add_callback('*', self._update_from_state, as_kwargs=True)
        except TypeError:  # glue-core >= 0.11
            self.state.add_global_callback(self._update_from_state)

        self._update_from_state(**self.state.as_dict())

        self._viewer_state.add_callback('x_att', nonpartial(self._update_data))
        self._viewer_state.add_callback('y_att', nonpartial(self._update_data))
        self._viewer_state.add_callback('z_att', nonpartial(self._update_data))

        self._update_data()

        self.visible = True

    @property
    def visual(self):
        return self._multiscat

    def _update_visibility(self):
        self._multiscat.set_visible(self.id, self.visible)
        self.redraw()

    def get_zorder(self):
        return self.zorder

    def redraw(self):
        """
        Redraw the Vispy canvas
        """
        self._multiscat._update()
        self.vispy_widget.canvas.update()

    def clear(self):
        """
        Remove the layer artist from the visualization
        """
        self._multiscat.set_data_values(self.id, [], [], [])

    def update(self):
        """
        Update the visualization to reflect the underlying data
        """
        self.redraw()
        self._changed = False

    def _update_from_state(self, **props):
        if any('size' in prop for prop in props):
            self._update_sizes()
        if any('color' in prop or 'cmap' in prop for prop in props):
            self._update_colors()
        if 'alpha' in props:
            self._update_alpha()
        self.redraw()

    def _update_sizes(self):
        if self.state.size_mode is None:
            pass
        elif self.state.size_mode == 'Fixed':
            self._multiscat.set_size(self.id, self.state.size * self.state.size_scaling)
        else:
            data = self.layer[self.state.size_attribute].ravel()
            data = data[self._marker_keep]
            if self.state.size_vmax == self.state.size_vmin:
                size = np.ones(data.shape) * 10
            else:
                size = (20 * (data - self.state.size_vmin) /
                        (self.state.size_vmax - self.state.size_vmin))
            size_data = size * self.state.size_scaling
            size_data[np.isnan(data)] = 0.
            self._multiscat.set_size(self.id, size_data)

    def _update_colors(self):
        if self.state.color_mode is None:
            pass
        elif self.state.color_mode == 'Fixed':
            self._multiscat.set_color(self.id, self.state.color)
        else:
            data = self.layer[self.state.cmap_attribute].ravel()
            data = data[self._marker_keep]
            if self.state.cmap_vmax == self.state.cmap_vmin:
                cmap_data = np.ones(data.shape) * 0.5
            else:
                cmap_data = ((data - self.state.cmap_vmin) /
                             (self.state.cmap_vmax - self.state.cmap_vmin))
            cmap_data = self.state.cmap(cmap_data)
            cmap_data[:, 3][np.isnan(data)] = 0.
            self._multiscat.set_color(self.id, cmap_data)

    def _update_alpha(self):
        self._multiscat.set_alpha(self.id, self.state.alpha)

    def _update_data(self):

        try:
            x = self.layer[self._viewer_state.x_att].ravel()
            y = self.layer[self._viewer_state.y_att].ravel()
            z = self.layer[self._viewer_state.z_att].ravel()
        except AttributeError:
            return
        except (IncompatibleAttribute, IndexError):
            # The following includes a call to self.clear()
            self.disable_invalid_attributes(self._viewer_state.x_att,
                                            self._viewer_state.y_att,
                                            self._viewer_state.z_att)
            return
        else:
            self._enabled = True

        if self._clip_limits is None:
            keep = Ellipsis
        else:
            xmin, xmax, ymin, ymax, zmin, zmax = self._clip_limits
            keep = (x >= xmin) & (x <= xmax) & (y >= ymin) & (y <= ymax) & (z >= zmin) & (z <= zmax)
            x, y, z = x[keep], y[keep], z[keep]

        self._marker_data = np.array([x, y, z]).transpose()
        self._marker_keep = keep

        # We need to make sure we update the sizes and colors in case
        # these were set as arrays, since the size of the data might have
        # changed (in the case of subsets)

        with self._multiscat.delay_update():
            self._multiscat.set_data_values(self.id, x, y, z)
            self._update_sizes()
            self._update_colors()

        self.redraw()

    @property
    def default_limits(self):
        if self._marker_data is None:
            raise ValueError("Data not yet set")
        dmin = np.nanmin(self._marker_data, axis=0)
        dmax = np.nanmax(self._marker_data, axis=0)
        # TODO: the following can be optimized
        return tuple(np.array([dmin, dmax]).transpose().ravel())

    def set_clip(self, limits):
        self._clip_limits = limits
        self._update_data()
<|endoftext|>


<commit_before># © 2004-2009 Tiny SPRL (<http://tiny.be>).
# © 2014-2017 Tecnativa - Pedro M. Baeza
# © 2016 ACSONE SA/NV (<http://acsone.eu>)
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html
{
    "name": "Purchase order lines with discounts",
    "author": "Tiny, "
    "Acysos S.L., "
    "Tecnativa, "
    "ACSONE SA/NV,"
    "GRAP,"
    "Odoo Community Association (OCA)",
    "version": "14.0.1.0.0",
    "category": "Purchase Management",
    "website": "https://github.com/OCA/purchase-workflow",
    "depends": ["purchase_stock"],
    "data": [
        "views/purchase_discount_view.xml",
        "views/report_purchaseorder.xml",
        "views/product_supplierinfo_view.xml",
        "views/res_partner_view.xml",
    ],
    "license": "AGPL-3",
    "installable": True,
    "images": ["images/purchase_discount.png"],
}
<commit_msg>purchase_discount 14.0.1.0.1<commit_after># © 2004-2009 Tiny SPRL (<http://tiny.be>).
# © 2014-2017 Tecnativa - Pedro M. Baeza
# © 2016 ACSONE SA/NV (<http://acsone.eu>)
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html
{
    "name": "Purchase order lines with discounts",
    "author": "Tiny, "
    "Acysos S.L., "
    "Tecnativa, "
    "ACSONE SA/NV,"
    "GRAP,"
    "Odoo Community Association (OCA)",
    "version": "14.0.1.0.1",
    "category": "Purchase Management",
    "website": "https://github.com/OCA/purchase-workflow",
    "depends": ["purchase_stock"],
    "data": [
        "views/purchase_discount_view.xml",
        "views/report_purchaseorder.xml",
        "views/product_supplierinfo_view.xml",
        "views/res_partner_view.xml",
    ],
    "license": "AGPL-3",
    "installable": True,
    "images": ["images/purchase_discount.png"],
}
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from . import core
from . import plotting
from .window import Window

import numpy as np
from scipy import signal
import matplotlib.pyplot as plt
import copy

class Pair:
    """
    The Pair: work with 2-component data.

    Usage: Pair()     => create Pair of synthetic data
           Pair(x,y) => creates Pair from two traces stored in numpy arrays x and y.

    Keyword Arguments:
        - delta = 1. (sample interval) [default] | float
        - units = 's' (for labelling) | string
        - angle = 0. (component 1 angle) | float

    Advanced Keyword Arguments (if in doubt don't use):
        - geom = 'geo' (N,E) [default] | 'ray' (SV,SH)
        - xy = np.ones(2) | custom numpy array
        - rcvloc = None
        - srcloc = None

    Methods:
        - plot()
        -
    """
    def __init__(self,*args,**kwargs):

        if ('delta' in kwargs):
            self.delta = kwargs['delta']
        else:
            self.delta = 1.

        if ('units' in kwargs):
            self.units = kwargs['units']
        else:
            self.units = 's'

        if ('angle' in kwargs):
            self.angle = kwargs['angle']
        else:
            self.angle = 0.

        if ('window' in kwargs):
            self.window = kwargs['window']
        else:
            self.window = None

        if len(args) == 0:
            if ('lag' in kwargs):
                # convert time shift to nsamples -- must be even
                nsamps = int(kwargs['lag']/self.delta)
                nsamps = nsamps if nsamps%2==0 else nsamps + 1
                kwargs['lag'] = nsamps
            self.x, self.y = _synth(**kwargs)
        elif len(args) == 2:
            self.x, self.y = args[0], args[1]
        else:
            raise Exception('Unexpected number of arguments')

        # some sanity checks
        if self.x.ndim != 1:
            raise Exception('data must be one dimensional')
        if self.x.size%2 == 0:
            raise Exception('data must have odd number of samples')
        if (self.x.size != self.y.size):
            raise Exception('x and y must be the same length')

        # add geometry info
        if ('geom' in kwargs):
            self.geom = kwargs['geom']
        else:
            # if using 2-component data I'll guess the user wants geo coordinates.
            self.geom = 'geo'

        if ('srcloc' in kwargs):
            self.srcloc = kwargs['srcloc']

        if ('rcvloc' in kwargs):
            self.rcvloc = kwargs['rcvloc']

        # if ('xyz' in kwargs):
        #     self.xyz = kargs['xyz']
        # else:
        #     self.xyz = np.ones(3)

    # methods

    # time from start
    def t(self):
        return np.arange(self.x.size) * self.delta

    def nsamps(self):
        return self.x.size

    def power(self):
        return self.x**2+self.y**2

    def centre(self):
        return int(self.x.size/2)

    def xy(self):
        return np.vstack((self.x,self.y))

    def plot(self,window=None):
        """
        Plot trace data and particle motion
        """
        from matplotlib import gridspec
        fig = plt.figure(figsize=(12, 3))
        if window is None:
            gs = gridspec.GridSpec(1, 2, width_ratios=[3, 1])
            ax0 = plt.subplot(gs[0])
            ax0.plot(self.t(),self.x)
            ax0.plot(self.t(),self.y)
            # particle  motion
            lim = abs(self.xy().max()) * 1.1
            # the polar axis:
            # ax_polar = plt.subplot(gs[1], polar=True, frameon=False)
            # ax_polar.set_rmax(lim)
            # ax_polar.patch.set_facecolor(111)
            # ax_polar.get_xaxis.set_visible(False)
            # ax_polar.grid(True)
            # the data
            ax1 = plt.subplot(gs[1])
            # ax1.patch.set_alpha(0)
            ax1.axis('equal')
            ax1.plot(self.y,self.x)
            ax1.set_xlim([-lim,lim])
            ax1.set_ylim([-lim,lim])
            ax1.axes.get_xaxis().set_visible(False)
            ax1.axes.get_yaxis().set_visible(False)
        else:
            gs = gridspec.GridSpec(1, 3, width_ratios=[3,1,1])
            ax0 = plt.subplot(gs[0])
            ax0.plot(self.t(),self.x)
            ax0.plot(self.t(),self.y)
            # the window limits
            nsamps = self.nsamps()
            wbeg = window.start(nsamps)*self.delta
            wend = window.end(nsamps)*self.delta
            ax0.axvline(wbeg,linewidth=2,color='r')
            ax0.axvline(wend,linewidth=2,color='r')
            # windowed data
            d2 = self.copy()
            d2.chop(window)
            ax1 = plt.subplot(gs[1])
            ax1.plot(d2.t()+wbeg,d2.x)
            ax1.plot(d2.t()+wbeg,d2.y)
            # particle  motion
            lim = abs(self.xy().max()) * 1.1
            ax2 = plt.subplot(gs[2])
            ax2.axis('equal')
            ax2.plot(d2.y,d2.x)
            ax2.set_xlim([-lim,lim])
            ax2.set_ylim([-lim,lim])
            ax2.axes.get_xaxis().set_visible(False)
            ax2.axes.get_yaxis().set_visible(False)

        # show
        plt.show()

    def split(self,degrees,tlag):
        """
        Applies splitting operator (phi,dt) to Pair.

        Rotates data so that trace1 is lined up with degrees (and trace2 90 degrees clockwise).
        Applies a relative time shift by the nearest even number of samples to tlag,
        trace1 is shifted tlag/2 forward in time, and trace2 tlag/2 backward in time.
        Then undoes the original rotation.
        """
        # convert time shift to nsamples -- must be even
        nsamps = int(tlag / self.delta)
        nsamps = nsamps if nsamps%2==0 else nsamps + 1
        # find appropriate rotation angle
        rangle = degrees - self.angle
        # apply splitting
        self.x, self.y = core.split(self.x,self.y,rangle,nsamps)

    def unsplit(self,degrees,tlag):
        """
        Applies reverse splitting operator (phi,dt) to Pair.

        Rotates data so that trace1 is lined up with degrees (and trace2 90 degrees clockwise).
        Applies a relative time shift by the nearest even number of samples to tlag,
        trace1 is shifted tlag/2 backward in time, and trace2 tlag/2 forward in time.
        Then undoes the original rotation.
        """
        # convert time shift to nsamples -- must be even
        nsamps = int(tlag / self.delta)
        nsamps = nsamps if nsamps%2==0 else nsamps + 1
        # find appropriate rotation angle
        rangle = degrees - self.angle
        self.x, self.y = core.unsplit(self.x,self.y,rangle,nsamps)

    def rotateto(self,degrees):
        """
        Rotate data so that trace1 lines up with *degrees*
        """
        # find appropriate rotation angle
        rangle = -degrees - self.angle
        self.x, self.y = core.rotate(self.x,self.y,rangle)
        self.angle = degrees

    def lag(self,tlag):
        """
        Relative shift trace1 and trace2 by tlag seconds
        """
        # convert time shift to nsamples -- must be even
        nsamps = int(tlag / self.delta)
        nsamps = nsamps if nsamps%2==0 else nsamps + 1
        self.x, self.y = core.lag(self.x,self.y,nsamps)

    def chop(self,window):
        """
        Chop data around window
        """
        self.x, self.y = core.chop(self.x,self.y,window=window)

    def genwindow(self,time_centre,time_width,tukey=None):
        """
        Return a window object about time_centre with time_width.
        """
        tcs = int(time_centre / self.delta)
        offset = tcs - self.centre()
        # convert time to nsamples -- must be odd
        width = int(time_width / self.delta)
        width = width if width%2==1 else width + 1
        return Window(width,offset,tukey=tukey)

    # def autowindow(self,time_centre=None):
    #     """
    #     Makes a guess based on energy near *time_centre* about a suitable window
    #
    #     *time centre* should be the shear wave pick at the centre of the energy packet.
    #     By default will use centre sample.
    #     """
    #     if time_centre is None:
    #         t0 = self.centre()
    #     else:
    #         t0 = int(time_centre / self.delta)

    def copy(self):
        return copy.copy(self)

def _synth(**kwargs):
    """return ricker wavelet synthetic data"""

    if ('pol' in kwargs):
        pol = kwargs['pol']
    else:
        pol = 0.

    if ('fast' in kwargs):
        fast = kwargs['fast']
    else:
        fast = 0.

    if ('lag' in kwargs):
        lag = kwargs['lag']
    else:
        lag = 0

    if ('noise' in kwargs):
        noise = kwargs['noise']
    else:
        noise = 0.03

    if ('nsamps' in kwargs):
        nsamps = kwargs['nsamps']
    else:
        nsamps = 501

    if ('width' in kwargs):
        width = kwargs['width']
    else:
        width = 16.

    if ('window' in kwargs):
        window = kwargs['window']
    else:
        window = Window(width*3)

    nsamps = int(nsamps)

    x = signal.ricker(nsamps, width) + core.noise(nsamps,noise,width/4)
    y = core.noise(nsamps,noise,width/4)

    # rotate to polarisation
    x,y = core.rotate(x,y,-pol)

    # add any splitting -- this will reduce nsamps
    x,y = core.split(x,y,fast,lag)

    return x,y

<commit_msg>Added Pair ppm function<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from . import core
from . import plotting
from .window import Window

import numpy as np
from scipy import signal
import matplotlib.pyplot as plt
import copy

class Pair:
    """
    The Pair: work with 2-component data.

    Usage: Pair()     => create Pair of synthetic data
           Pair(x,y) => creates Pair from two traces stored in numpy arrays x and y.

    Keyword Arguments:
        - delta = 1. (sample interval) [default] | float
        - units = 's' (for labelling) | string
        - angle = 0. (component 1 angle) | float

    Advanced Keyword Arguments (if in doubt don't use):
        - geom = 'geo' (N,E) [default] | 'ray' (SV,SH)
        - xy = np.ones(2) | custom numpy array
        - rcvloc = None
        - srcloc = None

    Methods:
        - plot()
        -
    """
    def __init__(self,*args,**kwargs):

        if ('delta' in kwargs):
            self.delta = kwargs['delta']
        else:
            self.delta = 1.

        if ('units' in kwargs):
            self.units = kwargs['units']
        else:
            self.units = 's'

        if ('angle' in kwargs):
            self.angle = kwargs['angle']
        else:
            self.angle = 0.

        if ('window' in kwargs):
            self.window = kwargs['window']
        else:
            self.window = None

        if len(args) == 0:
            if ('lag' in kwargs):
                # convert time shift to nsamples -- must be even
                nsamps = int(kwargs['lag']/self.delta)
                nsamps = nsamps if nsamps%2==0 else nsamps + 1
                kwargs['lag'] = nsamps
            self.x, self.y = _synth(**kwargs)
        elif len(args) == 2:
            self.x, self.y = args[0], args[1]
        else:
            raise Exception('Unexpected number of arguments')

        # some sanity checks
        if self.x.ndim != 1:
            raise Exception('data must be one dimensional')
        if self.x.size%2 == 0:
            raise Exception('data must have odd number of samples')
        if (self.x.size != self.y.size):
            raise Exception('x and y must be the same length')

        # add geometry info
        if ('geom' in kwargs):
            self.geom = kwargs['geom']
        else:
            # if using 2-component data I'll guess the user wants geo coordinates.
            self.geom = 'geo'

        if ('srcloc' in kwargs):
            self.srcloc = kwargs['srcloc']

        if ('rcvloc' in kwargs):
            self.rcvloc = kwargs['rcvloc']

        # if ('xyz' in kwargs):
        #     self.xyz = kargs['xyz']
        # else:
        #     self.xyz = np.ones(3)

    # methods

    # time from start
    def t(self):
        return np.arange(self.x.size) * self.delta

    def nsamps(self):
        return self.x.size

    def power(self):
        return self.x**2+self.y**2

    def centre(self):
        return int(self.x.size/2)

    def xy(self):
        return np.vstack((self.x,self.y))

    def plot(self,window=None):
        """
        Plot trace data and particle motion
        """
        from matplotlib import gridspec
        fig = plt.figure(figsize=(12, 3))
        if window is None:
            gs = gridspec.GridSpec(1, 2, width_ratios=[3, 1])
            ax0 = plt.subplot(gs[0])
            ax0.plot(self.t(),self.x)
            ax0.plot(self.t(),self.y)
            # particle  motion
            lim = abs(self.xy().max()) * 1.1
            # the polar axis:
            # ax_polar = plt.subplot(gs[1], polar=True, frameon=False)
            # ax_polar.set_rmax(lim)
            # ax_polar.patch.set_facecolor(111)
            # ax_polar.get_xaxis.set_visible(False)
            # ax_polar.grid(True)
            # the data
            ax1 = plt.subplot(gs[1])
            # ax1.patch.set_alpha(0)
            ax1.axis('equal')
            ax1.plot(self.y,self.x)
            ax1.set_xlim([-lim,lim])
            ax1.set_ylim([-lim,lim])
            ax1.axes.get_xaxis().set_visible(False)
            ax1.axes.get_yaxis().set_visible(False)
        else:
            gs = gridspec.GridSpec(1, 3, width_ratios=[3,1,1])
            ax0 = plt.subplot(gs[0])
            ax0.plot(self.t(),self.x)
            ax0.plot(self.t(),self.y)
            # the window limits
            nsamps = self.nsamps()
            wbeg = window.start(nsamps)*self.delta
            wend = window.end(nsamps)*self.delta
            ax0.axvline(wbeg,linewidth=2,color='r')
            ax0.axvline(wend,linewidth=2,color='r')
            # windowed data
            d2 = self.copy()
            d2.chop(window)
            ax1 = plt.subplot(gs[1])
            ax1.plot(d2.t()+wbeg,d2.x)
            ax1.plot(d2.t()+wbeg,d2.y)
            # particle  motion
            lim = abs(self.xy().max()) * 1.1
            ax2 = plt.subplot(gs[2])
            ax2.axis('equal')
            ax2.plot(d2.y,d2.x)
            ax2.set_xlim([-lim,lim])
            ax2.set_ylim([-lim,lim])
            ax2.axes.get_xaxis().set_visible(False)
            ax2.axes.get_yaxis().set_visible(False)

        # show
        plt.show()

    def ppm(self,window=None):
        """Plot particle motion."""
        fig = plt.figure()
        ax = fig.gca()
        ax.plot(self.y,self.x)
        lim = abs(self.xy().max()) * 1.1
        ax.set_aspect('equal')
        ax.set_xlim([-lim,lim])
        ax.set_ylim([-lim,lim])
        ax.set_xlabel('y')
        ax.set_ylabel('x')
        plt.show()

    def split(self,degrees,tlag):
        """
        Applies splitting operator (phi,dt) to Pair.

        Rotates data so that trace1 is lined up with degrees (and trace2 90 degrees clockwise).
        Applies a relative time shift by the nearest even number of samples to tlag,
        trace1 is shifted tlag/2 forward in time, and trace2 tlag/2 backward in time.
        Then undoes the original rotation.
        """
        # convert time shift to nsamples -- must be even
        nsamps = int(tlag / self.delta)
        nsamps = nsamps if nsamps%2==0 else nsamps + 1
        # find appropriate rotation angle
        rangle = degrees - self.angle
        # apply splitting
        self.x, self.y = core.split(self.x,self.y,rangle,nsamps)

    def unsplit(self,degrees,tlag):
        """
        Applies reverse splitting operator (phi,dt) to Pair.

        Rotates data so that trace1 is lined up with degrees (and trace2 90 degrees clockwise).
        Applies a relative time shift by the nearest even number of samples to tlag,
        trace1 is shifted tlag/2 backward in time, and trace2 tlag/2 forward in time.
        Then undoes the original rotation.
        """
        # convert time shift to nsamples -- must be even
        nsamps = int(tlag / self.delta)
        nsamps = nsamps if nsamps%2==0 else nsamps + 1
        # find appropriate rotation angle
        rangle = degrees - self.angle
        self.x, self.y = core.unsplit(self.x,self.y,rangle,nsamps)

    def rotateto(self,degrees):
        """
        Rotate data so that trace1 lines up with *degrees*
        """
        # find appropriate rotation angle
        rangle = -degrees - self.angle
        self.x, self.y = core.rotate(self.x,self.y,rangle)
        self.angle = degrees

    def lag(self,tlag):
        """
        Relative shift trace1 and trace2 by tlag seconds
        """
        # convert time shift to nsamples -- must be even
        nsamps = int(tlag / self.delta)
        nsamps = nsamps if nsamps%2==0 else nsamps + 1
        self.x, self.y = core.lag(self.x,self.y,nsamps)

    def chop(self,window):
        """
        Chop data around window
        """
        self.x, self.y = core.chop(self.x,self.y,window=window)

    def genwindow(self,time_centre,time_width,tukey=None):
        """
        Return a window object about time_centre with time_width.
        """
        tcs = int(time_centre / self.delta)
        offset = tcs - self.centre()
        # convert time to nsamples -- must be odd
        width = int(time_width / self.delta)
        width = width if width%2==1 else width + 1
        return Window(width,offset,tukey=tukey)

    # def autowindow(self,time_centre=None):
    #     """
    #     Makes a guess based on energy near *time_centre* about a suitable window
    #
    #     *time centre* should be the shear wave pick at the centre of the energy packet.
    #     By default will use centre sample.
    #     """
    #     if time_centre is None:
    #         t0 = self.centre()
    #     else:
    #         t0 = int(time_centre / self.delta)

    def copy(self):
        return copy.copy(self)

def _synth(**kwargs):
    """return ricker wavelet synthetic data"""

    if ('pol' in kwargs):
        pol = kwargs['pol']
    else:
        pol = 0.

    if ('fast' in kwargs):
        fast = kwargs['fast']
    else:
        fast = 0.

    if ('lag' in kwargs):
        lag = kwargs['lag']
    else:
        lag = 0

    if ('noise' in kwargs):
        noise = kwargs['noise']
    else:
        noise = 0.03

    if ('nsamps' in kwargs):
        nsamps = kwargs['nsamps']
    else:
        nsamps = 501

    if ('width' in kwargs):
        width = kwargs['width']
    else:
        width = 16.

    if ('window' in kwargs):
        window = kwargs['window']
    else:
        window = Window(width*3)

    nsamps = int(nsamps)

    x = signal.ricker(nsamps, width) + core.noise(nsamps,noise,width/4)
    y = core.noise(nsamps,noise,width/4)

    # rotate to polarisation
    x,y = core.rotate(x,y,-pol)

    # add any splitting -- this will reduce nsamps
    x,y = core.split(x,y,fast,lag)

    return x,y

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Tue Nov 26 10:57:30 2013

@author: Christian Muenker

Tab-Widget for displaying infos about filter and filter design method
"""
from __future__ import print_function, division, unicode_literals
import sys, os
import textwrap
from PyQt4 import QtGui
#import scipy.io
from scipy.signal import remez
#import numpy as np
from docutils.core import publish_string, publish_parts
# import filterbroker from one level above if this file is run as __main__
# for test purposes
if __name__ == "__main__":
    __cwd__ = os.path.dirname(os.path.abspath(__file__))
    sys.path.append(__cwd__ + '/..')

import filterbroker as fb # importing filterbroker initializes all its globals

# TODO: Setting the cursor position doesn't work yet
class InputInfo(QtGui.QWidget):
    """
    Create the window for entering exporting / importing and saving / loading data
    """
    def __init__(self, DEBUG = True):
        self.DEBUG = DEBUG
        super(InputInfo, self).__init__()

        self.initUI()
        self.showInfo()

    def initUI(self):
        """
        Intitialize the widget, consisting of:
        - Checkboxes for selecting the info to be displayed
        - A large text window for displaying infos about the filter design
          algorithm
        """
        # widget / subwindow for filter infos
        self.chkDocstring = QtGui.QCheckBox()
        self.chkDocstring.setChecked(False)
        self.chkDocstring.setToolTip("Display docstring from python filter method.")

        self.lblDocstring = QtGui.QLabel()
        self.lblDocstring.setText("Show Docstring")

        self.chkRichText = QtGui.QCheckBox()
        self.chkRichText.setChecked(True)
        self.chkRichText.setToolTip("Render documentation as Rich Text.")

        self.lblRichText = QtGui.QLabel()
        self.lblRichText.setText("Rich text")

        self.txtFiltInfoBox = QtGui.QTextBrowser()
        self.txtFiltInfoBox.setSizePolicy(QtGui.QSizePolicy.Minimum,
                                          QtGui.QSizePolicy.Expanding)

        # ============== UI Layout =====================================
        self.layHChkBoxes = QtGui.QHBoxLayout()
        self.layHChkBoxes.addWidget(self.chkDocstring)
        self.layHChkBoxes.addWidget(self.lblDocstring)
        self.layHChkBoxes.addStretch(10)
        self.layHChkBoxes.addWidget(self.chkRichText)
        self.layHChkBoxes.addWidget(self.lblRichText)

        self.layHChkBoxes.addStretch(10)

        layVMain = QtGui.QVBoxLayout()
        layVMain.addLayout(self.layHChkBoxes)
        layVMain.addWidget(self.txtFiltInfoBox)
#        layVMain.addStretch(10)
        self.setLayout(layVMain)

        # ============== Signals & Slots ================================
        self.chkDocstring.clicked.connect(self.showInfo)
        self.chkRichText.clicked.connect(self.showInfo)

    def showInfo(self):
        """
        Display info from filter design file and docstring
        """

        pos = self.txtFiltInfoBox.textCursor().position()
#        print(pos)

        if hasattr(fb.filObj,'info'):
            if self.chkRichText.isChecked():
                self.txtFiltInfoBox.setText(publish_string(
                    textwrap.dedent(fb.filObj.info), writer_name='html',
                    settings_overrides={'output_encoding': 'unicode'}))
            else:
                self.txtFiltInfoBox.setText(textwrap.dedent(fb.filObj.info))

        else:
            self.txtFiltInfoBox.setText("")

        if self.chkDocstring.isChecked() and hasattr(fb.filObj,'info_doc'):
            if self.chkRichText.isChecked():
                self.txtFiltInfoBox.append(
                '
Python module docstring:\n')
                for doc in fb.filObj.info_doc:
                    self.txtFiltInfoBox.append(publish_string(
                     self.cleanDoc(doc), writer_name='html'))
            else:
                self.txtFiltInfoBox.append('\nPython module docstring:\n')
                for doc in fb.filObj.info_doc:
                    self.txtFiltInfoBox.append(self.cleanDoc(doc))

#                self.txtFiltInfoBox.append(textwrap.dedent(fb.filObj.info_doc))

#
#                self.txtFiltInfoBox.append(publish_string(
#                    textwrap.dedent(fb.filObj.info_doc), writer_name='html'))

#        self.txtFiltInfoBox.textCursor().setPosition(pos) # no effect
        self.txtFiltInfoBox.moveCursor(QtGui.QTextCursor.Start)

    def cleanDoc(self, doc):
        lines = doc.splitlines()
        result = lines[0].lstrip() +\
         "\n" + textwrap.dedent("\n".join(lines[1:]))# + '\n'
        return result

#------------------------------------------------------------------------------

if __name__ == '__main__':

    app = QtGui.QApplication(sys.argv)
    form = InputInfo()
    form.show()

    app.exec_()<commit_msg>Added clean up for own docstrings as well Signed-off-by: chipmuenk <mail@chipmuenk.de><commit_after># -*- coding: utf-8 -*-
"""
Created on Tue Nov 26 10:57:30 2013

@author: Christian Muenker

Tab-Widget for displaying infos about filter and filter design method
"""
from __future__ import print_function, division, unicode_literals
import sys, os
import textwrap
from PyQt4 import QtGui
#import scipy.io
from scipy.signal import remez
#import numpy as np
from docutils.core import publish_string, publish_parts
# import filterbroker from one level above if this file is run as __main__
# for test purposes
if __name__ == "__main__":
    __cwd__ = os.path.dirname(os.path.abspath(__file__))
    sys.path.append(__cwd__ + '/..')

import filterbroker as fb # importing filterbroker initializes all its globals

# TODO: Setting the cursor position doesn't work yet
class InputInfo(QtGui.QWidget):
    """
    Create the window for entering exporting / importing and saving / loading data
    """
    def __init__(self, DEBUG = True):
        self.DEBUG = DEBUG
        super(InputInfo, self).__init__()

        self.initUI()
        self.showInfo()

    def initUI(self):
        """
        Intitialize the widget, consisting of:
        - Checkboxes for selecting the info to be displayed
        - A large text window for displaying infos about the filter design
          algorithm
        """
        # widget / subwindow for filter infos
        self.chkDocstring = QtGui.QCheckBox()
        self.chkDocstring.setChecked(False)
        self.chkDocstring.setToolTip("Display docstring from python filter method.")

        self.lblDocstring = QtGui.QLabel()
        self.lblDocstring.setText("Show Docstring")

        self.chkRichText = QtGui.QCheckBox()
        self.chkRichText.setChecked(True)
        self.chkRichText.setToolTip("Render documentation as Rich Text.")

        self.lblRichText = QtGui.QLabel()
        self.lblRichText.setText("Rich text")

        self.txtFiltInfoBox = QtGui.QTextBrowser()
        self.txtFiltInfoBox.setSizePolicy(QtGui.QSizePolicy.Minimum,
                                          QtGui.QSizePolicy.Expanding)

        # ============== UI Layout =====================================
        self.layHChkBoxes = QtGui.QHBoxLayout()
        self.layHChkBoxes.addWidget(self.chkDocstring)
        self.layHChkBoxes.addWidget(self.lblDocstring)
        self.layHChkBoxes.addStretch(10)
        self.layHChkBoxes.addWidget(self.chkRichText)
        self.layHChkBoxes.addWidget(self.lblRichText)

        self.layHChkBoxes.addStretch(10)

        layVMain = QtGui.QVBoxLayout()
        layVMain.addLayout(self.layHChkBoxes)
        layVMain.addWidget(self.txtFiltInfoBox)
#        layVMain.addStretch(10)
        self.setLayout(layVMain)

        # ============== Signals & Slots ================================
        self.chkDocstring.clicked.connect(self.showInfo)
        self.chkRichText.clicked.connect(self.showInfo)

    def showInfo(self):
        """
        Display info from filter design file and docstring
        """

        pos = self.txtFiltInfoBox.textCursor().position()
#        print(pos)

        if hasattr(fb.filObj,'info'):
            if self.chkRichText.isChecked():
                self.txtFiltInfoBox.setText(publish_string(
                    self.cleanDoc(fb.filObj.info), writer_name='html',
                    settings_overrides={'output_encoding': 'unicode'}))
            else:
                self.txtFiltInfoBox.setText(textwrap.dedent(fb.filObj.info))

        else:
            self.txtFiltInfoBox.setText("")

        if self.chkDocstring.isChecked() and hasattr(fb.filObj,'info_doc'):
            if self.chkRichText.isChecked():
                self.txtFiltInfoBox.append(
                '
Python module docstring:\n')
                for doc in fb.filObj.info_doc:
                    self.txtFiltInfoBox.append(publish_string(
                     self.cleanDoc(doc), writer_name='html'))
            else:
                self.txtFiltInfoBox.append('\nPython module docstring:\n')
                for doc in fb.filObj.info_doc:
                    self.txtFiltInfoBox.append(self.cleanDoc(doc))

#                self.txtFiltInfoBox.append(textwrap.dedent(fb.filObj.info_doc))

#
#                self.txtFiltInfoBox.append(publish_string(
#                    textwrap.dedent(fb.filObj.info_doc), writer_name='html'))

#        self.txtFiltInfoBox.textCursor().setPosition(pos) # no effect
        self.txtFiltInfoBox.moveCursor(QtGui.QTextCursor.Start)

    def cleanDoc(self, doc):
        lines = doc.splitlines()
        result = lines[0].lstrip() +\
         "\n" + textwrap.dedent("\n".join(lines[1:]))# + '\n'
        return result

#------------------------------------------------------------------------------

if __name__ == '__main__':

    app = QtGui.QApplication(sys.argv)
    form = InputInfo()
    form.show()

    app.exec_()<|endoftext|>