<commit_before># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Framework utilities.

See the @{$python/contrib.framework} guide.

@@assert_same_float_dtype
@@assert_scalar
@@assert_scalar_int
@@convert_to_tensor_or_sparse_tensor
@@get_graph_from_inputs
@@is_numeric_tensor
@@is_non_decreasing
@@is_strictly_increasing
@@is_tensor
@@reduce_sum_n
@@remove_squeezable_dimensions
@@with_shape
@@with_same_shape

@@deprecated
@@deprecated_args
@@deprecated_arg_values

@@arg_scope
@@add_arg_scope
@@current_arg_scope
@@has_arg_scope
@@arg_scoped_arguments

@@prepend_name_scope
@@strip_name_scope

@@add_model_variable
@@assert_global_step
@@assert_or_get_global_step
@@assign_from_checkpoint
@@assign_from_checkpoint_fn
@@assign_from_values
@@assign_from_values_fn
@@create_global_step
@@filter_variables
@@fuse_op
@@get_global_step
@@get_or_create_global_step
@@get_local_variables
@@get_model_variables
@@get_name_scope
@@get_trainable_variables
@@get_unique_variable
@@get_variables_by_name
@@get_variables_by_suffix
@@get_variable_full_name
@@get_variables_to_restore
@@get_variables
@@global_variable
@@local_variable
@@model_variable
@@variable
@@VariableDeviceChooser
@@convolutional_delta_orthogonal
@@convolutional_orthogonal_1d
@@convolutional_orthogonal_2d
@@convolutional_orthogonal_3d
@@zero_initializer

@@load_checkpoint
@@list_variables
@@load_variable
@@init_from_checkpoint
@@load_and_remap_matrix_initializer
@@load_embedding_initializer
@@load_linear_multiclass_bias_initializer
@@load_variable_slot_initializer

@@argsort
@@py_func
@@sort

@@get_placeholders

@@smart_cond
@@smart_constant_value
@@smart_case

@@CriticalSection

@@BoundedTensorSpec
@@TensorSpec
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

# pylint: disable=unused-import,wildcard-import
from tensorflow.contrib.framework.python.framework import *
from tensorflow.contrib.framework.python.framework import nest
from tensorflow.contrib.framework.python.ops import *
# pylint: enable=unused-import,wildcard-import

from tensorflow.python.framework.ops import prepend_name_scope
from tensorflow.python.framework.ops import strip_name_scope
from tensorflow.python.framework.smart_cond import smart_case
from tensorflow.python.framework.smart_cond import smart_cond
from tensorflow.python.framework.smart_cond import smart_constant_value
from tensorflow.python.framework.tensor_spec import BoundedTensorSpec
from tensorflow.python.framework.tensor_spec import TensorSpec
from tensorflow.python.ops.init_ops import convolutional_delta_orthogonal
from tensorflow.python.ops.init_ops import convolutional_orthogonal_1d
from tensorflow.python.ops.init_ops import convolutional_orthogonal_2d
from tensorflow.python.ops.init_ops import convolutional_orthogonal_3d
from tensorflow.python.util.all_util import remove_undocumented

_allowed_symbols = ['nest']
_nest_allowed_symbols = [
    'assert_same_structure',
    'is_sequence',
    'flatten',
    'flatten_dict_items',
    'pack_sequence_as',
    'map_structure',
    'assert_shallow_structure',
    'flatten_up_to',
    'map_structure_up_to',
    'get_traverse_shallow_structure',
    'yield_flat_paths',
    'flatten_with_joined_string_paths',
]

remove_undocumented(nest.__name__, allowed_exception_list=_nest_allowed_symbols)
remove_undocumented(__name__, allowed_exception_list=_allowed_symbols)
<commit_msg>Add map_structure_with_paths to list of documented nest symbols.<commit_after># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Framework utilities.

See the @{$python/contrib.framework} guide.

@@assert_same_float_dtype
@@assert_scalar
@@assert_scalar_int
@@convert_to_tensor_or_sparse_tensor
@@get_graph_from_inputs
@@is_numeric_tensor
@@is_non_decreasing
@@is_strictly_increasing
@@is_tensor
@@reduce_sum_n
@@remove_squeezable_dimensions
@@with_shape
@@with_same_shape

@@deprecated
@@deprecated_args
@@deprecated_arg_values

@@arg_scope
@@add_arg_scope
@@current_arg_scope
@@has_arg_scope
@@arg_scoped_arguments

@@prepend_name_scope
@@strip_name_scope

@@add_model_variable
@@assert_global_step
@@assert_or_get_global_step
@@assign_from_checkpoint
@@assign_from_checkpoint_fn
@@assign_from_values
@@assign_from_values_fn
@@create_global_step
@@filter_variables
@@fuse_op
@@get_global_step
@@get_or_create_global_step
@@get_local_variables
@@get_model_variables
@@get_name_scope
@@get_trainable_variables
@@get_unique_variable
@@get_variables_by_name
@@get_variables_by_suffix
@@get_variable_full_name
@@get_variables_to_restore
@@get_variables
@@global_variable
@@local_variable
@@model_variable
@@variable
@@VariableDeviceChooser
@@convolutional_delta_orthogonal
@@convolutional_orthogonal_1d
@@convolutional_orthogonal_2d
@@convolutional_orthogonal_3d
@@zero_initializer

@@load_checkpoint
@@list_variables
@@load_variable
@@init_from_checkpoint
@@load_and_remap_matrix_initializer
@@load_embedding_initializer
@@load_linear_multiclass_bias_initializer
@@load_variable_slot_initializer

@@argsort
@@py_func
@@sort

@@get_placeholders

@@smart_cond
@@smart_constant_value
@@smart_case

@@CriticalSection

@@BoundedTensorSpec
@@TensorSpec
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

# pylint: disable=unused-import,wildcard-import
from tensorflow.contrib.framework.python.framework import *
from tensorflow.contrib.framework.python.framework import nest
from tensorflow.contrib.framework.python.ops import *
# pylint: enable=unused-import,wildcard-import

from tensorflow.python.framework.ops import prepend_name_scope
from tensorflow.python.framework.ops import strip_name_scope
from tensorflow.python.framework.smart_cond import smart_case
from tensorflow.python.framework.smart_cond import smart_cond
from tensorflow.python.framework.smart_cond import smart_constant_value
from tensorflow.python.framework.tensor_spec import BoundedTensorSpec
from tensorflow.python.framework.tensor_spec import TensorSpec
from tensorflow.python.ops.init_ops import convolutional_delta_orthogonal
from tensorflow.python.ops.init_ops import convolutional_orthogonal_1d
from tensorflow.python.ops.init_ops import convolutional_orthogonal_2d
from tensorflow.python.ops.init_ops import convolutional_orthogonal_3d
from tensorflow.python.util.all_util import remove_undocumented

_allowed_symbols = ['nest']
_nest_allowed_symbols = [
    'assert_same_structure',
    'is_sequence',
    'flatten',
    'flatten_dict_items',
    'pack_sequence_as',
    'map_structure',
    'map_structure_with_paths',
    'assert_shallow_structure',
    'flatten_up_to',
    'map_structure_up_to',
    'get_traverse_shallow_structure',
    'yield_flat_paths',
    'flatten_with_joined_string_paths',
]

remove_undocumented(nest.__name__, allowed_exception_list=_nest_allowed_symbols)
remove_undocumented(__name__, allowed_exception_list=_allowed_symbols)
<|endoftext|>


<commit_before>"""
Tests for replacing master nodes.
"""

import textwrap
import uuid
from pathlib import Path
from typing import Iterator

import docker
import pytest
from _pytest.fixtures import SubRequest
from cluster_helpers import wait_for_dcos_oss
from dcos_e2e.backends import Docker
from dcos_e2e.cluster import Cluster
from dcos_e2e.node import Role
from docker.models.networks import Network

@pytest.fixture()
def docker_network_three_available_addresses() -> Iterator[Network]:
    """
    Return a custom Docker network with 3 assignable IP addresses.
    """
    # We want to return a Docker network with only three assignable IP
    # addresses.
    # To do this, we create a network with 8 IP addresses, where 5 are
    # reserved.
    #
    # Why we have 8 IP addresses available:
    # * The IP range is "172.28.0.0/29"
    # * We get 2 ^ (32 - 29) = 8 IP addresses
    #
    # The 8 IP addresses in the IPAM Pool are:
    # * 172.28.0.0 (reserved because this is the subnet identifier)
    # * 172.28.0.1 (reserved because this is the gateway address)
    # * 172.28.0.2 (available)
    # * 172.28.0.3 (available)
    # * 172.28.0.4 (available)
    # * 172.28.0.5 (reserved because we reserve this with `aux_addresses`)
    # * 172.28.0.6 (reserved because we reserve this with `aux_addresses`)
    # * 172.28.0.7 (reserved because this is the broadcast address)
    client = docker.from_env(version='auto')
    aux_addresses = {
        'reserved_address_0': '172.28.0.5',
        'reserved_address_1': '172.28.0.6',
    }
    ipam_pool = docker.types.IPAMPool(
        subnet='172.28.0.0/29',
        iprange='172.28.0.0/29',
        gateway='172.28.0.1',
        aux_addresses=aux_addresses,
    )
    network = client.networks.create(
        name='dcos-e2e-network-{random}'.format(random=uuid.uuid4()),
        driver='bridge',
        ipam=docker.types.IPAMConfig(pool_configs=[ipam_pool]),
        attachable=False,
    )
    try:
        yield network
    finally:
        network.remove()

def test_replace_all_static(
    artifact_path: Path,
    docker_network_three_available_addresses: Network,
    tmp_path: Path,
    request: SubRequest,
    log_dir: Path,
) -> None:
    """
    In a cluster with an Exhibitor backend consisting of a static ZooKeeper
    ensemble, after removing one master, and then adding another master with
    the same IP address, the cluster will get to a healthy state. This is
    repeated until all masters in the original cluster have been replaced.
    The purpose of this test is to assert that the ``node-poststart``
    procedure correctly prevents a master node replacement from being performed
    too quickly. A new master node should only become part of the cluster if
    there are no more underreplicated ranges reported by CockroachDB.

    Permanent CockroachDB data loss and a potential breakage of DC/OS occurs
    when a second master node is taken down for replacement while CockroachDB
    is recovering and there are still underreplicated ranges due to a recent
    other master node replacement.
    """
    docker_backend = Docker(network=docker_network_three_available_addresses)

    with Cluster(
        cluster_backend=docker_backend,
        # Allocate all 3 available IP addresses in the subnet.
        masters=3,
        agents=0,
        public_agents=0,
    ) as original_cluster:
        master = next(iter(original_cluster.masters))
        result = master.run(
            args=[
                'ifconfig',
                '|', 'grep', '-B1', str(master.public_ip_address),
                '|', 'grep', '-o', '"^\w*"',
            ],
            shell=True,
        )
        interface = result.stdout.strip().decode()
        ip_detect_contents = textwrap.dedent(
            """\
            #!/bin/bash -e
            if [ -f /sbin/ip ]; then
               IP_CMD=/sbin/ip
            else
               IP_CMD=/bin/ip
            fi

            $IP_CMD -4 -o addr show dev {interface} | awk '{{split($4,a,"/");print a[1]}}'
            """.format(interface=interface),
        )
        ip_detect_path = tmp_path / 'ip-detect'
        ip_detect_path.write_text(data=ip_detect_contents)
        static_config = {
            'master_discovery': 'static',
            'master_list': [str(master.private_ip_address)
                            for master in original_cluster.masters],
        }
        dcos_config = {
            **original_cluster.base_config,
            **static_config,
        }
        original_cluster.install_dcos_from_path(
            dcos_installer=artifact_path,
            dcos_config=dcos_config,
            ip_detect_path=ip_detect_path,
        )
        wait_for_dcos_oss(
            cluster=original_cluster,
            request=request,
            log_dir=log_dir,
        )
        current_cluster = original_cluster
        tmp_clusters = set()

        original_masters = original_cluster.masters

        try:
            for master_to_be_replaced in original_masters:
                # Destroy a master and free one IP address.
                original_cluster.destroy_node(node=master_to_be_replaced)

                temporary_cluster = Cluster(
                    cluster_backend=docker_backend,
                    # Allocate one container with the now free IP address.
                    masters=1,
                    agents=0,
                    public_agents=0,
                )
                tmp_clusters.add(temporary_cluster)

                # Install a new master on a new container with the same IP address.
                (new_master, ) = temporary_cluster.masters
                new_master.install_dcos_from_path(
                    dcos_installer=artifact_path,
                    dcos_config=dcos_config,
                    role=Role.MASTER,
                    ip_detect_path=ip_detect_path,
                )
                # Form a new cluster with the newly create master node.
                new_cluster = Cluster.from_nodes(
                    masters=current_cluster.masters.union({new_master}),
                    agents=current_cluster.agents,
                    public_agents=current_cluster.public_agents,
                )
                # The `wait_for_dcos_oss` function waits until the new master has
                # joined the cluster and all masters are healthy. Without the
                # cockroachdb check, this succeeds before all cockroachdb ranges
                # have finished replicating to the new master. That meant that the
                # next master would be replaced too quickly, while it had data that
                # was not present elsewhere in the cluster. This lead to
                # irrecoverable dataloss.  This function waits until the
                # master node is "healthy". This is a requirement for replacing the
                # next master node.
                #
                # We don't call the cockroachdb ranges check directly as the
                # purpose of this test is to ensure that when an operator follows
                # our documented procedure for replacing a master node multiple
                # times in a row (e.g. during a cluster upgrade) then the cluster
                # remains healthy throughout and afterwards.
                #
                # If we called the check directly here, we would be
                # sure the check is being called, but we would not be sure that
                # "wait_for_dcos_oss", i.e., the standard procedure for determining
                # whether a node is healthy, is sufficient to prevent the cluster
                # from breaking.
                #
                # We perform this check after every master is replaced, as that is
                # what we tell operators to do: "After installing the new master
                # node, wait until it becomes healthy before proceeding to the
                # next."
                #
                # The procedure for replacing multiple masters is documented here:
                # https://docs.mesosphere.com/1.12/installing/production/upgrading/#dcos-masters
                wait_for_dcos_oss(
                    cluster=new_cluster,
                    request=request,
                    log_dir=log_dir,
                )
                # Use the new cluster object in the next replacement iteration.
                current_cluster = new_cluster

        finally:
            for cluster in tmp_clusters:
                cluster.destroy()
<commit_msg>test-e2e/test_master_node_replacement: log output of ifconfig<commit_after>"""
Tests for replacing master nodes.
"""

import textwrap
import uuid
from pathlib import Path
from typing import Iterator

import docker
import pytest
from _pytest.fixtures import SubRequest
from cluster_helpers import wait_for_dcos_oss
from dcos_e2e.backends import Docker
from dcos_e2e.cluster import Cluster
from dcos_e2e.node import Output, Role
from docker.models.networks import Network

@pytest.fixture()
def docker_network_three_available_addresses() -> Iterator[Network]:
    """
    Return a custom Docker network with 3 assignable IP addresses.
    """
    # We want to return a Docker network with only three assignable IP
    # addresses.
    # To do this, we create a network with 8 IP addresses, where 5 are
    # reserved.
    #
    # Why we have 8 IP addresses available:
    # * The IP range is "172.28.0.0/29"
    # * We get 2 ^ (32 - 29) = 8 IP addresses
    #
    # The 8 IP addresses in the IPAM Pool are:
    # * 172.28.0.0 (reserved because this is the subnet identifier)
    # * 172.28.0.1 (reserved because this is the gateway address)
    # * 172.28.0.2 (available)
    # * 172.28.0.3 (available)
    # * 172.28.0.4 (available)
    # * 172.28.0.5 (reserved because we reserve this with `aux_addresses`)
    # * 172.28.0.6 (reserved because we reserve this with `aux_addresses`)
    # * 172.28.0.7 (reserved because this is the broadcast address)
    client = docker.from_env(version='auto')
    aux_addresses = {
        'reserved_address_0': '172.28.0.5',
        'reserved_address_1': '172.28.0.6',
    }
    ipam_pool = docker.types.IPAMPool(
        subnet='172.28.0.0/29',
        iprange='172.28.0.0/29',
        gateway='172.28.0.1',
        aux_addresses=aux_addresses,
    )
    network = client.networks.create(
        name='dcos-e2e-network-{random}'.format(random=uuid.uuid4()),
        driver='bridge',
        ipam=docker.types.IPAMConfig(pool_configs=[ipam_pool]),
        attachable=False,
    )
    try:
        yield network
    finally:
        network.remove()

def test_replace_all_static(
    artifact_path: Path,
    docker_network_three_available_addresses: Network,
    tmp_path: Path,
    request: SubRequest,
    log_dir: Path,
) -> None:
    """
    In a cluster with an Exhibitor backend consisting of a static ZooKeeper
    ensemble, after removing one master, and then adding another master with
    the same IP address, the cluster will get to a healthy state. This is
    repeated until all masters in the original cluster have been replaced.
    The purpose of this test is to assert that the ``node-poststart``
    procedure correctly prevents a master node replacement from being performed
    too quickly. A new master node should only become part of the cluster if
    there are no more underreplicated ranges reported by CockroachDB.

    Permanent CockroachDB data loss and a potential breakage of DC/OS occurs
    when a second master node is taken down for replacement while CockroachDB
    is recovering and there are still underreplicated ranges due to a recent
    other master node replacement.
    """
    docker_backend = Docker(network=docker_network_three_available_addresses)

    with Cluster(
        cluster_backend=docker_backend,
        # Allocate all 3 available IP addresses in the subnet.
        masters=3,
        agents=0,
        public_agents=0,
    ) as original_cluster:
        master = next(iter(original_cluster.masters))
        result = master.run(
            args=[
                'ifconfig',
                '|', 'grep', '-B1', str(master.public_ip_address),
                '|', 'grep', '-o', '"^\w*"',
            ],
            output=Output.LOG_AND_CAPTURE,
            shell=True,
        )
        interface = result.stdout.strip().decode()
        ip_detect_contents = textwrap.dedent(
            """\
            #!/bin/bash -e
            if [ -f /sbin/ip ]; then
               IP_CMD=/sbin/ip
            else
               IP_CMD=/bin/ip
            fi

            $IP_CMD -4 -o addr show dev {interface} | awk '{{split($4,a,"/");print a[1]}}'
            """.format(interface=interface),
        )
        ip_detect_path = tmp_path / 'ip-detect'
        ip_detect_path.write_text(data=ip_detect_contents)
        static_config = {
            'master_discovery': 'static',
            'master_list': [str(master.private_ip_address)
                            for master in original_cluster.masters],
        }
        dcos_config = {
            **original_cluster.base_config,
            **static_config,
        }
        original_cluster.install_dcos_from_path(
            dcos_installer=artifact_path,
            dcos_config=dcos_config,
            ip_detect_path=ip_detect_path,
        )
        wait_for_dcos_oss(
            cluster=original_cluster,
            request=request,
            log_dir=log_dir,
        )
        current_cluster = original_cluster
        tmp_clusters = set()

        original_masters = original_cluster.masters

        try:
            for master_to_be_replaced in original_masters:
                # Destroy a master and free one IP address.
                original_cluster.destroy_node(node=master_to_be_replaced)

                temporary_cluster = Cluster(
                    cluster_backend=docker_backend,
                    # Allocate one container with the now free IP address.
                    masters=1,
                    agents=0,
                    public_agents=0,
                )
                tmp_clusters.add(temporary_cluster)

                # Install a new master on a new container with the same IP address.
                (new_master, ) = temporary_cluster.masters
                new_master.install_dcos_from_path(
                    dcos_installer=artifact_path,
                    dcos_config=dcos_config,
                    role=Role.MASTER,
                    ip_detect_path=ip_detect_path,
                )
                # Form a new cluster with the newly create master node.
                new_cluster = Cluster.from_nodes(
                    masters=current_cluster.masters.union({new_master}),
                    agents=current_cluster.agents,
                    public_agents=current_cluster.public_agents,
                )
                # The `wait_for_dcos_oss` function waits until the new master has
                # joined the cluster and all masters are healthy. Without the
                # cockroachdb check, this succeeds before all cockroachdb ranges
                # have finished replicating to the new master. That meant that the
                # next master would be replaced too quickly, while it had data that
                # was not present elsewhere in the cluster. This lead to
                # irrecoverable dataloss.  This function waits until the
                # master node is "healthy". This is a requirement for replacing the
                # next master node.
                #
                # We don't call the cockroachdb ranges check directly as the
                # purpose of this test is to ensure that when an operator follows
                # our documented procedure for replacing a master node multiple
                # times in a row (e.g. during a cluster upgrade) then the cluster
                # remains healthy throughout and afterwards.
                #
                # If we called the check directly here, we would be
                # sure the check is being called, but we would not be sure that
                # "wait_for_dcos_oss", i.e., the standard procedure for determining
                # whether a node is healthy, is sufficient to prevent the cluster
                # from breaking.
                #
                # We perform this check after every master is replaced, as that is
                # what we tell operators to do: "After installing the new master
                # node, wait until it becomes healthy before proceeding to the
                # next."
                #
                # The procedure for replacing multiple masters is documented here:
                # https://docs.mesosphere.com/1.12/installing/production/upgrading/#dcos-masters
                wait_for_dcos_oss(
                    cluster=new_cluster,
                    request=request,
                    log_dir=log_dir,
                )
                # Use the new cluster object in the next replacement iteration.
                current_cluster = new_cluster

        finally:
            for cluster in tmp_clusters:
                cluster.destroy()
<|endoftext|>


<commit_before>"""Simulate a rotation dataset with a smoothly-varying beam position for
refinement testing. Script based on tst_nanoBragg_basic.py"""

from __future__ import absolute_import, division, print_function
from scitbx.array_family import flex
from simtbx.nanoBragg import shapetype
from simtbx.nanoBragg import nanoBragg
from scitbx import matrix
from dxtbx.model.detector import DetectorFactory
from dxtbx.model.beam import Beam, BeamFactory
from dxtbx.model.scan import ScanFactory
from dxtbx.model.goniometer import GoniometerFactory
from dxtbx.model import Crystal
from dxtbx.model.experiment_list import Experiment
from dxtbx.model.experiment_list import ExperimentList
from dxtbx.model.experiment_list import ExperimentListFactory
from dxtbx.model.experiment_list import ExperimentListDumper

pdb_lines = """HEADER TEST
CRYST1   50.000   60.000   70.000  90.00  90.00  90.00 P 1
ATOM      1  O   HOH A   1      56.829   2.920  55.702  1.00 20.00           O
ATOM      2  O   HOH A   2      49.515  35.149  37.665  1.00 20.00           O
ATOM      3  O   HOH A   3      52.667  17.794  69.925  1.00 20.00           O
ATOM      4  O   HOH A   4      40.986  20.409  18.309  1.00 20.00           O
ATOM      5  O   HOH A   5      46.896  37.790  41.629  1.00 20.00           O
ATOM      6 SED  MSE A   6       1.000   2.000   3.000  1.00 20.00          SE
END
"""

class Simulation(object):
    def __init__(self, override_fdp=None):

        # Set up detector
        distance = 100
        pixel_size = 0.1
        image_size = (1000, 1000)
        beam_centre_mm = (
            pixel_size * image_size[0] / 2,
            pixel_size * image_size[1] / 2,
        )
        self.detector = DetectorFactory().simple(
            "CCD",
            distance,
            beam_centre_mm,
            "+x",
            "-y",
            (pixel_size, pixel_size),
            image_size,
        )

        # Set up beam
        self.beam = BeamFactory().simple(wavelength=1)

        # Set up scan
        sweep_width = 90.0
        osc_start = 0.0
        image_width = 0.2
        oscillation = (osc_start, image_width)
        from math import ceil

        nframes = int(ceil(sweep_width / image_width))
        image_range = (1, nframes)
        exposure_times = 0.0
        epochs = [0] * nframes
        self.scan = ScanFactory().make_scan(
            image_range, exposure_times, oscillation, epochs, deg=True
        )

        # Set up goniometer
        self.goniometer = GoniometerFactory.known_axis(self.detector[0].get_fast_axis())

        # Set up simulated structure factors
        self.sfall = self.fcalc_from_pdb(
            resolution=1.6, algorithm="direct", override_fdp=override_fdp
        )

        # Set up crystal
        self.crystal = Crystal(
            real_space_a=(50, 0, 0),
            real_space_b=(0, 60, 0),
            real_space_c=(0, 0, 70),
            space_group_symbol="P1",
        )
        axis = matrix.col(
            elems=(-0.14480368275412925, -0.6202131724405818, -0.7709523423610766)
        )
        self.crystal.set_U(
            axis.axis_and_angle_as_r3_rotation_matrix(angle=0.625126343998969)
        )

        return

    def dump_experiments(self, file_name):
        e = Experiment(
            beam=self.beam,
            detector=self.detector,
            crystal=self.crystal,
            scan=self.scan,
            goniometer=self.goniometer,
        )
        el = ExperimentList()
        el.append(e)
        dump = ExperimentListDumper(el)
        dump.as_json(file_name)
        return

    def fcalc_from_pdb(self, resolution, algorithm=None, override_fdp=None):
        from iotbx import pdb

        pdb_inp = pdb.input(source_info=None, lines=pdb_lines)
        xray_structure = pdb_inp.xray_structure_simple()
        wavelength = self.beam.get_wavelength()
        #
        # take a detour to calculate anomalous contribution of every atom
        scatterers = xray_structure.scatterers()
        for sc in scatterers:
            from cctbx.eltbx import henke

            # expected_sasaki = sasaki.table(sc.element_symbol()).at_angstrom(wavelength)
            expected_henke = henke.table(sc.element_symbol()).at_angstrom(wavelength)
            sc.fp = expected_henke.fp()
            sc.fdp = override_fdp if override_fdp is not None else expected_henke.fdp()

        # how do we do bulk solvent?
        primitive_xray_structure = xray_structure.primitive_setting()
        P1_primitive_xray_structure = primitive_xray_structure.expand_to_p1()
        fcalc = P1_primitive_xray_structure.structure_factors(
            d_min=resolution, anomalous_flag=True, algorithm=algorithm
        ).f_calc()
        return fcalc.amplitudes()

    def set_varying_beam(self, along="fast", npixels_drift=5):

        assert along in ["fast", "slow", "both"]
        num_scan_points = self.scan.get_num_images() + 1
        s0 = matrix.col(self.beam.get_s0())
        beam_centre_px = self.detector[0].get_beam_centre_px(s0)
        if along == "fast":
            start_beam_centre = (
                beam_centre_px[0] - npixels_drift / 2,
                beam_centre_px[1],
            )
            end_beam_centre = (beam_centre_px[0] + npixels_drift / 2, beam_centre_px[1])
        elif along == "slow":
            start_beam_centre = (
                beam_centre_px[0],
                beam_centre_px[1] - npixels_drift / 2,
            )
            end_beam_centre = (beam_centre_px[0], beam_centre_px[1] + npixels_drift / 2)
        elif along == "both":
            from math import sqrt

            offset = sqrt(2.0) * npixels_drift / 4.0
            start_beam_centre = (beam_centre_px[0] - offset, beam_centre_px[1] - offset)
            end_beam_centre = (beam_centre_px[0] + offset, beam_centre_px[1] + offset)

        start_lab = matrix.col(self.detector[0].get_pixel_lab_coord(start_beam_centre))
        end_lab = matrix.col(self.detector[0].get_pixel_lab_coord(end_beam_centre))
        axis = start_lab.cross(end_lab).normalize()
        full_angle = start_lab.angle(end_lab)
        angle_step = full_angle / self.scan.get_num_images()
        angles = [e * angle_step for e in range(num_scan_points)]

        start_s0 = start_lab.normalize() * s0.length()
        s0_list = [start_s0.rotate_around_origin(axis=axis, angle=e) for e in angles]

        self.beam.set_s0_at_scan_points(s0_list)

        return

    def generate_image(self, image_no=1):

        # Set the beam for this image
        beam = Beam(self.beam)
        if self.beam.get_num_scan_points() == self.scan.get_num_images() + 1:
            # get s0 for the midpoint of this image
            s0_start = matrix.col(self.beam.get_s0_at_scan_point(image_no - 1))
            s0_end = matrix.col(self.beam.get_s0_at_scan_point(image_no))
            s0_mid = (s0_start + s0_end).normalize() * s0_start.length()
            beam.set_s0(s0_mid)

        print("Generating image {0}".format(image_no))
        phi_deg = self.scan.get_angle_from_image_index(image_no, deg=True)
        print("Rotation angle={0} degrees".format(phi_deg))
        print(
            "DIALS beam centre will be", self.detector[0].get_beam_centre(beam.get_s0())
        )

        # Construct simulation
        SIM = nanoBragg(self.detector, beam, verbose=0)
        SIM.Ncells_abc = (10, 10, 10)
        # set different random number seed for noise generation for each image
        SIM.seed = image_no
        SIM.oversample = 1
        SIM.progress_meter = False  # only really useful for long runs
        # SIM.default_F=100 # this will become F000, marking the beam center
        # use crystal structure to initialize Fhkl array
        SIM.Fhkl = self.sfall

        # This does not 'stick': "ERROR: cannot initialize without a cell"
        # SIM.Amatrix = self.crystal.get_A()

        # WORKAROUND: Instead, use nanoBragg to set the A matrix by missets, then
        # update the dxtbx crystal to keep track
        SIM.missets_deg = (10, 20, 30)
        # Apparently this needs the transpose
        self.crystal.set_A(matrix.sqr(SIM.Amatrix).transpose())

        SIM.xtal_shape = shapetype.Tophat  # fastest option, least realistic
        SIM.flux = 1e12  # photons/s
        SIM.beamsize_mm = 0.1  # assumes round beam
        SIM.exposure_s = 0.1

        # Set rotation image parameters
        SIM.phi_deg = phi_deg  # spindle phi rotation start angle (deg)
        SIM.osc_deg = self.scan.get_oscillation()[1]  # phi rotation range (deg)
        SIM.phisteps = 100  # 1000 would be better, for those with patience

        # Set low detector calibration noise of 1% (instead of default 3%)
        SIM.detector_calibration_noise_pct = 1

        # Now actually burn up some CPU
        SIM.add_nanoBragg_spots()

        # Simulated crystal is 1000 unit cells (50 nm wide). Amplify spot
        # signal to simulate physical crystal of 4000x larger: 200 um (64e9 x the
        # volume)
        SIM.raw_pixels *= 64e9

        # Write out the noise-free image with a pedestal matching that reported in
        # the header
        SIM.raw_pixels += SIM.adc_offset_adu
        fileout = "intimage_{0:03d}.img".format(image_no)
        SIM.to_smv_format(fileout=fileout, intfile_scale=1)
        SIM.raw_pixels -= SIM.adc_offset_adu

        # Add scatter from rough approximation to water: interpolation points for
        # sin(theta/lambda) vs structure factor
        bg = flex.vec2_double(
            [
                (0, 2.57),
                (0.0365, 2.58),
                (0.07, 2.8),
                (0.12, 5),
                (0.162, 8),
                (0.2, 6.75),
                (0.18, 7.32),
                (0.216, 6.75),
                (0.236, 6.5),
                (0.28, 4.5),
                (0.3, 4.3),
                (0.345, 4.36),
                (0.436, 3.77),
                (0.5, 3.17),
            ]
        )
        SIM.Fbg_vs_stol = bg
        SIM.amorphous_sample_thick_mm = 0.1
        SIM.amorphous_density_gcm3 = 1
        SIM.amorphous_molecular_weight_Da = 18
        SIM.add_background()

        # Add scatter from rough approximation to air
        bg = flex.vec2_double(
            [(0, 14.1), (0.045, 13.5), (0.174, 8.35), (0.35, 4.78), (0.5, 4.22)]
        )
        SIM.Fbg_vs_stol = bg
        SIM.amorphous_sample_thick_mm = 35  # between beamstop and collimator
        SIM.amorphous_density_gcm3 = 1.2e-3
        SIM.amorphous_sample_molecular_weight_Da = 28  # nitrogen = N2
        SIM.add_background()

        # detector PSF params for apply_psf(), called within add_noise()
        SIM.detector_psf_kernel_radius_pixels = 5
        SIM.detector_psf_fwhm_mm = 0.08
        SIM.detector_psf_type = shapetype.Fiber

        # Add the various noise sources
        SIM.add_noise()

        # Write out image with noise
        fileout = "noiseimage_{0:03d}.img".format(image_no)
        SIM.to_smv_format(fileout=fileout, intfile_scale=1)

        SIM.free_all()

    def generate_all_images(self):
        for i in range(self.scan.get_num_images()):
            self.generate_image(i + 1)

def check_anomalous_sign(mtz_file_name, override_fdp=None):
    """Test that the anomalous differences in a processed dataset are the right
  way round, so we haven't entered the 'inverted world' during the simulation.
  Using an MTZ from Aimless for this rather than data straight out of
  dials.integrate, mainly to use the simple any_reflection_file reader and to
  avoid having to decide between I and IPR etc."""

    # Extract miller array of intensities from the MTZ
    from iotbx.reflection_file_reader import any_reflection_file

    hkl_in = any_reflection_file(file_name=mtz_file_name)
    mtz_miller_arrays = hkl_in.as_miller_arrays()
    # assume I(+),SIGI(+),I(-),SIGI(-) is in the 2nd array
    mtz_intens = mtz_miller_arrays[1]

    assert mtz_intens.is_xray_intensity_array()
    mtz_ampl = mtz_intens.f_sq_as_f()
    print("MTZ anomalous signal = {0:.2f}%".format(mtz_ampl.anomalous_signal() * 100))
    mtz_anom_diff = mtz_ampl.anomalous_differences()

    # Take the largest 10% of anomalous differences
    # abs_diff = flex.abs(mtz_anom_diff.data())
    # cutoff = flex.sorted(abs_diff)[int(len(abs_diff) * 0.9)]
    # mtz_anom_diff = mtz_anom_diff.select(abs_diff > cutoff)

    # Simulation miller array of anomalous differences
    sim = Simulation(override_fdp=override_fdp)
    print(
        "Simulation anomalous signal = {0:.2f}%".format(
            sim.sfall.anomalous_signal() * 100
        )
    )
    sim_anom_diff = sim.sfall.anomalous_differences()

    # CCs of intensities and amplitudes
    sim_intens = sim.sfall.f_as_f_sq()
    mtz_intens_comm, sim_intens_comm = mtz_intens.common_sets(other=sim_intens)
    cc = flex.linear_correlation(
        mtz_intens_comm.data(), sim_intens_comm.data()
    ).coefficient()
    print("CC between intensities from MTZ and simulation = {0:.2f}%".format(cc * 100))
    mtz_ampl_comm, sim_ampl_comm = mtz_ampl.common_sets(other=sim.sfall)
    cc = flex.linear_correlation(
        mtz_ampl_comm.data(), sim_ampl_comm.data()
    ).coefficient()
    print("CC between amplitudes from MTZ and simulation = {0:.2f}%".format(cc * 100))

    # Write ideal SFs to an MTZ for comparison with viewhkl
    mtz_dataset = sim_intens_comm.as_mtz_dataset(column_root_label="I")
    mtz_dataset.add_miller_array(sim_ampl_comm, column_root_label="F")
    mtz_dataset.mtz_object().write("sim_data.mtz")

    # Get common sets of anomalous differences
    mtz_dano_comm, sim_dano_comm = mtz_anom_diff.common_sets(other=sim_anom_diff)

    cc = flex.linear_correlation(
        mtz_dano_comm.data(), sim_dano_comm.data()
    ).coefficient()
    msg = (
        "CC between anomalous differences of common " "reflections = {0:.2f}%"
    ).format(cc * 100)
    print(msg)

    # Compare signs
    mtz_pos = mtz_dano_comm.data() >= 0.0
    sim_pos = sim_dano_comm.data() >= 0.0
    n = len(mtz_pos)
    nsame = (mtz_pos == sim_pos).count(True)
    print(
        "Anomalous differences with the same sign: {0} ({1:0.2f}%)".format(
            nsame, 100.0 * nsame / n
        )
    )

def process_data(prefix="int"):
    assert prefix in ["int", "noise"]
    from libtbx import easy_run

    cmd = "dials.import {0}image_*.img".format(prefix)
    result = easy_run.fully_buffered(command=cmd)
    cmd = "dials.find_spots datablock.json nproc=8"
    result = easy_run.fully_buffered(command=cmd)
    cmd = "dials.index datablock.json strong.pickle"
    result = easy_run.fully_buffered(command=cmd)
    # now patch experiments.json to ensure the crystal model is indexed with the
    # right basis
    el = ExperimentListFactory.from_json_file("experiments.json")
    ref = ExperimentListFactory.from_json_file("sim_experiments.json")
    el[0].crystal = ref[0].crystal
    dump = ExperimentListDumper(el)
    dump.as_json("mod_experiments.json")
    cmd = "dials.index mod_experiments.json strong.pickle"
    result = easy_run.fully_buffered(command=cmd)
    cmd = "dials.refine experiments.json indexed.pickle"
    result = easy_run.fully_buffered(command=cmd)
    cmd = "dials.refine refined_experiments.json refined.pickle"
    result = easy_run.fully_buffered(command=cmd)
    cmd = "dials.integrate refined_experiments.json refined.pickle nproc=8"
    result = easy_run.fully_buffered(command=cmd)
    cmd = "dials.export integrated_experiments.json integrated.pickle"
    result = easy_run.fully_buffered(command=cmd)
    cmd = "aimless hklin integrated.mtz hklout {0}_scaled.mtz".format(prefix)
    stdin = ["resolution 2.0", "anomalous on"]
    result = easy_run.fully_buffered(command=cmd, stdin_lines=stdin)
    with open("{0}_aimless.log".format(prefix), "w") as f:
        f.writelines(result.stdout_lines)

def run_no_drift():
    sim = Simulation()
    sim.generate_all_images()
    # sim.generate_image(1)
    sim.dump_experiments("sim_experiments.json")
    process_data(prefix="int")
    process_data(prefix="noise")
    print("Checking processed data from noise-free images")
    check_anomalous_sign("int_scaled.mtz")
    print()
    print("Checking processed data from noisy images")
    check_anomalous_sign("noise_scaled.mtz")

def run_drift_along_fast():
    sim = Simulation()
    sim.set_varying_beam()
    sim.generate_all_images()

def run_drift_along_slow():
    sim = Simulation()
    sim.set_varying_beam(along="slow")
    sim.generate_all_images()

def run_drift_along_diag():
    sim = Simulation()
    sim.set_varying_beam(along="both")
    sim.generate_all_images()

if __name__ == "__main__":

    run_no_drift()
    # run_drift_along_fast()
<commit_msg>remove all unused code<commit_after>"""Simulate a rotation dataset with a smoothly-varying beam position for
refinement testing. Script based on tst_nanoBragg_basic.py"""

from __future__ import absolute_import, division, print_function

import math

from cctbx.eltbx import henke
from dxtbx.model import (
    BeamFactory,
    Crystal,
    DetectorFactory,
    GoniometerFactory,
    ScanFactory,
)
from iotbx import pdb
from scitbx import matrix

_pdb_lines = """HEADER TEST
CRYST1   50.000   60.000   70.000  90.00  90.00  90.00 P 1
ATOM      1  O   HOH A   1      56.829   2.920  55.702  1.00 20.00           O
ATOM      2  O   HOH A   2      49.515  35.149  37.665  1.00 20.00           O
ATOM      3  O   HOH A   3      52.667  17.794  69.925  1.00 20.00           O
ATOM      4  O   HOH A   4      40.986  20.409  18.309  1.00 20.00           O
ATOM      5  O   HOH A   5      46.896  37.790  41.629  1.00 20.00           O
ATOM      6 SED  MSE A   6       1.000   2.000   3.000  1.00 20.00          SE
END
"""

class Simulation(object):
    def __init__(self, override_fdp=None):

        # Set up detector
        distance = 100
        pixel_size = 0.1
        image_size = (1000, 1000)
        beam_centre_mm = (
            pixel_size * image_size[0] / 2,
            pixel_size * image_size[1] / 2,
        )
        self.detector = DetectorFactory().simple(
            "CCD",
            distance,
            beam_centre_mm,
            "+x",
            "-y",
            (pixel_size, pixel_size),
            image_size,
        )

        # Set up beam
        self.beam = BeamFactory().simple(wavelength=1)

        # Set up scan
        sweep_width = 90.0
        osc_start = 0.0
        image_width = 0.2
        oscillation = (osc_start, image_width)

        nframes = int(math.ceil(sweep_width / image_width))
        image_range = (1, nframes)
        exposure_times = 0.0
        epochs = [0] * nframes
        self.scan = ScanFactory().make_scan(
            image_range, exposure_times, oscillation, epochs, deg=True
        )

        # Set up goniometer
        self.goniometer = GoniometerFactory.known_axis(self.detector[0].get_fast_axis())

        # Set up simulated structure factors
        self.sfall = self.fcalc_from_pdb(
            resolution=1.6, algorithm="direct", override_fdp=override_fdp
        )

        # Set up crystal
        self.crystal = Crystal(
            real_space_a=(50, 0, 0),
            real_space_b=(0, 60, 0),
            real_space_c=(0, 0, 70),
            space_group_symbol="P1",
        )
        axis = matrix.col(
            elems=(-0.14480368275412925, -0.6202131724405818, -0.7709523423610766)
        )
        self.crystal.set_U(
            axis.axis_and_angle_as_r3_rotation_matrix(angle=0.625126343998969)
        )

    def fcalc_from_pdb(self, resolution, algorithm=None, override_fdp=None):
        pdb_inp = pdb.input(source_info=None, lines=_pdb_lines)
        xray_structure = pdb_inp.xray_structure_simple()
        wavelength = self.beam.get_wavelength()
        #
        # take a detour to calculate anomalous contribution of every atom
        scatterers = xray_structure.scatterers()
        for sc in scatterers:
            expected_henke = henke.table(sc.element_symbol()).at_angstrom(wavelength)
            sc.fp = expected_henke.fp()
            sc.fdp = override_fdp if override_fdp is not None else expected_henke.fdp()

        # how do we do bulk solvent?
        primitive_xray_structure = xray_structure.primitive_setting()
        P1_primitive_xray_structure = primitive_xray_structure.expand_to_p1()
        fcalc = P1_primitive_xray_structure.structure_factors(
            d_min=resolution, anomalous_flag=True, algorithm=algorithm
        ).f_calc()
        return fcalc.amplitudes()

    def set_varying_beam(self, along="fast", npixels_drift=5):
        assert along in ["fast", "slow", "both"]
        num_scan_points = self.scan.get_num_images() + 1
        s0 = matrix.col(self.beam.get_s0())
        beam_centre_px = self.detector[0].get_beam_centre_px(s0)
        if along == "fast":
            start_beam_centre = (
                beam_centre_px[0] - npixels_drift / 2,
                beam_centre_px[1],
            )
            end_beam_centre = (beam_centre_px[0] + npixels_drift / 2, beam_centre_px[1])
        elif along == "slow":
            start_beam_centre = (
                beam_centre_px[0],
                beam_centre_px[1] - npixels_drift / 2,
            )
            end_beam_centre = (beam_centre_px[0], beam_centre_px[1] + npixels_drift / 2)
        elif along == "both":
            offset = math.sqrt(2.0) * npixels_drift / 4.0
            start_beam_centre = (beam_centre_px[0] - offset, beam_centre_px[1] - offset)
            end_beam_centre = (beam_centre_px[0] + offset, beam_centre_px[1] + offset)

        start_lab = matrix.col(self.detector[0].get_pixel_lab_coord(start_beam_centre))
        end_lab = matrix.col(self.detector[0].get_pixel_lab_coord(end_beam_centre))
        axis = start_lab.cross(end_lab).normalize()
        full_angle = start_lab.angle(end_lab)
        angle_step = full_angle / self.scan.get_num_images()
        angles = [e * angle_step for e in range(num_scan_points)]

        start_s0 = start_lab.normalize() * s0.length()
        s0_list = [start_s0.rotate_around_origin(axis=axis, angle=e) for e in angles]

        self.beam.set_s0_at_scan_points(s0_list)
<|endoftext|>


<commit_before>import json
from functools import reduce
from collections import defaultdict
from node import Node, Interface
from link import Link, LinkConnector
from itertools import zip_longest

from bs4 import BeautifulSoup
from urllib.request import urlopen

class NodeDB:
  def __init__(self):
    self._nodes = []
    self._links = []

  # fetch list of links
  def get_links(self):
    self.update_vpn_links()
    return self.reduce_links()

  # fetch list of nodes
  def get_nodes(self):
    return self._nodes

  def maybe_node_by_fuzzy_mac(self, mac):
    mac_a = mac.lower()

    for node in self._nodes:
      for mac_b in node.macs:
        if is_similar(mac_a, mac_b):
          return node

    raise KeyError

  def maybe_node_by_mac(self, macs):
    for node in self._nodes:
      for mac in macs:
        if mac.lower() in node.macs:
          return node

    raise

  # import_batman(list(fileinput.input(options['batmanjson'])))
  def import_batman(self, lines):

    for line in lines:
      x = json.loads(line)

      if 'of' in x:
        try:
          node = self.maybe_node_by_mac((x['of'], x['secondary']))
        except:
          node = Node()
          node.flags['online'] = True
          self._nodes.append(node)

        node.add_mac(x['of'])
        node.add_mac(x['secondary'])

    for line in lines:
      x = json.loads(line)

      if 'router' in x:
        try:
          node = self.maybe_node_by_mac((x['router'], ))
        except:
          node = Node()
          node.flags['online'] = True
          node.add_mac(x['router'])
          self._nodes.append(node)

        # If it's a TT link and the MAC is very similar
        # consider this MAC as one of the routers
        # MACs
        if 'gateway' in x and x['label'] == "TT":
          if is_similar(x['router'], x['gateway']):
            node.add_mac(x['gateway'])

            # skip processing as regular link
            continue

        try:
          if 'neighbor' in x:
            try:
              node = self.maybe_node_by_mac((x['neighbor']))
            except:
              continue

          if 'gateway' in x:
            x['neighbor'] = x['gateway']

          node = self.maybe_node_by_mac((x['neighbor'], ))
        except:
          node = Node()
          node.flags['online'] = True
          if x['label'] == 'TT':
            node.flags['client'] = True

          node.add_mac(x['neighbor'])
          self._nodes.append(node)

    for line in lines:
      x = json.loads(line)

      if 'router' in x:
        try:
          if 'gateway' in x:
            x['neighbor'] = x['gateway']

          router = self.maybe_node_by_mac((x['router'], ))
          neighbor = self.maybe_node_by_mac((x['neighbor'], ))
        except:
          continue

        # filter TT links merged in previous step
        if router == neighbor:
          continue

        link = Link()
        link.source = LinkConnector()
        link.source.interface = x['router']
        link.source.id = self._nodes.index(router)
        link.target = LinkConnector()
        link.target.interface = x['neighbor']
        link.target.id = self._nodes.index(neighbor)
        link.quality = x['label']
        link.id = "-".join(sorted((link.source.interface, link.target.interface)))

        if x['label'] == "TT":
          link.type = "client"

        self._links.append(link)

    for line in lines:
      x = json.loads(line)

      if 'primary' in x:
        try:
          node = self.maybe_node_by_mac((x['primary'], ))
        except:
          continue

        node.id = x['primary']

  def reduce_links(self):
    tmp_links = defaultdict(list)

    for link in self._links:
      tmp_links[link.id].append(link)

    links = []

    def reduce_link(a, b):
      a.id = b.id
      a.source = b.source
      a.target = b.target
      a.type = b.type
      a.quality = ", ".join([x for x in (a.quality, b.quality) if x])

      return a

    for k, v in tmp_links.items():
      new_link = reduce(reduce_link, v, Link())
      links.append(new_link)

    return links

  def import_aliases(self, aliases):
    for mac, alias in aliases.items():
      try:
        node = self.maybe_node_by_mac((mac, ))
      except:
        continue

      node.name = alias['name']

      if 'vpn' in alias and alias['vpn']:
        node.interfaces[mac].vpn = True

      if 'gps' in alias:
        node.gps = alias['gps']

  # list of macs
  # if options['gateway']:
  #   mark_gateways(options['gateway'])
  def mark_gateways(self, gateways):
    for gateway in gateways:
      try:
        node = self.maybe_node_by_mac((gateway, ))
      except:
        continue

      node.flags['gateway'] = True

  def update_vpn_links(self):
    changes = 1
    while changes > 0:
      changes = 0
      for link in self._links:
        if link.type == "client":
          continue

        source_interface = self._nodes[link.source.id].interfaces[link.source.interface]
        target_interface = self._nodes[link.target.id].interfaces[link.target.interface]
        if source_interface.vpn or target_interface.vpn:
          source_interface.vpn = True
          target_interface.vpn = True
          if link.type != "vpn":
            changes += 1

          link.type = "vpn"

  def import_wikigps(self, url):
    def fetch_wikitable(url):
      f = urlopen(url)

      soup = BeautifulSoup(f)

      table = soup.find_all("table")[0]

      rows = table.find_all("tr")

      headers = []

      data = []

      def maybe_strip(x):
        if isinstance(x.string, str):
          return x.string.strip()
        else:
          return ""

      for row in rows:
        tds = list([maybe_strip(x) for x in row.find_all("td")])
        ths = list([maybe_strip(x) for x in row.find_all("th")])

        if any(tds):
          data.append(tds)

        if any(ths):
          headers = ths

      nodes = []

      for d in data:
        nodes.append(dict(zip(headers, d)))

      return nodes

    nodes = fetch_wikitable(url)

    for node in nodes:
      try:
        node['MAC'] = node['MAC'].split(',')
      except KeyError:
        pass

      try:
        node['GPS'] = node['GPS'].split(',')
      except KeyError:
        pass

      try:
        node['Nick'] = node['Nick'].split(',')
      except KeyError:
        pass

      nodes = zip_longest(node['MAC'], node['GPS'], node['Nick'])

      for data in nodes:
        if not data[0]:
          continue

        try:
          node = self.maybe_node_by_fuzzy_mac(data[0])
        except KeyError:
          node = Node()
          node.add_mac(data[0])
          self._nodes.append(node)

        if data[1]:
          node.gps = data[1]

        if data[2]:
          node.name = data[2]

# compares two MACs and decides whether they are
# similar and could be from the same node
def is_similar(a, b):
  if a == b:
    return True

  try:
    mac_a = list(int(i, 16) for i in a.split(":"))
    mac_b = list(int(i, 16) for i in b.split(":"))
  except ValueError:
    return False

  # first byte must only differ in bit 2
  if mac_a[0] | 2 == mac_b[0] | 2:
    # count different bytes
    c = [x for x in zip(mac_a[1:], mac_b[1:]) if x[0] != x[1]]
  else:
    return False

  # no more than two additional bytes must differ
  if len(c) <= 2:
    delta = 0

  if len(c) > 0:
    delta = sum(abs(i[0] -i[1]) for i in c)

  # These addresses look pretty similar!
  return delta < 8

<commit_msg>much better mac matching<commit_after>import json
from functools import reduce
from collections import defaultdict
from node import Node, Interface
from link import Link, LinkConnector
from itertools import zip_longest

from bs4 import BeautifulSoup
from urllib.request import urlopen

class NodeDB:
  def __init__(self):
    self._nodes = []
    self._links = []

  # fetch list of links
  def get_links(self):
    self.update_vpn_links()
    return self.reduce_links()

  # fetch list of nodes
  def get_nodes(self):
    return self._nodes

  def maybe_node_by_fuzzy_mac(self, mac):
    mac_a = mac.lower()

    for node in self._nodes:
      for mac_b in node.macs:
        if is_derived_mac(mac_a, mac_b):
          return node

    raise KeyError

  def maybe_node_by_mac(self, macs):
    for node in self._nodes:
      for mac in macs:
        if mac.lower() in node.macs:
          return node

    raise

  # import_batman(list(fileinput.input(options['batmanjson'])))
  def import_batman(self, lines):

    for line in lines:
      x = json.loads(line)

      if 'of' in x:
        try:
          node = self.maybe_node_by_mac((x['of'], x['secondary']))
        except:
          node = Node()
          node.flags['online'] = True
          self._nodes.append(node)

        node.add_mac(x['of'])
        node.add_mac(x['secondary'])

    for line in lines:
      x = json.loads(line)

      if 'router' in x:
        try:
          node = self.maybe_node_by_mac((x['router'], ))
        except:
          node = Node()
          node.flags['online'] = True
          node.add_mac(x['router'])
          self._nodes.append(node)

        # If it's a TT link and the MAC is very similar
        # consider this MAC as one of the routers
        # MACs
        if 'gateway' in x and x['label'] == "TT":
          if is_similar(x['router'], x['gateway']):
            node.add_mac(x['gateway'])

            # skip processing as regular link
            continue

        try:
          if 'neighbor' in x:
            try:
              node = self.maybe_node_by_mac((x['neighbor']))
            except:
              continue

          if 'gateway' in x:
            x['neighbor'] = x['gateway']

          node = self.maybe_node_by_mac((x['neighbor'], ))
        except:
          node = Node()
          node.flags['online'] = True
          if x['label'] == 'TT':
            node.flags['client'] = True

          node.add_mac(x['neighbor'])
          self._nodes.append(node)

    for line in lines:
      x = json.loads(line)

      if 'router' in x:
        try:
          if 'gateway' in x:
            x['neighbor'] = x['gateway']

          router = self.maybe_node_by_mac((x['router'], ))
          neighbor = self.maybe_node_by_mac((x['neighbor'], ))
        except:
          continue

        # filter TT links merged in previous step
        if router == neighbor:
          continue

        link = Link()
        link.source = LinkConnector()
        link.source.interface = x['router']
        link.source.id = self._nodes.index(router)
        link.target = LinkConnector()
        link.target.interface = x['neighbor']
        link.target.id = self._nodes.index(neighbor)
        link.quality = x['label']
        link.id = "-".join(sorted((link.source.interface, link.target.interface)))

        if x['label'] == "TT":
          link.type = "client"

        self._links.append(link)

    for line in lines:
      x = json.loads(line)

      if 'primary' in x:
        try:
          node = self.maybe_node_by_mac((x['primary'], ))
        except:
          continue

        node.id = x['primary']

  def reduce_links(self):
    tmp_links = defaultdict(list)

    for link in self._links:
      tmp_links[link.id].append(link)

    links = []

    def reduce_link(a, b):
      a.id = b.id
      a.source = b.source
      a.target = b.target
      a.type = b.type
      a.quality = ", ".join([x for x in (a.quality, b.quality) if x])

      return a

    for k, v in tmp_links.items():
      new_link = reduce(reduce_link, v, Link())
      links.append(new_link)

    return links

  def import_aliases(self, aliases):
    for mac, alias in aliases.items():
      try:
        node = self.maybe_node_by_mac((mac, ))
      except:
        continue

      node.name = alias['name']

      if 'vpn' in alias and alias['vpn']:
        node.interfaces[mac].vpn = True

      if 'gps' in alias:
        node.gps = alias['gps']

  # list of macs
  # if options['gateway']:
  #   mark_gateways(options['gateway'])
  def mark_gateways(self, gateways):
    for gateway in gateways:
      try:
        node = self.maybe_node_by_mac((gateway, ))
      except:
        continue

      node.flags['gateway'] = True

  def update_vpn_links(self):
    changes = 1
    while changes > 0:
      changes = 0
      for link in self._links:
        if link.type == "client":
          continue

        source_interface = self._nodes[link.source.id].interfaces[link.source.interface]
        target_interface = self._nodes[link.target.id].interfaces[link.target.interface]
        if source_interface.vpn or target_interface.vpn:
          source_interface.vpn = True
          target_interface.vpn = True
          if link.type != "vpn":
            changes += 1

          link.type = "vpn"

  def import_wikigps(self, url):
    def fetch_wikitable(url):
      f = urlopen(url)

      soup = BeautifulSoup(f)

      table = soup.find_all("table")[0]

      rows = table.find_all("tr")

      headers = []

      data = []

      def maybe_strip(x):
        if isinstance(x.string, str):
          return x.string.strip()
        else:
          return ""

      for row in rows:
        tds = list([maybe_strip(x) for x in row.find_all("td")])
        ths = list([maybe_strip(x) for x in row.find_all("th")])

        if any(tds):
          data.append(tds)

        if any(ths):
          headers = ths

      nodes = []

      for d in data:
        nodes.append(dict(zip(headers, d)))

      return nodes

    nodes = fetch_wikitable(url)

    for node in nodes:
      try:
        node['MAC'] = node['MAC'].split(',')
      except KeyError:
        pass

      try:
        node['GPS'] = node['GPS'].split(',')
      except KeyError:
        pass

      try:
        node['Nick'] = node['Nick'].split(',')
      except KeyError:
        pass

      nodes = zip_longest(node['MAC'], node['GPS'], node['Nick'])

      for data in nodes:
        if not data[0]:
          continue

        try:
          node = self.maybe_node_by_fuzzy_mac(data[0])
        except KeyError:
          node = Node()
          node.add_mac(data[0])
          self._nodes.append(node)

        if data[1]:
          node.gps = data[1]

        if data[2]:
          node.name = data[2]

# compares two MACs and decides whether they are
# similar and could be from the same node
def is_similar(a, b):
  if a == b:
    return True

  try:
    mac_a = list(int(i, 16) for i in a.split(":"))
    mac_b = list(int(i, 16) for i in b.split(":"))
  except ValueError:
    return False

  # first byte must only differ in bit 2
  if mac_a[0] | 2 == mac_b[0] | 2:
    # count different bytes
    c = [x for x in zip(mac_a[1:], mac_b[1:]) if x[0] != x[1]]
  else:
    return False

  # no more than two additional bytes must differ
  if len(c) <= 2:
    delta = 0

  if len(c) > 0:
    delta = sum(abs(i[0] -i[1]) for i in c)

  # These addresses look pretty similar!
  return delta < 8

def is_derived_mac(a, b):
  if a == b:
    return True

  try:
    mac_a = list(int(i, 16) for i in a.split(":"))
    mac_b = list(int(i, 16) for i in b.split(":"))
  except ValueError:
    return False

  x = mac_a
  x[5] += 1
  if mac_b == x:
    return True

  x[0] |= 2
  if mac_b == x:
    return True

  x[3] += 1
  if mac_b == x:
    return True

  x = mac_a
  x[0] |= 2
  x[5] += 2
  if mac_b == x:
    return True

  x = mac_a
  x[0] |= 2
  x[3] += 1
  if mac_b == x:
    return True

  return False
<|endoftext|>


<commit_before>import json
import os
import sys
from jinja2 import Environment, PackageLoader

env =  Environment(loader=PackageLoader('cat'))
template = env.get_template('email.html')
# add in e-mail from https://docs.python.org/3.4/library/email-examples.html
root = os.path.dirname((os.path.realpath(__file__)))
sys.path.insert(0, root)
from cat import tools
from cat.tools import read_json

with open('subscribers.json', 'r') as fh:
  subscriptions = json.load(fh)

print(subscriptions)

cat = read_json(root + '/html/cat.json')
conferences = tools.future(cat)

print(template.render(
  events = conferences, title="Hello"
))
<commit_msg>add mail service to send notification mails<commit_after>import json
import os
import sys
import smtplib
from jinja2 import Environment, PackageLoader

from email.mime.text import MIMEText

from email.mime.multipart import MIMEMultipart

env =  Environment(loader=PackageLoader('cat'))
template = env.get_template('email.html')
# add in e-mail from https://docs.python.org/3.4/library/email-examples.html
root = os.path.dirname((os.path.realpath(__file__)))
sys.path.insert(0, root)
from cat import tools
from cat.tools import read_json

with open('subscribers.json', 'r') as fh:
  subscriptions = json.load(fh)

print(subscriptions)

cat = read_json(root + '/html/cat.json')
conferences = tools.future(cat)

html =template.render(
  events = conferences, title="Hello"
)

text = "Hello"
me = "me@email"
you = "me@email"

msg = MIMEMultipart('alternative')
msg['Subject'] = "Upcoming Events"
msg['From'] = me
msg['To'] = you

part1 = MIMEText(text, 'plain')
part2 = MIMEText(html, 'html')

msg.attach(part1)
msg.attach(part2)

#s = smtplib.SMTP('localhost')
#s.sendmail(me, you, msg.as_string())
#s.quit()

with open ("file.txt", "wb") as fout:
    fout.write (bytes(msg))

<|endoftext|>


<commit_before>#!/usr/bin/python
# FIXME docstring wrong
"""
openmz -- Connecting mzIdentML with openMS

USAGE:
   openmz.py [option] [input files]
EXAMPLE:
   openmz.py -c quanttsv --mzidtsv psms.tsv --spectra file1.mzML file2.mzML -d /home/data/output
"""

import argparse
import os
from app.drivers import openmz as drivers

def parser_file_exists(currentparser, fn):
    if not os.path.exists(fn):
        currentparser.error('Input file %s not found!' % fn)
    else:
        return fn

parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('-c', dest='command', type=str,
                    help='How to manipulate the input:\n'
                    'quanttsv       - Add quantitative data from openMS '
                    'consensusXML to a tab separated file with '
                    'PSMs, their statistics and their quantification data. '
                    'Needs except these also the corresponding mzML spectra '
                    'files to correlate retention time to scan nrs.',
                    required=True
                    )
parser.add_argument('-d', dest='outdir', required=True,
                    help='Directory to output in',
                    type=lambda x: parser_file_exists(parser, x))
parser.add_argument('--mzidtsv', dest='mzidtsv', help='TSV table of mzIdentML',
                    type=lambda x: parser_file_exists(parser, x))
parser.add_argument('--quant', dest='quants', help='Quant data in '
                    'consensusXML format',
                    type=lambda x: parser_file_exists(parser, x))
parser.add_argument('--spectra', dest='spectra', help='mzML files', nargs='+',
                    type=lambda x: parser_file_exists(parser, x))

# TODO to be supported
#parser.add_argument('--allpsms', dest='allpsms', action='store_true',
#                    help='All PSMs from a single scan should be included, '
#                    'not only the best scoring one.')

args = parser.parse_args()

commandmap = {
    'quanttsv': drivers.TSVQuantDriver,
}

command = commandmap[args.command](**vars(args))
command.run()
<commit_msg>Settled for -i for openms output file instead of --quant<commit_after>#!/usr/bin/python
# FIXME docstring wrong
"""
openmz -- Connecting mzIdentML with openMS

USAGE:
   openmz.py [option] [input files]
EXAMPLE:
   openmz.py -c quanttsv --mzidtsv psms.tsv --spectra file1.mzML file2.mzML -d /home/data/output
"""

import argparse
import os
from app.drivers import openmz as drivers

def parser_file_exists(currentparser, fn):
    if not os.path.exists(fn):
        currentparser.error('Input file %s not found!' % fn)
    else:
        return fn

parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('-c', dest='command', type=str,
                    help='How to manipulate the input:\n'
                    'quanttsv       - Add quantitative data from openMS '
                    'consensusXML to a tab separated file with '
                    'PSMs, their statistics and their quantification data. '
                    'Needs except these also the corresponding mzML spectra '
                    'files to correlate retention time to scan nrs.',
                    required=True
                    )
parser.add_argument('-i', dest='infile',
                    type=lambda x: parser_file_exists(parser, x),
                    required=True, help='OpenMS result file')
parser.add_argument('-d', dest='outdir', required=True,
                    help='Directory to output in',
                    type=lambda x: parser_file_exists(parser, x))
parser.add_argument('--mzidtsv', dest='mzidtsv', help='TSV table of mzIdentML',
                    type=lambda x: parser_file_exists(parser, x))
parser.add_argument('--spectra', dest='spectra', help='mzML files', nargs='+',
                    type=lambda x: parser_file_exists(parser, x))

# TODO to be supported
#parser.add_argument('--allpsms', dest='allpsms', action='store_true',
#                    help='All PSMs from a single scan should be included, '
#                    'not only the best scoring one.')

args = parser.parse_args()

commandmap = {
    'quanttsv': drivers.TSVQuantDriver,
}

command = commandmap[args.command](**vars(args))
command.run()
<|endoftext|>


<commit_before>from ophyd import (EpicsMotor, PseudoSingle, PseudoPositioner,
                   Component as Cpt)
from ophyd.pseudopos import (pseudo_position_argument, real_position_argument)
import numpy as np
from scipy import interpolate

_ivu_gap = [ 6184.0,
       6368.0,
       6550.0,
       6730.0,
       6909.0,
       7086.0,
       7262.0,
       7438.0,
       7613.0,
       7788.0,
       7962.0,
       8138.0,
       8311.0,
       8486.0,
       8662.0,
       8840.0,
       9019.0,
       9200.0,
       9382.0,
       9567.0,
       9753.0,
       9943.0,
       10136.0,
       10332.0,
       10531.0,
       10740.0,
       10960.0,
       11385.0]

_Bragg = [ 16.401781,
       15.821071,
       15.280587,
       14.776273,
       14.304428,
       13.862182,
       13.446656,
       13.055574,
       12.686789,
       12.338436,
       12.008851,
       11.696607,
       11.400018,
       11.118472,
       10.850483,
       10.595196,
       10.351707,
       10.119222,
       9.896987,
       9.684357,
       9.480717,
       9.28554,
       9.098203,
       8.918375,
       8.7455,
       8.579178,
       8.41915,
       8.11645 ]

gcalc = interpolate.interp1d(_Bragg, _ivu_gap)

_hc = 12398.4193
_si_111 = 3.1363

class BLEnergy(PseudoPositioner):
    # limits and constants from Spec file, "site.mac"
    energy = Cpt(PseudoSingle, limits=(7.835, 17.7))

    theta = Cpt(EpicsMotor, 'XF:10IDA-OP{Mono:DCM-Ax:P}Mtr')
    z2 = Cpt(EpicsMotor, 'XF:10IDA-OP{Mono:DCM-Ax:Z2}Mtr')
    ugap = Cpt(Undulator, 'SR:C10-ID:G1{IVU22:1')

    def forward(self, pseudo_pos):
        _th = np.rad2deg(np.arcsin(_hc/(2.*_si_111*pseudo_pos)))
        _z2 = 15./np.cos(np.deg2rad(_th)) - 15.
        _ugap = gcalc(_th)

        return self.RealPosition(theta=_th, z2=_z2, ugap=_ugap)

    def inverse(self, real_pos):
        en = _hc/(2.*_si_111*np.sin(np.deg2rad(real_pos.theta)))
        en = float(en)
        return self.PseudoPosition(energy=en)

blE = BLEnergy('', name='blE', egu='eV')

class DCMEnergy(PseudoPositioner):
    # limits and constants from Spec file, "site.mac"
    energy = Cpt(PseudoSingle, limits=(7.835, 17.7))

    theta = Cpt(EpicsMotor, 'XF:10IDA-OP{Mono:DCM-Ax:P}Mtr')
    z2 = Cpt(EpicsMotor, 'XF:10IDA-OP{Mono:DCM-Ax:Z2}Mtr')

    def forward(self, pseudo_pos):
        _th = np.rad2deg(np.arcsin(_hc/(2.*_si_111*pseudo_pos)))
        _z2 = 15./np.cos(np.deg2rad(_th)) - 15.

        return self.RealPosition(theta=_th, z2=_z2)

    def inverse(self, real_pos):
        en = _hc/(2.*_si_111*np.sin(np.deg2rad(real_pos.theta)))
        en = float(en)
        return self.PseudoPosition(energy=en)

dcmE = DCMEnergy('', name='dcmE', egu='eV')

# constants for HRM2 energy conversion
_TB = 69.2943
_EB = 9.1317
class HRMEnergy(PseudoPositioner):
    energy = Cpt(PseudoSingle)

    uof = Cpt(EpicsMotor, 'XF:10IDB-OP{Mono:HRM2-Ax:UTO}Mtr')
    dof = Cpt(EpicsMotor, 'XF:10IDB-OP{Mono:HRM2-Ax:DTO}Mtr')

    def forward(self, pseudo_pos):
        _pos = -1.0*pseudo_pos.energy*np.tan(np.deg2rad(_TB))/_EB
        return self.RealPosition(uof=_pos, dof=_pos)

    def inverse(self, real_pos):
        _energy = -1.0*_EB*real_pos.uof/np.tan(np.deg2rad(_TB))
        return self.PseudoPosition(energy=_energy)

hrmE = HRMEnergy('', name='hrmE', egu='eV')

# ripped from Spec file, "site.mac"
th0 = 38.5857476

A0y = 1.21523
A0z = 7.20726
B0y = 3.08561
B0z = 6.57882
C0y = 0.36531
C0z = 3.16942
anz = 4.03784
bny = 1.87038
bnz = -0.62844
b2 = bny*bny + bnz*bnz
cny = 2.72030
cnz = 3.40940
c2 = cny*cny + cnz*cnz
c1 = np.sqrt(c2)
d2 = B0y*B0y + B0z*B0z
dn = np.sqrt(d2)
h2 = C0y*C0y + C0z*C0z
hn = np.sqrt(h2)

class AnalyzerCXtal(PseudoPositioner):
    theta  = Cpt(PseudoSingle, egu='deg')
    height = Cpt(PseudoSingle, egu='mm')

    uy = Cpt(EpicsMotor, 'XF:10IDD-OP{Analy:1-Ax:UY}Mtr')
    dy = Cpt(EpicsMotor, 'XF:10IDD-OP{Analy:1-Ax:DY}Mtr')

    @pseudo_position_argument
    def forward(self, pseudopos):
        ddy = np.zeros((2,), dtype=float)
        cth = pseudopos.theta + th0
        ccy = pseudopos.height

        cy = c1*np.sin(np.deg2rad(cth))
        cz = np.sqrt(c2 - cy*cy)
        a2 = anz*anz + c2 - b2 - 2*anz*cz
        d2 = np.sqrt(cy*cy - a2)
        a1 = (c2 + h2 -d2)/2.
        d1 = cz*np.sqrt(c2*h2 - a1*a1)
        hy = (a1*cy + d1)/c2
        #
        # first motor position (mm)
        y1 = 0.01*ccy - hy - C0y
        ddy[0] = 100.*y1
        #
        #
        # second motor position (mm)
        y2 = C0y + y1 + cy - d2 - A0y
        ddy[1] = 100.*y2

        return self.RealPosition(uy=ddy[0], dy=ddy[1])

    @real_position_argument
    def inverse(self, realpos):
        ccp = np.zeros((2,), dtype=float)
        # d2y == a2uy, d1y == a2dy
        d1y = self.uy.position
        d2y = self.dy.position

        any = 0.84992 + 0.01*(d2y - d1y)
        a2 = any*any + anz*anz
        a1 = (c2 + a2 - b2)/2.
        d1 = anz*np.sqrt(a2*c2 - a1*a1)
        cy = (a1*any + d1)/a2
        cz = np.sqrt(c2 - cy*cy)
        #
        # crystal angle (rad)
        ccp[0] = np.deg2rad(np.arcsin(cy/c1))
        # how to handle negative sqrts here??
        d2 = cz*np.sqrt(c2*h2 - a2*a2)
        a2 = (c2 + h2 - d2)/2.
        hy = (a2*cy + d2)/c2
        #
        # crystal y-position
        ccp[1] = 100.*(C0y + 0.01*d1y + hy)

        return self.PseudoPosition(theta=ccp[0]-th0, height=ccp[1])

anc_xtal = AnalyzerCXtal('', name='anc_xtal', egu=('deg', 'mm'))
<commit_msg>FIX: analyzer xtal calculations<commit_after>from ophyd import (EpicsMotor, PseudoSingle, PseudoPositioner,
                   Component as Cpt)
from ophyd.pseudopos import (pseudo_position_argument, real_position_argument)
import numpy as np
from scipy import interpolate

_ivu_gap = [ 6184.0,
       6368.0,
       6550.0,
       6730.0,
       6909.0,
       7086.0,
       7262.0,
       7438.0,
       7613.0,
       7788.0,
       7962.0,
       8138.0,
       8311.0,
       8486.0,
       8662.0,
       8840.0,
       9019.0,
       9200.0,
       9382.0,
       9567.0,
       9753.0,
       9943.0,
       10136.0,
       10332.0,
       10531.0,
       10740.0,
       10960.0,
       11385.0]

_Bragg = [ 16.401781,
       15.821071,
       15.280587,
       14.776273,
       14.304428,
       13.862182,
       13.446656,
       13.055574,
       12.686789,
       12.338436,
       12.008851,
       11.696607,
       11.400018,
       11.118472,
       10.850483,
       10.595196,
       10.351707,
       10.119222,
       9.896987,
       9.684357,
       9.480717,
       9.28554,
       9.098203,
       8.918375,
       8.7455,
       8.579178,
       8.41915,
       8.11645 ]

gcalc = interpolate.interp1d(_Bragg, _ivu_gap)

_hc = 12398.4193
_si_111 = 3.1363

class BLEnergy(PseudoPositioner):
    # limits and constants from Spec file, "site.mac"
    energy = Cpt(PseudoSingle, limits=(7.835, 17.7))

    theta = Cpt(EpicsMotor, 'XF:10IDA-OP{Mono:DCM-Ax:P}Mtr')
    z2 = Cpt(EpicsMotor, 'XF:10IDA-OP{Mono:DCM-Ax:Z2}Mtr')
    ugap = Cpt(Undulator, 'SR:C10-ID:G1{IVU22:1')

    def forward(self, pseudo_pos):
        _th = np.rad2deg(np.arcsin(_hc/(2.*_si_111*pseudo_pos)))
        _z2 = 15./np.cos(np.deg2rad(_th)) - 15.
        _ugap = gcalc(_th)

        return self.RealPosition(theta=_th, z2=_z2, ugap=_ugap)

    def inverse(self, real_pos):
        en = _hc/(2.*_si_111*np.sin(np.deg2rad(real_pos.theta)))
        en = float(en)
        return self.PseudoPosition(energy=en)

blE = BLEnergy('', name='blE', egu='eV')

class DCMEnergy(PseudoPositioner):
    # limits and constants from Spec file, "site.mac"
    energy = Cpt(PseudoSingle, limits=(7.835, 17.7))

    theta = Cpt(EpicsMotor, 'XF:10IDA-OP{Mono:DCM-Ax:P}Mtr')
    z2 = Cpt(EpicsMotor, 'XF:10IDA-OP{Mono:DCM-Ax:Z2}Mtr')

    def forward(self, pseudo_pos):
        _th = np.rad2deg(np.arcsin(_hc/(2.*_si_111*pseudo_pos)))
        _z2 = 15./np.cos(np.deg2rad(_th)) - 15.

        return self.RealPosition(theta=_th, z2=_z2)

    def inverse(self, real_pos):
        en = _hc/(2.*_si_111*np.sin(np.deg2rad(real_pos.theta)))
        en = float(en)
        return self.PseudoPosition(energy=en)

dcmE = DCMEnergy('', name='dcmE', egu='eV')

# constants for HRM2 energy conversion
_TB = 69.2943
_EB = 9.1317
class HRMEnergy(PseudoPositioner):
    energy = Cpt(PseudoSingle)

    uof = Cpt(EpicsMotor, 'XF:10IDB-OP{Mono:HRM2-Ax:UTO}Mtr')
    dof = Cpt(EpicsMotor, 'XF:10IDB-OP{Mono:HRM2-Ax:DTO}Mtr')

    def forward(self, pseudo_pos):
        _pos = -1.0*pseudo_pos.energy*np.tan(np.deg2rad(_TB))/_EB
        return self.RealPosition(uof=_pos, dof=_pos)

    def inverse(self, real_pos):
        _energy = -1.0*_EB*real_pos.uof/np.tan(np.deg2rad(_TB))
        return self.PseudoPosition(energy=_energy)

hrmE = HRMEnergy('', name='hrmE', egu='eV')

# ripped from Spec file, "site.mac"
th0 = 38.5857476

A0y = 1.21523
A0z = 7.20726
B0y = 3.08561
B0z = 6.57882
C0y = 0.36531
C0z = 3.16942
anz = 4.03784
bny = 1.87038
bnz = -0.62844
b2 = bny*bny + bnz*bnz
cny = 2.72030
cnz = 3.40940
c2 = cny*cny + cnz*cnz
c1 = np.sqrt(c2)
d2 = B0y*B0y + B0z*B0z
dn = np.sqrt(d2)
h2 = C0y*C0y + C0z*C0z
hn = np.sqrt(h2)

class AnalyzerCXtal(PseudoPositioner):
    theta  = Cpt(PseudoSingle, egu='deg')
    height = Cpt(PseudoSingle, egu='mm')

    uy = Cpt(EpicsMotor, 'XF:10IDD-OP{Analy:1-Ax:UY}Mtr')
    dy = Cpt(EpicsMotor, 'XF:10IDD-OP{Analy:1-Ax:DY}Mtr')

    @pseudo_position_argument
    def forward(self, pseudopos):
        ddy = np.zeros((2,), dtype=float)
        cth = pseudopos.theta + th0
        ccy = pseudopos.height

        cy = c1*np.sin(np.deg2rad(cth))
        cz = np.sqrt(c2 - cy*cy)
        a2 = anz*anz + c2 - b2 - 2*anz*cz
        d2 = np.sqrt(cy*cy - a2)
        a1 = (c2 + h2 -d2)/2.
        d1 = cz*np.sqrt(c2*h2 - a1*a1)
        hy = (a1*cy + d1)/c2
        #
        # first motor position (mm)
        y1 = 0.01*ccy - hy - C0y
        ddy[0] = 100.*y1
        #
        #
        # second motor position (mm)
        y2 = C0y + y1 + cy - d2 - A0y
        ddy[1] = 100.*y2

        return self.RealPosition(uy=ddy[0], dy=ddy[1])

    @real_position_argument
    def inverse(self, realpos):
        ccp = np.zeros((2,), dtype=float)
        # d2y == a2uy, d1y == a2dy
        d1y = self.uy.position
        d2y = self.dy.position

        _any = 0.84992 + 0.01*(d2y - d1y)
        a2 = _any*_any + anz*anz
        a1 = (c2 + a2 - b2)/2.
        d1 = anz*np.sqrt(a2*c2 - a1*a1)
        cy = (a1*_any + d1)/a2
        cz = np.sqrt(c2 - cy*cy)
        #
        # crystal angle (rad)
        ccp[0] = np.rad2deg(np.arcsin(cy/c1))
        a2 = (c2 + h2 - d2)/2.
        _d2 = cz*np.sqrt(c2*h2 - a2*a2)
        hy = (a2*cy + _d2)/c2
        #
        # crystal y-position
        ccp[1] = 100.*(C0y + 0.01*d1y + hy)

        return self.PseudoPosition(theta=ccp[0]-th0, height=ccp[1])

anc_xtal = AnalyzerCXtal('', name='anc_xtal', egu=('deg', 'mm'))
<|endoftext|>


<commit_before>import os
from distutils import file_util
from ConfigParser import SafeConfigParser

class PsiturkConfig(SafeConfigParser):
    def __init__(self, localConfig="config.txt", globalConfigName=".psiturkconfig", **kwargs):

        # If working in OpenShift, move global config file in data directory (has access rights)
        if 'OPENSHIFT_SECRET_TOKEN' in os.environ:
            globalConfig = os.environ['OPENSHIFT_DATA_DIR'] + globalConfigName
        elif 'PSITURK_GLOBAL_CONFIG_LOCATION' in os.environ:
            globalConfig = os.environ['PSITURK_GLOBAL_CONFIG_LOCATION'] + globalConfigName
        else: # if nothing is set default to user's home directory
            globalConfig = "~/" + globalConfigName

        self.parent = SafeConfigParser
        self.parent.__init__(self, **kwargs)
        self.localFile = localConfig
        self.globalFile = os.path.expanduser(globalConfig)

    def load_config(self):
        defaults_folder = os.path.join(os.path.dirname(__file__), "default_configs")
        local_defaults_file = os.path.join(defaults_folder, "local_config_defaults.txt")
        global_defaults_file = os.path.join(defaults_folder, "global_config_defaults.txt")
        if not os.path.exists(self.localFile):
            print "ERROR - no config.txt file in the current directory. \n\nAre you sure this directory is a valid psiTurk experiment?  If you are starting a new project run 'psiturk-setup-example' first."
            exit()
        if not os.path.exists(self.globalFile):
            if 'OPENSHIFT_SECRET_TOKEN' in os.environ:
                print "No '.psiturkconfig' file found in your " + os.environ['OPENSHIFT_DATA_DIR'] + " directory.\nCreating default " + self.globalFile + " file."
            elif 'PSITURK_GLOBAL_CONFIG_LOCATION' in os.environ:
                print "No '.psiturkconfig' file found in your " + os.environ['PSITURK_GLOBAL_CONFIG_LOCATION'] + " directory.\nCreating default " + self.globalFile + " file."
            else:
                print "No '.psiturkconfig' file found in your home directory.\nCreating default ~/.psiturkconfig file."
            file_util.copy_file(global_defaults_file, self.globalFile)
        # read default global and local, then user's global and local. This way
        # any field not in the user's files will be set to the default value.
        self.read([global_defaults_file, local_defaults_file, self.globalFile, self.localFile])

        # heroku reads from env vars instead of .psiturkconfig
        for section in ['psiTurk Access', 'AWS Access']:
            for (name, value) in self.items(section):
                if name in os.environ:
                    self.set(section, name, os.environ[name])
                    #print 'changed ' + name + ' in ' + section + ' from ' + value + ' to ' + os.environ[name]

        # heroku dynamically assigns your app a port, so you can't set the port to a fixed number
        if 'ON_HEROKU' in os.environ:
            self.set('Server Parameters', 'port', os.environ['PORT'])

<commit_msg>pull heroku database url from env<commit_after>import os
from distutils import file_util
from ConfigParser import SafeConfigParser

class PsiturkConfig(SafeConfigParser):
    def __init__(self, localConfig="config.txt", globalConfigName=".psiturkconfig", **kwargs):

        # If working in OpenShift, move global config file in data directory (has access rights)
        if 'OPENSHIFT_SECRET_TOKEN' in os.environ:
            globalConfig = os.environ['OPENSHIFT_DATA_DIR'] + globalConfigName
        elif 'PSITURK_GLOBAL_CONFIG_LOCATION' in os.environ:
            globalConfig = os.environ['PSITURK_GLOBAL_CONFIG_LOCATION'] + globalConfigName
        else: # if nothing is set default to user's home directory
            globalConfig = "~/" + globalConfigName

        self.parent = SafeConfigParser
        self.parent.__init__(self, **kwargs)
        self.localFile = localConfig
        self.globalFile = os.path.expanduser(globalConfig)

    def load_config(self):
        defaults_folder = os.path.join(os.path.dirname(__file__), "default_configs")
        local_defaults_file = os.path.join(defaults_folder, "local_config_defaults.txt")
        global_defaults_file = os.path.join(defaults_folder, "global_config_defaults.txt")
        if not os.path.exists(self.localFile):
            print "ERROR - no config.txt file in the current directory. \n\nAre you sure this directory is a valid psiTurk experiment?  If you are starting a new project run 'psiturk-setup-example' first."
            exit()
        if not os.path.exists(self.globalFile):
            if 'OPENSHIFT_SECRET_TOKEN' in os.environ:
                print "No '.psiturkconfig' file found in your " + os.environ['OPENSHIFT_DATA_DIR'] + " directory.\nCreating default " + self.globalFile + " file."
            elif 'PSITURK_GLOBAL_CONFIG_LOCATION' in os.environ:
                print "No '.psiturkconfig' file found in your " + os.environ['PSITURK_GLOBAL_CONFIG_LOCATION'] + " directory.\nCreating default " + self.globalFile + " file."
            else:
                print "No '.psiturkconfig' file found in your home directory.\nCreating default ~/.psiturkconfig file."
            file_util.copy_file(global_defaults_file, self.globalFile)
        # read default global and local, then user's global and local. This way
        # any field not in the user's files will be set to the default value.
        self.read([global_defaults_file, local_defaults_file, self.globalFile, self.localFile])

        # heroku reads from env vars instead of .psiturkconfig
        for section in ['psiTurk Access', 'AWS Access']:
            for (name, value) in self.items(section):
                if name in os.environ:
                    self.set(section, name, os.environ[name])
                    #print 'changed ' + name + ' in ' + section + ' from ' + value + ' to ' + os.environ[name]

        # heroku dynamically assigns your app a port, so you can't set the port to a fixed number
        # database url is also dynamic
        if 'ON_HEROKU' in os.environ:
            self.set('Server Parameters', 'port', os.environ['PORT'])
            self.set('Database Parameters', 'database_url', os.environ['DATABASE_URL'])
<|endoftext|>


<commit_before>"""Router of the PPP core."""

import json
import logging
import requests
import operator
import itertools
import functools
from ppp_datamodel import AbstractNode
from ppp_datamodel.communication import Request, Response, TraceItem
from .config import CoreConfig
from .exceptions import ClientError, BadGateway

s = lambda x:x if isinstance(x, str) else x.decode()

DEFAULT_ACCURACY = 0
DEFAULT_RELEVANCE = 0

def freeze(obj):
    if isinstance(obj, dict):
        return frozenset((freeze(x), freeze(y)) for (x,y) in obj.items())
    elif isinstance(obj, list):
        return tuple(map(freeze, obj))
    elif isinstance(obj, set):
        return frozenset(map(freeze, obj))
    elif isinstance(obj, TraceItem):
        return freeze(obj.as_dict())
    elif isinstance(obj, (str, tuple, frozenset, int, float)):
        return obj
    else:
        raise Exception(obj)

def answer_id(answer):
    return (answer.language, answer.tree,
            frozenset(answer.measures.items()),
            freeze(answer.trace))
def remove_duplicates(reference, new):
    result = []
    for x in new:
        id_ = answer_id(x)
        if id_ in reference:
            continue
        reference.add(id_)
        result.append(x)
    return result

class Router:
    def __init__(self, request):
        self.id = request.id
        self.language = request.language
        assert isinstance(request.tree, AbstractNode)
        self.tree = request.tree
        self.measures = request.measures
        self.trace = request.trace
        self.config = CoreConfig()

    def answer(self):
        answer_ids = set()
        answers = []
        new_answers = [Response(self.language, self.tree,
                                self.measures, self.trace)]
        for i in range(0, self.config.nb_passes):
            # Perform the pass
            requests = list(map(self.request_from_answer, new_answers))
            new_answers = []
            for request in requests:
                new_answers.extend(self.one_pass(request))
            # Remove duplicates, and update the answer list
            new_answers = list(remove_duplicates(answer_ids, new_answers))
            answers.extend(new_answers)
        # TODO: should sort according to accuracy too
        return sorted(answers,
                      key=lambda x:x.measures.get('relevance', DEFAULT_RELEVANCE),
                      reverse=True)

    def request_from_answer(self, answer):
        return Request(self.id, answer.language, answer.tree,
                       answer.measures, answer.trace)

    def one_pass(self, request):
        # First make all requests so modules can prepare their answer
        # while we send requests to other modules
        streams = self._get_streams(request)
        answers = map(self._stream_reader, streams)
        answers = map(self._process_answers, answers)
        answers = itertools.chain(*list(answers)) # Flatten answers lists
        answers = filter(bool, answers) # Eliminate None values
        return answers

    def _get_streams(self, request):
        headers = {'Content-type': 'application/json',
                   'Accept': 'application/json'}
        payload = request.as_json()
        getter = functools.partial(requests.post, stream=True,
                                   headers=headers, data=payload)
        streams = []
        for module in self.config.modules:
            try:
                streams.append((module, getter(module.url)))
            except requests.exceptions.ConnectionError as exc: # pragma: no cover
                logging.warning('Module %s could not be queried: %s' %
                                (module, exc.args[0]))
                pass
        return streams

    def _stream_reader(self, stream):
        (module, stream) = stream
        if stream.status_code != 200:
            logging.warning('Module %s returned %d: %s' %
                            (module, stream.status_code, stream.content))
            return None
        else:
            try:
                return (module, json.loads(s(stream.content)))
            except ValueError:
                logging.warning('Module %s returned %d: %s' %
                                (module, stream.status_code, stream.content))
                return None

    def _process_answers(self, t):
        if t:
            (module, answers) = t
            answers = map(Response.from_dict, answers)
            return list(map(functools.partial(self._process_answer, module), answers))
        else:
            return []

    def _process_answer(self, module, answer):
        missing = {'accuracy', 'relevance'} - set(answer.measures)
        if missing:
            logging.warning('Missing mandatory measures from module %s: %r' %
                             (module, missing))
        accuracy = answer.measures.get('accuracy', DEFAULT_ACCURACY)
        relevance = answer.measures.get('relevance', DEFAULT_RELEVANCE)
        if accuracy < 0 or accuracy > 1:
            logging.warning('Module %s answered with invalid accuracy: %r' %
                    (module, accuracy))
            return None
        answer.measures['accuracy'] = accuracy
        answer.measures['relevance'] = relevance * module.coefficient
        return answer
<commit_msg>Remove trace from the answer id.<commit_after>"""Router of the PPP core."""

import json
import logging
import requests
import operator
import itertools
import functools
from ppp_datamodel import AbstractNode
from ppp_datamodel.communication import Request, Response, TraceItem
from .config import CoreConfig
from .exceptions import ClientError, BadGateway

s = lambda x:x if isinstance(x, str) else x.decode()

DEFAULT_ACCURACY = 0
DEFAULT_RELEVANCE = 0

def freeze(obj):
    if isinstance(obj, dict):
        return frozenset((freeze(x), freeze(y)) for (x,y) in obj.items())
    elif isinstance(obj, list):
        return tuple(map(freeze, obj))
    elif isinstance(obj, set):
        return frozenset(map(freeze, obj))
    elif isinstance(obj, TraceItem):
        return freeze(obj.as_dict())
    elif isinstance(obj, (str, tuple, frozenset, int, float)):
        return obj
    else:
        raise Exception(obj)

def answer_id(answer):
    return (answer.language, answer.tree,
            frozenset(answer.measures.items()))
def remove_duplicates(reference, new):
    result = []
    for x in new:
        id_ = answer_id(x)
        if id_ in reference:
            continue
        reference.add(id_)
        result.append(x)
    return result

class Router:
    def __init__(self, request):
        self.id = request.id
        self.language = request.language
        assert isinstance(request.tree, AbstractNode)
        self.tree = request.tree
        self.measures = request.measures
        self.trace = request.trace
        self.config = CoreConfig()

    def answer(self):
        answer_ids = set()
        answers = []
        new_answers = [Response(self.language, self.tree,
                                self.measures, self.trace)]
        for i in range(0, self.config.nb_passes):
            # Perform the pass
            requests = list(map(self.request_from_answer, new_answers))
            new_answers = []
            for request in requests:
                new_answers.extend(self.one_pass(request))
            # Remove duplicates, and update the answer list
            new_answers = list(remove_duplicates(answer_ids, new_answers))
            answers.extend(new_answers)
        # TODO: should sort according to accuracy too
        return sorted(answers,
                      key=lambda x:x.measures.get('relevance', DEFAULT_RELEVANCE),
                      reverse=True)

    def request_from_answer(self, answer):
        return Request(self.id, answer.language, answer.tree,
                       answer.measures, answer.trace)

    def one_pass(self, request):
        # First make all requests so modules can prepare their answer
        # while we send requests to other modules
        streams = self._get_streams(request)
        answers = map(self._stream_reader, streams)
        answers = map(self._process_answers, answers)
        answers = itertools.chain(*list(answers)) # Flatten answers lists
        answers = filter(bool, answers) # Eliminate None values
        return answers

    def _get_streams(self, request):
        headers = {'Content-type': 'application/json',
                   'Accept': 'application/json'}
        payload = request.as_json()
        getter = functools.partial(requests.post, stream=True,
                                   headers=headers, data=payload)
        streams = []
        for module in self.config.modules:
            try:
                streams.append((module, getter(module.url)))
            except requests.exceptions.ConnectionError as exc: # pragma: no cover
                logging.warning('Module %s could not be queried: %s' %
                                (module, exc.args[0]))
                pass
        return streams

    def _stream_reader(self, stream):
        (module, stream) = stream
        if stream.status_code != 200:
            logging.warning('Module %s returned %d: %s' %
                            (module, stream.status_code, stream.content))
            return None
        else:
            try:
                return (module, json.loads(s(stream.content)))
            except ValueError:
                logging.warning('Module %s returned %d: %s' %
                                (module, stream.status_code, stream.content))
                return None

    def _process_answers(self, t):
        if t:
            (module, answers) = t
            answers = map(Response.from_dict, answers)
            return list(map(functools.partial(self._process_answer, module), answers))
        else:
            return []

    def _process_answer(self, module, answer):
        missing = {'accuracy', 'relevance'} - set(answer.measures)
        if missing:
            logging.warning('Missing mandatory measures from module %s: %r' %
                             (module, missing))
        accuracy = answer.measures.get('accuracy', DEFAULT_ACCURACY)
        relevance = answer.measures.get('relevance', DEFAULT_RELEVANCE)
        if accuracy < 0 or accuracy > 1:
            logging.warning('Module %s answered with invalid accuracy: %r' %
                    (module, accuracy))
            return None
        answer.measures['accuracy'] = accuracy
        answer.measures['relevance'] = relevance * module.coefficient
        return answer
<|endoftext|>