<commit_before># -*- coding: utf-8 -*-
"""This module defines Neroman.

To work with Neroman each experiment must have the following attributes
defined in its config:

* experiment_id: The unique identifier for the experiment
* run_command_prefix: The run command of the experiment
* main_code_file: The code file to be run
* destination_folder: Folder that the experiment is run in on the cluster.
* parameters: The definition of the experiment parameters
* parameters_format: The format of the experiment parameters
* logoutput: The location the experiment outputs its output
* state: A tuple of the experiment state which is set to 'defined' by this
  function and the time changed
* cluster: The cluster that the experiment is running on. Set to
  None by this function
* time_created: Sets the current time as the creation time
* time_modified: The time the experiment was last modified. Sets
  this time to the same as the time created
* path: The absolute path to the folder

"""

import os
import time
import datetime
import pathlib

import yaml

import neronet.core
import neronet.config_parser

class Neroman:

    """The part of Neronet that handles user side things.

    Attributes:
        database (str): Path to the database used, currently only .yaml
        clusters (Dict): A dictionary containing the specified clusters
        experiments (Dict): A dictionary containing the specified experiments
        preferences (Dict): A dictionary containing the preferences
    """

    def __init__(self, database='default.yaml',
                 preferences_file='preferences.yaml',
                 clusters_file='clusters.yaml'):
        """Initializes Neroman

        Reads the contents of its attributes from a database (currently just
        a .yaml file).

        Args:
            database (str): The path to the database file as a string, the
                rest of the attributes will be parsed from the database.
        """
        self.database = database
        self.clusters_file = clusters_file
        self.preferences_file = preferences_file
        self.config_parser = neronet.config_parser.ConfigParser()
        self.clusters = {}
        self.experiments = {}
        self.preferences = {}
        self._load_configurations(database, clusters_file, preferences_file)

    def _load_configurations(self, database, clusters, preferences):
        """Load the configurations from the yaml files or creates them if they
        don't exist

        Args:
            database (str): The filepath of the database file
            clusters (str): The filepath of the clusters file
            preferences (str): The filepath of the preferences file
        """
        if not os.path.exists(preferences):
            with open(preferences, 'w') as f:
                f.write("name:\nemail:\n")
        else:
            with open(preferences, 'r') as f:
                self.preferences = yaml.load(f.read())
        if not self.preferences:
            self.specify_user("","")
        if 'default_cluster' not in self.preferences:
            self.preferences['default_cluster'] = ""
        if 'email' not in self.preferences:
            raise FormatError('The user\'s email is not specified')
        if 'name' not in self.preferences:
            raise FormatError('The user\'s name is not specified')

        if not os.path.exists(clusters):
            with open(clusters, 'w') as f:
                f.write("clusters:\ngroups:\n")
        else:
            with open(clusters, 'r') as f:
                self.clusters = yaml.load(f.read())
        if not self.clusters:
            self.clusters = {'clusters': None}
        if not self.clusters['clusters']:
            self.clusters['clusters'] = {}

        # This checks the format of the clusters file
        for key in self.clusters['clusters']:
            if 'type' not in self.clusters['clusters'][key]:
                self.clusters['clusters'][key]['type'] = 'unmanaged'
            else:
                if self.clusters['clusters'][key]['type'] != 'unmanaged':
                    if self.clusters['clusters'][key]['type'] != 'slurm':
                        raise FormatError('The cluster type for the cluster ' +
                        key + ' is not valid.')
            if 'port' not in self.clusters['clusters'][key]:
                self.clusters['clusters'][key]['port'] = 22
            if 'ssh_address' not in self.clusters['clusters'][key]:
                raise FormatError('The ssh address for the cluster ' + key +
                ' is not defined.')
        if self.preferences['default_cluster']:
            if self.preferences['default_cluster'] not in self.clusters['clusters']:
                raise FormatError('The specified default cluster ' +
                self.preferences['default_cluster'] +' is not found')

        with open(self.clusters_file, 'w') as f:
            f.write(yaml.dump(self.clusters, default_flow_style=False))

        if not os.path.exists(database):
            with open(database, 'w') as f:
                f.write('')
        else:
            with open(database, 'r') as f:
                self.experiments = yaml.load(f.read())
        if not self.experiments:
            self.experiments = {}

    def save_database(self):
        """Save the contents of Neroman's attributes in the database
        """
        with open(self.database, 'w') as f:
            f.write(yaml.dump(self.experiments,
                              default_flow_style=False))

    def specify_cluster(
            self,
            cluster_name,
            ssh_address,
            cluster_type,
            portnumber=22):
        """Specify clusters so that Neroman is aware of them.

        Writes cluster name, address and type to the clusters config file

        Args:
            cluster_name (str): The name of the cluster, should be unique
            ssh_address (str): SSH address of the cluster
            cluster_type (str): Type of the cluster. Either slurm or unmanaged

        Raises:
            FormatError: if the cluster type isn't unmanaged or slurm

        """
        if cluster_type != 'slurm' and cluster_type != 'unmanaged':
            raise neronet.config_parser.FormatError( \
                        "Cluster type should be slurm or unmanaged")

        self.clusters['clusters'][cluster_name] = {'ssh_address': ssh_address,
                                                   'type': cluster_type,
                                                   'port': portnumber}
        with open(self.clusters_file, 'w') as f:
            f.write(yaml.dump(self.clusters, default_flow_style=False))

    def specify_experiments(self, folder):
        """Specify experiments so that Neroman is aware of them.

        Reads the contents of the experiment from a config file inside the
        specified folder.

        Args:
            folder (str): The path of the folder that includes
                the experiment that's being specified.

        Raises:
            FileNotFoundError: If the folder doesn't exists or the config file
                doesn't exists
            FormatError: If the config file is badly formated
        """

        experiments = self.config_parser.parse_experiments(folder)
        for experiment in experiments:
            if experiment.experiment_id in self.experiments:
                raise IOError("Experiment named %s already in the database" \
                                % experiment.experiment_id)
            else: self.experiments[experiment.experiment_id] = experiment
        self.save_database()

    def _time_now(self):
        return datetime.datetime.now().strftime('%H:%M:%S %d-%m-%Y')

    def update_state(self, experiment_id, state):
        self.experiments[experiment_id].update_state(state)

    def specify_user(self, name, email, default_cluster = ""):
        """Update user data"""
        self.preferences['name'] = name
        self.preferences['email'] = email
        self.preferences['default_cluster'] = default_cluster
        with open(self.preferences_file, 'w') as f:
            f.write(yaml.dump(self.preferences, default_flow_style=False))

    def get_experiment_results(self, experiment_id, remote_dir, local_dir):
        """Get the experiment results from neromum

        Args:
            experiment_id (str): the experiment ID.
            remote_dir (str): the file path to results folder on the remote
                machine.
            local_dir (str): the file path to results folder on the local
                machine.
        """
        experiment = self.experiments[experiment_id]
        cluster_ID = experiment.fields['cluster']
        cluster_port = self.clusters['clusters'][cluster_ID]['port']
        cluster_address = self.clusters['clusters'][cluster_ID]['ssh_address']
        neronet.core.osrun(
            'rsync -az -e "ssh -p%s" "%s:%s" "%s"'
            % (cluster_port, cluster_address,
                remote_dir, local_dir))

    def status(self, arg):
        """Display Neroman data on into stdstream"""
        if arg != 'all':
            if arg in self.experiments:
                experiment = self.experiments[arg]
                parameters = experiment.fields['parameters']
                time_modified = experiment.fields['time_modified']
                state, state_change_time = experiment.fields['state'][-1]
                parameters_string = ', '.join(
                    ["%s: %s" % (k, v) for k, v in parameters.items()])
                print(
                    'Experiment: %s\nParameters: %s' %
                    (arg, parameters_string))
                if state == 'defined':
                    print('State: %s - %s' % (state, state_change_time))
                else:
                    cluster = experiment['cluster']
                    print(
                        'State: %s - %s - %s' %
                        (state, cluster, state_change_time))
                print('Last modified: %s' % time_modified)
                return
            else:
                raise IOError('No experiment named %s' % arg)
        print("================Neroman=================")
        print("\n================User=================")
        print("Name: " + self.preferences['name'])
        print("Email: " + self.preferences['email'])
        if self.preferences['default_cluster']:
            print("Default Cluster: " + self.preferences['default_cluster'])
        print("\n================Clusters================")
        if not self.clusters['clusters']:
            print("No clusters defined")
        else:
            for cluster in self.clusters['clusters']:
                address = self.clusters['clusters'][cluster]['ssh_address']
                type = self.clusters['clusters'][cluster]['type']
                print("{} {} {}".format(cluster, address, type))
        print("\n================Experiments=============")
        if not len(self.experiments):
            print("No experiments defined")
        else:
            for experiment in sorted(self.experiments):
                print(self.experiments[experiment])

    def send_files(
        self,
        experiment_folder,
        remote_dir,
        cluster_address,
        cluster_port,
        neronet_root=pathlib.Path.cwd(),
    ):
        """Send experiment files to the cluster

        Args:
            experiment_folder (str): the file path to experiment folder on the local machine.
            remote_dir (str): the file path to experiment folder on the remote cluster.
            neronet_root (str): the file path to neronet folder.
            cluster_address (str): the address of the cluster.
            cluster_port (int): ssh port number of the cluster.
        """
        tmp_dir = pathlib.Path('/tmp/neronet-tmp')
        # rsync the neronet files to tmp
        neronet.core.osrun(
            'rsync -az "%s" "%s"' %
            (neronet_root /
             'neronet',
             tmp_dir))
        # rsync bin files to tmp
        neronet.core.osrun(
            'rsync -az "%s" "%s"' %
            (neronet_root / 'bin', tmp_dir))
        # rsync the experiment files to tmp
        neronet.core.osrun(
            'rsync -az "%s/" "%s"' %
            (experiment_folder, tmp_dir))
        neronet.core.osrun(
            'rsync -az -e "ssh -p%s" "%s/" "%s:%s"' %
            (cluster_port,
             tmp_dir,
             cluster_address,
             remote_dir))

    def submit(self, exp_id, cluster_ID = ""):
        """Main loop of neroman.

        Start the experiment in the cluster using ssh.

        Args:
            experiment_folder (str) : the file path to experiment folder in local machine.
            experiment_destination (str) : the file path to experiment folder on the remote cluster.
            experiment (str) : the name of the experiment.
            cluster_address (str) : the address of the cluster.
            cluster_port (int) : ssh port number of the cluster.
        """
        if not cluster_ID:
            cluster_ID = self.preferences['default_cluster']

        if cluster_ID not in self.clusters['clusters']:
            raise IOError('The given cluster ID or default cluster is not valid')

        remote_dir = pathlib.Path('/tmp/neronet-%d' % (time.time()))
        experiment_destination = self.experiments[exp_id].fields[
            'path'] + "/" + self.experiments[exp_id].fields['logoutput']
        experiment_folder = self.experiments[exp_id].fields["path"]
        #experiment = self.experiments[exp_id]["path"]+"/"+self.experiments[exp_id]["main_code_file"]
        experiment_parameters = self.experiments[exp_id].get_callstring()
        cluster_port = self.clusters['clusters'][cluster_ID]["port"]
        cluster_address = self.clusters["clusters"][cluster_ID]["ssh_address"]
        self.send_files(
            experiment_folder,
            remote_dir,
            cluster_address,
            cluster_port)
        # Magic do NOT touch:
        neronet.core.osrun(
            'ssh -p%s %s "cd %s; PATH="%s/bin:/usr/local/bin:/usr/bin:/bin" PYTHONPATH="%s" neromum %s"' %
            (cluster_port,
             cluster_address,
             remote_dir,
             remote_dir,
             remote_dir,
             experiment_parameters))
        self.experiments[exp_id].fields['cluster'] = cluster_ID
        self.update_state(exp_id, 'submitted')
        self.save_database()
        time.sleep(2)  # will be unnecessary as soon as daemon works
        # returns the results, should be called from cli
        self.get_experiment_results(exp_id, remote_dir, experiment_destination)
<commit_msg>Makes neroman use experiment methods<commit_after># -*- coding: utf-8 -*-
"""This module defines Neroman.

To work with Neroman each experiment must have the following attributes
defined in its config:

* experiment_id: The unique identifier for the experiment
* run_command_prefix: The run command of the experiment
* main_code_file: The code file to be run
* destination_folder: Folder that the experiment is run in on the cluster.
* parameters: The definition of the experiment parameters
* parameters_format: The format of the experiment parameters
* logoutput: The location the experiment outputs its output
* state: A tuple of the experiment state which is set to 'defined' by this
  function and the time changed
* cluster: The cluster that the experiment is running on. Set to
  None by this function
* time_created: Sets the current time as the creation time
* time_modified: The time the experiment was last modified. Sets
  this time to the same as the time created
* path: The absolute path to the folder

"""

import os
import time
import datetime
import pathlib

import yaml

import neronet.core
import neronet.config_parser

class Neroman:

    """The part of Neronet that handles user side things.

    Attributes:
        database (str): Path to the database used, currently only .yaml
        clusters (Dict): A dictionary containing the specified clusters
        experiments (Dict): A dictionary containing the specified experiments
        preferences (Dict): A dictionary containing the preferences
    """

    def __init__(self, database='default.yaml',
                 preferences_file='preferences.yaml',
                 clusters_file='clusters.yaml'):
        """Initializes Neroman

        Reads the contents of its attributes from a database (currently just
        a .yaml file).

        Args:
            database (str): The path to the database file as a string, the
                rest of the attributes will be parsed from the database.
        """
        self.database = database
        self.clusters_file = clusters_file
        self.preferences_file = preferences_file
        self.config_parser = neronet.config_parser.ConfigParser()
        self.clusters = {}
        self.experiments = {}
        self.preferences = {}
        self._load_configurations(database, clusters_file, preferences_file)

    def get_experiment(self, experiment_id):
        return self.experiments[experiment_id]

    def _load_configurations(self, database, clusters, preferences):
        """Load the configurations from the yaml files or creates them if they
        don't exist

        Args:
            database (str): The filepath of the database file
            clusters (str): The filepath of the clusters file
            preferences (str): The filepath of the preferences file
        """
        if not os.path.exists(preferences):
            with open(preferences, 'w') as f:
                f.write("name:\nemail:\n")
        else:
            with open(preferences, 'r') as f:
                self.preferences = yaml.load(f.read())
        if not self.preferences:
            self.specify_user("","")
        if 'default_cluster' not in self.preferences:
            self.preferences['default_cluster'] = ""
        if 'email' not in self.preferences:
            raise FormatError('The user\'s email is not specified')
        if 'name' not in self.preferences:
            raise FormatError('The user\'s name is not specified')

        if not os.path.exists(clusters):
            with open(clusters, 'w') as f:
                f.write("clusters:\ngroups:\n")
        else:
            with open(clusters, 'r') as f:
                self.clusters = yaml.load(f.read())
        if not self.clusters:
            self.clusters = {'clusters': None}
        if not self.clusters['clusters']:
            self.clusters['clusters'] = {}

        # This checks the format of the clusters file
        for key in self.clusters['clusters']:
            if 'type' not in self.clusters['clusters'][key]:
                self.clusters['clusters'][key]['type'] = 'unmanaged'
            else:
                if self.clusters['clusters'][key]['type'] != 'unmanaged':
                    if self.clusters['clusters'][key]['type'] != 'slurm':
                        raise FormatError('The cluster type for the cluster ' +
                        key + ' is not valid.')
            if 'port' not in self.clusters['clusters'][key]:
                self.clusters['clusters'][key]['port'] = 22
            if 'ssh_address' not in self.clusters['clusters'][key]:
                raise FormatError('The ssh address for the cluster ' + key +
                ' is not defined.')
        if self.preferences['default_cluster']:
            if self.preferences['default_cluster'] not in self.clusters['clusters']:
                raise FormatError('The specified default cluster ' +
                self.preferences['default_cluster'] +' is not found')

        with open(self.clusters_file, 'w') as f:
            f.write(yaml.dump(self.clusters, default_flow_style=False))

        if not os.path.exists(database):
            with open(database, 'w') as f:
                f.write('')
        else:
            with open(database, 'r') as f:
                self.experiments = yaml.load(f.read())
        if not self.experiments:
            self.experiments = {}

    def save_database(self):
        """Save the contents of Neroman's attributes in the database
        """
        with open(self.database, 'w') as f:
            f.write(yaml.dump(self.experiments,
                              default_flow_style=False))

    def specify_cluster(
            self,
            cluster_name,
            ssh_address,
            cluster_type,
            portnumber=22):
        """Specify clusters so that Neroman is aware of them.

        Writes cluster name, address and type to the clusters config file

        Args:
            cluster_name (str): The name of the cluster, should be unique
            ssh_address (str): SSH address of the cluster
            cluster_type (str): Type of the cluster. Either slurm or unmanaged

        Raises:
            FormatError: if the cluster type isn't unmanaged or slurm

        """
        if cluster_type != 'slurm' and cluster_type != 'unmanaged':
            raise neronet.config_parser.FormatError( \
                        "Cluster type should be slurm or unmanaged")

        self.clusters['clusters'][cluster_name] = {'ssh_address': ssh_address,
                                                   'type': cluster_type,
                                                   'port': portnumber}
        with open(self.clusters_file, 'w') as f:
            f.write(yaml.dump(self.clusters, default_flow_style=False))

    def specify_experiments(self, folder):
        """Specify experiments so that Neroman is aware of them.

        Reads the contents of the experiment from a config file inside the
        specified folder.

        Args:
            folder (str): The path of the folder that includes
                the experiment that's being specified.

        Raises:
            FileNotFoundError: If the folder doesn't exists or the config file
                doesn't exists
            FormatError: If the config file is badly formated
        """

        experiments = self.config_parser.parse_experiments(folder)
        for experiment in experiments:
            if experiment.id in self.experiments:
                raise IOError("Experiment named %s already in the database" \
                                % experiment.experiment_id)
            else: self.experiments[experiment.id] = experiment
        self.save_database()

    def _time_now(self):
        return datetime.datetime.now().strftime('%H:%M:%S %d-%m-%Y')

    def update_state(self, experiment_id, state):
        self.experiments[experiment_id].update_state(state)

    def specify_user(self, name, email, default_cluster = ""):
        """Update user data"""
        self.preferences['name'] = name
        self.preferences['email'] = email
        self.preferences['default_cluster'] = default_cluster
        with open(self.preferences_file, 'w') as f:
            f.write(yaml.dump(self.preferences, default_flow_style=False))

    def get_experiment_results(self, experiment_id, remote_dir, local_dir):
        """Get the experiment results from neromum

        Args:
            experiment_id (str): the experiment ID.
            remote_dir (str): the file path to results folder on the remote
                machine.
            local_dir (str): the file path to results folder on the local
                machine.
        """
        experiment = self.experiments[experiment_id]
        cluster_ID = experiment.cluster
        cluster_port = self.clusters['clusters'][cluster_ID]['port']
        cluster_address = self.clusters['clusters'][cluster_ID]['ssh_address']
        neronet.core.osrun(
            'rsync -az -e "ssh -p%s" "%s:%s" "%s"'
            % (cluster_port, cluster_address,
                remote_dir, local_dir))

    def status(self, arg):
        """Display Neroman data on into stdstream"""
        if arg != 'all':
            if arg in self.experiments:
                experiment = self.experiments[arg]
                parameters = experiment.parameters
                time_modified = experiment.time_modified
                state, state_change_time = experiment.state
                parameters_string = ', '.join(
                    ["%s: %s" % (k, v) for k, v in parameters.items()])
                print(
                    'Experiment: %s\nParameters: %s' %
                    (arg, parameters_string))
                if state == 'defined':
                    print('State: %s - %s' % (state, state_change_time))
                else:
                    cluster = experiment.cluster
                    print(
                        'State: %s - %s - %s' %
                        (state, cluster, state_change_time))
                print('Last modified: %s' % time_modified)
                return
            else:
                raise IOError('No experiment named %s' % arg)
        print("================Neroman=================")
        print("\n================User=================")
        print("Name: " + self.preferences['name'])
        print("Email: " + self.preferences['email'])
        if self.preferences['default_cluster']:
            print("Default Cluster: " + self.preferences['default_cluster'])
        print("\n================Clusters================")
        if not self.clusters['clusters']:
            print("No clusters defined")
        else:
            for cluster in self.clusters['clusters']:
                address = self.clusters['clusters'][cluster]['ssh_address']
                type = self.clusters['clusters'][cluster]['type']
                print("{} {} {}".format(cluster, address, type))
        print("\n================Experiments=============")
        if not len(self.experiments):
            print("No experiments defined")
        else:
            for experiment in sorted(self.experiments):
                print(self.experiments[experiment])

    def send_files(
        self,
        experiment_folder,
        remote_dir,
        cluster_address,
        cluster_port,
        neronet_root=pathlib.Path.cwd(),
    ):
        """Send experiment files to the cluster

        Args:
            experiment_folder (str): the file path to experiment folder on the local machine.
            remote_dir (str): the file path to experiment folder on the remote cluster.
            neronet_root (str): the file path to neronet folder.
            cluster_address (str): the address of the cluster.
            cluster_port (int): ssh port number of the cluster.
        """
        tmp_dir = pathlib.Path('/tmp/neronet-tmp')
        # rsync the neronet files to tmp
        neronet.core.osrun(
            'rsync -az "%s" "%s"' %
            (neronet_root /
             'neronet',
             tmp_dir))
        # rsync bin files to tmp
        neronet.core.osrun(
            'rsync -az "%s" "%s"' %
            (neronet_root / 'bin', tmp_dir))
        # rsync the experiment files to tmp
        neronet.core.osrun(
            'rsync -az "%s/" "%s"' %
            (experiment_folder, tmp_dir))
        neronet.core.osrun(
            'rsync -az -e "ssh -p%s" "%s/" "%s:%s"' %
            (cluster_port,
             tmp_dir,
             cluster_address,
             remote_dir))

    def submit(self, exp_id, cluster_ID = ""):
        """Main loop of neroman.

        Start the experiment in the cluster using ssh.

        Args:
            experiment_folder (str) : the file path to experiment folder in local machine.
            experiment_destination (str) : the file path to experiment folder on the remote cluster.
            experiment (str) : the name of the experiment.
            cluster_address (str) : the address of the cluster.
            cluster_port (int) : ssh port number of the cluster.
        """
        if not cluster_ID:
            cluster_ID = self.preferences['default_cluster']

        if cluster_ID not in self.clusters['clusters']:
            raise IOError('The given cluster ID or default cluster is not valid')

        remote_dir = pathlib.Path('/tmp/neronet-%d' % (time.time()))
        experiment_destination = self.experiments[exp_id].path + "/" + \
                                self.experiments[exp_id].logoutput
        experiment_folder = self.experiments[exp_id].path
        #experiment = self.experiments[exp_id]["path"]+"/"+self.experiments[exp_id]["main_code_file"]
        experiment_parameters = self.experiments[exp_id].callstring
        cluster_port = self.clusters['clusters'][cluster_ID]["port"]
        cluster_address = self.clusters["clusters"][cluster_ID]["ssh_address"]
        self.send_files(
            experiment_folder,
            remote_dir,
            cluster_address,
            cluster_port)
        # Magic do NOT touch:
        neronet.core.osrun(
            'ssh -p%s %s "cd %s; PATH="%s/bin:/usr/local/bin:/usr/bin:/bin" PYTHONPATH="%s" neromum %s"' %
            (cluster_port,
             cluster_address,
             remote_dir,
             remote_dir,
             remote_dir,
             experiment_parameters))
        self.experiments[exp_id].cluster = cluster_ID
        self.experiments[exp_id].update_state('submitted')
        self.save_database()
        time.sleep(2)  # will be unnecessary as soon as daemon works
        # returns the results, should be called from cli
        self.get_experiment_results(exp_id, remote_dir, experiment_destination)
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
from __future__ import print_function

import os
import urllib
import datetime

from platformSpecificHelper_Base import PlatformSpecificHelperBase
from platformSpecificHelper_Base import CopyToolBase
from platformSpecificHelper_Base import DownloadToolBase

def quoteme(to_qoute):
    return "".join( ('"', to_qoute, '"') )

class CopyToolMacRsync(CopyToolBase):
    def copy_dir_to_dir(self, src_dir, trg_dir, link_dest=None):
        if src_dir.endswith("/"):
            src_dir.rstrip("/")
        if link_dest is None:
            sync_command = "rsync -l -r -E --exclude=\'.svn/\' \"{src_dir}\" \"{trg_dir}\"".format(**locals())
        else:
            relative_link_dest = os.path.relpath(link_dest, trg_dir)
            sync_command = "rsync -l -r -E --exclude=\'.svn/\' --link-dest=\"{relative_link_dest}\" \"{src_dir}\" \"{trg_dir}\"".format(**locals())

        return sync_command

    def copy_file_to_dir(self, src_file, trg_dir, link_dest=None):
        assert not src_file.endswith("/")
        if link_dest is None:
            sync_command = "rsync -l -r -E --exclude=\'.svn/\' \"{src_file}\" \"{trg_dir}\"".format(**locals())
        else:
            relative_link_dest = os.path.relpath(link_dest, trg_dir)
            sync_command = "rsync -l -r -E --exclude=\'.svn/\' --link-dest=\"{relative_link_dest}\" \"{src_file}\" \"{trg_dir}\"".format(**locals())
        return sync_command

    def copy_dir_contents_to_dir(self, src_dir, trg_dir, link_dest=None):
        if not src_dir.endswith("/"):
            src_dir += "/"
        if link_dest is None:
            sync_command = "rsync -l -r -E --exclude=\'.svn/\' \"{src_dir}\" \"{trg_dir}\"".format(**locals())
        else:
            relative_link_dest = os.path.relpath(link_dest, trg_dir)
            sync_command = "rsync -l -r -E --exclude=\'.svn/\' --link-dest=\"{relative_link_dest}\" \"{src_dir}\" \"{trg_dir}\"".format(**locals())
        return sync_command

    def copy_dir_files_to_dir(self, src_dir, trg_dir, link_dest=None):
        if not src_dir.endswith("/"):
            src_dir += "/"
        # in order for * to correctly expand, it must be outside the quotes, e.g. to copy all files in folder a: A=a ; "${A}"/* and not "${A}/*"
        if link_dest is None:
            sync_command = "rsync -l -E -d --exclude=\'.svn/\' \"{src_dir}\"/* \"{trg_dir}\"".format(**locals())
        else:
            relative_link_dest = os.path.relpath(link_dest, trg_dir)
            sync_command = "rsync -l -E -d --exclude=\'.svn/\' --link-dest=\"{relative_link_dest}..\" \"{src_dir}\"/* \"{trg_dir}\"".format(**locals())

        return sync_command

class PlatformSpecificHelperMac(PlatformSpecificHelperBase):
    def __init__(self, instlInstance):
        super(PlatformSpecificHelperMac, self).__init__(instlInstance)
        self.var_replacement_pattern = "${\g<var_name>}"
        self.dl_tool = DownloadTool_mac_curl()

    def get_install_instructions_prefix(self):
        """ exec 2>&1 within a batch file will redirect stderr to stdout.
            .sync.sh >& out.txt on the command line will redirect stderr to stdout fom without.
        """
        retVal = (
            "#!/bin/sh",
            self.remark(self.instlInstance.get_version_str()),
            self.remark(datetime.datetime.today().isoformat()),
            self.save_dir("TOP_SAVE_DIR"))
        return retVal

    def get_install_instructions_postfix(self):
        return (self.restore_dir("TOP_SAVE_DIR"), "exit 0")

    def mkdir(self, directory):
        mk_command = " ".join( ("mkdir", "-p", quoteme(directory) ) )
        return mk_command

    def cd(self, directory):
        cd_command = " ".join( ("cd", quoteme(directory) ) )
        return cd_command

    def save_dir(self, var_name):
        save_dir_command = var_name+"=`pwd`"
        return save_dir_command

    def restore_dir(self, var_name):
        restore_dir_command = self.cd("$("+var_name+")")
        return restore_dir_command

    def rmdir(self, directory):
        rmdir_command = " ".join( ("rmdir", quoteme(directory) ) )
        return rmdir_command

    def rmfile(self, file):
        rmfile_command = " ".join( ("rm", "-f", quoteme(file) ) )
        return rmfile_command

    def get_svn_folder_cleanup_instructions(self):
        return 'find . -maxdepth 1 -mindepth 1 -type d -print0 | xargs -0 "$(SVN_CLIENT_PATH)" cleanup --non-interactive'

    def var_assign(self, identifier, value):
        return identifier+'="'+value+'"'

    def echo(self, message):
        echo_command = " ".join(('echo', quoteme(message)))
        return echo_command

    def remark(self, remark):
        remark_command = " ".join(('#', remark))
        return remark_command

    def use_copy_tool(self, tool):
        if tool == "rsync":
            self.copy_tool = CopyToolMacRsync()
        else:
            raise ValueError(tool, "is not a valid copy tool for Mac OS")

    def copy_file_to_file(self, src_file, trg_file):
        sync_command = "cp -f \"{src_file}\" \"{trg_file}\"".format(**locals())
        return sync_command

    def resolve_readlink_files(self, in_dir="."):
        """ create instructions to turn .readlink files into symlinks.
            Main problem was with files that had space in their name, just
            adding \" was no enough, had to separate each step to a single line
            which solved the spaces problem. Also find returns an empty string
            even when there were no files found, and therefor the check
        """
        resolve_commands = (
            "for readlink_file in \"$(find . -name '*.readlink')\" ; do",
            "   if [ \"$readlink_file\" ] ; then",             # avoid empty results
            "       file_contents=`cat \"$readlink_file\"`",    # avoid spaces in path
            "       link_file=\"${readlink_file%.*}\"",         # avoid spaces in path
            "       ln -s \"$file_contents\" \"$link_file\"",
            "       rm \"$readlink_file\"",
            "   fi",
            "done"
            )
        return resolve_commands

class DownloadTool_mac_curl(DownloadToolBase):
    def __init__(self):
        self.curl_instructions = list()

    def add_dl(self, url, path):
        self.curl_instructions.append( (urllib.quote(url, "$()/:"), path) )

    def create_download_file_to_file_command(self, src_url, trg_file):
        download_command_parts = list()
        download_command_parts.append("$(__RESOLVED_DOWNLOAD_TOOL_PATH__)")
        download_command_parts.append("--insecure")
        download_command_parts.append("--fail")
        download_command_parts.append("--raw")
        download_command_parts.append("--silent")
        download_command_parts.append("--show-error")
        download_command_parts.append("--compressed")
        download_command_parts.append("--connect-timeout")
        download_command_parts.append("60")
        download_command_parts.append("--max-time")
        download_command_parts.append("900")
        download_command_parts.append("-o")
        download_command_parts.append(quoteme(trg_file))
        download_command_parts.append(quoteme(urllib.quote(src_url, "$()/:")))
        return " ".join(download_command_parts)

    def create_config_file(self, curl_config_file_path):
        with open(curl_config_file_path, "w") as wfd:
            wfd.write("insecure\n")
            wfd.write("raw\n")
            wfd.write("create-dirs\n")
            wfd.write("connect-timeout = 60\n")
            wfd.write("\n")
            for url, path in self.curl_instructions:
                wfd.write('''url = "{url}"\noutput = "{path}"\n\n'''.format(**locals()))

    def create_download_from_config_file(self, config_file):

        download_command_parts = list()
        download_command_parts.append("$(__RESOLVED_DOWNLOAD_TOOL_PATH__)")
        #download_command_parts.append("--insecure")
        #download_command_parts.append("--raw")
        #download_command_parts.append("--silent")
        #download_command_parts.append("--create-dirs")
        #download_command_parts.append("--connect-timeout")
        #download_command_parts.append("60")
        download_command_parts.append("--max-time")
        download_command_parts.append(str(len(self.curl_instructions) * 3 + 180)) # 3 seconds for each item + 3 minutes
        download_command_parts.append("--config")
        download_command_parts.append(config_file)

        return " ".join(download_command_parts)
<commit_msg>--fail --silent --show-error and --compressed for curl downloads<commit_after>#!/usr/bin/env python2.7
from __future__ import print_function

import os
import urllib
import datetime

from platformSpecificHelper_Base import PlatformSpecificHelperBase
from platformSpecificHelper_Base import CopyToolBase
from platformSpecificHelper_Base import DownloadToolBase

def quoteme(to_qoute):
    return "".join( ('"', to_qoute, '"') )

class CopyToolMacRsync(CopyToolBase):
    def copy_dir_to_dir(self, src_dir, trg_dir, link_dest=None):
        if src_dir.endswith("/"):
            src_dir.rstrip("/")
        if link_dest is None:
            sync_command = "rsync -l -r -E --exclude=\'.svn/\' \"{src_dir}\" \"{trg_dir}\"".format(**locals())
        else:
            relative_link_dest = os.path.relpath(link_dest, trg_dir)
            sync_command = "rsync -l -r -E --exclude=\'.svn/\' --link-dest=\"{relative_link_dest}\" \"{src_dir}\" \"{trg_dir}\"".format(**locals())

        return sync_command

    def copy_file_to_dir(self, src_file, trg_dir, link_dest=None):
        assert not src_file.endswith("/")
        if link_dest is None:
            sync_command = "rsync -l -r -E --exclude=\'.svn/\' \"{src_file}\" \"{trg_dir}\"".format(**locals())
        else:
            relative_link_dest = os.path.relpath(link_dest, trg_dir)
            sync_command = "rsync -l -r -E --exclude=\'.svn/\' --link-dest=\"{relative_link_dest}\" \"{src_file}\" \"{trg_dir}\"".format(**locals())
        return sync_command

    def copy_dir_contents_to_dir(self, src_dir, trg_dir, link_dest=None):
        if not src_dir.endswith("/"):
            src_dir += "/"
        if link_dest is None:
            sync_command = "rsync -l -r -E --exclude=\'.svn/\' \"{src_dir}\" \"{trg_dir}\"".format(**locals())
        else:
            relative_link_dest = os.path.relpath(link_dest, trg_dir)
            sync_command = "rsync -l -r -E --exclude=\'.svn/\' --link-dest=\"{relative_link_dest}\" \"{src_dir}\" \"{trg_dir}\"".format(**locals())
        return sync_command

    def copy_dir_files_to_dir(self, src_dir, trg_dir, link_dest=None):
        if not src_dir.endswith("/"):
            src_dir += "/"
        # in order for * to correctly expand, it must be outside the quotes, e.g. to copy all files in folder a: A=a ; "${A}"/* and not "${A}/*"
        if link_dest is None:
            sync_command = "rsync -l -E -d --exclude=\'.svn/\' \"{src_dir}\"/* \"{trg_dir}\"".format(**locals())
        else:
            relative_link_dest = os.path.relpath(link_dest, trg_dir)
            sync_command = "rsync -l -E -d --exclude=\'.svn/\' --link-dest=\"{relative_link_dest}..\" \"{src_dir}\"/* \"{trg_dir}\"".format(**locals())

        return sync_command

class PlatformSpecificHelperMac(PlatformSpecificHelperBase):
    def __init__(self, instlInstance):
        super(PlatformSpecificHelperMac, self).__init__(instlInstance)
        self.var_replacement_pattern = "${\g<var_name>}"
        self.dl_tool = DownloadTool_mac_curl()

    def get_install_instructions_prefix(self):
        """ exec 2>&1 within a batch file will redirect stderr to stdout.
            .sync.sh >& out.txt on the command line will redirect stderr to stdout fom without.
        """
        retVal = (
            "#!/bin/sh",
            self.remark(self.instlInstance.get_version_str()),
            self.remark(datetime.datetime.today().isoformat()),
            self.save_dir("TOP_SAVE_DIR"))
        return retVal

    def get_install_instructions_postfix(self):
        return (self.restore_dir("TOP_SAVE_DIR"), "exit 0")

    def mkdir(self, directory):
        mk_command = " ".join( ("mkdir", "-p", quoteme(directory) ) )
        return mk_command

    def cd(self, directory):
        cd_command = " ".join( ("cd", quoteme(directory) ) )
        return cd_command

    def save_dir(self, var_name):
        save_dir_command = var_name+"=`pwd`"
        return save_dir_command

    def restore_dir(self, var_name):
        restore_dir_command = self.cd("$("+var_name+")")
        return restore_dir_command

    def rmdir(self, directory):
        rmdir_command = " ".join( ("rmdir", quoteme(directory) ) )
        return rmdir_command

    def rmfile(self, file):
        rmfile_command = " ".join( ("rm", "-f", quoteme(file) ) )
        return rmfile_command

    def get_svn_folder_cleanup_instructions(self):
        return 'find . -maxdepth 1 -mindepth 1 -type d -print0 | xargs -0 "$(SVN_CLIENT_PATH)" cleanup --non-interactive'

    def var_assign(self, identifier, value):
        return identifier+'="'+value+'"'

    def echo(self, message):
        echo_command = " ".join(('echo', quoteme(message)))
        return echo_command

    def remark(self, remark):
        remark_command = " ".join(('#', remark))
        return remark_command

    def use_copy_tool(self, tool):
        if tool == "rsync":
            self.copy_tool = CopyToolMacRsync()
        else:
            raise ValueError(tool, "is not a valid copy tool for Mac OS")

    def copy_file_to_file(self, src_file, trg_file):
        sync_command = "cp -f \"{src_file}\" \"{trg_file}\"".format(**locals())
        return sync_command

    def resolve_readlink_files(self, in_dir="."):
        """ create instructions to turn .readlink files into symlinks.
            Main problem was with files that had space in their name, just
            adding \" was no enough, had to separate each step to a single line
            which solved the spaces problem. Also find returns an empty string
            even when there were no files found, and therefor the check
        """
        resolve_commands = (
            "for readlink_file in \"$(find . -name '*.readlink')\" ; do",
            "   if [ \"$readlink_file\" ] ; then",             # avoid empty results
            "       file_contents=`cat \"$readlink_file\"`",    # avoid spaces in path
            "       link_file=\"${readlink_file%.*}\"",         # avoid spaces in path
            "       ln -s \"$file_contents\" \"$link_file\"",
            "       rm \"$readlink_file\"",
            "   fi",
            "done"
            )
        return resolve_commands

class DownloadTool_mac_curl(DownloadToolBase):
    def __init__(self):
        self.curl_instructions = list()

    def add_dl(self, url, path):
        self.curl_instructions.append( (urllib.quote(url, "$()/:"), path) )

    def create_download_file_to_file_command(self, src_url, trg_file):
        download_command_parts = list()
        download_command_parts.append("$(__RESOLVED_DOWNLOAD_TOOL_PATH__)")
        download_command_parts.append("--insecure")
        download_command_parts.append("--fail")
        download_command_parts.append("--raw")
        download_command_parts.append("--silent")
        download_command_parts.append("--show-error")
        download_command_parts.append("--compressed")
        download_command_parts.append("--connect-timeout")
        download_command_parts.append("60")
        download_command_parts.append("--max-time")
        download_command_parts.append("900")
        download_command_parts.append("-o")
        download_command_parts.append(quoteme(trg_file))
        download_command_parts.append(quoteme(urllib.quote(src_url, "$()/:")))
        return " ".join(download_command_parts)

    def create_config_file(self, curl_config_file_path):
        with open(curl_config_file_path, "w") as wfd:
            wfd.write("insecure\n")
            wfd.write("raw\n")
            wfd.write("fail\n")
            wfd.write("silent\n")
            wfd.write("show-error\n")
            wfd.write("compressed\n")
            wfd.write("create-dirs\n")
            wfd.write("connect-timeout = 60\n")
            wfd.write("\n")
            for url, path in self.curl_instructions:
                wfd.write('''url = "{url}"\noutput = "{path}"\n\n'''.format(**locals()))

    def create_download_from_config_file(self, config_file):

        download_command_parts = list()
        download_command_parts.append("$(__RESOLVED_DOWNLOAD_TOOL_PATH__)")
        download_command_parts.append("--max-time")
        download_command_parts.append(str(len(self.curl_instructions) * 6 + 300)) # 6 seconds for each item + 5 minutes
        download_command_parts.append("--config")
        download_command_parts.append(config_file)

        return " ".join(download_command_parts)
<|endoftext|>


<commit_before>from django.conf import global_settings
from django.template import Context, Template
from django.test import TestCase, modify_settings, override_settings

@modify_settings(INSTALLED_APPS={'append': 'django_react_templatetags'})
@override_settings(
    MIDDLEWARE_CLASSES=global_settings.MIDDLEWARE_CLASSES,
    TEMPLATES = [{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'debug': True,
            'context_processors': [
                'django.contrib.auth.context_processors.auth',
                'django.core.context_processors.debug',
                'django.core.context_processors.i18n',
                'django.core.context_processors.media',
                'django.core.context_processors.static',
                'django.core.context_processors.tz',
                'django.contrib.messages.context_processors.messages',
                'django.core.context_processors.request',

                # Project specific
                'django_react_templatetags.context_processors.react_context_processor',
            ],
        },
    }],
    SITE_ID=1
)
class ReactIncludeComponentTest(TestCase):
    def setUp(self):
        self.mocked_context = Context({'REACT_COMPONENTS': []})

    def test_react_tag(self):
        "The react_render inserts one components into the template"

        out = Template(
            "{% load react %}"
            "{% react_render component=\"Component\" %}"
        ).render(self.mocked_context)

        self.assertTrue('<div id="Component_' in out)

    def test_multiple_tags(self):
        "The react_render inserts two components into the template"

        out = Template(
            "{% load react %}"
            "{% react_render component=\"Component\" %}"
            "{% react_render component=\"Component\" %}"
        ).render(self.mocked_context)

        self.assertTrue('<div id="Component_' in out)
        self.assertEquals(len(self.mocked_context.get("REACT_COMPONENTS")), 2)

    def test_react_json_data_tag(self):
        "Tests that the data is added as correct json into the react render"

        self.mocked_context["component_data"] = {'name': 'Tom Waits'}

        out = Template(
            "{% load react %}"
            "{% react_render component=\"Component\" data=component_data %}"
            "{% react_print %}"
        ).render(self.mocked_context)

        self.assertTrue('{"name": "Tom Waits"}' in out)

    @override_settings(
        REACT_COMPONENT_PREFIX="Components."
    )
    def test_react_component_prefix(self):
        "Tests that a prefix is added to the component createElement script"

        out = Template(
            "{% load react %}"
            "{% react_render component=\"Component\" %}"
        ).render(self.mocked_context)

        self.assertTrue('<div id="Components.Component_' in out)

    def test_print_tag(self):
        "Makes sure the react_render gets emptied from context after print"

        out = Template(
            "{% load react %}"
            "{% react_render component=\"Component\" %}"
            "{% react_print %}"
        ).render(self.mocked_context)

        self.assertTrue('ReactDOM.render(' in out)
        self.assertTrue('React.createElement(Component' in out)
        self.assertEquals(len(self.mocked_context.get("REACT_COMPONENTS")), 0)

    def test_variable_identifier(self):
        "Tests that the identifier variable is evaluated as variable"

        self.mocked_context["component_identifier"] = "TomWaits"

        out = Template(
            "{% load react %}"
            "{% react_render component=\"Component\" identifier=component_identifier %}"
            "{% react_print %}"
        ).render(self.mocked_context)

        self.assertTrue('TomWaits' in out)

    def test_class_property(self):
        "Makes sure class property are applied"

        out = Template(
            "{% load react %}"
            "{% react_render component=\"Component\" css_class=\"component-class\" %}"
        ).render(self.mocked_context)

        self.assertTrue('class="component-class"' in out)
<commit_msg>Switched test to use class prop (instead of css_class)<commit_after>from django.conf import global_settings
from django.template import Context, Template
from django.test import TestCase, modify_settings, override_settings

@modify_settings(INSTALLED_APPS={'append': 'django_react_templatetags'})
@override_settings(
    MIDDLEWARE_CLASSES=global_settings.MIDDLEWARE_CLASSES,
    TEMPLATES = [{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'debug': True,
            'context_processors': [
                'django.contrib.auth.context_processors.auth',
                'django.core.context_processors.debug',
                'django.core.context_processors.i18n',
                'django.core.context_processors.media',
                'django.core.context_processors.static',
                'django.core.context_processors.tz',
                'django.contrib.messages.context_processors.messages',
                'django.core.context_processors.request',

                # Project specific
                'django_react_templatetags.context_processors.react_context_processor',
            ],
        },
    }],
    SITE_ID=1
)
class ReactIncludeComponentTest(TestCase):
    def setUp(self):
        self.mocked_context = Context({'REACT_COMPONENTS': []})

    def test_react_tag(self):
        "The react_render inserts one components into the template"

        out = Template(
            "{% load react %}"
            "{% react_render component=\"Component\" %}"
        ).render(self.mocked_context)

        self.assertTrue('<div id="Component_' in out)

    def test_multiple_tags(self):
        "The react_render inserts two components into the template"

        out = Template(
            "{% load react %}"
            "{% react_render component=\"Component\" %}"
            "{% react_render component=\"Component\" %}"
        ).render(self.mocked_context)

        self.assertTrue('<div id="Component_' in out)
        self.assertEquals(len(self.mocked_context.get("REACT_COMPONENTS")), 2)

    def test_react_json_data_tag(self):
        "Tests that the data is added as correct json into the react render"

        self.mocked_context["component_data"] = {'name': 'Tom Waits'}

        out = Template(
            "{% load react %}"
            "{% react_render component=\"Component\" data=component_data %}"
            "{% react_print %}"
        ).render(self.mocked_context)

        self.assertTrue('{"name": "Tom Waits"}' in out)

    @override_settings(
        REACT_COMPONENT_PREFIX="Components."
    )
    def test_react_component_prefix(self):
        "Tests that a prefix is added to the component createElement script"

        out = Template(
            "{% load react %}"
            "{% react_render component=\"Component\" %}"
        ).render(self.mocked_context)

        self.assertTrue('<div id="Components.Component_' in out)

    def test_print_tag(self):
        "Makes sure the react_render gets emptied from context after print"

        out = Template(
            "{% load react %}"
            "{% react_render component=\"Component\" %}"
            "{% react_print %}"
        ).render(self.mocked_context)

        self.assertTrue('ReactDOM.render(' in out)
        self.assertTrue('React.createElement(Component' in out)
        self.assertEquals(len(self.mocked_context.get("REACT_COMPONENTS")), 0)

    def test_variable_identifier(self):
        "Tests that the identifier variable is evaluated as variable"

        self.mocked_context["component_identifier"] = "TomWaits"

        out = Template(
            "{% load react %}"
            "{% react_render component=\"Component\" identifier=component_identifier %}"
            "{% react_print %}"
        ).render(self.mocked_context)

        self.assertTrue('TomWaits' in out)

    def test_class_property(self):
        "Makes sure class property are applied"

        out = Template(
            "{% load react %}"
            "{% react_render component=\"Component\" class=\"component-class\" %}"
        ).render(self.mocked_context)

        self.assertTrue('class="component-class"' in out)
<|endoftext|>


<commit_before># coding: utf-8
# Copyright (c) Pymatgen Development Team.
# Distributed under the terms of the MIT License.

from __future__ import division, unicode_literals

import collections
import abc

import numpy as np

from pymatgen.core.spectrum import Spectrum
from pymatgen.util.plotting import add_fig_kwargs

"""
This module implements an diffraction pattern calculator.
"""

__author__ = "Shyue Ping Ong"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "ongsp@ucsd.edu"
__date__ = "5/22/14"

class DiffractionPattern(Spectrum):
    """
    A representation of an XRDPattern
    """

    XLABEL = "$2\\Theta$"
    YLABEL = "Intensity"

    def __init__(self, x, y, hkls, d_hkls):
        """
        Args:
            x: Two theta angles.
            y: Intensities
            hkls: [{(h, k, l): mult}] {(h, k, l): mult} is a dict of Miller
                indices for all diffracted lattice facets contributing to each
                intensity.
            d_hkls: List of interplanar spacings.
        """
        super(DiffractionPattern, self).__init__(x, y, hkls,d_hkls)
        self.hkls = hkls
        self.d_hkls = d_hkls

class DiffractionPatternCalculator(abc.ABC):
    """
    Computes the XRD pattern of a crystal structure.

    This code is implemented by Shyue Ping Ong as part of UCSD's NANO106 -
    Crystallography of Materials. The formalism for this code is based on
    that given in Chapters 11 and 12 of Structure of Materials by Marc De
    Graef and Michael E. McHenry. This takes into account the atomic
    scattering factors and the Lorentz polarization factor, but not
    the Debye-Waller (temperature) factor (for which data is typically not
    available). Note that the multiplicity correction is not needed since
    this code simply goes through all reciprocal points within the limiting
    sphere, which includes all symmetrically equivalent facets. The algorithm
    is as follows

    1. Calculate reciprocal lattice of structure. Find all reciprocal points
       within the limiting sphere given by :math:`\\frac{2}{\\lambda}`.

    2. For each reciprocal point :math:`\\mathbf{g_{hkl}}` corresponding to
       lattice plane :math:`(hkl)`, compute the Bragg condition
       :math:`\\sin(\\theta) = \\frac{\\lambda}{2d_{hkl}}`

    3. Compute the structure factor as the sum of the atomic scattering
       factors. The atomic scattering factors are given by

       .. math::

           f(s) = Z - 41.78214 \\times s^2 \\times \\sum\\limits_{i=1}^n a_i \
           \\exp(-b_is^2)

       where :math:`s = \\frac{\\sin(\\theta)}{\\lambda}` and :math:`a_i`
       and :math:`b_i` are the fitted parameters for each element. The
       structure factor is then given by

       .. math::

           F_{hkl} = \\sum\\limits_{j=1}^N f_j \\exp(2\\pi i \\mathbf{g_{hkl}}
           \\cdot \\mathbf{r})

    4. The intensity is then given by the modulus square of the structure
       factor.

       .. math::

           I_{hkl} = F_{hkl}F_{hkl}^*

    5. Finally, the Lorentz polarization correction factor is applied. This
       factor is given by:

       .. math::

           P(\\theta) = \\frac{1 + \\cos^2(2\\theta)}
           {\\sin^2(\\theta)\\cos(\\theta)}
    """
    # Tolerance in which to treat two peaks as having the same two theta.
    TWO_THETA_TOL = 1e-5

    # Tolerance in which to treat a peak as effectively 0 if the scaled
    # intensity is less than this number. Since the max intensity is 100,
    # this means the peak must be less than 1e-5 of the peak intensity to be
    # considered as zero. This deals with numerical issues where systematic
    # absences do not cancel exactly to zero.
    SCALED_INTENSITY_TOL = 1e-3

    @abc.abstractmethod
    def get_pattern(self, structure, scaled=True, two_theta_range=(0, 90)):
        """
        Calculates the diffraction pattern for a structure.

        Args:
            structure (Structure): Input structure
            scaled (bool): Whether to return scaled intensities. The maximum
                peak is set to a value of 100. Defaults to True. Use False if
                you need the absolute values to combine XRD plots.
            two_theta_range ([float of length 2]): Tuple for range of
                two_thetas to calculate in degrees. Defaults to (0, 90). Set to
                None if you want all diffracted beams within the limiting
                sphere of radius 2 / wavelength.

        Returns:
            (DiffractionPattern)
        """
        pass

    def get_plot(self, structure, two_theta_range=(0, 90),
                     annotate_peaks=True, ax=None, with_labels=True,
                     fontsize=16):
        """
        Returns the XRD plot as a matplotlib.pyplot.

        Args:
            structure: Input structure
            two_theta_range ([float of length 2]): Tuple for range of
                two_thetas to calculate in degrees. Defaults to (0, 90). Set to
                None if you want all diffracted beams within the limiting
                sphere of radius 2 / wavelength.
            annotate_peaks: Whether to annotate the peaks with plane
                information.
            ax: matplotlib :class:`Axes` or None if a new figure should be created.
            with_labels: True to add xlabels and ylabels to the plot.
            fontsize: (int) fontsize for peak labels.

        Returns:
            (matplotlib.pyplot)
        """
        if ax is None:
            from pymatgen.util.plotting import pretty_plot
            plt = pretty_plot(16, 10)
            ax = plt.gca()
        else:
            # This to maintain the type of the return value.
            import matplotlib.pyplot as plt

        xrd = self.get_pattern(structure, two_theta_range=two_theta_range)

        for two_theta, i, hkls, d_hkl in zip(xrd.x, xrd.y, xrd.hkls, xrd.d_hkls):
            if two_theta_range[0] <= two_theta <= two_theta_range[1]:
                label = ", ".join([str(hkl) for hkl in hkls.keys()])
                ax.plot([two_theta, two_theta], [0, i], color='k',
                         linewidth=3, label=label)
                if annotate_peaks:
                    ax.annotate(label, xy=[two_theta, i],
                                xytext=[two_theta, i], fontsize=fontsize)

        if with_labels:
            ax.set_xlabel(r"$2\theta$ ($^\circ$)")
            ax.set_ylabel("Intensities (scaled)")

        if hasattr(ax, "tight_layout"):
            ax.tight_layout()

        return plt

    def show_plot(self, structure, two_theta_range=(0, 90),
                      annotate_peaks=True):
        """
        Shows the XRD plot.

        Args:
            structure (Structure): Input structure
            two_theta_range ([float of length 2]): Tuple for range of
                two_thetas to calculate in degrees. Defaults to (0, 90). Set to
                None if you want all diffracted beams within the limiting
                sphere of radius 2 / wavelength.
            annotate_peaks (bool): Whether to annotate the peaks with plane
                information.
        """
        self.get_plot(structure, two_theta_range=two_theta_range,
                          annotate_peaks=annotate_peaks).show()

    @add_fig_kwargs
    def plot_structures(self, structures, two_theta_range=(0, 90),
                       annotate_peaks=True, fontsize=6, **kwargs):
        """
        Plot XRD for multiple structures on the same figure.

        Args:
            structures (Structure): List of structures
            two_theta_range ([float of length 2]): Tuple for range of
                two_thetas to calculate in degrees. Defaults to (0, 90). Set to
                None if you want all diffracted beams within the limiting
                sphere of radius 2 / wavelength.
            annotate_peaks (bool): Whether to annotate the peaks with plane
                information.
            fontsize: (int) fontsize for peak labels.
        """
        import matplotlib.pyplot as plt
        nrows = len(structures)
        fig, axes = plt.subplots(nrows=nrows, ncols=1, sharex=True, squeeze=False)

        for i, (ax, structure) in enumerate(zip(axes.ravel(), structures)):
            self.get_plot(structure, two_theta_range=two_theta_range,
                          annotate_peaks=annotate_peaks,
                          fontsize=fontsize, ax=ax, with_labels=i == nrows - 1)
            spg_symbol, spg_number = structure.get_space_group_info()
            ax.set_title("{} {} ({}) ".format(structure.formula, spg_symbol,
                                              spg_number))

        return fig

def get_unique_families(hkls):
    """
    Returns unique families of Miller indices. Families must be permutations
    of each other.

    Args:
        hkls ([h, k, l]): List of Miller indices.

    Returns:
        {hkl: multiplicity}: A dict with unique hkl and multiplicity.
    """
    # TODO: Definitely can be sped up.
    def is_perm(hkl1, hkl2):
        h1 = np.abs(hkl1)
        h2 = np.abs(hkl2)
        return all([i == j for i, j in zip(sorted(h1), sorted(h2))])

    unique = collections.defaultdict(list)
    for hkl1 in hkls:
        found = False
        for hkl2 in unique.keys():
            if is_perm(hkl1, hkl2):
                found = True
                unique[hkl2].append(hkl1)
                break
        if not found:
            unique[hkl1].append(hkl1)

    pretty_unique = {}
    for k, v in unique.items():
        pretty_unique[sorted(v)[-1]] = len(v)

    return pretty_unique
<commit_msg>More streamlinging.<commit_after># coding: utf-8
# Copyright (c) Pymatgen Development Team.
# Distributed under the terms of the MIT License.

from __future__ import division, unicode_literals

import collections
import abc

import numpy as np

from pymatgen.core.spectrum import Spectrum
from pymatgen.util.plotting import add_fig_kwargs

"""
This module implements an diffraction pattern calculator.
"""

__author__ = "Shyue Ping Ong"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "ongsp@ucsd.edu"
__date__ = "5/22/14"

class DiffractionPattern(Spectrum):
    """
    A representation of a diffraction pattern
    """

    XLABEL = "$2\\Theta$"
    YLABEL = "Intensity"

    def __init__(self, x, y, hkls, d_hkls):
        """
        Args:
            x: Two theta angles.
            y: Intensities
            hkls: [{(h, k, l): mult}] {(h, k, l): mult} is a dict of Miller
                indices for all diffracted lattice facets contributing to each
                intensity.
            d_hkls: List of interplanar spacings.
        """
        super(DiffractionPattern, self).__init__(x, y, hkls,d_hkls)
        self.hkls = hkls
        self.d_hkls = d_hkls

class DiffractionPatternCalculator(abc.ABC):
    """
    Abstract base class for computing the diffraction pattern of a crystal.
    """
    # Tolerance in which to treat two peaks as having the same two theta.
    TWO_THETA_TOL = 1e-5

    # Tolerance in which to treat a peak as effectively 0 if the scaled
    # intensity is less than this number. Since the max intensity is 100,
    # this means the peak must be less than 1e-5 of the peak intensity to be
    # considered as zero. This deals with numerical issues where systematic
    # absences do not cancel exactly to zero.
    SCALED_INTENSITY_TOL = 1e-3

    @abc.abstractmethod
    def get_pattern(self, structure, scaled=True, two_theta_range=(0, 90)):
        """
        Calculates the diffraction pattern for a structure.

        Args:
            structure (Structure): Input structure
            scaled (bool): Whether to return scaled intensities. The maximum
                peak is set to a value of 100. Defaults to True. Use False if
                you need the absolute values to combine XRD plots.
            two_theta_range ([float of length 2]): Tuple for range of
                two_thetas to calculate in degrees. Defaults to (0, 90). Set to
                None if you want all diffracted beams within the limiting
                sphere of radius 2 / wavelength.

        Returns:
            (DiffractionPattern)
        """
        pass

    def get_plot(self, structure, two_theta_range=(0, 90),
                 annotate_peaks=True, ax=None, with_labels=True,
                 fontsize=16):
        """
        Returns the XRD plot as a matplotlib.pyplot.

        Args:
            structure: Input structure
            two_theta_range ([float of length 2]): Tuple for range of
                two_thetas to calculate in degrees. Defaults to (0, 90). Set to
                None if you want all diffracted beams within the limiting
                sphere of radius 2 / wavelength.
            annotate_peaks: Whether to annotate the peaks with plane
                information.
            ax: matplotlib :class:`Axes` or None if a new figure should be created.
            with_labels: True to add xlabels and ylabels to the plot.
            fontsize: (int) fontsize for peak labels.

        Returns:
            (matplotlib.pyplot)
        """
        if ax is None:
            from pymatgen.util.plotting import pretty_plot
            plt = pretty_plot(16, 10)
            ax = plt.gca()
        else:
            # This to maintain the type of the return value.
            import matplotlib.pyplot as plt

        xrd = self.get_pattern(structure, two_theta_range=two_theta_range)

        for two_theta, i, hkls, d_hkl in zip(xrd.x, xrd.y, xrd.hkls, xrd.d_hkls):
            if two_theta_range[0] <= two_theta <= two_theta_range[1]:
                label = ", ".join([str(hkl) for hkl in hkls.keys()])
                ax.plot([two_theta, two_theta], [0, i], color='k',
                         linewidth=3, label=label)
                if annotate_peaks:
                    ax.annotate(label, xy=[two_theta, i],
                                xytext=[two_theta, i], fontsize=fontsize)

        if with_labels:
            ax.set_xlabel(r"$2\theta$ ($^\circ$)")
            ax.set_ylabel("Intensities (scaled)")

        if hasattr(ax, "tight_layout"):
            ax.tight_layout()

        return plt

    def show_plot(self, structure, **kwargs):
        """
        Shows the XRD plot.

        Args:
            structure (Structure): Input structure
            two_theta_range ([float of length 2]): Tuple for range of
                two_thetas to calculate in degrees. Defaults to (0, 90). Set to
                None if you want all diffracted beams within the limiting
                sphere of radius 2 / wavelength.
            annotate_peaks (bool): Whether to annotate the peaks with plane
                information.
        """
        self.get_plot(structure, **kwargs).show()

    @add_fig_kwargs
    def plot_structures(self, structures, fontsize=6, **kwargs):
        """
        Plot XRD for multiple structures on the same figure.

        Args:
            structures (Structure): List of structures
            two_theta_range ([float of length 2]): Tuple for range of
                two_thetas to calculate in degrees. Defaults to (0, 90). Set to
                None if you want all diffracted beams within the limiting
                sphere of radius 2 / wavelength.
            annotate_peaks (bool): Whether to annotate the peaks with plane
                information.
            fontsize: (int) fontsize for peak labels.
        """
        import matplotlib.pyplot as plt
        nrows = len(structures)
        fig, axes = plt.subplots(nrows=nrows, ncols=1, sharex=True,
                                 squeeze=False)

        for i, (ax, structure) in enumerate(zip(axes.ravel(), structures)):
            self.get_plot(structure,
                          fontsize=fontsize, ax=ax, with_labels=i == nrows - 1,
                          **kwargs)
            spg_symbol, spg_number = structure.get_space_group_info()
            ax.set_title("{} {} ({}) ".format(structure.formula, spg_symbol,
                                              spg_number))

        return fig

def get_unique_families(hkls):
    """
    Returns unique families of Miller indices. Families must be permutations
    of each other.

    Args:
        hkls ([h, k, l]): List of Miller indices.

    Returns:
        {hkl: multiplicity}: A dict with unique hkl and multiplicity.
    """
    # TODO: Definitely can be sped up.
    def is_perm(hkl1, hkl2):
        h1 = np.abs(hkl1)
        h2 = np.abs(hkl2)
        return all([i == j for i, j in zip(sorted(h1), sorted(h2))])

    unique = collections.defaultdict(list)
    for hkl1 in hkls:
        found = False
        for hkl2 in unique.keys():
            if is_perm(hkl1, hkl2):
                found = True
                unique[hkl2].append(hkl1)
                break
        if not found:
            unique[hkl1].append(hkl1)

    pretty_unique = {}
    for k, v in unique.items():
        pretty_unique[sorted(v)[-1]] = len(v)

    return pretty_unique
<|endoftext|>


<commit_before>"""
Unit tests for filters
David Megginson
April 2015

License: Public Domain
"""

import unittest

from hxl import hxl

#
# Base data for tests
#

DATA = [
    ['Organisation', 'Cluster', 'District'],
    ['#org', '#sector', '#adm1'],
    ['NGO A', 'WASH', 'Coast'],
    ['NGO B', 'Education', 'Plains'],
    ['NGO B', 'Education', 'Coast']
]

SCHEMA_GOOD = [
    ['#valid_tag', '#valid_required'],
    ['#org', 'true']
]

SCHEMA_BAD = [
    ['#valid_tag', '#valid_required'],
    ['#severity', 'true']
]

class AbstractFilterTest(unittest.TestCase):
    """Base class for all tests."""

    def setUp(self):
        # use a cache filter so that we can run tests multiple times
        self.source = hxl(DATA).cache()

#
# Test classes
#

class TestCacheFilter(AbstractFilterTest):

    def test_headers(self):
        self.assertEqual(DATA[0], self.source.cache().headers)

    def test_columns(self):
        self.assertEqual(DATA[1], self.source.cache().tags)

    def test_rows(self):
        self.assertEqual(DATA[2:], self.source.cache().values)

class TestColumnFilter(AbstractFilterTest):

    def test_with_columns(self):
        expected = ['#sector']
        self.assertEqual(expected, self.source.with_columns('#sector').tags)
        self.assertEqual(expected, self.source.with_columns(['#sector']).tags)

    def test_without_columns(self):
        expected = ['#org', '#adm1']
        self.assertEqual(expected, self.source.without_columns('#sector').tags)
        self.assertEqual(expected, self.source.without_columns(['#sector']).tags)

class TestRowFilter(AbstractFilterTest):

    def test_with_rows(self):
        self.assertEqual(DATA[3:], self.source.with_rows(['#sector=education']).values)
        self.assertEqual(DATA[3:], self.source.with_rows('#sector=education').values)

    def test_without_rows(self):
        self.assertEqual(DATA[3:], self.source.without_rows(['#sector=wash']).values)
        self.assertEqual(DATA[3:], self.source.without_rows('#sector=wash').values)

class TestCountFilter(AbstractFilterTest):

    def test_tags(self):
        expected = ['#sector', '#x_count_num']
        self.assertEqual(expected, self.source.count('#sector').tags)

    # TODO test aggregation

class TestSortFilter(AbstractFilterTest):

    def test_forward(self):
        self.assertEqual(sorted(DATA[2:]), self.source.sort().values)

    def test_backward(self):
        self.assertEqual(sorted(DATA[2:], reverse=True), self.source.sort(reverse=True).values)

    def test_custom_keys(self):
        def key(r):
            return [r[2], r[1]]
        self.assertEqual(sorted(DATA[2:], key=key), self.source.sort(['#adm1', '#sector']).values)

class TestAddColumnsFilter(AbstractFilterTest):

    spec = 'Country#country=Country A'

    def test_before(self):
        self.assertEqual(
            ['#country', '#org', '#sector', '#adm1'],
            self.source.add_columns(self.spec, True).tags
        )

    def test_after(self):
        self.assertEqual(
            ['#org', '#sector', '#adm1', '#country'],
            self.source.add_columns(self.spec).tags
        )

    def test_headers(self):
        self.assertEqual(
            DATA[0] + ['Country'],
            self.source.add_columns(self.spec).headers
        )

    def test_rows(self):
        self.assertEqual(
            [values + ['Country A'] for values in DATA[2:]],
            self.source.add_columns(self.spec).values
        )

class TestRenameFilter(AbstractFilterTest):

    spec = '#sector:Sub-sector#subsector'

    def test_tags(self):
        self.assertEqual(
            ['#org', '#subsector', '#adm1'],
            self.source.rename_columns(self.spec).tags
        )

    def test_headers(self):
        self.assertEqual(
            ['Organisation', 'Sub-sector', 'District'],
            self.source.rename_columns(self.spec).headers
        )

class TestChaining(AbstractFilterTest):

    def test_rowfilter_countfilter(self):
        self.assertEqual(
            [['NGO A', 1]],
            self.source.with_rows('#sector=wash').count('#org').values
        )
        self.assertEqual(
            [['NGO B', 2]],
            self.source.without_rows('#sector=wash').count('#org').values
        )

class TestNonFilters(AbstractFilterTest):
    # TODO move elsewhere

    def test_validate(self):
        self.assertTrue(self.source.validate(SCHEMA_GOOD))
        self.assertFalse(self.source.validate(SCHEMA_BAD))
<commit_msg>Added aggregation tests for CountFilter.<commit_after>"""
Unit tests for filters
David Megginson
April 2015

License: Public Domain
"""

import unittest

from hxl import hxl

#
# Base data for tests
#

DATA = [
    ['Organisation', 'Cluster', 'District', 'Affected'],
    ['#org', '#sector', '#adm1', '#affected'],
    ['NGO A', 'WASH', 'Coast', '100'],
    ['NGO B', 'Education', 'Plains', '200'],
    ['NGO B', 'Education', 'Coast', '300']
]

SCHEMA_GOOD = [
    ['#valid_tag', '#valid_required'],
    ['#org', 'true']
]

SCHEMA_BAD = [
    ['#valid_tag', '#valid_required'],
    ['#severity', 'true']
]

class AbstractFilterTest(unittest.TestCase):
    """Base class for all tests."""

    def setUp(self):
        # use a cache filter so that we can run tests multiple times
        self.source = hxl(DATA).cache()

#
# Test classes
#

class TestCacheFilter(AbstractFilterTest):

    def test_headers(self):
        self.assertEqual(DATA[0], self.source.cache().headers)

    def test_columns(self):
        self.assertEqual(DATA[1], self.source.cache().tags)

    def test_rows(self):
        self.assertEqual(DATA[2:], self.source.cache().values)

class TestColumnFilter(AbstractFilterTest):

    def test_with_columns(self):
        expected = ['#sector']
        self.assertEqual(expected, self.source.with_columns('#sector').tags)
        self.assertEqual(expected, self.source.with_columns(['#sector']).tags)

    def test_without_columns(self):
        expected = ['#org', '#adm1', '#affected']
        self.assertEqual(expected, self.source.without_columns('#sector').tags)
        self.assertEqual(expected, self.source.without_columns(['#sector']).tags)

class TestRowFilter(AbstractFilterTest):

    def test_with_rows(self):
        self.assertEqual(DATA[3:], self.source.with_rows(['#sector=education']).values)
        self.assertEqual(DATA[3:], self.source.with_rows('#sector=education').values)

    def test_without_rows(self):
        self.assertEqual(DATA[3:], self.source.without_rows(['#sector=wash']).values)
        self.assertEqual(DATA[3:], self.source.without_rows('#sector=wash').values)

class TestCountFilter(AbstractFilterTest):

    def test_tags(self):
        expected = ['#sector', '#x_count_num']
        self.assertEqual(expected, self.source.count('#sector').tags)

    def test_values(self):
        expected = [
            ['Education', 2],
            ['WASH', 1]
        ]
        self.assertEqual(expected, self.source.count('#sector').values)

    def test_aggregation_tags(self):
        expected = ['#sector', '#x_count_num', '#x_sum_num', '#x_average_num', '#x_min_num', '#x_max_num']
        self.assertEqual(expected, self.source.count('#sector', '#affected').tags)

    def test_aggregation_values(self):
        expected = [
            ['Education', 2, 500, 250, 200, 300],
            ['WASH', 1, 100, 100, 100, 100]
        ]
        self.assertEqual(expected, self.source.count('#sector', '#affected').values)

class TestSortFilter(AbstractFilterTest):

    def test_forward(self):
        self.assertEqual(sorted(DATA[2:]), self.source.sort().values)

    def test_backward(self):
        self.assertEqual(sorted(DATA[2:], reverse=True), self.source.sort(reverse=True).values)

    def test_custom_keys(self):
        def key(r):
            return [r[2], r[1]]
        self.assertEqual(sorted(DATA[2:], key=key), self.source.sort(['#adm1', '#sector']).values)

class TestAddColumnsFilter(AbstractFilterTest):

    spec = 'Country#country=Country A'

    def test_before(self):
        self.assertEqual(
            ['#country', '#org', '#sector', '#adm1', '#affected'],
            self.source.add_columns(self.spec, True).tags
        )

    def test_after(self):
        self.assertEqual(
            ['#org', '#sector', '#adm1', '#affected', '#country'],
            self.source.add_columns(self.spec).tags
        )

    def test_headers(self):
        self.assertEqual(
            DATA[0] + ['Country'],
            self.source.add_columns(self.spec).headers
        )

    def test_rows(self):
        self.assertEqual(
            [values + ['Country A'] for values in DATA[2:]],
            self.source.add_columns(self.spec).values
        )

class TestRenameFilter(AbstractFilterTest):

    spec = '#sector:Sub-sector#subsector'

    def test_tags(self):
        self.assertEqual(
            ['#org', '#subsector', '#adm1', '#affected'],
            self.source.rename_columns(self.spec).tags
        )

    def test_headers(self):
        self.assertEqual(
            ['Organisation', 'Sub-sector', 'District', 'Affected'],
            self.source.rename_columns(self.spec).headers
        )

class TestChaining(AbstractFilterTest):

    def test_rowfilter_countfilter(self):
        self.assertEqual(
            [['NGO A', 1]],
            self.source.with_rows('#sector=wash').count('#org').values
        )
        self.assertEqual(
            [['NGO B', 2]],
            self.source.without_rows('#sector=wash').count('#org').values
        )

class TestNonFilters(AbstractFilterTest):
    # TODO move elsewhere

    def test_validate(self):
        self.assertTrue(self.source.validate(SCHEMA_GOOD))
        self.assertFalse(self.source.validate(SCHEMA_BAD))
<|endoftext|>