<commit_before>"""
This module gets thermodynamic datat from the JANAF database.
Files are downloaded from the NIST servers as needed and then cached locally.

Zack Gainsforth

Funding by NASA
"""

from __future__ import division
from __future__ import print_function

import os
import numpy as np
import pandas as pd
from scipy.interpolate import interp1d

try:
    # Python 3
    import urllib.request as urllib2
except ImportError:
    # Python 2
    import urllib2

try:
    # Python 2
    from StringIO import StringIO
except ImportError:
    # Python 3
    from io import StringIO

# Universal gas constant R
R = 8.314472

class JanafPhase(object):
    """
    Class which is created by Janafdb for a specific phase.

    It reads in the JANAF data file and produces functions which interpolate
    the thermodynamic constants.

    >>> db = Janafdb()
    >>> p = db.getphasedata(formula='O2Ti', name='Rutile', phase='cr')
    >>> print(p.cp([500, 550, 1800]))
    [ 67.203  68.567  78.283]
    >>> print(p.S([500, 550, 1800]))
    [  82.201    88.4565  176.876 ]
    >>> print(p.gef([500, 550, 1800]))
    [  57.077   59.704  115.753]
    >>> print(p.hef([500, 550, 1800]))
    [  12.562    15.9955  110.022 ]
    >>> print(p.DeltaH([500, 550, 1800]))
    [-943670.  -943229.5 -936679. ]
    >>> print(p.DeltaG([500, 550, 1800]))
    [-852157.  -843046.5 -621013. ]
    >>> print(p.logKf([500, 550, 1800]))
    [ 89.024   80.8125  18.021 ]
    >>> print(p.cp(50000))
    Traceback (most recent call last):
        ...
    ValueError: A value in x_new is above the interpolation range.
    """

    def __init__(self, rawdata_text):
        # Store the raw data text file from NIST.
        self.rawdata_text = rawdata_text

        # Read the text file into a DataFrame.
        data = pd.read_csv(
            StringIO(self.rawdata_text),
            skiprows=2,
            header=None,
            delimiter=r'[\t\s]*',
            engine='python')
        data.columns = ['T', 'Cp', 'S', '[G-H(Tr)]/T', 'H-H(Tr)', 'Delta_fH',
                        'Delta_fG', 'log(Kf)']
        self.rawdata = data

        # Sometimes the JANAF files have funky stuff written in them. (Old
        # school text format...)
        # Clean it up.
        for c in data.columns:
            # We only need to polish up columns that aren't floating point
            # numbers.
            if np.issubdtype(data.dtypes[c], np.floating):
                continue
            # Change INFINITE to inf
            data.loc[data[c] == 'INFINITE', c]
            # Anything else becomes a nan.
            # Convert to floats.
            data[c] = pd.to_numeric(data[c], errors='coerce')

        # Convert all units to Joules.
        data['Delta_fH'] *= 1000
        data['Delta_fG'] *= 1000

        # Now make interpolatable functions for each of these.
        self.cp = interp1d(self.rawdata['T'], self.rawdata['Cp'])
        self.S = interp1d(self.rawdata['T'], self.rawdata['S'])
        self.gef = interp1d(self.rawdata['T'], self.rawdata['[G-H(Tr)]/T'])
        self.hef = interp1d(self.rawdata['T'], self.rawdata['H-H(Tr)'])
        self.DeltaH = interp1d(self.rawdata['T'], self.rawdata['Delta_fH'])
        self.DeltaG = interp1d(self.rawdata['T'], self.rawdata['Delta_fG'])
        self.logKf = interp1d(self.rawdata['T'], self.rawdata['log(Kf)'])

        # TODO Deal well with crystal<->liquid transitions which have a below
        # and above value for Cp, S, etc.

class Janafdb(object):
    """
    Class that reads the NIST JANAF tables for thermodynamic data.

    Data is initially read from the web servers, and then cached.
    """
    VALIDPHASETYPES = ['cr', 'l', 'cr,l', 'g', 'ref', 'cd', 'fl', 'am', 'vit',
                       'mon', 'pol', 'sln', 'aq', 'sat']
    JANAF_URL = "http://kinetics.nist.gov/janaf/html/%s.txt"

    def __init__(self):
        """
        We have an index file which can be used to build the url for all phases
        on the NIST site.
        """

        # Read the index file which tells us the filenames for all the phases
        # in the JANAF database.
        dirname = os.path.dirname(__file__)
        janaf_index = os.path.join(dirname, 'JANAF_index.txt')
        self.db = pd.read_csv(janaf_index, delimiter='|')
        # Name the columns and trim whitespace off the text fields.
        self.db.columns = ['formula', 'name', 'phase', 'filename']
        self.db["formula"] = self.db["formula"].map(str.strip)
        self.db["name"] = self.db["name"].map(str.strip)
        self.db["phase"] = self.db["phase"].map(str.strip)
        self.db["filename"] = self.db["filename"].map(str.strip)

        # Make sure that the directory for cached JANAF files exists.
        self.JANAF_cachedir = os.path.join(dirname, 'JANAF_Cache')
        if not os.path.exists(self.JANAF_cachedir):
            os.mkdir(self.JANAF_cachedir)

    def search(self, searchstr):
        """
        List all the species containing a string. Helpful for
        interactive use of the database.
        returns a pandas dataframe containing valid phases.

        >>> db = Janafdb()
        >>> s = db.search('Rb-')
        >>> print(s)
             formula           name phase filename
        1710     Rb-  Rubidium, Ion     g   Rb-007
        >>> s = db.search('Ti')
        >>> print(len(s))
        88
        """

        formulasearch = self.db['formula'].str.contains(searchstr)
        namesearch = self.db['name'].str.contains(searchstr)

        return self.db[formulasearch | namesearch]

    def getphasedata(self, formula=None, name=None, phase=None, nocache=False):
        """
        Returns an element instance given the name of the element.
        formula, name and phase match the respective fields in the JANAF index.
        nocache = True means that we will always get the data from the web.

        >>> db = Janafdb()
        >>> db.getphasedata(formula='O2Ti', phase='cr')
        Traceback (most recent call last):
            ...
        ValueError: There are 2 records matching this pattern.
        >>> db.getphasedata(formula='Oxyz')
        Traceback (most recent call last):
            ...
        ValueError: Valid phase types are ['cr', 'l', 'cr,l', 'g', 'ref', 'cd', 'fl', 'am', 'vit', 'mon', 'pol', 'sln', 'aq', 'sat'].
        >>> db.getphasedata(formula='Oxyz', phase='l')
        Traceback (most recent call last):
            ...
        ValueError: Did not find Oxyz, None, (l)
        """

        # Check that the phase type requested is valid.
        if phase not in Janafdb.VALIDPHASETYPES:
            raise ValueError("Valid phase types are " + str(
                Janafdb.VALIDPHASETYPES) + ".")

        # We can search on either an exact formula, partial text match in the
        # name, and exact phase type.
        formulasearch = pd.Series(np.ones(len(self.db)), dtype=bool)
        namesearch = formulasearch.copy()
        phasesearch = formulasearch.copy()
        if formula is not None:
            formulasearch = self.db['formula'] == formula
        if name is not None:
            namesearch = self.db['name'].str.contains(name)
        if phase is not None:
            phasesearch = self.db['phase'] == phase
        searchmatch = formulasearch & namesearch & phasesearch

        # Get the record (should be one record) which specifies this phase.
        PhaseRecord = self.db[searchmatch]
        if len(PhaseRecord) == 0:
            raise ValueError("Did not find %s, %s, (%s)" %
                             (formula, name, phase))
        if len(PhaseRecord) > 1:
            raise ValueError("There are %d records matching this pattern." %
                             len(PhaseRecord))

        # At this point we have one record.  Check if we have that file cached.
        cachedfilename = os.path.join(
            self.JANAF_cachedir, PhaseRecord['filename'].values[0] + '.txt')
        if os.path.exists(cachedfilename) and not nocache:
            # Yes it was cached, so let's read it into memory.
            with open(cachedfilename, 'r') as f:
                textdata = f.read()
        else:
            # No it was not cached so let's get it from the web.
            response = urllib2.urlopen(Janafdb.JANAF_URL %
                                       PhaseRecord['filename'].values[0])
            textdata = response.read()

            # And cache the data so we aren't making unnecessary trips to the
            # web.
            if not nocache:
                with open(cachedfilename, 'w') as f:
                    f.write(textdata)

        # Create a phase class and return it.
        return JanafPhase(textdata)

    # def getmixturedata(self, components):
    #     """
    #     Creates a mixture of components given a list of tuples
    #     containing the formula and the volume percent
    #     """
    #     mixture = Mixture()
    #     for comp in components:
    #         mixture.add(self.getelementdata(comp[0]), comp[1])
    #
    #     return mixture
<commit_msg>Replace nocache with cache<commit_after>"""
This module gets thermodynamic datat from the JANAF database.
Files are downloaded from the NIST servers as needed and then cached locally.

Zack Gainsforth

Funding by NASA
"""

from __future__ import division
from __future__ import print_function

import os
import numpy as np
import pandas as pd
from scipy.interpolate import interp1d

try:
    # Python 3
    import urllib.request as urllib2
except ImportError:
    # Python 2
    import urllib2

try:
    # Python 2
    from StringIO import StringIO
except ImportError:
    # Python 3
    from io import StringIO

# Universal gas constant R
R = 8.314472

class JanafPhase(object):
    """
    Class which is created by Janafdb for a specific phase.

    It reads in the JANAF data file and produces functions which interpolate
    the thermodynamic constants.

    >>> db = Janafdb()
    >>> p = db.getphasedata(formula='O2Ti', name='Rutile', phase='cr')
    >>> print(p.cp([500, 550, 1800]))
    [ 67.203  68.567  78.283]
    >>> print(p.S([500, 550, 1800]))
    [  82.201    88.4565  176.876 ]
    >>> print(p.gef([500, 550, 1800]))
    [  57.077   59.704  115.753]
    >>> print(p.hef([500, 550, 1800]))
    [  12.562    15.9955  110.022 ]
    >>> print(p.DeltaH([500, 550, 1800]))
    [-943670.  -943229.5 -936679. ]
    >>> print(p.DeltaG([500, 550, 1800]))
    [-852157.  -843046.5 -621013. ]
    >>> print(p.logKf([500, 550, 1800]))
    [ 89.024   80.8125  18.021 ]
    >>> print(p.cp(50000))
    Traceback (most recent call last):
        ...
    ValueError: A value in x_new is above the interpolation range.
    """

    def __init__(self, rawdata_text):
        # Store the raw data text file from NIST.
        self.rawdata_text = rawdata_text

        # Read the text file into a DataFrame.
        data = pd.read_csv(
            StringIO(self.rawdata_text),
            skiprows=2,
            header=None,
            delimiter=r'[\t\s]*',
            engine='python')
        data.columns = ['T', 'Cp', 'S', '[G-H(Tr)]/T', 'H-H(Tr)', 'Delta_fH',
                        'Delta_fG', 'log(Kf)']
        self.rawdata = data

        # Sometimes the JANAF files have funky stuff written in them. (Old
        # school text format...)
        # Clean it up.
        for c in data.columns:
            # We only need to polish up columns that aren't floating point
            # numbers.
            if np.issubdtype(data.dtypes[c], np.floating):
                continue
            # Change INFINITE to inf
            data.loc[data[c] == 'INFINITE', c]
            # Anything else becomes a nan.
            # Convert to floats.
            data[c] = pd.to_numeric(data[c], errors='coerce')

        # Convert all units to Joules.
        data['Delta_fH'] *= 1000
        data['Delta_fG'] *= 1000

        # Now make interpolatable functions for each of these.
        self.cp = interp1d(self.rawdata['T'], self.rawdata['Cp'])
        self.S = interp1d(self.rawdata['T'], self.rawdata['S'])
        self.gef = interp1d(self.rawdata['T'], self.rawdata['[G-H(Tr)]/T'])
        self.hef = interp1d(self.rawdata['T'], self.rawdata['H-H(Tr)'])
        self.DeltaH = interp1d(self.rawdata['T'], self.rawdata['Delta_fH'])
        self.DeltaG = interp1d(self.rawdata['T'], self.rawdata['Delta_fG'])
        self.logKf = interp1d(self.rawdata['T'], self.rawdata['log(Kf)'])

        # TODO Deal well with crystal<->liquid transitions which have a below
        # and above value for Cp, S, etc.

class Janafdb(object):
    """
    Class that reads the NIST JANAF tables for thermodynamic data.

    Data is initially read from the web servers, and then cached.
    """
    VALIDPHASETYPES = ['cr', 'l', 'cr,l', 'g', 'ref', 'cd', 'fl', 'am', 'vit',
                       'mon', 'pol', 'sln', 'aq', 'sat']
    JANAF_URL = "http://kinetics.nist.gov/janaf/html/%s.txt"

    def __init__(self):
        """
        We have an index file which can be used to build the url for all phases
        on the NIST site.
        """

        # Read the index file which tells us the filenames for all the phases
        # in the JANAF database.
        dirname = os.path.dirname(__file__)
        janaf_index = os.path.join(dirname, 'JANAF_index.txt')
        self.db = pd.read_csv(janaf_index, delimiter='|')
        # Name the columns and trim whitespace off the text fields.
        self.db.columns = ['formula', 'name', 'phase', 'filename']
        self.db["formula"] = self.db["formula"].map(str.strip)
        self.db["name"] = self.db["name"].map(str.strip)
        self.db["phase"] = self.db["phase"].map(str.strip)
        self.db["filename"] = self.db["filename"].map(str.strip)

        # Make sure that the directory for cached JANAF files exists.
        self.JANAF_cachedir = os.path.join(dirname, 'JANAF_Cache')
        if not os.path.exists(self.JANAF_cachedir):
            os.mkdir(self.JANAF_cachedir)

    def search(self, searchstr):
        """
        List all the species containing a string. Helpful for
        interactive use of the database.
        returns a pandas dataframe containing valid phases.

        >>> db = Janafdb()
        >>> s = db.search('Rb-')
        >>> print(s)
             formula           name phase filename
        1710     Rb-  Rubidium, Ion     g   Rb-007
        >>> s = db.search('Ti')
        >>> print(len(s))
        88
        """

        formulasearch = self.db['formula'].str.contains(searchstr)
        namesearch = self.db['name'].str.contains(searchstr)

        return self.db[formulasearch | namesearch]

    def getphasedata(self, formula=None, name=None, phase=None, cache=True):
        """
        Returns an element instance given the name of the element.
        formula, name and phase match the respective fields in the JANAF index.
        cache = False means that we will always get the data from the web.

        >>> db = Janafdb()
        >>> db.getphasedata(formula='O2Ti', phase='cr')
        Traceback (most recent call last):
            ...
        ValueError: There are 2 records matching this pattern.
        >>> db.getphasedata(formula='Oxyz')
        Traceback (most recent call last):
            ...
        ValueError: Valid phase types are ['cr', 'l', 'cr,l', 'g', 'ref', 'cd', 'fl', 'am', 'vit', 'mon', 'pol', 'sln', 'aq', 'sat'].
        >>> db.getphasedata(formula='Oxyz', phase='l')
        Traceback (most recent call last):
            ...
        ValueError: Did not find Oxyz, None, (l)
        """

        # Check that the phase type requested is valid.
        if phase not in Janafdb.VALIDPHASETYPES:
            raise ValueError("Valid phase types are " + str(
                Janafdb.VALIDPHASETYPES) + ".")

        # We can search on either an exact formula, partial text match in the
        # name, and exact phase type.
        formulasearch = pd.Series(np.ones(len(self.db)), dtype=bool)
        namesearch = formulasearch.copy()
        phasesearch = formulasearch.copy()
        if formula is not None:
            formulasearch = self.db['formula'] == formula
        if name is not None:
            namesearch = self.db['name'].str.contains(name)
        if phase is not None:
            phasesearch = self.db['phase'] == phase
        searchmatch = formulasearch & namesearch & phasesearch

        # Get the record (should be one record) which specifies this phase.
        PhaseRecord = self.db[searchmatch]
        if len(PhaseRecord) == 0:
            raise ValueError("Did not find %s, %s, (%s)" %
                             (formula, name, phase))
        if len(PhaseRecord) > 1:
            raise ValueError("There are %d records matching this pattern." %
                             len(PhaseRecord))

        # At this point we have one record.  Check if we have that file cached.
        cachedfilename = os.path.join(
            self.JANAF_cachedir,
            "%s.txt" % PhaseRecord['filename'].values[0]
        )
        if cache and os.path.exists(cachedfilename):
            # Yes it was cached, so let's read it into memory.
            with open(cachedfilename, 'r') as f:
                textdata = f.read()
        else:
            # No it was not cached so let's get it from the web.
            response = urllib2.urlopen(Janafdb.JANAF_URL %
                                       PhaseRecord['filename'].values[0])
            textdata = response.read()

            # And cache the data so we aren't making unnecessary trips to the
            # web.
            if cache:
                with open(cachedfilename, 'w') as f:
                    f.write(textdata)

        # Create a phase class and return it.
        return JanafPhase(textdata)

    # def getmixturedata(self, components):
    #     """
    #     Creates a mixture of components given a list of tuples
    #     containing the formula and the volume percent
    #     """
    #     mixture = Mixture()
    #     for comp in components:
    #         mixture.add(self.getelementdata(comp[0]), comp[1])
    #
    #     return mixture
<|endoftext|>


<commit_before>#
# ========================================================
# Copyright (c) 2012 Whamcloud, Inc.  All rights reserved.
# ========================================================

from collections import defaultdict
from tastypie.authorization import DjangoAuthorization
from tastypie.resources import Resource
from tastypie import fields
from tastypie.validation import Validation
from chroma_api import api_log
from chroma_api.authentication import AnonymousAuthentication
from chroma_core.models.filesystem import ManagedFilesystem
from chroma_core.models.host import ManagedHost, VolumeNode
from chroma_core.models.target import ManagedMgs, ManagedMdt, ManagedOst, ManagedTargetMount
import chroma_core.lib.conf_param

class ConfigurationValidation(Validation):
    def is_valid(self, bundle, request = None):
        try:
            mgts = bundle.data['mgts']
        except KeyError:
            return {'mgts': ['This attribute is mandatory']}

        def validate_target(target):
            try:
                mounts = target['mounts']
            except KeyError:
                return ["'mounts' attribute mandatory for targets"]

            errors = []

            def validate_mount(mount):
                try:
                    mount['primary']
                except KeyError:
                    errors.append("'primary' attribute is mandatory for target mounts")
                else:
                    try:
                        host = ManagedHost.objects.get(address = mount['host'])
                    except KeyError:
                        errors.append("'host' attribute is mandatory for target mounts")
                    except ManagedHost.DoesNotExist:
                        errors.append("Host with address '%s' does not exist" % mount['host'])
                    else:
                        try:
                            VolumeNode.objects.get(path = mount['path'], host = host)
                        except KeyError:
                            errors.append("'path' attribute is mandatory for target mounts")
                        except VolumeNode.DoesNotExist:
                            errors.append("Device node '%s' on host '%s' not found" % (mount['path'], host))

                return errors

            for mount_data in mounts:
                validate_mount(mount_data)

            return errors

        def validate_filesystem(fs_data):
            errors = []
            if not 'name' in fs_data:
                errors.append("'name' attribute is mandatory for filesystems")

            def validate_target_list(name):
                try:
                    target_list = fs_data[name]
                except KeyError:
                    errors.append("'%s' attribute is mandatory for filesystems" % name)
                else:
                    for target_data in target_list:
                        errors.extend(validate_target(target_data))

            validate_target_list('mdts')
            validate_target_list('osts')

            return errors

        errors = defaultdict(list)
        for mgt_data in mgts:
            mgt_errors = validate_target(mgt_data)
            if not mgt_errors:
                try:
                    filesystems = mgt_data['filesystems']
                except KeyError:
                    mgt_errors.append("'filesystem' attribute mandatory for MGT")
                else:
                    for fs_data in filesystems:
                        mgt_errors.extend(validate_filesystem(fs_data))
            if mgt_errors:
                errors['mgts'].extend(mgt_errors)

        return errors

class Configuration(object):
    pass

class ConfigurationResource(Resource):
    """
    By default this will give a configuration dump that is suitable
    for re-creating a Chroma server managing the same targets that
    are managed by this Chroma server.

    Pass the 'template=True' as a parameter to a GET in order to receive
    a template suitable for creating a new filesystem on identically named
    hosts/devices.
    """
    mgts = fields.ListField()
    hosts = fields.ListField()

    class Meta:
        resource_name = 'configuration'
        object_class = Configuration
        authorization = DjangoAuthorization()
        authentication = AnonymousAuthentication()
        list_allowed_methods = ['get', 'post']
        get_allowed_methods = []
        validation = ConfigurationValidation()

    def dehydrate_hosts(self, bundle):
        return [{'address': h.address} for h in ManagedHost.objects.all()]

    def _dehydrate_target(self, target):
        data = {'mounts': []}
        for tm in target.managedtargetmount_set.all():
            data['mounts'].append({
                "host": tm.host.address,
                "path": tm.volume_node.path,
                "primary": tm.primary
            })

        if target.state in ['unmounted', 'mounted']:
            data['state'] = target.state
            data['uuid'] = target.uuid
            data['name'] = target.name
            data['inode_size'] = target.inode_size
            data['bytes_per_inode'] = target.bytes_per_inode
            data['inode_count'] = target.inode_count
            data['ha_label'] = target.ha_label
            data['immutable_state'] = target.immutable_state
            if not isinstance(target, ManagedMgs):
                data['conf_params'] = chroma_core.lib.conf_param.get_conf_params(target)

        return data

    def dehydrate_mgts(self, bundle):
        mgts = []
        for mgt in ManagedMgs.objects.all():
            mgt = self._dehydrate_target(mgt)
            mgt['filesystems'] = []
            for fs in ManagedFilesystem.objects.all():
                filesystem = {}
                filesystem['name'] = fs.name
                filesystem['state'] = fs.state
                filesystem['conf_params'] = chroma_core.lib.conf_param.get_conf_params(fs)
                filesystem['mdts'] = [self._dehydrate_target(mdt) for mdt in ManagedMdt.objects.filter(filesystem = fs)]
                filesystem['osts'] = [self._dehydrate_target(ost) for ost in ManagedOst.objects.filter(filesystem = fs)]
                filesystem['immutable_state'] = fs.immutable_state
                mgt['filesystems'].append(filesystem)
            mgts.append(mgt)

        return mgts

    def get_list(self, request = None, **kwargs):
        bundle = self.build_bundle(request = request)
        bundle = self.full_dehydrate(bundle)
        return self.create_response(request, bundle)

    def get_resource_uri(self, bundle_or_obj):
        kwargs = {
            'resource_name': self._meta.resource_name,
            'api_name': self._meta.api_name
            }

        return self._build_reverse_url('api_dispatch_list', kwargs = kwargs)

    def _load_target(self, target_data, klass, **kwargs):
        def get_node(mount):
            host_address = mount['host']
            path = mount['path']
            host = ManagedHost.objects.get(address = host_address)
            return VolumeNode.objects.get(host = host, path = path)

        volume = None
        for mount in target_data['mounts']:
            volume_node = get_node(mount)
            if not volume:
                volume = volume_node.volume
            else:
                if volume != volume_node.volume:
                    raise RuntimeError("Inconsistent volume paths for target %s: %s" %
                                       (target_data['name'], [m['path'] for m in target_data['mounts']]))

        mounts = target_data.pop('mounts')
        for k, v in target_data.items():
            if k != 'mounts' and k != 'filesystems' and k != 'conf_params':
                kwargs[k] = v
        kwargs['volume'] = volume

        try:
            target = klass.objects.get(volume = volume)
            # TODO: in validation stage, check attrs of this target
            # against attrs in input
        except klass.DoesNotExist:
            api_log.info("ConfigurationResource creating target %s" % target_data['name'])
            target = klass.objects.create(**kwargs)

        for mount in mounts:
            volume_node = get_node(mount)
            try:
                ManagedTargetMount.objects.get(
                    target = target,
                    volume_node = volume_node
                )
            except ManagedTargetMount.DoesNotExist:
                api_log.info("ConfigurationResource creating target mount %s:%s:%s" %
                             (target.name, volume_node.host, volume_node.path))
                ManagedTargetMount.objects.create(
                    target = target,
                    volume_node = volume_node,
                    host = volume_node.host,
                    mount_point = volume_node.path,
                    primary = mount['primary']
                )

        if 'conf_params' in target_data:
            conf_params = dict([(k, v) for k, v in target_data['conf_params'].items() if v is not None])
            chroma_core.lib.conf_param.set_conf_params(target, conf_params, new = False)

        return target

    def obj_create(self, bundle, request = None):
        for mgt_data in bundle.data['mgts']:
            mgt = self._load_target(mgt_data, ManagedMgs)
            for fs_data in mgt_data['filesystems']:
                filesystem, created = ManagedFilesystem.objects.get_or_create(mgs = mgt, name = fs_data['name'])
                if created:
                    api_log.info("ConfigurationResource: created filesystem %s" % filesystem)
                conf_params = dict([(k, v) for k, v in fs_data['conf_params'].items() if v is not None])
                chroma_core.lib.conf_param.set_conf_params(filesystem, conf_params, new = False)

                for ost_data in fs_data['osts']:
                    self._load_target(ost_data, ManagedOst, filesystem = filesystem)
                for mdt_data in fs_data['mdts']:
                    self._load_target(mdt_data, ManagedMdt, filesystem = filesystem)

        return bundle
<commit_msg>Fix detail_allowed_methods of ConfigurationResource<commit_after>#
# ========================================================
# Copyright (c) 2012 Whamcloud, Inc.  All rights reserved.
# ========================================================

from collections import defaultdict
from tastypie.authorization import DjangoAuthorization
from tastypie.resources import Resource
from tastypie import fields
from tastypie.validation import Validation
from chroma_api import api_log
from chroma_api.authentication import AnonymousAuthentication
from chroma_core.models.filesystem import ManagedFilesystem
from chroma_core.models.host import ManagedHost, VolumeNode
from chroma_core.models.target import ManagedMgs, ManagedMdt, ManagedOst, ManagedTargetMount
import chroma_core.lib.conf_param

class ConfigurationValidation(Validation):
    def is_valid(self, bundle, request = None):
        try:
            mgts = bundle.data['mgts']
        except KeyError:
            return {'mgts': ['This attribute is mandatory']}

        def validate_target(target):
            try:
                mounts = target['mounts']
            except KeyError:
                return ["'mounts' attribute mandatory for targets"]

            errors = []

            def validate_mount(mount):
                try:
                    mount['primary']
                except KeyError:
                    errors.append("'primary' attribute is mandatory for target mounts")
                else:
                    try:
                        host = ManagedHost.objects.get(address = mount['host'])
                    except KeyError:
                        errors.append("'host' attribute is mandatory for target mounts")
                    except ManagedHost.DoesNotExist:
                        errors.append("Host with address '%s' does not exist" % mount['host'])
                    else:
                        try:
                            VolumeNode.objects.get(path = mount['path'], host = host)
                        except KeyError:
                            errors.append("'path' attribute is mandatory for target mounts")
                        except VolumeNode.DoesNotExist:
                            errors.append("Device node '%s' on host '%s' not found" % (mount['path'], host))

                return errors

            for mount_data in mounts:
                validate_mount(mount_data)

            return errors

        def validate_filesystem(fs_data):
            errors = []
            if not 'name' in fs_data:
                errors.append("'name' attribute is mandatory for filesystems")

            def validate_target_list(name):
                try:
                    target_list = fs_data[name]
                except KeyError:
                    errors.append("'%s' attribute is mandatory for filesystems" % name)
                else:
                    for target_data in target_list:
                        errors.extend(validate_target(target_data))

            validate_target_list('mdts')
            validate_target_list('osts')

            return errors

        errors = defaultdict(list)
        for mgt_data in mgts:
            mgt_errors = validate_target(mgt_data)
            if not mgt_errors:
                try:
                    filesystems = mgt_data['filesystems']
                except KeyError:
                    mgt_errors.append("'filesystem' attribute mandatory for MGT")
                else:
                    for fs_data in filesystems:
                        mgt_errors.extend(validate_filesystem(fs_data))
            if mgt_errors:
                errors['mgts'].extend(mgt_errors)

        return errors

class Configuration(object):
    pass

class ConfigurationResource(Resource):
    """
    By default this will give a configuration dump that is suitable
    for re-creating a Chroma server managing the same targets that
    are managed by this Chroma server.

    Pass the 'template=True' as a parameter to a GET in order to receive
    a template suitable for creating a new filesystem on identically named
    hosts/devices.
    """
    mgts = fields.ListField()
    hosts = fields.ListField()

    class Meta:
        resource_name = 'configuration'
        object_class = Configuration
        authorization = DjangoAuthorization()
        authentication = AnonymousAuthentication()
        list_allowed_methods = ['get', 'post']
        detail_allowed_methods = []
        validation = ConfigurationValidation()

    def dehydrate_hosts(self, bundle):
        return [{'address': h.address} for h in ManagedHost.objects.all()]

    def _dehydrate_target(self, target):
        data = {'mounts': []}
        for tm in target.managedtargetmount_set.all():
            data['mounts'].append({
                "host": tm.host.address,
                "path": tm.volume_node.path,
                "primary": tm.primary
            })

        if target.state in ['unmounted', 'mounted']:
            data['state'] = target.state
            data['uuid'] = target.uuid
            data['name'] = target.name
            data['inode_size'] = target.inode_size
            data['bytes_per_inode'] = target.bytes_per_inode
            data['inode_count'] = target.inode_count
            data['ha_label'] = target.ha_label
            data['immutable_state'] = target.immutable_state
            if not isinstance(target, ManagedMgs):
                data['conf_params'] = chroma_core.lib.conf_param.get_conf_params(target)

        return data

    def dehydrate_mgts(self, bundle):
        mgts = []
        for mgt in ManagedMgs.objects.all():
            mgt = self._dehydrate_target(mgt)
            mgt['filesystems'] = []
            for fs in ManagedFilesystem.objects.all():
                filesystem = {}
                filesystem['name'] = fs.name
                filesystem['state'] = fs.state
                filesystem['conf_params'] = chroma_core.lib.conf_param.get_conf_params(fs)
                filesystem['mdts'] = [self._dehydrate_target(mdt) for mdt in ManagedMdt.objects.filter(filesystem = fs)]
                filesystem['osts'] = [self._dehydrate_target(ost) for ost in ManagedOst.objects.filter(filesystem = fs)]
                filesystem['immutable_state'] = fs.immutable_state
                mgt['filesystems'].append(filesystem)
            mgts.append(mgt)

        return mgts

    def get_list(self, request = None, **kwargs):
        bundle = self.build_bundle(request = request)
        bundle = self.full_dehydrate(bundle)
        return self.create_response(request, bundle)

    def get_resource_uri(self, bundle_or_obj):
        kwargs = {
            'resource_name': self._meta.resource_name,
            'api_name': self._meta.api_name
            }

        return self._build_reverse_url('api_dispatch_list', kwargs = kwargs)

    def _load_target(self, target_data, klass, **kwargs):
        def get_node(mount):
            host_address = mount['host']
            path = mount['path']
            host = ManagedHost.objects.get(address = host_address)
            return VolumeNode.objects.get(host = host, path = path)

        volume = None
        for mount in target_data['mounts']:
            volume_node = get_node(mount)
            if not volume:
                volume = volume_node.volume
            else:
                if volume != volume_node.volume:
                    raise RuntimeError("Inconsistent volume paths for target %s: %s" %
                                       (target_data['name'], [m['path'] for m in target_data['mounts']]))

        mounts = target_data.pop('mounts')
        for k, v in target_data.items():
            if k != 'mounts' and k != 'filesystems' and k != 'conf_params':
                kwargs[k] = v
        kwargs['volume'] = volume

        try:
            target = klass.objects.get(volume = volume)
            # TODO: in validation stage, check attrs of this target
            # against attrs in input
        except klass.DoesNotExist:
            api_log.info("ConfigurationResource creating target %s" % target_data['name'])
            target = klass.objects.create(**kwargs)

        for mount in mounts:
            volume_node = get_node(mount)
            try:
                ManagedTargetMount.objects.get(
                    target = target,
                    volume_node = volume_node
                )
            except ManagedTargetMount.DoesNotExist:
                api_log.info("ConfigurationResource creating target mount %s:%s:%s" %
                             (target.name, volume_node.host, volume_node.path))
                ManagedTargetMount.objects.create(
                    target = target,
                    volume_node = volume_node,
                    host = volume_node.host,
                    mount_point = volume_node.path,
                    primary = mount['primary']
                )

        if 'conf_params' in target_data:
            conf_params = dict([(k, v) for k, v in target_data['conf_params'].items() if v is not None])
            chroma_core.lib.conf_param.set_conf_params(target, conf_params, new = False)

        return target

    def obj_create(self, bundle, request = None):
        for mgt_data in bundle.data['mgts']:
            mgt = self._load_target(mgt_data, ManagedMgs)
            for fs_data in mgt_data['filesystems']:
                filesystem, created = ManagedFilesystem.objects.get_or_create(mgs = mgt, name = fs_data['name'])
                if created:
                    api_log.info("ConfigurationResource: created filesystem %s" % filesystem)
                conf_params = dict([(k, v) for k, v in fs_data['conf_params'].items() if v is not None])
                chroma_core.lib.conf_param.set_conf_params(filesystem, conf_params, new = False)

                for ost_data in fs_data['osts']:
                    self._load_target(ost_data, ManagedOst, filesystem = filesystem)
                for mdt_data in fs_data['mdts']:
                    self._load_target(mdt_data, ManagedMdt, filesystem = filesystem)

        return bundle
<|endoftext|>


<commit_before>import datetime

try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.db.models import Q, Avg, Count, Max, Min, Sum

from thing.models import *
from thing.stuff import *
from thing.templatetags.thing_extras import commas, duration, shortduration

# ---------------------------------------------------------------------------

ONE_DAY = 24 * 60 * 60
EXPIRE_WARNING = 10 * ONE_DAY

# ---------------------------------------------------------------------------
# Home page
@login_required
def home(request):
    tt = TimerThing('home')

    # Create the user's profile if it doesn't already exist
    try:
        profile = request.user.get_profile()
    except UserProfile.DoesNotExist:
        profile = UserProfile(user=request.user)
        profile.save()

    tt.add_time('profile')

    # Make a set of characters to hide
    hide_characters = set(int(c) for c in profile.home_hide_characters.split(',') if c)

    # Initialise various data structures
    now = datetime.datetime.utcnow()
    total_balance = 0

    api_keys = set()
    training = set()
    chars = {}
    ship_item_ids = set()

    # Try retrieving characters from cache
    cache_key = 'home:characters:%d' % (request.user.id)
    characters = cache.get(cache_key)
    # Not cached, fetch from database and cache
    if characters is None:
        characters = Character.objects.filter(apikeys__user=request.user, apikeys__key_type__in=(APIKey.ACCOUNT_TYPE, APIKey.CHARACTER_TYPE))
        characters = characters.prefetch_related('apikeys')
        characters = characters.select_related('config', 'details')
        characters = characters.annotate(total_sp=Sum('characterskill__points'))
        characters = characters.distinct()
        characters = [c for c in characters if c.details is not None]
        cache.set(cache_key, characters, 300)

    for character in characters:
        char_keys = [ak for ak in character.apikeys.all() if ak.user_id == request.user.id]
        api_keys.update(char_keys)

        chars[character.id] = character
        character.z_apikey = char_keys[0]
        character.z_training = {}

        total_balance += character.details.wallet_balance
        if character.details.ship_item_id is not None:
            ship_item_ids.add(character.details.ship_item_id)

    tt.add_time('characters')

    # Try retrieving ship map from cache
    cache_key = 'home:ship_map:%d' % (request.user.id)
    ship_map = cache.get(cache_key)
    # Not cached, fetch from database and cache
    if ship_map is None:
        ship_map = Item.objects.in_bulk(ship_item_ids)
        cache.set(cache_key, ship_map, 300)

    tt.add_time('ship_items')

    # Do skill training check - this can't be in the model because it
    # scales like crap doing individual queries
    skill_qs = []

    queues = SkillQueue.objects.filter(character__in=chars, end_time__gte=now)
    queues = queues.select_related('character__details', 'skill__item')
    for sq in queues:
        char = chars[sq.character_id]
        if 'sq' not in char.z_training:
            char.z_training['sq'] = sq
            char.z_training['skill_duration'] = total_seconds(sq.end_time - now)
            # if char.details:
            char.z_training['sp_per_hour'] = int(sq.skill.get_sp_per_minute(char) * 60)
            char.z_training['complete_per'] = sq.get_complete_percentage(now)
            training.add(char.z_apikey)

            skill_qs.append(Q(character=char, skill=sq.skill))

        char.z_training['queue_duration'] = total_seconds(sq.end_time - now)

    tt.add_time('training')

    # Retrieve training skill information
    if skill_qs:
        for cs in CharacterSkill.objects.filter(reduce(q_reduce_or, skill_qs)):
            chars[cs.character_id].z_tskill = cs

    tt.add_time('training skills')

    # Do total skill point aggregation
    total_sp = 0
    for char in characters:
        char.z_total_sp = char.total_sp
        if 'sq' in char.z_training and hasattr(char, 'z_tskill'):
            char.z_total_sp += int(char.z_training['sq'].get_completed_sp(char.z_tskill, now))

        total_sp += char.z_total_sp

    tt.add_time('total_sp')

    # Work out who is and isn't training
    not_training = api_keys - training

    # Do notifications
    for char_id, char in chars.items():
        char.z_notifications = []

        # Game time warnings
        if char.z_apikey.paid_until:
            timediff = total_seconds(char.z_apikey.paid_until - now)

            if timediff < 0:
                char.z_notifications.append({
                    'icon': 'time',
                    'text': 'Expired',
                    'tooltip': 'Game time has expired!',
                    'span_class': 'low-game-time',
                })

            elif timediff < EXPIRE_WARNING:
                char.z_notifications.append({
                    'icon': 'time',
                    'text': shortduration(timediff),
                    'tooltip': 'Remaining game time is low!',
                    'span_class': 'low-game-time',
                })

        # API key warnings
        if char.z_apikey.expires:
            timediff = total_seconds(char.z_apikey.expires - now)
            if timediff < EXPIRE_WARNING:
                char.z_notifications.append({
                    'icon': 'key',
                    'text': shortduration(timediff),
                    'tooltip': 'API key is close to expiring!',
                })

        # Empty skill queue
        if char.z_apikey in not_training:
            char.z_notifications.append({
                'icon': 'list-ol',
                'text': 'Empty!',
                'tooltip': 'Skill queue is empty!',
            })

        if char.z_training:
            # Room in skill queue
            if char.z_training['queue_duration'] < ONE_DAY:
                timediff = ONE_DAY - char.z_training['queue_duration']
                char.z_notifications.append({
                    'icon': 'list-ol',
                    'text': shortduration(timediff),
                    'tooltip': 'Skill queue is not full!',
                })

            # Missing implants
            skill = char.z_training['sq'].skill
            pri_attrs = Skill.ATTRIBUTE_MAP[skill.primary_attribute]
            sec_attrs = Skill.ATTRIBUTE_MAP[skill.secondary_attribute]
            pri_bonus = getattr(char.details, pri_attrs[1])
            sec_bonus = getattr(char.details, sec_attrs[1])

            if pri_bonus == 0 or sec_bonus == 0:
                t = []
                if pri_bonus == 0:
                    t.append(skill.get_primary_attribute_display())
                if sec_bonus == 0:
                    t.append(skill.get_secondary_attribute_display())

                char.z_notifications.append({
                    'icon': 'lightbulb',
                    'text': ', '.join(t),
                    'tooltip': 'Missing stat implants for currently training skill!',
                })

        # Insufficient clone
        if hasattr(char, 'z_total_sp') and char.z_total_sp > char.details.clone_skill_points:
            char.z_notifications.append({
                'icon': 'user-md',
                'text': '%s' % (commas(char.details.clone_skill_points)),
                'tooltip': 'Insufficient clone!',
            })

    tt.add_time('notifications')

    # Work out sort order
    char_list = chars.values()
    if profile.home_sort_order == 'apiname':
        temp = [(c.z_apikey.name, c.name.lower(), c) for c in char_list]
    elif profile.home_sort_order == 'charname':
        temp = [(c.name.lower(), c) for c in char_list]
    elif profile.home_sort_order == 'corpname':
        temp = [(c.corporation.name.lower(), c.name.lower(), c) for c in char_list]
    elif profile.home_sort_order == 'totalsp':
        temp = [(getattr(c, 'z_total_sp', 0), c) for c in char_list]
    elif profile.home_sort_order == 'wallet':
        temp = [(c.details and c.details.wallet_balance, c.name.lower(), c) for c in char_list]

    temp.sort()
    if profile.home_sort_descending:
        temp.reverse()

    char_list = [s[-1] for s in temp]

    tt.add_time('sort')

    # Make separate lists of training and not training characters
    first = [char for char in char_list if char.z_training and char.id not in hide_characters]
    last = [char for char in char_list if not char.z_training and char.id not in hide_characters]

    tt.add_time('training')

    # Try retrieving corporations from cache
    cache_key = 'home:corporations:%d' % (request.user.id)
    corporations = cache.get(cache_key)
    # Not cached, fetch from database and cache
    if corporations is None:
        corp_ids = APIKey.objects.filter(user=request.user.id).exclude(corp_character=None).values('corp_character__corporation')
        corp_map = OrderedDict()
        for corp_wallet in CorpWallet.objects.select_related().filter(corporation__in=corp_ids):
            if corp_wallet.corporation_id not in corp_map:
                corp_map[corp_wallet.corporation_id] = corp_wallet.corporation
                corp_map[corp_wallet.corporation_id].wallets = []

            corp_map[corp_wallet.corporation_id].wallets.append(corp_wallet)

        corporations = corp_map.values()
        cache.set(cache_key, corporations, 300)

    tt.add_time('corps')

    # Get old event stats for staff users
    if request.user.is_staff:
        task_count = TaskState.objects.filter(state=TaskState.QUEUED_STATE).aggregate(Count('id'))['id__count']
    else:
        task_count = 0

    tt.add_time('misc junk')

    out = render_page(
        'thing/home.html',
        {
            'profile': profile,
            'not_training': not_training,
            'total_balance': total_balance,
            'total_sp': total_sp,
            'corporations': corporations,
            'characters': first + last,
            'events': list(Event.objects.filter(user=request.user)[:10]),
            'ship_map': ship_map,
            'task_count': task_count,
        },
        request,
        chars.keys(),
        [c.id for c in corporations]
    )

    tt.add_time('template')
    if settings.DEBUG:
        tt.finished()

    return out
<commit_msg>Try a fix for an intermittent home view error with characters that have no related CharacterSkill objects<commit_after>import datetime

try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.db.models import Q, Avg, Count, Max, Min, Sum

from thing.models import *
from thing.stuff import *
from thing.templatetags.thing_extras import commas, duration, shortduration

# ---------------------------------------------------------------------------

ONE_DAY = 24 * 60 * 60
EXPIRE_WARNING = 10 * ONE_DAY

# ---------------------------------------------------------------------------
# Home page
@login_required
def home(request):
    tt = TimerThing('home')

    # Create the user's profile if it doesn't already exist
    try:
        profile = request.user.get_profile()
    except UserProfile.DoesNotExist:
        profile = UserProfile(user=request.user)
        profile.save()

    tt.add_time('profile')

    # Make a set of characters to hide
    hide_characters = set(int(c) for c in profile.home_hide_characters.split(',') if c)

    # Initialise various data structures
    now = datetime.datetime.utcnow()
    total_balance = 0

    api_keys = set()
    training = set()
    chars = {}
    ship_item_ids = set()

    # Try retrieving characters from cache
    cache_key = 'home:characters:%d' % (request.user.id)
    characters = cache.get(cache_key)
    # Not cached, fetch from database and cache
    if characters is None:
        characters = Character.objects.filter(apikeys__user=request.user, apikeys__key_type__in=(APIKey.ACCOUNT_TYPE, APIKey.CHARACTER_TYPE))
        characters = characters.prefetch_related('apikeys')
        characters = characters.select_related('config', 'details')
        characters = characters.annotate(total_sp=Sum('characterskill__points'))
        characters = characters.distinct()
        characters = [c for c in characters if c.details is not None]
        cache.set(cache_key, characters, 300)

    for character in characters:
        char_keys = [ak for ak in character.apikeys.all() if ak.user_id == request.user.id]
        api_keys.update(char_keys)

        chars[character.id] = character
        character.z_apikey = char_keys[0]
        character.z_training = {}

        total_balance += character.details.wallet_balance
        if character.details.ship_item_id is not None:
            ship_item_ids.add(character.details.ship_item_id)

    tt.add_time('characters')

    # Try retrieving ship map from cache
    cache_key = 'home:ship_map:%d' % (request.user.id)
    ship_map = cache.get(cache_key)
    # Not cached, fetch from database and cache
    if ship_map is None:
        ship_map = Item.objects.in_bulk(ship_item_ids)
        cache.set(cache_key, ship_map, 300)

    tt.add_time('ship_items')

    # Do skill training check - this can't be in the model because it
    # scales like crap doing individual queries
    skill_qs = []

    queues = SkillQueue.objects.filter(character__in=chars, end_time__gte=now)
    queues = queues.select_related('character__details', 'skill__item')
    for sq in queues:
        char = chars[sq.character_id]
        if 'sq' not in char.z_training:
            char.z_training['sq'] = sq
            char.z_training['skill_duration'] = total_seconds(sq.end_time - now)
            # if char.details:
            char.z_training['sp_per_hour'] = int(sq.skill.get_sp_per_minute(char) * 60)
            char.z_training['complete_per'] = sq.get_complete_percentage(now)
            training.add(char.z_apikey)

            skill_qs.append(Q(character=char, skill=sq.skill))

        char.z_training['queue_duration'] = total_seconds(sq.end_time - now)

    tt.add_time('training')

    # Retrieve training skill information
    if skill_qs:
        for cs in CharacterSkill.objects.filter(reduce(q_reduce_or, skill_qs)):
            chars[cs.character_id].z_tskill = cs

    tt.add_time('training skills')

    # Do total skill point aggregation
    total_sp = 0
    for char in characters:
        char.z_total_sp = char.total_sp or 0
        if 'sq' in char.z_training and hasattr(char, 'z_tskill'):
            char.z_total_sp += int(char.z_training['sq'].get_completed_sp(char.z_tskill, now))

        total_sp += char.z_total_sp

    tt.add_time('total_sp')

    # Work out who is and isn't training
    not_training = api_keys - training

    # Do notifications
    for char_id, char in chars.items():
        char.z_notifications = []

        # Game time warnings
        if char.z_apikey.paid_until:
            timediff = total_seconds(char.z_apikey.paid_until - now)

            if timediff < 0:
                char.z_notifications.append({
                    'icon': 'time',
                    'text': 'Expired',
                    'tooltip': 'Game time has expired!',
                    'span_class': 'low-game-time',
                })

            elif timediff < EXPIRE_WARNING:
                char.z_notifications.append({
                    'icon': 'time',
                    'text': shortduration(timediff),
                    'tooltip': 'Remaining game time is low!',
                    'span_class': 'low-game-time',
                })

        # API key warnings
        if char.z_apikey.expires:
            timediff = total_seconds(char.z_apikey.expires - now)
            if timediff < EXPIRE_WARNING:
                char.z_notifications.append({
                    'icon': 'key',
                    'text': shortduration(timediff),
                    'tooltip': 'API key is close to expiring!',
                })

        # Empty skill queue
        if char.z_apikey in not_training:
            char.z_notifications.append({
                'icon': 'list-ol',
                'text': 'Empty!',
                'tooltip': 'Skill queue is empty!',
            })

        if char.z_training:
            # Room in skill queue
            if char.z_training['queue_duration'] < ONE_DAY:
                timediff = ONE_DAY - char.z_training['queue_duration']
                char.z_notifications.append({
                    'icon': 'list-ol',
                    'text': shortduration(timediff),
                    'tooltip': 'Skill queue is not full!',
                })

            # Missing implants
            skill = char.z_training['sq'].skill
            pri_attrs = Skill.ATTRIBUTE_MAP[skill.primary_attribute]
            sec_attrs = Skill.ATTRIBUTE_MAP[skill.secondary_attribute]
            pri_bonus = getattr(char.details, pri_attrs[1])
            sec_bonus = getattr(char.details, sec_attrs[1])

            if pri_bonus == 0 or sec_bonus == 0:
                t = []
                if pri_bonus == 0:
                    t.append(skill.get_primary_attribute_display())
                if sec_bonus == 0:
                    t.append(skill.get_secondary_attribute_display())

                char.z_notifications.append({
                    'icon': 'lightbulb',
                    'text': ', '.join(t),
                    'tooltip': 'Missing stat implants for currently training skill!',
                })

        # Insufficient clone
        if hasattr(char, 'z_total_sp') and char.z_total_sp > char.details.clone_skill_points:
            char.z_notifications.append({
                'icon': 'user-md',
                'text': '%s' % (commas(char.details.clone_skill_points)),
                'tooltip': 'Insufficient clone!',
            })

    tt.add_time('notifications')

    # Work out sort order
    char_list = chars.values()
    if profile.home_sort_order == 'apiname':
        temp = [(c.z_apikey.name, c.name.lower(), c) for c in char_list]
    elif profile.home_sort_order == 'charname':
        temp = [(c.name.lower(), c) for c in char_list]
    elif profile.home_sort_order == 'corpname':
        temp = [(c.corporation.name.lower(), c.name.lower(), c) for c in char_list]
    elif profile.home_sort_order == 'totalsp':
        temp = [(getattr(c, 'z_total_sp', 0), c) for c in char_list]
    elif profile.home_sort_order == 'wallet':
        temp = [(c.details and c.details.wallet_balance, c.name.lower(), c) for c in char_list]

    temp.sort()
    if profile.home_sort_descending:
        temp.reverse()

    char_list = [s[-1] for s in temp]

    tt.add_time('sort')

    # Make separate lists of training and not training characters
    first = [char for char in char_list if char.z_training and char.id not in hide_characters]
    last = [char for char in char_list if not char.z_training and char.id not in hide_characters]

    tt.add_time('training')

    # Try retrieving corporations from cache
    cache_key = 'home:corporations:%d' % (request.user.id)
    corporations = cache.get(cache_key)
    # Not cached, fetch from database and cache
    if corporations is None:
        corp_ids = APIKey.objects.filter(user=request.user.id).exclude(corp_character=None).values('corp_character__corporation')
        corp_map = OrderedDict()
        for corp_wallet in CorpWallet.objects.select_related().filter(corporation__in=corp_ids):
            if corp_wallet.corporation_id not in corp_map:
                corp_map[corp_wallet.corporation_id] = corp_wallet.corporation
                corp_map[corp_wallet.corporation_id].wallets = []

            corp_map[corp_wallet.corporation_id].wallets.append(corp_wallet)

        corporations = corp_map.values()
        cache.set(cache_key, corporations, 300)

    tt.add_time('corps')

    # Get old event stats for staff users
    if request.user.is_staff:
        task_count = TaskState.objects.filter(state=TaskState.QUEUED_STATE).aggregate(Count('id'))['id__count']
    else:
        task_count = 0

    tt.add_time('misc junk')

    out = render_page(
        'thing/home.html',
        {
            'profile': profile,
            'not_training': not_training,
            'total_balance': total_balance,
            'total_sp': total_sp,
            'corporations': corporations,
            'characters': first + last,
            'events': list(Event.objects.filter(user=request.user)[:10]),
            'ship_map': ship_map,
            'task_count': task_count,
        },
        request,
        chars.keys(),
        [c.id for c in corporations]
    )

    tt.add_time('template')
    if settings.DEBUG:
        tt.finished()

    return out
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
threetaps.client
~~~~~~~~~~~~~~~~

This module provides a Threetaps object to make requests to the 3taps
api.

"""

from __future__ import unicode_literals
import inspect
import json
import urllib2
from urllib import urlencode
from urlparse import urljoin
import calendar

class Threetaps(object):
    """A 3taps API wrapper."""

    __attrs__ = ['auth_token']

    def __init__(self, auth_token):

        # prepare endpoints
        self.requester = self.Requester(auth_token)

        # dynamically attach endpoints
        self._attach_endpoints()

    def _attach_endpoints(self):
        """Dynamically attach endpoints to this client."""

        for name, endpoint in inspect.getmembers(self):
            if (inspect.isclass(endpoint) and
                    issubclass(endpoint, self._Endpoint) and
                    endpoint is not self._Endpoint):
                endpoint_instance = endpoint(self.requester)
                setattr(self, endpoint.name, endpoint_instance)

    class Requester(object):
        """An API requesting object."""

        def __init__(self, auth_token):

            self.auth_token = auth_token

        def GET(self, url, params={}):
            """Make a GET request to 3taps.

            :param url: String. 3taps endpoint.
            :param params: Dictionary. Params to send to 3taps.
            """
            params['auth_token'] = self.auth_token
            full_url = '{}?{}'.format(url, urlencode(params))
            request = urllib2.Request(full_url)

            try:
                f = urllib2.urlopen(request)
                response = json.loads(f.read())
            except (urllib2.HTTPError, ValueError) as err:
                response = {'error': err}

            return response

    class _Endpoint(object):
        """Base class for endpoints."""

        def __init__(self, requester):

            self.requester = requester

        def _GET(self, path='', params={}):
            """Make a GET request to the current endpoint.

            :param path: String. Path to append to the endpoint url.
            :param params: Dictionary. Params to send to 3taps.
            """
            url = urljoin(self.url, path)
            return self.requester.GET(url, params)

    class Search(_Endpoint):
        """Base class for endpoints."""

        name = 'search'
        url = 'http://search.3taps.com'

        def search(self, params={}):
            """Search the 3taps database of postings."""

            return self._GET(self.url, params)

        def count(self, field, params={}):
            """Count the number of postings matching a search.

            :param field: String. Field to use for calculating the count.
            """
            params['count'] = field
            return self._GET(self.url, params)

    class Polling(_Endpoint):
        """3taps Polling API endpoints."""

        name = 'polling'
        url = 'http://polling.3taps.com'

        def anchor(self, utc_date):
            """Get the value to use to find postings since `utc_date`.

            :param dt: UTC Datetime <datetime> object. Start-time for finding
                postings.
            """
            url = urljoin(self.url, 'anchor')

            # set anchor timestamp
            params = {'timestamp': self._timestamp_from_utc(utc_date)}

            return self._GET(url, params)

        def poll(self, params={}):
            """Retrieve postings that have changed since the last poll."""

            url = urljoin(self.url, 'poll')
            return self._GET(url, params)

        def _timestamp_from_utc(self, utc_date):
            """Convert a UTC date object to a UTC timestamp.
            Returns an Integer representing the number of seconds since the
            epoch.
            """
            return calendar.timegm(utc_date.timetuple())

    class Reference(_Endpoint):
        """3taps Reference API endpoints."""

        name = 'reference'
        url = 'http://reference.3taps.com'

        def sources(self):
            """Get a list of 3taps data sources."""

            return self._GET('sources')

        def category_groups(self):
            """Get a list of 3taps category groups."""

            return self._GET('category_groups')

        def categories(self):
            """Get a list of 3taps categories."""

            return self._GET('categories')

        def locations(self, level, params={}):
            """Get a list of 3taps locations.

            :param level: String. The level of the desired locations.
                Options: 'country', 'state', 'metro', 'region', 'county',
                         'city', 'locality', 'zipcode'.
            """
            params['level'] = level
            return self._GET('locations', params)

        def location_lookup(self, code):
            """Get the details for a location based on the 3taps location
            code.

            :param code: String. 3taps location code.
            """
            return self._GET('location/lookup', {'code': code})
<commit_msg>use Python 3 compatible imports<commit_after># -*- coding: utf-8 -*-

"""
threetaps.client
~~~~~~~~~~~~~~~~

This module provides a Threetaps object to make requests to the 3taps
api.

"""

from __future__ import unicode_literals
import inspect
import json
import calendar
try:
    from urllib2 import Request, HTTPError, urlopen
    from urllib import urlencode
    from urlparse import urljoin
except ImportError:
    from urllib.request import Request, urlopen
    from urllib.parse import urlencode, urljoin
    from urllib.error import HTTPError

class Threetaps(object):
    """A 3taps API wrapper."""

    __attrs__ = ['auth_token']

    def __init__(self, auth_token):

        # prepare endpoints
        self.requester = self.Requester(auth_token)

        # dynamically attach endpoints
        self._attach_endpoints()

    def _attach_endpoints(self):
        """Dynamically attach endpoints to this client."""

        for name, endpoint in inspect.getmembers(self):
            if (inspect.isclass(endpoint) and
                    issubclass(endpoint, self._Endpoint) and
                    endpoint is not self._Endpoint):
                endpoint_instance = endpoint(self.requester)
                setattr(self, endpoint.name, endpoint_instance)

    class Requester(object):
        """An API requesting object."""

        def __init__(self, auth_token):

            self.auth_token = auth_token

        def GET(self, url, params={}):
            """Make a GET request to 3taps.

            :param url: String. 3taps endpoint.
            :param params: Dictionary. Params to send to 3taps.
            """
            params['auth_token'] = self.auth_token
            full_url = '{}?{}'.format(url, urlencode(params))
            request = Request(full_url)

            try:
                f = urlopen(request)
                response = json.loads(f.read())
            except (HTTPError, ValueError) as err:
                response = {'error': err}

            return response

    class _Endpoint(object):
        """Base class for endpoints."""

        def __init__(self, requester):

            self.requester = requester

        def _GET(self, path='', params={}):
            """Make a GET request to the current endpoint.

            :param path: String. Path to append to the endpoint url.
            :param params: Dictionary. Params to send to 3taps.
            """
            url = urljoin(self.url, path)
            return self.requester.GET(url, params)

    class Search(_Endpoint):
        """Base class for endpoints."""

        name = 'search'
        url = 'http://search.3taps.com'

        def search(self, params={}):
            """Search the 3taps database of postings."""

            return self._GET(self.url, params)

        def count(self, field, params={}):
            """Count the number of postings matching a search.

            :param field: String. Field to use for calculating the count.
            """
            params['count'] = field
            return self._GET(self.url, params)

    class Polling(_Endpoint):
        """3taps Polling API endpoints."""

        name = 'polling'
        url = 'http://polling.3taps.com'

        def anchor(self, utc_date):
            """Get the value to use to find postings since `utc_date`.

            :param dt: UTC Datetime <datetime> object. Start-time for finding
                postings.
            """
            url = urljoin(self.url, 'anchor')

            # set anchor timestamp
            params = {'timestamp': self._timestamp_from_utc(utc_date)}

            return self._GET(url, params)

        def poll(self, params={}):
            """Retrieve postings that have changed since the last poll."""

            url = urljoin(self.url, 'poll')
            return self._GET(url, params)

        def _timestamp_from_utc(self, utc_date):
            """Convert a UTC date object to a UTC timestamp.
            Returns an Integer representing the number of seconds since the
            epoch.
            """
            return calendar.timegm(utc_date.timetuple())

    class Reference(_Endpoint):
        """3taps Reference API endpoints."""

        name = 'reference'
        url = 'http://reference.3taps.com'

        def sources(self):
            """Get a list of 3taps data sources."""

            return self._GET('sources')

        def category_groups(self):
            """Get a list of 3taps category groups."""

            return self._GET('category_groups')

        def categories(self):
            """Get a list of 3taps categories."""

            return self._GET('categories')

        def locations(self, level, params={}):
            """Get a list of 3taps locations.

            :param level: String. The level of the desired locations.
                Options: 'country', 'state', 'metro', 'region', 'county',
                         'city', 'locality', 'zipcode'.
            """
            params['level'] = level
            return self._GET('locations', params)

        def location_lookup(self, code):
            """Get the details for a location based on the 3taps location
            code.

            :param code: String. 3taps location code.
            """
            return self._GET('location/lookup', {'code': code})
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# thumbor imaging service
# https://github.com/globocom/thumbor/wiki

# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license
# Copyright (c) 2011 globo.com timehome@corp.globo.com

'''This is the main module in thumbor'''

__version__ = "2.6.12"
<commit_msg>Release 2.7.0<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# thumbor imaging service
# https://github.com/globocom/thumbor/wiki

# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license
# Copyright (c) 2011 globo.com timehome@corp.globo.com

'''This is the main module in thumbor'''

__version__ = "2.7.0"
<|endoftext|>


<commit_before>from ticketing import conf
from django.db import models

class BigAutoField(models.AutoField):

    def db_type(self, *args, **kwargs):
        connection = kwargs.get('connection', None)
        if connection and connection.settings_dict['ENGINE'] == 'django.db.backends.mysql':
            return 'bigint UNSIGNED AUTO_INCREMENT'
        else:
            return super(BigAutoField, self).db_type(*args, **kwargs)

class TicketField(models.BigIntegerField):

    description = "A big integer pre-populated from a ticket"

    def __init__(self, sequence=None, **kwargs):
        """
        Set up this field with the name of the sequence to use
        """
        if sequence:
            self.sequence = sequence
        else:
            # Use the default sequence
            self.sequence = conf.DEFAULT_SEQUENCE
        kwargs['editable'] = False
        super(TicketField, self).__init__(**kwargs)

    def pre_save(self, model_instance, add):
        """
        When saving the DB, if we are adding this instance, get a ticket
        value for this field. Otherwise pull it from the model instance.
        """
        if add:
            from ticketing.models import get_ticket
            value = get_ticket(self.sequence)
            setattr(model_instance, self.attname, value)
            return value
        else:
            return super(TicketField, self).pre_save(model_instance, add)

    def db_type(self, *args, **kwargs):
        connection = kwargs.get('connection', None)
        if connection and connection.settings_dict['ENGINE'] == 'django.db.backends.mysql':
            return 'bigint UNSIGNED'
        else:
            return super(BigAutoField, self).db_type(*args, **kwargs)

    def south_field_triple(self):
        "Returns a suitable description of this field for South."
        # We'll just introspect the _actual_ field.
        from south.modelsinspector import introspector
        field_class = 'ticketing.models.TicketField'
        args, kwargs = introspector(self)
        # That's our definition!
        return (field_class, args, kwargs)

try:
    from south.modelsinspector import add_introspection_rules
    add_introspection_rules(rules=[
        (
            [TicketField], # Class(es) these apply to
            [],         # Positional arguments (not used)
            {           # Keyword argument
                "sequence": ["sequence", {"default": None}],
            },
        ),
    ], patterns=["^ticketing\.fields\.TicketField"])
except ImportError:
    pass
<commit_msg>Fix a typo.<commit_after>from ticketing import conf
from django.db import models

class BigAutoField(models.AutoField):

    def db_type(self, *args, **kwargs):
        connection = kwargs.get('connection', None)
        if connection and connection.settings_dict['ENGINE'] == 'django.db.backends.mysql':
            return 'bigint UNSIGNED AUTO_INCREMENT'
        else:
            return super(BigAutoField, self).db_type(*args, **kwargs)

class TicketField(models.BigIntegerField):

    description = "A big integer pre-populated from a ticket"

    def __init__(self, sequence=None, **kwargs):
        """
        Set up this field with the name of the sequence to use
        """
        if sequence:
            self.sequence = sequence
        else:
            # Use the default sequence
            self.sequence = conf.DEFAULT_SEQUENCE
        kwargs['editable'] = False
        super(TicketField, self).__init__(**kwargs)

    def pre_save(self, model_instance, add):
        """
        When saving the DB, if we are adding this instance, get a ticket
        value for this field. Otherwise pull it from the model instance.
        """
        if add:
            from ticketing.models import get_ticket
            value = get_ticket(self.sequence)
            setattr(model_instance, self.attname, value)
            return value
        else:
            return super(TicketField, self).pre_save(model_instance, add)

    def db_type(self, *args, **kwargs):
        connection = kwargs.get('connection', None)
        if connection and connection.settings_dict['ENGINE'] == 'django.db.backends.mysql':
            return 'bigint UNSIGNED'
        else:
            return super(BigAutoField, self).db_type(*args, **kwargs)

    def south_field_triple(self):
        "Returns a suitable description of this field for South."
        # We'll just introspect the _actual_ field.
        from south.modelsinspector import introspector
        field_class = 'ticketing.fields.TicketField'
        args, kwargs = introspector(self)
        # That's our definition!
        return (field_class, args, kwargs)

try:
    from south.modelsinspector import add_introspection_rules
    add_introspection_rules(rules=[
        (
            [TicketField], # Class(es) these apply to
            [],         # Positional arguments (not used)
            {           # Keyword argument
                "sequence": ["sequence", {"default": None}],
            },
        ),
    ], patterns=["^ticketing\.fields\.TicketField"])
except ImportError:
    pass
<|endoftext|>


<commit_before>"""Models for the ``django-tinylinks`` app."""
import urllib2
from cookielib import CookieJar
from socket import gaierror

from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.utils import timezone

from urllib3 import PoolManager
from urllib3.exceptions import HTTPError, MaxRetryError, TimeoutError

def get_url_response(pool, link, url):
    """
    Function to open and check an URL. In case of failure it sets the relevant
    validation error.

    """
    response = False
    link.is_broken = True
    link.redirect_location = ''
    try:
        response = pool.urlopen('GET', url, retries=2, timeout=8.0)
    except TimeoutError:
        link.validation_error = _("Timeout after 8 seconds.")
    except MaxRetryError:
        link.validation_error = _("Failed after retrying twice.")
    except (HTTPError, gaierror):
        link.validation_error = _("Not found.")
    return response

def validate_long_url(link):
    """
    Function to validate a URL. The validator uses urllib3 to test the URL's
    availability.

    """
    http = PoolManager()
    response = get_url_response(http, link, link.long_url)
    if response and response.status == 200:
        link.is_broken = False
    elif response and response.status == 302:
        # If link is redirected, validate the redirect location.
        redirect = get_url_response(http, link,
                                    response.get_redirect_location())
        link.redirect_location = response.get_redirect_location()
        if redirect.status == 200:
            link.is_broken = False
        elif redirect.status == 302:
            # Seems like an infinite loop. Maybe the server is looking for a
            # cookie?
            cj = CookieJar()
            opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
            request = urllib2.Request(response.get_redirect_location())
            response = opener.open(request)
            if response.code == 200:
                link.is_broken = False
    elif response and response.status == 502:
        # Sometimes urllib3 repond with a 502er. Those pages might respond with
        # a 200er in the Browser, so re-check with urllib2
        try:
            response = urllib2.urlopen(link.long_url, timeout=8.0)
        except urllib2.HttpError:
            link.validation_error = _("URL not accessible.")
        else:
            link.is_broken = False
    else:
        link.validation_error = _("URL not accessible.")
    link.last_checked = timezone.now()
    link.save()

class Tinylink(models.Model):
    """
    Model to 'translate' long URLs into small ones.

    :user: The author of the tinylink.
    :long_url: Long URL version.
    :short_url: Shortened URL.
    :is_broken: Set if the given long URL couldn't be validated.
    :validation_error: Description of the occurred error.
    :last_checked: Datetime of the last validation process.
    :amount_of_views: Field to count the redirect views.
    :redirect_location: Redirect location if the long_url is redirected.

    """
    user = models.ForeignKey(
        'auth.User',
        verbose_name=_('Author'),
        related_name="tinylinks",
    )

    long_url = models.CharField(
        max_length=2500,
        verbose_name=_('Long URL'),
    )

    short_url = models.CharField(
        max_length=32,
        verbose_name=_('Short URL'),
        unique=True,
    )

    is_broken = models.BooleanField(
        default=False,
        verbose_name=_('Status'),
    )

    validation_error = models.CharField(
        max_length=100,
        verbose_name=_('Validation Error'),
        default='',
    )

    last_checked = models.DateTimeField(
        default=timezone.now(),
        verbose_name=_('Last validation'),
    )

    amount_of_views = models.PositiveIntegerField(
        default=0,
        verbose_name=_('Amount of views'),
    )

    redirect_location = models.CharField(
        max_length=2500,
        verbose_name=_('Redirect location'),
        default='',
    )

    def __unicode__(self):
        return self.short_url

    class Meta:
        ordering = ['-id']

    def can_be_validated(self):
        """
        URL can only be validated if the last validation was at least 1
        hour ago

        """
        if self.last_checked < timezone.now() - timezone.timedelta(minutes=60):
            return True
        return False

def tinylink_post_save(sender, instance, created, *args, **kwargs):
    if (created or instance.last_checked < timezone.now()
            - timezone.timedelta(minutes=1)):
        validate_long_url(instance)

models.signals.post_save.connect(tinylink_post_save, sender=Tinylink)
<commit_msg>fixed typo<commit_after>"""Models for the ``django-tinylinks`` app."""
import urllib2
from cookielib import CookieJar
from socket import gaierror

from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.utils import timezone

from urllib3 import PoolManager
from urllib3.exceptions import HTTPError, MaxRetryError, TimeoutError

def get_url_response(pool, link, url):
    """
    Function to open and check an URL. In case of failure it sets the relevant
    validation error.

    """
    response = False
    link.is_broken = True
    link.redirect_location = ''
    try:
        response = pool.urlopen('GET', url, retries=2, timeout=8.0)
    except TimeoutError:
        link.validation_error = _("Timeout after 8 seconds.")
    except MaxRetryError:
        link.validation_error = _("Failed after retrying twice.")
    except (HTTPError, gaierror):
        link.validation_error = _("Not found.")
    return response

def validate_long_url(link):
    """
    Function to validate a URL. The validator uses urllib3 to test the URL's
    availability.

    """
    http = PoolManager()
    response = get_url_response(http, link, link.long_url)
    if response and response.status == 200:
        link.is_broken = False
    elif response and response.status == 302:
        # If link is redirected, validate the redirect location.
        redirect = get_url_response(http, link,
                                    response.get_redirect_location())
        link.redirect_location = response.get_redirect_location()
        if redirect.status == 200:
            link.is_broken = False
        elif redirect.status == 302:
            # Seems like an infinite loop. Maybe the server is looking for a
            # cookie?
            cj = CookieJar()
            opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
            request = urllib2.Request(response.get_redirect_location())
            response = opener.open(request)
            if response.code == 200:
                link.is_broken = False
    elif response and response.status == 502:
        # Sometimes urllib3 repond with a 502er. Those pages might respond with
        # a 200er in the Browser, so re-check with urllib2
        try:
            response = urllib2.urlopen(link.long_url, timeout=8.0)
        except urllib2.HTTPError:
            link.validation_error = _("URL not accessible.")
        else:
            link.is_broken = False
    else:
        link.validation_error = _("URL not accessible.")
    link.last_checked = timezone.now()
    link.save()

class Tinylink(models.Model):
    """
    Model to 'translate' long URLs into small ones.

    :user: The author of the tinylink.
    :long_url: Long URL version.
    :short_url: Shortened URL.
    :is_broken: Set if the given long URL couldn't be validated.
    :validation_error: Description of the occurred error.
    :last_checked: Datetime of the last validation process.
    :amount_of_views: Field to count the redirect views.
    :redirect_location: Redirect location if the long_url is redirected.

    """
    user = models.ForeignKey(
        'auth.User',
        verbose_name=_('Author'),
        related_name="tinylinks",
    )

    long_url = models.CharField(
        max_length=2500,
        verbose_name=_('Long URL'),
    )

    short_url = models.CharField(
        max_length=32,
        verbose_name=_('Short URL'),
        unique=True,
    )

    is_broken = models.BooleanField(
        default=False,
        verbose_name=_('Status'),
    )

    validation_error = models.CharField(
        max_length=100,
        verbose_name=_('Validation Error'),
        default='',
    )

    last_checked = models.DateTimeField(
        default=timezone.now(),
        verbose_name=_('Last validation'),
    )

    amount_of_views = models.PositiveIntegerField(
        default=0,
        verbose_name=_('Amount of views'),
    )

    redirect_location = models.CharField(
        max_length=2500,
        verbose_name=_('Redirect location'),
        default='',
    )

    def __unicode__(self):
        return self.short_url

    class Meta:
        ordering = ['-id']

    def can_be_validated(self):
        """
        URL can only be validated if the last validation was at least 1
        hour ago

        """
        if self.last_checked < timezone.now() - timezone.timedelta(minutes=60):
            return True
        return False

def tinylink_post_save(sender, instance, created, *args, **kwargs):
    if (created or instance.last_checked < timezone.now()
            - timezone.timedelta(minutes=1)):
        validate_long_url(instance)

models.signals.post_save.connect(tinylink_post_save, sender=Tinylink)
<|endoftext|>


<commit_before>import os
import numpy as np
import ase.db
from tmd.bilayer.bilayer_util import _base_dir, global_config

def get_atoms(db, formula, phase):
    system_list = list(db.select('formula={},xc=PBE,phase={}'.format(formula, phase)))
    if len(system_list) > 1:
        # TODO - handle this better
        raise ValueError("found multiple matches for {}, PBE, H phase".format(formula))

    atoms = system_list[0]
    #if atoms["hform"] > 0.0:
    #    print("Warning: hform {} > 0".format(formula))

    return atoms

def min_energy(db, formula):
    H = get_atoms(db, formula, "H")
    T = get_atoms(db, formula, "T")

    if H.hform < T.hform:
        return "H"
    else:
        return "T"

# Consistent ordering of lattice vectors and atoms for (Mo,W)(S,Se,Te)2 family.
def a_from_2H(atoms):
    cell = atoms.get_cell()
    a = cell[0][0]

    return a

def h_from_2H(atoms):
    pos = atoms.get_positions()
    c_S2 = pos[2][2]
    c_S1 = pos[1][2]

    return c_S2 - c_S1

def symbols_from_2H(atoms):
    syms = atoms.get_chemical_symbols()
    # Consistent M, X, X order.
    return syms[0], syms[1]

def bilayer_setup(atoms_A, atoms_B=None, c_bulk=None, d_a=None, d_b=None, c_sep_input=None, ordering="2H"):
    # Choose lattice constant from A.
    a = a_from_2H(atoms_A)

    h_A = h_from_2H(atoms_A)

    # Setyawan and Curtarolo 2010 basis
    avec = [1/2, -float(np.sqrt(3)/2)]
    bvec = [1/2, float(np.sqrt(3)/2)]
    latvecs = [avec, bvec]

    # Monolayer special case.
    if atoms_B == None:
        X2_A = h_A/2.0
        M_A  = 0.0
        X1_A = -h_A/2.0

        lat_pos = [[0.0, 0.0],
                   [1/3, 2/3],
                   [1/3, 2/3]]

        cartpos_2D = []
        for pos in lat_pos:
            cartpos_2D.append(np.dot(pos, np.array(latvecs)))

        cartpos = []
        zvals = [M_A, X1_A, X2_A]

        sym_M_A, sym_X_A = symbols_from_2H(atoms_A)
        symbols = [sym_M_A, sym_X_A, sym_X_A]

        for pos, z, sym in zip(cartpos_2D, zvals, symbols):
            cartpos_3D = [float(pos[0]), float(pos[1]), float(z) / float(a)]
            cartpos.append([sym, cartpos_3D])

        return latvecs, cartpos, float(a)

    # Bilayer.
    h_B = h_from_2H(atoms_B)

    if c_sep_input is not None:
        c_sep = c_sep_input
    else:
        c_sep = (c_bulk - h_A - h_B) / 2

    X2_B = c_sep/2.0 + h_B
    M_B  = c_sep/2.0 + h_B/2.0
    X1_B = c_sep/2.0

    X2_A = -c_sep/2.0
    M_A  = -c_sep/2.0 - h_A/2.0
    X1_A = -c_sep/2.0 - h_A

    # M_A, X1_A, X2_A, M_B, X1_B, X2_B
    if ordering == "2H":
        # BAB/ABA
        lat_pos = [[0.0, 0.0], # M = A
                   [1/3, 2/3], # X = B
                   [1/3, 2/3],
                   [(1/3+d_a) % 1, (1/3+d_b) % 1], # Mp = B
                   [(0.0+d_a) % 1, (0.0+d_b) % 1], # Xp = A
                   [(0.0+d_a) % 1, (0.0+d_b) % 1]]
    elif ordering == "2H_top":
        # BAB/BAB
        lat_pos = [[0.0, 0.0], # M = A
                   [1/3, 2/3], # X = B
                   [1/3, 2/3],
                   [(0.0+d_a) % 1, (0.0+d_b) % 1], # Mp = A
                   [(1/3+d_a) % 1, (2/3+d_b) % 1], # Xp = B
                   [(1/3+d_a) % 1, (2/3+d_b) % 1]]
    else:
        raise ValueError("Ordering {} not recognized".format(ordering))

    cartpos_2D = []
    for pos in lat_pos:
        cartpos_2D.append(np.dot(pos, np.array(latvecs)))

    cartpos = []
    zvals = [M_A, X1_A, X2_A, M_B, X1_B, X2_B]

    sym_M_A, sym_X_A = symbols_from_2H(atoms_A)
    sym_M_B, sym_X_B = symbols_from_2H(atoms_B)
    symbols = [sym_M_A, sym_X_A, sym_X_A, sym_M_B, sym_X_B, sym_X_B]

    for pos, z, sym in zip(cartpos_2D, zvals, symbols):
        cartpos_3D = [float(pos[0]), float(pos[1]), float(z) / float(a)]
        cartpos.append([sym, cartpos_3D])

    return latvecs, cartpos, float(a)

def _emit_data(atoms):
    # For Atoms type docs, see https://wiki.fysik.dtu.dk/ase/ase/atoms.html
    print(atoms.get_chemical_symbols())
    print(atoms.get_masses())
    print("Forces")
    print(atoms.get_forces())
    print("Cart-position")
    print(atoms.get_positions())
    print("Cell")
    print(atoms.get_cell())
    print("Lat-position")
    print(atoms.get_scaled_positions()) # positions in direct lattice coordinates

def _main():
    base = _base_dir()
    db_path = os.path.join(base, "c2dm.db")
    db = ase.db.connect(db_path)
    #ws2 = get_atoms(db, "WS2", "H").toatoms()
    #_emit_data(ws2)
    #compounds = ["MoS2", "MoSe2", "MoTe2", "WS2", "WSe2", "WTe2"]
    #for f in compounds:
    #    print(f, min_energy(db, f))
    #    atoms = get_atoms(db, f, "H").toatoms()
    #    _emit_data(atoms)
    #    print("h = {} Ang".format(str(h_from_2H(atoms))))
    MoS2 = get_atoms(db, "MoS2", "H").toatoms()
    WS2 = get_atoms(db, "WS2", "H").toatoms()
    c_bulk = 12.296
    latvecs, cartpos, eq_latconst = bilayer_setup(MoS2, WS2, c_bulk, 0.1, 0.1)
    print(latvecs)
    print(cartpos)
    print(eq_latconst)

if __name__ == "__main__":
    _main()
<commit_msg>fix 2H cell Mp position<commit_after>import os
import numpy as np
import ase.db
from tmd.bilayer.bilayer_util import _base_dir, global_config

def get_atoms(db, formula, phase):
    system_list = list(db.select('formula={},xc=PBE,phase={}'.format(formula, phase)))
    if len(system_list) > 1:
        # TODO - handle this better
        raise ValueError("found multiple matches for {}, PBE, H phase".format(formula))

    atoms = system_list[0]
    #if atoms["hform"] > 0.0:
    #    print("Warning: hform {} > 0".format(formula))

    return atoms

def min_energy(db, formula):
    H = get_atoms(db, formula, "H")
    T = get_atoms(db, formula, "T")

    if H.hform < T.hform:
        return "H"
    else:
        return "T"

# Consistent ordering of lattice vectors and atoms for (Mo,W)(S,Se,Te)2 family.
def a_from_2H(atoms):
    cell = atoms.get_cell()
    a = cell[0][0]

    return a

def h_from_2H(atoms):
    pos = atoms.get_positions()
    c_S2 = pos[2][2]
    c_S1 = pos[1][2]

    return c_S2 - c_S1

def symbols_from_2H(atoms):
    syms = atoms.get_chemical_symbols()
    # Consistent M, X, X order.
    return syms[0], syms[1]

def bilayer_setup(atoms_A, atoms_B=None, c_bulk=None, d_a=None, d_b=None, c_sep_input=None, ordering="2H"):
    # Choose lattice constant from A.
    a = a_from_2H(atoms_A)

    h_A = h_from_2H(atoms_A)

    # Setyawan and Curtarolo 2010 basis
    avec = [1/2, -float(np.sqrt(3)/2)]
    bvec = [1/2, float(np.sqrt(3)/2)]
    latvecs = [avec, bvec]

    # Monolayer special case.
    if atoms_B == None:
        X2_A = h_A/2.0
        M_A  = 0.0
        X1_A = -h_A/2.0

        lat_pos = [[0.0, 0.0],
                   [1/3, 2/3],
                   [1/3, 2/3]]

        cartpos_2D = []
        for pos in lat_pos:
            cartpos_2D.append(np.dot(pos, np.array(latvecs)))

        cartpos = []
        zvals = [M_A, X1_A, X2_A]

        sym_M_A, sym_X_A = symbols_from_2H(atoms_A)
        symbols = [sym_M_A, sym_X_A, sym_X_A]

        for pos, z, sym in zip(cartpos_2D, zvals, symbols):
            cartpos_3D = [float(pos[0]), float(pos[1]), float(z) / float(a)]
            cartpos.append([sym, cartpos_3D])

        return latvecs, cartpos, float(a)

    # Bilayer.
    h_B = h_from_2H(atoms_B)

    if c_sep_input is not None:
        c_sep = c_sep_input
    else:
        c_sep = (c_bulk - h_A - h_B) / 2

    X2_B = c_sep/2.0 + h_B
    M_B  = c_sep/2.0 + h_B/2.0
    X1_B = c_sep/2.0

    X2_A = -c_sep/2.0
    M_A  = -c_sep/2.0 - h_A/2.0
    X1_A = -c_sep/2.0 - h_A

    # M_A, X1_A, X2_A, M_B, X1_B, X2_B
    if ordering == "2H":
        # BAB/ABA
        lat_pos = [[0.0, 0.0], # M = A
                   [1/3, 2/3], # X = B
                   [1/3, 2/3],
                   [(1/3+d_a) % 1, (2/3+d_b) % 1], # Mp = B
                   [(0.0+d_a) % 1, (0.0+d_b) % 1], # Xp = A
                   [(0.0+d_a) % 1, (0.0+d_b) % 1]]
    elif ordering == "2H_top":
        # BAB/BAB
        lat_pos = [[0.0, 0.0], # M = A
                   [1/3, 2/3], # X = B
                   [1/3, 2/3],
                   [(0.0+d_a) % 1, (0.0+d_b) % 1], # Mp = A
                   [(1/3+d_a) % 1, (2/3+d_b) % 1], # Xp = B
                   [(1/3+d_a) % 1, (2/3+d_b) % 1]]
    else:
        raise ValueError("Ordering {} not recognized".format(ordering))

    cartpos_2D = []
    for pos in lat_pos:
        cartpos_2D.append(np.dot(pos, np.array(latvecs)))

    cartpos = []
    zvals = [M_A, X1_A, X2_A, M_B, X1_B, X2_B]

    sym_M_A, sym_X_A = symbols_from_2H(atoms_A)
    sym_M_B, sym_X_B = symbols_from_2H(atoms_B)
    symbols = [sym_M_A, sym_X_A, sym_X_A, sym_M_B, sym_X_B, sym_X_B]

    for pos, z, sym in zip(cartpos_2D, zvals, symbols):
        cartpos_3D = [float(pos[0]), float(pos[1]), float(z) / float(a)]
        cartpos.append([sym, cartpos_3D])

    return latvecs, cartpos, float(a)

def _emit_data(atoms):
    # For Atoms type docs, see https://wiki.fysik.dtu.dk/ase/ase/atoms.html
    print(atoms.get_chemical_symbols())
    print(atoms.get_masses())
    print("Forces")
    print(atoms.get_forces())
    print("Cart-position")
    print(atoms.get_positions())
    print("Cell")
    print(atoms.get_cell())
    print("Lat-position")
    print(atoms.get_scaled_positions()) # positions in direct lattice coordinates

def _main():
    base = _base_dir()
    db_path = os.path.join(base, "c2dm.db")
    db = ase.db.connect(db_path)
    #ws2 = get_atoms(db, "WS2", "H").toatoms()
    #_emit_data(ws2)
    #compounds = ["MoS2", "MoSe2", "MoTe2", "WS2", "WSe2", "WTe2"]
    #for f in compounds:
    #    print(f, min_energy(db, f))
    #    atoms = get_atoms(db, f, "H").toatoms()
    #    _emit_data(atoms)
    #    print("h = {} Ang".format(str(h_from_2H(atoms))))
    MoS2 = get_atoms(db, "MoS2", "H").toatoms()
    WS2 = get_atoms(db, "WS2", "H").toatoms()
    c_bulk = 12.296
    latvecs, cartpos, eq_latconst = bilayer_setup(MoS2, WS2, c_bulk, 0.1, 0.1)
    print(latvecs)
    print(cartpos)
    print(eq_latconst)

if __name__ == "__main__":
    _main()
<|endoftext|>


<commit_before>
# Local modules
from common import debug, text_utils
from common.telegram import telegram_utils
from common.action import action_class

from bible import bible_utils

from tms import tms_utils

CMD_TMS = "/tms"
CMD_TMS_PROMPT = "Give me a Verse reference, or Pack and Verse number\n(P.S. you can even try giving me a topic)"
CMD_TMS_BADQUERY = "I can't find anything related to this, try another one?"

STATE_WAIT_TMS = "Waiting for TMS query"

class TMSAction(action_class.Action):
    def identifier(self):
        return '/tms'

    def resolve(self, user, msg):
        query = telegram_utils.strip_command(msg, self.identifier())

        if text_utils.is_valid(query):
            debug.log("Resolving TMS Query")
            verse = None

            verse_reference = bible_utils.get_reference(query)
            if text_utils.is_valid(verse_reference):
                verse = tms_utils.query_verse_by_reference(verse_reference)

            if verse is None:
                verse = tms_utils.query_verse_by_pack_pos(query)

            if verse is None:
                verse = tms_utils.query_verse_by_topic(query)

            if verse is not None:
                passage = bible_utils.get_passage_raw(verse.reference, user.get_version())
                verse_msg = tms_utils.format_verse(verse, passage)

                telegram_utils.send_msg(verse_msg, user.get_uid())
                user.set_state(None)
            else:
                telegram_utils.send_msg(CMD_TMS_BADQUERY, user.get_uid())
        else:
            telegram_utils.send_msg_keyboard(CMD_TMS_PROMPT, user.get_uid())
            user.set_state(STATE_WAIT_TMS)

        return True

ACTION = TMSAction()
def get_action():
    return ACTION
<commit_msg>Removing some debug. Not totally working<commit_after>
# Local modules
from common import debug, text_utils
from common.telegram import telegram_utils
from common.action import action_class

from bible import bible_utils

from tms import tms_utils

CMD_TMS = "/tms"
CMD_TMS_PROMPT = "Give me a Verse reference, or Pack and Verse number\n(P.S. you can even try giving me a topic)"
CMD_TMS_BADQUERY = "I can't find anything related to this, try another one?"

STATE_WAIT_TMS = "Waiting for TMS query"

class TMSAction(action_class.Action):
    def identifier(self):
        return '/tms'

    def resolve(self, user, msg):
        query = telegram_utils.strip_command(msg, self.identifier())

        if text_utils.is_valid(query):
            verse = None

            verse_reference = bible_utils.get_reference(query)
            if text_utils.is_valid(verse_reference):
                verse = tms_utils.query_verse_by_reference(verse_reference)

            if verse is None:
                verse = tms_utils.query_verse_by_pack_pos(query)

            if verse is None:
                verse = tms_utils.query_verse_by_topic(query)

            if verse is not None:
                passage = bible_utils.get_passage_raw(verse.reference, user.get_version())
                verse_msg = tms_utils.format_verse(verse, passage)

                telegram_utils.send_msg(verse_msg, user.get_uid())
                user.set_state(None)
            else:
                telegram_utils.send_msg(CMD_TMS_BADQUERY, user.get_uid())
        else:
            telegram_utils.send_msg_keyboard(CMD_TMS_PROMPT, user.get_uid())
            user.set_state(STATE_WAIT_TMS)

        return True

ACTION = TMSAction()
def get_action():
    return ACTION
<|endoftext|>


<commit_before>"""
A module for quick importing the essential core stuff.
(coroutine, Scheduler, events, sockets, priority)
"""
from cogen.core.coroutines import coroutine
from cogen.core.schedulers import Scheduler
from cogen.core import events
from cogen.core import sockets
from cogen.core import reactors
from cogen.core.events import priority
<commit_msg>added 'coro' to quick imports<commit_after>"""
A module for quick importing the essential core stuff.
(coroutine, Scheduler, events, sockets, priority)
"""
from cogen.core.coroutines import coroutine, coro
from cogen.core.schedulers import Scheduler
from cogen.core import events
from cogen.core import sockets
from cogen.core import reactors
from cogen.core.events import priority
<|endoftext|>