<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from __future__ import print_function

from copy import deepcopy
import functools
import io
import os
import zipfile

from .package import Package
from .. import tech
from . import layouts
from . import metakey

# technology modules
timestamp = tech.timestamp
securehash = tech.securehash
persistence = tech.persistence
path = tech.fs

class Archive(Package):

    def __init__(self, filename):
        self.archive_filename = filename
        self.zipfile = None
        self._meta = self._load_meta()

    def __zipfile_user(method):
        # method is called with the zipfile opened
        @functools.wraps(method)
        def f(self, *args, **kwargs):
            if self.zipfile:
                return method(self, *args, **kwargs)
            try:
                with zipfile.ZipFile(self.archive_filename) as self.zipfile:
                    return method(self, *args, **kwargs)
            finally:
                self.zipfile = None
        return f

    def export(self, exported_archive_path):
        '''
        I pack my content (everything!) as a zip-Archive to requested location.
        '''
        # FIXME: Implement Archive.export
        pass

    # -
    @property
    def is_valid(self):
        # TODO
        '''
        verify, that
        - all files under code, data, meta are present in the checksums
          file and they match their checksums
          (can extra files be allowed in the archive?)
        - the pkgmeta file is valid
            - has package uuid
            - has timestamp
            - has unofficial package name
            - has inputs (even if empty)
        '''
        valid = all((
            metakey.PACKAGE in self.meta,
            metakey.PACKAGE_TIMESTAMP in self.meta,
            metakey.INPUTS in self.meta,
            metakey.DEFAULT_NAME in self.meta,
        ))

        if valid:
            now = timestamp.time_from_timestamp(timestamp.timestamp())
            pkgtime = timestamp.time_from_timestamp(
                self.meta[metakey.PACKAGE_TIMESTAMP]
            )
            valid = pkgtime < now

        return valid

    @property
    @__zipfile_user
    def version(self):
        zipinfo = self.zipfile.getinfo(layouts.Archive.CHECKSUMS)
        with self.zipfile.open(zipinfo) as f:
            return securehash.file(f, zipinfo.file_size)

    @property
    def uuid(self):
        return self._meta[metakey.PACKAGE]

    @property
    def timestamp_str(self):
        return self._meta[metakey.PACKAGE_TIMESTAMP]

    @property
    def meta(self):
        # create a copy, so that returned meta can be modified without causing
        # harm to this Archive instance
        return deepcopy(self._meta)

    # -
    @__zipfile_user
    def _load_meta(self):
        with self.zipfile.open(layouts.Archive.PKGMETA) as f:
            return persistence.load(io.TextIOWrapper(f))

    @__zipfile_user
    def extract_file(self, zip_path, destination):
        '''Extract zip_path from zipfile to destination'''

        assert not os.path.exists(destination)

        with path.temp_dir(destination.parent) as unzip_dir:
            self.zipfile.extract(zip_path, unzip_dir)
            os.rename(unzip_dir / zip_path, destination)

    @__zipfile_user
    def extract_dir(self, zip_dir, destination):
        '''
        Extract all files from zipfile under zip_dir to destination.
        '''

        assert not os.path.exists(destination)

        zip_dir_prefix = zip_dir + '/'
        filelist = [
            name
            for name in self.zipfile.namelist()
            if name.startswith(zip_dir_prefix)
        ]

        if filelist:
            with path.temp_dir(destination.parent) as unzip_dir:
                self.zipfile.extractall(unzip_dir, filelist)
                os.rename(unzip_dir / zip_dir, destination)
        else:
            path.ensure_directory(destination)

    def unpack_code_to(self, destination):
        self.extract_dir(layouts.Archive.CODE, destination)

    def unpack_data_to(self, destination):
        self.extract_dir(layouts.Archive.DATA, destination)

    def unpack_meta_to(self, workspace):
        workspace.meta = self.meta
<commit_msg>FIXME: Archive.is_valid should verify checksums<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from __future__ import print_function

from copy import deepcopy
import functools
import io
import os
import zipfile

from .package import Package
from .. import tech
from . import layouts
from . import metakey

# technology modules
timestamp = tech.timestamp
securehash = tech.securehash
persistence = tech.persistence
path = tech.fs

class Archive(Package):

    def __init__(self, filename):
        self.archive_filename = filename
        self.zipfile = None
        self._meta = self._load_meta()

    def __zipfile_user(method):
        # method is called with the zipfile opened
        @functools.wraps(method)
        def f(self, *args, **kwargs):
            if self.zipfile:
                return method(self, *args, **kwargs)
            try:
                with zipfile.ZipFile(self.archive_filename) as self.zipfile:
                    return method(self, *args, **kwargs)
            finally:
                self.zipfile = None
        return f

    def export(self, exported_archive_path):
        '''
        I pack my content (everything!) as a zip-Archive to requested location.
        '''
        # FIXME: Implement Archive.export
        pass

    # -
    @property
    def is_valid(self):
        # FIXME verify checksums!
        '''
        verify, that
        - all files under code, data, meta are present in the checksums
          file and they match their checksums
          (can extra files be allowed in the archive?)
        - the pkgmeta file is valid
            - has package uuid
            - has timestamp
            - has unofficial package name
            - has inputs (even if empty)
        '''
        valid = all((
            metakey.PACKAGE in self.meta,
            metakey.PACKAGE_TIMESTAMP in self.meta,
            metakey.INPUTS in self.meta,
            metakey.DEFAULT_NAME in self.meta,
        ))

        if valid:
            now = timestamp.time_from_timestamp(timestamp.timestamp())
            pkgtime = timestamp.time_from_timestamp(
                self.meta[metakey.PACKAGE_TIMESTAMP]
            )
            valid = pkgtime < now

        return valid

    @property
    @__zipfile_user
    def version(self):
        zipinfo = self.zipfile.getinfo(layouts.Archive.CHECKSUMS)
        with self.zipfile.open(zipinfo) as f:
            return securehash.file(f, zipinfo.file_size)

    @property
    def uuid(self):
        return self._meta[metakey.PACKAGE]

    @property
    def timestamp_str(self):
        return self._meta[metakey.PACKAGE_TIMESTAMP]

    @property
    def meta(self):
        # create a copy, so that returned meta can be modified without causing
        # harm to this Archive instance
        return deepcopy(self._meta)

    # -
    @__zipfile_user
    def _load_meta(self):
        with self.zipfile.open(layouts.Archive.PKGMETA) as f:
            return persistence.load(io.TextIOWrapper(f))

    @__zipfile_user
    def extract_file(self, zip_path, destination):
        '''Extract zip_path from zipfile to destination'''

        assert not os.path.exists(destination)

        with path.temp_dir(destination.parent) as unzip_dir:
            self.zipfile.extract(zip_path, unzip_dir)
            os.rename(unzip_dir / zip_path, destination)

    @__zipfile_user
    def extract_dir(self, zip_dir, destination):
        '''
        Extract all files from zipfile under zip_dir to destination.
        '''

        assert not os.path.exists(destination)

        zip_dir_prefix = zip_dir + '/'
        filelist = [
            name
            for name in self.zipfile.namelist()
            if name.startswith(zip_dir_prefix)
        ]

        if filelist:
            with path.temp_dir(destination.parent) as unzip_dir:
                self.zipfile.extractall(unzip_dir, filelist)
                os.rename(unzip_dir / zip_dir, destination)
        else:
            path.ensure_directory(destination)

    def unpack_code_to(self, destination):
        self.extract_dir(layouts.Archive.CODE, destination)

    def unpack_data_to(self, destination):
        self.extract_dir(layouts.Archive.DATA, destination)

    def unpack_meta_to(self, workspace):
        workspace.meta = self.meta
<|endoftext|>


<commit_before>from django import forms
from django.conf import settings
from django.contrib.sites.models import Site
from django.utils.encoding import smart_str

class SpamPreventionForm(forms.Form):
    def __init__(self, request, *args, **kwargs):
        self.request = request
        super(SpamPreventionForm, self).__init__(*args, **kwargs)

class EarTriviaForm(SpamPreventionForm):
    """Form with the single purpose of making it difficult for a machine to
    pass validation by adding a question that is trivial to answer most
    humans.

    """
    (ONE, TWO) = (0, 1)
    CHOICES = ((ONE, 'One'), (TWO, 'Two'))
    CORRECT_ANSWER = TWO

    question = forms.ChoiceField(choices=CHOICES,
                                 label=u"How many ears does an average human "
                                       u"have?")

    def __init__(self, *args, **kwargs):
        super(EarTriviaForm, self).__init__(*args, **kwargs)

    def clean_question(self):
        value = self.cleaned_data["question"]
        if int(value) != self.CORRECT_ANSWER:
            raise forms.ValidationError(u'Think again!')
        return value

class HoneypotForm(SpamPreventionForm):
    """Form that adds a honeypot field -- a field that aims to lure
    spammers into entering data that would invalidate the form.

    """
    honeypot = forms.CharField(required=False)

    def clean_honeypot(self):
        """Check that nothing's been entered into the honeypot."""
        value = self.cleaned_data["honeypot"]
        if value:
            raise forms.ValidationError(u'This field must remain empty.')
        return value

class AkismetForm(SpamPreventionForm):
    """Form that adds the ability to check submitted data via the Wordpress
    Akismet spam filter.

    """
    def clean(self):
        """Run the body of the comment against the akismet API."""
        from akismet import Akismet
        data = {
            'comment_type': 'comment',
            'referrer': self.request.META.get('HTTP_REFERER', ''),
            'user_ip': self.request.META.get('REMOTE_ADDR', ''),
            'user_agent': self.request.META.get('HTTP_USER_AGENT', '')
        }
        body = self.cleaned_data.get('body')
        blog_url = 'http://%s/' % Site.objects.get_current().domain

        api = Akismet(key=settings.AKISMET_API_KEY, blog_url=blog_url)

        if api.verify_key():
            if not api.comment_check(smart_str(body), data=data):
                raise forms.ValidationError(u"Your comment appears to be "
                                            u"spam.")
        return self.cleaned_data
<commit_msg>Corrected akismet spam tests.<commit_after>from django import forms
from django.conf import settings
from django.contrib.sites.models import Site
from django.utils.encoding import smart_str

class SpamPreventionForm(forms.Form):
    def __init__(self, request, *args, **kwargs):
        self.request = request
        super(SpamPreventionForm, self).__init__(*args, **kwargs)

class EarTriviaForm(SpamPreventionForm):
    """Form with the single purpose of making it difficult for a machine to
    pass validation by adding a question that is trivial to answer most
    humans.

    """
    (ONE, TWO) = (0, 1)
    CHOICES = ((ONE, 'One'), (TWO, 'Two'))
    CORRECT_ANSWER = TWO

    question = forms.ChoiceField(choices=CHOICES,
                                 label=u"How many ears does an average human "
                                       u"have?")

    def __init__(self, *args, **kwargs):
        super(EarTriviaForm, self).__init__(*args, **kwargs)

    def clean_question(self):
        value = self.cleaned_data["question"]
        if int(value) != self.CORRECT_ANSWER:
            raise forms.ValidationError(u'Think again!')
        return value

class HoneypotForm(SpamPreventionForm):
    """Form that adds a honeypot field -- a field that aims to lure
    spammers into entering data that would invalidate the form.

    """
    honeypot = forms.CharField(required=False)

    def clean_honeypot(self):
        """Check that nothing's been entered into the honeypot."""
        value = self.cleaned_data["honeypot"]
        if value:
            raise forms.ValidationError(u'This field must remain empty.')
        return value

class AkismetForm(SpamPreventionForm):
    """Form that adds the ability to check submitted data via the Wordpress
    Akismet spam filter.

    """
    def clean(self):
        """Run the body of the comment against the akismet API."""
        from akismet import Akismet
        data = {
            'comment_type': 'comment',
            'referrer': self.request.META.get('HTTP_REFERER', ''),
            'user_ip': self.request.META.get('REMOTE_ADDR', ''),
            'user_agent': self.request.META.get('HTTP_USER_AGENT', '')
        }
        body = self.cleaned_data.get('body')
        blog_url = 'http://%s/' % Site.objects.get_current().domain

        api = Akismet(key=settings.AKISMET_API_KEY, blog_url=blog_url)

        if api.verify_key():
            if api.comment_check(smart_str(body), data=data):
                raise forms.ValidationError(u"Your comment appears to be "
                                            u"spam.")
        return self.cleaned_data
<|endoftext|>


<commit_before>import os
import socket
import warnings

import paramiko
from paramiko.ssh_exception import BadHostKeyException, AuthenticationException, SSHException
from libcloud.compute.base import NodeImage

from datasciencebox.core.logger import getLogger
logger = getLogger()
from datasciencebox.core.exceptions import DSBException, DSBWarning
from datasciencebox.core.utils import retry

class Instance(object):

    def __init__(self, uid=None, ip=None, port=None, username=None, keypair=None, settings=None, cluster=None):
        logger.debug('Initializing Instance')
        self.settings = settings
        self.cluster = cluster
        self._driver = None
        self._node = None
        self._uid = None
        self._ip = None
        self._port = 22
        self._username = None
        self._keypair = None

        self.uid = uid
        self.ip = ip
        self.port = port
        self.username = username
        self.keypair = keypair

    @classmethod
    def new(cls, settings, *args, **kwargs):
        """
        Create a new Cloud instance based on the Settings
        """
        logger.debug('Creating new "%s" Instance' % settings['CLOUD'])
        cloud = settings['CLOUD']
        if cloud == 'bare':
            self = BareInstance(settings=settings, *args, **kwargs)
        elif cloud == 'aws':
            self = AWSInstance(settings=settings, *args, **kwargs)
        elif cloud == 'gcp':
            self = GCPInstance(settings=settings, *args, **kwargs)
        else:
            raise DSBException('Cloud "%s" not supported' % cloud)
        return self

    def __repr__(self):
        return 'Instance(%s)' % self.to_dict()

    def to_dict(self):
        ret = {}
        ret['uid'] = self.uid
        ret['ip'] = self.ip
        ret['port'] = self.port
        return ret

    def get_driver(self):
        return self.cluster.driver

    driver = property(get_driver, None, None)

    def get_uid(self):
        if self._uid is None:
            self._uid = self.fetch_uid()
        return self._uid

    def fetch_uid(self):
        raise NotImplementedError()

    def set_uid(self, value):
        self._uid = value

    uid = property(get_uid, set_uid, None)

    def get_ip(self):
        if self._ip is None:
            self._ip = self.fetch_ip()
        return self._ip

    def fetch_ip(self):
        raise NotImplementedError('Subclass of Instance must implement "fetch_ip"')

    def set_ip(self, value):
        if value:
            try:
                self._ip, self.port = value.split(":")
            except ValueError:
                self._ip = value

    ip = property(get_ip, set_ip, None)

    def get_port(self):
        return self._port

    def set_port(self, value):
        self._port = value or self.port
        self._port = int(self._port)

    port = property(get_port, set_port, None)

    def get_node(self):
        if self._node is None:
            self._node = self.fetch_node()
        return self._node

    def fetch_node(self):
        raise NotImplementedError('Subclass of Instance must implement "fetch_node"')

    def set_node(self, value):
        self._node = value

    node = property(get_node, set_node, None)

    def get_username(self):
        default_username = self.settings['USERNAME'] if self.settings else None
        return self._username or default_username

    def set_username(self, value):
        self._username = value

    username = property(get_username, set_username, None)

    def get_keypair(self):
        default_keypair = self.settings['KEYPAIR'] if self.settings else None
        return self._keypair or default_keypair

    def set_keypair(self, value):
        self._keypair = value

    keypair = property(get_keypair, set_keypair, None)

    def create(self):
        raise NotImplementedError('Subclass of Instance must implement "create"')

    def destroy(self):
        raise NotImplementedError('Subclass of Instance must implement "destroy"')

    @retry(catch=(BadHostKeyException, AuthenticationException, SSHException, socket.error))
    def check_ssh(self):
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(self.ip, port=self.port, username=self.username, key_filename=self.keypair)
        return True

class BareInstance(Instance):

    def fetch_uid(self):
        pass

    def fetch_ip(self):
        warnings.warn('Bare Metal instance cannot fetch ip address', DSBWarning)

    def fetch_node(self):
        warnings.warn('Bare Metal instance cannot fetch a node', DSBWarning)

    def create(self):
        warnings.warn('Bare Metal instance cannot be created', DSBWarning)

    def destroy(self):
        warnings.warn('Bare Metal instance cannot be destroyed', DSBWarning)

class AWSInstance(Instance):

    def fetch_uid(self):
        return self.node.id

    def fetch_ip(self):
        return self.node.public_ips[0]

    def fetch_node(self):
        logger.debug('Fetching aws Instance: %s' % self.uid)
        return self.driver.list_nodes(ex_node_ids=[self.uid])[0]

    def create(self, suffix=None):
        name = self.settings['ID']
        ami_id = self.settings['AWS_IMAGE']
        image = NodeImage(id=ami_id, name=None, driver=self.driver)
        root_size = self.settings['AWS_ROOT_SIZE']
        root_type = self.settings['AWS_ROOT_TYPE']
        ex_keyname = self.settings['AWS_KEYNAME']
        ex_securitygroup = self.settings['AWS_SECURITY_GROUPS']

        instance_type = self.settings['AWS_SIZE']
        sizes = self.driver.list_sizes()
        size = [s for s in sizes if s.id == instance_type][0]

        ebs_mapping = [{
            'DeviceName': '/dev/sda1',
            'Ebs':
            {'DeleteOnTermination': 'true',
             'VolumeSize': root_size,
             'VolumeType': root_type},
            'VirtualName': None
        }]

        try:
            self.node = self.driver.create_node(name=name,
                                                size=size,
                                                image=image,
                                                ex_keyname=ex_keyname,
                                                ex_securitygroup=ex_securitygroup,
                                                ex_blockdevicemappings=ebs_mapping)
        except Exception, e:
            if 'EBS block device mappings not supported for instance-store AMIs' in str(e):
                self.node = self.driver.create_node(name=name,
                                                    size=size,
                                                    image=image,
                                                    ex_keyname=ex_keyname,
                                                    ex_securitygroup=ex_securitygroup)
            else:
                raise (e)

        return self.node

    def destroy(self):
        self.node.destroy()

class GCPInstance(Instance):

    def fetch_uid(self):
        return self.node.id

    def fetch_ip(self):
        return self.node.public_ips[0]

    def fetch_node(self):
        logger.debug('Fetching aws Instance: %s' % self.uid)
        all_nodes = self.driver.list_nodes()
        node = [node for node in all_nodes if node.id == self.uid]
        return node[0]

    def create(self, suffix=None):
        suffix = '-%s' % suffix if suffix is not None else ''
        name = '%s%s' % (self.settings['ID'], suffix)
        image = self.settings['GCP_IMAGE']
        size = self.settings['GCP_SIZE']
        network = self.settings['GCP_NETWORK']
        root_disk_size = self.settings['GCP_ROOT_SIZE']
        root_disk_type = self.settings['GCP_ROOT_TYPE']

        metadata = {}
        with open(os.path.expanduser(self.settings['GCP_PUBLIC_KEY'])) as f:
            metadata['sshKeys'] = '%s:%s' % (self.settings['USERNAME'], f.read())

        volume = self.driver.create_volume(size=root_disk_size,
                                           name=name,
                                           ex_disk_type=root_disk_type,
                                           image=image)

        sa_scopes = [{'scopes': ['compute', 'storage-full']}]
        self.node = self.driver.create_node(name=name,
                                            size=size,
                                            image=image,
                                            ex_metadata=metadata,
                                            ex_network=network,
                                            ex_boot_disk=volume,
                                            ex_disk_auto_delete=True,
                                            ex_service_accounts=sa_scopes)
        return self.node

    def destroy(self):
        self.node.destroy()
<commit_msg>expand user on keypair<commit_after>import os
import socket
import warnings

import paramiko
from paramiko.ssh_exception import BadHostKeyException, AuthenticationException, SSHException
from libcloud.compute.base import NodeImage

from datasciencebox.core.logger import getLogger
logger = getLogger()
from datasciencebox.core.exceptions import DSBException, DSBWarning
from datasciencebox.core.utils import retry

class Instance(object):

    def __init__(self, uid=None, ip=None, port=None, username=None, keypair=None, settings=None, cluster=None):
        logger.debug('Initializing Instance')
        self.settings = settings
        self.cluster = cluster
        self._driver = None
        self._node = None
        self._uid = None
        self._ip = None
        self._port = 22
        self._username = None
        self._keypair = None

        self.uid = uid
        self.ip = ip
        self.port = port
        self.username = username
        self.keypair = keypair

    @classmethod
    def new(cls, settings, *args, **kwargs):
        """
        Create a new Cloud instance based on the Settings
        """
        logger.debug('Creating new "%s" Instance' % settings['CLOUD'])
        cloud = settings['CLOUD']
        if cloud == 'bare':
            self = BareInstance(settings=settings, *args, **kwargs)
        elif cloud == 'aws':
            self = AWSInstance(settings=settings, *args, **kwargs)
        elif cloud == 'gcp':
            self = GCPInstance(settings=settings, *args, **kwargs)
        else:
            raise DSBException('Cloud "%s" not supported' % cloud)
        return self

    def __repr__(self):
        return 'Instance(%s)' % self.to_dict()

    def to_dict(self):
        ret = {}
        ret['uid'] = self.uid
        ret['ip'] = self.ip
        ret['port'] = self.port
        return ret

    def get_driver(self):
        return self.cluster.driver

    driver = property(get_driver, None, None)

    def get_uid(self):
        if self._uid is None:
            self._uid = self.fetch_uid()
        return self._uid

    def fetch_uid(self):
        raise NotImplementedError()

    def set_uid(self, value):
        self._uid = value

    uid = property(get_uid, set_uid, None)

    def get_ip(self):
        if self._ip is None:
            self._ip = self.fetch_ip()
        return self._ip

    def fetch_ip(self):
        raise NotImplementedError('Subclass of Instance must implement "fetch_ip"')

    def set_ip(self, value):
        if value:
            try:
                self._ip, self.port = value.split(":")
            except ValueError:
                self._ip = value

    ip = property(get_ip, set_ip, None)

    def get_port(self):
        return self._port

    def set_port(self, value):
        self._port = value or self.port
        self._port = int(self._port)

    port = property(get_port, set_port, None)

    def get_node(self):
        if self._node is None:
            self._node = self.fetch_node()
        return self._node

    def fetch_node(self):
        raise NotImplementedError('Subclass of Instance must implement "fetch_node"')

    def set_node(self, value):
        self._node = value

    node = property(get_node, set_node, None)

    def get_username(self):
        default_username = self.settings['USERNAME'] if self.settings else None
        return self._username or default_username

    def set_username(self, value):
        self._username = value

    username = property(get_username, set_username, None)

    def get_keypair(self):
        default_keypair = self.settings['KEYPAIR'] if self.settings else None
        key = self._keypair or default_keypair
        return os.path.expanduser(key)

    def set_keypair(self, value):
        self._keypair = value

    keypair = property(get_keypair, set_keypair, None)

    def create(self):
        raise NotImplementedError('Subclass of Instance must implement "create"')

    def destroy(self):
        raise NotImplementedError('Subclass of Instance must implement "destroy"')

    @retry(catch=(BadHostKeyException, AuthenticationException, SSHException, socket.error))
    def check_ssh(self):
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(self.ip, port=self.port, username=self.username, key_filename=self.keypair)
        return True

class BareInstance(Instance):

    def fetch_uid(self):
        pass

    def fetch_ip(self):
        warnings.warn('Bare Metal instance cannot fetch ip address', DSBWarning)

    def fetch_node(self):
        warnings.warn('Bare Metal instance cannot fetch a node', DSBWarning)

    def create(self):
        warnings.warn('Bare Metal instance cannot be created', DSBWarning)

    def destroy(self):
        warnings.warn('Bare Metal instance cannot be destroyed', DSBWarning)

class AWSInstance(Instance):

    def fetch_uid(self):
        return self.node.id

    def fetch_ip(self):
        return self.node.public_ips[0]

    def fetch_node(self):
        logger.debug('Fetching aws Instance: %s' % self.uid)
        return self.driver.list_nodes(ex_node_ids=[self.uid])[0]

    def create(self, suffix=None):
        name = self.settings['ID']
        ami_id = self.settings['AWS_IMAGE']
        image = NodeImage(id=ami_id, name=None, driver=self.driver)
        root_size = self.settings['AWS_ROOT_SIZE']
        root_type = self.settings['AWS_ROOT_TYPE']
        ex_keyname = self.settings['AWS_KEYNAME']
        ex_securitygroup = self.settings['AWS_SECURITY_GROUPS']

        instance_type = self.settings['AWS_SIZE']
        sizes = self.driver.list_sizes()
        size = [s for s in sizes if s.id == instance_type][0]

        ebs_mapping = [{
            'DeviceName': '/dev/sda1',
            'Ebs':
            {'DeleteOnTermination': 'true',
             'VolumeSize': root_size,
             'VolumeType': root_type},
            'VirtualName': None
        }]

        try:
            self.node = self.driver.create_node(name=name,
                                                size=size,
                                                image=image,
                                                ex_keyname=ex_keyname,
                                                ex_securitygroup=ex_securitygroup,
                                                ex_blockdevicemappings=ebs_mapping)
        except Exception, e:
            if 'EBS block device mappings not supported for instance-store AMIs' in str(e):
                self.node = self.driver.create_node(name=name,
                                                    size=size,
                                                    image=image,
                                                    ex_keyname=ex_keyname,
                                                    ex_securitygroup=ex_securitygroup)
            else:
                raise (e)

        return self.node

    def destroy(self):
        self.node.destroy()

class GCPInstance(Instance):

    def fetch_uid(self):
        return self.node.id

    def fetch_ip(self):
        return self.node.public_ips[0]

    def fetch_node(self):
        logger.debug('Fetching aws Instance: %s' % self.uid)
        all_nodes = self.driver.list_nodes()
        node = [node for node in all_nodes if node.id == self.uid]
        return node[0]

    def create(self, suffix=None):
        suffix = '-%s' % suffix if suffix is not None else ''
        name = '%s%s' % (self.settings['ID'], suffix)
        image = self.settings['GCP_IMAGE']
        size = self.settings['GCP_SIZE']
        network = self.settings['GCP_NETWORK']
        root_disk_size = self.settings['GCP_ROOT_SIZE']
        root_disk_type = self.settings['GCP_ROOT_TYPE']

        metadata = {}
        with open(os.path.expanduser(self.settings['GCP_PUBLIC_KEY'])) as f:
            metadata['sshKeys'] = '%s:%s' % (self.settings['USERNAME'], f.read())

        volume = self.driver.create_volume(size=root_disk_size,
                                           name=name,
                                           ex_disk_type=root_disk_type,
                                           image=image)

        sa_scopes = [{'scopes': ['compute', 'storage-full']}]
        self.node = self.driver.create_node(name=name,
                                            size=size,
                                            image=image,
                                            ex_metadata=metadata,
                                            ex_network=network,
                                            ex_boot_disk=volume,
                                            ex_disk_auto_delete=True,
                                            ex_service_accounts=sa_scopes)
        return self.node

    def destroy(self):
        self.node.destroy()
<|endoftext|>


<commit_before>import os
import csv

"""
format data for clustering
"""

results_dir = '../../results'

MINIMUM_POSTS = 100 #minimum to save a category

def write_rows(filename, rows):
	if len(rows) < MINIMUM_POSTS:
		return

	print 'Saving ' + str(len(rows)) + ' rows to ' + filename
	with open(filename, 'w') as out_file:
		csv_writer = csv.writer(out_file)
		for p in rows:
			csv_writer.writerow(p)

def by_city():
	for city in os.listdir(results_dir):
		d_path = os.path.join(results_dir, city)
		if os.path.isdir(d_path):
			city_posts = []
			for f in os.listdir(d_path):
				if f.endswith('.csv'):
					with open(os.path.join(d_path, f),'r') as in_file:
						next(in_file) # skip column labels
						csv_reader = csv.reader(in_file)
						for line in csv_reader:
							city = line[1]
							title = line[7]
							text = line[8] + title + title # DOUBLE WEIGHT WORDS IN TITLE
							category = line[5]
							posttype = line[6]
							city_posts.append([text, city, category, posttype]) #text

			write_rows('data/city/' + city + '.csv', city_posts)

def by_field(dirname, column_number):
	rows_by_field = {}
	for city in os.listdir(results_dir):
		d_path = os.path.join(results_dir, city)
		if os.path.isdir(d_path):
			posts = []
			for f in os.listdir(d_path):
				if f.endswith('.csv'):
					with open(os.path.join(d_path, f),'r') as in_file:
						next(in_file) # skip column labels
						csv_reader = csv.reader(in_file)
						for line in csv_reader:
							city = line[1]
							title = line[7]
							text = line[8] + title + title # DOUBLE WEIGHT WORDS IN TITLE
							category = line[5]
							posttype = line[6]
							# split based on value of the column we're concerned with
							value = line[column_number]
							if not value:
								value = 'none'
							if value not in rows_by_field:
								rows_by_field[value] = []
							rows_by_field[value].append([text, city, category, posttype]) #text

	for value in rows_by_field:
		write_rows('data/' + dirname + '/' + value + '.csv', rows_by_field[value])

"""
split up all posts into those posted by males (for anyone) and those posted by females (for anyone)
* this excludes posts coming from two people (ie mw4m, mm4m, etc)
* this includes both platonic and romantic postss

"""
def by_gender():
	male = []
	female = []
	for city in os.listdir(results_dir):
		d_path = os.path.join(results_dir, city)
		if os.path.isdir(d_path):
			posts = []
			for f in os.listdir(d_path):
				if f.endswith('.csv'):
					with open(os.path.join(d_path, f),'r') as in_file:
						next(in_file) # skip column labels
						csv_reader = csv.reader(in_file)
						for line in csv_reader:
							city = line[1]
							title = line[7]
							text = line[8] + title + title # DOUBLE WEIGHT WORDS IN TITLE
							category = line[5]
							posttype = line[6]
							# split on male and female posters
							if posttype.startswith('m4'):
								male.append([text, city, category, posttype])
							elif posttype.startswith('w4'):
								female.append([text, city, category, posttype])

	write_rows('data/gender/male.csv', male)
	write_rows('data/gender/female.csv', female)

def main():
	#by_city()
	#by_field('type', 6)
	by_gender()

if __name__ == '__main__':
	main()
<commit_msg>formatting category and type<commit_after>import os
import csv

"""
format data for clustering
"""

results_dir = '../../results'

MINIMUM_POSTS = 100 #minimum to save a category

def write_rows(filename, rows):
	if len(rows) < MINIMUM_POSTS:
		return

	print 'Saving ' + str(len(rows)) + ' rows to ' + filename
	with open(filename, 'w') as out_file:
		csv_writer = csv.writer(out_file)
		for p in rows:
			csv_writer.writerow(p)

def by_city():
	for city in os.listdir(results_dir):
		d_path = os.path.join(results_dir, city)
		if os.path.isdir(d_path):
			city_posts = []
			for f in os.listdir(d_path):
				if f.endswith('.csv'):
					with open(os.path.join(d_path, f),'r') as in_file:
						next(in_file) # skip column labels
						csv_reader = csv.reader(in_file)
						for line in csv_reader:
							city = line[1]
							title = line[7]
							text = line[8] + title + title # DOUBLE WEIGHT WORDS IN TITLE
							category = line[5]
							posttype = line[6]
							city_posts.append([text, city, category, posttype]) #text

			write_rows('data/city/' + city + '.csv', city_posts)

def by_field(dirname, column_number):
	rows_by_field = {}
	for city in os.listdir(results_dir):
		d_path = os.path.join(results_dir, city)
		if os.path.isdir(d_path):
			posts = []
			for f in os.listdir(d_path):
				if f.endswith('.csv'):
					with open(os.path.join(d_path, f),'r') as in_file:
						next(in_file) # skip column labels
						csv_reader = csv.reader(in_file)
						for line in csv_reader:
							city = line[1]
							title = line[7]
							text = line[8] + title + title # DOUBLE WEIGHT WORDS IN TITLE
							category = line[5]
							posttype = line[6]
							# split based on value of the column we're concerned with
							value = line[column_number]
							if not value:
								value = 'none'
							if value not in rows_by_field:
								rows_by_field[value] = []
							rows_by_field[value].append([text, city, category, posttype]) #text

	for value in rows_by_field:
		write_rows('data/' + dirname + '/' + value + '.csv', rows_by_field[value])

def by_category_and_type():
	rows_by_field = {}
	for city in os.listdir(results_dir):
		d_path = os.path.join(results_dir, city)
		if os.path.isdir(d_path):
			posts = []
			for f in os.listdir(d_path):
				if f.endswith('.csv'):
					with open(os.path.join(d_path, f),'r') as in_file:
						next(in_file) # skip column labels
						csv_reader = csv.reader(in_file)
						for line in csv_reader:
							city = line[1]
							title = line[7]
							text = line[8] + title + title # DOUBLE WEIGHT WORDS IN TITLE
							category = line[5]
							posttype = line[6]
							v1 = category
							v2 = posttype
							if not v1:
								v1 = 'none'
							if not v2:
								v2 = 'none'

							value = v1 + '_' + v2
							if value not in rows_by_field:
								rows_by_field[value] = []
							rows_by_field[value].append([text, city, category, posttype]) #text

	for value in rows_by_field:
		write_rows('data/category_type/' + value + '.csv', rows_by_field[value])

"""
split up all posts into those posted by males (for anyone) and those posted by females (for anyone)
* this excludes posts coming from two people (ie mw4m, mm4m, etc)
* this includes both platonic and romantic postss

"""
def by_gender():
	male = []
	female = []
	for city in os.listdir(results_dir):
		d_path = os.path.join(results_dir, city)
		if os.path.isdir(d_path):
			posts = []
			for f in os.listdir(d_path):
				if f.endswith('.csv'):
					with open(os.path.join(d_path, f),'r') as in_file:
						next(in_file) # skip column labels
						csv_reader = csv.reader(in_file)
						for line in csv_reader:
							city = line[1]
							title = line[7]
							text = line[8] + title + title # DOUBLE WEIGHT WORDS IN TITLE
							category = line[5]
							posttype = line[6]
							# split on male and female posters
							if posttype.startswith('m4'):
								male.append([text, city, category, posttype])
							elif posttype.startswith('w4'):
								female.append([text, city, category, posttype])

	write_rows('data/gender/male.csv', male)
	write_rows('data/gender/female.csv', female)

def main():
	#by_city()
	#by_field('type', 6)
	#by_gender()
	by_category_and_type()

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before>from flask import Flask, g, render_template
from app.database import db
from app.extensions import lm
from app.config import base_config
from app.user import user
from app.auth import auth

def create_app(config=base_config):
    app = Flask(__name__)
    app.config.from_object(config)

    from flask.ext.heroku import Heroku
    heroku = Heroku(app)

    db.init_app(app)
    lm.init_app(app)

    register_blueprints(app)
    register_errorhandlers(app)

    @app.before_request
    def before_request():
        import time
        g.request_start_time = time.time()
        g.request_time = lambda: '%.5fs' % (time.time() - g.request_start_time)

    @app.route('/', methods=['GET'])
    def index():
        return render_template('index.html')

    return app

def register_blueprints(app):
    app.register_blueprint(user, url_prefix='/user')
    app.register_blueprint(auth)

def register_errorhandlers(app):
    for e in [401, 404, 500]:
        app.errorhandler(e)(handle_error)

def handle_error(e):
    return render_template('%s.html' % e.code), e.code
<commit_msg>fix error templates path<commit_after>from flask import Flask, g, render_template
from app.database import db
from app.extensions import lm
from app.config import base_config
from app.user import user
from app.auth import auth

def create_app(config=base_config):
    app = Flask(__name__)
    app.config.from_object(config)

    from flask.ext.heroku import Heroku
    heroku = Heroku(app)

    db.init_app(app)
    lm.init_app(app)

    register_blueprints(app)
    register_errorhandlers(app)

    @app.before_request
    def before_request():
        import time
        g.request_start_time = time.time()
        g.request_time = lambda: '%.5fs' % (time.time() - g.request_start_time)

    @app.route('/', methods=['GET'])
    def index():
        return render_template('index.html')

    return app

def register_blueprints(app):
    app.register_blueprint(user, url_prefix='/user')
    app.register_blueprint(auth)

def register_errorhandlers(app):
    for e in [401, 404, 500]:
        app.errorhandler(e)(handle_error)

def handle_error(e):
    return render_template('errors/%s.html' % e.code), e.code
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7

# This scripts lists all files in an ObjectStore container and
# moves them one at a time to HDFS using the webhdfs protocol.
#
# You may need to install swiftclient: `pip install python-swiftclient`

import swiftclient
import requests

#### beg config ####

os_auth_url  = 'https://identity.open.softlayer.com/v3'
os_proj_id   = ''
os_user_id   = ''
os_region    = 'dallas'
os_pass      = ''
os_container = 'Filecontainer'

bi_host      = ''
bi_port      = '8443'
bi_user      = 'biadmin'
bi_pass      = ''
bi_folder    = '' # destination folder in hdfs

#### end config ####

bi_url  = 'https://{0}:{1}/gateway/default'.format(bi_host, bi_port)

# Get a Swift client connection object
def getConn():
    return swiftclient.Connection(
            key          = os_pass,
            authurl      = os_auth_url,
            auth_version = '3',
            os_options={
                "project_id"  : os_proj_id,
                "user_id"     : os_user_id,
                "region_name" : os_region
                })

for os_data in getConn().get_container(os_container)[1]:
    filename = os_data['name']
    filesize = os_data['bytes']

    print("Retrieving: {0} size {1} bytes".format(filename, filesize))

    # NOTE! This reads the whole file into memory - it may
    # crash if you have large files

    obj = getConn().get_object('Filecontainer', filename)

    print("Object size: {0}".format(len(obj[1])))

    url = "{0}/webhdfs/v1/{1}/{2}?op=CREATE".format(bi_url, bi_folder, filename)

    # WARNING! certification verifcation is disabled `verify = False` as per the bluemix
    # documentation example that uses curl with the -k flag

    # First request is to get 'Location' header which contains URL for PUT'ting the data
    response = requests.put(
        url,
        auth = (bi_user, bi_pass),
        verify = False,
        allow_redirects = False
    )

    if not response.status_code == requests.codes.ok:
        print(response.content)
        response.raise_for_status()

    # Save the file to the url obtained from the previous request
    response = requests.put(
        response.headers['Location'],
        auth = (bi_user, bi_pass),
        data = obj[1],
        verify = False
    )

    if not response.status_code == requests.codes.ok:
        print(response.content)
        response.raise_for_status()

    print("Uploaded: {0}".format(filename))
<commit_msg>Update copy_from_swift_to_hdfs.py<commit_after>#!/usr/bin/env python2.7

# This scripts lists all files in an ObjectStore container and
# moves them one at a time to HDFS using the webhdfs protocol.
#
# You may need to install swiftclient: `pip install python-swiftclient`

import swiftclient
import requests

#### beg config ####

os_auth_url  = 'https://identity.open.softlayer.com/v3'
os_proj_id   = ''
os_user_id   = ''
os_region    = 'dallas'
os_pass      = ''
os_container = 'Filecontainer'

ignore_files = ['somefile','someotherfile']

bi_host      = ''
bi_port      = '8443'
bi_user      = 'biadmin'
bi_pass      = ''
bi_folder    = '' # destination folder in hdfs

#### end config ####

bi_url  = 'https://{0}:{1}/gateway/default'.format(bi_host, bi_port)

# Get a Swift client connection object
def getConn():
    return swiftclient.Connection(
            key          = os_pass,
            authurl      = os_auth_url,
            auth_version = '3',
            os_options={
                "project_id"  : os_proj_id,
                "user_id"     : os_user_id,
                "region_name" : os_region
                })

for os_data in getConn().get_container(os_container)[1]:
    filename = os_data['name']
    filesize = os_data['bytes']

    if filename in ignore_files:
        continue

    print("Retrieving: {0} size {1} bytes".format(filename, filesize))

    # NOTE! This reads the whole file into memory - it may
    # crash if you have large files

    obj = getConn().get_object('Filecontainer', filename)

    print("Object size: {0}".format(len(obj[1])))

    url = "{0}/webhdfs/v1/{1}/{2}?op=CREATE".format(bi_url, bi_folder, filename)

    # WARNING! certification verifcation is disabled `verify = False` as per the bluemix
    # documentation example that uses curl with the -k flag

    # First request is to get 'Location' header which contains URL for PUT'ting the data
    response = requests.put(
        url,
        auth = (bi_user, bi_pass),
        verify = False,
        allow_redirects = False
    )

    if not response.status_code == requests.codes.ok:
        print(response.content)
        response.raise_for_status()

    # Save the file to the url obtained from the previous request
    response = requests.put(
        response.headers['Location'],
        auth = (bi_user, bi_pass),
        data = obj[1],
        verify = False
    )

    if not response.status_code == requests.codes.ok:
        print(response.content)
        response.raise_for_status()

    print("Uploaded: {0}".format(filename))
<|endoftext|>


<commit_before>import discord
from discord.ext import commands

from utils import aiohttp_wrap as aw

class MusicInfo:
    """A cog for retrieving music information (not playing it)"""
    URL = 'http://ws.audioscrobbler.com/2.0/'
    LOGO_URL = 'https://www.last.fm/static/images/lastfm_logo_facebook.1b63d4451dcc.png'
    NO_RESULT = "Sorry, couldn't find anything for {}."
    EM_COLOR = discord.Color.blurple()

    def __init__(self, bot):
        self.TOKEN = bot.api_keys['lastfm']

    @staticmethod
    def truncat(text: str, length=500) -> str:
        if len(text) > length:
            return f'{text[:length - 3]}...'

        return text

    @commands.group(aliases=['lastfm', 'fm'])
    async def music(self, ctx):
        """Main group cmd"""

    @music.command()
    async def album(self, ctx, *, query):
        """Search for some basic album information via album name"""
        search_params = {'method': 'album.search', 'album': query, 'limit': 5,
                         'format': 'json', 'api_key': self.TOKEN}
        search_resp = await aw.aio_get_json(ctx.bot.aio_session, self.URL, params=search_params)

        # API didn't respond
        if search_resp is None:
            return await ctx.error('There was a problem with the last.fm API, try again later.')

        # No results
        if len(search_resp['results']['albummatches']['album']) == 0:
            return await ctx.error(self.NO_RESULT.format(query))

        search_result = search_resp['results']['albummatches']['album'][0]
        name = search_result['name']
        artist = search_result['artist']

        # Once we've found the matching album, we've gotta do ANOTHER request
        info_params = {'method': 'album.getInfo', 'artist': artist, 'album': name,
                       'format': 'json', 'api_key': self.TOKEN}
        info_resp = await aw.aio_get_json(ctx.bot.aio_session, self.URL, params=info_params)

        em = discord.Embed(title=f'{artist} - {name}',
                           url=info_resp['album']['url'],
                           color=self.EM_COLOR)
        em.set_thumbnail(url=info_resp['album']['image'][-1]['#text'])
        # If there's some wiki info, include it
        if info_resp['album']['wiki']['summary']:
            em.description = self.truncat(info_resp['album']['wiki']['summary'])

        # Get and number the tracks in a list
        tracks = [f"{idx}. {x['name']}" for idx, x in enumerate(info_resp['album']['tracks']['track'])]
        em.add_field(name='Track List', value='\n'.join(tracks))

        # Attribution or whatever
        em.set_footer(text='Powered by last.fm', icon_url=self.LOGO_URL)

        await ctx.send(embed=em)

def setup(bot):
    bot.add_cog(MusicInfo(bot))
<commit_msg>fix numbering of tracks and description<commit_after>import discord
from discord.ext import commands

from utils import aiohttp_wrap as aw

class MusicInfo:
    """A cog for retrieving music information (not playing it)"""
    URL = 'http://ws.audioscrobbler.com/2.0/'
    LOGO_URL = 'https://www.last.fm/static/images/lastfm_logo_facebook.1b63d4451dcc.png'
    NO_RESULT = "Sorry, couldn't find anything for {}."
    EM_COLOR = discord.Color.blurple()

    def __init__(self, bot):
        self.TOKEN = bot.api_keys['lastfm']

    @staticmethod
    def truncat(text: str, length=500) -> str:
        if len(text) > length:
            return f'{text[:length - 3]}...'

        return text

    @commands.group(aliases=['lastfm', 'fm'])
    async def music(self, ctx):
        """Main group cmd"""

    @music.command()
    async def album(self, ctx, *, query):
        """Search for some basic album information via album name"""
        search_params = {'method': 'album.search', 'album': query, 'limit': 5,
                         'format': 'json', 'api_key': self.TOKEN}
        search_resp = await aw.aio_get_json(ctx.bot.aio_session, self.URL, params=search_params)

        # API didn't respond
        if search_resp is None:
            return await ctx.error('There was a problem with the last.fm API, try again later.')

        # No results
        if len(search_resp['results']['albummatches']['album']) == 0:
            return await ctx.error(self.NO_RESULT.format(query))

        search_result = search_resp['results']['albummatches']['album'][0]
        name = search_result['name']
        artist = search_result['artist']

        # Once we've found the matching album, we've gotta do ANOTHER request
        info_params = {'method': 'album.getInfo', 'artist': artist, 'album': name,
                       'format': 'json', 'api_key': self.TOKEN}
        info_resp = await aw.aio_get_json(ctx.bot.aio_session, self.URL, params=info_params)

        em = discord.Embed(title=f'{artist} - {name}',
                           url=info_resp['album']['url'],
                           color=self.EM_COLOR)
        em.set_thumbnail(url=info_resp['album']['image'][-1]['#text'])
        # If there's some wiki info, include it
        if 'wiki' in info_resp['album']:
            em.description = self.truncat(info_resp['album']['wiki']['summary'])

        # Get and number the tracks in a list
        tracks = [f"{idx + 1}. {x['name']}" for idx, x in enumerate(info_resp['album']['tracks']['track'])]
        em.add_field(name='Track List', value='\n'.join(tracks))

        # Attribution or whatever
        em.set_footer(text='Powered by last.fm', icon_url=self.LOGO_URL)

        await ctx.send(embed=em)

def setup(bot):
    bot.add_cog(MusicInfo(bot))
<|endoftext|>


<commit_before># This file is part of LiSE, a framework for life simulation games.
# Copyright (c) 2013-2014 Zachary Spector,  zacharyspector@gmail.com
import networkx as nx
from gorm.xjson import json_dump
from .node import Node
from .util import (
    path_len,
    CacheError,
    TravelException,
    encache,
    enkeycache,
    dekeycache,
    cache_forward,
    needcache
)

class Thing(Node):
    """The sort of item that has a particular location at any given time.

    If a Thing is in a Place, it is standing still. If it is in a
    Portal, it is moving through that Portal however fast it must in
    order to arrive at the other end when it is scheduled to. If it is
    in another Thing, then it is wherever that is, and moving the
    same.

    """
    def __init__(self, *args, **kwargs):
        """If caching, initialize the cache."""
        super().__init__(*args, **kwargs)
        self._loccache = {}

    def __iter__(self):
        """Iterate over a cached set of keys if possible and caching's
        enabled.

        Iterate over some special keys too.

        """
        extrakeys = [
            'name',
            'character',
            'location',
            'next_location',
            'arrival_time',
            'next_arrival_time'
        ]
        if not self.engine.caching:
            yield from extrakeys
            yield from super().__iter__()
            return
        (branch, tick) = self.engine.time
        if branch not in self._keycache:
            self._keycache[branch] = {}
        if tick not in self._keycache[branch]:
            self._keycache[branch][tick] = set(
                extrakeys + list(super().__iter__())
            )
        yield from self._keycache[branch][tick]

    def __contains__(self, key):
        if key in (
                'name',
                'character',
                'location',
                'next_location',
                'arrival_time',
                'next_arrival_time',
                'locations'
        ):
            return True
        return super().__contains__(key)

    def __getitem__(self, key):
        """Return one of my stats stored in the database, or a few
        special cases:

        ``name``: return the name that uniquely identifies me within
        my Character

        ``character``: return the name of my character

        ``location``: return the name of my location

        ``arrival_time``: return the tick when I arrived in the
        present location

        ``next_location``: if I'm in transit, return where to, else return None

        ``next_arrival_time``: return the tick when I'm going to
        arrive at ``next_location``

        ``locations``: return a pair of ``(location, next_location)``

        """
        if key == 'name':
            return self.name
        elif key == 'character':
            return self.character.name
        elif key == 'location':
            return self['locations'][0]
        elif key == 'arrival_time':
            if not self.engine.caching:
                return self._get_arrival_time()
            for (branch, tick) in self.engine._active_branches():
                self._load_locs_branch(branch)
                try:
                    return max(
                        t for t in self._loccache[branch]
                        if t <= tick
                    )
                except ValueError:
                    continue
            raise CacheError("Locations not cached correctly")
        elif key == 'next_location':
            return self['locations'][1]
        elif key == 'next_arrival_time':
            if not self.engine.caching:
                return self._get_next_arrival_time()
            for (branch, tick) in self.engine._active_branches():
                self._load_locs_branch(branch)
                try:
                    return min(
                        t for t in self._loccache[branch]
                        if t > tick
                    )
                except ValueError:
                    continue
            return None
        elif key == 'locations':
            if not self.engine.caching:
                return self._loc_and_next()
            for (branch, tick) in self.engine._active_branches():
                self._load_locs_branch(branch)
                try:
                    return self._loccache[branch][self['arrival_time']]
                except ValueError:
                    continue
            raise CacheError("Locations not cached correctly")
        else:
            if not self.engine.caching:
                return super().__getitem__(key)
            (branch, tick) = self.engine.time
            cache_forward(self._cache, key, branch, tick)
            if needcache(self._cache, key, branch, tick):
                encache(self, self._cache, key, super().__getitem__(key))
            r = self._cache[key][branch][tick]
            if r is None:
                raise KeyError("Key {} is not set now".format(key))
            return r

    def __setitem__(self, key, value):
        """Set ``key``=``value`` for the present game-time."""
        if key == 'name':
            raise ValueError("Can't change names")
        elif key == 'character':
            raise ValueError("Can't change characters")
        elif key == 'location':
            self['locations'] = (value, None)
        elif key == 'arrival_time':
            raise ValueError("Read-only")
        elif key == 'next_location':
            self['locations'] = (self['location'], value)
        elif key == 'next_arrival_time':
            raise ValueError("Read-only")
        elif key == 'locations':
            self._set_loc_and_next(*value)
            if hasattr(self._cache, '_fresh'):
                del self._cache._fresh
            if not self.engine.caching:
                return
            (branch, tick) = self.engine.time
            self._load_locs_branch(branch)
            self._loccache[branch][tick] = value
            self._dispatch_stat('locations', value)
        else:
            super().__setitem__(key, value)
            if self.engine.caching:
                encache(self, self._cache, key, value)
                enkeycache(self, self._keycache, key)

    def __delitem__(self, key):
        """As of now, this key isn't mine."""
        if key in (
                'name',
                'character',
                'location',
                'arrival_time',
                'next_location',
                'next_arrival_time',
                'locations'
        ):
            raise ValueError("Can't delete {}".format(key))
        super().__delitem__(key)
        if self.engine.caching:
            encache(self, self._cache, key, None)
            dekeycache(self, self._keycache, key)

    def _load_locs_branch(self, branch):
        """Private method. Cache stored location data for this branch."""
        self._loccache[branch] = {}
        for (tick, loc, nloc) in self.engine.db.thing_locs_data(
                self.character.name,
                self.name,
                branch
        ):
            self._loccache[branch][tick] = (loc, nloc)

    def _get_arrival_time(self):
        """Query the database for when I arrive at my present location."""
        loc = self['location']
        return self.engine.db.arrival_time_get(
            self.character.name,
            self.name,
            loc,
            *self.engine.time
        )

    def _get_next_arrival_time(self):
        """Query the database for when I will arrive at my next location, or
        ``None`` if I'm not traveling.

        """
        nextloc = self['next_location']
        if nextloc is None:
            return None
        return self.engine.db.next_arrival_time_get(
            self.character.name,
            self.name,
            nextloc,
            *self.engine.time
        )

    def delete(self):
        """Delete every reference to me."""
        del self.character.thing[self.name]
        super().delete()

    def clear(self):
        """Unset everything."""
        for k in list(self.keys()):
            if k not in (
                    'name',
                    'character',
                    'location',
                    'next_location',
                    'arrival_time',
                    'next_arrival_time',
                    'locations'
            ):
                del self[k]

    @property
    def container(self):
        """If I am in transit, this is the Portal I'm moving through. Otherwise
        it's the Thing or Place I'm located in.

        """
        (a, b) = self['locations']
        try:
            return self.character.portal[a][b]
        except KeyError:
            try:
                return self.character.thing[a]
            except KeyError:
                return self.character.place[a]

    @property
    def location(self):
        """The Thing or Place I'm in. If I'm in transit, it's where I
        started.

        """
        if not self['location']:
            return None
        return self.character.node[self['location']]

    @property
    def next_location(self):
        """If I'm not in transit, this is None. If I am, it's where I'm
        headed.

        """
        locn = self['next_location']
        if not locn:
            return None
        try:
            return self.character.thing[locn]
        except KeyError:
            return self.character.place[locn]

    def _loc_and_next(self):
        """Private method that returns a pair in which the first item is my
        present ``location`` and the second is my ``next_location``,
        to which I am presently travelling.

        """
        return self.engine.db.thing_loc_and_next_get(
            self.character.name,
            self.name,
            *self.engine.time
        )

    def _set_loc_and_next(self, loc, nextloc):
        """Private method to simultaneously set ``location`` and
        ``next_location``

        """
        self.engine.db.thing_loc_and_next_set(
            self.character.name,
            self.name,
            self.engine.branch,
            self.engine.tick,
            loc,
            nextloc
        )

    def go_to_place(self, place, weight=''):
        """Assuming I'm in a :class:`Place` that has a :class:`Portal` direct
        to the given :class:`Place`, schedule myself to travel to the
        given :class:`Place`, taking an amount of time indicated by
        the ``weight`` stat on the :class:`Portal`, if given; else 1
        tick.

        Return the number of ticks to travel.

        """
        if hasattr(place, 'name'):
            placen = place.name
        else:
            placen = place
        for fun in self.character.travel_reqs:
            if not fun(self.name, placen):
                (branch, tick) = self.engine.time
                raise TravelException(
                    "{} cannot travel through {}".format(self.name, placen),
                    traveller=self,
                    branch=branch,
                    tick=tick,
                    lastplace=self.location
                )
        curloc = self["location"]
        orm = self.character.engine
        curtick = orm.tick
        ticks = self.character.portal[curloc][placen].get(weight, 1)
        self['next_location'] = placen
        orm.tick += ticks
        self['locations'] = (placen, None)
        orm.tick = curtick
        return ticks

    def follow_path(self, path, weight=None):
        """Go to several :class:`Place`s in succession, deciding how long to
        spend in each by consulting the ``weight`` stat of the
        :class:`Portal` connecting the one :class:`Place` to the next.

        Return the total number of ticks the travel will take. Raise
        :class:`TravelException` if I can't follow the whole path,
        either because some of its nodes don't exist, or because I'm
        scheduled to be somewhere else.

        """
        curtick = self.character.engine.tick
        prevplace = path.pop(0)
        if prevplace != self['location']:
            raise ValueError("Path does not start at my present location")
        subpath = [prevplace]
        for place in path:
            if (
                    prevplace not in self.character.portal or
                    place not in self.character.portal[prevplace]
            ):
                raise TravelException(
                    "Couldn't follow portal from {} to {}".format(
                        prevplace,
                        place
                    ),
                    path=subpath,
                    traveller=self
                )
            subpath.append(place)
            prevplace = place
        ticks_total = 0
        prevsubplace = subpath.pop(0)
        subsubpath = [prevsubplace]
        for subplace in subpath:
            if prevsubplace != self["location"]:
                l = self["location"]
                fintick = self.character.engine.tick
                self.character.engine.tick = curtick
                raise TravelException(
                    "When I tried traveling to {}, at tick {}, "
                    "I ended up at {}".format(
                        prevsubplace,
                        fintick,
                        l
                    ),
                    path=subpath,
                    followed=subsubpath,
                    branch=self.character.engine.branch,
                    tick=fintick,
                    lastplace=l,
                    traveller=self
                )
            portal = self.character.portal[prevsubplace][subplace]
            tick_inc = portal.get(weight, 1)
            self.go_to_place(subplace, weight)
            self.character.engine.tick += tick_inc
            ticks_total += tick_inc
            subsubpath.append(subplace)
            prevsubplace = subplace
        self.character.engine.tick = curtick
        return ticks_total

    def travel_to(self, dest, weight=None, graph=None):
        """Find the shortest path to the given :class:`Place` from where I am
        now, and follow it.

        If supplied, the ``weight`` stat of the :class:`Portal`s along
        the path will be used in pathfinding, and for deciding how
        long to stay in each Place along the way.

        The ``graph`` argument may be any NetworkX-style graph. It
        will be used for pathfinding if supplied, otherwise I'll use
        my :class:`Character`. In either case, however, I will attempt
        to actually follow the path using my :class:`Character`, which
        might not be possible if the supplied ``graph`` and my
        :class:`Character` are too different. If it's not possible,
        I'll raise a :class:`TravelException`, whose ``subpath``
        attribute holds the part of the path that I *can* follow. To
        make me follow it, pass it to my ``follow_path`` method.

        Return value is the number of ticks the travel will take.

        """
        destn = dest.name if hasattr(dest, 'name') else dest
        graph = self.character if graph is None else graph
        path = nx.shortest_path(graph, self["location"], destn, weight)
        return self.follow_path(path, weight)

    def travel_to_by(self, dest, arrival_tick, weight=None, graph=None):
        """Arrange to travel to ``dest`` such that I arrive there at
        ``arrival_tick``.

        Optional argument ``weight`` indicates what attribute of
        portals will indicate how long they take to go through.

        Optional argument ``graph`` is the graph to perform
        pathfinding with. If it contains a viable path that my
        character does not, you'll get a TravelException.

        """
        curtick = self.character.engine.tick
        if arrival_tick <= curtick:
            raise ValueError("travel always takes positive amount of time")
        destn = dest.name if hasattr(dest, 'name') else dest
        graph = self.character if graph is None else graph
        curloc = self["location"]
        path = nx.shortest_path(graph, curloc, destn, weight)
        travel_time = path_len(graph, path, weight)
        start_tick = arrival_tick - travel_time
        if start_tick <= curtick:
            raise self.TravelException(
                "path too heavy to follow by the specified tick",
                path=path,
                traveller=self
            )
        self.character.engine.tick = start_tick
        self.follow_path(path, weight)
        self.character.engine.tick = curtick

    def _get_json_dict(self):
        (branch, tick) = self.character.engine.time
        return {
            "type": "Thing",
            "version": 0,
            "character": self.character.name,
            "name": self.name,
            "branch": branch,
            "tick": tick,
            "stat": dict(self)
        }

    def dump(self):
        """Return a JSON representation of my present state only, not any of
        my history.

        """
        return json_dump(self._get_json_dict())
<commit_msg>remove a bit left over from a failed attempt at implementing the cache<commit_after># This file is part of LiSE, a framework for life simulation games.
# Copyright (c) 2013-2014 Zachary Spector,  zacharyspector@gmail.com
import networkx as nx
from gorm.xjson import json_dump
from .node import Node
from .util import (
    path_len,
    CacheError,
    TravelException,
    encache,
    enkeycache,
    dekeycache,
    cache_forward,
    needcache
)

class Thing(Node):
    """The sort of item that has a particular location at any given time.

    If a Thing is in a Place, it is standing still. If it is in a
    Portal, it is moving through that Portal however fast it must in
    order to arrive at the other end when it is scheduled to. If it is
    in another Thing, then it is wherever that is, and moving the
    same.

    """
    def __init__(self, *args, **kwargs):
        """If caching, initialize the cache."""
        super().__init__(*args, **kwargs)
        self._loccache = {}

    def __iter__(self):
        """Iterate over a cached set of keys if possible and caching's
        enabled.

        Iterate over some special keys too.

        """
        extrakeys = [
            'name',
            'character',
            'location',
            'next_location',
            'arrival_time',
            'next_arrival_time'
        ]
        if not self.engine.caching:
            yield from extrakeys
            yield from super().__iter__()
            return
        (branch, tick) = self.engine.time
        if branch not in self._keycache:
            self._keycache[branch] = {}
        if tick not in self._keycache[branch]:
            self._keycache[branch][tick] = set(
                extrakeys + list(super().__iter__())
            )
        yield from self._keycache[branch][tick]

    def __contains__(self, key):
        if key in (
                'name',
                'character',
                'location',
                'next_location',
                'arrival_time',
                'next_arrival_time',
                'locations'
        ):
            return True
        return super().__contains__(key)

    def __getitem__(self, key):
        """Return one of my stats stored in the database, or a few
        special cases:

        ``name``: return the name that uniquely identifies me within
        my Character

        ``character``: return the name of my character

        ``location``: return the name of my location

        ``arrival_time``: return the tick when I arrived in the
        present location

        ``next_location``: if I'm in transit, return where to, else return None

        ``next_arrival_time``: return the tick when I'm going to
        arrive at ``next_location``

        ``locations``: return a pair of ``(location, next_location)``

        """
        if key == 'name':
            return self.name
        elif key == 'character':
            return self.character.name
        elif key == 'location':
            return self['locations'][0]
        elif key == 'arrival_time':
            if not self.engine.caching:
                return self._get_arrival_time()
            for (branch, tick) in self.engine._active_branches():
                self._load_locs_branch(branch)
                try:
                    return max(
                        t for t in self._loccache[branch]
                        if t <= tick
                    )
                except ValueError:
                    continue
            raise CacheError("Locations not cached correctly")
        elif key == 'next_location':
            return self['locations'][1]
        elif key == 'next_arrival_time':
            if not self.engine.caching:
                return self._get_next_arrival_time()
            for (branch, tick) in self.engine._active_branches():
                self._load_locs_branch(branch)
                try:
                    return min(
                        t for t in self._loccache[branch]
                        if t > tick
                    )
                except ValueError:
                    continue
            return None
        elif key == 'locations':
            if not self.engine.caching:
                return self._loc_and_next()
            for (branch, tick) in self.engine._active_branches():
                self._load_locs_branch(branch)
                try:
                    return self._loccache[branch][self['arrival_time']]
                except ValueError:
                    continue
            raise CacheError("Locations not cached correctly")
        else:
            if not self.engine.caching:
                return super().__getitem__(key)
            (branch, tick) = self.engine.time
            cache_forward(self._cache, key, branch, tick)
            if needcache(self._cache, key, branch, tick):
                encache(self, self._cache, key, super().__getitem__(key))
            r = self._cache[key][branch][tick]
            if r is None:
                raise KeyError("Key {} is not set now".format(key))
            return r

    def __setitem__(self, key, value):
        """Set ``key``=``value`` for the present game-time."""
        if key == 'name':
            raise ValueError("Can't change names")
        elif key == 'character':
            raise ValueError("Can't change characters")
        elif key == 'location':
            self['locations'] = (value, None)
        elif key == 'arrival_time':
            raise ValueError("Read-only")
        elif key == 'next_location':
            self['locations'] = (self['location'], value)
        elif key == 'next_arrival_time':
            raise ValueError("Read-only")
        elif key == 'locations':
            self._set_loc_and_next(*value)
            if not self.engine.caching:
                return
            (branch, tick) = self.engine.time
            self._load_locs_branch(branch)
            self._loccache[branch][tick] = value
            self._dispatch_stat('locations', value)
        else:
            super().__setitem__(key, value)
            if self.engine.caching:
                encache(self, self._cache, key, value)
                enkeycache(self, self._keycache, key)

    def __delitem__(self, key):
        """As of now, this key isn't mine."""
        if key in (
                'name',
                'character',
                'location',
                'arrival_time',
                'next_location',
                'next_arrival_time',
                'locations'
        ):
            raise ValueError("Can't delete {}".format(key))
        super().__delitem__(key)
        if self.engine.caching:
            encache(self, self._cache, key, None)
            dekeycache(self, self._keycache, key)

    def _load_locs_branch(self, branch):
        """Private method. Cache stored location data for this branch."""
        self._loccache[branch] = {}
        for (tick, loc, nloc) in self.engine.db.thing_locs_data(
                self.character.name,
                self.name,
                branch
        ):
            self._loccache[branch][tick] = (loc, nloc)

    def _get_arrival_time(self):
        """Query the database for when I arrive at my present location."""
        loc = self['location']
        return self.engine.db.arrival_time_get(
            self.character.name,
            self.name,
            loc,
            *self.engine.time
        )

    def _get_next_arrival_time(self):
        """Query the database for when I will arrive at my next location, or
        ``None`` if I'm not traveling.

        """
        nextloc = self['next_location']
        if nextloc is None:
            return None
        return self.engine.db.next_arrival_time_get(
            self.character.name,
            self.name,
            nextloc,
            *self.engine.time
        )

    def delete(self):
        """Delete every reference to me."""
        del self.character.thing[self.name]
        super().delete()

    def clear(self):
        """Unset everything."""
        for k in list(self.keys()):
            if k not in (
                    'name',
                    'character',
                    'location',
                    'next_location',
                    'arrival_time',
                    'next_arrival_time',
                    'locations'
            ):
                del self[k]

    @property
    def container(self):
        """If I am in transit, this is the Portal I'm moving through. Otherwise
        it's the Thing or Place I'm located in.

        """
        (a, b) = self['locations']
        try:
            return self.character.portal[a][b]
        except KeyError:
            try:
                return self.character.thing[a]
            except KeyError:
                return self.character.place[a]

    @property
    def location(self):
        """The Thing or Place I'm in. If I'm in transit, it's where I
        started.

        """
        if not self['location']:
            return None
        return self.character.node[self['location']]

    @property
    def next_location(self):
        """If I'm not in transit, this is None. If I am, it's where I'm
        headed.

        """
        locn = self['next_location']
        if not locn:
            return None
        try:
            return self.character.thing[locn]
        except KeyError:
            return self.character.place[locn]

    def _loc_and_next(self):
        """Private method that returns a pair in which the first item is my
        present ``location`` and the second is my ``next_location``,
        to which I am presently travelling.

        """
        return self.engine.db.thing_loc_and_next_get(
            self.character.name,
            self.name,
            *self.engine.time
        )

    def _set_loc_and_next(self, loc, nextloc):
        """Private method to simultaneously set ``location`` and
        ``next_location``

        """
        self.engine.db.thing_loc_and_next_set(
            self.character.name,
            self.name,
            self.engine.branch,
            self.engine.tick,
            loc,
            nextloc
        )

    def go_to_place(self, place, weight=''):
        """Assuming I'm in a :class:`Place` that has a :class:`Portal` direct
        to the given :class:`Place`, schedule myself to travel to the
        given :class:`Place`, taking an amount of time indicated by
        the ``weight`` stat on the :class:`Portal`, if given; else 1
        tick.

        Return the number of ticks to travel.

        """
        if hasattr(place, 'name'):
            placen = place.name
        else:
            placen = place
        for fun in self.character.travel_reqs:
            if not fun(self.name, placen):
                (branch, tick) = self.engine.time
                raise TravelException(
                    "{} cannot travel through {}".format(self.name, placen),
                    traveller=self,
                    branch=branch,
                    tick=tick,
                    lastplace=self.location
                )
        curloc = self["location"]
        orm = self.character.engine
        curtick = orm.tick
        ticks = self.character.portal[curloc][placen].get(weight, 1)
        self['next_location'] = placen
        orm.tick += ticks
        self['locations'] = (placen, None)
        orm.tick = curtick
        return ticks

    def follow_path(self, path, weight=None):
        """Go to several :class:`Place`s in succession, deciding how long to
        spend in each by consulting the ``weight`` stat of the
        :class:`Portal` connecting the one :class:`Place` to the next.

        Return the total number of ticks the travel will take. Raise
        :class:`TravelException` if I can't follow the whole path,
        either because some of its nodes don't exist, or because I'm
        scheduled to be somewhere else.

        """
        curtick = self.character.engine.tick
        prevplace = path.pop(0)
        if prevplace != self['location']:
            raise ValueError("Path does not start at my present location")
        subpath = [prevplace]
        for place in path:
            if (
                    prevplace not in self.character.portal or
                    place not in self.character.portal[prevplace]
            ):
                raise TravelException(
                    "Couldn't follow portal from {} to {}".format(
                        prevplace,
                        place
                    ),
                    path=subpath,
                    traveller=self
                )
            subpath.append(place)
            prevplace = place
        ticks_total = 0
        prevsubplace = subpath.pop(0)
        subsubpath = [prevsubplace]
        for subplace in subpath:
            if prevsubplace != self["location"]:
                l = self["location"]
                fintick = self.character.engine.tick
                self.character.engine.tick = curtick
                raise TravelException(
                    "When I tried traveling to {}, at tick {}, "
                    "I ended up at {}".format(
                        prevsubplace,
                        fintick,
                        l
                    ),
                    path=subpath,
                    followed=subsubpath,
                    branch=self.character.engine.branch,
                    tick=fintick,
                    lastplace=l,
                    traveller=self
                )
            portal = self.character.portal[prevsubplace][subplace]
            tick_inc = portal.get(weight, 1)
            self.go_to_place(subplace, weight)
            self.character.engine.tick += tick_inc
            ticks_total += tick_inc
            subsubpath.append(subplace)
            prevsubplace = subplace
        self.character.engine.tick = curtick
        return ticks_total

    def travel_to(self, dest, weight=None, graph=None):
        """Find the shortest path to the given :class:`Place` from where I am
        now, and follow it.

        If supplied, the ``weight`` stat of the :class:`Portal`s along
        the path will be used in pathfinding, and for deciding how
        long to stay in each Place along the way.

        The ``graph`` argument may be any NetworkX-style graph. It
        will be used for pathfinding if supplied, otherwise I'll use
        my :class:`Character`. In either case, however, I will attempt
        to actually follow the path using my :class:`Character`, which
        might not be possible if the supplied ``graph`` and my
        :class:`Character` are too different. If it's not possible,
        I'll raise a :class:`TravelException`, whose ``subpath``
        attribute holds the part of the path that I *can* follow. To
        make me follow it, pass it to my ``follow_path`` method.

        Return value is the number of ticks the travel will take.

        """
        destn = dest.name if hasattr(dest, 'name') else dest
        graph = self.character if graph is None else graph
        path = nx.shortest_path(graph, self["location"], destn, weight)
        return self.follow_path(path, weight)

    def travel_to_by(self, dest, arrival_tick, weight=None, graph=None):
        """Arrange to travel to ``dest`` such that I arrive there at
        ``arrival_tick``.

        Optional argument ``weight`` indicates what attribute of
        portals will indicate how long they take to go through.

        Optional argument ``graph`` is the graph to perform
        pathfinding with. If it contains a viable path that my
        character does not, you'll get a TravelException.

        """
        curtick = self.character.engine.tick
        if arrival_tick <= curtick:
            raise ValueError("travel always takes positive amount of time")
        destn = dest.name if hasattr(dest, 'name') else dest
        graph = self.character if graph is None else graph
        curloc = self["location"]
        path = nx.shortest_path(graph, curloc, destn, weight)
        travel_time = path_len(graph, path, weight)
        start_tick = arrival_tick - travel_time
        if start_tick <= curtick:
            raise self.TravelException(
                "path too heavy to follow by the specified tick",
                path=path,
                traveller=self
            )
        self.character.engine.tick = start_tick
        self.follow_path(path, weight)
        self.character.engine.tick = curtick

    def _get_json_dict(self):
        (branch, tick) = self.character.engine.time
        return {
            "type": "Thing",
            "version": 0,
            "character": self.character.name,
            "name": self.name,
            "branch": branch,
            "tick": tick,
            "stat": dict(self)
        }

    def dump(self):
        """Return a JSON representation of my present state only, not any of
        my history.

        """
        return json_dump(self._get_json_dict())
<|endoftext|>