<commit_before>import string
import os
import os.path
import time
import tempfile

from errno import EEXIST, ENOENT

from openid.consumer.stores import OpenIDStore, ConsumerAssociation
from openid import util

filename_allowed = string.letters + string.digits + '.-'
# Convert this to something whose __contains__ is fast
try:
    # 2.4
    alphanum = set(filename_allowed)
except NameError:
    try:
        # 2.3
        import sets
    except ImportError:
        # pre-2.3
        _d = {}
        for c in filename_allowed:
            _d[c] = None
        filename_allowed = _d
    else:
        filename_allowed = sets.Set(filename_allowed)

# The ordering and name of keys as stored by serializeAssociation
assoc_keys = [
    'version',
    'server_url',
    'handle',
    'secret',
    'issued',
    'lifetime',
    ]

def serializeAssociation(assoc):
    """Convert an association to kvform

    Inverse of deserializeAssociation
    """
    data = [
        '1',
        assoc.server_url,
        assoc.handle,
        util.to_b64(assoc.secret),
        str(int(assoc.issued)),
        str(int(assoc.lifetime)),
        ]

    assert len(data) == len(assoc_keys)

    lines = []
    for k, v in zip(assoc_keys, data):
        assert ':' not in k
        if '\n' in v:
            fmt = 'Invalid value serializing association: field %r, value %r'
            raise ValueError(fmt % (k, v))

        lines.append('%s: %s\n' % (k, v))

    return ''.join(lines)

def deserializeAssociation(assoc_s):
    """Parse an association as stored by serializeAssociation.

    inverse of serializeAssociation
    """
    lines = assoc_s.split('\n')
    if lines.pop() != '':
        raise ValueError('Missing trailing newline!')
    keys = []
    values = []
    for line in lines:
        k, v = line.split(': ')
        keys.append(k)
        values.append(v)

    if keys != assoc_keys:
        raise ValueError('Unexpected key values: %r', keys)

    version, server_url, handle, secret, issued, lifetime = values
    if version != '1':
        raise ValueError('Unknown version: %r' % version)
    issued = int(issued)
    lifetime = int(lifetime)
    secret = util.from_b64(secret)
    return ConsumerAssociation(server_url, handle, secret, issued, lifetime)

def removeIfPresent(filename):
    """Attempt to remove a file, returning whether the file existed at
    the time of the call.

    str -> bool
    """
    try:
        os.unlink(filename)
    except OSError, why:
        if why[0] == ENOENT:
            # Someone beat us to it, but it's gone, so that's OK
            return False
        else:
            raise
    else:
        # File was present
        return True

def ensureDir(dir_name):
    """Create dir_name as a directory if it does not exist. If it
    exists, make sure that it is, in fact, a directory.

    Can raise OSError

    str -> NoneType
    """
    try:
        os.makedirs(dir_name)
    except OSError, why:
        if why[0] != EEXIST or not os.path.isdir(dir_name):
            raise

class FilesystemOpenIDStore(object):
    """Filesystem-based store for OpenID associations and nonces.

    Methods of this object can raise OSError if unexpected filesystem
    conditions, such as bad permissions or missing directories, occur.
    """

    AUTH_KEY_LEN = 20

    def __init__(self, directory):
        """Initialize the nonce and association directories"""
        self.nonce_dir = os.path.join(directory, 'nonces')

        self.association_dir = os.path.join(directory, 'associations')

        # Temp dir must be on the same filesystem as the assciations
        # directory and the directory containing the auth key file.
        self.temp_dir = os.path.join(directory, 'temp')

        self.auth_key_name = os.path.join(directory, 'auth_key')

        self.max_nonce_age = 6 * 60 * 60 # Six hours, in seconds

        self._setup()

    def _setup(self):
        """Make sure that the directories in which we store our data
        exist.

        () -> NoneType
        """
        ensureDir(os.path.dirname(self.auth_key_name))
        ensureDir(self.nonce_dir)
        ensureDir(self.association_dir)
        ensureDir(self.temp_dir)

    def _mktemp(self):
        """Create a temporary file on the same filesystem as
        self.auth_key_name and self.association_dir.

        The temporary directory should not be cleaned if there are any
        processes using the store. If there is no active process using
        the store, it is safe to remove all of the files in the
        temporary directory.

        () -> (file, str)
        """
        fd, name = tempfile.mkstemp(dir=self.temp_dir)
        try:
            file_obj = os.fdopen(fd, 'wb')
            return file_obj, name
        except:
            removeIfPresent(name)
            raise

    def readAuthKey(self):
        """Read the auth key from the auth key file. Will return None
        if there is currently no key.

        () -> str or NoneType
        """
        try:
            auth_key_file = file(self.auth_key_name, 'rb')
        except IOError, why:
            if why[0] == ENOENT:
                return None
            else:
                raise

        try:
            return auth_key_file.read()
        finally:
            auth_key_file.close()

    def setAuthKey(self, auth_key):
        """Safely store the given auth key in the location specified
        by self.auth_key_name.

        It is possible that if there is a power loss or other fatal
        condition, this store will be left without an auth key. In
        that case, one will be generated unless this is called again.

        If other processes are setting different auth keys (especially
        if createAuthKey is being used), then it is possible that this
        function will fail with an OSError. To ensure that this
        function succeeds, it is necessary that there is no other
        process attempting to set a different key. Multiple processes
        attempting to set the same key should not cause any problems.

        str -> NoneType
        """
        if len(auth_key) != self.AUTH_KEY_LEN:
            fmt = ('Attempted to set invalid auth key. Expected %d byte '
                   'string. Got: %r')
            raise ValueError(fmt % (self.AUTH_KEY_LEN, auth_key))

        current_key = self.readAuthKey()
        if current_key == auth_key:
            # File exists and the contents match what we were trying
            # to set, so we're done.
            return

        # The contents did not match, so unlink the file.
        removeIfPresent(self.auth_key_name)

        file_obj, tmp = self._mktemp()
        try:
            file_obj.write(auth_key)
            # So that we know that the file will be consistent the
            # next time around.
            os.fsync(file_obj.fileno())

            try:
                os.rename(tmp, self.auth_key_name)
            except OSError, why:
                if why[0] == EEXIST:
                    current_key = self.readAuthKey()
                    if current_key != auth_key:
                        raise
                else:
                    raise
        finally:
            removeIfPresent(tmp)

    def createAuthKey(self):
        """Generate a new random auth key and safely store it in the
        location specified by self.auth_key_name.

        This function can interfere with setAuthKey. A given store
        should use only one or the other method for setting the auth
        key.

        () -> str"""

        # Do the import here because this should only get called at
        # most once from each process. Once the auth key file is
        # created, this should not get called at all.
        import random
        rand = random.SystemRandom()

        auth_key = util.random_string(self.AUTH_KEY_LEN, rand)

        file_obj, tmp = self._mktemp()
        try:
            file_obj.write(auth_key)
            os.fsync(file_obj.fileno())

            try:
                if hasattr(os, 'link'):
                    os.link(tmp, self.auth_key_name)
                else:
                    os.rename(tmp, self.auth_key_name)
            except OSError, why:
                if why[0] == EEXIST:
                    auth_key = self.readAuthKey()
                    if auth_key is None:
                        # This should only happen if someone deletes
                        # the auth key file out from under us.
                        raise
                else:
                    raise
        finally:
            removeIfPresent(tmp)

        return auth_key

    def getAuthKey(self):
        """Retrieve the auth key from the file specified by
        self.auth_key_name, creating it if it does not exist.

        () -> str
        """
        auth_key = self.readAuthKey()
        if auth_key is None:
            auth_key = self.createAuthKey()

        if len(auth_key) != self.AUTH_KEY_LEN:
            fmt = ('Got an invalid auth key from %s. Expected %d byte '
                   'string. Got: %r')
            msg = fmt % (self.auth_key_name, self.AUTH_KEY_LEN, auth_key)
            raise ValueError(msg)

        return auth_key

    def getAssociationFilename(self, server_url):
        """Create a unique filename for a given server url and
        handle. This implementation does not assume anything about the
        format of the handle. The filename that is returned will
        contain the domain name from the server URL for ease of human
        inspection of the data directory.

        (str, str) -> str
        """
        server_url = server_url.replace('://', '-', 1)
        filename_chunks = []
        for c in server_url:
            if c in alphanum:
                filename_chunks.append(c)
            else:
                filename_chunks.append('_%02X' % ord(c))
        filename = ''.join(filename_chunks)
        return os.path.join(self.association_dir, filename)

    def storeAssociation(self, association):
        """Store an association in the association directory.

        ConsumerAssociation -> NoneType
        """
        association_s = serializeAssociation(association)
        filename = self.getAssociationFilename(association.server_url)
        tmp_file, tmp = self._mktemp()

        try:
            try:
                tmp_file.write(association_s)
                os.fsync(tmp_file.fileno())
            finally:
                tmp_file.close()

            try:
                os.rename(tmp, filename)
            except OSError, why:
                if why[0] != EEXIST:
                    raise

                # We only expect EEXIST to happen only on Windows. It's
                # possible that we will succeed in unlinking the existing
                # file, but not in putting the temporary file in place.
                try:
                    os.unlink(filename)
                except OSError, why:
                    if why[0] == ENOENT:
                        pass
                    else:
                        raise

                # Now the target should not exist. Try renaming again,
                # giving up if it fails.
                os.rename(tmp, filename)
        except:
            # If there was an error, don't leave the temporary file
            # around.
            removeIfPresent(tmp)
            raise

    def getAssociation(self, server_url):
        """Retrieve an association.

        (str, str) -> ConsumerAssociation or NoneType
        """
        filename = self.getAssociationFilename(server_url)
        try:
            assoc_file = file(filename, 'rb')
        except IOError, why:
            if why[0] == ENOENT:
                # No association exists for that URL and handle
                return None
            else:
                raise
        else:
            try:
                assoc_s = assoc_file.read()
            finally:
                assoc_file.close()

            try:
                association = deserializeAssociation(assoc_s)
            except ValueError:
                removeIfPresent(filename)
                return None

        # Clean up expired associations
        if association.getExpiresIn() == 0:
            removeIfPresent(filename)
            return None
        else:
            return association

    def removeAssociation(self, server_url, handle):
        """Remove an association if it exists. Do nothing if it does not.

        (str, str) -> bool
        """
        assoc = self.getAssociation(server_url)
        if assoc is None or assoc.handle != handle:
            return False
        else:
            filename = self.getAssociationFilename(server_url)
            return removeIfPresent(filename)

    def storeNonce(self, nonce):
        """Mark this nonce as present.

        str -> NoneType
        """
        filename = os.path.join(self.nonce_dir, nonce)
        nonce_file = file(filename, 'w')
        nonce_file.close()

    def useNonce(self, nonce):
        """Return whether this nonce is present. As a side effect,
        mark it as no longer present.

        str -> bool
        """
        filename = os.path.join(self.nonce_dir, nonce)
        try:
            st = os.stat(filename)
        except OSError, why:
            if why[0] == ENOENT:
                # File was not present, so nonce is no good
                return False
            else:
                raise
        else:
            # Either it is too old or we are using it. Either way, we
            # must remove the file.
            try:
                os.unlink(filename)
            except OSError, why:
                if why[0] == ENOENT:
                    # someone beat us to it, so we cannot use this
                    # nonce anymore.
                    return False
                else:
                    raise

            now = time.time()
            nonce_age = now - st.st_mtime

            # We can us it if the age of the file is less than the
            # expiration time.
            return nonce_age <= self.max_nonce_age

    def clean(self):
        """Remove expired entries from the database. This is
        potentially expensive, so only run when it is acceptable to
        take time.

        () -> NoneType
        """
        nonces = os.listdir(self.nonce_dir)
        now = time.time()

        # Check all nonces for expiry
        for nonce in nonces:
            filename = os.path.join(self.nonce_dir, nonce)
            try:
                st = os.stat(filename)
            except OSError, why:
                if why[0] == ENOENT:
                    # The file did not exist by the time we tried to
                    # stat it.
                    pass
                else:
                    raise
            else:
                # Remove the nonce if it has expired
                nonce_age = now - st.st_mtime
                if nonce_age > self.max_nonce_age:
                    removeIfPresent(filename)

        association_filenames = os.listdir(self.association_dir)
        for association_filename in association_filenames:
            try:
                association_file = file(association_filename, 'rb')
            except IOError, why:
                if why[0] == ENOENT:
                    pass
                else:
                    raise
            else:
                try:
                    assoc_s = association_file.read()
                finally:
                    association_file.close()

                # Remove expired or corrupted associations
                try:
                    association = deserializeAssociation(assoc_s)
                except ValueError:
                    removeIfPresent(association_filename)
                else:
                    if association.getExpiresIn() == 0:
                        removeIfPresent(association_filename)
<commit_msg>[project @ Fix type signature in filestore getAssociation]<commit_after>import string
import os
import os.path
import time
import tempfile

from errno import EEXIST, ENOENT

from openid.consumer.stores import OpenIDStore, ConsumerAssociation
from openid import util

filename_allowed = string.letters + string.digits + '.-'
# Convert this to something whose __contains__ is fast
try:
    # 2.4
    alphanum = set(filename_allowed)
except NameError:
    try:
        # 2.3
        import sets
    except ImportError:
        # pre-2.3
        _d = {}
        for c in filename_allowed:
            _d[c] = None
        filename_allowed = _d
    else:
        filename_allowed = sets.Set(filename_allowed)

# The ordering and name of keys as stored by serializeAssociation
assoc_keys = [
    'version',
    'server_url',
    'handle',
    'secret',
    'issued',
    'lifetime',
    ]

def serializeAssociation(assoc):
    """Convert an association to kvform

    Inverse of deserializeAssociation
    """
    data = [
        '1',
        assoc.server_url,
        assoc.handle,
        util.to_b64(assoc.secret),
        str(int(assoc.issued)),
        str(int(assoc.lifetime)),
        ]

    assert len(data) == len(assoc_keys)

    lines = []
    for k, v in zip(assoc_keys, data):
        assert ':' not in k
        if '\n' in v:
            fmt = 'Invalid value serializing association: field %r, value %r'
            raise ValueError(fmt % (k, v))

        lines.append('%s: %s\n' % (k, v))

    return ''.join(lines)

def deserializeAssociation(assoc_s):
    """Parse an association as stored by serializeAssociation.

    inverse of serializeAssociation
    """
    lines = assoc_s.split('\n')
    if lines.pop() != '':
        raise ValueError('Missing trailing newline!')
    keys = []
    values = []
    for line in lines:
        k, v = line.split(': ')
        keys.append(k)
        values.append(v)

    if keys != assoc_keys:
        raise ValueError('Unexpected key values: %r', keys)

    version, server_url, handle, secret, issued, lifetime = values
    if version != '1':
        raise ValueError('Unknown version: %r' % version)
    issued = int(issued)
    lifetime = int(lifetime)
    secret = util.from_b64(secret)
    return ConsumerAssociation(server_url, handle, secret, issued, lifetime)

def removeIfPresent(filename):
    """Attempt to remove a file, returning whether the file existed at
    the time of the call.

    str -> bool
    """
    try:
        os.unlink(filename)
    except OSError, why:
        if why[0] == ENOENT:
            # Someone beat us to it, but it's gone, so that's OK
            return False
        else:
            raise
    else:
        # File was present
        return True

def ensureDir(dir_name):
    """Create dir_name as a directory if it does not exist. If it
    exists, make sure that it is, in fact, a directory.

    Can raise OSError

    str -> NoneType
    """
    try:
        os.makedirs(dir_name)
    except OSError, why:
        if why[0] != EEXIST or not os.path.isdir(dir_name):
            raise

class FilesystemOpenIDStore(object):
    """Filesystem-based store for OpenID associations and nonces.

    Methods of this object can raise OSError if unexpected filesystem
    conditions, such as bad permissions or missing directories, occur.
    """

    AUTH_KEY_LEN = 20

    def __init__(self, directory):
        """Initialize the nonce and association directories"""
        self.nonce_dir = os.path.join(directory, 'nonces')

        self.association_dir = os.path.join(directory, 'associations')

        # Temp dir must be on the same filesystem as the assciations
        # directory and the directory containing the auth key file.
        self.temp_dir = os.path.join(directory, 'temp')

        self.auth_key_name = os.path.join(directory, 'auth_key')

        self.max_nonce_age = 6 * 60 * 60 # Six hours, in seconds

        self._setup()

    def _setup(self):
        """Make sure that the directories in which we store our data
        exist.

        () -> NoneType
        """
        ensureDir(os.path.dirname(self.auth_key_name))
        ensureDir(self.nonce_dir)
        ensureDir(self.association_dir)
        ensureDir(self.temp_dir)

    def _mktemp(self):
        """Create a temporary file on the same filesystem as
        self.auth_key_name and self.association_dir.

        The temporary directory should not be cleaned if there are any
        processes using the store. If there is no active process using
        the store, it is safe to remove all of the files in the
        temporary directory.

        () -> (file, str)
        """
        fd, name = tempfile.mkstemp(dir=self.temp_dir)
        try:
            file_obj = os.fdopen(fd, 'wb')
            return file_obj, name
        except:
            removeIfPresent(name)
            raise

    def readAuthKey(self):
        """Read the auth key from the auth key file. Will return None
        if there is currently no key.

        () -> str or NoneType
        """
        try:
            auth_key_file = file(self.auth_key_name, 'rb')
        except IOError, why:
            if why[0] == ENOENT:
                return None
            else:
                raise

        try:
            return auth_key_file.read()
        finally:
            auth_key_file.close()

    def setAuthKey(self, auth_key):
        """Safely store the given auth key in the location specified
        by self.auth_key_name.

        It is possible that if there is a power loss or other fatal
        condition, this store will be left without an auth key. In
        that case, one will be generated unless this is called again.

        If other processes are setting different auth keys (especially
        if createAuthKey is being used), then it is possible that this
        function will fail with an OSError. To ensure that this
        function succeeds, it is necessary that there is no other
        process attempting to set a different key. Multiple processes
        attempting to set the same key should not cause any problems.

        str -> NoneType
        """
        if len(auth_key) != self.AUTH_KEY_LEN:
            fmt = ('Attempted to set invalid auth key. Expected %d byte '
                   'string. Got: %r')
            raise ValueError(fmt % (self.AUTH_KEY_LEN, auth_key))

        current_key = self.readAuthKey()
        if current_key == auth_key:
            # File exists and the contents match what we were trying
            # to set, so we're done.
            return

        # The contents did not match, so unlink the file.
        removeIfPresent(self.auth_key_name)

        file_obj, tmp = self._mktemp()
        try:
            file_obj.write(auth_key)
            # So that we know that the file will be consistent the
            # next time around.
            os.fsync(file_obj.fileno())

            try:
                os.rename(tmp, self.auth_key_name)
            except OSError, why:
                if why[0] == EEXIST:
                    current_key = self.readAuthKey()
                    if current_key != auth_key:
                        raise
                else:
                    raise
        finally:
            removeIfPresent(tmp)

    def createAuthKey(self):
        """Generate a new random auth key and safely store it in the
        location specified by self.auth_key_name.

        This function can interfere with setAuthKey. A given store
        should use only one or the other method for setting the auth
        key.

        () -> str"""

        # Do the import here because this should only get called at
        # most once from each process. Once the auth key file is
        # created, this should not get called at all.
        import random
        rand = random.SystemRandom()

        auth_key = util.random_string(self.AUTH_KEY_LEN, rand)

        file_obj, tmp = self._mktemp()
        try:
            file_obj.write(auth_key)
            os.fsync(file_obj.fileno())

            try:
                if hasattr(os, 'link'):
                    os.link(tmp, self.auth_key_name)
                else:
                    os.rename(tmp, self.auth_key_name)
            except OSError, why:
                if why[0] == EEXIST:
                    auth_key = self.readAuthKey()
                    if auth_key is None:
                        # This should only happen if someone deletes
                        # the auth key file out from under us.
                        raise
                else:
                    raise
        finally:
            removeIfPresent(tmp)

        return auth_key

    def getAuthKey(self):
        """Retrieve the auth key from the file specified by
        self.auth_key_name, creating it if it does not exist.

        () -> str
        """
        auth_key = self.readAuthKey()
        if auth_key is None:
            auth_key = self.createAuthKey()

        if len(auth_key) != self.AUTH_KEY_LEN:
            fmt = ('Got an invalid auth key from %s. Expected %d byte '
                   'string. Got: %r')
            msg = fmt % (self.auth_key_name, self.AUTH_KEY_LEN, auth_key)
            raise ValueError(msg)

        return auth_key

    def getAssociationFilename(self, server_url):
        """Create a unique filename for a given server url and
        handle. This implementation does not assume anything about the
        format of the handle. The filename that is returned will
        contain the domain name from the server URL for ease of human
        inspection of the data directory.

        (str, str) -> str
        """
        server_url = server_url.replace('://', '-', 1)
        filename_chunks = []
        for c in server_url:
            if c in alphanum:
                filename_chunks.append(c)
            else:
                filename_chunks.append('_%02X' % ord(c))
        filename = ''.join(filename_chunks)
        return os.path.join(self.association_dir, filename)

    def storeAssociation(self, association):
        """Store an association in the association directory.

        ConsumerAssociation -> NoneType
        """
        association_s = serializeAssociation(association)
        filename = self.getAssociationFilename(association.server_url)
        tmp_file, tmp = self._mktemp()

        try:
            try:
                tmp_file.write(association_s)
                os.fsync(tmp_file.fileno())
            finally:
                tmp_file.close()

            try:
                os.rename(tmp, filename)
            except OSError, why:
                if why[0] != EEXIST:
                    raise

                # We only expect EEXIST to happen only on Windows. It's
                # possible that we will succeed in unlinking the existing
                # file, but not in putting the temporary file in place.
                try:
                    os.unlink(filename)
                except OSError, why:
                    if why[0] == ENOENT:
                        pass
                    else:
                        raise

                # Now the target should not exist. Try renaming again,
                # giving up if it fails.
                os.rename(tmp, filename)
        except:
            # If there was an error, don't leave the temporary file
            # around.
            removeIfPresent(tmp)
            raise

    def getAssociation(self, server_url):
        """Retrieve an association.

        str -> ConsumerAssociation or NoneType
        """
        filename = self.getAssociationFilename(server_url)
        try:
            assoc_file = file(filename, 'rb')
        except IOError, why:
            if why[0] == ENOENT:
                # No association exists for that URL and handle
                return None
            else:
                raise
        else:
            try:
                assoc_s = assoc_file.read()
            finally:
                assoc_file.close()

            try:
                association = deserializeAssociation(assoc_s)
            except ValueError:
                removeIfPresent(filename)
                return None

        # Clean up expired associations
        if association.getExpiresIn() == 0:
            removeIfPresent(filename)
            return None
        else:
            return association

    def removeAssociation(self, server_url, handle):
        """Remove an association if it exists. Do nothing if it does not.

        (str, str) -> bool
        """
        assoc = self.getAssociation(server_url)
        if assoc is None or assoc.handle != handle:
            return False
        else:
            filename = self.getAssociationFilename(server_url)
            return removeIfPresent(filename)

    def storeNonce(self, nonce):
        """Mark this nonce as present.

        str -> NoneType
        """
        filename = os.path.join(self.nonce_dir, nonce)
        nonce_file = file(filename, 'w')
        nonce_file.close()

    def useNonce(self, nonce):
        """Return whether this nonce is present. As a side effect,
        mark it as no longer present.

        str -> bool
        """
        filename = os.path.join(self.nonce_dir, nonce)
        try:
            st = os.stat(filename)
        except OSError, why:
            if why[0] == ENOENT:
                # File was not present, so nonce is no good
                return False
            else:
                raise
        else:
            # Either it is too old or we are using it. Either way, we
            # must remove the file.
            try:
                os.unlink(filename)
            except OSError, why:
                if why[0] == ENOENT:
                    # someone beat us to it, so we cannot use this
                    # nonce anymore.
                    return False
                else:
                    raise

            now = time.time()
            nonce_age = now - st.st_mtime

            # We can us it if the age of the file is less than the
            # expiration time.
            return nonce_age <= self.max_nonce_age

    def clean(self):
        """Remove expired entries from the database. This is
        potentially expensive, so only run when it is acceptable to
        take time.

        () -> NoneType
        """
        nonces = os.listdir(self.nonce_dir)
        now = time.time()

        # Check all nonces for expiry
        for nonce in nonces:
            filename = os.path.join(self.nonce_dir, nonce)
            try:
                st = os.stat(filename)
            except OSError, why:
                if why[0] == ENOENT:
                    # The file did not exist by the time we tried to
                    # stat it.
                    pass
                else:
                    raise
            else:
                # Remove the nonce if it has expired
                nonce_age = now - st.st_mtime
                if nonce_age > self.max_nonce_age:
                    removeIfPresent(filename)

        association_filenames = os.listdir(self.association_dir)
        for association_filename in association_filenames:
            try:
                association_file = file(association_filename, 'rb')
            except IOError, why:
                if why[0] == ENOENT:
                    pass
                else:
                    raise
            else:
                try:
                    assoc_s = association_file.read()
                finally:
                    association_file.close()

                # Remove expired or corrupted associations
                try:
                    association = deserializeAssociation(assoc_s)
                except ValueError:
                    removeIfPresent(association_filename)
                else:
                    if association.getExpiresIn() == 0:
                        removeIfPresent(association_filename)
<|endoftext|>


<commit_before># start by importing version, because for some weird reason, I sometimes get
# networkx as over-riding the version here (no idea why...)
try:
    # should work if installed through normal means: setup.py-based with
    # pip, conda, easy_install, etc.
    from . import version
except ImportError:  # pragma: no cover
    import os
    # should work if someone just set the $PYTHONPATH to include OPS
    directory = os.path.dirname(os.path.realpath(__file__))
    prev_dir = os.path.split(directory)[0]
    setupfile = os.path.join(prev_dir, "setup.py")

    if not os.path.exists(setupfile):
        # now we're screwed
        raise ImportError("Unable to identify OPS version. "
                          + "OPS probably not installed correctly.")

    # continue force-setting version based on `setup.py`
    import imp  # may be Py2 only!
    ops_setup = imp.load_source("ops_setup", setupfile)
    version = imp.new_module("openpathsampling.version")

    version.version = ops_setup.preferences['version']
    version.short_version = ops_setup.preferences['version']
    version.git_version = ops_setup.get_git_version()
    version.full_version = ops_setup.preferences['version']
    if not ops_setup.preferences['released']:
        version.full_version += ".dev-" + version.git_version[:7]
    isrelease = str(ops_setup.preferences['released'])

from .analysis.path_histogram import PathDensityHistogram

from .analysis.replica_network import (
    ReplicaNetwork, trace_ensembles_for_replica,
    trace_replicas_for_ensemble, condense_repeats,
    ReplicaNetworkGraph
)

from .analysis.shooting_point_analysis import (
    ShootingPointAnalysis, SnapshotByCoordinateDict
)

from .analysis.trajectory_transition_analysis import (
    TrajectoryTransitionAnalysis,
    TrajectorySegmentContainer
)

from .analysis.channel_analysis import ChannelAnalysis

from .bias_function import (
    BiasFunction, BiasLookupFunction, BiasEnsembleTable,
    SRTISBiasFromNetwork
)

from .collectivevariable import (
    FunctionCV, MDTrajFunctionCV, MSMBFeaturizerCV,
    InVolumeCV, CollectiveVariable, CoordinateGeneratorCV,
    CoordinateFunctionCV, CallableCV, PyEMMAFeaturizerCV,
    GeneratorCV)

from .ensemble import (
    Ensemble, EnsembleCombination, EnsembleFactory, EntersXEnsemble,
    EmptyEnsemble, ExitsXEnsemble, FullEnsemble, PartInXEnsemble,
    AllInXEnsemble, AllOutXEnsemble, WrappedEnsemble,
    SuffixTrajectoryEnsemble, PrefixTrajectoryEnsemble,
    PartOutXEnsemble, LengthEnsemble, NegatedEnsemble,
    ReversedTrajectoryEnsemble, SequentialEnsemble, VolumeEnsemble,
    SequentialEnsemble, IntersectionEnsemble, UnionEnsemble,
    SingleFrameEnsemble, MinusInterfaceEnsemble, TISEnsemble,
    OptionalEnsemble, join_ensembles
)

from .step_visualizer_2D import StepVisualizer2D

from .movechange import (
    EmptyMoveChange, ConditionalSequentialMoveChange,
    MoveChange, PartialAcceptanceSequentialMoveChange,
    RandomChoiceMoveChange, SampleMoveChange,
    SequentialMoveChange, KeepLastSampleMoveChange,
    FilterSamplesMoveChange,
    PathSimulatorMoveChange, AcceptedSampleMoveChange,
    RejectedSampleMoveChange, SubMoveChange,
    FilterByEnsembleMoveChange, RejectedNaNSampleMoveChange,
    RejectedMaxLengthSampleMoveChange
)

from .pathmover import Details, MoveDetails, SampleDetails

from .pathmover import (
    RandomChoiceMover, PathMover, ConditionalSequentialMover,
    PartialAcceptanceSequentialMover, BackwardShootMover, ForwardShootMover,
    BackwardExtendMover, ForwardExtendMover, MinusMover,
    SingleReplicaMinusMover, PathMoverFactory, PathReversalMover,
    ReplicaExchangeMover, EnsembleHopMover,
    SequentialMover, ConditionalMover,
    PathSimulatorMover, PathReversalSet, NeighborEnsembleReplicaExchange,
    SampleMover, StateSwapMover, FinalSubtrajectorySelectMover, EngineMover,
    FirstSubtrajectorySelectMover,
    OneWayShootingMover, RandomSubtrajectorySelectMover, SubPathMover,
    EnsembleFilterMover, SelectionMover, FirstAllowedMover,
    LastAllowedMover, OneWayExtendMover, SubtrajectorySelectMover,
    IdentityPathMover, RandomAllowedChoiceMover,
    TwoWayShootingMover, ForwardFirstTwoWayShootingMover,
    BackwardFirstTwoWayShootingMover
)

from .pathsimulators import (
    PathSimulator, FullBootstrapping, Bootstrapping, PathSampling, MCStep,
    CommittorSimulation, DirectSimulation, ShootFromSnapshotsSimulation
)

from .sample import Sample, SampleSet

from .shooting import (
    ShootingPointSelector, UniformSelector, GaussianBiasSelector,
    FirstFrameSelector, FinalFrameSelector, InterfaceConstrainedSelector
)

from .snapshot_modifier import (
    NoModification, RandomVelocities, VelocityDirectionModifier,
    SingleAtomVelocityDirectionModifier
)

from .storage.storage import Storage, AnalysisStorage

from .volume import (
    Volume, VolumeCombination, VolumeFactory, VoronoiVolume,
    EmptyVolume, FullVolume, CVDefinedVolume, PeriodicCVDefinedVolume,
    IntersectionVolume, UnionVolume, SymmetricDifferenceVolume,
    RelativeComplementVolume, join_volumes
)

from .high_level import move_strategy as strategies

from .high_level.move_scheme import (
    MoveScheme, DefaultScheme, LockedMoveScheme, SRTISScheme,
    OneWayShootingMoveScheme
)

from .high_level.transition import (
    TISTransition, Transition, TPSTransition, FixedLengthTPSTransition
)

from .high_level.network import (
    MSTISNetwork, TransitionNetwork, MISTISNetwork, TPSNetwork,
    FixedLengthTPSNetwork
)

from .high_level.interface_set import (
    InterfaceSet, VolumeInterfaceSet, PeriodicVolumeInterfaceSet
)

from .high_level.ms_outer_interface import MSOuterTISInterface

from .high_level.part_in_b_tps import (
    PartInBFixedLengthTPSNetwork, PartInBFixedLengthTPSTransition
)

import openpathsampling.numerics as numerics

from openpathsampling.engines import Trajectory, BaseSnapshot
import openpathsampling.engines.openmm as openmm
import openpathsampling.engines.toy as toy

def git_HEAD():  # pragma: no cover
    from subprocess import check_output
    import os.path
    git_dir = os.path.dirname(os.path.realpath(__file__))
    return check_output(["git", "-C", git_dir, "rev-parse", "HEAD"])[:-1]
    # chops the newline at the end

import os.path

resources_directory = os.path.join(os.path.dirname(__file__), 'resources')
<commit_msg>remove factories from __init__<commit_after># start by importing version, because for some weird reason, I sometimes get
# networkx as over-riding the version here (no idea why...)
try:
    # should work if installed through normal means: setup.py-based with
    # pip, conda, easy_install, etc.
    from . import version
except ImportError:  # pragma: no cover
    import os
    # should work if someone just set the $PYTHONPATH to include OPS
    directory = os.path.dirname(os.path.realpath(__file__))
    prev_dir = os.path.split(directory)[0]
    setupfile = os.path.join(prev_dir, "setup.py")

    if not os.path.exists(setupfile):
        # now we're screwed
        raise ImportError("Unable to identify OPS version. "
                          + "OPS probably not installed correctly.")

    # continue force-setting version based on `setup.py`
    import imp  # may be Py2 only!
    ops_setup = imp.load_source("ops_setup", setupfile)
    version = imp.new_module("openpathsampling.version")

    version.version = ops_setup.preferences['version']
    version.short_version = ops_setup.preferences['version']
    version.git_version = ops_setup.get_git_version()
    version.full_version = ops_setup.preferences['version']
    if not ops_setup.preferences['released']:
        version.full_version += ".dev-" + version.git_version[:7]
    isrelease = str(ops_setup.preferences['released'])

from .analysis.path_histogram import PathDensityHistogram

from .analysis.replica_network import (
    ReplicaNetwork, trace_ensembles_for_replica,
    trace_replicas_for_ensemble, condense_repeats,
    ReplicaNetworkGraph
)

from .analysis.shooting_point_analysis import (
    ShootingPointAnalysis, SnapshotByCoordinateDict
)

from .analysis.trajectory_transition_analysis import (
    TrajectoryTransitionAnalysis,
    TrajectorySegmentContainer
)

from .analysis.channel_analysis import ChannelAnalysis

from .bias_function import (
    BiasFunction, BiasLookupFunction, BiasEnsembleTable,
    SRTISBiasFromNetwork
)

from .collectivevariable import (
    FunctionCV, MDTrajFunctionCV, MSMBFeaturizerCV,
    InVolumeCV, CollectiveVariable, CoordinateGeneratorCV,
    CoordinateFunctionCV, CallableCV, PyEMMAFeaturizerCV,
    GeneratorCV)

from .ensemble import (
    Ensemble, EnsembleCombination, EnsembleFactory, EntersXEnsemble,
    EmptyEnsemble, ExitsXEnsemble, FullEnsemble, PartInXEnsemble,
    AllInXEnsemble, AllOutXEnsemble, WrappedEnsemble,
    SuffixTrajectoryEnsemble, PrefixTrajectoryEnsemble,
    PartOutXEnsemble, LengthEnsemble, NegatedEnsemble,
    ReversedTrajectoryEnsemble, SequentialEnsemble, VolumeEnsemble,
    SequentialEnsemble, IntersectionEnsemble, UnionEnsemble,
    SingleFrameEnsemble, MinusInterfaceEnsemble, TISEnsemble,
    OptionalEnsemble, join_ensembles
)

from .step_visualizer_2D import StepVisualizer2D

from .movechange import (
    EmptyMoveChange, ConditionalSequentialMoveChange,
    MoveChange, PartialAcceptanceSequentialMoveChange,
    RandomChoiceMoveChange, SampleMoveChange,
    SequentialMoveChange, KeepLastSampleMoveChange,
    FilterSamplesMoveChange,
    PathSimulatorMoveChange, AcceptedSampleMoveChange,
    RejectedSampleMoveChange, SubMoveChange,
    FilterByEnsembleMoveChange, RejectedNaNSampleMoveChange,
    RejectedMaxLengthSampleMoveChange
)

from .pathmover import Details, MoveDetails, SampleDetails

from .pathmover import (
    RandomChoiceMover, PathMover, ConditionalSequentialMover,
    PartialAcceptanceSequentialMover, BackwardShootMover, ForwardShootMover,
    BackwardExtendMover, ForwardExtendMover, MinusMover,
    SingleReplicaMinusMover, PathReversalMover,
    ReplicaExchangeMover, EnsembleHopMover,
    SequentialMover, ConditionalMover,
    PathSimulatorMover, PathReversalSet,
    SampleMover, StateSwapMover, FinalSubtrajectorySelectMover, EngineMover,
    FirstSubtrajectorySelectMover,
    OneWayShootingMover, RandomSubtrajectorySelectMover, SubPathMover,
    EnsembleFilterMover, SelectionMover, FirstAllowedMover,
    LastAllowedMover, OneWayExtendMover, SubtrajectorySelectMover,
    IdentityPathMover, RandomAllowedChoiceMover,
    TwoWayShootingMover, ForwardFirstTwoWayShootingMover,
    BackwardFirstTwoWayShootingMover
)

from .pathsimulators import (
    PathSimulator, FullBootstrapping, Bootstrapping, PathSampling, MCStep,
    CommittorSimulation, DirectSimulation, ShootFromSnapshotsSimulation
)

from .sample import Sample, SampleSet

from .shooting import (
    ShootingPointSelector, UniformSelector, GaussianBiasSelector,
    FirstFrameSelector, FinalFrameSelector, InterfaceConstrainedSelector
)

from .snapshot_modifier import (
    NoModification, RandomVelocities, VelocityDirectionModifier,
    SingleAtomVelocityDirectionModifier
)

from .storage.storage import Storage, AnalysisStorage

from .volume import (
    Volume, VolumeCombination, VolumeFactory, VoronoiVolume,
    EmptyVolume, FullVolume, CVDefinedVolume, PeriodicCVDefinedVolume,
    IntersectionVolume, UnionVolume, SymmetricDifferenceVolume,
    RelativeComplementVolume, join_volumes
)

from .high_level import move_strategy as strategies

from .high_level.move_scheme import (
    MoveScheme, DefaultScheme, LockedMoveScheme, SRTISScheme,
    OneWayShootingMoveScheme
)

from .high_level.transition import (
    TISTransition, Transition, TPSTransition, FixedLengthTPSTransition
)

from .high_level.network import (
    MSTISNetwork, TransitionNetwork, MISTISNetwork, TPSNetwork,
    FixedLengthTPSNetwork
)

from .high_level.interface_set import (
    InterfaceSet, VolumeInterfaceSet, PeriodicVolumeInterfaceSet
)

from .high_level.ms_outer_interface import MSOuterTISInterface

from .high_level.part_in_b_tps import (
    PartInBFixedLengthTPSNetwork, PartInBFixedLengthTPSTransition
)

import openpathsampling.numerics as numerics

from openpathsampling.engines import Trajectory, BaseSnapshot
import openpathsampling.engines.openmm as openmm
import openpathsampling.engines.toy as toy

def git_HEAD():  # pragma: no cover
    from subprocess import check_output
    import os.path
    git_dir = os.path.dirname(os.path.realpath(__file__))
    return check_output(["git", "-C", git_dir, "rev-parse", "HEAD"])[:-1]
    # chops the newline at the end

import os.path

resources_directory = os.path.join(os.path.dirname(__file__), 'resources')
<|endoftext|>


<commit_before>r"""

**openpnm.geometry**

----

The ``geometry`` module contains the ``GenericGeometry`` class, and an
assortment of subclasses that implement specific pore-scale geometrical models

----

**The GenericGeometry Class**

Geometry objects (as well as Physics objects) are ``Subdomain`` subclasses,
which allow them to be assigned to subset of the full domain (although this is
not alway necessary).  This functionality was added so that networks with
distinct regions could be modelled by giving each region its own Geometry
with unique models (e.g. to give a bi-modal pore size distribution).

----

**Library of Preconfigured Geometry Classes**

This module contains a small selection of Geometry classes that are
pre-configured with a selection of pore-scale models.  These classes provide
a good starting point, but generally the choice of models and parameters used
will be specific to each problem and must be designed by the user.

The ``StickAndBall`` class, as it's name suggests assumes spherical pores and
cylindrical throats.  Pore sizes are assigned by finding the largest sphere
that can fit at each site (this will be dictated by the lattice spacing used
when generating the Network), then scaling that value by a random number
between 0 and 0.1.  Throat diameters are taken as half the size of the smaller
of it's two neighbors.  All other properties are calculated using the geometry
of spheres and throats.

The table belows shows the specific models used on ``StickAndBall``:

+----+----------------------+------------------+--------------------------+
| #  | Property Name        | Parameter        | Value                    |
+====+======================+==================+==========================+
| 1  | pore.seed            | model:           | random                   |
+----+----------------------+------------------+--------------------------+
| 2  | pore.max_size        | model:           | largest_sphere           |
+----+----------------------+------------------+--------------------------+
| 3  | pore.diameter        | model:           | product                  |
+----+----------------------+------------------+--------------------------+
| 4  | pore.area            | model:           | sphere                   |
+----+----------------------+------------------+--------------------------+
| 5  | pore.volume          | model:           | sphere                   |
+----+----------------------+------------------+--------------------------+
| 6  | throat.max_size      | model:           | from_neighbor_pores      |
+----+----------------------+------------------+--------------------------+
| 7  | throat.diameter      | model:           | scaled                   |
+----+----------------------+------------------+--------------------------+
| 8  | throat.length        | model:           | piecewise                |
+----+----------------------+------------------+--------------------------+
| 9  | throat.surface_area  | model:           | cylinder                 |
+----+----------------------+------------------+--------------------------+
| 10 | throat.volume        | model:           | cylinder                 |
+----+----------------------+------------------+--------------------------+
| 11 | throat.area          | model:           | cylinder                 |
+----+----------------------+------------------+--------------------------+

----

**Customizing a Preconfigured Geometry Instance**

Perhaps the ``StickAndBall`` class is almost suitable but you wish to decrease
the pores sizes.  The following example illustrates how to alter the
``'pore.size'`` model accordingly:

>>> import openpnm as op
>>> pn = op.network.Cubic([5, 5, 5])
>>> geo = op.geometry.StickAndBall(network=pn, pores=pn.Ps, throats=pn.Ts)

We can reach into the ``models`` attribute and change the parameters of any
model as follows:

>>> max(geo['pore.diameter']) < 1.0  # Confirm largest pore is less than 1.0
True
>>> geo.models['pore.seed']['num_range'] = [0.2, 0.7]
>>> geo.regenerate_models()  # Must regenerate all models
>>> max(geo['pore.diameter']) < 0.7  # Largest pore is now less than 0.7
True

This example illustrated that you can change one property ('pore.seed') and
that change can be cascaded to all dependent properties ('pore.diameter').

"""
from .GenericGeometry import GenericGeometry
from .StickAndBall import StickAndBall
from .StickAndBall_2D import StickAndBall_2D
from .Boundary import Boundary
from .Boundary_2D import Boundary_2D
from .Imported import Imported
<commit_msg>Fixed geometry __init__ file to account for the recent class name change<commit_after>r"""

**openpnm.geometry**

----

The ``geometry`` module contains the ``GenericGeometry`` class, and an
assortment of subclasses that implement specific pore-scale geometrical models

----

**The GenericGeometry Class**

Geometry objects (as well as Physics objects) are ``Subdomain`` subclasses,
which allow them to be assigned to subset of the full domain (although this is
not alway necessary).  This functionality was added so that networks with
distinct regions could be modelled by giving each region its own Geometry
with unique models (e.g. to give a bi-modal pore size distribution).

----

**Library of Preconfigured Geometry Classes**

This module contains a small selection of Geometry classes that are
pre-configured with a selection of pore-scale models.  These classes provide
a good starting point, but generally the choice of models and parameters used
will be specific to each problem and must be designed by the user.

The ``StickAndBall`` class, as it's name suggests assumes spherical pores and
cylindrical throats.  Pore sizes are assigned by finding the largest sphere
that can fit at each site (this will be dictated by the lattice spacing used
when generating the Network), then scaling that value by a random number
between 0 and 0.1.  Throat diameters are taken as half the size of the smaller
of it's two neighbors.  All other properties are calculated using the geometry
of spheres and throats.

The table belows shows the specific models used on ``StickAndBall``:

+----+----------------------+------------------+--------------------------+
| #  | Property Name        | Parameter        | Value                    |
+====+======================+==================+==========================+
| 1  | pore.seed            | model:           | random                   |
+----+----------------------+------------------+--------------------------+
| 2  | pore.max_size        | model:           | largest_sphere           |
+----+----------------------+------------------+--------------------------+
| 3  | pore.diameter        | model:           | product                  |
+----+----------------------+------------------+--------------------------+
| 4  | pore.area            | model:           | sphere                   |
+----+----------------------+------------------+--------------------------+
| 5  | pore.volume          | model:           | sphere                   |
+----+----------------------+------------------+--------------------------+
| 6  | throat.max_size      | model:           | from_neighbor_pores      |
+----+----------------------+------------------+--------------------------+
| 7  | throat.diameter      | model:           | scaled                   |
+----+----------------------+------------------+--------------------------+
| 8  | throat.length        | model:           | piecewise                |
+----+----------------------+------------------+--------------------------+
| 9  | throat.surface_area  | model:           | cylinder                 |
+----+----------------------+------------------+--------------------------+
| 10 | throat.volume        | model:           | cylinder                 |
+----+----------------------+------------------+--------------------------+
| 11 | throat.area          | model:           | cylinder                 |
+----+----------------------+------------------+--------------------------+

----

**Customizing a Preconfigured Geometry Instance**

Perhaps the ``StickAndBall`` class is almost suitable but you wish to decrease
the pores sizes.  The following example illustrates how to alter the
``'pore.size'`` model accordingly:

>>> import openpnm as op
>>> pn = op.network.Cubic([5, 5, 5])
>>> geo = op.geometry.StickAndBall(network=pn, pores=pn.Ps, throats=pn.Ts)

We can reach into the ``models`` attribute and change the parameters of any
model as follows:

>>> max(geo['pore.diameter']) < 1.0  # Confirm largest pore is less than 1.0
True
>>> geo.models['pore.seed']['num_range'] = [0.2, 0.7]
>>> geo.regenerate_models()  # Must regenerate all models
>>> max(geo['pore.diameter']) < 0.7  # Largest pore is now less than 0.7
True

This example illustrated that you can change one property ('pore.seed') and
that change can be cascaded to all dependent properties ('pore.diameter').

"""
from .GenericGeometry import GenericGeometry
from .StickAndBall import StickAndBall
from .StickAndBall2D import StickAndBall2D
from .Boundary import Boundary
from .Boundary2D import Boundary2D
from .Imported import Imported
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2016 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import os
import getpass

from openquake.commonlib import config

DB_SECTION = config.get_section('dbserver')

INSTALLED_APPS = ('openquake.server.db',)

OQSERVER_ROOT = os.path.dirname(__file__)

DEBUG = True
BASE_DIR = os.path.abspath(os.path.dirname(__file__))

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            # insert any extra TEMPLATE_DIRS here
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                # Insert your TEMPLATE_CONTEXT_PROCESSORS here or use this
                # list if you haven't customized them:
                'django.contrib.auth.context_processors.auth',
                'django.template.context_processors.debug',
                'django.template.context_processors.i18n',
                'django.template.context_processors.media',
                'django.template.context_processors.static',
                'django.template.context_processors.tz',
                'django.contrib.messages.context_processors.messages',
                'openquake.server.utils.oq_server_context_processor',
            ],
        },
    },
]

STATIC_URL = '/static/'

MEDIA_ROOT = '%(mediaroot)s'
STATIC_ROOT = '%(staticroot)s'

# Additional directories which hold static files
STATICFILES_DIRS = [
    os.path.join(OQSERVER_ROOT, 'static'),
]

# We need a 'default' database to make Django happy:
DATABASE = {
    'ENGINE': 'django.db.backends.sqlite3',
    'NAME': os.path.expanduser(DB_SECTION.get('file')),
    'LOG': os.path.expanduser(DB_SECTION.get('log')),
    'USER': getpass.getuser(),
    'HOST': DB_SECTION.get('host'),
    'PORT': DB_SECTION.get('port'),
}
DATABASES = {'default': DATABASE}

ALLOWED_HOSTS = ['*']

AUTHENTICATION_BACKENDS = ()

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'Europe/Zurich'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale
USE_L10N = True

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'f_6=^^_0%ygcpgmemxcp0p^xq%47yqe%u9pu!ad*2ym^zt+xq$'

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
)

LOCKDOWN = False
ACL_ON = False

# Add additional paths (as regular expressions) that don't require
# authentication.
AUTH_EXEMPT_URLS = ()

ROOT_URLCONF = 'openquake.server.urls'

INSTALLED_APPS += (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.staticfiles',
    'openquake.server',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'simple': {
            'format': '%(levelname)s %(message)s'
        },
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler'
        },
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'simple'
        },
    },
    'loggers': {
        'django': {
            'handlers': ['console'],
            'level': 'INFO',
            'propagate': True,
        },
        'django.request': {
            'handlers': ['console'],
            'level': 'DEBUG',
            'propagate': True,
        },
        'openquake.server': {
            'handlers': ['console'],
            'level': 'DEBUG',
            'propagate': True,
        },
    },
}

FILE_UPLOAD_MAX_MEMORY_SIZE = 1

# Enable this setting if used as backend for the OpenQuake Platform
# DEFAULT_USER = 'platform'

try:
    from local_settings import *
except ImportError:
    pass

if LOCKDOWN:

    ACL_ON = True

    AUTHENTICATION_BACKENDS += (
        'django.contrib.auth.backends.ModelBackend',
        # 'dpam.backends.PAMBackend',
    )

    MIDDLEWARE_CLASSES += (
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.contrib.messages.middleware.MessageMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'openquake.server.middleware.LoginRequiredMiddleware',
    )

    INSTALLED_APPS += (
        'django.contrib.contenttypes',
        'django.contrib.messages',
        'django.contrib.sessions',
        'django.contrib.admin',
        )

    LOGIN_REDIRECT_URL = '/engine'

    LOGIN_EXEMPT_URLS = ('/accounts/ajax_login/', )
<commit_msg>Remove some garbage from setup.py [skip ci]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2016 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import os
import getpass

from openquake.commonlib import config

DB_SECTION = config.get_section('dbserver')

INSTALLED_APPS = ('openquake.server.db',)

OQSERVER_ROOT = os.path.dirname(__file__)

DEBUG = True
BASE_DIR = os.path.abspath(os.path.dirname(__file__))

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            # insert any extra TEMPLATE_DIRS here
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.contrib.auth.context_processors.auth',
                'django.template.context_processors.debug',
                'django.template.context_processors.i18n',
                'django.template.context_processors.media',
                'django.template.context_processors.static',
                'django.template.context_processors.tz',
                'django.contrib.messages.context_processors.messages',
                'openquake.server.utils.oq_server_context_processor',
            ],
        },
    },
]

STATIC_URL = '/static/'

MEDIA_ROOT = '%(mediaroot)s'
STATIC_ROOT = '%(staticroot)s'

# Additional directories which hold static files
STATICFILES_DIRS = [
    os.path.join(OQSERVER_ROOT, 'static'),
]

DATABASE = {
    'ENGINE': 'django.db.backends.sqlite3',
    'NAME': os.path.expanduser(DB_SECTION.get('file')),
    'LOG': os.path.expanduser(DB_SECTION.get('log')),
    'USER': getpass.getuser(),
    'HOST': DB_SECTION.get('host'),
    'PORT': DB_SECTION.get('port'),
}
DATABASES = {'default': DATABASE}

ALLOWED_HOSTS = ['*']

AUTHENTICATION_BACKENDS = ()

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'Europe/Rome'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale
USE_L10N = True

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'f_6=^^_0%ygcpgmemxcp0p^xq%47yqe%u9pu!ad*2ym^zt+xq$'

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
)

LOCKDOWN = False
ACL_ON = False

# Add additional paths (as regular expressions) that don't require
# authentication.
AUTH_EXEMPT_URLS = ()

ROOT_URLCONF = 'openquake.server.urls'

INSTALLED_APPS += (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.staticfiles',
    'openquake.server',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'simple': {
            'format': '%(levelname)s %(message)s'
        },
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler'
        },
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'simple'
        },
    },
    'loggers': {
        'django': {
            'handlers': ['console'],
            'level': 'INFO',
            'propagate': True,
        },
        'django.request': {
            'handlers': ['console'],
            'level': 'DEBUG',
            'propagate': True,
        },
        'openquake.server': {
            'handlers': ['console'],
            'level': 'DEBUG',
            'propagate': True,
        },
    },
}

FILE_UPLOAD_MAX_MEMORY_SIZE = 1

try:
    from local_settings import *
except ImportError:
    pass

if LOCKDOWN:

    ACL_ON = True

    AUTHENTICATION_BACKENDS += (
        'django.contrib.auth.backends.ModelBackend',
        # 'dpam.backends.PAMBackend',
    )

    MIDDLEWARE_CLASSES += (
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.contrib.messages.middleware.MessageMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'openquake.server.middleware.LoginRequiredMiddleware',
    )

    INSTALLED_APPS += (
        'django.contrib.contenttypes',
        'django.contrib.messages',
        'django.contrib.sessions',
        'django.contrib.admin',
        )

    LOGIN_REDIRECT_URL = '/engine'

    LOGIN_EXEMPT_URLS = ('/accounts/ajax_login/', )
<|endoftext|>


<commit_before>import time
from datetime import timedelta
import logging

import pymongo
from pymongo.errors import DuplicateKeyError

from oplogwatcher import OplogWatcher

class OplogReplayer(OplogWatcher):
    """ Replays all oplogs from one mongo connection to another.

    Watches a mongo connection for write ops (source), and replays them
    into another mongo connection (destination).
    """

    @staticmethod
    def is_create_index(raw):
        """ Determines if the given operation is a "create index"" operation.

        { "op" : "i",
          "ns" : "testdb.system.indexes" }
        """
        return raw['op'] == 'i' and raw['ns'].endswith('.system.indexes')

    @staticmethod
    def is_drop_index(raw):
        """ Determines if the given operation is a "drop index" operation.

        { "op" : "c",
          "ns" : "testdb.$cmd",
          "o" : { "dropIndexes" : "testcoll",
        		  "index" : "nuie_1" } }
        """
        return raw['op'] == 'c' and 'dropIndexes' in raw['o']

    @staticmethod
    def is_index_operation(raw):
        return (OplogReplayer.is_create_index(raw) or
                OplogReplayer.is_drop_index(raw))

    def __init__(self, source, dest, replay_indexes=True, ts=None,
                 poll_time=1.0):
        # Create a one-time connection to source, to determine replicaset.
        c = pymongo.Connection(source)
        try:
            obj = c.local.system.replset.find_one()
            replicaset = obj['_id']
        except:
            raise ValueError('Could not determine replicaset for %r' % source)

        # Mongo source is a replica set, connect to it as such.
        self.source = pymongo.Connection(source, replicaset=replicaset)
        # Use ReadPreference.SECONDARY because we can afford to read oplogs
        # from secondaries: even if they're behind, everything will work
        # correctly because the oplog order will always be preserved.
        self.source.read_preference = pymongo.ReadPreference.SECONDARY

        self._lastts_id = '%s-lastts' % replicaset
        self.dest = pymongo.Connection(dest)

        self.replay_indexes = replay_indexes

        # When no ts argument is supplied, get the last timestamp from dest.
        if ts is None:
            ts = self._get_lastts()

        self._replay_count = 0
        # Compute velocity every few ops.
        self._started_at = self._last_velocity_at = time.time()
        self._last_replay_count = 0
        OplogWatcher.__init__(self, self.source, ts=ts, poll_time=poll_time)

    def print_replication_info(self):
        # Only print replication info every few hundred replayed ops.
        if self._replay_count % 5000 == 0:
            lprint = logging.info
        elif self._replay_count % 500 == 0:
            lprint = logging.debug
        else:
            return

        # Avoid multiple time.time() syscalls.
        now = time.time()

        # Print sync status.
        delay = now - self.ts.time
        lprint('synced = %dsecs ago (%.2fhrs)' % (delay, delay/3600.0))

        # Print current velocity (ops per second).
        new_ops_since_last_print = self._replay_count - self._last_replay_count
        velocity = new_ops_since_last_print / (now - self._last_velocity_at)
        self._last_replay_count = self._replay_count
        self._last_velocity_at = now
        lprint('current replay speed: %.2fops/sec' % velocity)

        # Print total number of oplogs replayed.
        tdiff = timedelta(seconds=int(now - self._started_at))
        lprint('replayed %s ops in %s' % (self._replay_count, tdiff))

    def _get_lastts(self):
        # Get the last oplog ts that was played on destination.
        obj = self.dest.oplogreplay.settings.find_one({'_id': self._lastts_id})
        if obj is None:
            return None
        else:
            return obj['value']

    def _update_lastts(self):
        self.dest.oplogreplay.settings.update({'_id': self._lastts_id},
                                              {'$set': {'value': self.ts}},
                                              upsert=True)

    def process_op(self, ns, raw):
        if not self.replay_indexes and OplogReplayer.is_index_operation(raw):
            # Do not replay index operations.
            pass
        else:
            # Treat "drop index" operations separately.
            if OplogReplayer.is_drop_index(raw):
                self.drop_index(raw)
            else:
                OplogWatcher.process_op(self, ns, raw)

        # Update the lastts on the destination
        self._update_lastts()
        self._replay_count += 1
        self.print_replication_info()

    def _dest_coll(self, ns):
        db, collection = ns.split('.', 1)
        return self.dest[db][collection]

    def insert(self, ns, docid, raw, **kw):
        """ Perform a single insert operation.

            {'docid': ObjectId('4e95ae77a20e6164850761cd'),
             'ns': u'mydb.tweets',
             'raw': {u'h': -1469300750073380169L,
                     u'ns': u'mydb.tweets',
                     u'o': {u'_id': ObjectId('4e95ae77a20e6164850761cd'),
                            u'content': u'Lorem ipsum',
                            u'nr': 16},
                     u'op': u'i',
                     u'ts': Timestamp(1318432375, 1)}}
        """
        try:
            self._dest_coll(ns).insert(raw['o'], safe=True)
        except DuplicateKeyError, e:
            logging.warning(e)

    def update(self, ns, docid, raw, **kw):
        """ Perform a single update operation.

            {'docid': ObjectId('4e95ae3616692111bb000001'),
             'ns': u'mydb.tweets',
             'raw': {u'h': -5295451122737468990L,
                     u'ns': u'mydb.tweets',
                     u'o': {u'$set': {u'content': u'Lorem ipsum'}},
                     u'o2': {u'_id': ObjectId('4e95ae3616692111bb000001')},
                     u'op': u'u',
                     u'ts': Timestamp(1318432339, 1)}}
        """
        self._dest_coll(ns).update(raw['o2'], raw['o'], safe=True)

    def delete(self, ns, docid, raw, **kw):
        """ Perform a single delete operation.

            {'docid': ObjectId('4e959ea11669210edc002902'),
             'ns': u'mydb.tweets',
             'raw': {u'b': True,
                     u'h': -8347418295715732480L,
                     u'ns': u'mydb.tweets',
                     u'o': {u'_id': ObjectId('4e959ea11669210edc002902')},
                     u'op': u'd',
                     u'ts': Timestamp(1318432261, 10499)}}
        """
        self._dest_coll(ns).remove(raw['o'], safe=True)

    def drop_index(self, raw):
        """ Executes a drop index command.

            { "op" : "c",
              "ns" : "testdb.$cmd",
              "o" : { "dropIndexes" : "testcoll",
            		  "index" : "nuie_1" } }
        """
        dbname = raw['ns'].split('.', 1)[0]
        collname = raw['o']['dropIndexes']
        self.dest[dbname][collname].drop_index(raw['o']['index'])
<commit_msg>implement "command" oplog operation<commit_after>import time
from datetime import timedelta
import logging

import pymongo
from pymongo.errors import DuplicateKeyError

from oplogwatcher import OplogWatcher

class OplogReplayer(OplogWatcher):
    """ Replays all oplogs from one mongo connection to another.

    Watches a mongo connection for write ops (source), and replays them
    into another mongo connection (destination).
    """

    @staticmethod
    def is_create_index(raw):
        """ Determines if the given operation is a "create index"" operation.

        { "op" : "i",
          "ns" : "testdb.system.indexes" }
        """
        return raw['op'] == 'i' and raw['ns'].endswith('.system.indexes')

    @staticmethod
    def is_drop_index(raw):
        """ Determines if the given operation is a "drop index" operation.

        { "op" : "c",
          "ns" : "testdb.$cmd",
          "o" : { "dropIndexes" : "testcoll",
        		  "index" : "nuie_1" } }
        """
        return raw['op'] == 'c' and 'dropIndexes' in raw['o']

    @staticmethod
    def is_index_operation(raw):
        return (OplogReplayer.is_create_index(raw) or
                OplogReplayer.is_drop_index(raw))

    def __init__(self, source, dest, replay_indexes=True, ts=None,
                 poll_time=1.0):
        # Create a one-time connection to source, to determine replicaset.
        c = pymongo.Connection(source)
        try:
            obj = c.local.system.replset.find_one()
            replicaset = obj['_id']
        except:
            raise ValueError('Could not determine replicaset for %r' % source)

        # Mongo source is a replica set, connect to it as such.
        self.source = pymongo.Connection(source, replicaset=replicaset)
        # Use ReadPreference.SECONDARY because we can afford to read oplogs
        # from secondaries: even if they're behind, everything will work
        # correctly because the oplog order will always be preserved.
        self.source.read_preference = pymongo.ReadPreference.SECONDARY

        self._lastts_id = '%s-lastts' % replicaset
        self.dest = pymongo.Connection(dest)

        self.replay_indexes = replay_indexes

        # When no ts argument is supplied, get the last timestamp from dest.
        if ts is None:
            ts = self._get_lastts()

        self._replay_count = 0
        # Compute velocity every few ops.
        self._started_at = self._last_velocity_at = time.time()
        self._last_replay_count = 0
        OplogWatcher.__init__(self, self.source, ts=ts, poll_time=poll_time)

    def print_replication_info(self):
        # Only print replication info every few hundred replayed ops.
        if self._replay_count % 5000 == 0:
            lprint = logging.info
        elif self._replay_count % 500 == 0:
            lprint = logging.debug
        else:
            return

        # Avoid multiple time.time() syscalls.
        now = time.time()

        # Print sync status.
        delay = now - self.ts.time
        lprint('synced = %dsecs ago (%.2fhrs)' % (delay, delay/3600.0))

        # Print current velocity (ops per second).
        new_ops_since_last_print = self._replay_count - self._last_replay_count
        velocity = new_ops_since_last_print / (now - self._last_velocity_at)
        self._last_replay_count = self._replay_count
        self._last_velocity_at = now
        lprint('current replay speed: %.2fops/sec' % velocity)

        # Print total number of oplogs replayed.
        tdiff = timedelta(seconds=int(now - self._started_at))
        lprint('replayed %s ops in %s' % (self._replay_count, tdiff))

    def _get_lastts(self):
        # Get the last oplog ts that was played on destination.
        obj = self.dest.oplogreplay.settings.find_one({'_id': self._lastts_id})
        if obj is None:
            return None
        else:
            return obj['value']

    def _update_lastts(self):
        self.dest.oplogreplay.settings.update({'_id': self._lastts_id},
                                              {'$set': {'value': self.ts}},
                                              upsert=True)

    def process_op(self, ns, raw):
        if not self.replay_indexes and OplogReplayer.is_index_operation(raw):
            # Do not replay index operations.
            pass
        else:
            # Treat "drop index" operations separately.
            if OplogReplayer.is_drop_index(raw):
                self.drop_index(raw)
            else:
                OplogWatcher.process_op(self, ns, raw)

        # Update the lastts on the destination
        self._update_lastts()
        self._replay_count += 1
        self.print_replication_info()

    def _dest_coll(self, ns):
        db, collection = ns.split('.', 1)
        return self.dest[db][collection]

    def insert(self, ns, docid, raw, **kw):
        """ Perform a single insert operation.

            {'docid': ObjectId('4e95ae77a20e6164850761cd'),
             'ns': u'mydb.tweets',
             'raw': {u'h': -1469300750073380169L,
                     u'ns': u'mydb.tweets',
                     u'o': {u'_id': ObjectId('4e95ae77a20e6164850761cd'),
                            u'content': u'Lorem ipsum',
                            u'nr': 16},
                     u'op': u'i',
                     u'ts': Timestamp(1318432375, 1)}}
        """
        try:
            self._dest_coll(ns).insert(raw['o'], safe=True)
        except DuplicateKeyError, e:
            logging.warning(e)

    def update(self, ns, docid, raw, **kw):
        """ Perform a single update operation.

            {'docid': ObjectId('4e95ae3616692111bb000001'),
             'ns': u'mydb.tweets',
             'raw': {u'h': -5295451122737468990L,
                     u'ns': u'mydb.tweets',
                     u'o': {u'$set': {u'content': u'Lorem ipsum'}},
                     u'o2': {u'_id': ObjectId('4e95ae3616692111bb000001')},
                     u'op': u'u',
                     u'ts': Timestamp(1318432339, 1)}}
        """
        self._dest_coll(ns).update(raw['o2'], raw['o'], safe=True)

    def delete(self, ns, docid, raw, **kw):
        """ Perform a single delete operation.

            {'docid': ObjectId('4e959ea11669210edc002902'),
             'ns': u'mydb.tweets',
             'raw': {u'b': True,
                     u'h': -8347418295715732480L,
                     u'ns': u'mydb.tweets',
                     u'o': {u'_id': ObjectId('4e959ea11669210edc002902')},
                     u'op': u'd',
                     u'ts': Timestamp(1318432261, 10499)}}
        """
        self._dest_coll(ns).remove(raw['o'], safe=True)

    def drop_index(self, raw):
        """ Executes a drop index command.

            { "op" : "c",
              "ns" : "testdb.$cmd",
              "o" : { "dropIndexes" : "testcoll",
            		  "index" : "nuie_1" } }
        """
        dbname = raw['ns'].split('.', 1)[0]
        collname = raw['o']['dropIndexes']
        self.dest[dbname][collname].drop_index(raw['o']['index'])

    def command(self, ns, raw, **kw):
        """ Executes command.

            { "op" : "c",
              "ns" : "testdb.$cmd",
              "o" : { "drop" : "fs.files"}
            }
        """
        try:
            dbname = raw['ns'].split('.', 1)[0]
            self.dest[dbname].command(raw['o'])
        except Exception, e:
            logging.warning(e)

<|endoftext|>


<commit_before>"""
Django settings for oppgavegenerator project.

For more information on this file, see
https://docs.djangoproject.com/en/1.7/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.7/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
#import dj_database_url

BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'x7m%)nsw^0^haut=++^iv0-k8!c-fctda)3j=$*cq*%cv#^i38'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = ['localhost', '158.38.101.36']

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',          # needed for registration
    'django.contrib.sites',         # needed for registration (?)
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',      # I don't think we need this -Einar / do something about it - Eirik
    'django.contrib.staticfiles',
    'oppgavegen',
    'bootstrap3',                   # django-bootstrap-3
    'registration',                 # django-registration-redux
    'gunicorn'                      # gunicorn (for unix deployment)
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'oppgavegenerator.urls'

WSGI_APPLICATION = 'oppgavegenerator.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.7/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.7/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.7/howto/static-files/

STATIC_ROOT = '/var/www/oppgavegenerator/static/'
STATIC_URL = '/static/'
STATICFILES_DIRS = (os.path.join(BASE_DIR, 'static'))

#PATH_PROJECT = os.path.realpath(os.path.dirname(__file__))
TEMPLATE_DIRS = [os.path.join(BASE_DIR, 'templates')]
TEMPLATE_LOADERS = ['django.template.loaders.filesystem.Loader',
                    'django.template.loaders.app_directories.Loader']

#REGISTRATION SETTINGS
ACCOUNT_ACTIVATION_DAYS = 7     # Amount of days a user has to activate their account
REGISTRATION_AUTO_LOGIN = True  # Auto-login after activation. True/False
SITE_ID = 1                     # Temporary solution for local production-environment (contrib.sites)
<commit_msg>Added trailing comma to STATICFILES_DIRS<commit_after>"""
Django settings for oppgavegenerator project.

For more information on this file, see
https://docs.djangoproject.com/en/1.7/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.7/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
#import dj_database_url

BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'x7m%)nsw^0^haut=++^iv0-k8!c-fctda)3j=$*cq*%cv#^i38'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = ['localhost', '158.38.101.36']

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',          # needed for registration
    'django.contrib.sites',         # needed for registration (?)
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',      # I don't think we need this -Einar / do something about it - Eirik
    'django.contrib.staticfiles',
    'oppgavegen',
    'bootstrap3',                   # django-bootstrap-3
    'registration',                 # django-registration-redux
    'gunicorn'                      # gunicorn (for unix deployment)
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'oppgavegenerator.urls'

WSGI_APPLICATION = 'oppgavegenerator.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.7/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.7/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.7/howto/static-files/

#STATIC_ROOT = '/var/www/oppgavegenerator/static/'
STATIC_URL = '/static/'
STATICFILES_DIRS = (os.path.join(BASE_DIR, 'static'),)

#PATH_PROJECT = os.path.realpath(os.path.dirname(__file__))
TEMPLATE_DIRS = [os.path.join(BASE_DIR, 'templates')]
TEMPLATE_LOADERS = ['django.template.loaders.filesystem.Loader',
                    'django.template.loaders.app_directories.Loader']

#REGISTRATION SETTINGS
ACCOUNT_ACTIVATION_DAYS = 7     # Amount of days a user has to activate their account
REGISTRATION_AUTO_LOGIN = True  # Auto-login after activation. True/False
SITE_ID = 1                     # Temporary solution for local production-environment (contrib.sites)
<|endoftext|>


<commit_before>import os
import codecs

import numpy as np
from Orange.data import Domain, StringVariable, Table, DiscreteVariable, ContinuousVariable

def get_sample_corpora_dir():
    path = os.path.dirname(__file__)
    dir = os.path.join(path, 'datasets')
    return os.path.realpath(dir)

def documents_to_numpy(documents, metadata):
    metadata2 = dict()
    for md in metadata:
        for key in md.keys():
            metadata2.setdefault(key, set()).add(md[key])

    meta_vars = [StringVariable('text')]
    for key in sorted(metadata2.keys()):
        if len(metadata2[key]) < 21:
            meta_vars.append(DiscreteVariable(key, values=list(metadata2[key])))
        else:
            meta_vars.append(StringVariable(key))

    metas = [[None] * len(meta_vars) for x in range(len(documents))]
    for i, md in enumerate(metadata):
        for j, var in enumerate(meta_vars):
            if var.name in md:
                metas[i][j] = var.to_val(md[var.name])

    text = np.array(documents).reshape(len(documents), 1)
    metas = np.array(metas, dtype=object)

    return np.hstack((text, metas)), meta_vars

class Corpus(Table):
    """
        Internal class for storing a corpus of orangecontrib.text.corpus.Corpus.
    """

    def __new__(cls, *args, **kwargs):
        """Bypass Table.__new__."""
        return object.__new__(Corpus)

    def __init__(self, documents, metadata=[]):
        self.documents = documents

        metas, meta_vars = documents_to_numpy(documents, metadata)

        self.metas = metas
        self.X = self._Y = self.W = np.zeros((len(documents), 0))
        self.domain = Domain([], metas=meta_vars)
        Table._init_ids(self)

    @classmethod
    def from_table(cls, domain, source, row_indices=...):
        c = super().from_table(domain, source, row_indices)
        c.documents = source.documents
        c._Y = c._Y.astype(np.float64)
        return c

    def get_number_of_categories(self):
        return len(self.domain['category'].values)

    @classmethod
    def from_file(cls, filename):
        documents = []
        metadata = []
        with codecs.open(filename, 'r', 'utf-8') as f:
            header = f.readline().strip().split('\t')
            if header.count('text') != 1:
                raise RuntimeError("File should contain exactly one column labeled 'text'.")
            text_ind = header.index('text')
            f.readline()

            for line in f:
                fields = line.strip().split("\t")
                if len(fields) != len(header):
                    raise RuntimeError("All lines should contain the same number of fields as a header.")
                documents.append(fields[text_ind])
                metadata.append({header[i]: fields[i] for i in range(len(fields)) if i != text_ind})
        return cls(documents, metadata)

    def extend_corpus(self, documents, metadata=[]):
        metas, meta_vars = documents_to_numpy(documents, metadata)

        # TODO check if Domains match!

        self.metas = np.vstack((self.metas, metas))
        self.documents += documents

        self.X = self._Y = self.W = np.zeros((len(self.documents), 0))
        Table._init_ids(self)

    def __len__(self):
        return len(self.documents)
<commit_msg>Fixed the documents_to_numpy method in corpus.py.<commit_after>import os
import codecs

import numpy as np
from Orange.data import Domain, StringVariable, Table, DiscreteVariable, ContinuousVariable

def get_sample_corpora_dir():
    path = os.path.dirname(__file__)
    dir = os.path.join(path, 'datasets')
    return os.path.realpath(dir)

def documents_to_numpy(documents, metadata):
    metadata2 = dict()
    for md in metadata:
        for key in md.keys():
            metadata2.setdefault(key, set()).add(md[key])

    meta_vars = []
    for key in sorted(metadata2.keys()):
        if len(metadata2[key]) < 21:
            meta_vars.append(DiscreteVariable(key, values=list(metadata2[key])))
        else:
            meta_vars.append(StringVariable(key))

    metas = [[None] * len(meta_vars) for x in range(len(documents))]
    for i, md in enumerate(metadata):
        for j, var in enumerate(meta_vars):
            if var.name in md:
                metas[i][j] = var.to_val(md[var.name])

    text = np.array(documents).reshape(len(documents), 1)
    metas = np.array(metas, dtype=object)
    meta_vars.append(StringVariable('text'))
    return np.hstack((text, metas)), meta_vars

class Corpus(Table):
    """
        Internal class for storing a corpus of orangecontrib.text.corpus.Corpus.
    """

    def __new__(cls, *args, **kwargs):
        """Bypass Table.__new__."""
        return object.__new__(Corpus)

    def __init__(self, documents, metadata=[]):
        self.documents = documents

        metas, meta_vars = documents_to_numpy(documents, metadata)

        self.metas = metas
        self.X = self._Y = self.W = np.zeros((len(documents), 0))
        self.domain = Domain([], metas=meta_vars)
        Table._init_ids(self)

    @classmethod
    def from_table(cls, domain, source, row_indices=...):
        c = super().from_table(domain, source, row_indices)
        c.documents = source.documents
        c._Y = c._Y.astype(np.float64)
        return c

    def get_number_of_categories(self):
        return len(self.domain['category'].values)

    @classmethod
    def from_file(cls, filename):
        documents = []
        metadata = []
        with codecs.open(filename, 'r', 'utf-8') as f:
            header = f.readline().strip().split('\t')
            if header.count('text') != 1:
                raise RuntimeError("File should contain exactly one column labeled 'text'.")
            text_ind = header.index('text')
            f.readline()

            for line in f:
                fields = line.strip().split("\t")
                if len(fields) != len(header):
                    raise RuntimeError("All lines should contain the same number of fields as a header.")
                documents.append(fields[text_ind])
                metadata.append({header[i]: fields[i] for i in range(len(fields)) if i != text_ind})
        return cls(documents, metadata)

    def extend_corpus(self, documents, metadata=[]):
        metas, meta_vars = documents_to_numpy(documents, metadata)

        # TODO check if Domains match!

        self.metas = np.vstack((self.metas, metas))
        self.documents += documents

        self.X = self._Y = self.W = np.zeros((len(self.documents), 0))
        Table._init_ids(self)

    def __len__(self):
        return len(self.documents)
<|endoftext|>


<commit_before># sudo apt-get install python-scapy
from scapy.all import *
# sudo pip install scapy_http
from scapy.layers import http
import sys

packets = rdpcap("task07_f1.pcap")
requests = {}
answers = {}

for pkt in packets:
    tcp = pkt['TCP']
    # destination port must be 80
    if tcp.dport == 80:
        ip_id = pkt[IP].id
        requests[ip_id] = pkt

for pkt in packets:
    tcp = pkt['TCP']
    # source port must be 80
    if tcp.sport == 80:
        ip_id = pkt['IP'].id
        answers[ip_id] = pkt

print '=============== REQUESTS =================='
for ip_id in requests:
    req = requests[ip_id]
    req.show()
#    if req.haslayer(http):
#    if 'HTTP Request' in req:
#        httpreq = req.getlayer(http)
#        print '\o/\o/\o/\o/ HTTP REQUEST \o/\o/\o/\o/\o/\o/'
#        print httpreq
#    else:
#        print '/o\ /o\ /o\ NO HTTP REQUEST'

print '=============== ANSWERS =================='
for ip_id in answers:
    ans = answers[ip_id]
    ans.show()

<commit_msg>Recognizing HTTP requests and HTTP Responses and their headers<commit_after># sudo apt-get install python-scapy
from scapy.all import *
# sudo pip install scapy_http
from scapy.layers import http
from scapy.layers.http import HTTPResponse
import sys

packets = rdpcap("task07_f1.pcap")
requests = {}
answers = {}

def has_http_header(packet):
    return ans.haslayer(HTTPResponse)

for pkt in packets:
    tcp = pkt['TCP']
    # destination port must be 80
    if tcp.dport == 80 and pkt.haslayer('HTTP'):
        ip_id = pkt[IP].id
        requests[ip_id] = pkt

for pkt in packets:
    tcp = pkt['TCP']
    # source port must be 80
    if tcp.sport == 80 and pkt.haslayer('HTTP'):
        ip_id = pkt['IP'].id
        answers[ip_id] = pkt

print '=============== REQUESTS =================='
i = 0
for ip_id in requests:
    print 'Packet: ', i, "\n"
    i = i + 1
    req = requests[ip_id]
    req.show()
'''    if req.haslayer('HTTP'):
#    if 'HTTP Request' in req:
        httpreq = req.getlayer('HTTP')
        print '\o/\o/\o/\o/ HTTP REQUEST \o/\o/\o/\o/\o/\o/'
        print httpreq
    else:
        print '/o\ /o\ /o\ NO HTTP REQUEST'
    	'''

print '=============== ANSWERS =================='
i = 0
for ip_id in answers:
    print 'Packet: ', i, "\n"
    i = i + 1
    ans = answers[ip_id]
    if has_http_header(ans):
    	print "HAS HEADER\n"
    else:
    	print "NO HEADER\n"
    ans.show()

<|endoftext|>


<commit_before>import pytest

from diofant.core import symbols
from diofant.matrices.expressions import MatrixSymbol, Inverse
from diofant.matrices import eye, Identity, ShapeError

__all__ = ()

n, m, l = symbols('n m l', integer=True)
A = MatrixSymbol('A', n, m)
B = MatrixSymbol('B', m, l)
C = MatrixSymbol('C', n, n)
D = MatrixSymbol('D', n, n)
E = MatrixSymbol('E', m, n)

def test_inverse():
    pytest.raises(ShapeError, lambda: Inverse(A))
    pytest.raises(ShapeError, lambda: Inverse(A*B))

    assert Inverse(C).shape == (n, n)
    assert Inverse(A*E).shape == (n, n)
    assert Inverse(E*A).shape == (m, m)
    assert Inverse(C).inverse() == C
    assert isinstance(Inverse(Inverse(C)), Inverse)

    assert C.inverse().inverse() == C

    assert C.inverse()*C == Identity(C.rows)

    assert Identity(n).inverse() == Identity(n)
    assert (3*Identity(n)).inverse() == Identity(n)/3

    # Simplifies Muls if possible (i.e. submatrices are square)
    assert (C*D).inverse() == D.I*C.I
    # But still works when not possible
    assert isinstance((A*E).inverse(), Inverse)

    assert Inverse(eye(3)).doit() == eye(3)
    assert Inverse(eye(3)).doit(deep=False) == eye(3)
<commit_msg>Add test for determinant of the inverse matrix expr<commit_after>import pytest

from diofant.core import symbols
from diofant.matrices.expressions import MatrixSymbol, Inverse
from diofant.matrices import eye, Identity, ShapeError, det

__all__ = ()

n, m, l = symbols('n m l', integer=True)
A = MatrixSymbol('A', n, m)
B = MatrixSymbol('B', m, l)
C = MatrixSymbol('C', n, n)
D = MatrixSymbol('D', n, n)
E = MatrixSymbol('E', m, n)

def test_inverse():
    pytest.raises(ShapeError, lambda: Inverse(A))
    pytest.raises(ShapeError, lambda: Inverse(A*B))

    assert Inverse(C).shape == (n, n)
    assert Inverse(A*E).shape == (n, n)
    assert Inverse(E*A).shape == (m, m)
    assert Inverse(C).inverse() == C
    assert isinstance(Inverse(Inverse(C)), Inverse)

    assert C.inverse().inverse() == C

    assert C.inverse()*C == Identity(C.rows)

    assert Identity(n).inverse() == Identity(n)
    assert (3*Identity(n)).inverse() == Identity(n)/3

    # Simplifies Muls if possible (i.e. submatrices are square)
    assert (C*D).inverse() == D.I*C.I
    # But still works when not possible
    assert isinstance((A*E).inverse(), Inverse)

    assert Inverse(eye(3)).doit() == eye(3)
    assert Inverse(eye(3)).doit(deep=False) == eye(3)

    assert det(Inverse(C)) == 1/det(C)
<|endoftext|>


<commit_before>from django.contrib import admin
from flag.models import FlaggedContent, FlagInstance

admin.site.register(FlaggedContent)
admin.site.register(FlagInstance)
<commit_msg>improved admin with inlining<commit_after>from django.contrib import admin
from flag.models import FlaggedContent, FlagInstance

class InlineFlagInstance(admin.TabularInline):
    model = FlagInstance
    extra = 0

class FlaggedContentAdmin(admin.ModelAdmin):
    inlines = [InlineFlagInstance]

admin.site.register(FlaggedContent, FlaggedContentAdmin)
<|endoftext|>