<commit_before>import json

import junit_xml
from typing import Any, Dict, Set, Text

from six.moves import range

MANDATORY = "Mandatory"

class TestResult(object):

    """Encapsulate relevant test result data."""

    def __init__(self, return_code, standard_output, error_output, duration, classname, message=''):
        # type: (int, Text, Text, float, Text, str) -> None
        self.return_code = return_code
        self.standard_output = standard_output
        self.error_output = error_output
        self.duration = duration
        self.message = message
        self.classname = classname

    def create_test_case(self, test):
        # type: (Dict[Text, Any]) -> junit_xml.TestCase
        doc = test.get(u'doc', 'N/A').strip()
        if test.get("tags"):
            category = ", ".join(test['tags'])
        else:
            category = MANDATORY
        case = junit_xml.TestCase(
            doc, elapsed_sec=self.duration, classname=self.classname,
            category=category, stdout=self.standard_output, stderr=self.error_output,
        )
        if self.return_code > 0:
            case.failure_message = self.message
        return case

class CompareFail(Exception):

    @classmethod
    def format(cls, expected, actual, cause=None):
        # type: (Any, Any, Any) -> CompareFail
        message = u"expected: %s\ngot: %s" % (
            json.dumps(expected, indent=4, sort_keys=True),
            json.dumps(actual, indent=4, sort_keys=True))
        if cause:
            message += u"\ncaused by: %s" % cause
        return cls(message)

def compare_location(expected, actual):
    # type: (Dict[str,Any], Dict[str,Any]) -> None
    if "path" in expected:
        comp = "path"
        if "path" not in actual:
            actual["path"] = actual["location"]
    elif "location" in expected:
        comp = "location"
    else:
        return
    if actual.get("class") == "Directory":
        actual[comp] = actual[comp].rstrip("/")

    if expected[comp] != "Any" and (not (actual[comp].endswith("/" + expected[comp]) or
                                    ("/" not in actual[comp] and expected[comp] == actual[comp]))):
        raise CompareFail.format(expected, actual, u"%s does not end with %s" % (actual[comp], expected[comp]))

def compare_contents(expected, actual):
    # type: (Dict[str,Any], Dict[str,Any]) -> None
    expected_contents = expected["contents"]
    with open(actual["path"]) as f:
        actual_contents = f.read()
    if expected_contents != actual_contents:
        raise CompareFail.format(expected, actual,
                                 json.dumps(u"Output file contents do not match: actual '%s' is not equal to expected '%s'"
                                 % (actual_contents, expected_contents)))

def check_keys(keys, expected, actual):
    # type: (Set[str], Dict[str,Any], Dict[str,Any]) -> None
    for k in keys:
        try:
            compare(expected.get(k), actual.get(k))
        except CompareFail as e:
            raise CompareFail.format(expected, actual, u"field '%s' failed comparison: %s" %(
                k, str(e)
            ))

def compare_file(expected, actual):
    # type: (Dict[str,Any], Dict[str,Any]) -> None
    compare_location(expected, actual)
    if 'contents' in expected:
        compare_contents(expected, actual)
    other_keys = set(expected.keys()) - {'path', 'location', 'listing', 'contents'}
    check_keys(other_keys, expected, actual)

def compare_directory(expected, actual):
    # type: (Dict[str,Any], Dict[str,Any]) -> None
    if actual.get("class") != 'Directory':
        raise CompareFail.format(expected, actual, u"expected object with a class 'Directory'")
    if "listing" not in actual:
        raise CompareFail.format(expected, actual, u"'listing' is mandatory field in Directory object")
    for i in expected["listing"]:
        found = False
        for j in actual["listing"]:
            try:
                compare(i, j)
                found = True
                break
            except CompareFail:
                pass
        if not found:
            raise CompareFail.format(expected, actual, u"%s not found" % json.dumps(i, indent=4, sort_keys=True))
    compare_file(expected, actual)

def compare_dict(expected, actual):
    # type: (Dict[str,Any], Dict[str,Any]) -> None
    for c in expected:
        try:
            compare(expected[c], actual.get(c))
        except CompareFail as e:
            raise CompareFail.format(expected, actual, u"failed comparison for key '%s': %s" % (c, e))
    extra_keys = set(actual.keys()).difference(list(expected.keys()))
    for k in extra_keys:
        if actual[k] is not None:
            raise CompareFail.format(expected, actual, u"unexpected key '%s'" % k)

def compare(expected, actual):  # type: (Any, Any) -> None
    if expected == "Any":
        return
    if expected is not None and actual is None:
        raise CompareFail.format(expected, actual)

    try:
        if isinstance(expected, dict):
            if not isinstance(actual, dict):
                raise CompareFail.format(expected, actual)

            if expected.get("class") == "File":
                compare_file(expected, actual)
            elif expected.get("class") == "Directory":
                compare_directory(expected, actual)
            else:
                compare_dict(expected, actual)

        elif isinstance(expected, list):
            if not isinstance(actual, list):
                raise CompareFail.format(expected, actual)

            if len(expected) != len(actual):
                raise CompareFail.format(expected, actual, u"lengths don't match")
            for c in range(0, len(expected)):
                try:
                    compare(expected[c], actual[c])
                except CompareFail as e:
                    raise CompareFail.format(expected, actual, e)
        else:
            if expected != actual:
                raise CompareFail.format(expected, actual)

    except Exception as e:
        raise CompareFail(str(e))
<commit_msg>Update utils.py<commit_after>import json

import junit_xml
from typing import Any, Dict, Set, Text

from six.moves import range

MANDATORY = "required"

class TestResult(object):

    """Encapsulate relevant test result data."""

    def __init__(self, return_code, standard_output, error_output, duration, classname, message=''):
        # type: (int, Text, Text, float, Text, str) -> None
        self.return_code = return_code
        self.standard_output = standard_output
        self.error_output = error_output
        self.duration = duration
        self.message = message
        self.classname = classname

    def create_test_case(self, test):
        # type: (Dict[Text, Any]) -> junit_xml.TestCase
        doc = test.get(u'doc', 'N/A').strip()
        if test.get("tags"):
            category = ", ".join(test['tags'])
        else:
            category = MANDATORY
        case = junit_xml.TestCase(
            doc, elapsed_sec=self.duration, classname=self.classname,
            category=category, stdout=self.standard_output, stderr=self.error_output,
        )
        if self.return_code > 0:
            case.failure_message = self.message
        return case

class CompareFail(Exception):

    @classmethod
    def format(cls, expected, actual, cause=None):
        # type: (Any, Any, Any) -> CompareFail
        message = u"expected: %s\ngot: %s" % (
            json.dumps(expected, indent=4, sort_keys=True),
            json.dumps(actual, indent=4, sort_keys=True))
        if cause:
            message += u"\ncaused by: %s" % cause
        return cls(message)

def compare_location(expected, actual):
    # type: (Dict[str,Any], Dict[str,Any]) -> None
    if "path" in expected:
        comp = "path"
        if "path" not in actual:
            actual["path"] = actual["location"]
    elif "location" in expected:
        comp = "location"
    else:
        return
    if actual.get("class") == "Directory":
        actual[comp] = actual[comp].rstrip("/")

    if expected[comp] != "Any" and (not (actual[comp].endswith("/" + expected[comp]) or
                                    ("/" not in actual[comp] and expected[comp] == actual[comp]))):
        raise CompareFail.format(expected, actual, u"%s does not end with %s" % (actual[comp], expected[comp]))

def compare_contents(expected, actual):
    # type: (Dict[str,Any], Dict[str,Any]) -> None
    expected_contents = expected["contents"]
    with open(actual["path"]) as f:
        actual_contents = f.read()
    if expected_contents != actual_contents:
        raise CompareFail.format(expected, actual,
                                 json.dumps(u"Output file contents do not match: actual '%s' is not equal to expected '%s'"
                                 % (actual_contents, expected_contents)))

def check_keys(keys, expected, actual):
    # type: (Set[str], Dict[str,Any], Dict[str,Any]) -> None
    for k in keys:
        try:
            compare(expected.get(k), actual.get(k))
        except CompareFail as e:
            raise CompareFail.format(expected, actual, u"field '%s' failed comparison: %s" %(
                k, str(e)
            ))

def compare_file(expected, actual):
    # type: (Dict[str,Any], Dict[str,Any]) -> None
    compare_location(expected, actual)
    if 'contents' in expected:
        compare_contents(expected, actual)
    other_keys = set(expected.keys()) - {'path', 'location', 'listing', 'contents'}
    check_keys(other_keys, expected, actual)

def compare_directory(expected, actual):
    # type: (Dict[str,Any], Dict[str,Any]) -> None
    if actual.get("class") != 'Directory':
        raise CompareFail.format(expected, actual, u"expected object with a class 'Directory'")
    if "listing" not in actual:
        raise CompareFail.format(expected, actual, u"'listing' is mandatory field in Directory object")
    for i in expected["listing"]:
        found = False
        for j in actual["listing"]:
            try:
                compare(i, j)
                found = True
                break
            except CompareFail:
                pass
        if not found:
            raise CompareFail.format(expected, actual, u"%s not found" % json.dumps(i, indent=4, sort_keys=True))
    compare_file(expected, actual)

def compare_dict(expected, actual):
    # type: (Dict[str,Any], Dict[str,Any]) -> None
    for c in expected:
        try:
            compare(expected[c], actual.get(c))
        except CompareFail as e:
            raise CompareFail.format(expected, actual, u"failed comparison for key '%s': %s" % (c, e))
    extra_keys = set(actual.keys()).difference(list(expected.keys()))
    for k in extra_keys:
        if actual[k] is not None:
            raise CompareFail.format(expected, actual, u"unexpected key '%s'" % k)

def compare(expected, actual):  # type: (Any, Any) -> None
    if expected == "Any":
        return
    if expected is not None and actual is None:
        raise CompareFail.format(expected, actual)

    try:
        if isinstance(expected, dict):
            if not isinstance(actual, dict):
                raise CompareFail.format(expected, actual)

            if expected.get("class") == "File":
                compare_file(expected, actual)
            elif expected.get("class") == "Directory":
                compare_directory(expected, actual)
            else:
                compare_dict(expected, actual)

        elif isinstance(expected, list):
            if not isinstance(actual, list):
                raise CompareFail.format(expected, actual)

            if len(expected) != len(actual):
                raise CompareFail.format(expected, actual, u"lengths don't match")
            for c in range(0, len(expected)):
                try:
                    compare(expected[c], actual[c])
                except CompareFail as e:
                    raise CompareFail.format(expected, actual, e)
        else:
            if expected != actual:
                raise CompareFail.format(expected, actual)

    except Exception as e:
        raise CompareFail(str(e))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
#
# Hello. This is make-release-mail.py from the Telepathy project. It's
# designed to turn an item from a NEWS file into a mail suitable for sending
# to <telepathy@lists.freedesktop.org>. I hope that you enjoy your stay.

import sys

def extract_description(package, version, news_path):
    release_name = []
    details = []

    with open(news_path) as f:
        lines = (line for line in f.readlines())
        for line in lines:
            # Find the 'telepathy-foo 0.1.2' header
            if line.startswith("%s %s" % (package, version)):
                break

        # Skip the ====== line, and the first blank line
        lines.next()
        lines.next()

        got_release_name = False

        for line in lines:
            line = line.rstrip()
            # If we hit the next version header, we're done
            if line.startswith(package):
                break
            # Else, if we hit a blank line and we're still reading the release
            # name, we're done with the release name.
            elif not got_release_name and line == '':
                got_release_name = True
            # Otherwise, append this to the relevant list
            elif not got_release_name:
                release_name.append(line)
            else:
                details.append(line)

        assert got_release_name, (release_name, details)

    # We rstrip details because it picks up a trailing blank line
    return ('\n'.join(release_name), '\n'.join(details).rstrip())

BASE_URL = 'http://telepathy.freedesktop.org/releases'
GIT_URL = 'http://cgit.freedesktop.org/telepathy'

def main(package, version, news_path):
    release_name, details = extract_description(package, version, news_path)

    print """
%(release_name)s

tarball: %(base_url)s/%(package)s/%(package)s-%(version)s.tar.gz
signature: %(base_url)s/%(package)s/%(package)s-%(version)s.tar.gz.asc
git: %(git_url)s/%(package)s

%(details)s""".strip().rstrip() % {
        'base_url': BASE_URL,
        'git_url': GIT_URL,
        'package': package,
        'version': version,
        'release_name': release_name,
        'details': details,
    }

if __name__ == '__main__':
    try:
        package, version, news_path = sys.argv[1:]

        main(package, version, news_path)
    except ValueError, e:
        sys.stderr.write(
            'Usage: %s package-name package.version.number path/to/NEWS\n' %
            sys.argv[0])
        sys.stderr.flush()
        sys.exit(1)
<commit_msg>Update make-release-mail.py from the Gabble.<commit_after>#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
#
# Hello. This is make-release-mail.py from the Telepathy project. It's
# designed to turn an item from a NEWS file into a mail suitable for sending
# to <telepathy@lists.freedesktop.org>. I hope that you enjoy your stay.

import sys
import re

def looks_like_a_header(line, package, version=None):
    if version is None:
        pattern = "^%s .* \(.*\)$" % package
    else:
        pattern = "^%s %s \(.*\)$" % (package, version)

    return re.match(pattern, line) is not None

def extract_description(package, version, news_path):
    release_name = []
    details = []

    with open(news_path) as f:
        lines = (line for line in f.readlines())
        for line in lines:
            # Find the 'telepathy-foo 0.1.2' header
            if looks_like_a_header(line, package, version):
                break

        # Skip the ====== line, and the first blank line
        lines.next()
        lines.next()

        got_release_name = False

        for line in lines:
            line = line.rstrip()
            # If we hit the next version header, we're done
            if looks_like_a_header(line, package):
                break
            # Else, if we hit a blank line and we're still reading the release
            # name, we're done with the release name.
            elif not got_release_name and line == '':
                got_release_name = True
            # Otherwise, append this to the relevant list
            elif not got_release_name:
                release_name.append(line)
            else:
                details.append(line)

        assert got_release_name, (release_name, details)

    # We rstrip details because it picks up a trailing blank line
    return ('\n'.join(release_name), '\n'.join(details).rstrip())

BASE_URL = 'http://telepathy.freedesktop.org/releases'
GIT_URL = 'http://cgit.freedesktop.org/telepathy'

def main(package, version, news_path):
    release_name, details = extract_description(package, version, news_path)

    print """
%(release_name)s

tarball: %(base_url)s/%(package)s/%(package)s-%(version)s.tar.gz
signature: %(base_url)s/%(package)s/%(package)s-%(version)s.tar.gz.asc
git: %(git_url)s/%(package)s

%(details)s""".strip().rstrip() % {
        'base_url': BASE_URL,
        'git_url': GIT_URL,
        'package': package,
        'version': version,
        'release_name': release_name,
        'details': details,
    }

if __name__ == '__main__':
    try:
        package, version, news_path = sys.argv[1:]

        main(package, version, news_path)
    except ValueError, e:
        sys.stderr.write(
            'Usage: %s package-name package.version.number path/to/NEWS\n' %
            sys.argv[0])
        sys.stderr.flush()
        sys.exit(1)
<|endoftext|>


<commit_before>import bson.objectid

from .. import config
from ..auth import INTEGER_ROLES

CONT_TYPES = ['acquisition', 'analysis', 'collection', 'group', 'project', 'session']

def getPerm(name):
    return INTEGER_ROLES[name]

def add_id_to_subject(subject, pid):
    """
    Add a mongo id field to given subject object (dict)

    Use the same _id as other subjects in the session's project with the same code
    If no _id is found, generate a new _id
    """
    result = None
    if subject is None:
        subject = {}
    if subject.get('_id') is not None:
        # Ensure _id is bson ObjectId
        subject['_id'] = bson.ObjectId(str(subject['_id']))
        return subject

    # Attempt to match with another session in the project
    if subject.get('code') is not None and pid is not None:
        query = {'subject.code': subject['code'],
                 'project': pid,
                 'subject._id': {'$exists': True}}
        result = config.db.sessions.find_one(query)

    if result is not None:
        subject['_id'] = result['subject']['_id']
    else:
        subject['_id'] = bson.ObjectId()
    return subject

def get_stats(cont, cont_type):
    """
    Add a session, subject, non-compliant session and attachment count to a project or collection
    """

    if cont_type not in ['projects', 'collections']:
        return cont

    # Get attachment count from file array length
    cont['attachment_count'] = len(cont.get('files', []))

    # Get session and non-compliant session count
    match_q = {}
    if cont_type == 'projects':
        match_q = {'project': cont['_id'], 'archived': {'$in': [None, False]}}
    elif cont_type == 'collections':
        result = config.db.acquisitions.find({'collections': cont['_id'], 'archived': {'$in': [None, False]}}, {'session': 1})
        session_ids = list(set([s['session'] for s in result]))
        match_q = {'_id': {'$in': session_ids}}

    pipeline = [
        {'$match': match_q},
        {'$project': {'_id': 1, 'non_compliant':  {'$cond': [{'$eq': ['$satisfies_template', False]}, 1, 0]}}},
        {'$group': {'_id': 1, 'noncompliant_count': {'$sum': '$non_compliant'}, 'total': {'$sum': 1}}}
    ]

    result = config.db.command('aggregate', 'sessions', pipeline=pipeline).get('result', [])

    if len(result) > 0:
        cont['session_count'] = result[0].get('total', 0)
        cont['noncompliant_session_count'] = result[0].get('noncompliant_count', 0)
    else:
        # If there are no sessions, return zero'd out stats
        cont['session_count'] = 0
        cont['noncompliant_session_count'] = 0
        cont['subject_count'] = 0
        return cont

    # Get subject count
    match_q['subject._id'] = {'$ne': None}
    pipeline = [
        {'$match': match_q},
        {'$group': {'_id': '$subject._id'}},
        {'$group': {'_id': 1, 'count': { '$sum': 1 }}}
    ]

    result = config.db.command('aggregate', 'sessions', pipeline=pipeline).get('result', [])

    if len(result) > 0:
        cont['subject_count'] = result[0].get('count', 0)
    else:
        cont['subject_count'] = 0

    return cont

class ContainerReference(object):
    # pylint: disable=redefined-builtin
    # TODO: refactor to resolve pylint warning

    def __init__(self, type, id):
        if type not in CONT_TYPES:
            raise Exception('Container type must be one of {}'.format(CONT_TYPES))
        if type == 'analysis':
            self.__class__ = AnalysisReference

        if not isinstance(type, basestring):
            raise Exception('Container type must be of type str')
        if not isinstance(id, basestring):
            raise Exception('Container id must be of type str')

        self.type   = type
        self.id     = id

    @classmethod
    def from_dictionary(cls, d):
        return cls(
            type = d['type'],
            id = d['id']
        )

    @classmethod
    def from_filereference(cls, fr):
        return cls(
            type = fr.type,
            id = fr.id
        )

    def get(self):
        result = config.db[self.type + 's'].find_one({'_id': bson.ObjectId(self.id)})
        if result is None:
            raise Exception('No such {} {} in database'.format(self.type, self.id))
        return result

    def find_file(self, filename):
        cont = self.get()
        for f in cont.get('files', []):
            if f['name'] == filename:
                return f
        return None

    def file_uri(self, filename):
        return '/' + self.type + 's/' + self.id + '/files/' + filename

    def check_access(self, userID, perm_name):
        perm = getPerm(perm_name)
        for p in self.get()['permissions']:
            if p['_id'] == userID and getPerm(p['access']) > perm:
                return

        raise Exception('User {} does not have {} access to {} {}'.format(userID, perm_name, self.type, self.id))

class AnalysisReference(ContainerReference):
    # pylint: disable=redefined-builtin
    # TODO: refactor to resolve pylint warning

    def get(self):
        result = config.db.sessions.find_one({'analyses._id': self.id}, {'permissions':1, 'analyses': {'$elemMatch': {'_id': self.id}}})
        if result is None or result.get('analyses') is None:
            raise Exception('No such analysis {} in database'.format(self.id))
        analysis = result['analyses'][0]
        analysis['permissions'] = result['permissions']
        analysis['session_id'] = result['_id']
        return analysis

    def file_uri(self, filename):
        analysis = self.get()
        return '/sessions/' + str(analysis['session_id']) + '/analyses/' + self.id + '/files/' + filename

class FileReference(ContainerReference):
    # pylint: disable=redefined-builtin
    # TODO: refactor to resolve pylint warning

    def __init__(self, type, id, name):
        super(FileReference, self).__init__(type, id)
        self.name = name

    @classmethod
    def from_dictionary(cls, d):
        return cls(
            type = d['type'],
            id = d['id'],
            name = d['name']
        )

def create_filereference_from_dictionary(d):
    return FileReference.from_dictionary(d)

def create_containerreference_from_dictionary(d):
    return ContainerReference.from_dictionary(d)

def create_containerreference_from_filereference(fr):
    return ContainerReference.from_filereference(fr)
<commit_msg>update containerutil to use analyses coll<commit_after>import bson.objectid

from .. import config
from ..auth import INTEGER_ROLES

CONT_TYPES = ['acquisition', 'analysis', 'collection', 'group', 'project', 'session']

def get_perm(name):
    return INTEGER_ROLES[name]

def add_id_to_subject(subject, pid):
    """
    Add a mongo id field to given subject object (dict)

    Use the same _id as other subjects in the session's project with the same code
    If no _id is found, generate a new _id
    """
    result = None
    if subject is None:
        subject = {}
    if subject.get('_id') is not None:
        # Ensure _id is bson ObjectId
        subject['_id'] = bson.ObjectId(str(subject['_id']))
        return subject

    # Attempt to match with another session in the project
    if subject.get('code') is not None and pid is not None:
        query = {'subject.code': subject['code'],
                 'project': pid,
                 'subject._id': {'$exists': True}}
        result = config.db.sessions.find_one(query)

    if result is not None:
        subject['_id'] = result['subject']['_id']
    else:
        subject['_id'] = bson.ObjectId()
    return subject

def get_stats(cont, cont_type):
    """
    Add a session, subject, non-compliant session and attachment count to a project or collection
    """

    if cont_type not in ['projects', 'collections']:
        return cont

    # Get attachment count from file array length
    cont['attachment_count'] = len(cont.get('files', []))

    # Get session and non-compliant session count
    match_q = {}
    if cont_type == 'projects':
        match_q = {'project': cont['_id'], 'archived': {'$in': [None, False]}}
    elif cont_type == 'collections':
        result = config.db.acquisitions.find({'collections': cont['_id'], 'archived': {'$in': [None, False]}}, {'session': 1})
        session_ids = list(set([s['session'] for s in result]))
        match_q = {'_id': {'$in': session_ids}}

    pipeline = [
        {'$match': match_q},
        {'$project': {'_id': 1, 'non_compliant':  {'$cond': [{'$eq': ['$satisfies_template', False]}, 1, 0]}}},
        {'$group': {'_id': 1, 'noncompliant_count': {'$sum': '$non_compliant'}, 'total': {'$sum': 1}}}
    ]

    result = config.db.command('aggregate', 'sessions', pipeline=pipeline).get('result', [])

    if len(result) > 0:
        cont['session_count'] = result[0].get('total', 0)
        cont['noncompliant_session_count'] = result[0].get('noncompliant_count', 0)
    else:
        # If there are no sessions, return zero'd out stats
        cont['session_count'] = 0
        cont['noncompliant_session_count'] = 0
        cont['subject_count'] = 0
        return cont

    # Get subject count
    match_q['subject._id'] = {'$ne': None}
    pipeline = [
        {'$match': match_q},
        {'$group': {'_id': '$subject._id'}},
        {'$group': {'_id': 1, 'count': { '$sum': 1 }}}
    ]

    result = config.db.command('aggregate', 'sessions', pipeline=pipeline).get('result', [])

    if len(result) > 0:
        cont['subject_count'] = result[0].get('count', 0)
    else:
        cont['subject_count'] = 0

    return cont

class ContainerReference(object):
    # pylint: disable=redefined-builtin
    # TODO: refactor to resolve pylint warning

    def __init__(self, type, id):
        if type not in CONT_TYPES:
            raise Exception('Container type must be one of {}'.format(CONT_TYPES))

        if not isinstance(type, basestring):
            raise Exception('Container type must be of type str')
        if not isinstance(id, basestring):
            raise Exception('Container id must be of type str')

        self.type = type
        self.collection = singular_to_plural[type]
        self.id = id

    @classmethod
    def from_dictionary(cls, d):
        return cls(
            type = d['type'],
            id = d['id']
        )

    @classmethod
    def from_filereference(cls, fr):
        return cls(
            type = fr.type,
            id = fr.id
        )

    def get(self):
        result = config.db[self.collection].find_one({'_id': bson.ObjectId(self.id)})
        if result is None:
            raise Exception('No such {} {} in database'.format(self.type, self.id))
        return result

    def find_file(self, filename):
        cont = self.get()
        for f in cont.get('files', []):
            if f['name'] == filename:
                return f
        return None

    def file_uri(self, filename):
        if self.type == 'analysis':
            analysis = self.get()
            par_coll, par_id = singular_to_plural[analysis['parent']['type']], analysis['parent']['id']
            return '/{}/{}/analyses/{}/files/{}'.format(par_coll, par_id, self.id, filename)
        return '/{}/{}/files/{}'.format(self.collection, self.id, filename)

    def check_access(self, uid, perm_name):
        perm = get_perm(perm_name)
        for p in self.get()['permissions']:
            if p['_id'] == uid and get_perm(p['access']) > perm:
                return
        raise Exception('User {} does not have {} access to {} {}'.format(uid, perm_name, self.type, self.id))

class FileReference(ContainerReference):
    # pylint: disable=redefined-builtin
    # TODO: refactor to resolve pylint warning

    def __init__(self, type, id, name):
        super(FileReference, self).__init__(type, id)
        self.name = name

    @classmethod
    def from_dictionary(cls, d):
        return cls(
            type = d['type'],
            id = d['id'],
            name = d['name']
        )

def create_filereference_from_dictionary(d):
    return FileReference.from_dictionary(d)

def create_containerreference_from_dictionary(d):
    return ContainerReference.from_dictionary(d)

def create_containerreference_from_filereference(fr):
    return ContainerReference.from_filereference(fr)

singular_to_plural = {
    'group':       'groups',
    'project':     'projects',
    'session':     'sessions',
    'acquisition': 'acquisitions',
    'analysis':    'analyses',
    'file':        'files',
}

plural_to_singular = {p: s for s, p in singular_to_plural.iteritems()}
<|endoftext|>


<commit_before># Copyright (c) 2014, Salesforce.com, Inc.  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# - Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
# - Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
# - Neither the name of Salesforce.com nor the names of its contributors
#   may be used to endorse or promote products derived from this
#   software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import re
import sys
import csv
import urllib
import subprocess
import datetime
import dateutil.parser
import parsable
from collections import Counter
from itertools import izip
from distributions.io.stream import open_compressed, json_dump, json_load
import loom.util
import loom.tasks

# see https://www.lendingclub.com/info/download-data.action
URL = 'https://resources.lendingclub.com/'
FILES = [
    'LoanStats3a.csv.zip',
    'LoanStats3b.csv.zip',
    'LoanStats3c.csv.zip',
    # 'RejectStatsA.csv.zip',
    # 'RejectStatsB.csv.zip',
]
DATA = os.path.join(os.path.dirname(__file__), 'data')
NAME = os.path.join(DATA, 'results')
DOWNLOADS = os.path.join(DATA, 'downloads')
SCHEMA_JSON = os.path.join(DATA, 'rows_csv')
ROWS_CSV = os.path.join(DATA, 'rows_csv')
SCHEMA_JSON = os.path.join(DATA, 'schema.json')
MIN_ROW_LENGTH = 10
ROW_COUNTS = {
    'LoanStats3a.csv': 39787,
    'LoanStats3b.csv': 197787,
    'LoanStats3c.csv': 138735,
}
FEATURES = os.path.join(DATA, 'features.{}.json')
NOW = datetime.datetime.now()
FEATURE_COUNT = 1000
SAMPLE_COUNT = 10

dot_counter = 0

def print_dot(every=1):
    global dot_counter
    dot_counter += 1
    if dot_counter >= every:
        sys.stdout.write('.')
        sys.stdout.flush()
        dot_counter = 0

@parsable.command
def download():
    '''
    Download datset from website.
    '''
    loom.util.mkdir_p(DOWNLOADS)
    with loom.util.chdir(DOWNLOADS):
        for filename in FILES:
            if not os.path.exists(filename):
                url = URL + filename
                print 'fetching {}'.format(url)
                urllib.urlretrieve(url, filename)
            subprocess.check_call(['unzip', '-n', filename])

def load_rows(filename):
    row_count = ROW_COUNTS[filename]
    with open_compressed(os.path.join('data', 'downloads', filename)) as f:
        reader = csv.reader(f)
        header = []
        while len(header) < MIN_ROW_LENGTH:
            header = reader.next()
        for i, row in enumerate(reader):
            if i == row_count:
                raise StopIteration()
            if len(row) < MIN_ROW_LENGTH:
                continue
            yield header, row

@parsable.command
def explore_schema():
    '''
    Print header with some example values.
    '''
    for filename in ROW_COUNTS:
        print '-' * 80
        print filename
        for header, row in load_rows(filename):
            for name, value in sorted(zip(header, row)):
                print name.rjust(20), value
            break

def transform_string(string):
    return string.lower() if string else None

def transform_count(string):
    return int(string) if string else None

def transform_real(string):
    return float(string) if string else None

def transform_percent(string):
    return float(string.replace('%', '')) * 0.01 if string else None

def transform_sparse_real(string):
    if string:
        x = float(string)
        return [True, x] if x else [False, None]
    else:
        return [None, None]

def transform_maybe_count(string):
    return [True, int(string)] if string else [False, None]

def transform_days_ago(string):
    if string:
        delta = NOW - dateutil.parser.parse(string)
        return delta.total_seconds() / (24 * 60 * 60)
    else:
        return None

re_word = re.compile('[A-Za-z]{2,}')

def get_word_set(text):
    return frozenset(m.group().lower() for m in re_word.finditer(text))

@parsable.command
def find_text_features(field, feature_count=FEATURE_COUNT):
    '''
    Build a list of most-common words to extract as sparse boolean features.
    '''
    print 'finding most common words in text field:', field
    counter = Counter()
    for filename in ROW_COUNTS:
        for header, row in load_rows(filename):
            pos = header.index(field)
            break
        for header, row in load_rows(filename):
            text = row[pos]
            counter.update(get_word_set(text))
            print_dot(1000)
    features = [word for word, count in counter.most_common(feature_count)]
    json_dump(features, FEATURES.format(field))
    return features

def transform_text(features):
    prototype = [0] * len(features)
    index = {word: pos for pos, word in enumerate(features)}

    def transform(text):
        result = prototype[:]
        for word in get_word_set(text):
            try:
                result[index[word]] = 1
            except KeyError:
                pass
        return result

    return transform

def text_datatype(field):
    filename = FEATURES.format(field)
    if not os.path.exists(filename):
        print 'first run `python main.py find_text_features {}`'.format(field)
        return None
    else:
        features = json_load(filename)
        return {
            'parts': features,
            'model': ['bb'] * len(features),
            'transform': transform_text(features),
        }

DATATYPES = {
    'count': {'model': 'gp', 'transform': transform_count},
    'real': {'model': 'nich', 'transform': transform_real},
    'date': {'model': 'nich', 'transform': transform_days_ago},
    'percent': {'model': 'nich', 'transform': transform_percent},
    'categorical': {'model': 'dd', 'transform': transform_string},
    'unbounded_categorical': {'model': 'dpd', 'transform': transform_string},
    'maybe_count': {
        'parts': ['nonzero', 'value'],
        'model': ['bb', 'gp'],
        'transform': transform_maybe_count,
    },
    'sparse_real': {
        'parts': ['nonzero', 'value'],
        'model': ['bb', 'nich'],
        'transform': transform_sparse_real,
    },
    'text_desc': text_datatype('desc'),
    'text_emp_title': text_datatype('emp_title'),
    'text_title': text_datatype('title'),
}

SCHEMA_IN = {
    'acc_now_delinq': 'count',
    'acc_open_past_24mths': 'count',
    'accept_d': 'date',
    'addr_city': 'unbounded_categorica',
    'addr_state': 'categorical',
    'annual_inc': 'real',
    'avg_cur_bal': 'real',
    'bc_open_to_buy': 'real',
    'bc_util': 'real',
    'chargeoff_within_12_mths': 'count',
    'collection_recovery_fee': 'sparse_real',
    'collections_12_mths_ex_med': 'count',
    'delinq_2yrs': 'count',
    'delinq_amnt': 'count',
    'desc': 'text_desc',
    'dti': 'real',
    'earliest_cr_line': 'date',
    'emp_length': 'categorical',
    'emp_title': 'text_emp_title',
    'exp_d': 'date',
    'funded_amnt': 'real',
    'funded_amnt_inv': 'real',
    'grade': 'categorical',
    'home_ownership': 'categorical',
    'id': None,
    'initial_list_status': 'categorical',
    'inq_last_6mths': 'count',
    'installment': 'real',
    'int_rate': 'percent',
    'is_inc_v': 'categorical',
    'issue_d': 'date',
    'last_credit_pull_d': 'date',
    'last_pymnt_amnt': 'real',
    'last_pymnt_d': 'date',
    'list_d': 'date',
    'loan_amnt': 'real',
    'loan_status': 'categorical',
    'member_id': None,
    'mo_sin_old_il_acct': 'count',
    'mo_sin_old_rev_tl_op': 'count',
    'mo_sin_rcnt_rev_tl_op': 'count',
    'mo_sin_rcnt_tl': 'count',
    'mort_acc': 'count',
    'mths_since_last_delinq': 'maybe_count',
    'mths_since_last_major_derog': 'maybe_count',
    'mths_since_last_record': 'maybe_count',
    'mths_since_recent_bc': 'maybe_count',
    'mths_since_recent_bc_dlq': 'maybe_count',
    'mths_since_recent_inq': 'maybe_count',
    'mths_since_recent_revol_delinq': 'maybe_count',
    'next_pymnt_d': 'date',
    'num_accts_ever_120_pd': 'count',
    'num_actv_bc_tl': 'count',
    'num_actv_rev_tl': 'count',
    'num_bc_sats': 'count',
    'num_bc_tl': 'count',
    'num_il_tl': 'count',
    'num_op_rev_tl': 'count',
    'num_rev_accts': 'count',
    'num_rev_tl_bal_gt_0': 'count',
    'num_sats': 'count',
    'num_tl_120dpd_2m': 'count',
    'num_tl_30dpd': 'count',
    'num_tl_90g_dpd_24m': 'count',
    'num_tl_op_past_12m': 'count',
    'open_acc': 'count',
    'out_prncp': 'real',
    'out_prncp_inv': 'real',
    'pct_tl_nvr_dlq': 'real',
    'percent_bc_gt_75': 'real',
    'policy_code': 'categorical',
    'pub_rec': 'maybe_count',
    'pub_rec_bankruptcies': 'maybe_count',
    'purpose': 'categorical',
    'pymnt_plan': 'categorical',
    'recoveries': 'sparse_real',
    'revol_bal': 'real',
    'revol_util': 'percent',
    'sub_grade': 'categorical',
    'tax_liens': 'count',
    'term': 'categorical',
    'title': 'text_title',
    'tot_coll_amt': 'sparse_real',
    'tot_cur_bal': 'sparse_real',
    'tot_hi_cred_lim': 'real',
    'total_acc': 'count',
    'total_bal_ex_mort': 'real',
    'total_bc_limit': 'real',
    'total_il_high_credit_limit': 'real',
    'total_pymnt': 'real',
    'total_pymnt_inv': 'real',
    'total_rec_int': 'real',
    'total_rec_late_fee': 'real',
    'total_rec_prncp': 'real',
    'total_rev_hi_lim': 'real',
    'url': None,
}

def transform_schema():
    schema = {}
    for key in SCHEMA_IN:
        datatype = DATATYPES.get(SCHEMA_IN[key])
        if datatype is not None:
            model = datatype['model']
            if 'parts' in datatype:
                for part, model_part in izip(datatype['parts'], model):
                    schema['{}_{}'.format(key, part)] = model_part
            else:
                schema[key] = model
    return schema

def transform_header():
    header = []
    for key in sorted(SCHEMA_IN.keys()):
        datatype = DATATYPES.get(SCHEMA_IN[key])
        if datatype is not None:
            if 'parts' in datatype:
                for part in datatype['parts']:
                    header.append('{}_{}'.format(key, part))
            else:
                header.append(key)
    return header

def transform_row(header_in, row_in):
    row_out = {}
    assert len(header_in) == len(row_in), row_in
    for key, value in izip(header_in, row_in):
        datatype = DATATYPES.get(SCHEMA_IN[key])
        if datatype is not None:
            try:
                value = datatype['transform'](value)
            except Exception as e:
                sys.stdout.write('{}: {}'.format(key, e))
                sys.stdout.write('\n')
            if 'parts' in datatype:
                for part, value_part in izip(datatype['parts'], value):
                    row_out['{}_{}'.format(key, part)] = value_part
            else:
                row_out[key] = value
    return row_out

def transform_rows(filename):
    header = transform_header()
    filename_out = os.path.join(ROWS_CSV, filename + '.gz')
    with open_compressed(filename_out, 'wb') as f:
        print 'writing', filename_out
        writer = csv.writer(f)
        writer.writerow(header)
        for header_in, row_in in load_rows(filename):
            row_out = transform_row(header_in, row_in)
            writer.writerow([row_out[key] for key in header])
            print_dot(1000)

@parsable.command
def transform():
    '''
    Transform dataset.
    '''
    schema = transform_schema()
    print 'writing', SCHEMA_JSON
    json_dump(schema, SCHEMA_JSON)
    loom.util.mkdir_p(ROWS_CSV)
    loom.util.parallel_map(transform_rows, ROW_COUNTS.keys())

@parsable.command
def ingest():
    '''
    Ingest dataset into loom.
    '''
    loom.tasks.ingest(NAME, SCHEMA_JSON, ROWS_CSV)

@parsable.command
def infer(sample_count=SAMPLE_COUNT):
    '''
    Infer model.
    '''
    loom.tasks.infer(NAME, sample_count=sample_count)

@parsable.command
def run():
    '''
    download; find text features; transform; ingest; infer.
    '''
    download()
    loom.util.parallel_map(find_text_features, ['desc', 'emp_title', 'title'])
    transform()
    ingest()
    infer()

if __name__ == '__main__':
    parsable.dispatch()
<commit_msg>Add related to lending-club example<commit_after># Copyright (c) 2014, Salesforce.com, Inc.  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# - Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
# - Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
# - Neither the name of Salesforce.com nor the names of its contributors
#   may be used to endorse or promote products derived from this
#   software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import re
import sys
import csv
import urllib
import subprocess
import datetime
import dateutil.parser
import parsable
from collections import Counter
from itertools import izip
from distributions.io.stream import open_compressed, json_dump, json_load
import loom.util
import loom.tasks

# see https://www.lendingclub.com/info/download-data.action
URL = 'https://resources.lendingclub.com/'
FILES = [
    'LoanStats3a.csv.zip',
    'LoanStats3b.csv.zip',
    'LoanStats3c.csv.zip',
    # 'RejectStatsA.csv.zip',
    # 'RejectStatsB.csv.zip',
]
DATA = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data')
NAME = os.path.join(DATA, 'results')
DOWNLOADS = os.path.join(DATA, 'downloads')
SCHEMA_JSON = os.path.join(DATA, 'rows_csv')
ROWS_CSV = os.path.join(DATA, 'rows_csv')
SCHEMA_JSON = os.path.join(DATA, 'schema.json')
RELATED = os.path.join(DATA, 'related.csv')
MIN_ROW_LENGTH = 10
ROW_COUNTS = {
    'LoanStats3a.csv': 39787,
    'LoanStats3b.csv': 197787,
    'LoanStats3c.csv': 138735,
}
FEATURES = os.path.join(DATA, 'features.{}.json')
NOW = datetime.datetime.now()
FEATURE_COUNT = 1000
SAMPLE_COUNT = 10

dot_counter = 0

def print_dot(every=1):
    global dot_counter
    dot_counter += 1
    if dot_counter >= every:
        sys.stdout.write('.')
        sys.stdout.flush()
        dot_counter = 0

@parsable.command
def download():
    '''
    Download datset from website.
    '''
    loom.util.mkdir_p(DOWNLOADS)
    with loom.util.chdir(DOWNLOADS):
        for filename in FILES:
            if not os.path.exists(filename):
                url = URL + filename
                print 'fetching {}'.format(url)
                urllib.urlretrieve(url, filename)
            subprocess.check_call(['unzip', '-n', filename])

def load_rows(filename):
    row_count = ROW_COUNTS[filename]
    with open_compressed(os.path.join('data', 'downloads', filename)) as f:
        reader = csv.reader(f)
        header = []
        while len(header) < MIN_ROW_LENGTH:
            header = reader.next()
        for i, row in enumerate(reader):
            if i == row_count:
                raise StopIteration()
            if len(row) < MIN_ROW_LENGTH:
                continue
            yield header, row

@parsable.command
def explore_schema():
    '''
    Print header with some example values.
    '''
    for filename in ROW_COUNTS:
        print '-' * 80
        print filename
        for header, row in load_rows(filename):
            for name, value in sorted(zip(header, row)):
                print name.rjust(20), value
            break

def transform_string(string):
    return string.lower() if string else None

def transform_count(string):
    return int(string) if string else None

def transform_real(string):
    return float(string) if string else None

def transform_percent(string):
    return float(string.replace('%', '')) * 0.01 if string else None

def transform_sparse_real(string):
    if string:
        x = float(string)
        return [True, x] if x else [False, None]
    else:
        return [None, None]

def transform_maybe_count(string):
    return [True, int(string)] if string else [False, None]

def transform_days_ago(string):
    if string:
        delta = NOW - dateutil.parser.parse(string)
        return delta.total_seconds() / (24 * 60 * 60)
    else:
        return None

re_word = re.compile('[A-Za-z]{2,}')

def get_word_set(text):
    return frozenset(m.group().lower() for m in re_word.finditer(text))

@parsable.command
def find_text_features(field, feature_count=FEATURE_COUNT):
    '''
    Build a list of most-common words to extract as sparse boolean features.
    '''
    print 'finding most common words in text field:', field
    counter = Counter()
    for filename in ROW_COUNTS:
        for header, row in load_rows(filename):
            pos = header.index(field)
            break
        for header, row in load_rows(filename):
            text = row[pos]
            counter.update(get_word_set(text))
            print_dot(1000)
    features = [word for word, count in counter.most_common(feature_count)]
    json_dump(features, FEATURES.format(field))
    return features

def transform_text(features):
    prototype = [0] * len(features)
    index = {word: pos for pos, word in enumerate(features)}

    def transform(text):
        result = prototype[:]
        for word in get_word_set(text):
            try:
                result[index[word]] = 1
            except KeyError:
                pass
        return result

    return transform

def text_datatype(field):
    filename = FEATURES.format(field)
    if not os.path.exists(filename):
        print 'first run `python main.py find_text_features {}`'.format(field)
        return None
    else:
        features = json_load(filename)
        return {
            'parts': features,
            'model': ['bb'] * len(features),
            'transform': transform_text(features),
        }

DATATYPES = {
    'count': {'model': 'gp', 'transform': transform_count},
    'real': {'model': 'nich', 'transform': transform_real},
    'date': {'model': 'nich', 'transform': transform_days_ago},
    'percent': {'model': 'nich', 'transform': transform_percent},
    'categorical': {'model': 'dd', 'transform': transform_string},
    'unbounded_categorical': {'model': 'dpd', 'transform': transform_string},
    'maybe_count': {
        'parts': ['nonzero', 'value'],
        'model': ['bb', 'gp'],
        'transform': transform_maybe_count,
    },
    'sparse_real': {
        'parts': ['nonzero', 'value'],
        'model': ['bb', 'nich'],
        'transform': transform_sparse_real,
    },
    'text_desc': text_datatype('desc'),
    'text_emp_title': text_datatype('emp_title'),
    'text_title': text_datatype('title'),
}

SCHEMA_IN = {
    'acc_now_delinq': 'count',
    'acc_open_past_24mths': 'count',
    'accept_d': 'date',
    'addr_city': 'unbounded_categorica',
    'addr_state': 'categorical',
    'annual_inc': 'real',
    'avg_cur_bal': 'real',
    'bc_open_to_buy': 'real',
    'bc_util': 'real',
    'chargeoff_within_12_mths': 'count',
    'collection_recovery_fee': 'sparse_real',
    'collections_12_mths_ex_med': 'count',
    'delinq_2yrs': 'count',
    'delinq_amnt': 'count',
    'desc': 'text_desc',
    'dti': 'real',
    'earliest_cr_line': 'date',
    'emp_length': 'categorical',
    'emp_title': 'text_emp_title',
    'exp_d': 'date',
    'funded_amnt': 'real',
    'funded_amnt_inv': 'real',
    'grade': 'categorical',
    'home_ownership': 'categorical',
    'id': None,
    'initial_list_status': 'categorical',
    'inq_last_6mths': 'count',
    'installment': 'real',
    'int_rate': 'percent',
    'is_inc_v': 'categorical',
    'issue_d': 'date',
    'last_credit_pull_d': 'date',
    'last_pymnt_amnt': 'real',
    'last_pymnt_d': 'date',
    'list_d': 'date',
    'loan_amnt': 'real',
    'loan_status': 'categorical',
    'member_id': None,
    'mo_sin_old_il_acct': 'count',
    'mo_sin_old_rev_tl_op': 'count',
    'mo_sin_rcnt_rev_tl_op': 'count',
    'mo_sin_rcnt_tl': 'count',
    'mort_acc': 'count',
    'mths_since_last_delinq': 'maybe_count',
    'mths_since_last_major_derog': 'maybe_count',
    'mths_since_last_record': 'maybe_count',
    'mths_since_recent_bc': 'maybe_count',
    'mths_since_recent_bc_dlq': 'maybe_count',
    'mths_since_recent_inq': 'maybe_count',
    'mths_since_recent_revol_delinq': 'maybe_count',
    'next_pymnt_d': 'date',
    'num_accts_ever_120_pd': 'count',
    'num_actv_bc_tl': 'count',
    'num_actv_rev_tl': 'count',
    'num_bc_sats': 'count',
    'num_bc_tl': 'count',
    'num_il_tl': 'count',
    'num_op_rev_tl': 'count',
    'num_rev_accts': 'count',
    'num_rev_tl_bal_gt_0': 'count',
    'num_sats': 'count',
    'num_tl_120dpd_2m': 'count',
    'num_tl_30dpd': 'count',
    'num_tl_90g_dpd_24m': 'count',
    'num_tl_op_past_12m': 'count',
    'open_acc': 'count',
    'out_prncp': 'real',
    'out_prncp_inv': 'real',
    'pct_tl_nvr_dlq': 'real',
    'percent_bc_gt_75': 'real',
    'policy_code': 'categorical',
    'pub_rec': 'maybe_count',
    'pub_rec_bankruptcies': 'maybe_count',
    'purpose': 'categorical',
    'pymnt_plan': 'categorical',
    'recoveries': 'sparse_real',
    'revol_bal': 'real',
    'revol_util': 'percent',
    'sub_grade': 'categorical',
    'tax_liens': 'count',
    'term': 'categorical',
    'title': 'text_title',
    'tot_coll_amt': 'sparse_real',
    'tot_cur_bal': 'sparse_real',
    'tot_hi_cred_lim': 'real',
    'total_acc': 'count',
    'total_bal_ex_mort': 'real',
    'total_bc_limit': 'real',
    'total_il_high_credit_limit': 'real',
    'total_pymnt': 'real',
    'total_pymnt_inv': 'real',
    'total_rec_int': 'real',
    'total_rec_late_fee': 'real',
    'total_rec_prncp': 'real',
    'total_rev_hi_lim': 'real',
    'url': None,
}

def transform_schema():
    schema = {}
    for key in SCHEMA_IN:
        datatype = DATATYPES.get(SCHEMA_IN[key])
        if datatype is not None:
            model = datatype['model']
            if 'parts' in datatype:
                for part, model_part in izip(datatype['parts'], model):
                    schema['{}_{}'.format(key, part)] = model_part
            else:
                schema[key] = model
    return schema

def transform_header():
    header = []
    for key in sorted(SCHEMA_IN.keys()):
        datatype = DATATYPES.get(SCHEMA_IN[key])
        if datatype is not None:
            if 'parts' in datatype:
                for part in datatype['parts']:
                    header.append('{}_{}'.format(key, part))
            else:
                header.append(key)
    return header

def transform_row(header_in, row_in):
    row_out = {}
    assert len(header_in) == len(row_in), row_in
    for key, value in izip(header_in, row_in):
        datatype = DATATYPES.get(SCHEMA_IN[key])
        if datatype is not None:
            try:
                value = datatype['transform'](value)
            except Exception as e:
                sys.stdout.write('{}: {}'.format(key, e))
                sys.stdout.write('\n')
            if 'parts' in datatype:
                for part, value_part in izip(datatype['parts'], value):
                    row_out['{}_{}'.format(key, part)] = value_part
            else:
                row_out[key] = value
    return row_out

def transform_rows(filename):
    header = transform_header()
    filename_out = os.path.join(ROWS_CSV, filename + '.gz')
    with open_compressed(filename_out, 'wb') as f:
        print 'writing', filename_out
        writer = csv.writer(f)
        writer.writerow(header)
        for header_in, row_in in load_rows(filename):
            row_out = transform_row(header_in, row_in)
            writer.writerow([row_out[key] for key in header])
            print_dot(1000)

@parsable.command
def transform():
    '''
    Transform dataset.
    '''
    schema = transform_schema()
    print 'writing', SCHEMA_JSON
    json_dump(schema, SCHEMA_JSON)
    loom.util.mkdir_p(ROWS_CSV)
    loom.util.parallel_map(transform_rows, ROW_COUNTS.keys())

@parsable.command
def ingest():
    '''
    Ingest dataset into loom.
    '''
    loom.tasks.ingest(NAME, SCHEMA_JSON, ROWS_CSV)

@parsable.command
def infer(sample_count=SAMPLE_COUNT):
    '''
    Infer model.
    '''
    loom.tasks.infer(NAME, sample_count=sample_count)

@parsable.command
def related():
    '''
    Compute feature relatedness.
    '''
    with loom.tasks.query(NAME) as preql:
        preql.relate(preql.feature_names, result_out=RELATED)

@parsable.command
def plot():
    '''
    Plot results.
    '''
    raise NotImplementedError('TODO')

@parsable.command
def run():
    '''
    Run entire pipeline:
        download
        find text features
        transform
        ingest
        infer
        related
        plot
    '''
    download()
    loom.util.parallel_map(find_text_features, ['desc', 'emp_title', 'title'])
    transform()
    ingest()
    infer()
    related()
    plot()

if __name__ == '__main__':
    parsable.dispatch()
<|endoftext|>


<commit_before>"""Set up a model directory.

Requires:

    lang_data --- Rules for the tokenizer
        * prefix.txt
        * suffix.txt
        * infix.txt
        * morphs.json
        * specials.json

    corpora --- Data files
        * WordNet
        * words.sgt.prob --- Smoothed unigram probabilities
        * clusters.txt --- Output of hierarchical clustering, e.g. Brown clusters
        * vectors.tgz --- output of something like word2vec
"""
import plac
from pathlib import Path

from shutil import copyfile
from shutil import copytree
import codecs

from spacy.en import get_lex_props
from spacy.vocab import Vocab
from spacy.vocab import write_binary_vectors

def setup_tokenizer(lang_data_dir, tok_dir):
    if not tok_dir.exists():
        tok_dir.mkdir()

    for filename in ('infix.txt', 'morphs.json', 'prefix.txt', 'specials.json',
                     'suffix.txt'):
        src = lang_data_dir / filename
        dst = tok_dir / filename
        if not dst.exists():
            copyfile(str(src), str(dst))

def _read_clusters(loc):
    clusters = {}
    for line in codecs.open(str(loc), 'r', 'utf8'):
        try:
            cluster, word, freq = line.split()
        except ValueError:
            continue
        # If the clusterer has only seen the word a few times, its cluster is
        # unreliable.
        if int(freq) >= 3:
            clusters[word] = cluster
    return clusters

def _read_probs(loc):
    probs = {}
    for i, line in enumerate(codecs.open(str(loc), 'r', 'utf8')):
        prob, word = line.split()
        prob = float(prob)
        probs[word] = prob
    return probs

def setup_vocab(src_dir, dst_dir):
    if not dst_dir.exists():
        dst_dir.mkdir()

    vectors_src = src_dir / 'vectors.tgz'
    if vectors_src.exists():
        write_binary_vectors(str(vectors_src), str(dst_dir / 'vec.bin'))
    vocab = Vocab(data_dir=None, get_lex_props=get_lex_props)
    clusters = _read_clusters(src_dir / 'clusters.txt')
    probs = _read_probs(src_dir / 'words.sgt.prob')
    lexicon = []
    for word, prob in reversed(sorted(probs.items(), key=lambda item: item[1])):
        entry = get_lex_props(word)
        if word in clusters or float(prob) >= -17:
            entry['prob'] = float(prob)
            cluster = clusters.get(word, '0')
            # Decode as a little-endian string, so that we can do & 15 to get
            # the first 4 bits. See _parse_features.pyx
            entry['cluster'] = int(cluster[::-1], 2)
            vocab[word] = entry
    vocab.dump(str(dst_dir / 'lexemes.bin'))
    vocab.strings.dump(str(dst_dir / 'strings.txt'))

def main(lang_data_dir, corpora_dir, model_dir):
    model_dir = Path(model_dir)
    lang_data_dir = Path(lang_data_dir)
    corpora_dir = Path(corpora_dir)

    assert corpora_dir.exists()
    assert lang_data_dir.exists()

    if not model_dir.exists():
        model_dir.mkdir()

    setup_tokenizer(lang_data_dir, model_dir / 'tokenizer')
    setup_vocab(corpora_dir, model_dir / 'vocab')
    if not (model_dir / 'wordnet').exists():
        copytree(str(corpora_dir / 'wordnet'), str(model_dir / 'wordnet'))

if __name__ == '__main__':
    plac.call(main)
<commit_msg>* Add cluster=0 by default in init_model<commit_after>"""Set up a model directory.

Requires:

    lang_data --- Rules for the tokenizer
        * prefix.txt
        * suffix.txt
        * infix.txt
        * morphs.json
        * specials.json

    corpora --- Data files
        * WordNet
        * words.sgt.prob --- Smoothed unigram probabilities
        * clusters.txt --- Output of hierarchical clustering, e.g. Brown clusters
        * vectors.tgz --- output of something like word2vec
"""
import plac
from pathlib import Path

from shutil import copyfile
from shutil import copytree
import codecs

from spacy.en import get_lex_props
from spacy.vocab import Vocab
from spacy.vocab import write_binary_vectors

def setup_tokenizer(lang_data_dir, tok_dir):
    if not tok_dir.exists():
        tok_dir.mkdir()

    for filename in ('infix.txt', 'morphs.json', 'prefix.txt', 'specials.json',
                     'suffix.txt'):
        src = lang_data_dir / filename
        dst = tok_dir / filename
        if not dst.exists():
            copyfile(str(src), str(dst))

def _read_clusters(loc):
    clusters = {}
    for line in codecs.open(str(loc), 'r', 'utf8'):
        try:
            cluster, word, freq = line.split()
        except ValueError:
            continue
        # If the clusterer has only seen the word a few times, its cluster is
        # unreliable.
        if int(freq) >= 3:
            clusters[word] = cluster
        else:
            clusters[word] = '0'
    return clusters

def _read_probs(loc):
    probs = {}
    for i, line in enumerate(codecs.open(str(loc), 'r', 'utf8')):
        prob, word = line.split()
        prob = float(prob)
        probs[word] = prob
    return probs

def setup_vocab(src_dir, dst_dir):
    if not dst_dir.exists():
        dst_dir.mkdir()

    vectors_src = src_dir / 'vectors.tgz'
    if vectors_src.exists():
        write_binary_vectors(str(vectors_src), str(dst_dir / 'vec.bin'))
    vocab = Vocab(data_dir=None, get_lex_props=get_lex_props)
    clusters = _read_clusters(src_dir / 'clusters.txt')
    probs = _read_probs(src_dir / 'words.sgt.prob')
    lexicon = []
    for word, prob in reversed(sorted(probs.items(), key=lambda item: item[1])):
        entry = get_lex_props(word)
        if word in clusters or float(prob) >= -17:
            entry['prob'] = float(prob)
            cluster = clusters.get(word, '0')
            # Decode as a little-endian string, so that we can do & 15 to get
            # the first 4 bits. See _parse_features.pyx
            entry['cluster'] = int(cluster[::-1], 2)
            vocab[word] = entry
    vocab.dump(str(dst_dir / 'lexemes.bin'))
    vocab.strings.dump(str(dst_dir / 'strings.txt'))

def main(lang_data_dir, corpora_dir, model_dir):
    model_dir = Path(model_dir)
    lang_data_dir = Path(lang_data_dir)
    corpora_dir = Path(corpora_dir)

    assert corpora_dir.exists()
    assert lang_data_dir.exists()

    if not model_dir.exists():
        model_dir.mkdir()

    setup_tokenizer(lang_data_dir, model_dir / 'tokenizer')
    setup_vocab(corpora_dir, model_dir / 'vocab')
    if not (model_dir / 'wordnet').exists():
        copytree(str(corpora_dir / 'wordnet'), str(model_dir / 'wordnet'))

if __name__ == '__main__':
    plac.call(main)
<|endoftext|>


<commit_before>"""
Copyright (c) 2015 Robert Hodgen

All rights reserved.
"""

from google.appengine.ext import ndb

from datetime import datetime, timedelta, tzinfo

import utilities

from ndb_users import users

class UTC(tzinfo):
  """ Defines a `tzinfo` subclass for UTC/Universal Coordindated Time.
  All dates are stored in NDB without time zone information. This UTC class
  is needed so `datetime.isoformat()` includes timezone information. """

  def utcoffset(self, dt):
    return timedelta(0);

  def tzname(self, dt):
    return 'UTC'

  def dst(self, dt):
    return timedelta(0)

class Project(ndb.Model):
  """ Project """

  # The name/short description of this Project
  name = ndb.StringProperty(required=True)

  # The long description of this Project
  description = ndb.StringProperty(default=None)

  # The permalink (or web safe name, used in URLs) of this Project
  permalink = ndb.StringProperty(required=True)

  # The primary User's email address
  owner = ndb.StringProperty(required=True)

  # Other User's email address whom can contribute (create/end) Time Records
  contributors = ndb.StringProperty(repeated=True)

  # Other User's email address whom can view the Project and Time Records
  observers = ndb.StringProperty(repeated=True)

  # Holds the `owner`, every `contributors` and every `observers`;
  # Used for querying Projects to avoid massive OR queries...
  users = ndb.StringProperty(repeated=True)

  # The number of seconds this Project has recorded
  completed = ndb.FloatProperty(default=0)

  # Whether this Project should appear in a User's Projects list (not archived)
  active = ndb.BooleanProperty(required=True, default=True)

  # Record WHEN this record was truly created and last updated
  created = ndb.DateTimeProperty(auto_now_add=True)
  updated = ndb.DateTimeProperty(auto_now=True)

  @property
  def created_utc(self):
    """ Return `created` with tzinfo set to `UTC`. """
    return self.created.replace(tzinfo=UTC())

  @property
  def updated_utc(self):
    """ Return `updated` with tzinfo set to `UTC`. """
    return self.updated.replace(tzinfo=UTC())

  @property
  def has_uncompleted_time_records(self):
    """ Run a query to see if this Project has uncompleted Time Records. """
    query = TimeRecord.query(TimeRecord.completed==None, ancestor=self.key)
    return (query.count(limit=1) > 0)

  @classmethod
  def create_project(cls, name):
    """ Create a new Project for the currently logged in user named `name`. """
    user = users.get_current_user()
    new_project = cls(
      name=name,
      permalink=utilities.permalinkify(name),
      owner=user.email,
      users=[user.email]
    )
    return new_project.put()

  def add_contributors(self, contributors):
    """ Add `contributors` to `self.contributors`.
    `contributors` is expected as an array of email address strings.
    Update `self.users` as needed. """
    for contributor in contributors:
      if contributor != self.owner:
        if contributor not in self.contributors:
          self.contributors.append(contributor)
        if contributor not in self.users:
          self.users.append(contributor)
        if contributor in self.observers:
          self.observers.remove(contributor)
    return self.put()

  def remove_contributors(self, contributors):
    """ Remove `contributors` from `self.contributors`.
    `contributors` is expected to be an array of email address strings.
    Update `self.users` as needed. """
    for contributor in contributors:
      if contributor in self.contributors:
        self.contributors.remove(contributor)
        if contributor in self.users and contributor != self.owner:
          self.users.remove(contributor)
    return self.put()

  def is_owner(self, email):
    """ """
    return email == self.owner

  def has_contributor(self, contributor):
    """ Returns whether `contributor` is a Contributor of this Project. """
    return contributor in self.contributors

  def has_observor(self, observer):
    """ Returns whether `observer` is an Observer of this Project. """
    return observer in self.observers

  def json_object(self):
    """ Return a dictionary representing this Project.
    Will be used for JSON requests! """
    return {
      'id': self.key.urlsafe(),
      'name': self.name,
      'owner': self.owner,
      'contributors': self.contributors,
      'observers': self.observers,
      'completed': self.completed,
      'has_uncompleted_time_records': self.has_uncompleted_time_records,
      'active': self.active,
      'created': self.created_utc.isoformat(),
      'updated': self.updated_utc.isoformat()
    }

class TimeRecord(ndb.Model):
  """ Time Record """

  tags = ndb.StringProperty(repeated=True)

  # The name/short description of this Time Record
  name = ndb.StringProperty(default=None)

  # The date/time (in UTC) this Time Record was started
  start = ndb.DateTimeProperty(required=True, auto_now_add=True)

  # Store a the User's email address who (created) this Time Record
  user = ndb.StringProperty(required=True)

  # The date/time (in UTC) this Time Record was ended
  end = ndb.DateTimeProperty(default=None)

  # The number of seconds this Time Record counted for
  completed = ndb.FloatProperty(default=None)

  # Record WHEN this record was truly created and last updated
  created = ndb.DateTimeProperty(auto_now_add=True)
  updated = ndb.DateTimeProperty(auto_now=True)

  @classmethod
  @ndb.transactional()
  def create_time_record(cls, project_key, user_email):
    """ Create a new TimeRecord for the Project identified by `project_key`. """
    new_time_record = cls(
      parent=project_key,
      user=user_email
    )
    project_key.get().put() # Update the `updated` property
    return new_time_record.put()

  def complete_time_record(self):
    """ Complete this Time Record by setting the `end` property, computing
    it's total seconds and setting this Time Record's `completed`
    property and add its seconds to it's parent Project. """

    # Only proceed if `self.completed` is None
    if not self.completed:
      # Update and record
      self.end = datetime.now()
      self.completed = (self.end - self.start).total_seconds()
      self.put()

      # Update the parent Project
      project_key = self.key.parent()
      project = project_key.get()
      project.completed += self.completed
      project.put()
    return self

  def json_object(self):
    """ Return a dictionary representing this Time Record. Will be used for
    sending information about this Time Record via JSON requests. """
    start = self.start.replace(tzinfo=UTC())
    end_string_value = None
    if self.end:
      end_string_value = self.end.replace(tzinfo=UTC()).isoformat()

    return {
      'id': self.key.urlsafe(),
      'start': start.isoformat(),
      'end': end_string_value,
      'completed': self.completed,
      'created': self.created.replace(tzinfo=UTC()).isoformat(),
      'updated': self.updated.replace(tzinfo=UTC()).isoformat(),
      'name': self.name,
      'user': self.user
    }

class Account(ndb.Model):

  pass

<commit_msg>Added Comment class in model.py; will be used for comments on Projects, Time Records, etc.<commit_after>"""
Copyright (c) 2015 Robert Hodgen

All rights reserved.
"""

from google.appengine.ext import ndb

from datetime import datetime, timedelta, tzinfo

import utilities

from ndb_users import users

class UTC(tzinfo):
  """ Defines a `tzinfo` subclass for UTC/Universal Coordindated Time.
  All dates are stored in NDB without time zone information. This UTC class
  is needed so `datetime.isoformat()` includes timezone information. """

  def utcoffset(self, dt):
    return timedelta(0);

  def tzname(self, dt):
    return 'UTC'

  def dst(self, dt):
    return timedelta(0)

class Project(ndb.Model):
  """ Project """
  # The name/short description of this Project
  name = ndb.StringProperty(required=True)
  # The long description of this Project
  description = ndb.StringProperty(default=None)
  # The permalink (or web safe name, used in URLs) of this Project
  permalink = ndb.StringProperty(required=True)
  # The primary User's email address
  owner = ndb.StringProperty(required=True)
  # Other User's email address whom can contribute (create/end) Time Records
  contributors = ndb.StringProperty(repeated=True)
  # Other User's email address whom can view the Project and Time Records
  observers = ndb.StringProperty(repeated=True)
  # Holds the `owner`, every `contributors` and every `observers`;
  # Used for querying Projects to avoid massive OR queries...
  users = ndb.StringProperty(repeated=True)
  # The number of seconds this Project has recorded
  completed = ndb.FloatProperty(default=0)
  # Whether this Project should appear in a User's Projects list (not archived)
  active = ndb.BooleanProperty(required=True, default=True)
  # Record WHEN this record was truly created and last updated
  created = ndb.DateTimeProperty(auto_now_add=True)
  updated = ndb.DateTimeProperty(auto_now=True)

  @property
  def created_utc(self):
    """ Return `created` with tzinfo set to `UTC`. """
    return self.created.replace(tzinfo=UTC())

  @property
  def updated_utc(self):
    """ Return `updated` with tzinfo set to `UTC`. """
    return self.updated.replace(tzinfo=UTC())

  @property
  def has_uncompleted_time_records(self):
    """ Run a query to see if this Project has uncompleted Time Records. """
    query = TimeRecord.query(TimeRecord.completed==None, ancestor=self.key)
    return (query.count(limit=1) > 0)

  @classmethod
  def create_project(cls, name):
    """ Create a new Project for the currently logged in user named `name`. """
    user = users.get_current_user()
    new_project = cls(
      name=name,
      permalink=utilities.permalinkify(name),
      owner=user.email,
      users=[user.email]
    )
    return new_project.put()

  def add_contributors(self, contributors):
    """ Add `contributors` to `self.contributors`.
    `contributors` is expected as an array of email address strings.
    Update `self.users` as needed. """
    for contributor in contributors:
      if contributor != self.owner:
        if contributor not in self.contributors:
          self.contributors.append(contributor)
        if contributor not in self.users:
          self.users.append(contributor)
        if contributor in self.observers:
          self.observers.remove(contributor)
    return self.put()

  def remove_contributors(self, contributors):
    """ Remove `contributors` from `self.contributors`.
    `contributors` is expected to be an array of email address strings.
    Update `self.users` as needed. """
    for contributor in contributors:
      if contributor in self.contributors:
        self.contributors.remove(contributor)
        if contributor in self.users and contributor != self.owner:
          self.users.remove(contributor)
    return self.put()

  def is_owner(self, email):
    """ """
    return email == self.owner

  def has_contributor(self, contributor):
    """ Returns whether `contributor` is a Contributor of this Project. """
    return contributor in self.contributors

  def has_observor(self, observer):
    """ Returns whether `observer` is an Observer of this Project. """
    return observer in self.observers

  def json_object(self):
    """ Return a dictionary representing this Project.
    Will be used for JSON requests! """
    return {
      'id': self.key.urlsafe(),
      'name': self.name,
      'owner': self.owner,
      'contributors': self.contributors,
      'observers': self.observers,
      'completed': self.completed,
      'has_uncompleted_time_records': self.has_uncompleted_time_records,
      'active': self.active,
      'created': self.created_utc.isoformat(),
      'updated': self.updated_utc.isoformat()
    }

class TimeRecord(ndb.Model):
  """ Time Record """
  tags = ndb.StringProperty(repeated=True)
  # The name/short description of this Time Record
  name = ndb.StringProperty(default=None)
  # The date/time (in UTC) this Time Record was started
  start = ndb.DateTimeProperty(required=True, auto_now_add=True)
  # Store a the User's email address who (created) this Time Record
  user = ndb.StringProperty(required=True)
  # The date/time (in UTC) this Time Record was ended
  end = ndb.DateTimeProperty(default=None)
  # The number of seconds this Time Record counted for
  completed = ndb.FloatProperty(default=None)
  # Record WHEN this record was truly created and last updated
  created = ndb.DateTimeProperty(auto_now_add=True)
  updated = ndb.DateTimeProperty(auto_now=True)

  @classmethod
  @ndb.transactional(xg=True)
  def create_time_record(cls, project_key, user_email):
    """ Create a new TimeRecord for the Project identified by `project_key`. """
    new_time_record = cls(
      parent=project_key,
      user=user_email
    )
    project_key.get().put() # Update the `updated` property of our parent
    return new_time_record.put()

  def complete_time_record(self):
    """ Complete this Time Record by setting the `end` property, computing
    it's total seconds and setting this Time Record's `completed`
    property and add its seconds to it's parent Project. """
    # Only proceed if `self.completed` is None
    if not self.completed:
      # Update and record
      self.end = datetime.now()
      self.completed = (self.end - self.start).total_seconds()
      self.put()
      # Update the parent Project
      project_key = self.key.parent()
      project = project_key.get()
      project.completed += self.completed
      project.put()
    return self

  def json_object(self):
    """ Return a dictionary representing this Time Record. Will be used for
    sending information about this Time Record via JSON requests. """
    start = self.start.replace(tzinfo=UTC())
    end_string_value = None
    if self.end:
      end_string_value = self.end.replace(tzinfo=UTC()).isoformat()

    return {
      'id': self.key.urlsafe(),
      'start': start.isoformat(),
      'end': end_string_value,
      'completed': self.completed,
      'created': self.created.replace(tzinfo=UTC()).isoformat(),
      'updated': self.updated.replace(tzinfo=UTC()).isoformat(),
      'name': self.name,
      'user': self.user
    }

class Comment(ndb.Model):
  """ Comments left on TimeRecords or general Project comments. """
  # The content of the comment
  comment = ndb.StringProperty(required=True)
  # Store the User's email address who (created) this Comment
  user = ndb.StringProperty(required=True)
  # Store the Project to which this Comment is held
  project = ndb.KeyProperty(required=True, kind=Project)
  # Record WHEN this comment was truly created and last updated
  created = ndb.DateTimeProperty(auto_now_add=True)
  updated = ndb.DateTimeProperty(auto_now=True)

  @classmethod
  @ndb.transactional(xg=True)
  def create_comment(cls, comment, parent_key, project_key, user_email):
    """ Create a new Comment belonging to `parent_key`. """
    new_comment = cls(
      parent=parent_key,
      project=project_key,
      comment=comment,
      user=user_email
    )
    parent_key.get().put() # Update the `updated` property of our parent
    if parent_key != project_key:
      project_key.get().put() # Update the `updated` property of our Project
    return new_comment.put()

class Account(ndb.Model):

  pass

<|endoftext|>


<commit_before>import pytest
import numpy as np

import astropy
from astropy.wcs import WCS
from astropy import units as u
from astropy.io import fits

from ..spectral_cube import SpectralCube
from .helpers import assert_allclose

from distutils.version import StrictVersion

# the back of the book
dv = 3e-2 * u.Unit('m/s')
dy = 2e-5 * u.Unit('deg')
dx = 1e-5 * u.Unit('deg')
data_unit = u.K

m0v = np.array([[27, 30, 33],
                [36, 39, 42],
                [45, 48, 51]]) * data_unit * dv
m0y = np.array([[9, 12, 15],
                [36, 39, 42],
                [63, 66, 69]]) * data_unit * dy
m0x = np.array([[3, 12, 21],
                [30, 39, 48],
                [57, 66, 75]]) * data_unit * dx

# M1V is a special case, where we return the actual coordinate
m1v = np.array([[1.66666667, 1.6, 1.54545455],
                [1.5, 1.46153846, 1.42857143],
                [1.4, 1.375, 1.35294118]]) * dv + 2 * u.Unit('m/s')
m1y = np.array([[1.66666667, 1.5, 1.4],
                [1.16666667, 1.15384615, 1.14285714],
                [1.0952381, 1.09090909, 1.08695652]]) * dy
m1x = np.array([[1.66666667, 1.16666667, 1.0952381],
                [1.06666667, 1.05128205, 1.04166667],
                [1.03508772, 1.03030303, 1.02666667]]) * dx

m2v = np.array([[0.22222222, 0.30666667, 0.36914601],
               [0.41666667, 0.45364892, 0.4829932],
               [0.50666667, 0.52604167, 0.54209919]]) * dv ** 2
m2y = np.array([[0.22222222, 0.41666667, 0.50666667],
                [0.63888889, 0.64299803, 0.6462585],
                [0.65759637, 0.6584022, 0.65910523]]) * dy ** 2
m2x = np.array([[0.22222222, 0.63888889, 0.65759637],
                [0.66222222, 0.66403682, 0.66493056],
                [0.66543552, 0.66574839, 0.66595556]]) * dx ** 2
MOMENTS = [[m0v, m0y, m0x], [m1v, m1y, m1x], [m2v, m2y, m2x]]
# In issue 184, the cubes were corrected such that they all have valid units
# Therefore, no separate tests are needed for moments-with-units and those
# without
MOMENTSu = MOMENTS

def moment_cube():
    cube = np.arange(27).reshape([3, 3, 3]).astype(np.float)
    wcs = WCS(naxis=3)
    wcs.wcs.ctype = ['RA---TAN', 'DEC--TAN', 'VELO']
    # choose values to minimize spherical distortions
    wcs.wcs.cdelt = np.array([-1, 2, 3], dtype='float32') / 1e5
    wcs.wcs.crpix = np.array([1, 1, 1], dtype='float32')
    wcs.wcs.crval = np.array([0, 1e-3, 2e-3], dtype='float32')
    wcs.wcs.cunit = ['deg', 'deg', 'km/s']

    header = wcs.to_header()
    header['BUNIT'] = 'K'

    hdu = fits.PrimaryHDU(data=cube, header=header)
    return hdu

axis_order = pytest.mark.parametrize(('axis', 'order'),
                                    ((0, 0), (0, 1), (0, 2),
                                     (1, 0), (1, 1), (1, 2),
                                     (2, 0), (2, 1), (2, 2)))

if StrictVersion(astropy.__version__[:3]) > StrictVersion('1.0'):
    # The relative error is slightly larger on astropy-dev
    # There is no obvious reason for this.
    rtol = 2e-7
    atol = 1e-40
else:
    rtol = 1e-7
    atol = 0.0

@axis_order
def test_strategies_consistent(axis, order):
    mc_hdu = moment_cube()
    sc = SpectralCube.read(mc_hdu)

    cwise = sc.moment(axis=axis, order=order, how='cube')
    swise = sc.moment(axis=axis, order=order, how='slice')
    rwise = sc.moment(axis=axis, order=order, how='ray')
    assert_allclose(cwise, swise, rtol=rtol, atol=atol)
    assert_allclose(cwise, rwise, rtol=rtol, atol=atol)

@pytest.mark.parametrize(('order', 'axis', 'how'),
                         [(o, a, h)
                          for o in [0, 1, 2]
                          for a in [0, 1, 2]
                          for h in ['cube', 'slice', 'auto', 'ray']])
def test_reference(order, axis, how):
    mc_hdu = moment_cube()
    sc = SpectralCube.read(mc_hdu)
    mom_sc = sc.moment(order=order, axis=axis, how=how)
    assert_allclose(mom_sc, MOMENTS[order][axis])

@axis_order
def test_consistent_mask_handling(axis, order):
    mc_hdu = moment_cube()
    sc = SpectralCube.read(mc_hdu)
    sc._mask = sc > 4*u.K

    cwise = sc.moment(axis=axis, order=order, how='cube')
    swise = sc.moment(axis=axis, order=order, how='slice')
    rwise = sc.moment(axis=axis, order=order, how='ray')
    assert_allclose(cwise, swise, rtol=rtol, atol=atol)
    assert_allclose(cwise, rwise, rtol=rtol, atol=atol)

def test_convenience_methods():
    mc_hdu = moment_cube()
    sc = SpectralCube.read(mc_hdu)

    assert_allclose(sc.moment0(axis=0), MOMENTS[0][0])
    assert_allclose(sc.moment1(axis=2), MOMENTS[1][2])
    assert_allclose(sc.moment2(axis=1), MOMENTS[2][1])

def test_preserve_unit():
    mc_hdu = moment_cube()
    sc = SpectralCube.read(mc_hdu)
    sc_kms = sc.with_spectral_unit(u.km/u.s)
    m0 = sc_kms.moment0(axis=0)
    m1 = sc_kms.moment1(axis=0)

    assert_allclose(m0, MOMENTS[0][0].to(u.K*u.km/u.s))
    assert_allclose(m1, MOMENTS[1][0].to(u.km/u.s))

def test_with_flux_unit():
    """
    As of Issue 184, redundant with test_reference
    """
    mc_hdu = moment_cube()
    sc = SpectralCube.read(mc_hdu)
    sc._unit = u.K
    sc_kms = sc.with_spectral_unit(u.km/u.s)
    m0 = sc_kms.moment0(axis=0)
    m1 = sc_kms.moment1(axis=0)

    assert sc.unit == u.K
    assert sc.filled_data[:].unit == u.K

    assert_allclose(m0, MOMENTS[0][0].to(u.K*u.km/u.s))
    assert_allclose(m1, MOMENTS[1][0].to(u.km/u.s))

@pytest.mark.parametrize(('order', 'axis', 'how'),
                         [(o, a, h)
                          for o in [0, 1, 2]
                          for a in [0, 1, 2]
                          for h in ['cube', 'slice', 'auto', 'ray']])
def test_how_withfluxunit(order, axis, how):
    """
    Regression test for issue 180
    As of issue 184, this is mostly redundant with test_reference except that
    it (kind of) checks that units are set
    """
    mc_hdu = moment_cube()
    sc = SpectralCube.read(mc_hdu)
    sc._unit = u.K
    mom_sc = sc.moment(order=order, axis=axis, how=how)

    assert sc.unit == u.K
    assert sc.filled_data[:].unit == u.K

    assert_allclose(mom_sc, MOMENTSu[order][axis])
<commit_msg>add a test for the metadata<commit_after>import pytest
import numpy as np

import astropy
from astropy.wcs import WCS
from astropy import units as u
from astropy.io import fits

from ..spectral_cube import SpectralCube
from .helpers import assert_allclose

from distutils.version import StrictVersion

# the back of the book
dv = 3e-2 * u.Unit('m/s')
dy = 2e-5 * u.Unit('deg')
dx = 1e-5 * u.Unit('deg')
data_unit = u.K

m0v = np.array([[27, 30, 33],
                [36, 39, 42],
                [45, 48, 51]]) * data_unit * dv
m0y = np.array([[9, 12, 15],
                [36, 39, 42],
                [63, 66, 69]]) * data_unit * dy
m0x = np.array([[3, 12, 21],
                [30, 39, 48],
                [57, 66, 75]]) * data_unit * dx

# M1V is a special case, where we return the actual coordinate
m1v = np.array([[1.66666667, 1.6, 1.54545455],
                [1.5, 1.46153846, 1.42857143],
                [1.4, 1.375, 1.35294118]]) * dv + 2 * u.Unit('m/s')
m1y = np.array([[1.66666667, 1.5, 1.4],
                [1.16666667, 1.15384615, 1.14285714],
                [1.0952381, 1.09090909, 1.08695652]]) * dy
m1x = np.array([[1.66666667, 1.16666667, 1.0952381],
                [1.06666667, 1.05128205, 1.04166667],
                [1.03508772, 1.03030303, 1.02666667]]) * dx

m2v = np.array([[0.22222222, 0.30666667, 0.36914601],
               [0.41666667, 0.45364892, 0.4829932],
               [0.50666667, 0.52604167, 0.54209919]]) * dv ** 2
m2y = np.array([[0.22222222, 0.41666667, 0.50666667],
                [0.63888889, 0.64299803, 0.6462585],
                [0.65759637, 0.6584022, 0.65910523]]) * dy ** 2
m2x = np.array([[0.22222222, 0.63888889, 0.65759637],
                [0.66222222, 0.66403682, 0.66493056],
                [0.66543552, 0.66574839, 0.66595556]]) * dx ** 2
MOMENTS = [[m0v, m0y, m0x], [m1v, m1y, m1x], [m2v, m2y, m2x]]
# In issue 184, the cubes were corrected such that they all have valid units
# Therefore, no separate tests are needed for moments-with-units and those
# without
MOMENTSu = MOMENTS

def moment_cube():
    cube = np.arange(27).reshape([3, 3, 3]).astype(np.float)
    wcs = WCS(naxis=3)
    wcs.wcs.ctype = ['RA---TAN', 'DEC--TAN', 'VELO']
    # choose values to minimize spherical distortions
    wcs.wcs.cdelt = np.array([-1, 2, 3], dtype='float32') / 1e5
    wcs.wcs.crpix = np.array([1, 1, 1], dtype='float32')
    wcs.wcs.crval = np.array([0, 1e-3, 2e-3], dtype='float32')
    wcs.wcs.cunit = ['deg', 'deg', 'km/s']

    header = wcs.to_header()
    header['BUNIT'] = 'K'

    hdu = fits.PrimaryHDU(data=cube, header=header)
    return hdu

axis_order = pytest.mark.parametrize(('axis', 'order'),
                                    ((0, 0), (0, 1), (0, 2),
                                     (1, 0), (1, 1), (1, 2),
                                     (2, 0), (2, 1), (2, 2)))

if StrictVersion(astropy.__version__[:3]) > StrictVersion('1.0'):
    # The relative error is slightly larger on astropy-dev
    # There is no obvious reason for this.
    rtol = 2e-7
    atol = 1e-40
else:
    rtol = 1e-7
    atol = 0.0

@axis_order
def test_strategies_consistent(axis, order):
    mc_hdu = moment_cube()
    sc = SpectralCube.read(mc_hdu)

    cwise = sc.moment(axis=axis, order=order, how='cube')
    swise = sc.moment(axis=axis, order=order, how='slice')
    rwise = sc.moment(axis=axis, order=order, how='ray')
    assert_allclose(cwise, swise, rtol=rtol, atol=atol)
    assert_allclose(cwise, rwise, rtol=rtol, atol=atol)

@pytest.mark.parametrize(('order', 'axis', 'how'),
                         [(o, a, h)
                          for o in [0, 1, 2]
                          for a in [0, 1, 2]
                          for h in ['cube', 'slice', 'auto', 'ray']])
def test_reference(order, axis, how):
    mc_hdu = moment_cube()
    sc = SpectralCube.read(mc_hdu)
    mom_sc = sc.moment(order=order, axis=axis, how=how)
    assert_allclose(mom_sc, MOMENTS[order][axis])

@axis_order
def test_consistent_mask_handling(axis, order):
    mc_hdu = moment_cube()
    sc = SpectralCube.read(mc_hdu)
    sc._mask = sc > 4*u.K

    cwise = sc.moment(axis=axis, order=order, how='cube')
    swise = sc.moment(axis=axis, order=order, how='slice')
    rwise = sc.moment(axis=axis, order=order, how='ray')
    assert_allclose(cwise, swise, rtol=rtol, atol=atol)
    assert_allclose(cwise, rwise, rtol=rtol, atol=atol)

def test_convenience_methods():
    mc_hdu = moment_cube()
    sc = SpectralCube.read(mc_hdu)

    assert_allclose(sc.moment0(axis=0), MOMENTS[0][0])
    assert_allclose(sc.moment1(axis=2), MOMENTS[1][2])
    assert_allclose(sc.moment2(axis=1), MOMENTS[2][1])

def test_preserve_unit():
    mc_hdu = moment_cube()
    sc = SpectralCube.read(mc_hdu)
    sc_kms = sc.with_spectral_unit(u.km/u.s)
    m0 = sc_kms.moment0(axis=0)
    m1 = sc_kms.moment1(axis=0)

    assert_allclose(m0, MOMENTS[0][0].to(u.K*u.km/u.s))
    assert_allclose(m1, MOMENTS[1][0].to(u.km/u.s))

def test_with_flux_unit():
    """
    As of Issue 184, redundant with test_reference
    """
    mc_hdu = moment_cube()
    sc = SpectralCube.read(mc_hdu)
    sc._unit = u.K
    sc_kms = sc.with_spectral_unit(u.km/u.s)
    m0 = sc_kms.moment0(axis=0)
    m1 = sc_kms.moment1(axis=0)

    assert sc.unit == u.K
    assert sc.filled_data[:].unit == u.K

    assert_allclose(m0, MOMENTS[0][0].to(u.K*u.km/u.s))
    assert_allclose(m1, MOMENTS[1][0].to(u.km/u.s))

@pytest.mark.parametrize(('order', 'axis', 'how'),
                         [(o, a, h)
                          for o in [0, 1, 2]
                          for a in [0, 1, 2]
                          for h in ['cube', 'slice', 'auto', 'ray']])
def test_how_withfluxunit(order, axis, how):
    """
    Regression test for issue 180
    As of issue 184, this is mostly redundant with test_reference except that
    it (kind of) checks that units are set
    """
    mc_hdu = moment_cube()
    sc = SpectralCube.read(mc_hdu)
    sc._unit = u.K
    mom_sc = sc.moment(order=order, axis=axis, how=how)

    assert sc.unit == u.K
    assert sc.filled_data[:].unit == u.K
    assert 'beam' in sc.meta
    assert 'beam' in sc.hdu.header

    assert_allclose(mom_sc, MOMENTSu[order][axis])
<|endoftext|>


<commit_before><commit_msg>Simplified scrollbar code by switching to after_idle(); needs to be tested on all platforms.<commit_after><|endoftext|>