<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# This script computes the new "current" toolstate for the toolstate repo (not to be
# confused with publishing the test results, which happens in `src/bootstrap/toolstate.rs`).
# It gets called from `src/ci/publish_toolstate.sh` when a new commit lands on `master`
# (i.e., after it passed all checks on `auto`).

from __future__ import print_function

import sys
import re
import os
import json
import datetime
import collections
import textwrap
try:
    import urllib2
except ImportError:
    import urllib.request as urllib2

# List of people to ping when the status of a tool or a book changed.
# These should be collaborators of the rust-lang/rust repository (with at least
# read privileges on it). CI will fail otherwise.
MAINTAINERS = {
    'miri': {'oli-obk', 'RalfJung', 'eddyb'},
    'rls': {'Xanewok'},
    'rustfmt': {'topecongiro', 'calebcartwright'},
    'book': {'carols10cents', 'steveklabnik'},
    'nomicon': {'frewsxcv', 'Gankra'},
    'reference': {'steveklabnik', 'Havvy', 'matthewjasper', 'ehuss'},
    'rust-by-example': {'steveklabnik', 'marioidival'},
    'embedded-book': {'adamgreig', 'andre-richter', 'jamesmunns', 'therealprof'},
    'edition-guide': {'ehuss', 'steveklabnik'},
    'rustc-dev-guide': {'mark-i-m', 'spastorino', 'amanjeev', 'JohnTitor'},
}

LABELS = {
    'miri': ['A-miri', 'C-bug'],
    'rls': ['A-rls', 'C-bug'],
    'rustfmt': ['A-rustfmt', 'C-bug'],
    'book': ['C-bug'],
    'nomicon': ['C-bug'],
    'reference': ['C-bug'],
    'rust-by-example': ['C-bug'],
    'embedded-book': ['C-bug'],
    'edition-guide': ['C-bug'],
    'rustc-dev-guide': ['C-bug'],
}

REPOS = {
    'miri': 'https://github.com/rust-lang/miri',
    'rls': 'https://github.com/rust-lang/rls',
    'rustfmt': 'https://github.com/rust-lang/rustfmt',
    'book': 'https://github.com/rust-lang/book',
    'nomicon': 'https://github.com/rust-lang/nomicon',
    'reference': 'https://github.com/rust-lang/reference',
    'rust-by-example': 'https://github.com/rust-lang/rust-by-example',
    'embedded-book': 'https://github.com/rust-embedded/book',
    'edition-guide': 'https://github.com/rust-lang/edition-guide',
    'rustc-dev-guide': 'https://github.com/rust-lang/rustc-dev-guide',
}

def load_json_from_response(resp):
    content = resp.read()
    if isinstance(content, bytes):
        content = content.decode('utf-8')
    else:
        print("Refusing to decode " + str(type(content)) + " to str")
    return json.loads(content)

def validate_maintainers(repo, github_token):
    '''Ensure all maintainers are assignable on a GitHub repo'''
    next_link_re = re.compile(r'<([^>]+)>; rel="next"')

    # Load the list of assignable people in the GitHub repo
    assignable = []
    url = 'https://api.github.com/repos/%s/collaborators?per_page=100' % repo
    while url is not None:
        response = urllib2.urlopen(urllib2.Request(url, headers={
            'Authorization': 'token ' + github_token,
            # Properly load nested teams.
            'Accept': 'application/vnd.github.hellcat-preview+json',
        }))
        assignable.extend(user['login'] for user in load_json_from_response(response))
        # Load the next page if available
        url = None
        link_header = response.headers.get('Link')
        if link_header:
            matches = next_link_re.match(link_header)
            if matches is not None:
                url = matches.group(1)

    errors = False
    for tool, maintainers in MAINTAINERS.items():
        for maintainer in maintainers:
            if maintainer not in assignable:
                errors = True
                print(
                    "error: %s maintainer @%s is not assignable in the %s repo"
                    % (tool, maintainer, repo),
                )

    if errors:
        print()
        print("  To be assignable, a person needs to be explicitly listed as a")
        print("  collaborator in the repository settings. The simple way to")
        print("  fix this is to ask someone with 'admin' privileges on the repo")
        print("  to add the person or whole team as a collaborator with 'read'")
        print("  privileges. Those privileges don't grant any extra permissions")
        print("  so it's safe to apply them.")
        print()
        print("The build will fail due to this.")
        exit(1)

def read_current_status(current_commit, path):
    '''Reads build status of `current_commit` from content of `history/*.tsv`
    '''
    with open(path, 'rU') as f:
        for line in f:
            (commit, status) = line.split('\t', 1)
            if commit == current_commit:
                return json.loads(status)
    return {}

def gh_url():
    return os.environ['TOOLSTATE_ISSUES_API_URL']

def maybe_delink(message):
    if os.environ.get('TOOLSTATE_SKIP_MENTIONS') is not None:
        return message.replace("@", "")
    return message

def issue(
    tool,
    status,
    assignees,
    relevant_pr_number,
    relevant_pr_user,
    labels,
):
    # Open an issue about the toolstate failure.
    if status == 'test-fail':
        status_description = 'has failing tests'
    else:
        status_description = 'no longer builds'
    request = json.dumps({
        'body': maybe_delink(textwrap.dedent('''\
        Hello, this is your friendly neighborhood mergebot.
        After merging PR {}, I observed that the tool {} {}.
        A follow-up PR to the repository {} is needed to fix the fallout.

        cc @{}, do you think you would have time to do the follow-up work?
        If so, that would be great!

        And nominating for compiler team prioritization.

        ''').format(
            relevant_pr_number, tool, status_description,
            REPOS.get(tool), relevant_pr_user
        )),
        'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number),
        'assignees': list(assignees),
        'labels': labels,
    })
    print("Creating issue:\n{}".format(request))
    response = urllib2.urlopen(urllib2.Request(
        gh_url(),
        request,
        {
            'Authorization': 'token ' + github_token,
            'Content-Type': 'application/json',
        }
    ))
    response.read()

def update_latest(
    current_commit,
    relevant_pr_number,
    relevant_pr_url,
    relevant_pr_user,
    pr_reviewer,
    current_datetime
):
    '''Updates `_data/latest.json` to match build result of the given commit.
    '''
    with open('_data/latest.json', 'r+') as f:
        latest = json.load(f, object_pairs_hook=collections.OrderedDict)

        current_status = {
            os: read_current_status(current_commit, 'history/' + os + '.tsv')
            for os in ['windows', 'linux']
        }

        slug = 'rust-lang/rust'
        message = textwrap.dedent('''\
            📣 Toolstate changed by {}!

            Tested on commit {}@{}.
            Direct link to PR: <{}>

        ''').format(relevant_pr_number, slug, current_commit, relevant_pr_url)
        anything_changed = False
        for status in latest:
            tool = status['tool']
            changed = False
            create_issue_for_status = None  # set to the status that caused the issue

            for os, s in current_status.items():
                old = status[os]
                new = s.get(tool, old)
                status[os] = new
                maintainers = ' '.join('@'+name for name in MAINTAINERS.get(tool, ()))
                # comparing the strings, but they are ordered appropriately:
                # "test-pass" > "test-fail" > "build-fail"
                if new > old:
                    # things got fixed or at least the status quo improved
                    changed = True
                    message += '🎉 {} on {}: {} → {} (cc {}).\n' \
                        .format(tool, os, old, new, maintainers)
                elif new < old:
                    # tests or builds are failing and were not failing before
                    changed = True
                    title = '💔 {} on {}: {} → {}' \
                        .format(tool, os, old, new)
                    message += '{} (cc {}).\n' \
                        .format(title, maintainers)
                    # See if we need to create an issue.
                    if tool == 'miri':
                        # Create issue if tests used to pass before. Don't open a *second*
                        # issue when we regress from "test-fail" to "build-fail".
                        if old == 'test-pass':
                            create_issue_for_status = new
                    else:
                        # Create issue if things no longer build.
                        # (No issue for mere test failures to avoid spurious issues.)
                        if new == 'build-fail':
                            create_issue_for_status = new

            if create_issue_for_status is not None:
                try:
                    issue(
                        tool, create_issue_for_status, MAINTAINERS.get(tool, ''),
                        relevant_pr_number, relevant_pr_user, LABELS.get(tool, ''),
                    )
                except urllib2.HTTPError as e:
                    # network errors will simply end up not creating an issue, but that's better
                    # than failing the entire build job
                    print("HTTPError when creating issue for status regression: {0}\n{1}"
                          .format(e, e.read()))
                except IOError as e:
                    print("I/O error when creating issue for status regression: {0}".format(e))
                except:
                    print("Unexpected error when creating issue for status regression: {0}"
                          .format(sys.exc_info()[0]))
                    raise

            if changed:
                status['commit'] = current_commit
                status['datetime'] = current_datetime
                anything_changed = True

        if not anything_changed:
            return ''

        f.seek(0)
        f.truncate(0)
        json.dump(latest, f, indent=4, separators=(',', ': '))
        return message

def main():
    repo = os.environ.get('TOOLSTATE_VALIDATE_MAINTAINERS_REPO')
    if repo:
        github_token = os.environ.get('TOOLSTATE_REPO_ACCESS_TOKEN')
        if github_token:
            validate_maintainers(repo, github_token)
        else:
            print('skipping toolstate maintainers validation since no GitHub token is present')
        # When validating maintainers don't run the full script.
        exit(0)

    cur_commit = sys.argv[1]
    cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
    cur_commit_msg = sys.argv[2]
    save_message_to_path = sys.argv[3]
    github_token = sys.argv[4]

    # assume that PR authors are also owners of the repo where the branch lives
    relevant_pr_match = re.search(
        r'Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)',
        cur_commit_msg,
    )
    if relevant_pr_match:
        number = relevant_pr_match.group(1)
        relevant_pr_user = relevant_pr_match.group(2)
        relevant_pr_number = 'rust-lang/rust#' + number
        relevant_pr_url = 'https://github.com/rust-lang/rust/pull/' + number
        pr_reviewer = relevant_pr_match.group(3)
    else:
        number = '-1'
        relevant_pr_user = 'ghost'
        relevant_pr_number = '<unknown PR>'
        relevant_pr_url = '<unknown>'
        pr_reviewer = 'ghost'

    message = update_latest(
        cur_commit,
        relevant_pr_number,
        relevant_pr_url,
        relevant_pr_user,
        pr_reviewer,
        cur_datetime
    )
    if not message:
        print('<Nothing changed>')
        sys.exit(0)

    print(message)

    if not github_token:
        print('Dry run only, not committing anything')
        sys.exit(0)

    with open(save_message_to_path, 'w') as f:
        f.write(message)

    # Write the toolstate comment on the PR as well.
    issue_url = gh_url() + '/{}/comments'.format(number)
    response = urllib2.urlopen(urllib2.Request(
        issue_url,
        json.dumps({'body': maybe_delink(message)}),
        {
            'Authorization': 'token ' + github_token,
            'Content-Type': 'application/json',
        }
    ))
    response.read()

if __name__ == '__main__':
    try:
        main()
    except urllib2.HTTPError as e:
        print("HTTPError: %s\n%s" % (e, e.read()))
        raise
<commit_msg>Rollup merge of #75958 - Mark-Simulacrum:fix-toolstate, r=kennytm<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# This script computes the new "current" toolstate for the toolstate repo (not to be
# confused with publishing the test results, which happens in `src/bootstrap/toolstate.rs`).
# It gets called from `src/ci/publish_toolstate.sh` when a new commit lands on `master`
# (i.e., after it passed all checks on `auto`).

from __future__ import print_function

import sys
import re
import os
import json
import datetime
import collections
import textwrap
try:
    import urllib2
except ImportError:
    import urllib.request as urllib2

# List of people to ping when the status of a tool or a book changed.
# These should be collaborators of the rust-lang/rust repository (with at least
# read privileges on it). CI will fail otherwise.
MAINTAINERS = {
    'miri': {'oli-obk', 'RalfJung', 'eddyb'},
    'rls': {'Xanewok'},
    'rustfmt': {'topecongiro', 'calebcartwright'},
    'book': {'carols10cents', 'steveklabnik'},
    'nomicon': {'frewsxcv', 'Gankra'},
    'reference': {'steveklabnik', 'Havvy', 'matthewjasper', 'ehuss'},
    'rust-by-example': {'steveklabnik', 'marioidival'},
    'embedded-book': {'adamgreig', 'andre-richter', 'jamesmunns', 'therealprof'},
    'edition-guide': {'ehuss', 'steveklabnik'},
    'rustc-dev-guide': {'mark-i-m', 'spastorino', 'amanjeev', 'JohnTitor'},
}

LABELS = {
    'miri': ['A-miri', 'C-bug'],
    'rls': ['A-rls', 'C-bug'],
    'rustfmt': ['A-rustfmt', 'C-bug'],
    'book': ['C-bug'],
    'nomicon': ['C-bug'],
    'reference': ['C-bug'],
    'rust-by-example': ['C-bug'],
    'embedded-book': ['C-bug'],
    'edition-guide': ['C-bug'],
    'rustc-dev-guide': ['C-bug'],
}

REPOS = {
    'miri': 'https://github.com/rust-lang/miri',
    'rls': 'https://github.com/rust-lang/rls',
    'rustfmt': 'https://github.com/rust-lang/rustfmt',
    'book': 'https://github.com/rust-lang/book',
    'nomicon': 'https://github.com/rust-lang/nomicon',
    'reference': 'https://github.com/rust-lang/reference',
    'rust-by-example': 'https://github.com/rust-lang/rust-by-example',
    'embedded-book': 'https://github.com/rust-embedded/book',
    'edition-guide': 'https://github.com/rust-lang/edition-guide',
    'rustc-dev-guide': 'https://github.com/rust-lang/rustc-dev-guide',
}

def load_json_from_response(resp):
    content = resp.read()
    if isinstance(content, bytes):
        content = content.decode('utf-8')
    else:
        print("Refusing to decode " + str(type(content)) + " to str")
    return json.loads(content)

def validate_maintainers(repo, github_token):
    '''Ensure all maintainers are assignable on a GitHub repo'''
    next_link_re = re.compile(r'<([^>]+)>; rel="next"')

    # Load the list of assignable people in the GitHub repo
    assignable = []
    url = 'https://api.github.com/repos/%s/collaborators?per_page=100' % repo
    while url is not None:
        response = urllib2.urlopen(urllib2.Request(url, headers={
            'Authorization': 'token ' + github_token,
            # Properly load nested teams.
            'Accept': 'application/vnd.github.hellcat-preview+json',
        }))
        assignable.extend(user['login'] for user in load_json_from_response(response))
        # Load the next page if available
        url = None
        link_header = response.headers.get('Link')
        if link_header:
            matches = next_link_re.match(link_header)
            if matches is not None:
                url = matches.group(1)

    errors = False
    for tool, maintainers in MAINTAINERS.items():
        for maintainer in maintainers:
            if maintainer not in assignable:
                errors = True
                print(
                    "error: %s maintainer @%s is not assignable in the %s repo"
                    % (tool, maintainer, repo),
                )

    if errors:
        print()
        print("  To be assignable, a person needs to be explicitly listed as a")
        print("  collaborator in the repository settings. The simple way to")
        print("  fix this is to ask someone with 'admin' privileges on the repo")
        print("  to add the person or whole team as a collaborator with 'read'")
        print("  privileges. Those privileges don't grant any extra permissions")
        print("  so it's safe to apply them.")
        print()
        print("The build will fail due to this.")
        exit(1)

def read_current_status(current_commit, path):
    '''Reads build status of `current_commit` from content of `history/*.tsv`
    '''
    with open(path, 'rU') as f:
        for line in f:
            (commit, status) = line.split('\t', 1)
            if commit == current_commit:
                return json.loads(status)
    return {}

def gh_url():
    return os.environ['TOOLSTATE_ISSUES_API_URL']

def maybe_delink(message):
    if os.environ.get('TOOLSTATE_SKIP_MENTIONS') is not None:
        return message.replace("@", "")
    return message

def issue(
    tool,
    status,
    assignees,
    relevant_pr_number,
    relevant_pr_user,
    labels,
):
    # Open an issue about the toolstate failure.
    if status == 'test-fail':
        status_description = 'has failing tests'
    else:
        status_description = 'no longer builds'
    request = json.dumps({
        'body': maybe_delink(textwrap.dedent('''\
        Hello, this is your friendly neighborhood mergebot.
        After merging PR {}, I observed that the tool {} {}.
        A follow-up PR to the repository {} is needed to fix the fallout.

        cc @{}, do you think you would have time to do the follow-up work?
        If so, that would be great!

        And nominating for compiler team prioritization.

        ''').format(
            relevant_pr_number, tool, status_description,
            REPOS.get(tool), relevant_pr_user
        )),
        'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number),
        'assignees': list(assignees),
        'labels': labels,
    })
    print("Creating issue:\n{}".format(request))
    response = urllib2.urlopen(urllib2.Request(
        gh_url(),
        request,
        {
            'Authorization': 'token ' + github_token,
            'Content-Type': 'application/json',
        }
    ))
    response.read()

def update_latest(
    current_commit,
    relevant_pr_number,
    relevant_pr_url,
    relevant_pr_user,
    pr_reviewer,
    current_datetime
):
    '''Updates `_data/latest.json` to match build result of the given commit.
    '''
    with open('_data/latest.json', 'r+') as f:
        latest = json.load(f, object_pairs_hook=collections.OrderedDict)

        current_status = {
            os: read_current_status(current_commit, 'history/' + os + '.tsv')
            for os in ['windows', 'linux']
        }

        slug = 'rust-lang/rust'
        message = textwrap.dedent('''\
            📣 Toolstate changed by {}!

            Tested on commit {}@{}.
            Direct link to PR: <{}>

        ''').format(relevant_pr_number, slug, current_commit, relevant_pr_url)
        anything_changed = False
        for status in latest:
            tool = status['tool']
            changed = False
            create_issue_for_status = None  # set to the status that caused the issue

            for os, s in current_status.items():
                old = status[os]
                new = s.get(tool, old)
                status[os] = new
                maintainers = ' '.join('@'+name for name in MAINTAINERS.get(tool, ()))
                # comparing the strings, but they are ordered appropriately:
                # "test-pass" > "test-fail" > "build-fail"
                if new > old:
                    # things got fixed or at least the status quo improved
                    changed = True
                    message += '🎉 {} on {}: {} → {} (cc {}).\n' \
                        .format(tool, os, old, new, maintainers)
                elif new < old:
                    # tests or builds are failing and were not failing before
                    changed = True
                    title = '💔 {} on {}: {} → {}' \
                        .format(tool, os, old, new)
                    message += '{} (cc {}).\n' \
                        .format(title, maintainers)
                    # See if we need to create an issue.
                    if tool == 'miri':
                        # Create issue if tests used to pass before. Don't open a *second*
                        # issue when we regress from "test-fail" to "build-fail".
                        if old == 'test-pass':
                            create_issue_for_status = new
                    else:
                        # Create issue if things no longer build.
                        # (No issue for mere test failures to avoid spurious issues.)
                        if new == 'build-fail':
                            create_issue_for_status = new

            if create_issue_for_status is not None:
                try:
                    issue(
                        tool, create_issue_for_status, MAINTAINERS.get(tool, ''),
                        relevant_pr_number, relevant_pr_user, LABELS.get(tool, ''),
                    )
                except urllib2.HTTPError as e:
                    # network errors will simply end up not creating an issue, but that's better
                    # than failing the entire build job
                    print("HTTPError when creating issue for status regression: {0}\n{1}"
                          .format(e, e.read()))
                except IOError as e:
                    print("I/O error when creating issue for status regression: {0}".format(e))
                except:
                    print("Unexpected error when creating issue for status regression: {0}"
                          .format(sys.exc_info()[0]))
                    raise

            if changed:
                status['commit'] = current_commit
                status['datetime'] = current_datetime
                anything_changed = True

        if not anything_changed:
            return ''

        f.seek(0)
        f.truncate(0)
        json.dump(latest, f, indent=4, separators=(',', ': '))
        return message

# Warning: Do not try to add a function containing the body of this try block.
# There are variables declared within that are implicitly global; it is unknown
# which ones precisely but at least this is true for `github_token`.
try:
    if __name__ != '__main__':
        exit(0)
    repo = os.environ.get('TOOLSTATE_VALIDATE_MAINTAINERS_REPO')
    if repo:
        github_token = os.environ.get('TOOLSTATE_REPO_ACCESS_TOKEN')
        if github_token:
            validate_maintainers(repo, github_token)
        else:
            print('skipping toolstate maintainers validation since no GitHub token is present')
        # When validating maintainers don't run the full script.
        exit(0)

    cur_commit = sys.argv[1]
    cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
    cur_commit_msg = sys.argv[2]
    save_message_to_path = sys.argv[3]
    github_token = sys.argv[4]

    # assume that PR authors are also owners of the repo where the branch lives
    relevant_pr_match = re.search(
        r'Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)',
        cur_commit_msg,
    )
    if relevant_pr_match:
        number = relevant_pr_match.group(1)
        relevant_pr_user = relevant_pr_match.group(2)
        relevant_pr_number = 'rust-lang/rust#' + number
        relevant_pr_url = 'https://github.com/rust-lang/rust/pull/' + number
        pr_reviewer = relevant_pr_match.group(3)
    else:
        number = '-1'
        relevant_pr_user = 'ghost'
        relevant_pr_number = '<unknown PR>'
        relevant_pr_url = '<unknown>'
        pr_reviewer = 'ghost'

    message = update_latest(
        cur_commit,
        relevant_pr_number,
        relevant_pr_url,
        relevant_pr_user,
        pr_reviewer,
        cur_datetime
    )
    if not message:
        print('<Nothing changed>')
        sys.exit(0)

    print(message)

    if not github_token:
        print('Dry run only, not committing anything')
        sys.exit(0)

    with open(save_message_to_path, 'w') as f:
        f.write(message)

    # Write the toolstate comment on the PR as well.
    issue_url = gh_url() + '/{}/comments'.format(number)
    response = urllib2.urlopen(urllib2.Request(
        issue_url,
        json.dumps({'body': maybe_delink(message)}),
        {
            'Authorization': 'token ' + github_token,
            'Content-Type': 'application/json',
        }
    ))
    response.read()
except urllib2.HTTPError as e:
    print("HTTPError: %s\n%s" % (e, e.read()))
    raise
<|endoftext|>


<commit_before>import argparse
from nltk.classify import MaxentClassifier
from nltk.metrics import BigramAssocMeasures

parser = argparse.ArgumentParser(description='Train a NLTK Classifier')

parser.add_argument('corpus',
	help='corpus name/path relative to nltk_data directory')
parser.add_argument('--filename', help='''filename/path for where to store the
	pickled classifier. the default is {corpus}_{algorithm}.pickle''')
parser.add_argument('--algorithm', default='naivebayes',
	choices=['NaiveBayes', 'DecisionTree', 'Maxent'] + MaxentClassifier.ALGORITHMS,
	help='training algorithm to use. maxent used the default maxent training algorithm, either CG or iis')
parser.add_argument('--trace', default=0, type=int,
	help='how much trace output you want')

corpus_group = parser.add_argument_group('Training Corpus')
corpus_group.add_argument('--reader', choices=('plaintext', 'tagged'),
	default='plaintext',
	help='specify categorized plaintext or part-of-speech tagged corpus')
corpus_group.add_argument('--cat_pattern', default='(.+)/.+',
	help='regular expression pattern to identify categories based on file paths')
corpus_group.add_argument('--cat_file',
	help='relative path to a file containing category listings')
corpus_group.add_argument('--delimiter', default=' ',
	help='category delimiter for category file')
corpus_group.add_argument('--instances', default='files',
	choices=('sents', 'paras', 'files'),
	help='which groups of words represent a single training instance')
corpus_group.add_argument('--fraction', default=1.0, type=float,
	help='''what fraction of the corpus to use for training. the rest will be used for evaulation.
			the default is to use all of it, and to test the classifier against the training data.
			any other number < 1 will test against the remaining portion.''')

classifier_group = parser.add_argument_group('Classifier Type',
	'''A binary classifier has only 2 labels, and is the default classifier.
	A multi-class classifier chooses one of many possible labels.
	A multi-binary classifier choose zero or more labels by combining multiple
	binary classifiers, 1 for each label.''')
classifier_group.add_argument('--binary', action='store_true', default=False,
	help='train a binary classifier, or a multi-binary classifier if --multi is also given')
classifier_group.add_argument('--multi', action='store_true', default=False,
	help='train a multi-class classifier, or a multi-binary classifier if --binary is also given')

score_group = parser.add_argument_group('Feature Scoring')
score_group.add_argument('--score_fn', default='chi_sq',
	choices=[f for f in dir(BigramAssocMeasures) if not f.startswith('_')],
	help='scoring function for information gain and bigram collocations')
score_group.add_argument('--min_score', default=0, type=int,
	help='minimum score for a word to be included. if 0, all words are used.')
score_group.add_argument('--max-feats', default=0, type=int,
	help='maximum number of words to include, ordered by highest score. if 0, all words are used.')

maxent_group = parser.add_argument_group('Maxent Classifier',
	'These options only apply when a Maxent algorithm is chosen.')
maxent_group.add_argument('--max_iter', default=10, type=int,
	help='maximum number of training iterations')
maxent_group.add_argument('--min_ll', default=0, type=float,
	help='stop classification when average log-likelihood is less than this')
maxent_group.add_argument('--min_lldelta', default=0.1, type=float,
	help='stop classification when the change in average log-likelihood is less than this')

decisiontree_group = parser.add_argument_group('Decision Tree Classifier',
	'These options only apply when the DecisionTree algorithm is chosen')
decisiontree_group.add_argument('--entropy_cutoff', default=0.05, type=float)
decisiontree_group.add_argument('--depth_cutoff', default=100, type=int)
decisiontree_group.add_argument('--support_cutoff', default=10, type=int)

args = parser.parse_args()<commit_msg>corpus reader creation, instance extraction, feat & eval options<commit_after>import argparse, itertools, re
from nltk.classify import MaxentClassifier
from nltk.corpus import stopwords
from nltk.corpus.reader import CategorizedPlaintextCorpusReader, CategorizedTaggedCorpusReader
from nltk.corpus.util import LazyCorpusLoader
from nltk.metrics import BigramAssocMeasures
from nltk.util import bigrams

########################################
## command options & argument parsing ##
########################################

parser = argparse.ArgumentParser(description='Train a NLTK Classifier')

parser.add_argument('corpus',
	help='corpus name/path relative to nltk_data directory')
parser.add_argument('--filename', help='''filename/path for where to store the
	pickled classifier. the default is {corpus}_{algorithm}.pickle''')
parser.add_argument('--algorithm', default='naivebayes',
	choices=['NaiveBayes', 'DecisionTree', 'Maxent'] + MaxentClassifier.ALGORITHMS,
	help='training algorithm to use. maxent used the default maxent training algorithm, either CG or iis')
parser.add_argument('--trace', default=0, type=int,
	help='how much trace output you want')

corpus_group = parser.add_argument_group('Training Corpus')
corpus_group.add_argument('--reader', choices=('plaintext', 'tagged'),
	default='plaintext',
	help='specify categorized plaintext or part-of-speech tagged corpus')
corpus_group.add_argument('--cat_pattern', default='(.+)/.+',
	help='''regular expression pattern to identify categories based on file paths.
	if cat_file is also given, this pattern is used to identify corpus file ids.''')
corpus_group.add_argument('--cat_file',
	help='relative path to a file containing category listings')
corpus_group.add_argument('--delimiter', default=' ',
	help='category delimiter for category file')
corpus_group.add_argument('--instances', default='files',
	choices=('sents', 'paras', 'files'),
	help='which groups of words represent a single training instance')
corpus_group.add_argument('--fraction', default=1.0, type=float,
	help='''what fraction of the corpus to use for training. the rest will be used for evaulation.
			the default is to use all of it, and to test the classifier against the training data.
			any other number < 1 will test against the remaining portion.''')

classifier_group = parser.add_argument_group('Classifier Type',
	'''A binary classifier has only 2 labels, and is the default classifier.
	A multi-class classifier chooses one of many possible labels.
	A multi-binary classifier choose zero or more labels by combining multiple
	binary classifiers, 1 for each label.''')
classifier_group.add_argument('--binary', action='store_true', default=False,
	help='train a binary classifier, or a multi-binary classifier if --multi is also given')
classifier_group.add_argument('--multi', action='store_true', default=False,
	help='train a multi-class classifier, or a multi-binary classifier if --binary is also given')

feat_group = parser.add_argument_group('Feature Extraction',
	'The default is to lowercase every word and filter out stopwords')
feat_group.add_argument('--bigrams', action='store_true', default=False,
	help='include bigrams as features')
feat_group.add_argument('--no-lowercase', action='store_true', default=False,
	help="don't lowercase every word")
feat_group.add_argument('--filter-stopwords', default='english',
	choices=['no']+stopwords.fileids(),
	help='stopwords to filter, or "no" if want to keep stopwords')

score_group = parser.add_argument_group('Feature Scoring',
	'The default is no scoring, all words are included as features')
score_group.add_argument('--score_fn', default='chi_sq',
	choices=[f for f in dir(BigramAssocMeasures) if not f.startswith('_')],
	help='scoring function for information gain and bigram collocations')
score_group.add_argument('--min_score', default=0, type=int,
	help='minimum score for a word to be included. if 0, all words are used.')
score_group.add_argument('--max-feats', default=0, type=int,
	help='maximum number of words to include, ordered by highest score. if 0, all words are used.')

eval_group = parser.add_argument_group('Classifier Evaluation',
	'''The default is to test the classifier against the unused fraction of the
	corpus, or against the entire corpus if the whole corpus is used for training.''')
eval_group.add_argument('--no-eval', action='store_true', default=False,
	help="don't do any evaluation")
eval_group.add_argument('--no-accuracy', action='store_true', default=False,
	help="don't evaluate accuracy")
eval_group.add_argument('--no-precision', action='store_true', default=False,
	help="don't evaluate precision")
eval_group.add_argument('--no-recall', action='store_true', default=False,
	help="don't evaluate recall")
eval_group.add_argument('--no-fmeasure', action='store_true', default=False,
	help="don't evaluate f-measure")
eval_group.add_argument('--no-masi-distance', action='store_true', default=False,
	help="don't evaluate masi distance (only applies to a multi binary classifier)")

maxent_group = parser.add_argument_group('Maxent Classifier',
	'These options only apply when a Maxent algorithm is chosen.')
maxent_group.add_argument('--max_iter', default=10, type=int,
	help='maximum number of training iterations')
maxent_group.add_argument('--min_ll', default=0, type=float,
	help='stop classification when average log-likelihood is less than this')
maxent_group.add_argument('--min_lldelta', default=0.1, type=float,
	help='stop classification when the change in average log-likelihood is less than this')

decisiontree_group = parser.add_argument_group('Decision Tree Classifier',
	'These options only apply when the DecisionTree algorithm is chosen')
decisiontree_group.add_argument('--entropy_cutoff', default=0.05, type=float)
decisiontree_group.add_argument('--depth_cutoff', default=100, type=int)
decisiontree_group.add_argument('--support_cutoff', default=10, type=int)

args = parser.parse_args()

###################
## corpus reader ##
###################

reader_class = {
	'plaintext': CategorizedPlaintextCorpusReader,
	'tagged': CategorizedTaggedCorpusReader
}

reader_args = []
reader_kwargs = {}

if args.cat_file:
	reader_kwargs['cat_file'] = args.cat_file

	if args.delimiter and args.delimiter != ' ':
		reader_kwargs['delimiter'] = args.delimiter

	if args.cat_pattern:
		reader_args.append(args.cat_pattern)
	else:
		reader_args.append('.+/.+')
elif args.cat_pattern:
	reader_args.append(args.cat_pattern)
	reader_kwargs['cat_pattern'] = re.compile(args.cat_pattern)

categorized_corpus = LazyCorpusLoader(args.corpus, reader_class[args.reader],
	*reader_args, **reader_kwargs)
labels = categorized_corpus.categories()
nlabels = len(labels)

if args.trace:
	print '%d labels: %s' % (nlabels, labels)

if not nlabels:
	raise ValueError('corpus does not have any categories')
elif nlabels == 1:
	raise ValueError('corpus must have more than 1 category')
elif nlabels == 2 and args.multi:
	raise ValueError('corpus must have more than 2 categories if --multi is specified')

#####################
## text extraction ##
#####################

if args.filter_stopwords == 'no':
	stopset = set()
else:
	stopset = set(stopwords.words(args.filter_stopwords))

def norm_sent(words):
	if not args.no_lowercase:
		words = [w.lower() for w in words]

	if stopset:
		words = [w for w in words if w.lower() not in stopset]

	if args.bigrams:
		return words + bigrams(words)
	else:
		return words

def label_sents(label):
	for sent in categorized_corpus.sents(categories=[label]):
		yield norm_sent(sent)

def label_paras(label):
	for para in categorized_corpus.paras(categories=[label]):
		yield itertools.chain([norm_sent(sent) for sent in para])

def label_files(label):
	for fileid in categorized_corpus.fileids(categories=[label]):
		sents = categorized_corpus.sents(fileids=[fileid])
		yield itertools.chain([norm_sent(sent) for sent in sents])

label_instance_function = {
	'sents': label_sents,
	'paras': label_paras,
	'files': label_files
}

lif = label_instance_function[args.instances]
label_words = {}

for label in labels:
	label_words[label] = lif(label)<|endoftext|>


<commit_before># -*- coding:utf-8 -*-

"""
Select a specific simulation from a HDF5 file and plot its granule figure.

"""

import numpy as np
import matplotlib.pyplot as plt
import tables

import h5manager as h5m
from plotting import granule_pop_figure
from analysis import fftmax

DB_FILENAME = "db_two_glom_beta_new_ps_interco_strength0_1_interco_rate0_1.h5"
DB = tables.openFile(DB_FILENAME)

# Get all simulation data
ATTRS = (('paramset', '_v_attrs', 'Common', 'inter_conn_rate', 0, 1),
         ('paramset', '_v_attrs', 'Common', 'inter_conn_strength', 0, 1),
         ('paramset', 'arrays', 'times'),
         ('paramset', '_v_attrs', 'Common', 'simu_dt'),
         ('results', 's_granule'),
         ('results', 's_syn_self'),
         ('results', '_v_attrs'))
ALL_SIMU_ATTRS = h5m.get_all_attrs(DB, ATTRS)

# Some useful functions
def deq(a, b, delta):
    """Delta equality"""
    return abs(a - b) < delta

class DummyPkg:
    def __init__(self, values, times):
        self.values = values
        self.times = times

# Get the specific simulations
def get_interco(simu, interco_rate, interco_strength):
    return deq(simu[0], interco_rate, 0.02) and deq(simu[1], interco_strength, 0.02)

GOOD_SIMUS = filter(lambda x : get_interco(x, 0.793, 0.918), ALL_SIMU_ATTRS)

REDO_FFTMAX = []
for simu in GOOD_SIMUS:
    gr_s = simu[4].read()
    gr_s_syn_self = simu[5].read()
    times = simu[2].read()
    dt = float(simu[3])
    granule_pop_figure(gr_s, gr_s_syn_self, times, dt)

    dummy = DummyPkg(gr_s, times)
    REDO_FFTMAX.append(fftmax(dummy, 2, dt))

print REDO_FFTMAX
plt.show()

# Finally close the DB
DB.close()
<commit_msg>Add better simulation selection<commit_after># -*- coding:utf-8 -*-

"""
Select a specific simulation from a HDF5 file and plot its granule figure.

"""

import numpy as np
import matplotlib.pyplot as plt
import tables

import h5manager as h5m
from plotting import granule_pop_figure
from analysis import fftmax

DB_FILENAME = "data/db30x30_two_glom_beta_new_ps_interco_strength0_1_interco_rate0_1.h5"
DB = tables.openFile(DB_FILENAME)

# Get all simulation data
ATTRS = (('paramset', '_v_attrs', 'Common', 'inter_conn_rate', 0, 1),
         ('paramset', '_v_attrs', 'Common', 'inter_conn_strength', 0, 1),
         ('paramset', 'arrays', 'times'),
         ('paramset', '_v_attrs', 'Common', 'simu_dt'),
         ('results', 's_granule'),
         ('results', 's_syn_self'),
         ('results', '_v_attrs'))
ALL_SIMU_ATTRS = h5m.get_all_attrs(DB, ATTRS)

SET_INTERCO_RATE = list(set(s[0] for s in ALL_SIMU_ATTRS))
SET_INTERCO_RATE.sort()
SET_INTERCO_STRENGTH = list(set(s[1] for s in ALL_SIMU_ATTRS))
SET_INTERCO_STRENGTH.sort()

# Some useful functions
def deq(a, b, delta):
    """Delta equality"""
    return abs(a - b) < delta

class DummyPkg:
    def __init__(self, values, times):
        self.values = values
        self.times = times

# Get the specific simulations
def get_interco(simu, interco_rate, interco_strength):
    return deq(simu[0], interco_rate, 0.02) and deq(simu[1], interco_strength, 0.02)

INTERCO_RATE = SET_INTERCO_RATE[25]
INTERCO_STRENGTH = SET_INTERCO_STRENGTH[4]
GOOD_SIMUS = filter(lambda x : get_interco(x, INTERCO_RATE, INTERCO_STRENGTH), ALL_SIMU_ATTRS)

REDO_FFTMAX = []
for simu in GOOD_SIMUS:
    gr_s = simu[4].read()
    gr_s_syn_self = simu[5].read()
    times = simu[2].read()
    dt = float(simu[3])
    granule_pop_figure(gr_s, gr_s_syn_self, times, dt)

    dummy = DummyPkg(gr_s, times)
    REDO_FFTMAX.append(fftmax(dummy, 2, dt))

print REDO_FFTMAX
plt.show()

# Finally close the DB
DB.close()
<|endoftext|>


<commit_before>from skbeam.core.accumulators.binned_statistic import (RadialBinnedStatistic,
                                                       RPhiBinnedStatistic,
                                                       BinnedStatistic1D)
from nose.tools import assert_raises
from numpy.testing import assert_array_equal, assert_array_almost_equal
import numpy as np
import scipy.stats
from ...utils import bin_edges_to_centers

stats_list = [('mean', np.mean), ('median', np.median), ('count', len),
              ('sum', np.sum), ('std', np.std)]

class TestRadialBinnedStatistic(object):
    oscillation_rate = 10.0

    def setup(self):

        # Create test image - a sinc function.
        # Integrating in phi will produce sin(x)

        rowsize, colsize = (90, 102)

        rowarr = np.arange(rowsize)
        colarr = np.arange(colsize)
        rowgrid, colgrid = np.meshgrid(rowarr, colarr, indexing='ij')
        self.rgrid = np.sqrt(rowgrid**2 + colgrid**2)
        self.phigrid = np.arctan2(rowgrid, colgrid)

        self.image = np.sinc(self.rgrid / self.oscillation_rate)

    def testRadialBinnedStatistic(self):

        mykwargs = [{'origin': (0, 0),
                     'range': (10, 90)},
                    {'origin': (0, 0)}]
        bins, shape = 100, self.image.shape
        mask_ones = np.ones_like(self.image)
        mask_random = np.random.randint(2, size=self.image.shape)

        for kwargs in mykwargs:
            for stat, stat_func in stats_list:

                if stat is 'sum':
                    # in this case we can compare our masked
                    # result to binned_statistic
                    mask = mask_random
                else:
                    mask = mask_ones

                # test radial case
                radbinstat = RadialBinnedStatistic(shape, bins,
                                                   statistic=stat,
                                                   mask=mask,
                                                   **kwargs)
                radbinstat_f = RadialBinnedStatistic(shape, bins,
                                                     statistic=stat_func,
                                                     mask=mask,
                                                     **kwargs)
                binned = radbinstat(self.image)
                binned_f = radbinstat_f(self.image)

                assert_array_almost_equal(binned_f, binned)

                kwrange = kwargs.get('range', None)
                ref, edges, _ = scipy.stats.binned_statistic(
                    x=self.rgrid.ravel(),
                    values=(self.image*mask).ravel(),
                    statistic=stat,
                    range=kwrange,
                    bins=bins,
                )
                centers = bin_edges_to_centers(edges)

                assert_array_equal(ref, binned)
                assert_array_equal(edges, radbinstat.bin_edges[0])
                assert_array_equal(edges, radbinstat_f.bin_edges[0])
                assert_array_equal(centers, radbinstat.bin_centers[0])
                assert_array_equal(centers, radbinstat_f.bin_centers[0])

        bins = (100, 2)
        myrphikwargs = [{'origin': (0, 0),
                         'range': ((10, 90), (0, np.pi/2))},
                        {'origin': (0, 0)}]
        for kwargs in myrphikwargs:
            for stat, stat_func in stats_list:

                if stat is 'sum':
                    # in this case we can compare our masked
                    # result to binned_statistic
                    mask = mask_random
                else:
                    mask = mask_ones

                # test radial case
                rphibinstat = RPhiBinnedStatistic(shape, bins,
                                                  statistic=stat,
                                                  mask=mask,
                                                  **kwargs)
                rphibinstat_f = RPhiBinnedStatistic(shape, bins,
                                                    statistic=stat_func,
                                                    mask=mask,
                                                    **kwargs)
                binned = rphibinstat(self.image)
                binned_f = rphibinstat_f(self.image)

                # this test fails only for the standard deviation where
                # there is a disagreement in the number of nan's.  I
                # don't believe this is the fault of the binned_statistic
                # code
                if stat != 'std':
                    assert_array_almost_equal(binned_f, binned)

                kwrange = kwargs.get('range', None)
                ref, redges, phiedges, _ = scipy.stats.binned_statistic_2d(
                    x=self.rgrid.ravel(),
                    y=self.phigrid.ravel(),
                    values=(self.image*mask).ravel(),
                    statistic=stat,
                    range=kwrange,
                    bins=bins,
                )

                assert_array_equal(ref, binned)
                assert_array_equal(redges, rphibinstat.bin_edges[0])
                assert_array_equal(redges, rphibinstat_f.bin_edges[0])
                assert_array_equal(phiedges, rphibinstat.bin_edges[1])
                assert_array_equal(phiedges, rphibinstat_f.bin_edges[1])

        # test exception when BinnedStatistic is given array of incorrect shape
        with assert_raises(ValueError):
            radbinstat(self.image[:10, :10])

        # test exception when RadialBinnedStatistic is given 1D array
        with assert_raises(ValueError):
            RadialBinnedStatistic(self.image.shape, 10,
                                  mask=np.array([1, 2, 3, 4]))

def test_BinnedStatistics1D():
    x = np.linspace(0, 2*np.pi, 100)
    values = np.sin(x * 5)

    for stat, stat_f in stats_list:
        bs = BinnedStatistic1D(x, statistic=stat, bins=10)
        bs_f = BinnedStatistic1D(x, statistic=stat_f, bins=10)

        ref, edges, _ = scipy.stats.binned_statistic(x, values,
                                                     statistic=stat, bins=10)

        assert_array_equal(bs(values), ref)
        assert_array_almost_equal(bs_f(values), ref)
        assert_array_equal(edges, bs.bin_edges[0])
        assert_array_equal(edges, bs_f.bin_edges[0])
<commit_msg>added origin to test, made one dimension odd for test<commit_after>from skbeam.core.accumulators.binned_statistic import (RadialBinnedStatistic,
                                                       RPhiBinnedStatistic,
                                                       BinnedStatistic1D)
from nose.tools import assert_raises
from numpy.testing import assert_array_equal, assert_array_almost_equal
import numpy as np
import scipy.stats
from ...utils import bin_edges_to_centers

stats_list = [('mean', np.mean), ('median', np.median), ('count', len),
              ('sum', np.sum), ('std', np.std)]

class TestRadialBinnedStatistic(object):
    oscillation_rate = 10.0

    def setup(self):

        # Create test image - a sinc function.
        # Integrating in phi will produce sin(x)

        self.rowsize, self.colsize = (91, 102)
        self.shape = self.rowsize, self.colsize

        rowarr = np.arange(self.rowsize)
        colarr = np.arange(self.colsize)
        self.rowgrid, self.colgrid = np.meshgrid(rowarr, colarr, indexing='ij')

    def testRadialBinnedStatistic(self):

        mykwargs = [{'origin': (0, 0),
                     'range': (10, 90)},
                    {'origin': (0, 0)},
                    {'origin': None}
                    ]
        bins, shape = 100, self.shape
        mask_ones = np.ones(self.shape)
        mask_random = np.random.randint(2, size=self.shape)

        for kwargs in mykwargs:
            for stat, stat_func in stats_list:

                if 'origin' in kwargs:
                    origin = kwargs['origin']
                else:
                    origin = None
                if origin is None:
                    origin = self.rowsize/2., self.colsize/2.

                # need to calculate these every time in loop since origin
                # changes
                # rows are y, cols are x, as in angle_grid in core.utils
                self.rgrid = np.sqrt((self.rowgrid-origin[0])**2 +
                                     (self.colgrid-origin[1])**2)
                self.phigrid = np.arctan2(self.rowgrid-origin[0],
                                          self.colgrid-origin[1])

                self.image = np.sinc(self.rgrid / self.oscillation_rate)

                if stat is 'sum':
                    # in this case we can compare our masked
                    # result to binned_statistic
                    mask = mask_random
                else:
                    mask = mask_ones

                # test radial case
                radbinstat = RadialBinnedStatistic(shape, bins,
                                                   statistic=stat,
                                                   mask=mask,
                                                   **kwargs)
                radbinstat_f = RadialBinnedStatistic(shape, bins,
                                                     statistic=stat_func,
                                                     mask=mask,
                                                     **kwargs)
                binned = radbinstat(self.image)
                binned_f = radbinstat_f(self.image)

                assert_array_almost_equal(binned_f, binned)

                kwrange = kwargs.get('range', None)
                ref, edges, _ = scipy.stats.binned_statistic(
                    x=self.rgrid.ravel(),
                    values=(self.image*mask).ravel(),
                    statistic=stat,
                    range=kwrange,
                    bins=bins,
                )
                centers = bin_edges_to_centers(edges)

                assert_array_equal(ref, binned)
                assert_array_equal(edges, radbinstat.bin_edges[0])
                assert_array_equal(edges, radbinstat_f.bin_edges[0])
                assert_array_equal(centers, radbinstat.bin_centers[0])
                assert_array_equal(centers, radbinstat_f.bin_centers[0])

        bins = (100, 2)
        myrphikwargs = [{'origin': (0, 0),
                         'range': ((10, 90), (0, np.pi/2))},
                        {'origin': (0, 0)},
                        {'origin': None}]
        for kwargs in myrphikwargs:
            for stat, stat_func in stats_list:
                if 'origin' in kwargs:
                    origin = kwargs['origin']
                else:
                    origin = None
                if origin is None:
                    origin = self.rowsize/2., self.colsize/2.

                # need to calculate these every time in loop since origin
                # changes
                # rows are y, cols are x, as in angle_grid in core.utils
                self.rgrid = np.sqrt((self.rowgrid-origin[0])**2 +
                                     (self.colgrid-origin[1])**2)
                self.phigrid = np.arctan2(self.rowgrid-origin[0],
                                          self.colgrid-origin[1])

                self.image = np.sinc(self.rgrid / self.oscillation_rate)

                if stat is 'sum':
                    # in this case we can compare our masked
                    # result to binned_statistic
                    mask = mask_random
                else:
                    mask = mask_ones

                # test radial case
                rphibinstat = RPhiBinnedStatistic(shape, bins,
                                                  statistic=stat,
                                                  mask=mask,
                                                  **kwargs)
                rphibinstat_f = RPhiBinnedStatistic(shape, bins,
                                                    statistic=stat_func,
                                                    mask=mask,
                                                    **kwargs)
                binned = rphibinstat(self.image)
                binned_f = rphibinstat_f(self.image)

                # this test fails only for the standard deviation where
                # there is a disagreement in the number of nan's.  I
                # don't believe this is the fault of the binned_statistic
                # code
                if stat != 'std':
                    assert_array_almost_equal(binned_f, binned)

                kwrange = kwargs.get('range', None)
                ref, redges, phiedges, _ = scipy.stats.binned_statistic_2d(
                    x=self.rgrid.ravel(),
                    y=self.phigrid.ravel(),
                    values=(self.image*mask).ravel(),
                    statistic=stat,
                    range=kwrange,
                    bins=bins,
                )

                assert_array_equal(ref, binned)
                assert_array_equal(redges, rphibinstat.bin_edges[0])
                assert_array_equal(redges, rphibinstat_f.bin_edges[0])
                assert_array_equal(phiedges, rphibinstat.bin_edges[1])
                assert_array_equal(phiedges, rphibinstat_f.bin_edges[1])

        # test exception when BinnedStatistic is given array of incorrect shape
        with assert_raises(ValueError):
            radbinstat(self.image[:10, :10])

        # test exception when RadialBinnedStatistic is given 1D array
        with assert_raises(ValueError):
            RadialBinnedStatistic(self.image.shape, 10,
                                  mask=np.array([1, 2, 3, 4]))

def test_BinnedStatistics1D():
    x = np.linspace(0, 2*np.pi, 100)
    values = np.sin(x * 5)

    for stat, stat_f in stats_list:
        bs = BinnedStatistic1D(x, statistic=stat, bins=10)
        bs_f = BinnedStatistic1D(x, statistic=stat_f, bins=10)

        ref, edges, _ = scipy.stats.binned_statistic(x, values,
                                                     statistic=stat, bins=10)

        assert_array_equal(bs(values), ref)
        assert_array_almost_equal(bs_f(values), ref)
        assert_array_equal(edges, bs.bin_edges[0])
        assert_array_equal(edges, bs_f.bin_edges[0])
<|endoftext|>


<commit_before>from datetime import datetime, date
from dateutil.parser import parse

import sys
import os
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../lib'))
import pytz

import tzlocal

UTC_TIMEZONE_COUNT = 0
def utc_timezone():
    ''' This is a mock function, so when we run tests
        we trick the system into thinking we're on UTC
        no matter what the real timezone is
    '''
    global UTC_TIMEZONE_COUNT
    UTC_TIMEZONE_COUNT += 1
    return pytz.timezone('UTC')

tzlocal.get_localzone = utc_timezone

from utilities.toUTC import toUTC

class TestToUTC():

    def result_is_datetime(self, result):
        assert type(result) == datetime

    def test_timezone_function_call_count(self):
        toUTC("2016-07-11 14:33:31.625443")
        toUTC("2016-07-12 14:33:31.625444")
        toUTC("2016-07-13 14:33:31.625445")
        toUTC("2016-07-14 14:33:31.625446")
        assert UTC_TIMEZONE_COUNT == 1

    def test_normal_date_str_with_default_timezone(self):
        result = toUTC("2016-07-13 14:33:31.625443")
        self.result_is_datetime(result)
        assert str(result) == '2016-07-13 14:33:31.625443+00:00'

    def test_normal_date_str_with_utc_timezone(self):
        result = toUTC("2016-07-13 22:33:31.625443+00:00")
        self.result_is_datetime(result)
        assert str(result) == '2016-07-13 22:33:31.625443+00:00'

    def test_normal_date_str_with_timezone(self):
        result = toUTC("2016-07-13 14:33:31.625443-08:00")
        self.result_is_datetime(result)
        assert str(result) == '2016-07-13 22:33:31.625443+00:00'

    def test_abnormal_date_str_without_timezone(self):
        result = toUTC("Jan  2 08:01:57")
        self.result_is_datetime(result)
        assert str(result) == str(date.today().year) + '-01-02 08:01:57+00:00'

    def test_abnormal_date_obj_with_timezone_in_date(self):
        result = toUTC(parse("2016-01-02 08:01:57+06:00"))
        self.result_is_datetime(result)
        assert str(result) == '2016-01-02 02:01:57+00:00'

    def test_long_epoch_without_timezone(self):
        result = toUTC(1468443523000000000)
        self.result_is_datetime(result)
        assert str(result) == '2016-07-13 20:58:43+00:00'

    def test_short_epoch_without_timezone(self):
        result = toUTC(1468443523)
        self.result_is_datetime(result)
        assert str(result) == '2016-07-13 20:58:43+00:00'

    def test_float_epoch(self):
        result = toUTC(1468443523.0)
        self.result_is_datetime(result)
        assert str(result) == '2016-07-13 20:58:43+00:00'

    def test_long_float_epoch(self):
        result = toUTC(1.468443523e+18)
        self.result_is_datetime(result)
        assert str(result) == '2016-07-13 20:58:43+00:00'

    def test_float_epoch_milliseconds(self):
        result = toUTC(1.468443523e+11)
        self.result_is_datetime(result)
        assert str(result) == '2016-07-13 20:58:43+00:00'
<commit_msg>Force reload of toUTC if already loaded in tests<commit_after>from datetime import datetime, date
from dateutil.parser import parse

import sys
import os
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../lib'))
import pytz

import tzlocal

UTC_TIMEZONE_COUNT = 0
def utc_timezone():
    ''' This is a mock function, so when we run tests
        we trick the system into thinking we're on UTC
        no matter what the real timezone is
    '''
    global UTC_TIMEZONE_COUNT
    UTC_TIMEZONE_COUNT += 1
    return pytz.timezone('UTC')

tzlocal.get_localzone = utc_timezone

import sys
if 'utilities.toUTC' in sys.modules:
    reload(sys.modules['utilities.toUTC'])

from utilities.toUTC import toUTC

class TestToUTC():

    def result_is_datetime(self, result):
        assert type(result) == datetime

    def test_timezone_function_call_count(self):
        toUTC("2016-07-11 14:33:31.625443")
        toUTC("2016-07-12 14:33:31.625444")
        toUTC("2016-07-13 14:33:31.625445")
        toUTC("2016-07-14 14:33:31.625446")
        assert UTC_TIMEZONE_COUNT == 1

    def test_normal_date_str_with_default_timezone(self):
        result = toUTC("2016-07-13 14:33:31.625443")
        self.result_is_datetime(result)
        assert str(result) == '2016-07-13 14:33:31.625443+00:00'

    def test_normal_date_str_with_utc_timezone(self):
        result = toUTC("2016-07-13 22:33:31.625443+00:00")
        self.result_is_datetime(result)
        assert str(result) == '2016-07-13 22:33:31.625443+00:00'

    def test_normal_date_str_with_timezone(self):
        result = toUTC("2016-07-13 14:33:31.625443-08:00")
        self.result_is_datetime(result)
        assert str(result) == '2016-07-13 22:33:31.625443+00:00'

    def test_abnormal_date_str_without_timezone(self):
        result = toUTC("Jan  2 08:01:57")
        self.result_is_datetime(result)
        assert str(result) == str(date.today().year) + '-01-02 08:01:57+00:00'

    def test_abnormal_date_obj_with_timezone_in_date(self):
        result = toUTC(parse("2016-01-02 08:01:57+06:00"))
        self.result_is_datetime(result)
        assert str(result) == '2016-01-02 02:01:57+00:00'

    def test_long_epoch_without_timezone(self):
        result = toUTC(1468443523000000000)
        self.result_is_datetime(result)
        assert str(result) == '2016-07-13 20:58:43+00:00'

    def test_short_epoch_without_timezone(self):
        result = toUTC(1468443523)
        self.result_is_datetime(result)
        assert str(result) == '2016-07-13 20:58:43+00:00'

    def test_float_epoch(self):
        result = toUTC(1468443523.0)
        self.result_is_datetime(result)
        assert str(result) == '2016-07-13 20:58:43+00:00'

    def test_long_float_epoch(self):
        result = toUTC(1.468443523e+18)
        self.result_is_datetime(result)
        assert str(result) == '2016-07-13 20:58:43+00:00'

    def test_float_epoch_milliseconds(self):
        result = toUTC(1.468443523e+11)
        self.result_is_datetime(result)
        assert str(result) == '2016-07-13 20:58:43+00:00'
<|endoftext|>


<commit_before>import mbuild as mb

class Monolayer(mb.Compound):
    """A general monolayer recipe. """

    def __init__(self, surface, chain, backfill=None, mask=None, tile_x=1, tile_y=1):
        """Create an alkylsilane monolayer on beta-cristobalite.

        Parameters
        ----------
        surface : mb.Compound
            Surface on which the monolayer will be built.
        chain : list of mb.Compound
            The chain to be replicated and attached to the surface.
        backfill : list of mb.Compound, optional, default=None
            If there are fewer chains than there are ports on the surface,
            copies of `backfill` will be used to fill the remaining ports.
        mask : np.ndarray, shape=(n, 3), optional, default=None
            An array of planar binding locations. If not provided, the entire
            surface will be filled with `chain`.
        tile_x : int, optional, default=1
            Number of times to replicate substrate in x-direction.
        tile_y : int, optional, default=1
            Number of times to replicate substrate in y-direction.

        """
        super(Monolayer, self).__init__()

        # Replicate the surface.
        tc = mb.TiledCompound(surface, n_tiles=(tile_x, tile_y, 1))
        self.add(tc, 'tiled_surface')

        if mask is None:  # Fill the surface.
            mask = mb.random_mask_2d(len(tc.referenced_ports()))

        # Attach chains to specified binding sites. Remaining sites get a backfill.
        chains, backfills = mb.apply_mask(host=self.tiled_surface, guest=chain,
                                          mask=mask, backfill=backfill)
        self.add(chains)
        self.add(backfills)

if __name__ == "__main__":
    from mbuild.components.surfaces.betacristobalite import Betacristobalite
    from mbuild.examples.alkane_monolayer.alkylsilane import AlkylSilane
    from mbuild.components.atoms.H import H

    mask = mb.grid_mask_2d(8, 8)  # Evenly spaced, 2D grid of points.
    monolayer = Monolayer(surface=Betacristobalite(), chain=AlkylSilane(10),
                          backfill=H(), mask=mask)
    monolayer.visualize()
<commit_msg>pass kwargs to apply_mask<commit_after>import mbuild as mb

class Monolayer(mb.Compound):
    """A general monolayer recipe. """

    def __init__(self, surface, chain, backfill=None, mask=None, tile_x=1, tile_y=1, **kwargs):
        """Create an alkylsilane monolayer on beta-cristobalite.

        Parameters
        ----------
        surface : mb.Compound
            Surface on which the monolayer will be built.
        chain : list of mb.Compound
            The chain to be replicated and attached to the surface.
        backfill : list of mb.Compound, optional, default=None
            If there are fewer chains than there are ports on the surface,
            copies of `backfill` will be used to fill the remaining ports.
        mask : np.ndarray, shape=(n, 3), optional, default=None
            An array of planar binding locations. If not provided, the entire
            surface will be filled with `chain`.
        tile_x : int, optional, default=1
            Number of times to replicate substrate in x-direction.
        tile_y : int, optional, default=1
            Number of times to replicate substrate in y-direction.

        """
        super(Monolayer, self).__init__()

        # Replicate the surface.
        tc = mb.TiledCompound(surface, n_tiles=(tile_x, tile_y, 1))
        self.add(tc, 'tiled_surface')

        if mask is None:  # Fill the surface.
            mask = mb.random_mask_2d(len(tc.referenced_ports()))

        # Attach chains to specified binding sites. Remaining sites get a backfill.
        chains, backfills = mb.apply_mask(host=self.tiled_surface, guest=chain,
                                          mask=mask, backfill=backfill, **kwargs)
        self.add(chains)
        self.add(backfills)

if __name__ == "__main__":
    from mbuild.components.surfaces.betacristobalite import Betacristobalite
    from mbuild.examples.alkane_monolayer.alkylsilane import AlkylSilane
    from mbuild.components.atoms.H import H

    mask = mb.grid_mask_2d(8, 8)  # Evenly spaced, 2D grid of points.
    monolayer = Monolayer(surface=Betacristobalite(), chain=AlkylSilane(10),
                          backfill=H(), mask=mask)
    monolayer.visualize()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

from models import *
from rapidsms.message import StatusCodes
from apps.reporters.models import *
from apps.form.formslogic import FormsLogic

class NigeriaFormsLogic(FormsLogic):
    ''' This class will hold the nigeria-specific forms logic.
        I'm not sure whether this will be the right structure
        this was just for getting something hooked up '''

    # this is a simple structure we use to describe the forms.
    # maps token names to db names
    _form_lookups = {
                     "nets" : {
                               "class" : NetDistribution,
                               "display" : "nets",
                               "fields" : {
                                           "location" : "location",
                                           "distributed" : "distributed",
                                           "expected" : "expected",
                                           "actual" : "actual",
                                           "discrepancy" : "discrepancy",
                                           }
                               },
                     "netcards" : {"class" : CardDistribution,
                              "display" : "net cards",
                              "fields" : {
                                          "location" : "location",
                                          "settlements" : "settlements",
                                          "people" : "people",
                                          "issued" : "distributed",
                                          }
                              }
                     }

    _foreign_key_lookups = {"Location" : "code" }

    def validate(self, *args, **kwargs):
        message = args[0]
        form_entry = args[1]
        # in case we need help, build a valid reminder string
        # TODO put this in the db!
        required = ["location", "role", "firstname"]
        help = ("%s register " % form_entry.domain.code.abbreviation.lower()) +\
                " ".join(["<%s>" % t for t in required])
        if form_entry.form.code.abbreviation == "register":
            data = form_entry.to_dict()
            print "\n\n%r\n\n" % data

            # check that ALL FIELDS were provided
            missing = [t for t in required if data[t] is None]

            # missing fields! collate them, and
            # send back a friendly non-localized
            # error message, then abort
            if missing:
                mis_str = ", ".join(missing)
                return ["Missing fields: %s" % mis_str, help]

            # parse the name via Reporter
            flat_name = data.pop("firstname") + " " + data.pop("lastname") + " " + data.pop("othername")
            data["alias"], data["first_name"], data["last_name"] =\
                Reporter.parse_name(flat_name.strip())

            # all fields were present and correct, so copy them into the
            # form_entry, for "actions" to pick up again without re-fetching
            form_entry.rep_data = data

            # nothing went wrong. the data structure
            # is ready to spawn a Reporter object
            return None
        elif form_entry.form.code.abbreviation in self._form_lookups.keys():
            # we know all the fields in this form are required, so make sure they're set
            # TODO check the token's required flag
            required_token_names = self._form_lookups[form_entry.form.code.abbreviation]["fields"].keys()
            for token in form_entry.tokenentry_set.all():
                if token.token.abbreviation in required_token_names:
                    # found it, as long as the data isn't empty remove it
                    if token.data:
                        required_token_names.remove(token.token.abbreviation)
            if required_token_names:
                errors = "The following fields are required: " + ", ".join(required_token_names)
                return [errors]
            return None

    def actions(self, *args, **kwargs):
        message = args[0]
        form_entry = args[1]
        print(form_entry.form.code.abbreviation)
        if form_entry.form.code.abbreviation== "register":

            data = form_entry.rep_data
            # load the location and role objects via their codes
            data["location"] = Location.objects.get(code__iexact=data["location"])
            data["role"]     = Role.objects.get(code__iexact=data["role"])

            # spawn and save the reporter using the
            # data we collected in self.validate
            rep = Reporter.objects.create(**data)

            # we can assume that the new reporter will be using
            # this device again, so register a connection. this
            # automatically logs them in, so they can start
            # reporting straight away
            conn = PersistantConnection.from_message(message)
            conn.reporter = rep
            conn.save()

            # notify the user that everyting went okay
            # TODO: proper (localized?) messages here
            message.respond("Hello %s! You are now registered as %s at %s %s."\
                % (rep.first_name, rep.role, rep.location, rep.location.type), StatusCodes.OK)

        elif self._form_lookups.has_key(form_entry.form.code.abbreviation):
            to_use = self._form_lookups[form_entry.form.code.abbreviation]
            form_class = to_use["class"]
            field_map = to_use["fields"]
            # create and save the model from the form data
            instance = self._model_from_form(message, form_entry, form_class, field_map, self._foreign_key_lookups)
            instance.time = message.date

            # if the reporter isn't set then populate the connection object.
            # this means that at least one (actually exactly one) is set
            # the method above sets this property in the instance
            # if it was found.
            if not hasattr(instance, "reporter") or not instance.reporter:
                instance.connection = message.persistant_connection
            instance.save()
            response = "Received report for %s %s: " % (form_entry.domain.code.abbreviation.upper(), to_use["display"].upper())
            # this line pulls any attributes that are present into 2-item lists
            attrs = [[attr_name, str(getattr(instance, attr_name))] for attr_name in field_map.values() if hasattr(instance, attr_name)]
            # joins the inner list on "=" and the outer on ", " so we get
            # attr1=value1, attr2=value2
            response = response + ", ".join(["=".join(t) for t in attrs])
            if not instance.reporter:
                response = response + ". Please register your phone"
            message.respond(response, StatusCodes.OK)
<commit_msg>small renaming of name tokens<commit_after>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

from models import *
from rapidsms.message import StatusCodes
from apps.reporters.models import *
from apps.form.formslogic import FormsLogic

class NigeriaFormsLogic(FormsLogic):
    ''' This class will hold the nigeria-specific forms logic.
        I'm not sure whether this will be the right structure
        this was just for getting something hooked up '''

    # this is a simple structure we use to describe the forms.
    # maps token names to db names
    _form_lookups = {
                     "nets" : {
                               "class" : NetDistribution,
                               "display" : "nets",
                               "fields" : {
                                           "location" : "location",
                                           "distributed" : "distributed",
                                           "expected" : "expected",
                                           "actual" : "actual",
                                           "discrepancy" : "discrepancy",
                                           }
                               },
                     "netcards" : {"class" : CardDistribution,
                              "display" : "net cards",
                              "fields" : {
                                          "location" : "location",
                                          "settlements" : "settlements",
                                          "people" : "people",
                                          "issued" : "distributed",
                                          }
                              }
                     }

    _foreign_key_lookups = {"Location" : "code" }

    def validate(self, *args, **kwargs):
        message = args[0]
        form_entry = args[1]
        # in case we need help, build a valid reminder string
        # TODO put this in the db!
        required = ["location", "role", "firstname"]
        help = ("%s register " % form_entry.domain.code.abbreviation.lower()) +\
                " ".join(["<%s>" % t for t in required])
        if form_entry.form.code.abbreviation == "register":
            data = form_entry.to_dict()
            print "\n\n%r\n\n" % data

            # check that ALL FIELDS were provided
            missing = [t for t in required if data[t] is None]

            # missing fields! collate them, and
            # send back a friendly non-localized
            # error message, then abort
            if missing:
                mis_str = ", ".join(missing)
                return ["Missing fields: %s" % mis_str, help]

            # parse the name via Reporter
            flat_name = data.pop("firstname") + " " + data.pop("secondname") + " " + data.pop("thirdname")
            data["alias"], data["first_name"], data["last_name"] =\
                Reporter.parse_name(flat_name.strip())

            # all fields were present and correct, so copy them into the
            # form_entry, for "actions" to pick up again without re-fetching
            form_entry.rep_data = data

            # nothing went wrong. the data structure
            # is ready to spawn a Reporter object
            return None
        elif form_entry.form.code.abbreviation in self._form_lookups.keys():
            # we know all the fields in this form are required, so make sure they're set
            # TODO check the token's required flag
            required_token_names = self._form_lookups[form_entry.form.code.abbreviation]["fields"].keys()
            for token in form_entry.tokenentry_set.all():
                if token.token.abbreviation in required_token_names:
                    # found it, as long as the data isn't empty remove it
                    if token.data:
                        required_token_names.remove(token.token.abbreviation)
            if required_token_names:
                errors = "The following fields are required: " + ", ".join(required_token_names)
                return [errors]
            return None

    def actions(self, *args, **kwargs):
        message = args[0]
        form_entry = args[1]
        print(form_entry.form.code.abbreviation)
        if form_entry.form.code.abbreviation== "register":

            data = form_entry.rep_data
            # load the location and role objects via their codes
            data["location"] = Location.objects.get(code__iexact=data["location"])
            data["role"]     = Role.objects.get(code__iexact=data["role"])

            # spawn and save the reporter using the
            # data we collected in self.validate
            rep = Reporter.objects.create(**data)

            # we can assume that the new reporter will be using
            # this device again, so register a connection. this
            # automatically logs them in, so they can start
            # reporting straight away
            conn = PersistantConnection.from_message(message)
            conn.reporter = rep
            conn.save()

            # notify the user that everyting went okay
            # TODO: proper (localized?) messages here
            message.respond("Hello %s! You are now registered as %s at %s %s."\
                % (rep.first_name, rep.role, rep.location, rep.location.type), StatusCodes.OK)

        elif self._form_lookups.has_key(form_entry.form.code.abbreviation):
            to_use = self._form_lookups[form_entry.form.code.abbreviation]
            form_class = to_use["class"]
            field_map = to_use["fields"]
            # create and save the model from the form data
            instance = self._model_from_form(message, form_entry, form_class, field_map, self._foreign_key_lookups)
            instance.time = message.date

            # if the reporter isn't set then populate the connection object.
            # this means that at least one (actually exactly one) is set
            # the method above sets this property in the instance
            # if it was found.
            if not hasattr(instance, "reporter") or not instance.reporter:
                instance.connection = message.persistant_connection
            instance.save()
            response = "Received report for %s %s: " % (form_entry.domain.code.abbreviation.upper(), to_use["display"].upper())
            # this line pulls any attributes that are present into 2-item lists
            attrs = [[attr_name, str(getattr(instance, attr_name))] for attr_name in field_map.values() if hasattr(instance, attr_name)]
            # joins the inner list on "=" and the outer on ", " so we get
            # attr1=value1, attr2=value2
            response = response + ", ".join(["=".join(t) for t in attrs])
            if not instance.reporter:
                response = response + ". Please register your phone"
            message.respond(response, StatusCodes.OK)
<|endoftext|>


<commit_before># Copyright (c) 2003-2005 Maxim Sobolev. All rights reserved.
# Copyright (c) 2006-2014 Sippy Software, Inc. All rights reserved.
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from datetime import datetime
from twisted.internet import task, reactor
from traceback import print_exc, format_list, extract_stack
from sys import stdout
from random import random

class TimeoutInact(object):
    _task = None
    _interval = None
    _ticks_left = None
    _timeout_callback = None
    _timeout_cb_args = None
    _randomize_runs = None

    def __init__(self, timeout_callback, interval, ticks = 1, *callback_arguments):
        #print('TimeoutInact.__init__(%s)' % (str(callback_arguments),))
        self._timeout_callback = timeout_callback
        self._interval = interval
        self._ticks_left = ticks
        self._timeout_cb_args = callback_arguments

    def _get_randomizer(self, p):
        return lambda x: x * (1.0 + p * (1.0 - 2.0 * random()))

    def spread_runs(self, p):
        self._randomize_runs = self._get_randomizer(p)

    def go(self):
        if self._ticks_left == 1 or self._randomize_runs != None:
            # Special case for just one call, we also use it when we need
            # to add some random noise into a schedule so that LoopingCall()
            # is not very convinient
            self._schedule_call_later()
            self.cancel = self.cancel_callLater
            if self._randomize_runs == None:
                self._timeout_cb_args = None
            return
        self._task = task.LoopingCall(self._run, *self._timeout_cb_args)
        self._timeout_cb_args = None
        self._task.start(self._interval, False)

    def _schedule_call_later(self):
        if self._randomize_runs == None:
            ival = self._interval
        else:
            ival = self._randomize_runs(self._interval)
        self._task = reactor.callLater(ival, self._run_once, \
          *self._timeout_cb_args)

    def _run(self, *callback_arguments):
        try:
            self._timeout_callback(*callback_arguments)
        except:
            print datetime.now(), 'Timeout: unhandled exception in timeout callback'
            print '-' * 70
            print_exc(file = stdout)
            print '-' * 70
            stdout.flush()
        if self._ticks_left == 1:
            self.cancel()
        elif self._ticks_left != -1:
            self._ticks_left -= 1

    def _run_once(self, *callback_arguments):
        try:
            self._timeout_callback(*callback_arguments)
        except:
            print datetime.now(), 'Timeout: unhandled exception in timeout callback'
            print '-' * 70
            print_exc(file = stdout)
            print '-' * 70
            stdout.flush()
        if self._ticks_left == 1:
            self._cleanup()
            return
        self._ticks_left -= 1
        self._schedule_call_later()

    def cancel(self):
        self._task.stop()
        self._cleanup()

    def cancel_callLater(self):
        self._task.cancel()
        self._cleanup()

    def _cleanup(self):
        self._task = None
        self._timeout_callback = None
        self._timeout_cb_args = None

class Timeout(TimeoutInact):
    def __init__(self, *args):
        TimeoutInact.__init__(self, *args)
        self.go()

class TimeoutAbs:
    _task = None
    _timeout_callback = None

    def __init__(self, timeout_callback, etime, *callback_arguments):
        etime -= reactor.seconds()
        if etime < 0:
            etime = 0
        self._timeout_callback = timeout_callback
        self._task = reactor.callLater(etime, self._run_once, *callback_arguments)

    def _run_once(self, *callback_arguments):
        try:
            self._timeout_callback(*callback_arguments)
        except:
            print datetime.now(), 'Timeout: unhandled exception in timeout callback'
            print '-' * 70
            print_exc(file = stdout)
            print '-' * 70
            stdout.flush()
        self._task = None
        self._timeout_callback = None

    def cancel(self):
        self._task.cancel()
        self._task = None
        self._timeout_callback = None

class Timeout_debug(Timeout):
    _traceback = None

    def __init__(self, *parameters, **kparameters):
        self._traceback = format_list(extract_stack())
        Timeout.__init__(self, *parameters, **kparameters)

    def cancel(self):
        if self._task == None:
            print self._traceback
        Timeout.cancel(self)

if __name__ == '__main__':
    from twisted.internet import reactor

    def test1(arguments, testnum):
        print testnum
        arguments['test'] = True
        reactor.crash()

    def test2(arguments, testnum):
        print testnum
        arguments['test'] = 'bar'
        reactor.crash()

    arguments = {'test':False}
    timeout_1 = Timeout(test1, 0, 1, arguments, 'test1')
    reactor.run()
    assert(arguments['test'])
    timeout_1 = Timeout(test1, 0.1, 1, arguments, 'test2')
    timeout_2 = Timeout(test2, 0.2, 1, arguments, 'test3')
    timeout_1.cancel()
    reactor.run()
    assert(arguments['test'] == 'bar')

    arguments = {'test':False}
    timeout_1 = TimeoutAbs(test1, reactor.seconds(), arguments, 'test4')
    reactor.run()
    assert(arguments['test'])

    timeout_1 = TimeoutAbs(test1, reactor.seconds() + 0.1, arguments, 'test5')
    timeout_2 = TimeoutAbs(test2, reactor.seconds() + 0.2, arguments, 'test6')
    timeout_1.cancel()
    reactor.run()
    assert(arguments['test'] == 'bar')
<commit_msg>Mangle self.cancel when we are in the _run_once routine, so if that cancel is called here it would not call reactor's cancel on event that has not been scheduled yet.<commit_after># Copyright (c) 2003-2005 Maxim Sobolev. All rights reserved.
# Copyright (c) 2006-2014 Sippy Software, Inc. All rights reserved.
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from datetime import datetime
from twisted.internet import task, reactor
from traceback import print_exc, format_list, extract_stack
from sys import stdout
from random import random

class TimeoutInact(object):
    _task = None
    _interval = None
    _ticks_left = None
    _timeout_callback = None
    _timeout_cb_args = None
    _randomize_runs = None

    def __init__(self, timeout_callback, interval, ticks = 1, *callback_arguments):
        #print('TimeoutInact.__init__(%s)' % (str(callback_arguments),))
        self._timeout_callback = timeout_callback
        self._interval = interval
        self._ticks_left = ticks
        self._timeout_cb_args = callback_arguments

    def _get_randomizer(self, p):
        return lambda x: x * (1.0 + p * (1.0 - 2.0 * random()))

    def spread_runs(self, p):
        self._randomize_runs = self._get_randomizer(p)

    def go(self):
        if self._ticks_left == 1 or self._randomize_runs != None:
            # Special case for just one call, we also use it when we need
            # to add some random noise into a schedule so that LoopingCall()
            # is not very convinient
            self._schedule_call_later()
            self.cancel = self.cancel_callLater
            if self._randomize_runs == None:
                self._timeout_cb_args = None
            return
        self._task = task.LoopingCall(self._run, *self._timeout_cb_args)
        self._timeout_cb_args = None
        self._task.start(self._interval, False)

    def _schedule_call_later(self):
        if self._randomize_runs == None:
            ival = self._interval
        else:
            ival = self._randomize_runs(self._interval)
        self._task = reactor.callLater(ival, self._run_once, \
          *self._timeout_cb_args)

    def _run(self, *callback_arguments):
        try:
            self._timeout_callback(*callback_arguments)
        except:
            print datetime.now(), 'Timeout: unhandled exception in timeout callback'
            print '-' * 70
            print_exc(file = stdout)
            print '-' * 70
            stdout.flush()
        if self._ticks_left == 1:
            self.cancel()
        elif self._ticks_left != -1:
            self._ticks_left -= 1

    def _run_once(self, *callback_arguments):
        self.cancel = self.cancel_callLater_in_run_once
        try:
            self._timeout_callback(*callback_arguments)
        except:
            print datetime.now(), 'Timeout: unhandled exception in timeout callback'
            print '-' * 70
            print_exc(file = stdout)
            print '-' * 70
            stdout.flush()
        self._ticks_left -= 1
        if self._ticks_left == 0:
            self._cleanup()
            return
        self._schedule_call_later()
        self.cancel = self.cancel_callLater

    def cancel(self):
        self._task.stop()
        self._cleanup()

    def cancel_callLater(self):
        self._task.cancel()
        self._cleanup()

    def cancel_callLater_in_run_once(self):
        self._ticks_left = 1

    def _cleanup(self):
        self._task = None
        self._timeout_callback = None
        self._timeout_cb_args = None

class Timeout(TimeoutInact):
    def __init__(self, *args):
        TimeoutInact.__init__(self, *args)
        self.go()

class TimeoutAbs:
    _task = None
    _timeout_callback = None

    def __init__(self, timeout_callback, etime, *callback_arguments):
        etime -= reactor.seconds()
        if etime < 0:
            etime = 0
        self._timeout_callback = timeout_callback
        self._task = reactor.callLater(etime, self._run_once, *callback_arguments)

    def _run_once(self, *callback_arguments):
        try:
            self._timeout_callback(*callback_arguments)
        except:
            print datetime.now(), 'Timeout: unhandled exception in timeout callback'
            print '-' * 70
            print_exc(file = stdout)
            print '-' * 70
            stdout.flush()
        self._task = None
        self._timeout_callback = None

    def cancel(self):
        self._task.cancel()
        self._task = None
        self._timeout_callback = None

class Timeout_debug(Timeout):
    _traceback = None

    def __init__(self, *parameters, **kparameters):
        self._traceback = format_list(extract_stack())
        Timeout.__init__(self, *parameters, **kparameters)

    def cancel(self):
        if self._task == None:
            print self._traceback
        Timeout.cancel(self)

if __name__ == '__main__':
    from twisted.internet import reactor

    def test1(arguments, testnum):
        print testnum
        arguments['test'] = True
        reactor.crash()

    def test2(arguments, testnum):
        print testnum
        arguments['test'] = 'bar'
        reactor.crash()

    arguments = {'test':False}
    timeout_1 = Timeout(test1, 0, 1, arguments, 'test1')
    reactor.run()
    assert(arguments['test'])
    timeout_1 = Timeout(test1, 0.1, 1, arguments, 'test2')
    timeout_2 = Timeout(test2, 0.2, 1, arguments, 'test3')
    timeout_1.cancel()
    reactor.run()
    assert(arguments['test'] == 'bar')

    arguments = {'test':False}
    timeout_1 = TimeoutAbs(test1, reactor.seconds(), arguments, 'test4')
    reactor.run()
    assert(arguments['test'])

    timeout_1 = TimeoutAbs(test1, reactor.seconds() + 0.1, arguments, 'test5')
    timeout_2 = TimeoutAbs(test2, reactor.seconds() + 0.2, arguments, 'test6')
    timeout_1.cancel()
    reactor.run()
    assert(arguments['test'] == 'bar')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models
from django.template.defaultfilters import slugify
import datetime
from django.core.validators import RegexValidator

class Genero(models.Model):
    nombre = models.CharField(blank=False, max_length=100,primary_key=True,unique=True)

    def get(self):
        return nombre

    def __unicode__(self):
            return self.nombre

class Usuario(models.Model):
    nombre = models.CharField(blank=False, max_length=15)
    apellidos = models.CharField(blank=False, max_length=30)
    phone_regex = RegexValidator(regex=r'^\d{9,15}$', message="Telefono debe ser: 123456789.")
    telefono = models.CharField(validators=[phone_regex], blank=True,max_length=15) # validators should be a list
    direccion = models.CharField(blank=True,max_length=100)

    def __unicode__(self):
            return self.nombre+" "+self.apellidos

class Recurso(models.Model):
    titulo = models.CharField(blank=False, max_length=100)
    autor = models.CharField(blank=False, max_length=100)
    descripcion = models.CharField(blank=True, max_length=250)
    codigo = models.CharField(blank=True, max_length=100)
    estanteria = models.CharField(blank=True, max_length=100)
    genero = models.ForeignKey(Genero, on_delete=models.CASCADE)
    anio = models.IntegerField(blank=True, null=True)
    editorial = models.CharField(blank=True, max_length=250)
    dvd = models.BooleanField(default=False)

    def __unicode__(self):
            return self.titulo

class Prestamo(models.Model):
    """( Prestamo Recursos)"""
    fechaInicio = models.DateField(default=datetime.datetime.today)
    fechaFin = models.DateField(default=datetime.datetime.today()+datetime.timedelta(days=7)) #por defecto 7 dias despues de la creacion
    Recurso = models.OneToOneField(Recurso,unique=True)
    usuarioPrestamo = models.OneToOneField(Usuario,unique=True)

    def __unicode__(self):
        return self.fechaFin.strftime("%Y-%m-%d")+" "+self.Recurso.__unicode__()+" "+self.usuarioPrestamo.__unicode__()
<commit_msg>models otra vez<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models
from django.template.defaultfilters import slugify
import datetime
from django.core.validators import RegexValidator

class Genero(models.Model):
    nombre = models.CharField(blank=False, max_length=100,primary_key=True,unique=True)

    def get(self):
        return nombre

    def __unicode__(self):
            return self.nombre

class Usuario(models.Model):
    nombre = models.CharField(blank=False, max_length=15)
    apellidos = models.CharField(blank=False, max_length=30)
    phone_regex = RegexValidator(regex=r'^\d{9,15}$', message="Telefono debe ser: 123456789.")
    telefono = models.CharField(validators=[phone_regex], blank=True,max_length=15) # validators should be a list
    direccion = models.CharField(blank=True,max_length=100)

    def __unicode__(self):
            return self.nombre+" "+self.apellidos

class Recurso(models.Model):
    titulo = models.CharField(blank=False, max_length=100)
    autor = models.CharField(blank=False, max_length=100)
    descripcion = models.CharField(blank=True, max_length=250)
    codigo = models.CharField(blank=True, max_length=100)
    estanteria = models.CharField(blank=True, max_length=100)
    genero = models.ForeignKey(Genero, on_delete=models.CASCADE)
    anio = models.IntegerField(blank=True, null=True)
    editorial = models.CharField(blank=True, max_length=250)
    dvd = models.BooleanField(default=False)

    def __unicode__(self):
            return self.titulo

class Prestamo(models.Model):
    """( Prestamo Recursos)"""
    fechaInicio = models.DateField(default=datetime.datetime.today)
    fechaFin = models.DateField(default=datetime.datetime.today()+datetime.timedelta(days=7)) #por defecto 7 dias despues de la creacion
    Recurso = models.OneToOneField(Recurso,unique=True)
    usuarioPrestamo = models.OneToOneField(Usuario,unique=True)

    def __unicode__(self):
        return self.fechaFin.strftime("%Y-%m-%d")+" "+self.Recurso.__unicode__()+" "+self.usuarioPrestamo.__unicode__()
<|endoftext|>