<commit_before>from os import path, getcwd
import json
import copy

from unittest import TestCase

#import geopandas
from mander.districts import district
from mander.metrics import calculatePolsby, calculateConvexHull, calculateReock, calculateSchwartzberg

base_dir = path.dirname(path.realpath(__file__))

test_districts = ['CD_CA_9', 'CD_IL_4', 'CD_PA_7'] # TODO also test 'CD_CA_24', 'CD_MA_9'

metrics = {
    'polsbypopper': [],
    'convexhull': [],
    'reock': [],
    'schwartzberg': []
}

test_expected = copy.deepcopy(metrics)
test_results = copy.deepcopy(metrics)

for d in test_districts:

  district_boundaries_file = path.join(base_dir, 'data', d + '.geojson')
  district_scores_file = path.join(base_dir, 'data', d + '_scores.json')

  with open(district_boundaries_file) as district_boundary_data:
    with open(district_scores_file) as district_scores_data:

      district_boundary = json.load(district_boundary_data)
      district_scores = json.load(district_scores_data)

      test_district = district(district_boundaries_file) # TODO use python object instead of file path parameter

      # Polsby Popper
      test_expected['polsbypopper'].append('%.4f' % district_scores['polsbypopper'])
      test_results['polsbypopper'].append('%.4f' % calculatePolsby(test_district))

      # Convex Hull
      test_expected['convexhull'].append('%.4f' % district_scores['convexhull'])
      test_results['convexhull'].append('%.4f' % calculateConvexHull(test_district))

      # Reock
      test_expected['reock'].append('%.4f' % district_scores['convexhull'])
      test_results['reock'].append('%.4f' % calculateReock(test_district))

      # Schwartzberg
      test_expected['schwartzberg'].append('%.4f' % district_scores['schwartzberg'])
      test_results['schwartzberg'].append('%.4f' % calculateSchwartzberg(test_district))

class TestMetrics(TestCase):

    def test_polsbypopper(self):
      self.assertEqual(test_expected['polsbypopper'], test_results['polsbypopper'])

    def test_convexhull(self):
      self.assertEqual(test_expected['convexhull'], test_results['convexhull'])

    def test_reock(self):
      self.assertEqual(test_expected['reock'], test_results['reock'])

    def test_schwartzberg(self):
      self.assertEqual(test_expected['schwartzberg'], test_results['schwartzberg'])
<commit_msg>test multipolygons<commit_after>from os import path, getcwd
import json
import copy

from unittest import TestCase

#import geopandas
from mander.districts import District
from mander.metrics import calculatePolsby, calculateConvexHull, calculateReock, calculateSchwartzberg

base_dir = path.dirname(path.realpath(__file__))

test_districts = ['CD_CA_9', 'CD_IL_4', 'CD_PA_7', 'CD_CA_24', 'CD_MA_9']

metrics = {
    'polsbypopper': [],
    'convexhull': [],
    'reock': [],
    'schwartzberg': []
}

test_expected = copy.deepcopy(metrics)
test_results = copy.deepcopy(metrics)

for d in test_districts:

  district_boundaries_file = path.join(base_dir, 'data', d + '.geojson')
  district_scores_file = path.join(base_dir, 'data', d + '_scores.json')

  with open(district_boundaries_file) as district_boundary_data:
    with open(district_scores_file) as district_scores_data:

      district_boundary = json.load(district_boundary_data)
      district_scores = json.load(district_scores_data)

      test_district = District(district_boundaries_file) # TODO use python object instead of file path parameter

      # Polsby Popper
      test_expected['polsbypopper'].append('%.4f' % district_scores['polsbypopper'])
      test_results['polsbypopper'].append('%.4f' % calculatePolsby(test_district))

      # Convex Hull
      test_expected['convexhull'].append('%.4f' % district_scores['convexhull'])
      test_results['convexhull'].append('%.4f' % calculateConvexHull(test_district))

      # Reock
      test_expected['reock'].append('%.4f' % district_scores['convexhull'])
      test_results['reock'].append('%.4f' % calculateReock(test_district))

      # Schwartzberg
      test_expected['schwartzberg'].append('%.4f' % district_scores['schwartzberg'])
      test_results['schwartzberg'].append('%.4f' % calculateSchwartzberg(test_district))

class TestMetrics(TestCase):

    def test_polsbypopper(self):
      self.assertEqual(test_expected['polsbypopper'], test_results['polsbypopper'])

    def test_convexhull(self):
      self.assertEqual(test_expected['convexhull'], test_results['convexhull'])

    def test_reock(self):
      self.assertEqual(test_expected['reock'], test_results['reock'])

    def test_schwartzberg(self):
      self.assertEqual(test_expected['schwartzberg'], test_results['schwartzberg'])
<|endoftext|>


<commit_before>"""
This module hears to estimation messages and response back to them by using
predictive machine/deep learning
"""
from jira_client import JiraClient
import random

class EstimationMessage:

  def __init__(self, bot):
    self.bot = bot
    self.jira_client = JiraClient()
    self.story_points = [1, 2, 3, 5, 8]
    self.words = ['estima']

  def listenTo(self, from_data, message):
    return any([word in message.lower() for word in self.words])

  def getEstimation(self, jira_issue):
    return random.choice(self.story_points)

  def reactTo(self, from_data, message):
    jira_issue_id = message.split(' ')[1]
    jira_issue = self.jira_client.getIssue(jira_issue_id)
    estimation_points = self.getEstimation(jira_issue)
    response_msg = '{0} story points for {1} ({2})!'.format(str(estimation_points), jira_issue_id, jira_issue.fields.summary)
    self.bot.sendMessage(from_data, response_msg)<commit_msg>fix(estimation_message): use regex for jira issues id's<commit_after>"""
This module hears to estimation messages and response back to them by using
predictive machine/deep learning
"""
from jira_client import JiraClient
import re
import random

class EstimationMessage:

  def __init__(self, bot):
    self.bot = bot
    self.jira_client = JiraClient()
    self.story_points = [1, 2, 3, 5, 8]
    self.words = ['estima']

  def listenTo(self, from_data, message):
    return any([word in message.lower() for word in self.words])

  def getJiraID(self, message):
    matchID = re.search('uif-[0-9]+', message, re.IGNORECASE)
    return None if matchID is None else message[matchID.start():matchID.end()]

  def getEstimation(self, jira_issue):
    return random.choice(self.story_points)

  def reactTo(self, from_data, message):
    jira_issue_id = self.getJiraID(message)
    if jira_issue_id is None:
      return self.bot.sendMessage(from_data, 'No jira issue found')
    jira_issue = self.jira_client.getIssue(jira_issue_id)
    estimation_points = self.getEstimation(jira_issue)
    response_msg = '{0} story points for {1} ({2})!'.format(str(estimation_points), jira_issue_id, jira_issue.fields.summary)
    self.bot.sendMessage(from_data, response_msg)<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2014 Eugene Zhuk.
# Use of this source code is governed by the MIT license that can be found
# in the LICENSE file.

"""Checks AWS usage.

This script retrieves and displays an estimated total statement amount for
the specified billing period.

Usage:
    ./check_usage.py [options]
"""

import boto.cloudfront
import boto.dynamodb2
import boto.ec2
import boto.ec2.autoscale
import boto.ec2.cloudwatch
import boto.elasticache
import boto.emr
import boto.glacier
import boto.iam
import boto.kinesis
import boto.opsworks
import boto.rds2
import boto.redshift
import boto.route53
import boto.s3
import boto.sns
import boto.sdb
import boto.sqs
import boto.vpc
import csv
import itertools
import optparse
import re
import sys
import time

from boto.ec2.cloudwatch import MetricAlarm

class Error(Exception):
    pass

def connect_to_regions(service, regions):
    if regions is not None:
        return [service.connect_to_region(r.name) for r in service.regions() \
            if r.name in regions]
    else:
        return [service.connect_to_region(r.name) for r in service.regions() \
            if not (r.name.startswith('cn-') or r.name.startswith('us-gov-'))]

def get_ec2_usage(regions):
    cs = connect_to_regions(boto.ec2, regions)
    instances = list(itertools.chain.from_iterable([x.instances \
        for c in cs for x in c.get_all_reservations()]))
    running = sum(16 == i.state_code for i in instances)
    volumes = list(itertools.chain.from_iterable([c.get_all_volumes() \
        for c in cs]))
    size = sum(v.size for v in volumes)
    addresses = list(itertools.chain.from_iterable([c.get_all_addresses() \
        for c in cs]))
    unassigned = sum(a.instance_id is None for a in addresses)
    print '{0} EC2 Instance(s){1}\n' \
        '{2} EC2 Reserved Instance(s)\n' \
        '{3} EC2 Spot Instance Request(s)\n' \
        '{4} EBS Volume(s){5}\n' \
        '{6} EBS Snapshot(s)\n' \
        '{7} Amazon Machine Image(s)\n' \
        '{8} Network Interface(s)\n' \
        '{9} Elastic IP Address(es){10}\n' \
        '{11} Security Group(s)\n' \
        '{12} Key Pair(s)\n' \
        '{13} Tag(s)' \
        .format(len(instances), \
            '[{0} running]'.format(running) if 0 != running else '', \
            sum(len(c.get_all_reserved_instances()) for c in cs), \
            sum(len(c.get_all_spot_instance_requests()) for c in cs), \
            len(volumes), \
            '[{0} GB]'.format(size) if 0 != size else '', \
            sum(len(c.get_all_snapshots(owner=['self'])) for c in cs), \
            sum(len(c.get_all_images(owners=['self'])) for c in cs), \
            sum(len(c.get_all_network_interfaces()) for c in cs), \
            len(addresses), \
            '[{0} unassigned]'.format(unassigned) if 0 != unassigned else '', \
            sum(len(c.get_all_security_groups()) for c in cs), \
            sum(len(c.get_all_key_pairs()) for c in cs), \
            sum(len(c.get_all_tags()) for c in cs))

def get_autoscale_usage(regions):
    cs = connect_to_regions(boto.ec2.autoscale, regions)
    print '{0} Auto Scaling Group(s)\n' \
        '{1} Auto Scaling Launch Configuration(s)\n' \
        '{2} Auto Scaling Policie(s)' \
        .format(sum(len(c.get_all_groups()) for c in cs), \
            sum(len(c.get_all_launch_configurations()) for c in cs), \
            sum(len(c.get_all_policies()) for c in cs))

def get_elb_usage(regions):
    cs = connect_to_regions(boto.ec2.elb, regions)
    print '{0} Elastic Load Balancer(s)' \
        .format(sum(len(c.get_all_load_balancers()) for c in cs))

def get_vpc_usage(regions):
    cs = connect_to_regions(boto.vpc, regions)
    vpcs = list(itertools.chain.from_iterable([c.get_all_vpcs() for c in cs]))
    print '{0} Virtual Private Cloud(s) [{1} default]\n' \
        '{2} Internet Gateway(s)\n' \
        '{3} Customer Gateway(s)\n' \
        '{4} VPN Gateway(s)\n' \
        '{5} Subnet(s)' \
        .format(len(vpcs), \
            sum(v.is_default for v in vpcs), \
            sum(len(c.get_all_internet_gateways()) for c in cs), \
            sum(len(c.get_all_customer_gateways()) for c in cs), \
            sum(len(c.get_all_vpn_gateways()) for c in cs), \
            sum(len(c.get_all_subnets()) for c in cs))

def get_route53_usage():
    r53 = boto.connect_route53()
    zones = r53.get_zones()
    records = sum(len(z.get_records()) for z in zones)
    print '{0} Route53 Hosted Zone(s) [{1} record(s)]' \
        .format(len(zones), records)

def get_s3_usage():
    s3 = boto.connect_s3()
    buckets = s3.get_all_buckets()
    res = sum([k.size for k in itertools.chain.from_iterable( \
        [b.get_all_keys() for b in buckets])])
    print '{0} S3 Bucket(s) [{1:.3f} GB]' \
        .format(len(buckets), res / float(1024 * 1024 * 1024))

def get_glacier_usage():
    gc = boto.connect_glacier()
    print '{0} Glacier Vault(s)' \
        .format(len(gc.list_vaults()))

def get_cloudfront_usage():
    cf = boto.connect_cloudfront()
    ds = cf.get_all_distributions()
    os = len(list(itertools.chain.from_iterable( \
        [x.get_distribution().get_objects() for x in ds])))
    print '{0} CloudFront Distribution(s){1}' \
        .format(len(ds), ' [{0} object(s)]'.format(os) if 0 != os else '')

def get_sdb_usage(regions):
    cs = connect_to_regions(boto.sdb, regions)
    print '{0} SimpleDB Domain(s)' \
        .format(sum(len(c.get_all_domains()) for c in cs))

def get_rds_usage(regions):
    cs = connect_to_regions(boto.rds2, regions)
    print '{0} RDS Instance(s)\n' \
        '{1} RDS Reserved Instance(s)\n' \
        '{2} RDS Snapshot(s)' \
        .format(sum(len(c.describe_db_instances() \
                ['DescribeDBInstancesResponse'] \
                ['DescribeDBInstancesResult'] \
                ['DBInstances']) for c in cs),
            sum(len(c.describe_reserved_db_instances() \
                ['DescribeReservedDBInstancesResponse'] \
                ['DescribeReservedDBInstancesResult'] \
                ['ReservedDBInstances']) for c in cs),
            sum(len(c.describe_db_snapshots() \
                ['DescribeDBSnapshotsResponse'] \
                ['DescribeDBSnapshotsResult'] \
                ['DBSnapshots']) for c in cs))

def get_dynamodb_usage(regions):
    cs = connect_to_regions(boto.dynamodb2, regions)
    print '{0} DynamoDB Table(s)' \
        .format(sum(len(c.list_tables() \
                ['TableNames']) for c in cs))

def get_elasticache_usage(regions):
    cs = connect_to_regions(boto.elasticache, regions)
    print '{0} ElastiCache Cluster(s)' \
        .format(sum(len(c.describe_cache_clusters() \
                ['DescribeCacheClustersResponse'] \
                ['DescribeCacheClustersResult'] \
                ['CacheClusters']) for c in cs))

def get_redshift_usage(regions):
    cs = connect_to_regions(boto.redshift, regions)
    clusters = list(itertools.chain.from_iterable( \
        [c.describe_clusters() \
            ['DescribeClustersResponse'] \
            ['DescribeClustersResult'] \
            ['Clusters'] for c in cs]))
    print '{0} Redshift Cluster(s)' \
        .format(len(clusters))

def get_emr_usage(regions):
    cs = connect_to_regions(boto.emr, regions)
    clusters = list(itertools.chain.from_iterable( \
        [[c.describe_cluster(s.id)] for c in cs \
        for s in c.list_clusters().clusters]))
    print '{0} EMR Cluster(s)' \
        .format(len(clusters))

def get_kinesis_usage(regions):
    cs = connect_to_regions(boto.kinesis, regions)
    streams = list(itertools.chain.from_iterable( \
        [c.list_streams()['StreamNames'] for c in cs]))
    shards = sum(len(c.describe_stream(s) \
        ['StreamDescription'] \
        ['Shards']) for c in cs for s in streams)
    print '{0} Kinesis Stream(s){1}' \
        .format(len(streams), \
            '[{0} shard(s)]'.format(shards) if 0 != shards else '')

def get_sns_usage(regions):
    cs = connect_to_regions(boto.sns, regions)
    print '{0} SNS Topic(s)\n' \
        '{1} SNS Subscription(s)\n' \
        '{2} SNS Platform Application(s)' \
        .format(sum(len(c.get_all_topics() \
                ['ListTopicsResponse'] \
                ['ListTopicsResult'] \
                ['Topics']) for c in cs),
            sum(len(c.get_all_subscriptions() \
                ['ListSubscriptionsResponse'] \
                ['ListSubscriptionsResult'] \
                ['Subscriptions']) for c in cs), \
            sum(len(c.list_platform_applications() \
                ['ListPlatformApplicationsResponse'] \
                ['ListPlatformApplicationsResult'] \
                ['PlatformApplications']) for c in cs))

def get_sqs_usage(regions):
    cs = connect_to_regions(boto.sqs, regions)
    queues = list(itertools.chain.from_iterable( \
        [c.get_all_queues() for c in cs]))
    messages = sum(q.count() for q in queues)
    print '{0} SQS Queue(s){1}' \
        .format(len(queues), \
            '[{0} message(s)]'.format(messages) if 0 != messages else '')

def get_iam_usage(regions):
    cs = connect_to_regions(boto.iam, regions)
    users = list(itertools.chain.from_iterable([c.get_all_users() \
        ['list_users_response'] \
        ['list_users_result'] \
        ['users'] for c in cs]))
    groups = list(itertools.chain.from_iterable([c.get_all_groups() \
        ['list_groups_response'] \
        ['list_groups_result'] \
        ['groups'] for c in cs]))
    print '{0} IAM User(s)\n' \
        '{1} IAM Group(s)' \
        .format(len(users), len(groups))

def get_cloudwatch_usage(regions):
    cs = connect_to_regions(boto.ec2.cloudwatch, regions)
    alarms = list(itertools.chain.from_iterable( \
        [c.describe_alarms() for c in cs]))
    triggered = sum(a.state_value == MetricAlarm.ALARM for a in alarms)
    print '{0} CloudWatch Alarm(s){1}' \
        .format(len(alarms), \
            '[{0} triggered]'.format(triggered) if 0 != triggered else '')

def get_opsworks_usage():
    ow = boto.connect_opsworks()
    print '{0} OpsWorks Stack(s)' \
        .format(len(ow.describe_stacks()['Stacks']))

def get_aws_cost(bucket_name, time_period):
    s3 = boto.connect_s3()
    bucket = s3.lookup(bucket_name)
    if bucket is None:
        raise Error('could not find \'{0}\''.format(bucket_name))

    period = time_period if time_period is not None \
        else time.strftime('%Y-%m', time.gmtime())

    data = ''
    for key in bucket.list():
        p = re.match(r'(\w+)-aws-billing-csv-{0}.csv' \
            .format(period), key.name)
        if p:
            data = key.get_contents_as_string()
            break
    if not data:
        raise Error('could not find billing data for this month')

    cost = dict()
    total = list()

    doc = csv.reader(data.rstrip('\n').split('\n'), delimiter=',')
    for row in doc:
        code = row[12]
        if code and code != 'ProductCode':
            value = float(row[28])
            if value >= 0:
                if not code in cost:
                    cost[code] = [row[13].split(' ', 1)[1], value, row[23]]
                else:
                    cost[code][1] += value
        if row[3] == 'StatementTotal':
            total.extend([['Cost', float(row[24]), row[23]], \
                ['Credit', float(row[25]), row[23]], \
                ['Total', float(row[28]), row[23]]])

    print '---'
    for k, v in cost.items():
        print '{0:<30} {1:>8.2f} {2}'.format(v[0], v[1], v[2])
    for v in total:
        print '{0:>29}: {1:>8.2f} {2}'.format(v[0], v[1], v[2])

def main():
    parser = optparse.OptionParser('Usage: %prog [options]')
    parser.add_option('-b', '--bucket', dest='bucket',
        help='The name of the S3 bucket that holds billing reports. This '
             'option is required.')
    parser.add_option('-p', '--period', dest='period',
        help='The billing period to check the usage for (e.g., \'2014-02\' '
             'without quotes). Defaults to the current billing period if '
             'not specified.')
    parser.add_option('-r', '--region', dest='regions', action='append',
        help='The name of the region to usage for.')
    (opts, args) = parser.parse_args()

    if len(args) != 0 or \
       opts.bucket is None:
        parser.print_help()
        return 1

    try:
        get_ec2_usage(opts.regions)
        get_autoscale_usage(opts.regions)
        get_elb_usage(opts.regions)
        get_vpc_usage(opts.regions)
        get_route53_usage()

        get_s3_usage()
        get_glacier_usage()
        get_cloudfront_usage()

        get_sdb_usage(opts.regions)
        get_rds_usage(opts.regions)
        get_dynamodb_usage(opts.regions)
        get_elasticache_usage(opts.regions)
        get_redshift_usage(opts.regions)

        get_emr_usage(opts.regions)
        get_kinesis_usage(opts.regions)

        get_sns_usage(opts.regions)
        get_sqs_usage(opts.regions)

        get_cloudwatch_usage(opts.regions)
        get_opsworks_usage()
        get_iam_usage(opts.regions)

        get_aws_cost(opts.bucket, opts.period)
    except (Error, Exception), err:
        sys.stderr.write('[ERROR] {0}\n'.format(err))
        return 1

    return 0

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Better ElastiCache stats.<commit_after>#!/usr/bin/env python
# Copyright (c) 2014 Eugene Zhuk.
# Use of this source code is governed by the MIT license that can be found
# in the LICENSE file.

"""Checks AWS usage.

This script retrieves and displays an estimated total statement amount for
the specified billing period.

Usage:
    ./check_usage.py [options]
"""

import boto.cloudfront
import boto.dynamodb2
import boto.ec2
import boto.ec2.autoscale
import boto.ec2.cloudwatch
import boto.elasticache
import boto.emr
import boto.glacier
import boto.iam
import boto.kinesis
import boto.opsworks
import boto.rds2
import boto.redshift
import boto.route53
import boto.s3
import boto.sns
import boto.sdb
import boto.sqs
import boto.vpc
import csv
import itertools
import optparse
import re
import sys
import time

from boto.ec2.cloudwatch import MetricAlarm

class Error(Exception):
    pass

def connect_to_regions(service, regions):
    if regions is not None:
        return [service.connect_to_region(r.name) for r in service.regions() \
            if r.name in regions]
    else:
        return [service.connect_to_region(r.name) for r in service.regions() \
            if not (r.name.startswith('cn-') or r.name.startswith('us-gov-'))]

def get_ec2_usage(regions):
    cs = connect_to_regions(boto.ec2, regions)
    instances = list(itertools.chain.from_iterable([x.instances \
        for c in cs for x in c.get_all_reservations()]))
    running = sum(16 == i.state_code for i in instances)
    volumes = list(itertools.chain.from_iterable([c.get_all_volumes() \
        for c in cs]))
    size = sum(v.size for v in volumes)
    addresses = list(itertools.chain.from_iterable([c.get_all_addresses() \
        for c in cs]))
    unassigned = sum(a.instance_id is None for a in addresses)
    print '{0} EC2 Instance(s){1}\n' \
        '{2} EC2 Reserved Instance(s)\n' \
        '{3} EC2 Spot Instance Request(s)\n' \
        '{4} EBS Volume(s){5}\n' \
        '{6} EBS Snapshot(s)\n' \
        '{7} Amazon Machine Image(s)\n' \
        '{8} Network Interface(s)\n' \
        '{9} Elastic IP Address(es){10}\n' \
        '{11} Security Group(s)\n' \
        '{12} Key Pair(s)\n' \
        '{13} Tag(s)' \
        .format(len(instances), \
            '[{0} running]'.format(running) if 0 != running else '', \
            sum(len(c.get_all_reserved_instances()) for c in cs), \
            sum(len(c.get_all_spot_instance_requests()) for c in cs), \
            len(volumes), \
            '[{0} GB]'.format(size) if 0 != size else '', \
            sum(len(c.get_all_snapshots(owner=['self'])) for c in cs), \
            sum(len(c.get_all_images(owners=['self'])) for c in cs), \
            sum(len(c.get_all_network_interfaces()) for c in cs), \
            len(addresses), \
            '[{0} unassigned]'.format(unassigned) if 0 != unassigned else '', \
            sum(len(c.get_all_security_groups()) for c in cs), \
            sum(len(c.get_all_key_pairs()) for c in cs), \
            sum(len(c.get_all_tags()) for c in cs))

def get_autoscale_usage(regions):
    cs = connect_to_regions(boto.ec2.autoscale, regions)
    print '{0} Auto Scaling Group(s)\n' \
        '{1} Auto Scaling Launch Configuration(s)\n' \
        '{2} Auto Scaling Policie(s)' \
        .format(sum(len(c.get_all_groups()) for c in cs), \
            sum(len(c.get_all_launch_configurations()) for c in cs), \
            sum(len(c.get_all_policies()) for c in cs))

def get_elb_usage(regions):
    cs = connect_to_regions(boto.ec2.elb, regions)
    print '{0} Elastic Load Balancer(s)' \
        .format(sum(len(c.get_all_load_balancers()) for c in cs))

def get_vpc_usage(regions):
    cs = connect_to_regions(boto.vpc, regions)
    vpcs = list(itertools.chain.from_iterable([c.get_all_vpcs() for c in cs]))
    print '{0} Virtual Private Cloud(s) [{1} default]\n' \
        '{2} Internet Gateway(s)\n' \
        '{3} Customer Gateway(s)\n' \
        '{4} VPN Gateway(s)\n' \
        '{5} Subnet(s)' \
        .format(len(vpcs), \
            sum(v.is_default for v in vpcs), \
            sum(len(c.get_all_internet_gateways()) for c in cs), \
            sum(len(c.get_all_customer_gateways()) for c in cs), \
            sum(len(c.get_all_vpn_gateways()) for c in cs), \
            sum(len(c.get_all_subnets()) for c in cs))

def get_route53_usage():
    r53 = boto.connect_route53()
    zones = r53.get_zones()
    records = sum(len(z.get_records()) for z in zones)
    print '{0} Route53 Hosted Zone(s) [{1} record(s)]' \
        .format(len(zones), records)

def get_s3_usage():
    s3 = boto.connect_s3()
    buckets = s3.get_all_buckets()
    res = sum([k.size for k in itertools.chain.from_iterable( \
        [b.get_all_keys() for b in buckets])])
    print '{0} S3 Bucket(s) [{1:.3f} GB]' \
        .format(len(buckets), res / float(1024 * 1024 * 1024))

def get_glacier_usage():
    gc = boto.connect_glacier()
    print '{0} Glacier Vault(s)' \
        .format(len(gc.list_vaults()))

def get_cloudfront_usage():
    cf = boto.connect_cloudfront()
    ds = cf.get_all_distributions()
    os = len(list(itertools.chain.from_iterable( \
        [x.get_distribution().get_objects() for x in ds])))
    print '{0} CloudFront Distribution(s){1}' \
        .format(len(ds), ' [{0} object(s)]'.format(os) if 0 != os else '')

def get_sdb_usage(regions):
    cs = connect_to_regions(boto.sdb, regions)
    print '{0} SimpleDB Domain(s)' \
        .format(sum(len(c.get_all_domains()) for c in cs))

def get_rds_usage(regions):
    cs = connect_to_regions(boto.rds2, regions)
    print '{0} RDS Instance(s)\n' \
        '{1} RDS Reserved Instance(s)\n' \
        '{2} RDS Snapshot(s)' \
        .format(sum(len(c.describe_db_instances() \
                ['DescribeDBInstancesResponse'] \
                ['DescribeDBInstancesResult'] \
                ['DBInstances']) for c in cs),
            sum(len(c.describe_reserved_db_instances() \
                ['DescribeReservedDBInstancesResponse'] \
                ['DescribeReservedDBInstancesResult'] \
                ['ReservedDBInstances']) for c in cs),
            sum(len(c.describe_db_snapshots() \
                ['DescribeDBSnapshotsResponse'] \
                ['DescribeDBSnapshotsResult'] \
                ['DBSnapshots']) for c in cs))

def get_dynamodb_usage(regions):
    cs = connect_to_regions(boto.dynamodb2, regions)
    print '{0} DynamoDB Table(s)' \
        .format(sum(len(c.list_tables() \
                ['TableNames']) for c in cs))

def get_elasticache_usage(regions):
    cs = connect_to_regions(boto.elasticache, regions)
    clusters = list(itertools.chain.from_iterable( \
        [c.describe_cache_clusters() \
            ['DescribeCacheClustersResponse'] \
            ['DescribeCacheClustersResult'] \
            ['CacheClusters'] for c in cs]))
    print '{0} ElastiCache Cluster(s)' \
        .format(len(clusters))

def get_redshift_usage(regions):
    cs = connect_to_regions(boto.redshift, regions)
    clusters = list(itertools.chain.from_iterable( \
        [c.describe_clusters() \
            ['DescribeClustersResponse'] \
            ['DescribeClustersResult'] \
            ['Clusters'] for c in cs]))
    print '{0} Redshift Cluster(s)' \
        .format(len(clusters))

def get_emr_usage(regions):
    cs = connect_to_regions(boto.emr, regions)
    clusters = list(itertools.chain.from_iterable( \
        [[c.describe_cluster(s.id)] for c in cs \
        for s in c.list_clusters().clusters]))
    print '{0} EMR Cluster(s)' \
        .format(len(clusters))

def get_kinesis_usage(regions):
    cs = connect_to_regions(boto.kinesis, regions)
    streams = list(itertools.chain.from_iterable( \
        [c.list_streams()['StreamNames'] for c in cs]))
    shards = sum(len(c.describe_stream(s) \
        ['StreamDescription'] \
        ['Shards']) for c in cs for s in streams)
    print '{0} Kinesis Stream(s){1}' \
        .format(len(streams), \
            '[{0} shard(s)]'.format(shards) if 0 != shards else '')

def get_sns_usage(regions):
    cs = connect_to_regions(boto.sns, regions)
    print '{0} SNS Topic(s)\n' \
        '{1} SNS Subscription(s)\n' \
        '{2} SNS Platform Application(s)' \
        .format(sum(len(c.get_all_topics() \
                ['ListTopicsResponse'] \
                ['ListTopicsResult'] \
                ['Topics']) for c in cs),
            sum(len(c.get_all_subscriptions() \
                ['ListSubscriptionsResponse'] \
                ['ListSubscriptionsResult'] \
                ['Subscriptions']) for c in cs), \
            sum(len(c.list_platform_applications() \
                ['ListPlatformApplicationsResponse'] \
                ['ListPlatformApplicationsResult'] \
                ['PlatformApplications']) for c in cs))

def get_sqs_usage(regions):
    cs = connect_to_regions(boto.sqs, regions)
    queues = list(itertools.chain.from_iterable( \
        [c.get_all_queues() for c in cs]))
    messages = sum(q.count() for q in queues)
    print '{0} SQS Queue(s){1}' \
        .format(len(queues), \
            '[{0} message(s)]'.format(messages) if 0 != messages else '')

def get_iam_usage(regions):
    cs = connect_to_regions(boto.iam, regions)
    users = list(itertools.chain.from_iterable([c.get_all_users() \
        ['list_users_response'] \
        ['list_users_result'] \
        ['users'] for c in cs]))
    groups = list(itertools.chain.from_iterable([c.get_all_groups() \
        ['list_groups_response'] \
        ['list_groups_result'] \
        ['groups'] for c in cs]))
    print '{0} IAM User(s)\n' \
        '{1} IAM Group(s)' \
        .format(len(users), len(groups))

def get_cloudwatch_usage(regions):
    cs = connect_to_regions(boto.ec2.cloudwatch, regions)
    alarms = list(itertools.chain.from_iterable( \
        [c.describe_alarms() for c in cs]))
    triggered = sum(a.state_value == MetricAlarm.ALARM for a in alarms)
    print '{0} CloudWatch Alarm(s){1}' \
        .format(len(alarms), \
            '[{0} triggered]'.format(triggered) if 0 != triggered else '')

def get_opsworks_usage():
    ow = boto.connect_opsworks()
    print '{0} OpsWorks Stack(s)' \
        .format(len(ow.describe_stacks()['Stacks']))

def get_aws_cost(bucket_name, time_period):
    s3 = boto.connect_s3()
    bucket = s3.lookup(bucket_name)
    if bucket is None:
        raise Error('could not find \'{0}\''.format(bucket_name))

    period = time_period if time_period is not None \
        else time.strftime('%Y-%m', time.gmtime())

    data = ''
    for key in bucket.list():
        p = re.match(r'(\w+)-aws-billing-csv-{0}.csv' \
            .format(period), key.name)
        if p:
            data = key.get_contents_as_string()
            break
    if not data:
        raise Error('could not find billing data for this month')

    cost = dict()
    total = list()

    doc = csv.reader(data.rstrip('\n').split('\n'), delimiter=',')
    for row in doc:
        code = row[12]
        if code and code != 'ProductCode':
            value = float(row[28])
            if value >= 0:
                if not code in cost:
                    cost[code] = [row[13].split(' ', 1)[1], value, row[23]]
                else:
                    cost[code][1] += value
        if row[3] == 'StatementTotal':
            total.extend([['Cost', float(row[24]), row[23]], \
                ['Credit', float(row[25]), row[23]], \
                ['Total', float(row[28]), row[23]]])

    print '---'
    for k, v in cost.items():
        print '{0:<30} {1:>8.2f} {2}'.format(v[0], v[1], v[2])
    for v in total:
        print '{0:>29}: {1:>8.2f} {2}'.format(v[0], v[1], v[2])

def main():
    parser = optparse.OptionParser('Usage: %prog [options]')
    parser.add_option('-b', '--bucket', dest='bucket',
        help='The name of the S3 bucket that holds billing reports. This '
             'option is required.')
    parser.add_option('-p', '--period', dest='period',
        help='The billing period to check the usage for (e.g., \'2014-02\' '
             'without quotes). Defaults to the current billing period if '
             'not specified.')
    parser.add_option('-r', '--region', dest='regions', action='append',
        help='The name of the region to usage for.')
    (opts, args) = parser.parse_args()

    if len(args) != 0 or \
       opts.bucket is None:
        parser.print_help()
        return 1

    try:
        get_ec2_usage(opts.regions)
        get_autoscale_usage(opts.regions)
        get_elb_usage(opts.regions)
        get_vpc_usage(opts.regions)
        get_route53_usage()

        get_s3_usage()
        get_glacier_usage()
        get_cloudfront_usage()

        get_sdb_usage(opts.regions)
        get_rds_usage(opts.regions)
        get_dynamodb_usage(opts.regions)
        get_elasticache_usage(opts.regions)
        get_redshift_usage(opts.regions)

        get_emr_usage(opts.regions)
        get_kinesis_usage(opts.regions)

        get_sns_usage(opts.regions)
        get_sqs_usage(opts.regions)

        get_cloudwatch_usage(opts.regions)
        get_opsworks_usage()
        get_iam_usage(opts.regions)

        get_aws_cost(opts.bucket, opts.period)
    except (Error, Exception), err:
        sys.stderr.write('[ERROR] {0}\n'.format(err))
        return 1

    return 0

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Once the analysis of a folder has been done, we use the TarToArchive.py script
to pack it up and sent to the PSI tape archive, the DarkDeleter.py script is
then used to delete all the unnecessary files.

If you want to get some stuff back, it's annoying manual work to go in each
folder and unpack the tar files.

This script should alleviate this problem.
"""

import os
import subprocess
import sys
import fnmatch
import glob

from functions import get_experiment_list
from functions import get_git_hash
from functions import myLogger

RootFolder = ('/afs/psi.ch/project/EssentialMed/MasterArbeitBFH/' +
    'XrayImages/20140920')

ListOfTARs = []
for root, dirnames, filenames in os.walk(os.path.join(RootFolder)):
    for filename in fnmatch.filter(filenames, '*.gz'):
        ListOfTARs.append(os.path.join(root, filename))

if not ListOfTARs:
    print 'Nothing to do!'

for counter, item in enumerate(ListOfTARs):
    print counter, 'of', len(ListOfTARs), '| unpacking', \
        item[len(RootFolder) + 1:]
    UnpackCommand = ['tar', '-xf', item, '--directory', os.path.dirname(item)]
    unpackit = subprocess.Popen(UnpackCommand, stdout=subprocess.PIPE)
    output, error = unpackit.communicate()
    if output:
        print output
    if error:
        print error
    print 'Unpacking done, now removing', item[len(RootFolder) + 1:],
    try:
        os.remove(item)
    except:
        print 'It is already gone!'
    ExperimentID = os.path.splitext(os.path.splitext(item)[0])[0]
    DeletionLog = ExperimentID + '.deletion.log'
    print 'as well as', os.path.basename(DeletionLog), \
        'and results from Analyis (all', \
        os.path.basename(ExperimentID) + '*.png)'
    try:
        os.remove(DeletionLog)
        for i in glob.glob(ExperimentID + '*.png'):
            try:
                os.remove(i)
            except:
                print 'The analysis images are already gone!'
    except:
            print 'The log file is already gone!'
    print 80 * '-'
print 'Fertig'
<commit_msg>Search everywhere<commit_after># -*- coding: utf-8 -*-

"""
Once the analysis of a folder has been done, we use the TarToArchive.py script
to pack it up and sent to the PSI tape archive, the DarkDeleter.py script is
then used to delete all the unnecessary files.

If you want to get some stuff back, it's annoying manual work to go in each
folder and unpack the tar files.

This script should alleviate this problem.
"""

import os
import subprocess
import sys
import fnmatch
import glob

from functions import get_experiment_list
from functions import get_git_hash
from functions import myLogger

RootFolder = ('/afs/psi.ch/project/EssentialMed/MasterArbeitBFH/' +
    'XrayImages')

ListOfTARs = []
for root, dirnames, filenames in os.walk(os.path.join(RootFolder)):
    for filename in fnmatch.filter(filenames, '*.gz'):
        ListOfTARs.append(os.path.join(root, filename))

if not ListOfTARs:
    print 'Nothing to do!'

for counter, item in enumerate(ListOfTARs):
    print counter, 'of', len(ListOfTARs), '| unpacking', \
        item[len(RootFolder) + 1:]
    UnpackCommand = ['tar', '-xf', item, '--directory', os.path.dirname(item)]
    unpackit = subprocess.Popen(UnpackCommand, stdout=subprocess.PIPE)
    output, error = unpackit.communicate()
    if output:
        print output
    if error:
        print error
    print 'Unpacking done, now removing', item[len(RootFolder) + 1:],
    try:
        os.remove(item)
    except:
        print 'It is already gone!'
    ExperimentID = os.path.splitext(os.path.splitext(item)[0])[0]
    DeletionLog = ExperimentID + '.deletion.log'
    print 'as well as', os.path.basename(DeletionLog), \
        'and results from Analyis (all', \
        os.path.basename(ExperimentID) + '*.png)'
    try:
        os.remove(DeletionLog)
        for i in glob.glob(ExperimentID + '*.png'):
            try:
                os.remove(i)
            except:
                print 'The analysis images are already gone!'
    except:
            print 'The log file is already gone!'
    print 80 * '-'
print 'Fertig'
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding:utf8 -*-
# Copyright (C) 2012  FinalsClub Foundation
"""Before running add the downloaded docs to this folder:
bin/import_json/downloaded_notes/

Run from the django shell with:
import bin.import_json.run
"""

import json
import os
import shutil
import datetime
import taggit
import re

from os.path import join, exists, split, isdir

from django.core.files import File as DjangoFile
from django.template.defaultfilters import slugify

from apps.courses.models import *
from apps.notes.models import *

# read json files
with open('bin/import_json/schools.json', 'r') as f:
	school_dicts = json.load(f)

with open('bin/import_json/notes.json', 'r') as f:
	note_dicts = json.load(f)

with open('bin/import_json/courses.json', 'r') as f:
	course_dicts = json.load(f)

print 'Schools found:', len(school_dicts)
print 'Notes found:', len(note_dicts)
print 'Courses found:', len(course_dicts)

def fix_file_path(note_dict):
	"""
	The old file path was an absolute path on the server:
	/var/www/uploads/filename.txt

	Look for those files in this folder:
	bin/import_json/downloaded_notes

	update the note_dictionary's file_path key with the new spec
	"""
	path, name = split(note_dict['file_path'])

	# If we have the file, copy it to the new location
	new_filename = join('bin/import_json/downloaded_notes', name)

	if exists(new_filename) and name:
		note_dict['file_path'] = new_filename

for note in note_dicts:
	fix_file_path(note)

# Import from json
print 'updating %i schools' % len(school_dicts)
for school in school_dicts:
	s = School(**school)
	s.save()

print 'updating %i courses' % len(course_dicts)
for course in course_dicts:
	course['updated_at'] = datetime.datetime.utcnow()
	course['created_at'] = datetime.datetime.utcnow()

	# Somc courses have no school_id using arbitrary one for these
	if not course['school_id']:
		print 'Using arbitrary school_id for course id:', course['id'], '-', course['name']
		course['school_id'] = School.objects.all()[0].id

	c = Course(**course)
	c.save()

# Import the Notes
print 'updating %i notes' % len(note_dicts)
for note in note_dicts:

	if not note['course_id']:
		print 'using arbitrary course id for note_id:', note['id'], '-', note['name']
		note['course_id'] = Course.objects.all()[0].id

	if 'slug' not in note:
		path, fn = split(note['file_path'])
		note['slug'] = slugify(fn)

	course = Course.objects.get(id=note['course_id'])
	if Note.objects.filter(slug=note['slug']).filter(course__slug=course.slug).exists():
		print 'note_slug and course slug not unique together. skipping note with id:', note['id'], '-', note['name']
		continue

	# These keys cannot be pased as keyword arguments
	tags = note['tags']
	del note['tags']
	file_path = note['file_path']
	del note['file_path']

	# replace the string with this value
	note['uploaded_at'] = datetime.datetime.utcnow()

	try:
		n = Note(**note)

		# fix the weird javascript issue
		if n.html:
			n.html = re.sub(r'\\(.)', r'\1', n.html)
			n.html = re.sub(r'\\\n', r'', n.html)

		# Add the tags, if any
		for t in tags: n.tags.add(t)

		# double check if the file exists before trying to open file
		if exists(file_path):
			with open(file_path) as f:
				df = DjangoFile(f)
				dir, filename = split(file_path)
				print 'copying file', note['id'], ' - ', filename
				n.note_file.save(join('imported', filename), df)

		n.save()

	except (TypeError, ) as error:
		print '\nError found in note:', error
		print note['name']

<commit_msg>tweaking json db import script<commit_after>#!/usr/bin/env python
# -*- coding:utf8 -*-
# Copyright (C) 2012  FinalsClub Foundation
"""Before running add the downloaded docs to this folder:
bin/import_json/downloaded_notes/

Run from the django shell with:
import bin.import_json.run
"""

import json
import os
import shutil
import datetime
import taggit
import re

from os.path import join, exists, split, isdir

from django.core.files import File as DjangoFile
from django.template.defaultfilters import slugify

from apps.courses.models import *
from apps.notes.models import *

# read json files
with open('bin/import_json/schools.json', 'r') as f:
	school_dicts = json.load(f)

with open('bin/import_json/notes.json', 'r') as f:
	note_dicts = json.load(f)

with open('bin/import_json/courses.json', 'r') as f:
	course_dicts = json.load(f)

print 'Schools found:', len(school_dicts)
print 'Notes found:', len(note_dicts)
print 'Courses found:', len(course_dicts)

def fix_file_path(note_dict):
	"""
	The old file path was an absolute path on the server:
	/var/www/uploads/filename.txt

	Look for those files in this folder:
	bin/import_json/downloaded_notes

	update the note_dictionary's file_path key with the new spec
	"""
	path, name = split(note_dict['file_path'])

	# If we have the file, copy it to the new location
	new_filename = join('bin/import_json/downloaded_notes', name)

	if exists(new_filename) and name:
		note_dict['file_path'] = new_filename

for note in note_dicts:
	fix_file_path(note)

# Import from json
print 'updating %i schools' % len(school_dicts)
for school in school_dicts:
	s = School(**school)
	s.save()

# Only Save this scool if we actually need it.
arbitrary_school = School(name='No School', slug='no_school')

print 'updating %i courses' % len(course_dicts)
for course in course_dicts:
	course['updated_at'] = datetime.datetime.utcnow()
	course['created_at'] = datetime.datetime.utcnow()

	# Somc courses have no school_id using arbitrary one for these
	if not course['school_id']:
		print 'Using arbitrary school_id for course id:', course['id'], '-', course['name']
		course['school_id'] = arbitrary_school.id
		arbitrary_school.save()

	c = Course(**course)
	c.save()

# Import the Notes
print 'updating %i notes' % len(note_dicts)

arbitrary_course = Course(name='No Course', slug='no_course', school=arbitrary_school)

for note in note_dicts:

	if not note['course_id']:
		print 'using arbitrary course id for note_id:', note['id'], '-', note['name']
		note['course_id'] = arbitrary_course.id
		arbitrary_school.save()
		arbitrary_course.save()

	if 'slug' not in note:
		path, fn = split(note['file_path'])
		note['slug'] = slugify(fn)

	course = Course.objects.get(id=note['course_id'])
	if Note.objects.filter(slug=note['slug']).filter(course__slug=course.slug).exists():
		print 'note_slug and course slug not unique together. skipping note with id:', note['id'], '-', note['name']
		continue

	# These keys cannot be pased as keyword arguments
	tags = note['tags']
	del note['tags']
	file_path = note['file_path']
	del note['file_path']

	# replace the string with this value
	note['uploaded_at'] = datetime.datetime.utcnow()

	try:
		n = Note(**note)

		# fix the weird javascript issue
		if n.html:
			n.html = re.sub(r'\\(.)', r'\1', n.html)
			n.html = re.sub(r'\\\n', r'', n.html)

		# Add the tags, if any
		for t in tags: n.tags.add(t)

		# double check if the file exists before trying to open file
		if exists(file_path):
			with open(file_path) as f:
				df = DjangoFile(f)
				dir, filename = split(file_path)
				print 'copying file', note['id'], ' - ', filename
				n.note_file.save(join('imported', filename), df)

		n.save()

	except (TypeError, ) as error:
		print '\nError found in note:', error
		print note['name']

<|endoftext|>


<commit_before>from collections import namedtuple, OrderedDict, defaultdict

from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import models
from django.utils import timezone
from django.utils.functional import cached_property
from django.utils.translation import ugettext_lazy as _

from data_sources.models import DataSource

class Experiment(models.Model):
    """A model to store relations between data sources.

    """
    owner = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        on_delete=models.CASCADE,
        related_name='experiments',
    )

    name = models.CharField(
        max_length=512,
        verbose_name=_("name"),
    )

    description = models.TextField(
        blank=True,
        verbose_name=_("description"),
        help_text=_(
            "Description about this experiment. Can be written in "
            "Markdown markup language."
        ),
    )

    data_sources = models.ManyToManyField(
        DataSource, through="Condition",
    )

    date_created = models.DateTimeField(
        verbose_name=_('date created'),
        default=timezone.now,
        editable=False,
    )

    @cached_property
    def sample_names(self):
        """Return all distinct sample names"""
        return (
            self.conditions
                .values_list('sample_name', flat=True)
                .distinct()
                .order_by('sample_name')
        )

    @cached_property
    def condition_names(self):
        """Return all distinct condition names

        Note that condition (ALL) is ignored.
        """
        return (
            self.conditions
                .filter(condition_order__gt=0)
                .values_list('condition', flat=True)
                .distinct()
                .order_by('condition_order')
        )

    @cached_property
    def group_data_sources(self):
        """
        Return dict of condition to samples to data sources.
        """
        grouped_data_sources = OrderedDict()

        conditions = (
            self.conditions
                .select_related('data_source')
                .order_by("condition_order", "sample_name")
        )
        for condition in conditions:
            cond_lab = condition.condition_label
            sample = condition.sample_name
            if cond_lab not in grouped_data_sources:
                grouped_data_sources[cond_lab] = defaultdict(list)
            grouped_data_sources[cond_lab][sample].append(condition)
        # If we do not disable the default_factory, it cannot be rendered
        # correctly in templates.
        # Ref: http://stackoverflow.com/a/12842716
        for samples in grouped_data_sources.values():
            samples.default_factory = None
        return grouped_data_sources

    def get_absolute_url(self):
        return reverse('detail_experiment', kwargs={'pk': self.pk})

    @cached_property
    def summary(self):
        return (
            '{name:s} was created by {owner:s} involving '
            '{num_sources:d} data sources. '
            'It defined {num_sample:d} samples: {samples:s}, '
            'and defined {num_condition:d} conditions: {conditions:s}.'
            .format(
                owner=self.owner.name,
                name=self.name,
                num_sample=len(self.sample_names),
                samples=', '.join(self.sample_names),
                num_sources=self.conditions.count(),
                conditions=', '.join(self.condition_names),
                num_condition=len(self.condition_names),
            )
        )

    def __str__(self):
        return (
            '{owner:s}\'s {name:s} '
            '(involving {num_sources:d} data sources '
            'grouped as {num_sample:d} samples '
            'and {num_condition:d} conditions)'
            .format(
                owner=self.owner.name,
                name=self.name,
                num_sample=len(self.sample_names),
                num_sources=self.conditions.count(),
                num_condition=len(self.condition_names),
            )
        )

    class Meta:
        get_latest_by = "date_created"
        ordering = ['-date_created']

ConditionLabel = namedtuple('ConditionLabel', ['order', 'label'])

class Condition(models.Model):
    """Intermediary model to connect experiments and data sources.

    """
    STRAND_CHOICES = (
        ("1", _('R1 (forward)')),
        ("2", _('R2 (reversed)')),
    )

    experiment = models.ForeignKey(
        Experiment,
        related_name="conditions",
        on_delete=models.CASCADE,
    )

    condition = models.CharField(
        max_length=512,
        help_text=_("Name of the condition. (All) is a special condition "
                    "which implies this data source appears in every "
                    "condition.")
    )

    condition_order = models.IntegerField(
        help_text=_("Order of the condition. 0 should always represent the "
                    "special condition (All).")
    )

    data_source = models.ForeignKey(
        DataSource,
        related_name="exp_conditions",
        on_delete=models.CASCADE,
    )

    sample_name = models.CharField(
        max_length=512,
        blank=True,
        verbose_name=_("sample name"),
    )

    strand = models.CharField(
        choices=STRAND_CHOICES,
        max_length=4,
        blank=True,
        verbose_name=_("read strand"),
        help_text=_(
            "Strand for the sequencing type of data sources. Usually the file "
            "is sufficed with R1 (forward) or R2 (reversed). If the data "
            "source is not stranded or not of sequencing type, leave this "
            "field blank."
        )
    )

    class Meta:
        unique_together = ("experiment", "condition", "data_source")

    def __str__(self):
        return (
            '{condition:s} ({source_file_path:} of {sample_name:s})'
            .format(
                condition=self.condition,
                sample_name=self.sample_name,
                source_file_path=self.data_source.get_rel_file_path(),
            )
        )

    @property
    def condition_label(self):
        """Return a ConditionLabel tuple"""
        return ConditionLabel(self.condition_order, self.condition)
<commit_msg>Always sort by strand order<commit_after>from collections import namedtuple, OrderedDict, defaultdict

from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import models
from django.utils import timezone
from django.utils.functional import cached_property
from django.utils.translation import ugettext_lazy as _

from data_sources.models import DataSource

class Experiment(models.Model):
    """A model to store relations between data sources.

    """
    owner = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        on_delete=models.CASCADE,
        related_name='experiments',
    )

    name = models.CharField(
        max_length=512,
        verbose_name=_("name"),
    )

    description = models.TextField(
        blank=True,
        verbose_name=_("description"),
        help_text=_(
            "Description about this experiment. Can be written in "
            "Markdown markup language."
        ),
    )

    data_sources = models.ManyToManyField(
        DataSource, through="Condition",
    )

    date_created = models.DateTimeField(
        verbose_name=_('date created'),
        default=timezone.now,
        editable=False,
    )

    @cached_property
    def sample_names(self):
        """Return all distinct sample names"""
        return (
            self.conditions
                .values_list('sample_name', flat=True)
                .distinct()
                .order_by('sample_name')
        )

    @cached_property
    def condition_names(self):
        """Return all distinct condition names

        Note that condition (ALL) is ignored.
        """
        return (
            self.conditions
                .filter(condition_order__gt=0)
                .values_list('condition', flat=True)
                .distinct()
                .order_by('condition_order')
        )

    @cached_property
    def group_data_sources(self):
        """
        Return dict of condition to samples to data sources.
        """
        grouped_data_sources = OrderedDict()

        conditions = (
            self.conditions
                .select_related('data_source')
                .order_by("condition_order", "sample_name", "strand")
        )
        for condition in conditions:
            cond_lab = condition.condition_label
            sample = condition.sample_name
            if cond_lab not in grouped_data_sources:
                grouped_data_sources[cond_lab] = defaultdict(list)
            grouped_data_sources[cond_lab][sample].append(condition)
        # If we do not disable the default_factory, it cannot be rendered
        # correctly in templates.
        # Ref: http://stackoverflow.com/a/12842716
        for samples in grouped_data_sources.values():
            samples.default_factory = None
        return grouped_data_sources

    def get_absolute_url(self):
        return reverse('detail_experiment', kwargs={'pk': self.pk})

    @cached_property
    def summary(self):
        return (
            '{name:s} was created by {owner:s} involving '
            '{num_sources:d} data sources. '
            'It defined {num_sample:d} samples: {samples:s}, '
            'and defined {num_condition:d} conditions: {conditions:s}.'
            .format(
                owner=self.owner.name,
                name=self.name,
                num_sample=len(self.sample_names),
                samples=', '.join(self.sample_names),
                num_sources=self.conditions.count(),
                conditions=', '.join(self.condition_names),
                num_condition=len(self.condition_names),
            )
        )

    def __str__(self):
        return (
            '{owner:s}\'s {name:s} '
            '(involving {num_sources:d} data sources '
            'grouped as {num_sample:d} samples '
            'and {num_condition:d} conditions)'
            .format(
                owner=self.owner.name,
                name=self.name,
                num_sample=len(self.sample_names),
                num_sources=self.conditions.count(),
                num_condition=len(self.condition_names),
            )
        )

    class Meta:
        get_latest_by = "date_created"
        ordering = ['-date_created']

ConditionLabel = namedtuple('ConditionLabel', ['order', 'label'])

class Condition(models.Model):
    """Intermediary model to connect experiments and data sources.

    """
    STRAND_CHOICES = (
        ("1", _('R1 (forward)')),
        ("2", _('R2 (reversed)')),
    )

    experiment = models.ForeignKey(
        Experiment,
        related_name="conditions",
        on_delete=models.CASCADE,
    )

    condition = models.CharField(
        max_length=512,
        help_text=_("Name of the condition. (All) is a special condition "
                    "which implies this data source appears in every "
                    "condition.")
    )

    condition_order = models.IntegerField(
        help_text=_("Order of the condition. 0 should always represent the "
                    "special condition (All).")
    )

    data_source = models.ForeignKey(
        DataSource,
        related_name="exp_conditions",
        on_delete=models.CASCADE,
    )

    sample_name = models.CharField(
        max_length=512,
        blank=True,
        verbose_name=_("sample name"),
    )

    strand = models.CharField(
        choices=STRAND_CHOICES,
        max_length=4,
        blank=True,
        verbose_name=_("read strand"),
        help_text=_(
            "Strand for the sequencing type of data sources. Usually the file "
            "is sufficed with R1 (forward) or R2 (reversed). If the data "
            "source is not stranded or not of sequencing type, leave this "
            "field blank."
        )
    )

    class Meta:
        unique_together = ("experiment", "condition", "data_source")

    def __str__(self):
        return (
            '{condition:s} ({source_file_path:} of {sample_name:s})'
            .format(
                condition=self.condition,
                sample_name=self.sample_name,
                source_file_path=self.data_source.get_rel_file_path(),
            )
        )

    @property
    def condition_label(self):
        """Return a ConditionLabel tuple"""
        return ConditionLabel(self.condition_order, self.condition)
<|endoftext|>


<commit_before>import numpy as np

from test import *  # noqa
import stella

class B(object):
    x = 0
    y = 0

    def __init__(self, x=1, y=2):
        self.x = x
        self.y = y

    def __eq__(self, other):
        return self.x == other.x and self.y == other.y

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        return "{}:{}, {}>".format(str(type(self))[:-1], self.x, self.y)

class C(object):
    """
    %"class 'test.objects.C'_<Int*6>_Int" = type { [6 x i64]*, i64 }
    """
    def __init__(self, obj, i=0):
        if isinstance(obj, int):
            self.a = np.zeros(shape=obj, dtype=int)
            self.a[0] = 42
        else:
            self.a = np.array(obj)
        self.i = i

    def __eq__(self, other):
        return self.i == other.i and (self.a == other.a).all()

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        return "{}:{}, {}>".format(str(type(self))[:-1], self.i, self.a)

class D(object):
    z = 0
    a = 0
    y = 0.0
    g = 0.0

    def __init__(self):
        pass

    def __eq__(self, other):
        return (self.z == other.z and
                self.a == other.a and
                self.y == other.y and
                self.g == other.g)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        return "{}: {}>".format(str(type(self))[:-1], [self.z, self.a, self.y, self.g])

class E(object):
    def __init__(self, x=0):
        self.x = x

    def inc(self, p=1):
        self.x += p
        return self.x

    def __eq__(self, other):
        return (self.x == other.x)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        return "{}[x={}]".format(str(type(self))[8:-2], self.x)

class F(object):
    def __init__(self, l):
        self.l = l

    def __eq__(self, other):
        return (self.l == other.l)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        return "{}[l={}]".format(str(type(self))[8:-2], self.l)

class G(B):
    def __init__(self, x=1, y=2):
        super().__init__(x, y)

G.origin = G(0, 0)

def justPassing(a):
    x = 1  # noqa

def cmpAttrib(a):
    return a.x == a.y

def setAttrib(a):
    a.x = 42

def setAttribFloat(a):
    a.x = 42.0

def setUnknownAttrib(a):
    a.z = 42

def getAttrib(a):
    return a.x

def addAttribs(a):
    return a.x + a.y

def returnUnknownAttrib(a):
    return a.z

def getAndSetAttrib1(a):
    a.x *= a.y

def getAndSetAttrib2(a):
    a.x -= 1

def callBoundMethod(e):
    e.inc()
    return e.x

def callBoundMethod2(e):
    e.inc(42)

def callBoundMethod3(e, x):
    e.inc(x)

def callBoundMethodTwice(e, x):
    e.inc(x)
    e.inc(x)

def callBoundMethodOnTwo(e1, e2):
    e1.inc(1)
    e2.inc(2)

def objList1(l):
    return l[0].x + l[1].x

def objList2(l):
    r = 0
    for i in range(len(l)):
        r += l[i].x
    return r

def objList3(l):
    r = 0
    for i in range(len(l)):
        for j in range(len(l)):
            r += l[j].x + i
    return r

def objList4(l):
    for i in range(len(l)):
        l[i].x = i

def objContainingList1(f):
    return f.l[0].x + f.l[1].x

def objContainingList2(f):
    r = 0
    for i in range(len(f.l)):
        r += f.l[i].x
    return r

def objContainingList3(f):
    for i in range(len(f.l)):
        f.l[i].x = i

def selfRef(g):
    return ((g.x - G.origin.x)**2 + (g.y - G.origin.y)**2)**0.5

def nextB(b):
    return b.x == b.next.x and b.y == b.next.y

args1 = [(1, 1), (24, 42), (0.0, 1.0), (1.0, 1.0), (3.0, 0.0)]

def getFirstArrayValue(c):
    return c.a[0]

def getSomeArrayValue(c, i):
    return c.a[i]

def sumC(c):
    for i in range(len(c.a)):
        c.i += c.a[i]

@mark.parametrize('f', [justPassing, addAttribs, getAttrib])
@mark.parametrize('args', args1)
def test_no_mutation(f, args):
    b1 = B(*args)
    b2 = B(*args)

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

@mark.parametrize('f', [])
@unimplemented
def test_no_mutation_u(f):
    b1 = B()
    b2 = B()

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

@mark.parametrize('f', [setAttrib])
def test_mutation(f):
    b1 = B()
    b2 = B()

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 != B() and b1 == b2 and py == st

@mark.parametrize('f', [setAttribFloat])
@mark.xfail(raises=TypeError)
def test_mutation_f(f):
    """
    The opposite, setting an int when the struct member is float does not
    raise a TypeError since the int will be promoted to a float.
    """
    b1 = B()
    b2 = B()

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 != B() and b1 == b2 and py == st

@mark.parametrize('args', args1)
@mark.parametrize('f', [cmpAttrib, getAndSetAttrib1, getAndSetAttrib2])
def test_mutation2(f, args):
    b1 = B(*args)
    b2 = B(*args)

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

@mark.parametrize('args', [])
@mark.parametrize('f', [])
@unimplemented
def test_mutation2_u(f, args):
    b1 = B(*args)
    b2 = B(*args)

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

@mark.parametrize('f', [returnUnknownAttrib])
@mark.xfail(raises=AttributeError)
def test_mutation2_f(f):
    b1 = B()
    b2 = B()

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

args2 = [(1, 2, 3, 4), (1.0, 2.0, 3.0)]
args3 = list(zip(args2, [0, 0.0]))

@mark.parametrize('f', [getFirstArrayValue, sumC])
@mark.parametrize('args', args3)
def test_no_mutation2(f, args):
    b1 = C(*args)
    b2 = C(*args)

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

@mark.parametrize('f', [])
@mark.parametrize('args', args2)
@unimplemented
def test_no_mutation2_f(f, args):
    b1 = C(args)
    b2 = C(args)

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

@mark.parametrize('f', [getSomeArrayValue])
@mark.parametrize('args', args2)
def test_no_mutation3(f, args):
    b1 = C(args)
    b2 = C(args)

    assert b1 == b2
    py = f(b1, 1)
    st = stella.wrap(f)(b2, 1)

    assert b1 == b2 and py == st

def manipulate_d1(d):
    d.z = 1
    d.a = 2
    d.y = 3.0
    d.g = 4.0

def pass_struct(d):
    manipulate_d1(d)

@mark.parametrize('f', [manipulate_d1, pass_struct])
def test_mutation3(f):
    b1 = D()
    b2 = D()

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

@mark.parametrize('f', [callBoundMethod, callBoundMethod2])
def test_mutation4(f):
    e1 = E()
    e2 = E()

    assert e1 == e2

    py = f(e1)
    st = stella.wrap(f)(e2)

    assert e1 == e2 and py == st

@mark.parametrize('f', [callBoundMethod3, callBoundMethodTwice])
@mark.parametrize('arg', [0, -1, 5])
def test_mutation5(f, arg):
    e1 = E()
    e2 = E()

    assert e1 == e2

    py = f(e1, arg)
    st = stella.wrap(f)(e2, arg)

    assert e1 == e2 and py == st

@mark.parametrize('f', [callBoundMethodOnTwo])
def test_mutation6(f):
    e1 = E()
    e2 = E()
    e3 = E()
    e4 = E()

    assert e1 == e2 and e3 == e4

    py = f(e1, e3)
    st = stella.wrap(f)(e2, e4)

    assert e1 == e2 and e3 == e4 and py == st

@mark.parametrize('f', [objList1, objList2, objList3])
def test_no_mutation7(f):
    l1 = [E(4), E(1)]
    l2 = [E(4), E(1)]

    py = f(l1)
    st = stella.wrap(f)(l2)

    assert l1 == l2 and py == st

@mark.parametrize('f', [objList4])
def test_mutation7(f):
    l1 = [E(4), E(1)]
    l2 = [E(4), E(1)]

    py = f(l1)
    st = stella.wrap(f)(l2)

    assert l1 == l2 and py == st

@mark.parametrize('f', [objContainingList1, objContainingList2])
def test_no_mutation8(f):
    l1 = [E(2), E(5)]
    l2 = [E(2), E(5)]
    f1 = F(l1)
    f2 = F(l2)

    py = f(f1)
    st = stella.wrap(f)(f2)

    assert f1 == f2 and py == st

@mark.parametrize('f', [objContainingList3])
def test_mutation8(f):
    l1 = [E(2), E(5)]
    l2 = [E(2), E(5)]
    f1 = F(l1)
    f2 = F(l2)

    py = f(f1)
    st = stella.wrap(f)(f2)

    assert f1 == f2 and py == st

args3 = [(4, 8), (9.0, 27.0)]

@mark.parametrize('f', [selfRef])
@mark.parametrize('args', args3)
@unimplemented
def test_no_mutation9(f, args):
    b1 = B(*args)
    b2 = B(*args)

    b1.next = b1
    b2.next = b2

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

@mark.parametrize('f', [selfRef])
@mark.parametrize('args', args3)
@unimplemented
def test_no_mutation9_u(f, args):
    b1 = G(*args)
    b2 = G(*args)

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st
<commit_msg>Bugfix: run test for self reference feature<commit_after>import numpy as np

from test import *  # noqa
import stella

class B(object):
    x = 0
    y = 0

    def __init__(self, x=1, y=2):
        self.x = x
        self.y = y

    def __eq__(self, other):
        return self.x == other.x and self.y == other.y

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        return "{}:{}, {}>".format(str(type(self))[:-1], self.x, self.y)

class C(object):
    """
    %"class 'test.objects.C'_<Int*6>_Int" = type { [6 x i64]*, i64 }
    """
    def __init__(self, obj, i=0):
        if isinstance(obj, int):
            self.a = np.zeros(shape=obj, dtype=int)
            self.a[0] = 42
        else:
            self.a = np.array(obj)
        self.i = i

    def __eq__(self, other):
        return self.i == other.i and (self.a == other.a).all()

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        return "{}:{}, {}>".format(str(type(self))[:-1], self.i, self.a)

class D(object):
    z = 0
    a = 0
    y = 0.0
    g = 0.0

    def __init__(self):
        pass

    def __eq__(self, other):
        return (self.z == other.z and
                self.a == other.a and
                self.y == other.y and
                self.g == other.g)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        return "{}: {}>".format(str(type(self))[:-1], [self.z, self.a, self.y, self.g])

class E(object):
    def __init__(self, x=0):
        self.x = x

    def inc(self, p=1):
        self.x += p
        return self.x

    def __eq__(self, other):
        return (self.x == other.x)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        return "{}[x={}]".format(str(type(self))[8:-2], self.x)

class F(object):
    def __init__(self, l):
        self.l = l

    def __eq__(self, other):
        return (self.l == other.l)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        return "{}[l={}]".format(str(type(self))[8:-2], self.l)

class G(B):
    def __init__(self, x=1, y=2):
        super().__init__(x, y)

G.origin = G(0, 0)

def justPassing(a):
    x = 1  # noqa

def cmpAttrib(a):
    return a.x == a.y

def setAttrib(a):
    a.x = 42

def setAttribFloat(a):
    a.x = 42.0

def setUnknownAttrib(a):
    a.z = 42

def getAttrib(a):
    return a.x

def addAttribs(a):
    return a.x + a.y

def returnUnknownAttrib(a):
    return a.z

def getAndSetAttrib1(a):
    a.x *= a.y

def getAndSetAttrib2(a):
    a.x -= 1

def callBoundMethod(e):
    e.inc()
    return e.x

def callBoundMethod2(e):
    e.inc(42)

def callBoundMethod3(e, x):
    e.inc(x)

def callBoundMethodTwice(e, x):
    e.inc(x)
    e.inc(x)

def callBoundMethodOnTwo(e1, e2):
    e1.inc(1)
    e2.inc(2)

def objList1(l):
    return l[0].x + l[1].x

def objList2(l):
    r = 0
    for i in range(len(l)):
        r += l[i].x
    return r

def objList3(l):
    r = 0
    for i in range(len(l)):
        for j in range(len(l)):
            r += l[j].x + i
    return r

def objList4(l):
    for i in range(len(l)):
        l[i].x = i

def objContainingList1(f):
    return f.l[0].x + f.l[1].x

def objContainingList2(f):
    r = 0
    for i in range(len(f.l)):
        r += f.l[i].x
    return r

def objContainingList3(f):
    for i in range(len(f.l)):
        f.l[i].x = i

def selfRef(g):
    return ((g.x - G.origin.x)**2 + (g.y - G.origin.y)**2)**0.5

def nextB(b):
    return b.x == b.next.x and b.y == b.next.y

args1 = [(1, 1), (24, 42), (0.0, 1.0), (1.0, 1.0), (3.0, 0.0)]

def getFirstArrayValue(c):
    return c.a[0]

def getSomeArrayValue(c, i):
    return c.a[i]

def sumC(c):
    for i in range(len(c.a)):
        c.i += c.a[i]

@mark.parametrize('f', [justPassing, addAttribs, getAttrib])
@mark.parametrize('args', args1)
def test_no_mutation(f, args):
    b1 = B(*args)
    b2 = B(*args)

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

@mark.parametrize('f', [])
@unimplemented
def test_no_mutation_u(f):
    b1 = B()
    b2 = B()

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

@mark.parametrize('f', [setAttrib])
def test_mutation(f):
    b1 = B()
    b2 = B()

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 != B() and b1 == b2 and py == st

@mark.parametrize('f', [setAttribFloat])
@mark.xfail(raises=TypeError)
def test_mutation_f(f):
    """
    The opposite, setting an int when the struct member is float does not
    raise a TypeError since the int will be promoted to a float.
    """
    b1 = B()
    b2 = B()

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 != B() and b1 == b2 and py == st

@mark.parametrize('args', args1)
@mark.parametrize('f', [cmpAttrib, getAndSetAttrib1, getAndSetAttrib2])
def test_mutation2(f, args):
    b1 = B(*args)
    b2 = B(*args)

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

@mark.parametrize('args', [])
@mark.parametrize('f', [])
@unimplemented
def test_mutation2_u(f, args):
    b1 = B(*args)
    b2 = B(*args)

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

@mark.parametrize('f', [returnUnknownAttrib])
@mark.xfail(raises=AttributeError)
def test_mutation2_f(f):
    b1 = B()
    b2 = B()

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

args2 = [(1, 2, 3, 4), (1.0, 2.0, 3.0)]
args3 = list(zip(args2, [0, 0.0]))

@mark.parametrize('f', [getFirstArrayValue, sumC])
@mark.parametrize('args', args3)
def test_no_mutation2(f, args):
    b1 = C(*args)
    b2 = C(*args)

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

@mark.parametrize('f', [])
@mark.parametrize('args', args2)
@unimplemented
def test_no_mutation2_f(f, args):
    b1 = C(args)
    b2 = C(args)

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

@mark.parametrize('f', [getSomeArrayValue])
@mark.parametrize('args', args2)
def test_no_mutation3(f, args):
    b1 = C(args)
    b2 = C(args)

    assert b1 == b2
    py = f(b1, 1)
    st = stella.wrap(f)(b2, 1)

    assert b1 == b2 and py == st

def manipulate_d1(d):
    d.z = 1
    d.a = 2
    d.y = 3.0
    d.g = 4.0

def pass_struct(d):
    manipulate_d1(d)

@mark.parametrize('f', [manipulate_d1, pass_struct])
def test_mutation3(f):
    b1 = D()
    b2 = D()

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

@mark.parametrize('f', [callBoundMethod, callBoundMethod2])
def test_mutation4(f):
    e1 = E()
    e2 = E()

    assert e1 == e2

    py = f(e1)
    st = stella.wrap(f)(e2)

    assert e1 == e2 and py == st

@mark.parametrize('f', [callBoundMethod3, callBoundMethodTwice])
@mark.parametrize('arg', [0, -1, 5])
def test_mutation5(f, arg):
    e1 = E()
    e2 = E()

    assert e1 == e2

    py = f(e1, arg)
    st = stella.wrap(f)(e2, arg)

    assert e1 == e2 and py == st

@mark.parametrize('f', [callBoundMethodOnTwo])
def test_mutation6(f):
    e1 = E()
    e2 = E()
    e3 = E()
    e4 = E()

    assert e1 == e2 and e3 == e4

    py = f(e1, e3)
    st = stella.wrap(f)(e2, e4)

    assert e1 == e2 and e3 == e4 and py == st

@mark.parametrize('f', [objList1, objList2, objList3])
def test_no_mutation7(f):
    l1 = [E(4), E(1)]
    l2 = [E(4), E(1)]

    py = f(l1)
    st = stella.wrap(f)(l2)

    assert l1 == l2 and py == st

@mark.parametrize('f', [objList4])
def test_mutation7(f):
    l1 = [E(4), E(1)]
    l2 = [E(4), E(1)]

    py = f(l1)
    st = stella.wrap(f)(l2)

    assert l1 == l2 and py == st

@mark.parametrize('f', [objContainingList1, objContainingList2])
def test_no_mutation8(f):
    l1 = [E(2), E(5)]
    l2 = [E(2), E(5)]
    f1 = F(l1)
    f2 = F(l2)

    py = f(f1)
    st = stella.wrap(f)(f2)

    assert f1 == f2 and py == st

@mark.parametrize('f', [objContainingList3])
def test_mutation8(f):
    l1 = [E(2), E(5)]
    l2 = [E(2), E(5)]
    f1 = F(l1)
    f2 = F(l2)

    py = f(f1)
    st = stella.wrap(f)(f2)

    assert f1 == f2 and py == st

args3 = [(4, 8), (9.0, 27.0)]

@mark.parametrize('f', [nextB])
@mark.parametrize('args', args3)
def test_no_mutation9(f, args):
    b1 = B(*args)
    b2 = B(*args)

    b1.next = b1
    b2.next = b2

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st

@mark.parametrize('f', [selfRef])
@mark.parametrize('args', args3)
@unimplemented
def test_no_mutation9_u(f, args):
    b1 = G(*args)
    b2 = G(*args)

    assert b1 == b2
    py = f(b1)
    st = stella.wrap(f)(b2)

    assert b1 == b2 and py == st
<|endoftext|>


<commit_before>#!/usr/bin/env python

#  Copyright 2008-2012 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

USAGE = """robot.tidy -- Robot Framework test data clean-up tool.

Version:  <VERSION>

Usage:  python -m robot.tidy [options] inputfile
   or:  python -m robot.tidy [options] inputfile [outputfile]
   or:  python -m robot.tidy --inplace [options] inputfile [more input files]
   or:  python -m robot.tidy --recursive [options] directory

Tidy tool can be used to clean up and change format of Robot Framework test
data files. The output is written into the standard output stream by default,
but an optional output file can be given starting from Robot Framework 2.7.5.
Files can also be modified in-place using --inplace or --recursive options.

Options
=======

 -i --inplace    Tidy given file(s) so that original file(s) are overwritten
                 (or removed, if the format is changed). When this option is
                 used, it is possible to give multiple input files.
                 Examples:
                   python -m robot.tidy --inplace tests.html
                   python -m robot.tidy --inplace --format txt *.html
 -r --recursive  Process given directory recursively. Files in the directory
                 are processed in-place similarly as when --inplace option
                 is used.
 -f --format txt|html|tsv
                 Output file format. If omitted, the format of the input
                 file is used.
 -p --usepipes   Use pipe (`|`) as a cell separator in the txt format.
 -s --spacecount number
                 The number of spaces between cells in the txt format.
                 New in Robot Framework 2.7.3.
 -h -? --help    Show this help.

Cleaning up the test data
=========================

Test case files created with HTML editors or written by hand can be normalized
using tidy. Tidy always writes consistent headers, consistent order for
settings, and consistent amount of whitespace between cells and tables.

Examples:
  python -m robot.tidy messed_up_tests.html cleaned_tests.html
  python -m robot.tidy --inplace tests.txt

Changing the test data format
=============================

Robot Framework supports test data in HTML, TSV and TXT formats and this tool
makes changing between the formats trivial. Input format is always determined
based on the extension of the input file. Output format can be set using
the --format option.

Examples:
  python -m robot.tidy --format tsv --inplace tests.html
  python -m robot.tidy --format txt --recursive mytests

Output encoding
===============

All output files are written using UTF-8 encoding. Outputs written to the
console use the current console encoding.

Alternative execution
=====================

In the above examples tidy is used only with Python, but it works also with
Jython and IronPython. Above tidy is executed as an installed module, but
it can also be executed as a script like `python path/robot/tidy.py`.
"""

import os
import sys
from StringIO import StringIO

# Allows running as a script. __name__ check needed with multiprocessing:
# http://code.google.com/p/robotframework/issues/detail?id=1137
if 'robot' not in sys.modules and __name__ == '__main__':
    import pythonpathsetter

from robot.errors import DataError
from robot.parsing import (ResourceFile, TestDataDirectory, TestCaseFile,
                           disable_curdir_processing)
from robot.utils import Application

class Tidy(object):

    def __init__(self, **options):
        self._options = options

    def file(self, path, output=None):
        data = self._create_datafile(path)
        outfile = open(output, 'w') if output else StringIO()
        try:
            self._save_file(data, outfile)
            if not output:
                return outfile.getvalue().replace('\r\n', '\n').decode('UTF-8')
        finally:
            outfile.close()

    def directory(self, path):
        self._save_directory(self._create_datafile(path))

    def inplace(self, path):
        self._save_file(self._create_datafile(path))

    def _save_file(self, data, output=None):
        source = data.initfile if self._is_directory(data) else data.source
        if source and not output:
            os.remove(source)
        data.save(output=output, **self._options)

    def _save_directory(self, data):
        if not self._is_directory(data):
            self._save_file(data)
            return
        if data.initfile:
            self._save_file(data)
        for child in data.children:
            self._save_directory(child)

    def _is_directory(self, data):
        return hasattr(data, 'initfile')

    @disable_curdir_processing
    def _create_datafile(self, source):
        if os.path.isdir(source):
            return TestDataDirectory(source=source).populate()
        if self._is_init_file(source):
            dir_ = os.path.dirname(source)
            return TestDataDirectory(source=dir_).populate(recurse=False)
        try:
            return TestCaseFile(source=source).populate()
        except DataError:
            try:
                return ResourceFile(source=source).populate()
            except DataError:
                raise DataError("Invalid data source '%s'." % source)

    def _is_init_file(self, source):
        return os.path.splitext(os.path.basename(source))[0] == '__init__'

class TidyCommandLine(Application):

    def __init__(self):
        Application.__init__(self, USAGE, arg_limits=(1,))

    def main(self, arguments, recursive=False, inplace=False, format='txt',
             usepipes=False, spacecount=4):
        tidy = Tidy(format=format, pipe_separated=usepipes,
                    txt_separating_spaces=spacecount)
        if recursive:
            tidy.directory(arguments[0])
        elif inplace:
            for source in arguments:
                tidy.inplace(source)
        else:
            output = tidy.file(*arguments)
            self.console(output)

    def validate(self, opts, args):
        self._validate_mode_and_arguments(args, **opts)
        opts['format'] = self._validate_format(args, **opts)
        if not opts['spacecount']:
            opts.pop('spacecount')
        else:
            opts['spacecount'] = self._validate_spacecount(opts['spacecount'])
        return opts, args

    def _validate_mode_and_arguments(self, args, inplace, recursive, **others):
        if inplace and recursive:
            raise DataError('--recursive and --inplace can not be used together.')
        if recursive and (len(args) > 1 or not os.path.isdir(args[0])):
            raise DataError('--recursive requires exactly one directory as argument.')
        if not (inplace or recursive) and len(args) > 2:
            raise DataError('Default mode requires 1 or 2 arguments.')

    def _validate_format(self, args, format, inplace, recursive, **others):
        if not format:
            if inplace or recursive or len(args) < 2:
                return None
            format = os.path.splitext(args[1])[1][1:]
        format = format.upper()
        if format not in ['TXT', 'TSV', 'HTML']:
            raise DataError("Invalid format: '%s'." % format)
        return format

    def _validate_spacecount(self, spacecount):
        try:
            spacecount = int(spacecount)
            if spacecount < 2:
                raise ValueError
        except ValueError:
            raise DataError('--spacecount must be an integer greater than 1.')
        return spacecount

def tidy_cli(args):
    """Executes tidy similarly as from the command line.

    :param args: command line arguments as a list of strings.

    Example:
        tidy_cli(['--format', 'txt', 'mytests.html'])
    """
    TidyCommandLine().execute_cli(args)

if __name__ == '__main__':
    tidy_cli(sys.argv[1:])
<commit_msg>Tidy: Renamed and moved methods.<commit_after>#!/usr/bin/env python

#  Copyright 2008-2012 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

USAGE = """robot.tidy -- Robot Framework test data clean-up tool.

Version:  <VERSION>

Usage:  python -m robot.tidy [options] inputfile
   or:  python -m robot.tidy [options] inputfile [outputfile]
   or:  python -m robot.tidy --inplace [options] inputfile [more input files]
   or:  python -m robot.tidy --recursive [options] directory

Tidy tool can be used to clean up and change format of Robot Framework test
data files. The output is written into the standard output stream by default,
but an optional output file can be given starting from Robot Framework 2.7.5.
Files can also be modified in-place using --inplace or --recursive options.

Options
=======

 -i --inplace    Tidy given file(s) so that original file(s) are overwritten
                 (or removed, if the format is changed). When this option is
                 used, it is possible to give multiple input files.
                 Examples:
                   python -m robot.tidy --inplace tests.html
                   python -m robot.tidy --inplace --format txt *.html
 -r --recursive  Process given directory recursively. Files in the directory
                 are processed in-place similarly as when --inplace option
                 is used.
 -f --format txt|html|tsv
                 Output file format. If omitted, the format of the input
                 file is used.
 -p --usepipes   Use pipe (`|`) as a cell separator in the txt format.
 -s --spacecount number
                 The number of spaces between cells in the txt format.
                 New in Robot Framework 2.7.3.
 -h -? --help    Show this help.

Cleaning up the test data
=========================

Test case files created with HTML editors or written by hand can be normalized
using tidy. Tidy always writes consistent headers, consistent order for
settings, and consistent amount of whitespace between cells and tables.

Examples:
  python -m robot.tidy messed_up_tests.html cleaned_tests.html
  python -m robot.tidy --inplace tests.txt

Changing the test data format
=============================

Robot Framework supports test data in HTML, TSV and TXT formats and this tool
makes changing between the formats trivial. Input format is always determined
based on the extension of the input file. Output format can be set using
the --format option.

Examples:
  python -m robot.tidy --format tsv --inplace tests.html
  python -m robot.tidy --format txt --recursive mytests

Output encoding
===============

All output files are written using UTF-8 encoding. Outputs written to the
console use the current console encoding.

Alternative execution
=====================

In the above examples tidy is used only with Python, but it works also with
Jython and IronPython. Above tidy is executed as an installed module, but
it can also be executed as a script like `python path/robot/tidy.py`.
"""

import os
import sys
from StringIO import StringIO

# Allows running as a script. __name__ check needed with multiprocessing:
# http://code.google.com/p/robotframework/issues/detail?id=1137
if 'robot' not in sys.modules and __name__ == '__main__':
    import pythonpathsetter

from robot.errors import DataError
from robot.parsing import (ResourceFile, TestDataDirectory, TestCaseFile,
                           disable_curdir_processing)
from robot.utils import Application

class Tidy(object):

    def __init__(self, **options):
        self._options = options

    def file(self, path, output=None):
        data = self._parse_data(path)
        outfile = open(output, 'w') if output else StringIO()
        try:
            self._save_file(data, outfile)
            if not output:
                return outfile.getvalue().replace('\r\n', '\n').decode('UTF-8')
        finally:
            outfile.close()

    def directory(self, path):
        self._save_directory(self._parse_data(path))

    def inplace(self, path):
        self._save_file(self._parse_data(path))

    @disable_curdir_processing
    def _parse_data(self, path):
        if os.path.isdir(path):
            return TestDataDirectory(source=path).populate()
        if self._is_init_file(path):
            path = os.path.dirname(path)
            return TestDataDirectory(source=path).populate(recurse=False)
        try:
            return TestCaseFile(source=path).populate()
        except DataError:
            try:
                return ResourceFile(source=path).populate()
            except DataError:
                raise DataError("Invalid data source '%s'." % path)

    def _is_init_file(self, path):
        return os.path.splitext(os.path.basename(path))[0].lower() == '__init__'

    def _save_file(self, data, output=None):
        source = data.initfile if self._is_directory(data) else data.source
        if source and not output:
            os.remove(source)
        data.save(output=output, **self._options)

    def _save_directory(self, data):
        if not self._is_directory(data):
            self._save_file(data)
            return
        if data.initfile:
            self._save_file(data)
        for child in data.children:
            self._save_directory(child)

    def _is_directory(self, data):
        return hasattr(data, 'initfile')

class TidyCommandLine(Application):

    def __init__(self):
        Application.__init__(self, USAGE, arg_limits=(1,))

    def main(self, arguments, recursive=False, inplace=False, format='txt',
             usepipes=False, spacecount=4):
        tidy = Tidy(format=format, pipe_separated=usepipes,
                    txt_separating_spaces=spacecount)
        if recursive:
            tidy.directory(arguments[0])
        elif inplace:
            for source in arguments:
                tidy.inplace(source)
        else:
            output = tidy.file(*arguments)
            self.console(output)

    def validate(self, opts, args):
        self._validate_mode_and_arguments(args, **opts)
        opts['format'] = self._validate_format(args, **opts)
        if not opts['spacecount']:
            opts.pop('spacecount')
        else:
            opts['spacecount'] = self._validate_spacecount(opts['spacecount'])
        return opts, args

    def _validate_mode_and_arguments(self, args, inplace, recursive, **others):
        if inplace and recursive:
            raise DataError('--recursive and --inplace can not be used together.')
        if recursive and (len(args) > 1 or not os.path.isdir(args[0])):
            raise DataError('--recursive requires exactly one directory as argument.')
        if not (inplace or recursive) and len(args) > 2:
            raise DataError('Default mode requires 1 or 2 arguments.')

    def _validate_format(self, args, format, inplace, recursive, **others):
        if not format:
            if inplace or recursive or len(args) < 2:
                return None
            format = os.path.splitext(args[1])[1][1:]
        format = format.upper()
        if format not in ['TXT', 'TSV', 'HTML']:
            raise DataError("Invalid format: '%s'." % format)
        return format

    def _validate_spacecount(self, spacecount):
        try:
            spacecount = int(spacecount)
            if spacecount < 2:
                raise ValueError
        except ValueError:
            raise DataError('--spacecount must be an integer greater than 1.')
        return spacecount

def tidy_cli(args):
    """Executes tidy similarly as from the command line.

    :param args: command line arguments as a list of strings.

    Example:
        tidy_cli(['--format', 'txt', 'mytests.html'])
    """
    TidyCommandLine().execute_cli(args)

if __name__ == '__main__':
    tidy_cli(sys.argv[1:])
<|endoftext|>