<commit_before>#!/usr/bin/env python
import drmaa
import os
import sys
import glob
import time
import yaml
import subprocess

ANALYSIS_SCRIPT="distributed_nextgen_pipeline.py"
#ANALYSIS_SCRIPT="automated_initial_analysis.py"
ANALYSIS_DIR="/proj/a2010002/nobackup/illumina/"
EMAIL="seqmaster@scilifelab.se"
#SLURM_ARGS="-A a2010002 -p node"
SLURM_ARGS="-A a2010002 -p node"
RUN_TIME="168:00:00"
#RUN_TIME="48:00:00"
PROCESS_YAML_SCRIPT = "/bubo/home/h27/pontusla/scilifelab/scripts/process_run_info.py"
PROCESS_YAML = True
FC_SPECIFIC_AMPQ = True

def main():

    # Ugly hack for overriding module system's GATK verison (will it work?)
    # Not needed with customized module
    #gatk_home = os.path.join(os.getenv("HOME"),"src","GATK","1.1")
    #print "Setting GATK_HOME = %s" % gatk_home
    #os.environ['GATK_HOME'] = gatk_home

    post_process_config = sys.argv[1]
    archive_dir = sys.argv[2]
    if len(sys.argv) > 3:
        run_info_file = sys.argv[3]
    else:
        run_info_file = None

    # Set the barcode type in run_info.yaml to "illumina", strip the 7th nucleotide and set analysis to 'Minimal'
    if run_info_file is not None and PROCESS_YAML:
        print "---------\nProcessing run_info:"
        run_info_backup = "%s.orig" % run_info_file
        os.rename(run_info_file,run_info_backup)
        #cl = ["python","%s" % PROCESS_YAML_SCRIPT,run_info_backup,"--barcode_type","illumina","--trim","1","--analysis","Minimal","--out_file",run_info_file]
        #cl = ["python","%s" % PROCESS_YAML_SCRIPT,run_info_backup,"--barcode_type","illumina","--analysis","Minimal","--out_file",run_info_file,"--ascii"]
        cl = ["python","%s" % PROCESS_YAML_SCRIPT,run_info_backup,"--analysis","Align_illumina","--out_file",run_info_file,"--ascii","--clear_description"]
        print subprocess.check_output(cl)
        print "\n---------\n"

    # Check that the specified paths exist
    print "Checking input paths"
    for path in (post_process_config,archive_dir,run_info_file):
        if path is not None and not os.path.exists(path):
            raise Exception("The path %s does not exist" % path)

    print "Getting base_dir from %s" % post_process_config
    # Parse the config to get the analysis directory
    with open(post_process_config) as ppc:
        config = yaml.load(ppc)

    analysis = config.get("analysis",{})
    base_dir = analysis.get("base_dir",ANALYSIS_DIR)

    print "Getting run name from %s" % archive_dir
    # Get the run name from the archive dir
    _,run_name = os.path.split(os.path.normpath(archive_dir))

    # Create the working directory if necessary and change into it
    work_dir = os.path.join(base_dir,run_name)
    os.chdir(base_dir)
    print "Creating/changing to %s" % work_dir
    try:
        os.mkdir(run_name,0770)
    except OSError:
        pass
    os.chdir(run_name)

    # make sure that the work dir exists
    if not os.path.exists(work_dir):
        raise Exception("The path %s does not exist and was not created" % work_dir)

    # if required, parse the machine id and flowcell position and use an ampq vhost specific for it
    if FC_SPECIFIC_AMPQ:
        machine_id = None
        flowcell_position = None
        for p in run_name.upper().split("_"):
            if p.startswith("SN"):
                machine_id = p
            elif p[0] in ("A","B") and p.endswith("XX"):
                flowcell_position = p[0]
        assert machine_id and flowcell_position, "Machine id and flowcell position could not be parsed from run name '%s'" % run_name

        # write a dedicated post_process.yaml for the ampq queue
        if config.get("distributed",False):
            config["distributed"]["rabbitmq_vhost"] = "bionextgen-%s-%s" % (machine_id,flowcell_position)

        post_process_config_orig = post_process_config
        parts = os.path.splitext(post_process_config)
        post_process_config = "%s-%s-%s%s" % (parts[0],machine_id,flowcell_position,parts[1])

        if os.path.exists(post_process_config):
            print "Customized post process yaml '%s' already exists, will not overwrite" % post_process_config
        else:
            with open(post_process_config,"w") as fh:
                fh.write(yaml.safe_dump(config, default_flow_style=False, allow_unicode=True))

    # Translate the time string into seconds
    multiplier = (86400,3600,60,1)
    units = RUN_TIME.split(":")
    seconds = 0
    for i in range(1,len(units)+1):
        seconds += multiplier[-1*i]*int(units[-1*i])
    print "Will request job to run for %s seconds" % seconds
    print "Initializing session"
    s = drmaa.Session()
    s.initialize()

    jt = s.createJobTemplate()
    jt.remoteCommand = ANALYSIS_SCRIPT
    args = [post_process_config,archive_dir]
    if run_info_file is not None:
        args.append(run_info_file)
    jt.args = args

    # TODO: job name is always (null), must fix slurm_drmaa C library and its
    # custom parsing (substitute "slurmdrmaa_parse_native"
    # for GNU GetOpt on slurm_drmaa/util.c)
    #jt.job_name = run_name
    #jt.blockEmail = 0
    #jt.mail = EMAIL
    #jt.hardRunDurationLimit(seconds)
    output = os.path.join(work_dir,"%s.out" % run_name)
    error = os.path.join(work_dir,"%s.err" % run_name)
    print "Setting output and error paths (%s,%s)" % (output,error)

    #jt.outputPath = output
    #jt.errorPath = error
    jt.nativeSpecification = "%s -t %s" % (SLURM_ARGS,RUN_TIME)

    print "Submitting job"
    jobid = s.runJob(jt)
    print 'Your job has been submitted with id ' + jobid

    s.deleteJobTemplate(jt)
    s.exit()

    exit(0)

if __name__ == "__main__":
    main()
<commit_msg>cleanup<commit_after>#!/usr/bin/env python
import drmaa
import os
import sys
import glob
import time
import yaml
import subprocess

ANALYSIS_SCRIPT="distributed_nextgen_pipeline.py"
ANALYSIS_DIR="/proj/a2010002/nobackup/illumina/"
EMAIL="seqmaster@scilifelab.se"
SLURM_ARGS="-A a2010002 -p node"
RUN_TIME="168:00:00"
PROCESS_YAML_SCRIPT = "process_run_info.py"
PROCESS_YAML = True
FC_SPECIFIC_AMPQ = True

def main():

    post_process_config = sys.argv[1]
    archive_dir = sys.argv[2]
    if len(sys.argv) > 3:
        run_info_file = sys.argv[3]
    else:
        run_info_file = None

    # Set the barcode type in run_info.yaml to "illumina", strip the 7th nucleotide and set analysis to 'Minimal'
    if run_info_file is not None and PROCESS_YAML:
        print "---------\nProcessing run_info:"
        run_info_backup = "%s.orig" % run_info_file
        os.rename(run_info_file,run_info_backup)
        cl = ["python","%s" % PROCESS_YAML_SCRIPT,run_info_backup,"--analysis","Align_illumina","--out_file",run_info_file,"--ascii","--clear_description"]
        print subprocess.check_output(cl)
        print "\n---------\n"

    # Check that the specified paths exist
    print "Checking input paths"
    for path in (post_process_config,archive_dir,run_info_file):
        if path is not None and not os.path.exists(path):
            raise Exception("The path %s does not exist" % path)

    print "Getting base_dir from %s" % post_process_config
    # Parse the config to get the analysis directory
    with open(post_process_config) as ppc:
        config = yaml.load(ppc)

    analysis = config.get("analysis",{})
    base_dir = analysis.get("base_dir",ANALYSIS_DIR)

    print "Getting run name from %s" % archive_dir
    # Get the run name from the archive dir
    _,run_name = os.path.split(os.path.normpath(archive_dir))

    # Create the working directory if necessary and change into it
    work_dir = os.path.join(base_dir,run_name)
    os.chdir(base_dir)
    print "Creating/changing to %s" % work_dir
    try:
        os.mkdir(run_name,0770)
    except OSError:
        pass
    os.chdir(run_name)

    # make sure that the work dir exists
    if not os.path.exists(work_dir):
        raise Exception("The path %s does not exist and was not created" % work_dir)

    # if required, parse the machine id and flowcell position and use an ampq vhost specific for it
    if FC_SPECIFIC_AMPQ:
        machine_id = None
        flowcell_position = None
        for p in run_name.upper().split("_"):
            if p.startswith("SN"):
                machine_id = p
            elif p[0] in ("A","B") and p.endswith("XX"):
                flowcell_position = p[0]
        assert machine_id and flowcell_position, "Machine id and flowcell position could not be parsed from run name '%s'" % run_name

        # write a dedicated post_process.yaml for the ampq queue
        if config.get("distributed",False):
            config["distributed"]["rabbitmq_vhost"] = "bionextgen-%s-%s" % (machine_id,flowcell_position)

        post_process_config_orig = post_process_config
        parts = os.path.splitext(post_process_config)
        post_process_config = "%s-%s-%s%s" % (parts[0],machine_id,flowcell_position,parts[1])

        if os.path.exists(post_process_config):
            print "Customized post process yaml '%s' already exists, will not overwrite" % post_process_config
        else:
            with open(post_process_config,"w") as fh:
                fh.write(yaml.safe_dump(config, default_flow_style=False, allow_unicode=True))

    # Translate the time string into seconds
    multiplier = (86400,3600,60,1)
    units = RUN_TIME.split(":")
    seconds = 0
    for i in range(1,len(units)+1):
        seconds += multiplier[-1*i]*int(units[-1*i])
    print "Will request job to run for %s seconds" % seconds
    print "Initializing session"
    s = drmaa.Session()
    s.initialize()

    jt = s.createJobTemplate()
    jt.remoteCommand = ANALYSIS_SCRIPT
    args = [post_process_config,archive_dir]
    if run_info_file is not None:
        args.append(run_info_file)
    jt.args = args

    # TODO: job name is always (null), must fix slurm_drmaa C library and its
    # custom parsing (substitute "slurmdrmaa_parse_native"
    # for GNU GetOpt on slurm_drmaa/util.c)
    #jt.job_name = run_name
    #jt.blockEmail = 0
    #jt.mail = EMAIL
    #jt.hardRunDurationLimit(seconds)
    output = os.path.join(work_dir,"%s.out" % run_name)
    error = os.path.join(work_dir,"%s.err" % run_name)
    print "Setting output and error paths (%s,%s)" % (output,error)

    #jt.outputPath = output
    #jt.errorPath = error
    jt.nativeSpecification = "%s -t %s" % (SLURM_ARGS,RUN_TIME)

    print "Submitting job"
    jobid = s.runJob(jt)
    print 'Your job has been submitted with id ' + jobid

    s.deleteJobTemplate(jt)
    s.exit()

    exit(0)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#coding:utf-8
import logging

from boto import s3
from boto.exception import S3ResponseError, S3CreateError
from boto.route53.exception import DNSServerError

from cactus.deployment.engine import BaseDeploymentEngine
from cactus.deployment.s3.auth import AWSCredentialsManager
from cactus.deployment.s3.file import S3File
from cactus.deployment.s3.domain import AWSBucket, AWSDomain
from cactus.exceptions import InvalidCredentials
from cactus.utils import ipc

logger = logging.getLogger(__name__)

class S3DeploymentEngine(BaseDeploymentEngine):
    FileClass = S3File
    CredentialsManagerClass = AWSCredentialsManager

    config_bucket_name = "aws-bucket-name"
    config_bucket_website = "aws-bucket-website"
    config_bucket_region = "aws-bucket-region"

    _s3_default_region = "us-east-1"
    _s3_port = 443
    _s3_is_secure = True
    _s3_https_connection_factory = None

    def _get_buckets(self):
        """
        :returns: The list of buckets found for this account
        """
        try:
            return self.get_connection().get_all_buckets()
        except S3ResponseError as e:
            if e.error_code == u'InvalidAccessKeyId':
                logger.info("Received an Error from AWS:\n %s", e.body)
                raise InvalidCredentials()
            raise

    def _get_bucket_region(self):
        return self.site.config.get(self.config_bucket_region, self._s3_default_region)

    def _create_connection(self):
        """
        Create a new S3 Connection
        """
        aws_access_key, aws_secret_key = self.credentials_manager.get_credentials()

        return s3.connect_to_region(self._get_bucket_region(),
                aws_access_key_id=aws_access_key.strip(),
                aws_secret_access_key=aws_secret_key.strip(),
                is_secure=self._s3_is_secure, port=self._s3_port,
                https_connection_factory=self._s3_https_connection_factory
        )

    def get_bucket(self):
        """
        :returns: The Bucket if found, None otherwise.
        :raises: InvalidCredentials if we can't connect to AWS
        """
        buckets = self._get_buckets()
        buckets = dict((bucket.name, bucket) for bucket in buckets)
        return buckets.get(self.bucket_name)

    def create_bucket(self):
        """
        :returns: The newly created bucket
        """
        try:
            bucket = self.get_connection().create_bucket(self.bucket_name,
                policy='public-read', location=self._get_bucket_region()
            )
        except S3CreateError:
            logger.info(
                'Bucket with name %s already is used by someone else, '
                'please try again with another name', self.bucket_name)
            return  #TODO: These should be exceptions

        # Configure S3 to use the index.html and error.html files for indexes and 404/500s.
        bucket.configure_website(self._index_page, self._error_page)

        return bucket

    def get_website_endpoint(self):
        return self.bucket.get_website_endpoint()

    def domain_setup(self):
        bucket_name = self.site.config.get(self.config_bucket_name)
        if not bucket_name:
            logger.warning("No bucket name")
            return

        aws_access_key, aws_secret_key = self.credentials_manager.get_credentials()
        domain = AWSDomain(aws_access_key, aws_secret_key, bucket_name)

        try:
            domain.setup()
        except DNSServerError, e:
            logger.debug(e)
            ipc.signal("domain.setup.error", {"errorKey": "AccountDisabled"})
            logger.error("Account cannot use route 53")
            logger.error(e)

    def domain_list(self):
        bucket_name = self.site.config.get(self.config_bucket_name)

        if not bucket_name:
            logger.warning("No bucket name")
            return

        aws_access_key, aws_secret_key = self.credentials_manager.get_credentials()
        domain = AWSDomain(aws_access_key, aws_secret_key, bucket_name)

        try:
            domain_list = domain.nameServers()
        except DNSServerError, e:
            print e
            ipc.signal("domain.list.error", {"errorKey": "AccountDisabled"})
            logger.error("Account cannot use route 53")
            logger.error(e)
            return

        if domain_list:
            ipc.signal("domain.list.result", {"nameservers": domain_list})
            for domain in domain_list:
                logger.info(domain)
        else:
            logger.error("No name servers configured")

    def domain_remove(self):
        pass
<commit_msg>Fix default region for creating buckets<commit_after>#coding:utf-8
import logging

from boto import s3
from boto.exception import S3ResponseError, S3CreateError
from boto.route53.exception import DNSServerError

from cactus.deployment.engine import BaseDeploymentEngine
from cactus.deployment.s3.auth import AWSCredentialsManager
from cactus.deployment.s3.file import S3File
from cactus.deployment.s3.domain import AWSBucket, AWSDomain
from cactus.exceptions import InvalidCredentials
from cactus.utils import ipc

logger = logging.getLogger(__name__)

class S3DeploymentEngine(BaseDeploymentEngine):
    FileClass = S3File
    CredentialsManagerClass = AWSCredentialsManager

    config_bucket_name = "aws-bucket-name"
    config_bucket_website = "aws-bucket-website"
    config_bucket_region = "aws-bucket-region"

    _s3_default_region = "us-east-1"
    _s3_port = 443
    _s3_is_secure = True
    _s3_https_connection_factory = None

    def _get_buckets(self):
        """
        :returns: The list of buckets found for this account
        """
        try:
            return self.get_connection().get_all_buckets()
        except S3ResponseError as e:
            if e.error_code == u'InvalidAccessKeyId':
                logger.info("Received an Error from AWS:\n %s", e.body)
                raise InvalidCredentials()
            raise

    def _get_bucket_region(self):
        return self.site.config.get(self.config_bucket_region, self._s3_default_region)

    def _create_connection(self):
        """
        Create a new S3 Connection
        """
        aws_access_key, aws_secret_key = self.credentials_manager.get_credentials()

        return s3.connect_to_region(self._get_bucket_region(),
                aws_access_key_id=aws_access_key.strip(),
                aws_secret_access_key=aws_secret_key.strip(),
                is_secure=self._s3_is_secure, port=self._s3_port,
                https_connection_factory=self._s3_https_connection_factory
        )

    def get_bucket(self):
        """
        :returns: The Bucket if found, None otherwise.
        :raises: InvalidCredentials if we can't connect to AWS
        """
        buckets = self._get_buckets()
        buckets = dict((bucket.name, bucket) for bucket in buckets)
        return buckets.get(self.bucket_name)

    def create_bucket(self):
        """
        :returns: The newly created bucket
        """
        try:

            # When creating a bucket, the region cannot be "us-east-1" but needs
            # to be an empty string, so we do that for now.
            # https://github.com/boto/boto3/issues/125#issuecomment-109408790
            if self._get_bucket_region() == "us-east-1":
                region = ""
            else:
                region = self._get_bucket_region()

            bucket = self.get_connection().create_bucket(self.bucket_name,
                policy='public-read', location=region
            )
        except S3CreateError:
            logger.info(
                'Bucket with name %s already is used by someone else, '
                'please try again with another name', self.bucket_name)
            return  #TODO: These should be exceptions

        # Configure S3 to use the index.html and error.html files for indexes and 404/500s.
        bucket.configure_website(self._index_page, self._error_page)

        return bucket

    def get_website_endpoint(self):
        return self.bucket.get_website_endpoint()

    def domain_setup(self):
        bucket_name = self.site.config.get(self.config_bucket_name)
        if not bucket_name:
            logger.warning("No bucket name")
            return

        aws_access_key, aws_secret_key = self.credentials_manager.get_credentials()
        domain = AWSDomain(aws_access_key, aws_secret_key, bucket_name)

        try:
            domain.setup()
        except DNSServerError, e:
            logger.debug(e)
            ipc.signal("domain.setup.error", {"errorKey": "AccountDisabled"})
            logger.error("Account cannot use route 53")
            logger.error(e)

    def domain_list(self):
        bucket_name = self.site.config.get(self.config_bucket_name)

        if not bucket_name:
            logger.warning("No bucket name")
            return

        aws_access_key, aws_secret_key = self.credentials_manager.get_credentials()
        domain = AWSDomain(aws_access_key, aws_secret_key, bucket_name)

        try:
            domain_list = domain.nameServers()
        except DNSServerError, e:
            print e
            ipc.signal("domain.list.error", {"errorKey": "AccountDisabled"})
            logger.error("Account cannot use route 53")
            logger.error(e)
            return

        if domain_list:
            ipc.signal("domain.list.result", {"nameservers": domain_list})
            for domain in domain_list:
                logger.info(domain)
        else:
            logger.error("No name servers configured")

    def domain_remove(self):
        pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Fri Oct  2 15:08:22 2015

@author: GILLES Armand
"""

import pandas as pd
import glob

df = pd.read_csv('../source/ConfigFamiliale2009.csv', sep=";")

df.columns = ['Communes', 'Codes_Insee', 'NB_Allocataires_2009',
              'COUP_0_ENF_2009', 'COUP_1_ENF_2009', 'COUP_2_ENF_2009', 'COUP_3_ENF_2009', 'COUP_4plus_ENF_2009',
              'Homme_Isole_2009', 'Femme_Isolee_2009', 'MONO_1_ENF_2009', 'MONO_2_ENF_2009',
              'MONO_3_ENF_2009', 'MONO_4plus_ENF_2009']

files = glob.glob('../source/ConfigFamiliale*')

for path_file in files:
    year = str(path_file[-8:-4])
    if (year != '2009'):
        df_temp = pd.read_csv(path_file, sep=';')

        # Rename Col with year
        year_col = ['Communes', 'Codes_Insee']
        features_col = []
        for col in df_temp.columns[-12:]:
            year_col.append(col +"_"+ year)
            features_col.append(col +"_"+ year)

        # Adding key for mergeing
        features_col.append('Codes_Insee')
        df_temp.columns = year_col
        df = pd.merge(df, df_temp[features_col], how='inner', on='Codes_Insee')

# Rename col to have unique name in futur merge
list_col = []
for col in df.columns:
    if "nb_allocataires" in col.lower(): # NB_Allocataires (2009) != NB_allocataires (2010)
        list_col.append(col+"_CF") # CF = ConfigFamiliale
    else:
        list_col.append(col)
df.columns = list_col

df.to_csv('../data/full_ConfigFamiliale.csv', encoding='utf-8', index=False)

## Features
#u'NB_Allocataires_2009_CF',
#       u'COUP_0_ENF_2009', u'COUP_1_ENF_2009', u'COUP_2_ENF_2009',
#       u'COUP_3_ENF_2009', u'COUP_4plus_ENF_2009', u'Homme_Isole_2009',
#       u'Femme_Isolee_2009', u'MONO_1_ENF_2009', u'MONO_2_ENF_2009',
#       u'MONO_3_ENF_2009', u'MONO_4plus_ENF_2009', u'NB_allocataires_2010_CF',
#       u'COUP_0_ENF_2010', u'COUP_1_ENF_2010', u'COUP_2_ENF_2010',
#       u'COUP_3_ENF_2010', u'COUP_4plus_ENF_2010', u'Homme_Isole_2010',
#       u'Femme_Isolee_2010', u'MONO_1_ENF_2010', u'MONO_2_ENF_2010',
#       u'MONO_3_ENF_2010', u'MONO_4plus_ENF_2010', u'NB_allocataires_2011_CF',
#       u'COUP_0_ENF_2011', u'COUP_1_ENF_2011', u'COUP_2_ENF_2011',
#       u'COUP_3_ENF_2011', u'COUP_4plus_ENF_2011', u'Homme_Isole_2011',
#       u'Femme_Isolee_2011', u'MONO_1_ENF_2011', u'MONO_2_ENF_2011',
#       u'MONO_3_ENF_2011', u'MONO_4plus_ENF_2011', u'NB_allocataires_2012_CF',
#       u'COUP_0_ENF_2012', u'COUP_1_ENF_2012', u'COUP_2_ENF_2012',
#       u'COUP_3_ENF_2012', u'COUP_4plus_ENF_2012', u'Homme_Isole_2012',
#       u'Femme_Isolee_2012', u'MONO_1_ENF_2012', u'MONO_2_ENF_2012',
#       u'MONO_3_ENF_2012', u'MONO_4plus_ENF_2012', u'NB_allocataires_2013_CF',
#       u'COUP_0_ENF_2013', u'COUP_1_ENF_2013', u'COUP_2_ENF_2013',
#       u'COUP_3_ENF_2013', u'COUP_4plus_ENF_2013', u'Homme_Isole_2013',
#       u'Femme_Isolee_2013', u'MONO_1_ENF_2013', u'MONO_2_ENF_2013',
#       u'MONO_3_ENF_2013', u'MONO_4plus_ENF_2013', u'NB_allocataires_2014_CF',
#       u'COUP_0_ENF_2014', u'COUP_1_ENF_2014', u'COUP_2_ENF_2014',
#       u'COUP_3_ENF_2014', u'COUP_4plus_ENF_2014', u'Homme_Isole_2014',
#       u'Femme_Isolee_2014', u'MONO_1_ENF_2014', u'MONO_2_ENF_2014',
#       u'MONO_3_ENF_2014', u'MONO_4plus_ENF_2014'

<commit_msg>Fix path in script<commit_after># -*- coding: utf-8 -*-
"""
Created on Fri Oct  2 15:08:22 2015

@author: GILLES Armand
"""

import pandas as pd
import glob

df = pd.read_csv('source/ConfigFamiliale2009.csv', sep=";")

df.columns = ['Communes', 'Codes_Insee', 'NB_Allocataires_2009',
              'COUP_0_ENF_2009', 'COUP_1_ENF_2009', 'COUP_2_ENF_2009', 'COUP_3_ENF_2009', 'COUP_4plus_ENF_2009',
              'Homme_Isole_2009', 'Femme_Isolee_2009', 'MONO_1_ENF_2009', 'MONO_2_ENF_2009',
              'MONO_3_ENF_2009', 'MONO_4plus_ENF_2009']

files = glob.glob('source/ConfigFamiliale*')

for path_file in files:
    year = str(path_file[-8:-4])
    if (year != '2009'):
        df_temp = pd.read_csv(path_file, sep=';')

        # Rename Col with year
        year_col = ['Communes', 'Codes_Insee']
        features_col = []
        for col in df_temp.columns[-12:]:
            year_col.append(col +"_"+ year)
            features_col.append(col +"_"+ year)

        # Adding key for mergeing
        features_col.append('Codes_Insee')
        df_temp.columns = year_col
        df = pd.merge(df, df_temp[features_col], how='inner', on='Codes_Insee')

# Rename col to have unique name in futur merge
list_col = []
for col in df.columns:
    if "nb_allocataires" in col.lower(): # NB_Allocataires (2009) != NB_allocataires (2010)
        list_col.append(col+"_CF") # CF = ConfigFamiliale
    else:
        list_col.append(col)
df.columns = list_col

df.to_csv('data/full_ConfigFamiliale.csv', encoding='utf-8', index=False)

## Features
#u'NB_Allocataires_2009_CF',
#       u'COUP_0_ENF_2009', u'COUP_1_ENF_2009', u'COUP_2_ENF_2009',
#       u'COUP_3_ENF_2009', u'COUP_4plus_ENF_2009', u'Homme_Isole_2009',
#       u'Femme_Isolee_2009', u'MONO_1_ENF_2009', u'MONO_2_ENF_2009',
#       u'MONO_3_ENF_2009', u'MONO_4plus_ENF_2009', u'NB_allocataires_2010_CF',
#       u'COUP_0_ENF_2010', u'COUP_1_ENF_2010', u'COUP_2_ENF_2010',
#       u'COUP_3_ENF_2010', u'COUP_4plus_ENF_2010', u'Homme_Isole_2010',
#       u'Femme_Isolee_2010', u'MONO_1_ENF_2010', u'MONO_2_ENF_2010',
#       u'MONO_3_ENF_2010', u'MONO_4plus_ENF_2010', u'NB_allocataires_2011_CF',
#       u'COUP_0_ENF_2011', u'COUP_1_ENF_2011', u'COUP_2_ENF_2011',
#       u'COUP_3_ENF_2011', u'COUP_4plus_ENF_2011', u'Homme_Isole_2011',
#       u'Femme_Isolee_2011', u'MONO_1_ENF_2011', u'MONO_2_ENF_2011',
#       u'MONO_3_ENF_2011', u'MONO_4plus_ENF_2011', u'NB_allocataires_2012_CF',
#       u'COUP_0_ENF_2012', u'COUP_1_ENF_2012', u'COUP_2_ENF_2012',
#       u'COUP_3_ENF_2012', u'COUP_4plus_ENF_2012', u'Homme_Isole_2012',
#       u'Femme_Isolee_2012', u'MONO_1_ENF_2012', u'MONO_2_ENF_2012',
#       u'MONO_3_ENF_2012', u'MONO_4plus_ENF_2012', u'NB_allocataires_2013_CF',
#       u'COUP_0_ENF_2013', u'COUP_1_ENF_2013', u'COUP_2_ENF_2013',
#       u'COUP_3_ENF_2013', u'COUP_4plus_ENF_2013', u'Homme_Isole_2013',
#       u'Femme_Isolee_2013', u'MONO_1_ENF_2013', u'MONO_2_ENF_2013',
#       u'MONO_3_ENF_2013', u'MONO_4plus_ENF_2013', u'NB_allocataires_2014_CF',
#       u'COUP_0_ENF_2014', u'COUP_1_ENF_2014', u'COUP_2_ENF_2014',
#       u'COUP_3_ENF_2014', u'COUP_4plus_ENF_2014', u'Homme_Isole_2014',
#       u'Femme_Isolee_2014', u'MONO_1_ENF_2014', u'MONO_2_ENF_2014',
#       u'MONO_3_ENF_2014', u'MONO_4plus_ENF_2014'

<|endoftext|>


<commit_before># Copyright 2016 Joao Cardoso
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import six

from palettable.colorbrewer.colorbrewer import _load_maps_by_type as load_colorbrewer_palette  # noqa

from palettable.cubehelix.cubehelix import _get_all_maps as load_cubehelix_palettes
from palettable.tableau.tableau import _get_all_maps as load_tableau_palettes
from palettable.wesanderson.wesanderson import _get_all_maps as load_wesanderson_palettes

from palettable.palette import Palette

class PaletteMapper(object):
    def __init__(self):
        self._palettes = {}

    @staticmethod
    def _split_palette_name(name):
        split = name.split("_")
        if len(split) == 1:
            raise ValueError("Invalid palette description '%s'" % name)
        name = split[0]
        reverse = False
        if len(split) == 2:
            size = int(split[1])
        else:
            if split[-1] == "r":
                reverse = True
                size = int(split[-2])
            else:
                size = int(split[-1])

        return name, size, reverse

    def add_palettalbe_palette(self, name, palette):
        assert isinstance(palette, Palette)
        name, size, reverse = self._split_palette_name(name)

        if name not in self._palettes:
            self._palettes[name] = {}
        if size not in self._palettes[name]:
            self._palettes[name][size] = {}
        self._palettes[name][size][reverse] = palette

    def map_palette(self, name, n=3, reverse=False):
        try:
            name, size, reverse = self._split_palette_name(name)
        except ValueError:
            size = n

        palette = self._palettes[name]
        max_size = max(palette.keys())
        min_size = min(palette.keys())
        if size < min_size:
            size = min_size
        if size > max_size:
            size = max_size

        return palette[size][reverse]

mapper = PaletteMapper()

for key, obj in six.iteritems(load_colorbrewer_palette("diverging")):
    mapper.add_palettalbe_palette(key, obj)

for key, obj in six.iteritems(load_colorbrewer_palette("sequential")):
    mapper.add_palettalbe_palette(key, obj)

for key, obj in six.iteritems(load_colorbrewer_palette("qualitative")):
    mapper.add_palettalbe_palette(key, obj)

for key, obj in six.iteritems(load_cubehelix_palettes()):
    mapper.add_palettalbe_palette(key, obj)

for key, obj in six.iteritems(load_tableau_palettes()):
    mapper.add_palettalbe_palette(key, obj)

for key, obj in six.iteritems(load_wesanderson_palettes()):
    mapper.add_palettalbe_palette(key, obj)
<commit_msg>fix: copyright header<commit_after># Copyright 2016 The Novo Nordisk Foundation Center for Biosustainability
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import six

from palettable.colorbrewer.colorbrewer import _load_maps_by_type as load_colorbrewer_palette  # noqa

from palettable.cubehelix.cubehelix import _get_all_maps as load_cubehelix_palettes
from palettable.tableau.tableau import _get_all_maps as load_tableau_palettes
from palettable.wesanderson.wesanderson import _get_all_maps as load_wesanderson_palettes

from palettable.palette import Palette

class PaletteMapper(object):
    def __init__(self):
        self._palettes = {}

    @staticmethod
    def _split_palette_name(name):
        split = name.split("_")
        if len(split) == 1:
            raise ValueError("Invalid palette description '%s'" % name)
        name = split[0]
        reverse = False
        if len(split) == 2:
            size = int(split[1])
        else:
            if split[-1] == "r":
                reverse = True
                size = int(split[-2])
            else:
                size = int(split[-1])

        return name, size, reverse

    def add_palettalbe_palette(self, name, palette):
        assert isinstance(palette, Palette)
        name, size, reverse = self._split_palette_name(name)

        if name not in self._palettes:
            self._palettes[name] = {}
        if size not in self._palettes[name]:
            self._palettes[name][size] = {}
        self._palettes[name][size][reverse] = palette

    def map_palette(self, name, n=3, reverse=False):
        try:
            name, size, reverse = self._split_palette_name(name)
        except ValueError:
            size = n

        palette = self._palettes[name]
        max_size = max(palette.keys())
        min_size = min(palette.keys())
        if size < min_size:
            size = min_size
        if size > max_size:
            size = max_size

        return palette[size][reverse]

mapper = PaletteMapper()

for key, obj in six.iteritems(load_colorbrewer_palette("diverging")):
    mapper.add_palettalbe_palette(key, obj)

for key, obj in six.iteritems(load_colorbrewer_palette("sequential")):
    mapper.add_palettalbe_palette(key, obj)

for key, obj in six.iteritems(load_colorbrewer_palette("qualitative")):
    mapper.add_palettalbe_palette(key, obj)

for key, obj in six.iteritems(load_cubehelix_palettes()):
    mapper.add_palettalbe_palette(key, obj)

for key, obj in six.iteritems(load_tableau_palettes()):
    mapper.add_palettalbe_palette(key, obj)

for key, obj in six.iteritems(load_wesanderson_palettes()):
    mapper.add_palettalbe_palette(key, obj)
<|endoftext|>


<commit_before>#!/usr/bin/python

import json
import logging
import os
import socket
import subprocess

# vttop is the toplevel of the vitess source tree
vttop = os.environ['VTTOP']

# vtroot is where everything gets installed
vtroot = os.environ['VTROOT']

# vtdataroot is where to put all the data files
vtdataroot = os.environ.get('VTDATAROOT', '/vt')

# tmproot is the temporary place to put all test files
tmproot = os.path.join(vtdataroot, 'tmp')

# vtlogroot is where to put all the log files
vtlogroot = tmproot

# where to start allocating ports from
vtportstart = int(os.environ.get('VTPORTSTART', '6700'))

# url in which binaries export their status.
status_url = '/debug/status'

# which flavor of mysql to use in tests
mysql_flavor = os.environ.get('MYSQL_FLAVOR', 'GoogleMysql')

# location of the curl binary, used for some tests.
curl_bin = '/usr/bin/curl'

mysql_client_bin = 'mysql'

def memcached_bin():
  in_vt = os.path.join(vtroot, 'bin', 'memcached')
  if os.path.exists(in_vt):
    return in_vt
  return 'memcached'

# url to hit to force the logs to flush.
flush_logs_url = '/debug/flushlogs'

def setup():
  global tmproot
  try:
    os.makedirs(tmproot)
  except OSError:
    # directory already exists
    pass

# port management: reserve count consecutive ports, returns the first one
def reserve_ports(count):
  global vtportstart
  result = vtportstart
  vtportstart += count
  return result

# simple run command, cannot use utils.run to avoid circular dependencies
def run(args, raise_on_error=True, **kargs):
  proc = subprocess.Popen(args,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE,
                          **kargs)
  stdout, stderr = proc.communicate()
  if proc.returncode:
    if raise_on_error:
      raise Exception('Command failed: ' + ' '.join(args) + ':\n' + stdout +
                      stderr)
    else:
      logging.error('Command failed: %s:\n%s%s', ' '.join(args), stdout, stderr)

# compile command line programs, only once
compiled_progs = []
def prog_compile(name):
  if name in compiled_progs:
    return
  compiled_progs.append(name)
  logging.debug('Compiling %s', name)
  run(['go', 'install'], cwd=os.path.join(vttop, 'go', 'cmd', name))

# binary management: returns the full path for a binary
def binary_path(name):
  prog_compile(name)
  return os.path.join(vtroot, 'bin', name)

# binary management for the MySQL distribution.
def mysql_binary_path(name):
  return name

# topology server management: we use zookeeper in all the tests
topo_server_implementation = 'zookeeper'
hostname = socket.gethostname()
zk_port_base = reserve_ports(3)
zkocc_port_base = reserve_ports(3)
def topo_server_setup(add_bad_host=False):
  global zk_port_base
  global zkocc_port_base
  zk_ports = ":".join([str(zk_port_base), str(zk_port_base+1), str(zk_port_base+2)])
  run([binary_path('zkctl'),
       '-log_dir', vtlogroot,
       '-zk.cfg', '1@%s:%s' % (hostname, zk_ports),
       'init'])
  config = tmproot+'/test-zk-client-conf.json'
  with open(config, 'w') as f:
    ca_server = 'localhost:%u' % (zk_port_base+2)
    if add_bad_host:
      ca_server += ',does.not.exists:1234'
    zk_cell_mapping = {'test_nj': 'localhost:%u'%(zk_port_base+2),
                       'test_ny': 'localhost:%u'%(zk_port_base+2),
                       'test_ca': ca_server,
                       'global': 'localhost:%u'%(zk_port_base+2),
                       'test_nj:_zkocc': 'localhost:%u,localhost:%u,localhost:%u'%(zkocc_port_base,zkocc_port_base+1,zkocc_port_base+2),
                       'test_ny:_zkocc': 'localhost:%u'%(zkocc_port_base),
                       'test_ca:_zkocc': 'localhost:%u'%(zkocc_port_base),
                       'global:_zkocc': 'localhost:%u'%(zkocc_port_base),}
    json.dump(zk_cell_mapping, f)
  os.putenv('ZK_CLIENT_CONFIG', config)
  run([binary_path('zk'), 'touch', '-p', '/zk/test_nj/vt'])
  run([binary_path('zk'), 'touch', '-p', '/zk/test_ny/vt'])
  run([binary_path('zk'), 'touch', '-p', '/zk/test_ca/vt'])

def topo_server_teardown():
  global zk_port_base
  zk_ports = ":".join([str(zk_port_base), str(zk_port_base+1), str(zk_port_base+2)])
  run([binary_path('zkctl'),
       '-log_dir', vtlogroot,
       '-zk.cfg', '1@%s:%s' % (hostname, zk_ports),
       'teardown'], raise_on_error=False)

def topo_server_wipe():
  # Work around safety check on recursive delete.
  run([binary_path('zk'), 'rm', '-rf', '/zk/test_nj/vt/*'])
  run([binary_path('zk'), 'rm', '-rf', '/zk/test_ny/vt/*'])
  run([binary_path('zk'), 'rm', '-rf', '/zk/global/vt/*'])

  run([binary_path('zk'), 'rm', '-f', '/zk/test_nj/vt'])
  run([binary_path('zk'), 'rm', '-f', '/zk/test_ny/vt'])
  run([binary_path('zk'), 'rm', '-f', '/zk/global/vt'])

def topo_server_flags():
  return ['-topo_implementation', 'zookeeper']

def tablet_manager_protocol_flags():
  return ['-tablet_manager_protocol', 'bson']

def tabletconn_protocol_flags():
  return ['-tablet_protocol', 'gorpc']
<commit_msg>Fixing mysql_binary_path.<commit_after>#!/usr/bin/python

import json
import logging
import os
import socket
import subprocess

# vttop is the toplevel of the vitess source tree
vttop = os.environ['VTTOP']

# vtroot is where everything gets installed
vtroot = os.environ['VTROOT']

# vtdataroot is where to put all the data files
vtdataroot = os.environ.get('VTDATAROOT', '/vt')

# tmproot is the temporary place to put all test files
tmproot = os.path.join(vtdataroot, 'tmp')

# vtlogroot is where to put all the log files
vtlogroot = tmproot

# where to start allocating ports from
vtportstart = int(os.environ.get('VTPORTSTART', '6700'))

# url in which binaries export their status.
status_url = '/debug/status'

# which flavor of mysql to use in tests
mysql_flavor = os.environ.get('MYSQL_FLAVOR', 'GoogleMysql')

# location of the curl binary, used for some tests.
curl_bin = '/usr/bin/curl'

def memcached_bin():
  in_vt = os.path.join(vtroot, 'bin', 'memcached')
  if os.path.exists(in_vt):
    return in_vt
  return 'memcached'

# url to hit to force the logs to flush.
flush_logs_url = '/debug/flushlogs'

def setup():
  global tmproot
  try:
    os.makedirs(tmproot)
  except OSError:
    # directory already exists
    pass

# port management: reserve count consecutive ports, returns the first one
def reserve_ports(count):
  global vtportstart
  result = vtportstart
  vtportstart += count
  return result

# simple run command, cannot use utils.run to avoid circular dependencies
def run(args, raise_on_error=True, **kargs):
  proc = subprocess.Popen(args,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE,
                          **kargs)
  stdout, stderr = proc.communicate()
  if proc.returncode:
    if raise_on_error:
      raise Exception('Command failed: ' + ' '.join(args) + ':\n' + stdout +
                      stderr)
    else:
      logging.error('Command failed: %s:\n%s%s', ' '.join(args), stdout, stderr)

# compile command line programs, only once
compiled_progs = []
def prog_compile(name):
  if name in compiled_progs:
    return
  compiled_progs.append(name)
  logging.debug('Compiling %s', name)
  run(['go', 'install'], cwd=os.path.join(vttop, 'go', 'cmd', name))

# binary management: returns the full path for a binary
def binary_path(name):
  prog_compile(name)
  return os.path.join(vtroot, 'bin', name)

# binary management for the MySQL distribution.
def mysql_binary_path(name):
  return os.path.join(vtroot, 'dist', 'mysql', 'bin', name)

# topology server management: we use zookeeper in all the tests
topo_server_implementation = 'zookeeper'
hostname = socket.gethostname()
zk_port_base = reserve_ports(3)
zkocc_port_base = reserve_ports(3)
def topo_server_setup(add_bad_host=False):
  global zk_port_base
  global zkocc_port_base
  zk_ports = ":".join([str(zk_port_base), str(zk_port_base+1), str(zk_port_base+2)])
  run([binary_path('zkctl'),
       '-log_dir', vtlogroot,
       '-zk.cfg', '1@%s:%s' % (hostname, zk_ports),
       'init'])
  config = tmproot+'/test-zk-client-conf.json'
  with open(config, 'w') as f:
    ca_server = 'localhost:%u' % (zk_port_base+2)
    if add_bad_host:
      ca_server += ',does.not.exists:1234'
    zk_cell_mapping = {'test_nj': 'localhost:%u'%(zk_port_base+2),
                       'test_ny': 'localhost:%u'%(zk_port_base+2),
                       'test_ca': ca_server,
                       'global': 'localhost:%u'%(zk_port_base+2),
                       'test_nj:_zkocc': 'localhost:%u,localhost:%u,localhost:%u'%(zkocc_port_base,zkocc_port_base+1,zkocc_port_base+2),
                       'test_ny:_zkocc': 'localhost:%u'%(zkocc_port_base),
                       'test_ca:_zkocc': 'localhost:%u'%(zkocc_port_base),
                       'global:_zkocc': 'localhost:%u'%(zkocc_port_base),}
    json.dump(zk_cell_mapping, f)
  os.putenv('ZK_CLIENT_CONFIG', config)
  run([binary_path('zk'), 'touch', '-p', '/zk/test_nj/vt'])
  run([binary_path('zk'), 'touch', '-p', '/zk/test_ny/vt'])
  run([binary_path('zk'), 'touch', '-p', '/zk/test_ca/vt'])

def topo_server_teardown():
  global zk_port_base
  zk_ports = ":".join([str(zk_port_base), str(zk_port_base+1), str(zk_port_base+2)])
  run([binary_path('zkctl'),
       '-log_dir', vtlogroot,
       '-zk.cfg', '1@%s:%s' % (hostname, zk_ports),
       'teardown'], raise_on_error=False)

def topo_server_wipe():
  # Work around safety check on recursive delete.
  run([binary_path('zk'), 'rm', '-rf', '/zk/test_nj/vt/*'])
  run([binary_path('zk'), 'rm', '-rf', '/zk/test_ny/vt/*'])
  run([binary_path('zk'), 'rm', '-rf', '/zk/global/vt/*'])

  run([binary_path('zk'), 'rm', '-f', '/zk/test_nj/vt'])
  run([binary_path('zk'), 'rm', '-f', '/zk/test_ny/vt'])
  run([binary_path('zk'), 'rm', '-f', '/zk/global/vt'])

def topo_server_flags():
  return ['-topo_implementation', 'zookeeper']

def tablet_manager_protocol_flags():
  return ['-tablet_manager_protocol', 'bson']

def tabletconn_protocol_flags():
  return ['-tablet_protocol', 'gorpc']
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Run the Skia benchmarking executable."""

import os
import sys

from build_step import BuildStep
from utils import gclient_utils

import builder_name_schema

class RunNanobench(BuildStep):
  """A BuildStep that runs nanobench."""

  def __init__(self, timeout=9600, no_output_timeout=9600, **kwargs):
    super(RunNanobench, self).__init__(timeout=timeout,
                                       no_output_timeout=no_output_timeout,
                                       **kwargs)

  def _KeyParams(self):
    """Build a unique key from the builder name (as a list).

    E.g.:  os Mac10.6 model MacMini4.1 gpu GeForce320M arch x86

    This info is used by nanobench in its JSON output.
    """
    params = builder_name_schema.DictForBuilderName(self._builder_name)
    blacklist = ['configuration', 'role', 'is_trybot']
    # Don't include role (always Perf) or configuration (always Release).
    # TryBots can use the same exact key as they are uploaded to a different
    # location.
    #
    # It would be great to simplify this even further, but right now we have
    # two models for the same GPU (GalaxyNexus/NexusS for SGX540) and two
    # gpus for the same model (ShuttleA for GTX660/HD7770).
    for name in blacklist:
      if name in params:
        del params[name]
    flat = []
    for k in sorted(params.keys()):
      flat.append(k)
      flat.append(params[k])
    return flat

  def _JSONPath(self):
    git_timestamp = gclient_utils.GetGitRepoPOSIXTimestamp()
    return os.path.join(
            self._device_dirs.PerfDir(),
            'nanobench_%s_%s.json' % ( self._got_revision, git_timestamp))

  def _AnyMatch(self, *args):
    return any(arg in self._builder_name for arg in args)

  def _Run(self):
    args = ['-i', self._device_dirs.ResourceDir(),
            '--skps', self._device_dirs.SKPDir(),
            '--scales', '1.0', '1.1']
    if self._AnyMatch('Valgrind'):
      args.extend(['--loops', '1'])  # Don't care about performance on Valgrind.
    elif self._perf_data_dir:
      args.extend(['--outResultsFile', self._JSONPath()])
      args.append('--key')
      args.extend(self._KeyParams())
      args.append('--properties')
      args.extend(['gitHash', self._got_revision,
                   'build_number', str(self._build_number)])

    match  = []
    # Disable known problems.
    if self._AnyMatch('Android'):
      # Segfaults when run as GPU bench.  Very large texture?
      match.append('~blurroundrect')
      match.append('~patch_grid')  # skia:2847

    if self._AnyMatch('GalaxyNexus'):
      # Covered by faster CPUs in the same processor family (N7).
      args.append('--nocpu')

    if self._AnyMatch('HD2000'):
      # skia:2895
      match.extend(['~gradient', '~etc1bitmap'])

    if match:
      args.append('--match')
      args.extend(match)

    self._flavor_utils.RunFlavoredCmd('nanobench', args)

    # See skia:2789
    if self._AnyMatch('Valgrind'):
      abandonGpuContext = list(args)
      abandonGpuContext.append('--abandonGpuContext')
      abandonGpuContext.append('--nocpu')
      self._flavor_utils.RunFlavoredCmd('nanobench', abandonGpuContext)

if '__main__' == __name__:
  sys.exit(BuildStep.RunBuildStep(RunNanobench))
<commit_msg>Revert of experimentally re-enable N7 GPU benchmarking (patchset #1 id:1 of https://codereview.chromium.org/523933002/)<commit_after>#!/usr/bin/env python
# Copyright (c) 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Run the Skia benchmarking executable."""

import os
import sys

from build_step import BuildStep
from utils import gclient_utils

import builder_name_schema

class RunNanobench(BuildStep):
  """A BuildStep that runs nanobench."""

  def __init__(self, timeout=9600, no_output_timeout=9600, **kwargs):
    super(RunNanobench, self).__init__(timeout=timeout,
                                       no_output_timeout=no_output_timeout,
                                       **kwargs)

  def _KeyParams(self):
    """Build a unique key from the builder name (as a list).

    E.g.:  os Mac10.6 model MacMini4.1 gpu GeForce320M arch x86

    This info is used by nanobench in its JSON output.
    """
    params = builder_name_schema.DictForBuilderName(self._builder_name)
    blacklist = ['configuration', 'role', 'is_trybot']
    # Don't include role (always Perf) or configuration (always Release).
    # TryBots can use the same exact key as they are uploaded to a different
    # location.
    #
    # It would be great to simplify this even further, but right now we have
    # two models for the same GPU (GalaxyNexus/NexusS for SGX540) and two
    # gpus for the same model (ShuttleA for GTX660/HD7770).
    for name in blacklist:
      if name in params:
        del params[name]
    flat = []
    for k in sorted(params.keys()):
      flat.append(k)
      flat.append(params[k])
    return flat

  def _JSONPath(self):
    git_timestamp = gclient_utils.GetGitRepoPOSIXTimestamp()
    return os.path.join(
            self._device_dirs.PerfDir(),
            'nanobench_%s_%s.json' % ( self._got_revision, git_timestamp))

  def _AnyMatch(self, *args):
    return any(arg in self._builder_name for arg in args)

  def _Run(self):
    args = ['-i', self._device_dirs.ResourceDir(),
            '--skps', self._device_dirs.SKPDir(),
            '--scales', '1.0', '1.1']
    if self._AnyMatch('Valgrind'):
      args.extend(['--loops', '1'])  # Don't care about performance on Valgrind.
    elif self._perf_data_dir:
      args.extend(['--outResultsFile', self._JSONPath()])
      args.append('--key')
      args.extend(self._KeyParams())
      args.append('--properties')
      args.extend(['gitHash', self._got_revision,
                   'build_number', str(self._build_number)])

    match  = []
    # Disable known problems.
    if self._AnyMatch('Android'):
      # Segfaults when run as GPU bench.  Very large texture?
      match.append('~blurroundrect')
      match.append('~patch_grid')  # skia:2847

    if self._AnyMatch('Nexus7'):
      args.append('--nogpu')  # skia:2774

    if self._AnyMatch('GalaxyNexus'):
      # Covered by faster CPUs in the same processor family (N7).
      args.append('--nocpu')

    if self._AnyMatch('HD2000'):
      # skia:2895
      match.extend(['~gradient', '~etc1bitmap'])

    if match:
      args.append('--match')
      args.extend(match)

    self._flavor_utils.RunFlavoredCmd('nanobench', args)

    # See skia:2789
    if self._AnyMatch('Valgrind'):
      abandonGpuContext = list(args)
      abandonGpuContext.append('--abandonGpuContext')
      abandonGpuContext.append('--nocpu')
      self._flavor_utils.RunFlavoredCmd('nanobench', abandonGpuContext)

if '__main__' == __name__:
  sys.exit(BuildStep.RunBuildStep(RunNanobench))
<|endoftext|>


<commit_before>from __future__ import (absolute_import, print_function)

import boto
from boto.s3.connection import S3Connection
from yaml import load

try:
    # LibYAML based parser and emitter
    from yaml import CLoader as Loader
except ImportError:
    from yaml import Loader
import os
import time
import glob
import logging
import subprocess
import multiprocessing
from multiprocessing.dummy import Pool

from cassandra_snapshotter import logging_helper
from cassandra_snapshotter.timeout import timeout
from cassandra_snapshotter.utils import (add_s3_arguments, base_parser,
                                         map_wrap, get_s3_connection_host,
                                         check_lzop, check_pv, compressed_pipe)

DEFAULT_CONCURRENCY = max(multiprocessing.cpu_count() - 1, 1)
BUFFER_SIZE = 64  # Default bufsize is 64M
MBFACTOR = float(1 << 20)
MAX_RETRY_COUNT = 4
SLEEP_TIME = 2
SLEEP_MULTIPLIER = 3
UPLOAD_TIMEOUT = 600
DEFAULT_REDUCED_REDUNDANCY=False

logging_helper.configure(
    format='%(name)-12s %(levelname)-8s %(message)s')

logger = logging_helper.CassandraSnapshotterLogger('cassandra_snapshotter.agent')
boto.set_stream_logger('boto', logging.WARNING)

def get_bucket(
        s3_bucket, aws_access_key_id,
        aws_secret_access_key, s3_connection_host):
    connection = S3Connection(
        aws_access_key_id=aws_access_key_id,
        aws_secret_access_key=aws_secret_access_key,
        host=s3_connection_host
    )
    return connection.get_bucket(s3_bucket, validate=False)

def destination_path(s3_base_path, file_path, compressed=True):
    suffix = compressed and '.lzo' or ''
    return '/'.join([s3_base_path, file_path + suffix])

def s3_progress_update_callback(*args):
    # TODO: use this to display some nice progress bar
    pass

@map_wrap
def upload_file(bucket, source, destination, s3_ssenc, bufsize, reduced_redundancy, rate_limit):
    mp = None
    retry_count = 0
    sleep_time = SLEEP_TIME
    while True:
        try:
            if mp is None:
                # Initiate the multi-part upload.
                try:
                    mp = bucket.initiate_multipart_upload(destination, encrypt_key=s3_ssenc, reduced_redundancy=reduced_redundancy)
                    logger.info("Initialized multipart upload for file {!s} to {!s}".format(source, destination))
                except Exception as exc:
                    logger.error("Error while initializing multipart upload for file {!s} to {!s}".format(source, destination))
                    logger.error(exc.message)
                    raise

            try:
                for i, chunk in enumerate(compressed_pipe(source, bufsize, rate_limit)):
                    mp.upload_part_from_file(chunk, i + 1, cb=s3_progress_update_callback)
            except Exception as exc:
                logger.error("Error uploading file {!s} to {!s}".format(source, destination))
                logger.error(exc.message)
                raise

            try:
                mp.complete_upload()
            except Exception as exc:
                logger.error("Error completing multipart file upload for file {!s} to {!s}".format(source, destination))
                logger.error(exc.message)
                # The multi-part object may be in a bad state.  Extract an error
                # message if we can, then discard it.
                try:
                    logger.error(mp.to_xml())
                except Exception as exc:
                    pass
                cancel_upload(bucket, mp, destination)
                mp = None
                raise

            # Successful upload, return the uploaded file.
            return source
        except Exception as exc:
            # Failure anywhere reaches here.
            retry_count = retry_count + 1
            if retry_count > MAX_RETRY_COUNT:
                logger.error("Retried too many times uploading file {!s}".format(source))
                # Abort the multi-part upload if it was ever initiated.
                if mp is not None:
                    cancel_upload(bucket, mp, destination)
                return None
            else:
                logger.info("Sleeping before retry")
                time.sleep(sleep_time)
                sleep_time = sleep_time * SLEEP_MULTIPLIER
                logger.info("Retrying {}/{}".format(retry_count, MAX_RETRY_COUNT))
                # Go round again.

@timeout(UPLOAD_TIMEOUT)
def upload_chunk(mp, chunk, index):
    mp.upload_part_from_file(chunk, index)

def cancel_upload(bucket, mp, remote_path):
    """
    Safe way to cancel a multipart upload
    sleeps SLEEP_TIME seconds and then makes sure that there are not parts left
    in storage
    """
    attempts = 0
    while attempts < 5:
        try:
            time.sleep(SLEEP_TIME)
            mp.cancel_upload()
            time.sleep(SLEEP_TIME)
            for mp in bucket.list_multipart_uploads():
                if mp.key_name == remote_path:
                    mp.cancel_upload()
            return
        except Exception:
            logger.error("Error while cancelling multipart upload")
            attempts += 1

def put_from_manifest(
        s3_bucket, s3_connection_host, s3_ssenc, s3_base_path,
        aws_access_key_id, aws_secret_access_key, manifest,
        bufsize, reduced_redundancy, rate_limit, concurrency=None, incremental_backups=False):
    """
    Uploads files listed in a manifest to amazon S3
    to support larger than 5GB files multipart upload is used (chunks of 60MB)
    files are uploaded compressed with lzop, the .lzo suffix is appended
    """
    exit_code = 0
    bucket = get_bucket(
        s3_bucket, aws_access_key_id,
        aws_secret_access_key, s3_connection_host)
    manifest_fp = open(manifest, 'r')
    buffer_size = int(bufsize * MBFACTOR)
    files = manifest_fp.read().splitlines()
    pool = Pool(concurrency)
    for f in pool.imap(upload_file,
                       ((bucket, f, destination_path(s3_base_path, f), s3_ssenc, buffer_size, reduced_redundancy, rate_limit) for f in files if f)):
        if f is None:
            # Upload failed.
            exit_code = 1
        elif incremental_backups:
            # Delete files that were successfully uploaded.
            os.remove(f)
    pool.terminate()
    exit(exit_code)

def get_data_path(conf_path):
    """Retrieve cassandra data_file_directories from cassandra.yaml"""
    config_file_path = os.path.join(conf_path, 'cassandra.yaml')
    cassandra_configs = {}
    with open(config_file_path, 'r') as f:
        cassandra_configs = load(f, Loader=Loader)
    data_paths = cassandra_configs['data_file_directories']
    return data_paths

def create_upload_manifest(
        snapshot_name, snapshot_keyspaces, snapshot_table,
        conf_path, manifest_path, exclude_tables, incremental_backups=False):
    if snapshot_keyspaces:
        keyspace_globs = snapshot_keyspaces.split(',')
    else:
        keyspace_globs = ['*']

    if snapshot_table:
        table_glob = snapshot_table
    else:
        table_glob = '*'

    data_paths = get_data_path(conf_path)
    files = []
    exclude_tables_list = exclude_tables.split(',')
    for data_path in data_paths:
        for keyspace_glob in keyspace_globs:
            path = [
                data_path,
                keyspace_glob,
                table_glob
            ]
            if incremental_backups:
                path += ['backups']
            else:
                path += ['snapshots', snapshot_name]
            path += ['*']

            path = os.path.join(*path)
            if len(exclude_tables_list) > 0:
                for f in glob.glob(os.path.join(path)):
                    # Get the table name
                    # The current format of a file path looks like:
                    # /var/lib/cassandra/data03/system/compaction_history/snapshots/20151102182658/system-compaction_history-jb-6684-Summary.db
                    if f.split('/')[-4] not in exclude_tables_list:
                        files.append(f.strip())
            else:
                files.append(f.strip() for f in glob.glob(os.path.join(path)))

    with open(manifest_path, 'w') as manifest:
        manifest.write('\n'.join("%s" % f for f in files))

def main():
    subparsers = base_parser.add_subparsers(
        title='subcommands', dest='subcommand')
    base_parser.add_argument(
        '--incremental_backups', action='store_true', default=False)

    put_parser = subparsers.add_parser(
        'put', help="put files on s3 from a manifest")
    manifest_parser = subparsers.add_parser(
        'create-upload-manifest', help="put files on s3 from a manifest")

    # put arguments
    put_parser = add_s3_arguments(put_parser)
    put_parser.add_argument(
        '--bufsize',
        required=False,
        default=BUFFER_SIZE,
        type=int,
        help="Compress and upload buffer size")

    put_parser.add_argument(
        '--manifest',
        required=True,
        help="The manifest containing the files to put on s3")

    put_parser.add_argument(
        '--concurrency',
        required=False,
        default=DEFAULT_CONCURRENCY,
        type=int,
        help="Compress and upload concurrent processes")

    put_parser.add_argument(
        '--reduced-redundancy',
        required=False,
        default=DEFAULT_REDUCED_REDUNDANCY,
        action="store_true",
        help="Compress and upload concurrent processes")

    put_parser.add_argument(
        '--rate-limit',
        required=False,
        default=0,
        type=int,
        help="Limit the upload speed to S3 (by using 'pv'). Value expressed in kilobytes (*1024)")

    # create-upload-manifest arguments
    manifest_parser.add_argument('--snapshot_name', required=True, type=str)
    manifest_parser.add_argument('--conf_path', required=True, type=str)
    manifest_parser.add_argument('--manifest_path', required=True, type=str)
    manifest_parser.add_argument(
        '--snapshot_keyspaces', default='', required=False, type=str)
    manifest_parser.add_argument(
        '--snapshot_table', required=False, default='', type=str)
    manifest_parser.add_argument(
        '--exclude_tables', required=False, type=str)

    args = base_parser.parse_args()
    subcommand = args.subcommand

    if subcommand == 'create-upload-manifest':
        create_upload_manifest(
            args.snapshot_name,
            args.snapshot_keyspaces,
            args.snapshot_table,
            args.conf_path,
            args.manifest_path,
            args.exclude_tables,
            args.incremental_backups
        )

    if subcommand == 'put':
        check_lzop()

        if args.rate_limit > 0:
            check_pv()

        put_from_manifest(
            args.s3_bucket_name,
            get_s3_connection_host(args.s3_bucket_region),
            args.s3_ssenc,
            args.s3_base_path,
            args.aws_access_key_id,
            args.aws_secret_access_key,
            args.manifest,
            args.bufsize,
            args.reduced_redundancy,
            args.rate_limit,
            args.concurrency,
            args.incremental_backups,
            args.compress
        )

if __name__ == '__main__':
    # TODO: if lzop is not available we should fail or run without it
    check_lzop()
<commit_msg>In newer versions of cassandra, data_file_directories may be omitted<commit_after>from __future__ import (absolute_import, print_function)

import boto
from boto.s3.connection import S3Connection
from yaml import load

try:
    # LibYAML based parser and emitter
    from yaml import CLoader as Loader
except ImportError:
    from yaml import Loader
import os
import time
import glob
import logging
import subprocess
import multiprocessing
from multiprocessing.dummy import Pool

from cassandra_snapshotter import logging_helper
from cassandra_snapshotter.timeout import timeout
from cassandra_snapshotter.utils import (add_s3_arguments, base_parser,
                                         map_wrap, get_s3_connection_host,
                                         check_lzop, check_pv, compressed_pipe)

DEFAULT_CONCURRENCY = max(multiprocessing.cpu_count() - 1, 1)
BUFFER_SIZE = 64  # Default bufsize is 64M
MBFACTOR = float(1 << 20)
MAX_RETRY_COUNT = 4
SLEEP_TIME = 2
SLEEP_MULTIPLIER = 3
UPLOAD_TIMEOUT = 600
DEFAULT_REDUCED_REDUNDANCY=False

logging_helper.configure(
    format='%(name)-12s %(levelname)-8s %(message)s')

logger = logging_helper.CassandraSnapshotterLogger('cassandra_snapshotter.agent')
boto.set_stream_logger('boto', logging.WARNING)

def get_bucket(
        s3_bucket, aws_access_key_id,
        aws_secret_access_key, s3_connection_host):
    connection = S3Connection(
        aws_access_key_id=aws_access_key_id,
        aws_secret_access_key=aws_secret_access_key,
        host=s3_connection_host
    )
    return connection.get_bucket(s3_bucket, validate=False)

def destination_path(s3_base_path, file_path, compressed=True):
    suffix = compressed and '.lzo' or ''
    return '/'.join([s3_base_path, file_path + suffix])

def s3_progress_update_callback(*args):
    # TODO: use this to display some nice progress bar
    pass

@map_wrap
def upload_file(bucket, source, destination, s3_ssenc, bufsize, reduced_redundancy, rate_limit):
    mp = None
    retry_count = 0
    sleep_time = SLEEP_TIME
    while True:
        try:
            if mp is None:
                # Initiate the multi-part upload.
                try:
                    mp = bucket.initiate_multipart_upload(destination, encrypt_key=s3_ssenc, reduced_redundancy=reduced_redundancy)
                    logger.info("Initialized multipart upload for file {!s} to {!s}".format(source, destination))
                except Exception as exc:
                    logger.error("Error while initializing multipart upload for file {!s} to {!s}".format(source, destination))
                    logger.error(exc.message)
                    raise

            try:
                for i, chunk in enumerate(compressed_pipe(source, bufsize, rate_limit)):
                    mp.upload_part_from_file(chunk, i + 1, cb=s3_progress_update_callback)
            except Exception as exc:
                logger.error("Error uploading file {!s} to {!s}".format(source, destination))
                logger.error(exc.message)
                raise

            try:
                mp.complete_upload()
            except Exception as exc:
                logger.error("Error completing multipart file upload for file {!s} to {!s}".format(source, destination))
                logger.error(exc.message)
                # The multi-part object may be in a bad state.  Extract an error
                # message if we can, then discard it.
                try:
                    logger.error(mp.to_xml())
                except Exception as exc:
                    pass
                cancel_upload(bucket, mp, destination)
                mp = None
                raise

            # Successful upload, return the uploaded file.
            return source
        except Exception as exc:
            # Failure anywhere reaches here.
            retry_count = retry_count + 1
            if retry_count > MAX_RETRY_COUNT:
                logger.error("Retried too many times uploading file {!s}".format(source))
                # Abort the multi-part upload if it was ever initiated.
                if mp is not None:
                    cancel_upload(bucket, mp, destination)
                return None
            else:
                logger.info("Sleeping before retry")
                time.sleep(sleep_time)
                sleep_time = sleep_time * SLEEP_MULTIPLIER
                logger.info("Retrying {}/{}".format(retry_count, MAX_RETRY_COUNT))
                # Go round again.

@timeout(UPLOAD_TIMEOUT)
def upload_chunk(mp, chunk, index):
    mp.upload_part_from_file(chunk, index)

def cancel_upload(bucket, mp, remote_path):
    """
    Safe way to cancel a multipart upload
    sleeps SLEEP_TIME seconds and then makes sure that there are not parts left
    in storage
    """
    attempts = 0
    while attempts < 5:
        try:
            time.sleep(SLEEP_TIME)
            mp.cancel_upload()
            time.sleep(SLEEP_TIME)
            for mp in bucket.list_multipart_uploads():
                if mp.key_name == remote_path:
                    mp.cancel_upload()
            return
        except Exception:
            logger.error("Error while cancelling multipart upload")
            attempts += 1

def put_from_manifest(
        s3_bucket, s3_connection_host, s3_ssenc, s3_base_path,
        aws_access_key_id, aws_secret_access_key, manifest,
        bufsize, reduced_redundancy, rate_limit, concurrency=None, incremental_backups=False):
    """
    Uploads files listed in a manifest to amazon S3
    to support larger than 5GB files multipart upload is used (chunks of 60MB)
    files are uploaded compressed with lzop, the .lzo suffix is appended
    """
    exit_code = 0
    bucket = get_bucket(
        s3_bucket, aws_access_key_id,
        aws_secret_access_key, s3_connection_host)
    manifest_fp = open(manifest, 'r')
    buffer_size = int(bufsize * MBFACTOR)
    files = manifest_fp.read().splitlines()
    pool = Pool(concurrency)
    for f in pool.imap(upload_file,
                       ((bucket, f, destination_path(s3_base_path, f), s3_ssenc, buffer_size, reduced_redundancy, rate_limit) for f in files if f)):
        if f is None:
            # Upload failed.
            exit_code = 1
        elif incremental_backups:
            # Delete files that were successfully uploaded.
            os.remove(f)
    pool.terminate()
    exit(exit_code)

def get_data_path(conf_path):
    """Retrieve cassandra data_file_directories from cassandra.yaml"""
    config_file_path = os.path.join(conf_path, 'cassandra.yaml')
    cassandra_configs = {}
    with open(config_file_path, 'r') as f:
        cassandra_configs = load(f, Loader=Loader)
    data_paths = cassandra_configs.get("data_file_directories")
    if data_paths is None:
        data_paths = ['/var/lib/cassandra/data']
    return data_paths

def create_upload_manifest(
        snapshot_name, snapshot_keyspaces, snapshot_table,
        conf_path, manifest_path, exclude_tables, incremental_backups=False):
    if snapshot_keyspaces:
        keyspace_globs = snapshot_keyspaces.split(',')
    else:
        keyspace_globs = ['*']

    if snapshot_table:
        table_glob = snapshot_table
    else:
        table_glob = '*'

    data_paths = get_data_path(conf_path)
    files = []
    exclude_tables_list = exclude_tables.split(',')
    for data_path in data_paths:
        for keyspace_glob in keyspace_globs:
            path = [
                data_path,
                keyspace_glob,
                table_glob
            ]
            if incremental_backups:
                path += ['backups']
            else:
                path += ['snapshots', snapshot_name]
            path += ['*']

            path = os.path.join(*path)
            if len(exclude_tables_list) > 0:
                for f in glob.glob(os.path.join(path)):
                    # Get the table name
                    # The current format of a file path looks like:
                    # /var/lib/cassandra/data03/system/compaction_history/snapshots/20151102182658/system-compaction_history-jb-6684-Summary.db
                    if f.split('/')[-4] not in exclude_tables_list:
                        files.append(f.strip())
            else:
                files.append(f.strip() for f in glob.glob(os.path.join(path)))

    with open(manifest_path, 'w') as manifest:
        manifest.write('\n'.join("%s" % f for f in files))

def main():
    subparsers = base_parser.add_subparsers(
        title='subcommands', dest='subcommand')
    base_parser.add_argument(
        '--incremental_backups', action='store_true', default=False)

    put_parser = subparsers.add_parser(
        'put', help="put files on s3 from a manifest")
    manifest_parser = subparsers.add_parser(
        'create-upload-manifest', help="put files on s3 from a manifest")

    # put arguments
    put_parser = add_s3_arguments(put_parser)
    put_parser.add_argument(
        '--bufsize',
        required=False,
        default=BUFFER_SIZE,
        type=int,
        help="Compress and upload buffer size")

    put_parser.add_argument(
        '--manifest',
        required=True,
        help="The manifest containing the files to put on s3")

    put_parser.add_argument(
        '--concurrency',
        required=False,
        default=DEFAULT_CONCURRENCY,
        type=int,
        help="Compress and upload concurrent processes")

    put_parser.add_argument(
        '--reduced-redundancy',
        required=False,
        default=DEFAULT_REDUCED_REDUNDANCY,
        action="store_true",
        help="Compress and upload concurrent processes")

    put_parser.add_argument(
        '--rate-limit',
        required=False,
        default=0,
        type=int,
        help="Limit the upload speed to S3 (by using 'pv'). Value expressed in kilobytes (*1024)")

    # create-upload-manifest arguments
    manifest_parser.add_argument('--snapshot_name', required=True, type=str)
    manifest_parser.add_argument('--conf_path', required=True, type=str)
    manifest_parser.add_argument('--manifest_path', required=True, type=str)
    manifest_parser.add_argument(
        '--snapshot_keyspaces', default='', required=False, type=str)
    manifest_parser.add_argument(
        '--snapshot_table', required=False, default='', type=str)
    manifest_parser.add_argument(
        '--exclude_tables', required=False, type=str)

    args = base_parser.parse_args()
    subcommand = args.subcommand

    if subcommand == 'create-upload-manifest':
        create_upload_manifest(
            args.snapshot_name,
            args.snapshot_keyspaces,
            args.snapshot_table,
            args.conf_path,
            args.manifest_path,
            args.exclude_tables,
            args.incremental_backups
        )

    if subcommand == 'put':
        check_lzop()

        if args.rate_limit > 0:
            check_pv()

        put_from_manifest(
            args.s3_bucket_name,
            get_s3_connection_host(args.s3_bucket_region),
            args.s3_ssenc,
            args.s3_base_path,
            args.aws_access_key_id,
            args.aws_secret_access_key,
            args.manifest,
            args.bufsize,
            args.reduced_redundancy,
            args.rate_limit,
            args.concurrency,
            args.incremental_backups,
            args.compress
        )

if __name__ == '__main__':
    # TODO: if lzop is not available we should fail or run without it
    check_lzop()
<|endoftext|>


<commit_before>import enum
import json
import weakref
import functools

import cassiopeia.type.api.exception

class CassiopeiaObject(object):
    """An object storing data from the API, with various helpful utilities and shortcuts"""
    def __init__(self, data):
        """
        data    CassiopeiaDto    the underlying DTO object with the data for this type
        """
        if(data.__class__ is not self.dto_type):
            raise cassiopeia.type.api.exception.CassiopeiaException("Tried to instantiate a core {class_} with a {dto} dto!".format(class_=self.__class__.__name__, dto=data.__class__.__name__))
        self.data = data

    def to_json(self):
        """Gets a JSON representation of the object

        return    str    a JSON representation of the object
        """
        return self.data.to_json()

    def __str__(self):
        return str(self.data)

    def __eq__(self, other):
        return self.data == other.data

    def __ne__(self, other):
        return self.data != other.data

    def __hash__(self):
        return hash(id(self))

class LazyProperty(object):
    def __init__(self, method):
        """
        method    function    the method to turn into a lazy property
        """
        self.method = method
        self.values = weakref.WeakKeyDictionary()

    def __set__(self, obj, value):
        raise AttributeError("can't set attribute")

    def __delete__(self, obj):
        raise AttributeError("can't delete attribute")

    def __get__(self, obj, t=None):
        try:
            return self.values[obj]
        except(KeyError):
            self.values[obj] = self.method(obj)
            return self.values[obj]

def lazyproperty(method):
    """Makes a property load only once and store the result value to be returned to all later calls

    method    function    the method to turn into a lazy property

    return    function    the method as a lazy property
    """
    prop = LazyProperty(method)

    @property
    @functools.wraps(method)
    def lazy(self):
        return prop.__get__(self)

    return lazy

class immutablemethod(object):
    """Makes a method un-deletable and un-repleacable
    @decorator
    """
    def __init__(self, method):
        """
        method    function    the method to make immutable
        """
        self.method = method

    def __set__(self, obj, value):
        raise AttributeError("can't set method")

    def __delete__(self, obj):
        raise AttributeError("can't delete method")

    def __get__(self, obj, type=None):
        @functools.wraps(self.method)
        def curried(*args, **kwargs):
            return self.method(obj, *args, **kwargs)
        return curried

def inheritdocs(class_):
    """Makes a class inherit the documentation from any overridden methods
    @decorator

    class_    class    the class to make inherit documentation

    return    class    the class with inherited documentation
    """
    for name, method in vars(class_).items():
        if(not method.__doc__):
            for parent in class_.__bases__:
                try:
                    p_method = getattr(parent, name)
                    if(p_method and p_method.__doc__):
                        method.__doc__ = p_method.__doc__
                except AttributeError:
                    continue
    return class_

class LoadPolicy(enum.Enum):
    lazy = "LAZY"
    eager = "EAGER"

class Lane(enum.Enum):
    top_lane = "TOP"
    jungle = "JUNGLE"
    mid_lane = "MIDDLE"
    bot_lane = "BOTTOM"

    def for_id(id_):
        try:
            return Lane.by_id[id_]
        except:
            return None

Lane.by_id = {
    1: Lane.top_lane,
    2: Lane.mid_lane,
    3: Lane.jungle,
    4: Lane.bot_lane
}

class Role(enum.Enum):
    none = "NONE"
    solo = "SOLO"
    duo = "DUO"
    carry = "DUO_CARRY"
    support = "DUO_SUPPORT"

    def for_id(id_):
        try:
            return Role.by_id[id_]
        except:
            return None

Role.by_id = {
    1: Role.duo,
    2: Role.support,
    3: Role.carry,
    4: Role.solo
}

class Side(enum.Enum):
    blue = 100
    red = 200

class Queue(enum.Enum):
    custom = "CUSTOM"
    normal_blind_fives = "NORMAL_5x5_BLIND"
    bot_fives = "BOT_5x5"
    bot_intro_fives = "BOT_5x5_INTRO"
    bot_beginner_fives = "BOT_5x5_BEGINNER"
    bot_intermediate_fives = "BOT_5x5_INTERMEDIATE"
    normal_blind_threes = "NORMAL_3x3"
    normal_draft_fives = "NORMAL_5x5_DRAFT"
    dominion_blind = "ODIN_5x5_BLIND"
    dominion_draft = "ODIN_5x5_DRAFT"
    bot_dominion = "BOT_ODIN_5x5"
    ranked_solo = "RANKED_SOLO_5x5"
    ranked_premade_threes = "RANKED_PREMADE_3x3"
    ranked_premade_fives = "RANKED_PREMADE_5x5"
    ranked_threes = "RANKED_TEAM_3x3"
    ranked_fives = "RANKED_TEAM_5x5"
    bot_threes = "BOT_TT_3x3"
    team_builder = "GROUP_FINDER_5x5"
    aram = "ARAM_5x5"
    one_for_all = "ONEFORALL_5x5"
    showdown_solo = "FIRSTBLOOD_1x1"
    showdown_duo = "FIRSTBLOOD_2x2"
    hexakill_summoners_rift = "SR_6x6"
    urf = "URF_5x5"
    bot_urf = "BOT_URF_5x5"
    doom_bots_1 = "NIGHTMARE_BOT_5x5_RANK1"
    doom_bots_2 = "NIGHTMARE_BOT_5x5_RANK2"
    doom_bots_5 = "NIGHTMARE_BOT_5x5_RANK5"
    ascension = "ASCENSION_5x5"
    hexakill_twisted_treeline = "HEXAKILL"
    butchers_bridge = "BILGEWATER_ARAM_5x5"
    poro_king = "KING_PORO_5x5"
    nemesis_draft = "COUNTER_PICK"
    black_market = "BILGEWATER_5x5"

    def for_id(id_):
        try:
            return Queue.by_id[id_]
        except:
            return None

Queue.by_id = {
    0: Queue.custom,
    65: Queue.aram,
    2: Queue.normal_blind_fives,
    4: Queue.ranked_solo,
    6: Queue.ranked_premade_fives,
    7: Queue.bot_fives,
    8: Queue.normal_blind_threes,
    9: Queue.ranked_premade_threes,
    75: Queue.hexakill_summoners_rift,
    76: Queue.urf,
    14: Queue.normal_draft_fives,
    16: Queue.dominion_blind,
    17: Queue.dominion_draft,
    83: Queue.bot_urf,
    25: Queue.bot_dominion,
    91: Queue.doom_bots_1,
    92: Queue.doom_bots_2,
    93: Queue.doom_bots_5,
    31: Queue.bot_intro_fives,
    32: Queue.bot_beginner_fives,
    33: Queue.bot_intermediate_fives,
    98: Queue.hexakill_twisted_treeline,
    100: Queue.butchers_bridge,
    70: Queue.one_for_all,
    41: Queue.ranked_threes,
    42: Queue.ranked_fives,
    300: Queue.poro_king,
    96: Queue.ascension,
    72: Queue.showdown_solo,
    52: Queue.bot_threes,
    310: Queue.nemesis_draft,
    73: Queue.showdown_duo,
    313: Queue.black_market,
    61: Queue.team_builder
}
ranked_queues = {Queue.ranked_solo, Queue.ranked_threes, Queue.ranked_fives}

class Tier(enum.Enum):
    challenger = "CHALLENGER"
    master = "MASTER"
    diamond = "DIAMOND"
    platinum = "PLATINUM"
    gold = "GOLD"
    silver = "SILVER"
    bronze = "BRONZE"

class Division(enum.Enum):
    one = "I"
    two = "II"
    three = "III"
    four = "IV"
    five = "V"

class Season(enum.Enum):
    preseason_3 = "PRESEASON3"
    season_3 = "SEASON3"
    preseason_4 = "PRESEASON2014"
    season_4 = "SEASON2014"
    preseason_5 = "PRESEASON2015"
    season_5 = "SEASON2015"
    preseason_6 = "PRESEASON2016"
    season_6 = "SEASON2016"

stats_seasons = {Season.season_3, Season.season_4, Season.season_5}

class Region(enum.Enum):
    brazil = "br"
    europe_north_east = "eune"
    europe_west = "euw"
    korea = "kr"
    latin_america_north = "lan"
    latin_america_south = "las"
    north_america = "na"
    oceania = "oce"
    russia = "ru"
    turkey = "tr"

class Platform(enum.Enum):
    brazil = "BR1"
    europe_north_east = "EUN1"
    europe_west = "EUW1"
    korea = "KR"
    latin_america_north = "LA1"
    latin_america_south = "LA2"
    north_america = "NA1"
    oceania = "OC1"
    russia = "RU"
    turkey = "TR1"

class Map(enum.Enum):
    summoners_rift_summer = 1
    summoners_rift_autumn = 2
    the_proving_grounds = 3
    twisted_treeline_original = 4
    the_crystal_scar = 8
    twisted_treeline = 10
    summoners_rift = 11
    howling_abyss = 12
    butchers_bridge = 14

class GameMode(enum.Enum):
    aram = "ARAM"
    ascension = "ASCENSION"
    classic = "CLASSIC"
    showdown = "FIRSTBLOOD"
    poro_king = "KINGPORO"
    dominion = "ODIN"
    one_for_all = "ONEFORALL"
    tutorial = "TUTORIAL"

class GameType(enum.Enum):
    custom = "CUSTOM_GAME"
    tutorial = "TUTORIAL_GAME"
    matched = "MATCHED_GAME"

class SubType(enum.Enum):
    custom = "NONE"
    normal_fives = "NORMAL"
    normal_threes = "NORMAL_3x3"
    dominion = "ODIN_UNRANKED"
    aram = "ARAM_UNRANKED_5x5"
    bot_fives = "BOT"
    bot_threes = "BOT_3x3"
    ranked_solo = "RANKED_SOLO_5x5"
    ranked_threes = "RANKED_TEAM_3x3"
    ranked_fives = "RANKED_TEAM_5x5"
    one_for_all = "ONEFORALL_5x5"
    showdown_solo = "FIRSTBLOOD_1x1"
    showdown_duo = "FIRSTBLOOD_2x2"
    hexakill_summoners_rift = "SR_6x6"
    team_builder = "CAP_5x5"
    urf = "URF"
    bot_urf = "URF_BOT"
    doom_bots = "NIGHTMARE_BOT"
    ascension = "ASCENSION"
    hexakill_twisted_treeline = "HEXAKILL"
    poro_king = "KING_PORO"
    nemesis_draft = "COUNTER_PICK"
    black_market = "BILGEWATER"

class StatSummaryType(enum.Enum):
    normal_fives = "Unranked"
    normal_threes = "Unranked3x3"
    dominion = "OdinUnranked"
    aram = "AramUnranked5x5"
    bot_fives = "CoopVsAI"
    bot_threes = "CoopVsAI3x3"
    ranked_solo = "RankedSolo5x5"
    ranked_threes = "RankedTeam3x3"
    ranked_fives = "RankedTeam5x5"
    one_for_all = "OneForAll5x5"
    showdown_solo = "FirstBlood1x1"
    showdown_duo = "FirstBlood2x2"
    hexakill_summoners_rift = "SummonersRift6x6"
    team_builder = "CAP5x5"
    urf = "URF"
    bot_urf = "URFBots"
    doom_bots = "NightmareBot"
    ascension = "Ascension"
    hexakill_twisted_treeline = "Hexakill"
    poro_king = "KingPoro"
    nemesis_draft = "CounterPick"
    black_market = "Bilgewater"

class Ascended(enum.Enum):
    player = "CHAMPION_ASCENDED"
    death = "CLEAR_ASCENDED"
    npc = "MINION_ASCENDED"

class Building(enum.Enum):
    inhibitor = "INHIBITOR_BUILDING"
    turret = "TOWER_BUILDING"

class EventType(enum.Enum):
    ascension = "ASCENDED_EVENT"
    building_kill = "BUILDING_KILL"
    point_capture = "CAPTURE_POINT"
    kill = "CHAMPION_KILL"
    elite_monster_kill = "ELITE_MONSTER_KILL"
    item_destruction = "ITEM_DESTROYED"
    item_purchase = "ITEM_PURCHASED"
    item_sale = "ITEM_SOLD"
    item_undo = "ITEM_UNDO"
    summoning = "PORO_KING_SUMMON"
    skill_up = "SKILL_LEVEL_UP"
    ward_kill = "WARD_KILL"
    ward_placement = "WARD_PLACED"

class LaneType(enum.Enum):
    top_lane = "TOP_LANE"
    mid_lane = "MID_LANE"
    bot_lane = "BOT_LANE"

class LevelUp(enum.Enum):
    evolution = "EVOLVE"
    normal = "NORMAL"

class Monster(enum.Enum):
    baron = "BARON_NASHOR"
    dragon = "DRAGON"
    blue = "BLUE_GOLEM"
    red = "RED_LIZARD"
    spider = "VILEMAW"

class Point(enum.Enum):
    windmill = "POINT_C"
    drill = "POINT_D"
    boneyard = "POINT_E"
    quarry = "POINT_A"
    refinery = "POINT_B"

class Turret(enum.Enum):
    outer = "OUTER_TURRET"
    inner = "INNER_TURRET"
    inhibitor = "BASE_TURRET"
    nexus = "NEXUS_TURRET"
    fountain = "FOUNTAIN_TURRET"
    undefined = "UNDEFINED_TURRET"

class Ward(enum.Enum):
    sight = "SIGHT_WARD"
    vision = "VISION_WARD"
    trinket = "YELLOW_TRINKET"
    upgraded_trinket = "YELLOW_TRINKET_UPGRADE"
    mushroom = "TEEMO_MUSHROOM"
    undefined = "UNDEFINED"
<commit_msg>needed to add two more enums for the /stats/ endpoint<commit_after>import enum
import json
import weakref
import functools

import cassiopeia.type.api.exception

class CassiopeiaObject(object):
    """An object storing data from the API, with various helpful utilities and shortcuts"""
    def __init__(self, data):
        """
        data    CassiopeiaDto    the underlying DTO object with the data for this type
        """
        if(data.__class__ is not self.dto_type):
            raise cassiopeia.type.api.exception.CassiopeiaException("Tried to instantiate a core {class_} with a {dto} dto!".format(class_=self.__class__.__name__, dto=data.__class__.__name__))
        self.data = data

    def to_json(self):
        """Gets a JSON representation of the object

        return    str    a JSON representation of the object
        """
        return self.data.to_json()

    def __str__(self):
        return str(self.data)

    def __eq__(self, other):
        return self.data == other.data

    def __ne__(self, other):
        return self.data != other.data

    def __hash__(self):
        return hash(id(self))

class LazyProperty(object):
    def __init__(self, method):
        """
        method    function    the method to turn into a lazy property
        """
        self.method = method
        self.values = weakref.WeakKeyDictionary()

    def __set__(self, obj, value):
        raise AttributeError("can't set attribute")

    def __delete__(self, obj):
        raise AttributeError("can't delete attribute")

    def __get__(self, obj, t=None):
        try:
            return self.values[obj]
        except(KeyError):
            self.values[obj] = self.method(obj)
            return self.values[obj]

def lazyproperty(method):
    """Makes a property load only once and store the result value to be returned to all later calls

    method    function    the method to turn into a lazy property

    return    function    the method as a lazy property
    """
    prop = LazyProperty(method)

    @property
    @functools.wraps(method)
    def lazy(self):
        return prop.__get__(self)

    return lazy

class immutablemethod(object):
    """Makes a method un-deletable and un-repleacable
    @decorator
    """
    def __init__(self, method):
        """
        method    function    the method to make immutable
        """
        self.method = method

    def __set__(self, obj, value):
        raise AttributeError("can't set method")

    def __delete__(self, obj):
        raise AttributeError("can't delete method")

    def __get__(self, obj, type=None):
        @functools.wraps(self.method)
        def curried(*args, **kwargs):
            return self.method(obj, *args, **kwargs)
        return curried

def inheritdocs(class_):
    """Makes a class inherit the documentation from any overridden methods
    @decorator

    class_    class    the class to make inherit documentation

    return    class    the class with inherited documentation
    """
    for name, method in vars(class_).items():
        if(not method.__doc__):
            for parent in class_.__bases__:
                try:
                    p_method = getattr(parent, name)
                    if(p_method and p_method.__doc__):
                        method.__doc__ = p_method.__doc__
                except AttributeError:
                    continue
    return class_

class LoadPolicy(enum.Enum):
    lazy = "LAZY"
    eager = "EAGER"

class Lane(enum.Enum):
    top_lane = "TOP"
    jungle = "JUNGLE"
    mid_lane = "MIDDLE"
    bot_lane = "BOTTOM"

    def for_id(id_):
        try:
            return Lane.by_id[id_]
        except:
            return None

Lane.by_id = {
    1: Lane.top_lane,
    2: Lane.mid_lane,
    3: Lane.jungle,
    4: Lane.bot_lane
}

class Role(enum.Enum):
    none = "NONE"
    solo = "SOLO"
    duo = "DUO"
    carry = "DUO_CARRY"
    support = "DUO_SUPPORT"

    def for_id(id_):
        try:
            return Role.by_id[id_]
        except:
            return None

Role.by_id = {
    1: Role.duo,
    2: Role.support,
    3: Role.carry,
    4: Role.solo
}

class Side(enum.Enum):
    blue = 100
    red = 200

class Queue(enum.Enum):
    custom = "CUSTOM"
    normal_blind_fives = "NORMAL_5x5_BLIND"
    bot_fives = "BOT_5x5"
    bot_intro_fives = "BOT_5x5_INTRO"
    bot_beginner_fives = "BOT_5x5_BEGINNER"
    bot_intermediate_fives = "BOT_5x5_INTERMEDIATE"
    normal_blind_threes = "NORMAL_3x3"
    normal_draft_fives = "NORMAL_5x5_DRAFT"
    dominion_blind = "ODIN_5x5_BLIND"
    dominion_draft = "ODIN_5x5_DRAFT"
    bot_dominion = "BOT_ODIN_5x5"
    ranked_solo = "RANKED_SOLO_5x5"
    ranked_premade_threes = "RANKED_PREMADE_3x3"
    ranked_premade_fives = "RANKED_PREMADE_5x5"
    ranked_threes = "RANKED_TEAM_3x3"
    ranked_fives = "RANKED_TEAM_5x5"
    bot_threes = "BOT_TT_3x3"
    team_builder = "GROUP_FINDER_5x5"
    aram = "ARAM_5x5"
    one_for_all = "ONEFORALL_5x5"
    showdown_solo = "FIRSTBLOOD_1x1"
    showdown_duo = "FIRSTBLOOD_2x2"
    hexakill_summoners_rift = "SR_6x6"
    urf = "URF_5x5"
    bot_urf = "BOT_URF_5x5"
    doom_bots_1 = "NIGHTMARE_BOT_5x5_RANK1"
    doom_bots_2 = "NIGHTMARE_BOT_5x5_RANK2"
    doom_bots_5 = "NIGHTMARE_BOT_5x5_RANK5"
    ascension = "ASCENSION_5x5"
    hexakill_twisted_treeline = "HEXAKILL"
    butchers_bridge = "BILGEWATER_ARAM_5x5"
    poro_king = "KING_PORO_5x5"
    nemesis_draft = "COUNTER_PICK"
    black_market = "BILGEWATER_5x5"

    def for_id(id_):
        try:
            return Queue.by_id[id_]
        except:
            return None

Queue.by_id = {
    0: Queue.custom,
    65: Queue.aram,
    2: Queue.normal_blind_fives,
    4: Queue.ranked_solo,
    6: Queue.ranked_premade_fives,
    7: Queue.bot_fives,
    8: Queue.normal_blind_threes,
    9: Queue.ranked_premade_threes,
    75: Queue.hexakill_summoners_rift,
    76: Queue.urf,
    14: Queue.normal_draft_fives,
    16: Queue.dominion_blind,
    17: Queue.dominion_draft,
    83: Queue.bot_urf,
    25: Queue.bot_dominion,
    91: Queue.doom_bots_1,
    92: Queue.doom_bots_2,
    93: Queue.doom_bots_5,
    31: Queue.bot_intro_fives,
    32: Queue.bot_beginner_fives,
    33: Queue.bot_intermediate_fives,
    98: Queue.hexakill_twisted_treeline,
    100: Queue.butchers_bridge,
    70: Queue.one_for_all,
    41: Queue.ranked_threes,
    42: Queue.ranked_fives,
    300: Queue.poro_king,
    96: Queue.ascension,
    72: Queue.showdown_solo,
    52: Queue.bot_threes,
    310: Queue.nemesis_draft,
    73: Queue.showdown_duo,
    313: Queue.black_market,
    61: Queue.team_builder
}
ranked_queues = {Queue.ranked_solo, Queue.ranked_threes, Queue.ranked_fives}

class Tier(enum.Enum):
    challenger = "CHALLENGER"
    master = "MASTER"
    diamond = "DIAMOND"
    platinum = "PLATINUM"
    gold = "GOLD"
    silver = "SILVER"
    bronze = "BRONZE"

class Division(enum.Enum):
    one = "I"
    two = "II"
    three = "III"
    four = "IV"
    five = "V"

class Season(enum.Enum):
    preseason_3 = "PRESEASON3"
    season_3 = "SEASON3"
    preseason_4 = "PRESEASON2014"
    season_4 = "SEASON2014"
    preseason_5 = "PRESEASON2015"
    season_5 = "SEASON2015"
    preseason_6 = "PRESEASON2016"
    season_6 = "SEASON2016"

stats_seasons = {Season.season_3, Season.season_4, Season.season_5}

class Region(enum.Enum):
    brazil = "br"
    europe_north_east = "eune"
    europe_west = "euw"
    korea = "kr"
    latin_america_north = "lan"
    latin_america_south = "las"
    north_america = "na"
    oceania = "oce"
    russia = "ru"
    turkey = "tr"

class Platform(enum.Enum):
    brazil = "BR1"
    europe_north_east = "EUN1"
    europe_west = "EUW1"
    korea = "KR"
    latin_america_north = "LA1"
    latin_america_south = "LA2"
    north_america = "NA1"
    oceania = "OC1"
    russia = "RU"
    turkey = "TR1"

class Map(enum.Enum):
    summoners_rift_summer = 1
    summoners_rift_autumn = 2
    the_proving_grounds = 3
    twisted_treeline_original = 4
    the_crystal_scar = 8
    twisted_treeline = 10
    summoners_rift = 11
    howling_abyss = 12
    butchers_bridge = 14

class GameMode(enum.Enum):
    aram = "ARAM"
    ascension = "ASCENSION"
    classic = "CLASSIC"
    showdown = "FIRSTBLOOD"
    poro_king = "KINGPORO"
    dominion = "ODIN"
    one_for_all = "ONEFORALL"
    tutorial = "TUTORIAL"

class GameType(enum.Enum):
    custom = "CUSTOM_GAME"
    tutorial = "TUTORIAL_GAME"
    matched = "MATCHED_GAME"

class SubType(enum.Enum):
    custom = "NONE"
    normal_fives = "NORMAL"
    normal_threes = "NORMAL_3x3"
    dominion = "ODIN_UNRANKED"
    aram = "ARAM_UNRANKED_5x5"
    bot_fives = "BOT"
    bot_threes = "BOT_3x3"
    ranked_solo = "RANKED_SOLO_5x5"
    ranked_threes = "RANKED_TEAM_3x3"
    ranked_fives = "RANKED_TEAM_5x5"
    one_for_all = "ONEFORALL_5x5"
    showdown_solo = "FIRSTBLOOD_1x1"
    showdown_duo = "FIRSTBLOOD_2x2"
    hexakill_summoners_rift = "SR_6x6"
    team_builder = "CAP_5x5"
    urf = "URF"
    bot_urf = "URF_BOT"
    doom_bots = "NIGHTMARE_BOT"
    ascension = "ASCENSION"
    hexakill_twisted_treeline = "HEXAKILL"
    poro_king = "KING_PORO"
    nemesis_draft = "COUNTER_PICK"
    black_market = "BILGEWATER"

class StatSummaryType(enum.Enum):
    normal_fives = "Unranked"
    normal_threes = "Unranked3x3"
    dominion = "OdinUnranked"
    aram = "AramUnranked5x5"
    bot_fives = "CoopVsAI"
    bot_threes = "CoopVsAI3x3"
    ranked_solo = "RankedSolo5x5"
    ranked_threes = "RankedTeam3x3"
    ranked_premade_threes = "RankedPremade3x3"
    ranked_fives = "RankedTeam5x5"
    ranked_premade_fives = "RankedPremade5x5"
    one_for_all = "OneForAll5x5"
    showdown_solo = "FirstBlood1x1"
    showdown_duo = "FirstBlood2x2"
    hexakill_summoners_rift = "SummonersRift6x6"
    team_builder = "CAP5x5"
    urf = "URF"
    bot_urf = "URFBots"
    doom_bots = "NightmareBot"
    ascension = "Ascension"
    hexakill_twisted_treeline = "Hexakill"
    poro_king = "KingPoro"
    nemesis_draft = "CounterPick"
    black_market = "Bilgewater"

class Ascended(enum.Enum):
    player = "CHAMPION_ASCENDED"
    death = "CLEAR_ASCENDED"
    npc = "MINION_ASCENDED"

class Building(enum.Enum):
    inhibitor = "INHIBITOR_BUILDING"
    turret = "TOWER_BUILDING"

class EventType(enum.Enum):
    ascension = "ASCENDED_EVENT"
    building_kill = "BUILDING_KILL"
    point_capture = "CAPTURE_POINT"
    kill = "CHAMPION_KILL"
    elite_monster_kill = "ELITE_MONSTER_KILL"
    item_destruction = "ITEM_DESTROYED"
    item_purchase = "ITEM_PURCHASED"
    item_sale = "ITEM_SOLD"
    item_undo = "ITEM_UNDO"
    summoning = "PORO_KING_SUMMON"
    skill_up = "SKILL_LEVEL_UP"
    ward_kill = "WARD_KILL"
    ward_placement = "WARD_PLACED"

class LaneType(enum.Enum):
    top_lane = "TOP_LANE"
    mid_lane = "MID_LANE"
    bot_lane = "BOT_LANE"

class LevelUp(enum.Enum):
    evolution = "EVOLVE"
    normal = "NORMAL"

class Monster(enum.Enum):
    baron = "BARON_NASHOR"
    dragon = "DRAGON"
    blue = "BLUE_GOLEM"
    red = "RED_LIZARD"
    spider = "VILEMAW"

class Point(enum.Enum):
    windmill = "POINT_C"
    drill = "POINT_D"
    boneyard = "POINT_E"
    quarry = "POINT_A"
    refinery = "POINT_B"

class Turret(enum.Enum):
    outer = "OUTER_TURRET"
    inner = "INNER_TURRET"
    inhibitor = "BASE_TURRET"
    nexus = "NEXUS_TURRET"
    fountain = "FOUNTAIN_TURRET"
    undefined = "UNDEFINED_TURRET"

class Ward(enum.Enum):
    sight = "SIGHT_WARD"
    vision = "VISION_WARD"
    trinket = "YELLOW_TRINKET"
    upgraded_trinket = "YELLOW_TRINKET_UPGRADE"
    mushroom = "TEEMO_MUSHROOM"
    undefined = "UNDEFINED"
<|endoftext|>


<commit_before>import cea.plots

__author__ = "Daren Thomas"
__copyright__ = "Copyright 2016, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Daren Thomas"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

class ScenarioPlotsTool(object):
    def __init__(self):
        self.label = 'Scenario Plots'
        self.description = 'Create summary plots of scenarios in a folder'
        self.canRunInBackground = False

    def getParameterInfo(self):
        import arcpy
        scenarios = arcpy.Parameter(
            displayName="Path to the scenarios to plot",
            name="scenarios",
            datatype="DEFolder",
            parameterType="Required",
            direction="Input",
            multiValue=True)
        output_file = arcpy.Parameter(
            displayName="Path to output PDF",
            name="output_file",
            datatype="DEFile",
            parameterType="Required",
            direction="Output")
        output_file.filter.list = ['pdf']
        return [scenarios, output_file]

    def execute(self, parameters, messages):
        scenarios = parameters[0].valueAsText
        scenarios = scenarios.replace("'", "")
        scenarios = scenarios.replace('"', '')
        scenarios = scenarios.split(';')
        output_file = parameters[1].valueAsText

        arcpy.AddMessage(scenarios)

        import cea.plots.scenario_plots
        reload(cea.plots.scenario_plots)
        cea.plots.scenario_plots.plot_scenarios(scenarios=scenarios, output_file=output_file)
        return<commit_msg>Small changes<commit_after>import arcpy
__author__ = "Daren Thomas"
__copyright__ = "Copyright 2016, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Daren Thomas"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

class ScenarioPlotsTool(object):
    def __init__(self):
        self.label = 'Scenario Plots'
        self.description = 'Create summary plots of scenarios in a folder'
        self.canRunInBackground = False

    def getParameterInfo(self):
        scenarios = arcpy.Parameter(
            displayName="Path to the scenarios to plot",
            name="scenarios",
            datatype="DEFolder",
            parameterType="Required",
            direction="Input",
            multiValue=True)
        output_file = arcpy.Parameter(
            displayName="Path to output PDF",
            name="output_file",
            datatype="DEFile",
            parameterType="Required",
            direction="Output")
        output_file.filter.list = ['pdf']
        return [scenarios, output_file]

    def execute(self, parameters, messages):
        scenarios = parameters[0].valueAsText
        scenarios = scenarios.replace("'", "")
        scenarios = scenarios.replace('"', '')
        scenarios = scenarios.split(';')
        output_file = parameters[1].valueAsText

        arcpy.AddMessage(scenarios)

        import cea.plots.scenario_plots
        reload(cea.plots.scenario_plots)
        cea.plots.scenario_plots.plot_scenarios(scenarios=scenarios, output_file=output_file)
        return<|endoftext|>