<commit_before># Copyright 2015 Hewlett-Packard Development Company, L.P.
# Copyright 2016 Rackspace Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import ipaddress

from tempest import config
from tempest import test
from tempest.lib.common.utils import data_utils

from f5.bigip import ManagementRoot
from f5lbaasdriver.test.tempest.tests.api import base

config = config.CONF

class MemberSubnetTestJSON(base.BaseAdminTestCase):
    """Test creating two members on different subnets, same network.

    Verifies that a self IP is created for configurations that have
    multiple pool members on separate subnets. To do so, the test
    creates a network and three subnest: one for lb, one for first
    member, and one for the second memeber. A loadbalancer with listener,
    pool, and members is created, and the test verifies that a self IP
    is created for each member (that the self IP address and member address
    overlap).

    Note: this test will only execute correctly on undercloud deployments.
    """

    @classmethod
    def resource_setup(cls):
        """Setup client fixtures for test suite."""
        super(MemberSubnetTestJSON, cls).resource_setup()
        network_name = data_utils.rand_name('network')
        cls.network = cls.create_network(network_name)

        # subnet for load balancer
        cls.subnet1 = cls.create_subnet(cls.network)

        # subnet for first member
        cls.subnet2 = cls.create_subnet(cls.network)

        # subnet for second member
        cls.subnet3 = cls.create_subnet(cls.network)

        cls.create_lb_kwargs = {'tenant_id': cls.subnet1['tenant_id'],
                                'vip_subnet_id': cls.subnet1['id']}
        cls.load_balancer = \
            cls._create_active_load_balancer(**cls.create_lb_kwargs)
        cls.load_balancer_id = cls.load_balancer['id']

        # create listener
        listener_kwargs = {'loadbalancer_id': cls.load_balancer_id,
                           'protocol': 'HTTP',
                           'protocol_port': 80}
        cls.listener = cls._create_listener(**listener_kwargs)

        # get an RPC client for calling into driver
        cls.client = cls.plugin_rpc.get_client()
        cls.context = cls.plugin_rpc.get_context()

    @test.attr(type='smoke')
    def test_member_selfips(self):

        self.bigip = ManagementRoot(config.f5_lbaasv2_driver.icontrol_hostname,
                                    config.f5_lbaasv2_driver.icontrol_username,
                                    config.f5_lbaasv2_driver.icontrol_password)

        # create pool
        pool_kwargs = {'listener_id': self.listener['id'],
                       'protocol': 'HTTP',
                       'lb_algorithm': 'ROUND_ROBIN'}
        pool = self._create_pool(**pool_kwargs)
        self.addCleanup(self._delete_pool, pool.get('id'))

        # create first member on subnet2
        allocation_pool = self.subnet2['allocation_pools'][0]
        member_kwargs = {'pool_id': pool['id'],
                         'address': allocation_pool['start'],
                         'protocol_port': 8080,
                         'subnet_id': self.subnet2['id']}
        member1 = self._create_member(**member_kwargs)
        self.addCleanup(self._delete_member, pool.get('id'), member1.get('id'))
        assert self._has_selfip_for_member(member1)

        # create second member on subnet3
        allocation_pool = self.subnet3['allocation_pools'][0]
        member_kwargs = {'pool_id': pool['id'],
                         'address': allocation_pool['start'],
                         'protocol_port': 8080,
                         'subnet_id': self.subnet3['id']}
        member2 = self._create_member(**member_kwargs)
        self.addCleanup(self._delete_member, pool.get('id'), member2.get('id'))
        assert self._has_selfip_for_member(member2)

        # delete first and check second self IP
        self._delete_member(pool.get('id'), member1.get('id'))
        assert self._has_selfip_for_member(member2)

        # re-create first member and check both self IPs
        allocation_pool = self.subnet2['allocation_pools'][0]
        member_kwargs = {'pool_id': pool['id'],
                         'address': allocation_pool['start'],
                         'protocol_port': 8080,
                         'subnet_id': self.subnet2['id']}
        member1 = self._create_member(**member_kwargs)
        self.addCleanup(self._delete_member, pool.get('id'), member1.get('id'))
        assert self._has_selfip_for_member(member1)
        assert self._has_selfip_for_member(member2)

    def _has_selfip_for_member(self, member):
        # Verify that self IP overlaps with member IP
        address = self._split_address(member['address'])
        member_address = ipaddress.ip_address(address)

        selfips = self.bigip.tm.net.selfips.get_collection()
        for selfip in selfips:
            selfip_address, selfip_rd = self._split_address(selfip.address)
            network = ipaddress.ip_network(selfip_address, False)
            if member_address in network:
                return True

        return False

    def _split_address(self, address):
        '''Extract IP address which might have a route domain.

        Input address may or may not have route domains, mask.
        Some examples of different possible forms:
           10.1.1.1
           10.1.1.1%3
           10.1.1.1/24
           10.1.1.1%3/24
        '''

        if '%' in address:
            # has route domain
            parts = address.split('%')
            ip_address = parts[0]
            rd = parts[1]
            if '/' in rd:
                # retain CIDR form without route domain
                parts = rd.split('/')
                ip_address += '/' + parts[1]
        else:
            # no route domain
            ip_address = address

        return unicode(ip_address)
<commit_msg>Fix flake8 errors.<commit_after># Copyright 2015 Hewlett-Packard Development Company, L.P.
# Copyright 2016 Rackspace Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import ipaddress

from tempest import config
from tempest.lib.common.utils import data_utils
from tempest import test

from f5.bigip import ManagementRoot
from f5lbaasdriver.test.tempest.tests.api import base

config = config.CONF

class MemberSubnetTestJSON(base.BaseAdminTestCase):
    """Test creating two members on different subnets, same network.

    Verifies that a self IP is created for configurations that have
    multiple pool members on separate subnets. To do so, the test
    creates a network and three subnest: one for lb, one for first
    member, and one for the second memeber. A loadbalancer with listener,
    pool, and members is created, and the test verifies that a self IP
    is created for each member (that the self IP address and member address
    overlap).

    Note: this test will only execute correctly on undercloud deployments.
    """

    @classmethod
    def resource_setup(cls):
        """Setup client fixtures for test suite."""
        super(MemberSubnetTestJSON, cls).resource_setup()
        network_name = data_utils.rand_name('network')
        cls.network = cls.create_network(network_name)

        # subnet for load balancer
        cls.subnet1 = cls.create_subnet(cls.network)

        # subnet for first member
        cls.subnet2 = cls.create_subnet(cls.network)

        # subnet for second member
        cls.subnet3 = cls.create_subnet(cls.network)

        cls.create_lb_kwargs = {'tenant_id': cls.subnet1['tenant_id'],
                                'vip_subnet_id': cls.subnet1['id']}
        cls.load_balancer = \
            cls._create_active_load_balancer(**cls.create_lb_kwargs)
        cls.load_balancer_id = cls.load_balancer['id']

        # create listener
        listener_kwargs = {'loadbalancer_id': cls.load_balancer_id,
                           'protocol': 'HTTP',
                           'protocol_port': 80}
        cls.listener = cls._create_listener(**listener_kwargs)

        # get an RPC client for calling into driver
        cls.client = cls.plugin_rpc.get_client()
        cls.context = cls.plugin_rpc.get_context()

    @test.attr(type='smoke')
    def test_member_selfips(self):

        self.bigip = ManagementRoot(config.f5_lbaasv2_driver.icontrol_hostname,
                                    config.f5_lbaasv2_driver.icontrol_username,
                                    config.f5_lbaasv2_driver.icontrol_password)

        # create pool
        pool_kwargs = {'listener_id': self.listener['id'],
                       'protocol': 'HTTP',
                       'lb_algorithm': 'ROUND_ROBIN'}
        pool = self._create_pool(**pool_kwargs)
        self.addCleanup(self._delete_pool, pool.get('id'))

        # create first member on subnet2
        allocation_pool = self.subnet2['allocation_pools'][0]
        member_kwargs = {'pool_id': pool['id'],
                         'address': allocation_pool['start'],
                         'protocol_port': 8080,
                         'subnet_id': self.subnet2['id']}
        member1 = self._create_member(**member_kwargs)
        self.addCleanup(self._delete_member, pool.get('id'), member1.get('id'))
        assert self._has_selfip_for_member(member1)

        # create second member on subnet3
        allocation_pool = self.subnet3['allocation_pools'][0]
        member_kwargs = {'pool_id': pool['id'],
                         'address': allocation_pool['start'],
                         'protocol_port': 8080,
                         'subnet_id': self.subnet3['id']}
        member2 = self._create_member(**member_kwargs)
        self.addCleanup(self._delete_member, pool.get('id'), member2.get('id'))
        assert self._has_selfip_for_member(member2)

        # delete first and check second self IP
        self._delete_member(pool.get('id'), member1.get('id'))
        assert self._has_selfip_for_member(member2)

        # re-create first member and check both self IPs
        allocation_pool = self.subnet2['allocation_pools'][0]
        member_kwargs = {'pool_id': pool['id'],
                         'address': allocation_pool['start'],
                         'protocol_port': 8080,
                         'subnet_id': self.subnet2['id']}
        member1 = self._create_member(**member_kwargs)
        self.addCleanup(self._delete_member, pool.get('id'), member1.get('id'))
        assert self._has_selfip_for_member(member1)
        assert self._has_selfip_for_member(member2)

    def _has_selfip_for_member(self, member):
        # Verify that a self IP exists that overlaps with member IP
        address = self._split_address(member['address'])
        member_address = ipaddress.ip_address(address)

        selfips = self.bigip.tm.net.selfips.get_collection()
        for selfip in selfips:
            selfip_address, selfip_rd = self._split_address(selfip.address)
            network = ipaddress.ip_network(selfip_address, False)
            if member_address in network:
                return True

        return False

    def _split_address(self, address):
        '''Extract IP address which might have a route domain.

        Input address may or may not have route domains, mask.
        Some examples of different possible forms:
           10.1.1.1
           10.1.1.1%3
           10.1.1.1/24
           10.1.1.1%3/24
        '''

        if '%' in address:
            # has route domain
            parts = address.split('%')
            ip_address = parts[0]
            rd = parts[1]
            if '/' in rd:
                # retain CIDR form without route domain
                parts = rd.split('/')
                ip_address += '/' + parts[1]
        else:
            # no route domain
            ip_address = address

        return unicode(ip_address)
<|endoftext|>


<commit_before>import os
import logging

from .framework import framework

log = logging.getLogger(__name__)

class _StepRunner(object):
    """
    Imports all test files in the project scope. Specs within these files
    should either be at the top level of the module or in a function or class
    that is invoked from the top-level. This service is managed and invoked
    by the framework.
    """
    def load_steps(self, working, registry):
        for root, dirs, files in os.walk(working):
            for f in files:
                if self._is_test_module(f):
                    path = os.path.join(root, f)
                    self._exec_in(path, registry)

    def _is_test_module(self, f):
        return f.endswith('.pyspecs')

    def _exec_in(self, path, registry):
        log.debug('Procesing file %s', path)

        config = framework(registry)
        with open(path) as fd:
            code = compile(fd.read(), path, 'exec', optimize=0)
            exec(code, config)
        return config
<commit_msg>python 2.7 compatibility<commit_after>import os
import logging

from .framework import framework

log = logging.getLogger(__name__)

class _StepRunner(object):
    """
    Imports all test files in the project scope. Specs within these files
    should either be at the top level of the module or in a function or class
    that is invoked from the top-level. This service is managed and invoked
    by the framework.
    """
    def load_steps(self, working, registry):
        for root, dirs, files in os.walk(working):
            for f in files:
                if self._is_test_module(f):
                    path = os.path.join(root, f)
                    self._exec_in(path, registry)

    def _is_test_module(self, f):
        return f.endswith('.pyspecs')

    def _exec_in(self, path, registry):
        log.debug('Procesing file %s', path)

        config = framework(registry)
        with open(path) as fd:
            code = compile(fd.read(), path, 'exec')
            exec(code, config)
        return config
<|endoftext|>


<commit_before>import sys
sys.path.insert(1,"../../../")
import h2o
from tests import pyunit_utils
from h2o.estimators.gbm import H2OGradientBoostingEstimator

def ignored_columns():
    airlines = h2o.upload_file(pyunit_utils.locate("smalldata/airlines/allyears2k_headers.zip"))

    # convert columns to factors
    airlines["Year"]= airlines["Year"].asfactor()
    airlines["Month"]= airlines["Month"].asfactor()
    airlines["DayOfWeek"] = airlines["DayOfWeek"].asfactor()
    airlines["Cancelled"] = airlines["Cancelled"].asfactor()
    airlines['FlightNum'] = airlines['FlightNum'].asfactor()

    # set the predictor names and the response column name
    predictors = airlines.columns[:9]
    response = "IsDepDelayed"

    # split into train and validation sets
    train, valid= airlines.split_frame(ratios=[.8], seed=1234)

    # create a list of column names to ignore
    col_list = ['DepTime','CRSDepTime','ArrTime','CRSArrTime']

    # initialize the estimator and train the model
    airlines_gbm = H2OGradientBoostingEstimator(ignored_columns=col_list, seed=1234)

    # throws H2OValueError: Properties x and ignored_columns cannot be specified simultaneously
    message = "Properties x and ignored_columns cannot be specified simultaneously"
    try:
        airlines_gbm.train(x=predictors, y=response, training_frame=train, validation_frame=valid)
        assert False, "It should throw H2OValueError: "+message
    except Exception as e:
        assert message in e.args, "It should throw H2OValueError: "+message

    airlines_gbm.train(y=response, training_frame=train, validation_frame=valid)

    # get the ignored columns from the model JSON
    ignored_cols_from_model = [param_dict["actual_value"] for param_dict in airlines_gbm._model_json["parameters"] if param_dict["name"] == "ignored_columns"][0]
    assert set(col_list) <= set(ignored_cols_from_model)

    # get the list of columns used by the model
    used_columns = airlines_gbm._model_json['output']['names']
    assert set(col_list).isdisjoint(set(used_columns))

if __name__ == "__main__":
    pyunit_utils.standalone_test(ignored_columns)
else:
    ignored_columns()
<commit_msg>[AUTOMATIC] HEXDEV-727: remove trailing spaces in recently changed source files<commit_after>import sys
sys.path.insert(1,"../../../")
import h2o
from tests import pyunit_utils
from h2o.estimators.gbm import H2OGradientBoostingEstimator

def ignored_columns():
    airlines = h2o.upload_file(pyunit_utils.locate("smalldata/airlines/allyears2k_headers.zip"))

    # convert columns to factors
    airlines["Year"]= airlines["Year"].asfactor()
    airlines["Month"]= airlines["Month"].asfactor()
    airlines["DayOfWeek"] = airlines["DayOfWeek"].asfactor()
    airlines["Cancelled"] = airlines["Cancelled"].asfactor()
    airlines['FlightNum'] = airlines['FlightNum'].asfactor()

    # set the predictor names and the response column name
    predictors = airlines.columns[:9]
    response = "IsDepDelayed"

    # split into train and validation sets
    train, valid= airlines.split_frame(ratios=[.8], seed=1234)

    # create a list of column names to ignore
    col_list = ['DepTime','CRSDepTime','ArrTime','CRSArrTime']

    # initialize the estimator and train the model
    airlines_gbm = H2OGradientBoostingEstimator(ignored_columns=col_list, seed=1234)

    # throws H2OValueError: Properties x and ignored_columns cannot be specified simultaneously
    message = "Properties x and ignored_columns cannot be specified simultaneously"
    try:
        airlines_gbm.train(x=predictors, y=response, training_frame=train, validation_frame=valid)
        assert False, "It should throw H2OValueError: "+message
    except Exception as e:
        assert message in e.args, "It should throw H2OValueError: "+message

    airlines_gbm.train(y=response, training_frame=train, validation_frame=valid)

    # get the ignored columns from the model JSON
    ignored_cols_from_model = [param_dict["actual_value"] for param_dict in airlines_gbm._model_json["parameters"] if param_dict["name"] == "ignored_columns"][0]
    assert set(col_list) <= set(ignored_cols_from_model)

    # get the list of columns used by the model
    used_columns = airlines_gbm._model_json['output']['names']
    assert set(col_list).isdisjoint(set(used_columns))

if __name__ == "__main__":
    pyunit_utils.standalone_test(ignored_columns)
else:
    ignored_columns()
<|endoftext|>


<commit_before>from cbint.utils.detonation import DetonationDaemon, ConfigurationError
from cbint.utils.detonation.binary_analysis import (BinaryAnalysisProvider, AnalysisPermanentError,
                                                    AnalysisTemporaryError, AnalysisResult)
import cbint.utils.feed
import time
import logging
import os
import requests
import traceback
from time import sleep

logging.getLogger("requests").setLevel(logging.WARNING)

log = logging.getLogger(__name__)

class BluecoatProvider(BinaryAnalysisProvider):
    def __init__(self, name, bluecoat_url, bluecoat_api_key, bluecoat_owner):
        super(BluecoatProvider, self).__init__(name)

        # TODO -- pass in whether or not to verify ???

        self.bluecoat_url = bluecoat_url
        if not self.bluecoat_url.endswith('/'):
            self.bluecoat_url += "/"

        self.bluecoat_api_key = bluecoat_api_key
        self.bluecoat_owner = bluecoat_owner
        self.headers = {'X-API-TOKEN': self.bluecoat_api_key}

        self.sample_upload_url = "%srapi/samples/basic" % self.bluecoat_url
        self.create_task_url = "%srapi/tasks" % self.bluecoat_url

        self.check_url_format_str = "%srapi/samples?md5=%%s" % (self.bluecoat_url)
        self.get_tasks_url_format_str = "%srapi/samples/%%d/tasks" % self.bluecoat_url

    def check_result_for(self, md5sum, sample_id=None):
        print "check_result_for"
        try:
            if not sample_id:
                #
                # if no sample_id then try md5sum
                #
                url = self.check_url_format_str % md5sum

                #
                # Send the get request
                #
                resp = requests.get(url, headers=self.headers, verify=False)

                #
                # Parse the results
                #
                sample_results = resp.json()
                result_count = sample_results.get('results_count', 0)
                if result_count == 0:
                    #
                    # if there are no results return None
                    #
                    return None

                result = sample_results.get('results', [{}])[0]
                sample_id = result.get('samples_sample_id', -1)

            url = self.get_tasks_url_format_str % sample_id
            resp = requests.get(url, headers=self.headers, verify=False)
            log.warn("%s | %d" % (url, resp.status_code))

            tasks_results = resp.json()
            #
            # Inside of tasks_results if there are none then it will be an empty list
            #
            task_result = tasks_results.get('results', [{}])
            if len(task_result) == 0:
                return None

            #
            # Here task_result is a [{}]
            #
            task_result = task_result[0]

            task_id = task_result.get('tasks_task_id', -1)
            if not task_id: #
                #
                # Do it over basically.
                # No result found, return None
                #
                return None

            #
            # Get the current task state
            #
            task_status = task_result.get('task_state_state', 'UNKNOWN')
            if task_status == 'CORE_COMPLETE':

                #
                # Pull the score from json
                #
                score = task_result['tasks_global_risk_score']

                log.info("Binary %s score %d" % (md5sum, score))

                #
                # Hardcoded value for potential malware
                #
                if score > 8:
                    malware_result = "Potential Malware"
                else:
                    malware_result = "Benign"

                #
                # Normalize score by just multiplying
                #
                score *= 10

                #
                # generate the task link to send back with the Analysis Result
                #
                task_link = "%sanalysis_center/view_task/%d" % (self.bluecoat_url, task_id)

                return AnalysisResult(message=malware_result, extended_message="",
                                      link=task_link,
                                      score=score)
            else:
                #
                # Since this is called from a quick scan thread just return None
                # if the bluecoat provider returns anything besides CORE_COMPLETE
                #
                return None

        except Exception as e:
            print traceback.format_exc()
            log.error("check_result_for: an exception occurred while querying bluecoat for %s: %s" % (md5sum, e))
            log.error(traceback.format_exc())
            raise AnalysisTemporaryError(message=e.message, retry_in=120)

    def analyze_binary(self, md5sum, binary_file_stream):
        print "analyze_binary"
        try:
            description = 'Uploaded from Carbon Black'
            label = 'cb-%s' % md5sum

            sample_file = {'file': binary_file_stream}
            form_data = {'owner': self.bluecoat_owner, 'description': description, 'label': label}

            #
            # Upload the binary
            #
            resp = requests.post(self.sample_upload_url, files=sample_file, data=form_data, headers=self.headers, verify=False)
            log.info("%s | %d" % (self.sample_upload_url, resp.status_code))

            if resp.status_code != 200:
                raise AnalysisTemporaryError(message=resp.content, retry_in=120)

            #
            # Check the response of the upload
            #
            sample_upload_data = resp.json()
            sample_result = sample_upload_data.get('results', [{}])[0]

            #
            # Now create the task to analyze the binary
            #
            sample_id = sample_result.get('samples_sample_id')
            task_data = {"sample_id":  sample_id, "env": "ivm"}

            #
            # Send the Http Post to create the task
            #
            resp = requests.post(self.create_task_url, data=task_data, headers=self.headers, verify=False)
            log.info("%s | %d" % (self.create_task_url, resp.status_code))

            if resp.status_code != 200:
                raise AnalysisTemporaryError(message=resp.content, retry_in=120)

            #
            # Try to get the results if we can
            #
            retries = 20
                while retries:
                    sleep(10)
                    result = self.check_result_for(md5sum, event_id=event_id)
                    if result:
                        return result
                    retries -= 1

            raise AnalysisTemporaryError(message="Maximum retries (20) exceeded submitting to Bluecoat", retry_in=120)

        except:
            print traceback.format_exc()
            log.error("analyze_binary: an exception occurred while submitting to bluecoat for %s: %s" % (md5sum, e))
            log.error(traceback.format_exc())
            raise AnalysisTemporaryError(traceback.format_exc(), retry_in=120)

class BluecoatConnector(DetonationDaemon):
    @property
    def filter_spec(self):
        filters = []
        max_module_len = 10 * 1024 * 1024
        filters.append('os_type:windows orig_mod_len:[1 TO %d]' % max_module_len)
        additional_filter_requirements = self.get_config_string("binary_filter_query", None)
        if additional_filter_requirements:
            filters.append(additional_filter_requirements)
        return ' '.join(filters)

    @property
    def num_quick_scan_threads(self):
        return 0

    @property
    def num_deep_scan_threads(self):
        return 4

    def get_provider(self):
        bluecoat_provider = BluecoatProvider(self.name, self.bluecoat_url, self.bluecoat_api_key, self.bluecoat_owner)
        return bluecoat_provider

    def get_metadata(self):
        return cbint.utils.feed.generate_feed(self.name, summary="Bluecoat Malware Analysis Appliance Detonation",
                        tech_data="There are no requirements to share any data with Carbon Black to use this feed.",
                        provider_url="http://www.bluecoat.com",
                        icon_path='/usr/share/cb/integrations/bluecoat/bluecoat-logo.png',
                        display_name="Bluecoat", category="Connectors")

    def validate_config(self):
        super(BluecoatConnector, self).validate_config()
        self.check_required_options(["bluecoat_url", "bluecoat_api_key"])
        self.bluecoat_url = self.get_config_string("bluecoat_url", None)
        self.bluecoat_api_key = self.get_config_string("bluecoat_api_key", None)
        self.bluecoat_owner = self.get_config_string("bluecoat_owner", "admin")
        return True

if __name__ == '__main__':
    import os

    my_path = os.path.dirname(os.path.abspath(__file__))
    temp_directory = "/tmp/bluecoat"

    config_path = os.path.join(my_path, "testing.conf")
    daemon = BluecoatConnector('bluecoattest', configfile=config_path, work_directory=temp_directory,
                                logfile=os.path.join(temp_directory, 'test.log'), debug=True)
    daemon.start()<commit_msg>minor bugfixes<commit_after>from cbint.utils.detonation import DetonationDaemon, ConfigurationError
from cbint.utils.detonation.binary_analysis import (BinaryAnalysisProvider, AnalysisPermanentError,
                                                    AnalysisTemporaryError, AnalysisResult)
import cbint.utils.feed
import time
import logging
import os
import requests
import traceback
from time import sleep

logging.getLogger("requests").setLevel(logging.WARNING)

log = logging.getLogger(__name__)

class BluecoatProvider(BinaryAnalysisProvider):
    def __init__(self, name, bluecoat_url, bluecoat_api_key, bluecoat_owner):
        super(BluecoatProvider, self).__init__(name)

        # TODO -- pass in whether or not to verify ???

        self.bluecoat_url = bluecoat_url
        if not self.bluecoat_url.endswith('/'):
            self.bluecoat_url += "/"

        self.bluecoat_api_key = bluecoat_api_key
        self.bluecoat_owner = bluecoat_owner
        self.headers = {'X-API-TOKEN': self.bluecoat_api_key}

        self.sample_upload_url = "%srapi/samples/basic" % self.bluecoat_url
        self.create_task_url = "%srapi/tasks" % self.bluecoat_url

        self.check_url_format_str = "%srapi/samples?md5=%%s" % (self.bluecoat_url)
        self.get_tasks_url_format_str = "%srapi/samples/%%d/tasks" % self.bluecoat_url

    def check_result_for(self, md5sum, sample_id = None):
        try:
            if not sample_id:
                #
                # if no sample_id then try md5sum
                #
                url = self.check_url_format_str % md5sum

                #
                # Send the get request
                #
                resp = requests.get(url, headers=self.headers, verify=False)

                #
                # Parse the results
                #
                sample_results = resp.json()
                result_count = sample_results.get('results_count', 0)
                if result_count == 0:
                    #
                    # if there are no results return None
                    #
                    return None

                result = sample_results.get('results', [{}])[0]
                sample_id = result.get('samples_sample_id', -1)

            url = self.get_tasks_url_format_str % sample_id
            resp = requests.get(url, headers=self.headers, verify=False)
            log.warn("%s | %d" % (url, resp.status_code))

            tasks_results = resp.json()
            #
            # Inside of tasks_results if there are none then it will be an empty list
            #
            task_result = tasks_results.get('results', [{}])
            if len(task_result) == 0:
                return None

            #
            # Here task_result is a [{}]
            #
            task_result = task_result[0]

            task_id = task_result.get('tasks_task_id', -1)
            if not task_id: #
                #
                # No task associated with this sample id
                #
                return None

            #
            # Get the current task state
            #
            task_status = task_result.get('task_state_state', 'UNKNOWN')
            if task_status == 'CORE_COMPLETE':

                #
                # Pull the score from json
                #
                score = task_result['tasks_global_risk_score']

                log.info("Binary %s score %d" % (md5sum, score))

                #
                # Hardcoded value for potential malware
                #
                if score > 8:
                    malware_result = "Potential Malware"
                else:
                    malware_result = "Benign"

                #
                # Normalize score by just multiplying
                #
                score *= 10

                #
                # generate the task link to send back with the Analysis Result
                #
                task_link = "%sanalysis_center/view_task/%d" % (self.bluecoat_url, task_id)

                return AnalysisResult(message=malware_result, extended_message="",
                                      link=task_link,
                                      score=score)
            else:
                #
                # Since this is called from a quick scan thread just return None
                # if the bluecoat provider returns anything besides CORE_COMPLETE
                #
                return None

        except Exception as e:
            print traceback.format_exc()
            log.error("check_result_for: an exception occurred while querying bluecoat for %s: %s" % (md5sum, e))
            log.error(traceback.format_exc())
            raise AnalysisTemporaryError(message=e.message, retry_in=120)

    def analyze_binary(self, md5sum, binary_file_stream):
        try:
            description = 'Uploaded from Carbon Black'
            label = 'cb-%s' % md5sum

            sample_file = {'file': binary_file_stream}
            form_data = {'owner': self.bluecoat_owner, 'description': description, 'label': label}

            #
            # Upload the binary
            #
            resp = requests.post(self.sample_upload_url, files=sample_file, data=form_data, headers=self.headers, verify=False)
            log.info("%s | %d" % (self.sample_upload_url, resp.status_code))

            if resp.status_code != 200:
                raise AnalysisTemporaryError(message=resp.content, retry_in=120)

            #
            # Check the response of the upload
            #
            sample_upload_data = resp.json()
            sample_result = sample_upload_data.get('results', [{}])[0]

            #
            # Now create the task to analyze the binary
            #
            sample_id = sample_result.get('samples_sample_id')
            task_data = {"sample_id":  sample_id, "env": "ivm"}

            #
            # Send the Http Post to create the task
            #
            resp = requests.post(self.create_task_url, data=task_data, headers=self.headers, verify=False)
            log.info("%s | %d" % (self.create_task_url, resp.status_code))

            if resp.status_code != 200:
                raise AnalysisTemporaryError(message=resp.content, retry_in=120)

            #
            # Try to get the results if we can
            #
            retries = 20
            while retries:
                sleep(10)
                result = self.check_result_for(md5sum, sample_id=sample_id)
                if result:
                    return result
                retries -= 1

            raise AnalysisTemporaryError(message="Maximum retries (20) exceeded submitting to Bluecoat", retry_in=120)

        except:
            print traceback.format_exc()
            log.error("analyze_binary: an exception occurred while submitting to bluecoat for %s: %s" % (md5sum, e))
            log.error(traceback.format_exc())
            raise AnalysisTemporaryError(traceback.format_exc(), retry_in=120)

class BluecoatConnector(DetonationDaemon):
    @property
    def filter_spec(self):
        filters = []
        max_module_len = 10 * 1024 * 1024
        filters.append('os_type:windows orig_mod_len:[1 TO %d]' % max_module_len)
        additional_filter_requirements = self.get_config_string("binary_filter_query", None)
        if additional_filter_requirements:
            filters.append(additional_filter_requirements)
        return ' '.join(filters)

    @property
    def num_quick_scan_threads(self):
        return 0

    @property
    def num_deep_scan_threads(self):
        return 4

    def get_provider(self):
        bluecoat_provider = BluecoatProvider(self.name, self.bluecoat_url, self.bluecoat_api_key, self.bluecoat_owner)
        return bluecoat_provider

    def get_metadata(self):
        return cbint.utils.feed.generate_feed(self.name, summary="Bluecoat Malware Analysis Appliance Detonation",
                        tech_data="There are no requirements to share any data with Carbon Black to use this feed.",
                        provider_url="http://www.bluecoat.com",
                        icon_path='/usr/share/cb/integrations/bluecoat/bluecoat-logo.png',
                        display_name="Bluecoat", category="Connectors")

    def validate_config(self):
        super(BluecoatConnector, self).validate_config()
        self.check_required_options(["bluecoat_url", "bluecoat_api_key"])
        self.bluecoat_url = self.get_config_string("bluecoat_url", None)
        self.bluecoat_api_key = self.get_config_string("bluecoat_api_key", None)
        self.bluecoat_owner = self.get_config_string("bluecoat_owner", "admin")
        return True

if __name__ == '__main__':
    import os

    my_path = os.path.dirname(os.path.abspath(__file__))
    temp_directory = "/tmp/bluecoat"

    config_path = os.path.join(my_path, "testing.conf")
    daemon = BluecoatConnector('bluecoattest', configfile=config_path, work_directory=temp_directory,
                                logfile=os.path.join(temp_directory, 'test.log'), debug=True)
    daemon.start()<|endoftext|>


<commit_before><commit_msg>[python][02] Even Fibonacci numbers<commit_after><|endoftext|>


<commit_before>from pyparsing import *
import lang

doc_next = None
doc_prev_component = None

def component(com):
	global doc_next, doc_prev_component
	if doc_next:
		com.doc = doc_next
		doc_next = None
	doc_prev_component = com
	return com

def document(text, line, prev):
	text = text.strip()
	if not text:
		print 'WARNING: empty documentation string at line %d' %line
		return

	global doc_next, doc_prev_component
	if prev:
		if doc_prev_component:
			if doc_prev_component.doc is not None:
				print 'WARNING: duplicate documentation string %s at line %d' %(text, line)
			doc_prev_component.doc = lang.DocumentationString(text)
		else:
			print 'WARNING: unused documentation string %s at line %d' %(text, line)
	else:
		if doc_next is not None:
			print 'WARNING: duplicate documentation string %s at line %d' %(text, line)
		doc_next = lang.DocumentationString(text)

def handle_component_declaration(s, l, t):
	#print "component>", t
	return component(lang.Component(t[0], t[1]))

def handle_assignment(s, l, t):
	#print "assignment>", t
	return component(lang.Assignment(t[0], t[1]))

def handle_property_declaration(s, l, t):
	#print "property>", t
	default = t[2] if len(t) > 2 else None
	return component(lang.Property(t[0], t[1], default))

def handle_alias_property_declaration(s, l, t):
	return component(lang.AliasProperty(t[0], t[1]))

def handle_enum_property_declaration(s, l, t):
	return component(lang.EnumProperty(t[0], t[1], t[2] if len(t) > 2 else None))

def handle_method_declaration(s, l, t):
	event_handler = t[0] != 'function'
	if event_handler:
		name, args, code = t[0], t[1], t[2]
	else:
		name, args, code = t[1], t[2], t[3]
	return component(lang.Method(name, args, code, event_handler) if name != 'constructor' else lang.Constructor(args, code))

def handle_assignment_scope(s, l, t):
	#print "assignment-scope>", t
	return component(lang.AssignmentScope(t[0], t[1]))

def handle_nested_identifier_rvalue(s, l, t):
	#print "nested-id>", t
	path = t[0].split(".")
	if path[0] == 'this':
		return 'this'
	if path[0] == "model":
		return "this._get('model').%s" %".".join(path[1:])
	path = ["_get('%s')" %x for x in path]
	return "this.%s" % ".".join(path)

def handle_enum_value(s, l, t):
	#print "enum>", t
	return "".join(t)

def handle_id_declaration(s, l, t):
	#print "id>", t
	return component(lang.IdAssignment(t[0]))

def handle_behavior_declaration(s, l, t):
	#print "behavior>", t
	return component(lang.Behavior(t[0], t[1]))

def handle_signal_declaration(s, l, t):
	return component(lang.Signal(t[0]))

def handle_function_call(s, l, t):
	#print "func> ", t
	name = t[0]
	if name[0].islower():
		name = '_globals.' + name
	return "%s(%s)" % (name, ",".join(t[1:]))

def handle_documentation_string(s, l, t):
	text = t[0]
	if text.startswith('///<'):
		document(text[4:], l, True)
	elif text.startswith('///'):
		document(text[3:], l, False)
	elif text.startswith('/**'):
		end = text.rfind('*/')
		document(text[3:end], l, False)

def handle_json_object(s, l, tokens):
	obj = {}
	for key, value in tokens:
		obj[key] = value
	return obj

def handle_list_element(s, l, t):
	return lang.ListElement(t[0])

def handle_number(s, l, t):
	value = t[0]
	return float(value) if '.' in value else int(value)

expression = Forward()
expression_list = Forward()
component_declaration = Forward()

type = Word(alphas, alphanums)
component_type = Word(srange("[A-Za-z_]"), alphanums + "._")
identifier = Word(srange("[a-z_]"), alphanums + "_")
code = originalTextFor(nestedExpr("{", "}", None, None))
null_value = Keyword("null")
bool_value = Keyword("true") | Keyword("false")
number = Word("01234567890+-.")
number.setParseAction(handle_number)

quoted_string_value = \
	QuotedString('"', escChar='\\', unquoteResults = False, multiline=True) | \
	QuotedString("'", escChar='\\', unquoteResults = False, multiline=True)

unquoted_string_value = \
	QuotedString('"', escChar='\\', unquoteResults = True, multiline=True) | \
	QuotedString("'", escChar='\\', unquoteResults = True, multiline=True)

enum_element = Word(srange("[A-Z_]"), alphanums)
enum_value = Word(srange("[A-Z_]"), alphanums) + Literal(".") + enum_element
enum_value.setParseAction(handle_enum_value)

function_call = Word(alphanums + '._') + Literal("(").suppress() + expression_list + Literal(")").suppress() + \
	ZeroOrMore(Literal(".").suppress() + Literal("arg").suppress() + Literal("(").suppress() + expression + Literal(")").suppress())
function_call.setParseAction(handle_function_call)

nested_identifier_lvalue = Word(srange("[a-z_]"), alphanums + "._")
nested_identifier_lvalue_list = Group(nested_identifier_lvalue + ZeroOrMore(Literal(",").suppress() + nested_identifier_lvalue))

nested_identifier_rvalue = Word(srange("[a-z_]"), alphanums + "._")
nested_identifier_rvalue.setParseAction(handle_nested_identifier_rvalue)

expression_end = Literal(";").suppress()

signal_declaration = Keyword("signal").suppress() + identifier + expression_end
signal_declaration.setParseAction(handle_signal_declaration)

id_declaration = Keyword("id").suppress() + Literal(":").suppress() + identifier + expression_end
id_declaration.setParseAction(handle_id_declaration)

assign_declaration = nested_identifier_lvalue + Literal(":").suppress() + expression + expression_end
assign_declaration.setParseAction(handle_assignment)

assign_component_declaration = nested_identifier_lvalue + Literal(":").suppress() + component_declaration
assign_component_declaration.setParseAction(handle_assignment)

property_declaration = (((Keyword("property").suppress() + type + identifier + Literal(":").suppress() + expression) | \
		(Keyword("property").suppress() + type + identifier)) + expression_end) | \
	(Keyword("property").suppress() + type + identifier + Literal(":").suppress() + component_declaration) \

alias_property_declaration = Keyword("property").suppress() + Keyword("alias").suppress() + identifier + Literal(":").suppress() + nested_identifier_lvalue + expression_end
alias_property_declaration.setParseAction(handle_alias_property_declaration)

enum_property_declaration = Keyword("property").suppress() + Keyword("enum").suppress() + identifier + \
	Literal("{").suppress() + Group(enum_element + ZeroOrMore(Literal(",").suppress() + enum_element)) + Literal("}").suppress() + Optional(Literal(':').suppress() + enum_element) + expression_end
enum_property_declaration.setParseAction(handle_enum_property_declaration)

property_declaration.setParseAction(handle_property_declaration)

assign_scope_declaration = identifier + Literal(":").suppress() + expression + expression_end
assign_scope_declaration.setParseAction(handle_assignment)
assign_scope = nested_identifier_lvalue + Literal("{").suppress() + Group(OneOrMore(assign_scope_declaration)) + Literal("}").suppress()
assign_scope.setParseAction(handle_assignment_scope)

method_declaration = nested_identifier_lvalue + Group(Optional(Literal("(").suppress() + identifier + ZeroOrMore(Literal(",").suppress() + identifier) + Literal(")").suppress() )) + Literal(":").suppress() + code
method_declaration.setParseAction(handle_method_declaration)

method_declaration_qml = Keyword("function") + nested_identifier_lvalue + Group(Literal("(").suppress() + Optional(identifier + ZeroOrMore(Literal(",").suppress() + identifier)) + Literal(")").suppress() ) + code
method_declaration_qml.setParseAction(handle_method_declaration)

behavior_declaration = Keyword("Behavior").suppress() + Keyword("on").suppress() + nested_identifier_lvalue_list + Literal("{").suppress() + component_declaration + Literal("}").suppress()
behavior_declaration.setParseAction(handle_behavior_declaration)

json_value = Forward()
json_object = Suppress("{") + delimitedList(Group((unquoted_string_value | identifier) + Suppress(":") + json_value) | empty, Suppress(";") | Suppress(",")) + Suppress('}')
json_object.setParseAction(handle_json_object)
json_array = Suppress("[") + delimitedList(json_value) + Suppress("]")
json_value << (null_value | bool_value | number | unquoted_string_value | json_array | json_object)

list_element_declaration = Keyword("ListElement").suppress() - json_object
list_element_declaration.setParseAction(handle_list_element)

scope_declaration = list_element_declaration | behavior_declaration | signal_declaration | alias_property_declaration | enum_property_declaration | property_declaration | id_declaration | assign_declaration | assign_component_declaration | component_declaration | method_declaration | method_declaration_qml | assign_scope
component_scope = (Literal("{").suppress() + Group(ZeroOrMore(scope_declaration)) + Literal("}").suppress())

component_declaration << (component_type + component_scope)
component_declaration.setParseAction(handle_component_declaration)

def handle_unary_op(s, l, t):
	#print "EXPR", t
	return " ".join(map(str, t[0]))
def handle_binary_op(s, l, t):
	#print "EXPR", t
	return " ".join(map(str, t[0]))
def handle_ternary_op(s, l, t):
	#print "EXPR", t
	return " ".join(map(str, t[0]))

expression_array = Literal("[") + Optional(expression + ZeroOrMore(Literal(",") + expression)) + Literal("]")
def handle_expression_array(s, l, t):
	return "".join(t)

expression_array.setParseAction(handle_expression_array)

expression_definition = bool_value | number | quoted_string_value | function_call | nested_identifier_rvalue | enum_value | expression_array

expression_ops = infixNotation(expression_definition, [
	('!', 1, opAssoc.RIGHT, handle_unary_op),
	('%', 2, opAssoc.LEFT, handle_binary_op),
	('*', 2, opAssoc.LEFT, handle_binary_op),
	('/', 2, opAssoc.LEFT, handle_binary_op),
	('+', 2, opAssoc.LEFT, handle_binary_op),
	('-', 2, opAssoc.LEFT, handle_binary_op),
	('-', 1, opAssoc.RIGHT, handle_unary_op),
	('&&', 2, opAssoc.LEFT, handle_binary_op),
	('||', 2, opAssoc.LEFT, handle_binary_op),
	(('?', ':'), 3, opAssoc.RIGHT, handle_ternary_op),

	('<',	2, opAssoc.LEFT, handle_binary_op),
	('<=',	2, opAssoc.LEFT, handle_binary_op),
	('>',	2, opAssoc.LEFT, handle_binary_op),
	('>=',	2, opAssoc.LEFT, handle_binary_op),
	('==',	2, opAssoc.LEFT, handle_binary_op),
	('!=',	2, opAssoc.LEFT, handle_binary_op),
	('===',	2, opAssoc.LEFT, handle_binary_op),
	('!==',	2, opAssoc.LEFT, handle_binary_op),
])
expression_ops.setParseAction(lambda s, l, t: "(%s)" %t[0])

expression << expression_ops

expression_list_definition = expression + ZeroOrMore(Suppress(",") + expression)
expression_list << Optional(expression_list_definition)

source = component_declaration
cStyleComment.setParseAction(handle_documentation_string)
source = source.ignore(cStyleComment)
dblSlashComment.setParseAction(handle_documentation_string)
source = source.ignore(dblSlashComment)
#source.setDefaultWhitespaceChars(" \t\r\f")
ParserElement.enablePackrat()

def parse(data):
	tree = source.parseString(data, parseAll = True)
	return tree
<commit_msg>replaced custom lists with 'delimitedList' object<commit_after>from pyparsing import *
import lang

doc_next = None
doc_prev_component = None

def component(com):
	global doc_next, doc_prev_component
	if doc_next:
		com.doc = doc_next
		doc_next = None
	doc_prev_component = com
	return com

def document(text, line, prev):
	text = text.strip()
	if not text:
		print 'WARNING: empty documentation string at line %d' %line
		return

	global doc_next, doc_prev_component
	if prev:
		if doc_prev_component:
			if doc_prev_component.doc is not None:
				print 'WARNING: duplicate documentation string %s at line %d' %(text, line)
			doc_prev_component.doc = lang.DocumentationString(text)
		else:
			print 'WARNING: unused documentation string %s at line %d' %(text, line)
	else:
		if doc_next is not None:
			print 'WARNING: duplicate documentation string %s at line %d' %(text, line)
		doc_next = lang.DocumentationString(text)

def handle_component_declaration(s, l, t):
	#print "component>", t
	return component(lang.Component(t[0], t[1]))

def handle_assignment(s, l, t):
	#print "assignment>", t
	return component(lang.Assignment(t[0], t[1]))

def handle_property_declaration(s, l, t):
	#print "property>", t
	default = t[2] if len(t) > 2 else None
	return component(lang.Property(t[0], t[1], default))

def handle_alias_property_declaration(s, l, t):
	return component(lang.AliasProperty(t[0], t[1]))

def handle_enum_property_declaration(s, l, t):
	return component(lang.EnumProperty(t[0], t[1], t[2] if len(t) > 2 else None))

def handle_method_declaration(s, l, t):
	event_handler = t[0] != 'function'
	if event_handler:
		name, args, code = t[0], t[1], t[2]
	else:
		name, args, code = t[1], t[2], t[3]
	return component(lang.Method(name, args, code, event_handler) if name != 'constructor' else lang.Constructor(args, code))

def handle_assignment_scope(s, l, t):
	#print "assignment-scope>", t
	return component(lang.AssignmentScope(t[0], t[1]))

def handle_nested_identifier_rvalue(s, l, t):
	#print "nested-id>", t
	path = t[0].split(".")
	if path[0] == 'this':
		return 'this'
	if path[0] == "model":
		return "this._get('model').%s" %".".join(path[1:])
	path = ["_get('%s')" %x for x in path]
	return "this.%s" % ".".join(path)

def handle_enum_value(s, l, t):
	#print "enum>", t
	return "".join(t)

def handle_id_declaration(s, l, t):
	#print "id>", t
	return component(lang.IdAssignment(t[0]))

def handle_behavior_declaration(s, l, t):
	#print "behavior>", t
	return component(lang.Behavior(t[0], t[1]))

def handle_signal_declaration(s, l, t):
	return component(lang.Signal(t[0]))

def handle_function_call(s, l, t):
	#print "func> ", t
	name = t[0]
	if name[0].islower():
		name = '_globals.' + name
	return "%s(%s)" % (name, ",".join(t[1:]))

def handle_documentation_string(s, l, t):
	text = t[0]
	if text.startswith('///<'):
		document(text[4:], l, True)
	elif text.startswith('///'):
		document(text[3:], l, False)
	elif text.startswith('/**'):
		end = text.rfind('*/')
		document(text[3:end], l, False)

def handle_json_object(s, l, tokens):
	obj = {}
	for key, value in tokens:
		obj[key] = value
	return obj

def handle_list_element(s, l, t):
	return lang.ListElement(t[0])

def handle_number(s, l, t):
	value = t[0]
	return float(value) if '.' in value else int(value)

expression = Forward()
expression_list = Forward()
component_declaration = Forward()

type = Word(alphas, alphanums)
component_type = Word(srange("[A-Za-z_]"), alphanums + "._")
identifier = Word(srange("[a-z_]"), alphanums + "_")
code = originalTextFor(nestedExpr("{", "}", None, None))
null_value = Keyword("null")
bool_value = Keyword("true") | Keyword("false")
number = Word("01234567890+-.")
number.setParseAction(handle_number)

quoted_string_value = \
	QuotedString('"', escChar='\\', unquoteResults = False, multiline=True) | \
	QuotedString("'", escChar='\\', unquoteResults = False, multiline=True)

unquoted_string_value = \
	QuotedString('"', escChar='\\', unquoteResults = True, multiline=True) | \
	QuotedString("'", escChar='\\', unquoteResults = True, multiline=True)

enum_element = Word(srange("[A-Z_]"), alphanums)
enum_value = Word(srange("[A-Z_]"), alphanums) + Literal(".") + enum_element
enum_value.setParseAction(handle_enum_value)

function_call = Word(alphanums + '._') + Literal("(").suppress() + expression_list + Literal(")").suppress() + \
	ZeroOrMore(Literal(".").suppress() + Literal("arg").suppress() + Literal("(").suppress() + expression + Literal(")").suppress())
function_call.setParseAction(handle_function_call)

nested_identifier_lvalue = Word(srange("[a-z_]"), alphanums + "._")

nested_identifier_rvalue = Word(srange("[a-z_]"), alphanums + "._")
nested_identifier_rvalue.setParseAction(handle_nested_identifier_rvalue)

expression_end = Literal(";").suppress()

signal_declaration = Keyword("signal").suppress() + identifier + expression_end
signal_declaration.setParseAction(handle_signal_declaration)

id_declaration = Keyword("id").suppress() + Literal(":").suppress() + identifier + expression_end
id_declaration.setParseAction(handle_id_declaration)

assign_declaration = nested_identifier_lvalue + Literal(":").suppress() + expression + expression_end
assign_declaration.setParseAction(handle_assignment)

assign_component_declaration = nested_identifier_lvalue + Literal(":").suppress() + component_declaration
assign_component_declaration.setParseAction(handle_assignment)

property_declaration = (((Keyword("property").suppress() + type + identifier + Literal(":").suppress() + expression) | \
		(Keyword("property").suppress() + type + identifier)) + expression_end) | \
	(Keyword("property").suppress() + type + identifier + Literal(":").suppress() + component_declaration) \

alias_property_declaration = Keyword("property").suppress() + Keyword("alias").suppress() + identifier + Literal(":").suppress() + nested_identifier_lvalue + expression_end
alias_property_declaration.setParseAction(handle_alias_property_declaration)

enum_property_declaration = Keyword("property").suppress() + Keyword("enum").suppress() + identifier + \
	Literal("{").suppress() + Group(delimitedList(enum_element, ',')) + Literal("}").suppress() + Optional(Literal(':').suppress() + enum_element) + expression_end
enum_property_declaration.setParseAction(handle_enum_property_declaration)

property_declaration.setParseAction(handle_property_declaration)

assign_scope_declaration = identifier + Literal(":").suppress() + expression + expression_end
assign_scope_declaration.setParseAction(handle_assignment)
assign_scope = nested_identifier_lvalue + Literal("{").suppress() + Group(OneOrMore(assign_scope_declaration)) + Literal("}").suppress()
assign_scope.setParseAction(handle_assignment_scope)

method_declaration = nested_identifier_lvalue + Group(Optional(Literal("(").suppress() + delimitedList(identifier, ",") + Literal(")").suppress() )) + Literal(":").suppress() + code
method_declaration.setParseAction(handle_method_declaration)

method_declaration_qml = Keyword("function") + nested_identifier_lvalue + Group(Literal("(").suppress() + Optional(delimitedList(identifier, ",")) + Literal(")").suppress() ) + code
method_declaration_qml.setParseAction(handle_method_declaration)

behavior_declaration = Keyword("Behavior").suppress() + Keyword("on").suppress() + Group(delimitedList(nested_identifier_lvalue, ',')) + Literal("{").suppress() + component_declaration + Literal("}").suppress()
behavior_declaration.setParseAction(handle_behavior_declaration)

json_value = Forward()
json_object = Suppress("{") + delimitedList(Group((unquoted_string_value | identifier) + Suppress(":") + json_value) | empty, Suppress(";") | Suppress(",")) + Suppress('}')
json_object.setParseAction(handle_json_object)
json_array = Suppress("[") + delimitedList(json_value) + Suppress("]")
json_value << (null_value | bool_value | number | unquoted_string_value | json_array | json_object)

list_element_declaration = Keyword("ListElement").suppress() - json_object
list_element_declaration.setParseAction(handle_list_element)

scope_declaration = list_element_declaration | behavior_declaration | signal_declaration | alias_property_declaration | enum_property_declaration | property_declaration | id_declaration | assign_declaration | assign_component_declaration | component_declaration | method_declaration | method_declaration_qml | assign_scope
component_scope = (Literal("{").suppress() + Group(ZeroOrMore(scope_declaration)) + Literal("}").suppress())

component_declaration << (component_type + component_scope)
component_declaration.setParseAction(handle_component_declaration)

def handle_unary_op(s, l, t):
	#print "EXPR", t
	return " ".join(map(str, t[0]))
def handle_binary_op(s, l, t):
	#print "EXPR", t
	return " ".join(map(str, t[0]))
def handle_ternary_op(s, l, t):
	#print "EXPR", t
	return " ".join(map(str, t[0]))

expression_array = Literal("[") + Optional(delimitedList(expression, ",")) + Literal("]")
def handle_expression_array(s, l, t):
	return "".join(t)

expression_array.setParseAction(handle_expression_array)

expression_definition = bool_value | number | quoted_string_value | function_call | nested_identifier_rvalue | enum_value | expression_array

expression_ops = infixNotation(expression_definition, [
	('!', 1, opAssoc.RIGHT, handle_unary_op),
	('%', 2, opAssoc.LEFT, handle_binary_op),
	('*', 2, opAssoc.LEFT, handle_binary_op),
	('/', 2, opAssoc.LEFT, handle_binary_op),
	('+', 2, opAssoc.LEFT, handle_binary_op),
	('-', 2, opAssoc.LEFT, handle_binary_op),
	('-', 1, opAssoc.RIGHT, handle_unary_op),
	('&&', 2, opAssoc.LEFT, handle_binary_op),
	('||', 2, opAssoc.LEFT, handle_binary_op),
	(('?', ':'), 3, opAssoc.RIGHT, handle_ternary_op),

	('<',	2, opAssoc.LEFT, handle_binary_op),
	('<=',	2, opAssoc.LEFT, handle_binary_op),
	('>',	2, opAssoc.LEFT, handle_binary_op),
	('>=',	2, opAssoc.LEFT, handle_binary_op),
	('==',	2, opAssoc.LEFT, handle_binary_op),
	('!=',	2, opAssoc.LEFT, handle_binary_op),
	('===',	2, opAssoc.LEFT, handle_binary_op),
	('!==',	2, opAssoc.LEFT, handle_binary_op),
])
expression_ops.setParseAction(lambda s, l, t: "(%s)" %t[0])

expression << expression_ops
expression_list << Optional(delimitedList(expression, ","))

source = component_declaration
cStyleComment.setParseAction(handle_documentation_string)
source = source.ignore(cStyleComment)
dblSlashComment.setParseAction(handle_documentation_string)
source = source.ignore(dblSlashComment)
#source.setDefaultWhitespaceChars(" \t\r\f")
ParserElement.enablePackrat()

def parse(data):
	tree = source.parseString(data, parseAll = True)
	return tree
<|endoftext|>


<commit_before>"""
changes.

Usage:
  changes [options] <app_name> changelog
  changes [options] <app_name> release
  changes [options] <app_name> version
  changes [options] <app_name> test
  changes [options] <app_name> install
  changes [options] <app_name> upload
  changes [options] <app_name> pypi
  changes [options] <app_name> tag

  changes -h | --help

Options:
  --new-version=<ver>   Specify version.
  -p --patch            Patch-level version increment.
  -m --minor            Minor-level version increment.
  -M --major            Minor-level version increment.

  -h --help             Show this screen.

  --pypi=<pypi>         Use alternative package index
  --dry-run             Prints the commands that would have been executed.
  --skip-changelog      For the release task: should the changelog be generated
                        and committed?
  --tox                 Use tox instead of nosetests
  --test-command=<cmd>  Command to use to test the newly installed package
  --debug               Debug output.

The commands do the following:
   changelog   Generates an automatic changelog from your commit messages
   version     Increments the __version__ attribute of your module's __init__
   test        Runs your tests with nosetests
   install     Attempts to install the sdist
   tag         Tags your git repo with the new version number
   upload      Uploads your project with setup.py clean sdist upload
   pypi        Attempts to install your package from pypi
   release     Runs all the previous commands
"""

import ast
import re
import subprocess
import tempfile

from docopt import docopt
from path import path
import semantic_version
import logging
import virtualenv

import changes

log = logging.getLogger(__name__)
CHANGELOG = 'CHANGELOG.md'

def extract(dictionary, keys):
    """
    Extract only the specified keys from a dict

    :param dictionary: source dictionary
    :param keys: list of keys to extract
    :return dict: extracted dictionary
    """
    return dict(
        (k, dictionary[k]) for k in keys if k in dictionary
    )

def extract_version_arguments(arguments):
    version_arguments = extract(arguments, ['--major', '--minor', '--patch'])
    return dict([
        (key[2:], value) for key, value in version_arguments.items()
    ])

def increment(version, major=False, minor=False, patch=True):
    """
    Increment a semantic version

    :param version: str of the version to increment
    :param major: bool specifying major level version increment
    :param minor: bool specifying minor level version increment
    :param patch: bool specifying patch level version increment
    :return: str of the incremented version
    """
    version = semantic_version.Version(version)
    if major:
        version.major += 1
        version.minor = 0
        version.patch = 0
    elif minor:
        version.minor += 1
        version.patch = 0
    elif patch:
        version.patch += 1

    return str(version)

def get_new_version(app_name, current_version,
                    major=False, minor=False, patch=True):

    guess_new_version = increment(
        current_version,
        major=major,
        minor=minor,
        patch=patch
    )

    new_version = raw_input(
        'What is the release version for "%s" '
        '[Default: %s]: ' % (
            app_name, guess_new_version
        )
    )
    if not new_version:
        new_version = guess_new_version
    return new_version.strip()

def extract_attribute(app_name, attribute_name):
    """Extract metatdata property from a module"""
    with open('%s/__init__.py' % app_name) as input_file:
        for line in input_file:
            if line.startswith(attribute_name):
                return ast.literal_eval(line.split('=')[1].strip())

def replace_attribute(app_name, attribute_name, new_value, dry_run=True):
    init_file = '%s/__init__.py' % app_name
    _, tmp_file = tempfile.mkstemp()

    with open(init_file) as input_file:
        with open(tmp_file, 'w') as output_file:
            for line in input_file:
                if line.startswith(attribute_name):
                    line = "%s = '%s'\n" % (attribute_name, new_value)

                output_file.write(line)

    if not dry_run:
        path(tmp_file).move(init_file)
    else:
        log.debug(execute(['diff', tmp_file, init_file], dry_run=False))

def current_version(app_name):
    return extract_attribute(app_name, '__version__')

def execute(commands, dry_run=True):
    log.debug('executing %s', commands)
    if not dry_run:
        try:
            return subprocess.check_output(commands).split('\n')
        except subprocess.CalledProcessError, e:
            log.debug('return code: %s, output: %s', e.returncode, e.output)
            return False
    else:
        return True

def write_new_changelog(app_name, filename, content_lines, dry_run=True):
    heading_and_newline = (
        '# [Changelog](%s/releases)\n' %
        extract_attribute(app_name, '__url__')
    )

    with open(filename, 'r+') as f:
        existing = f.readlines()

    output = existing[2:]
    output.insert(0, '\n')

    for index, line in enumerate(content_lines):
        output.insert(0, content_lines[len(content_lines) - index - 1])

    output.insert(0, heading_and_newline)

    output = ''.join(output)

    if not dry_run:
        with open(filename, 'w+') as f:
            f.write(output)
    else:
        log.info('New changelog:\n%s' % output)

def changelog(arguments):
    dry_run = arguments['--dry-run']
    app_name = arguments['<app_name>']
    new_version = arguments['new_version']

    changelog_content = [
        '\n## [%s](%s/compare/%s...%s)\n\n' % (
            new_version, extract_attribute(app_name, '__url__'),
            current_version(app_name), new_version,
        )
    ]

    git_log_commands = [
        'git', 'log',  '--oneline', '--no-merges',
        '%s..master' % current_version(app_name),
    ]

    git_log_content = execute(git_log_commands, dry_run=False)

    if not git_log_content:
        git_log_commands.pop()
        log.debug('sniffing initial release, drop tags: %s', git_log_commands)
        git_log_content = execute(git_log_commands, dry_run=False)

    for index, line in enumerate(git_log_content):
        # http://stackoverflow.com/a/468378/5549
        sha1_re = re.match(r'^[0-9a-f]{5,40}\b', line)
        if sha1_re:
            sha1 = sha1_re.group()

            new_line = line.replace(
                sha1,
                '[%s](%s/commit/%s)' % (
                    sha1,
                    extract_attribute(app_name, '__url__'),
                    sha1
                )
            )
            log.debug('old line: %s\nnew line: %s' % (line, new_line))
            git_log_content[index] = new_line

    if git_log_content:
        [
            changelog_content.append('* %s\n' % line)
            if line else line
            for line in git_log_content[:-1]
        ]

    write_new_changelog(
        app_name,
        CHANGELOG,
        changelog_content,
        dry_run=dry_run
    )
    log.info('Added content to CHANGELOG.md')

def version(arguments):
    dry_run = arguments['--dry-run']
    app_name = arguments['<app_name>']
    new_version = arguments['new_version']

    replace_attribute(
        app_name,
        '__version__',
        new_version,
        dry_run=dry_run)

def commit_version_change(arguments):
    dry_run = arguments['--dry-run']
    app_name = arguments['<app_name>']
    new_version = arguments['new_version']

    commands = [
        'git', 'ci', '-mx', new_version,
        '%s/__init__.py' % app_name, CHANGELOG
    ]

    if not (execute(commands, dry_run=dry_run) and
            execute(['git', 'push'], dry_run=dry_run)):
        raise Exception('Version change commit failed')

def test(arguments):
    dry_run = arguments['--dry-run']
    command = 'nosetests'
    if arguments['--tox']:
        command = 'tox'

    if not execute([command], dry_run=False):
        raise Exception('Test command failed')

def make_virtualenv():
    tmp_dir = tempfile.mkdtemp()
    log.debug('tmp ve dir: %s' % tmp_dir)
    virtualenv.create_environment(tmp_dir, site_packages=False)
    return tmp_dir

def run_test_command(arguments):
    if arguments['--test-command']:
        test_command = arguments['--test-command'].split(' ')
        result = execute(test_command, dry_run=arguments['--dry-run'])
        log.info('Test command "%s" result: %s', test_command, result)

def install(arguments):
    dry_run = arguments['--dry-run']
    app_name = arguments['<app_name>']
    new_version = arguments['new_version']

    result = execute(
        ['python', 'setup.py', 'clean', 'sdist'],
        dry_run=dry_run
    )
    if result:
        tmp_dir = make_virtualenv()
        try:
            virtualenv.install_sdist(
                arguments['<app_name>'],
                'dist/%s-%s.tar.gz' % (app_name, new_version),
                '%s/bin/python' % tmp_dir
            )
            log.info('Successfully installed %s sdist', app_name)
            run_test_command(arguments)
        except:
            log.info('Error installing %s sdist', app_name)

        path(tmp_dir).rmtree(path(tmp_dir))

def upload(arguments):
    dry_run = arguments['--dry-run']
    pypi = arguments['--pypi']

    upload = ['python', 'setup.py', 'clean', 'sdist', 'upload']
    if pypi:
        upload.append('-r')
        upload.append(pypi)

    execute(upload, dry_run=dry_run)

def pypi(arguments):
    dry_run = arguments['--dry-run']
    app_name = arguments['<app_name>']
    pypi = arguments['--pypi']
    package_index = 'pypi'

    tmp_dir = make_virtualenv()

    install = ['%s/bin/pip' % tmp_dir, 'install', app_name]
    if pypi:
        install.append('-i')
        install.append(pypi)
        package_index = pypi

    try:
        result = execute(install, dry_run=dry_run)
        if result:
            log.info('Successfully installed %s from %s',
                     app_name, package_index)
        else:
            log.error('Failed to install %s from %s',
                      app_name, package_index)
        run_test_command(arguments)
    except:
        log.exception('')
        log.info('Error installing %s from %s', app_name, package_index)

    path(tmp_dir).rmtree(path(tmp_dir))

def tag(arguments):
    dry_run = arguments['--dry-run']
    new_version = arguments['new_version']

    execute(
        ['git', 'tag', '-a', new_version, '-m', '"%s"' % new_version],
        dry_run=dry_run
    )
    execute(['git', 'push', '--tags'], dry_run=dry_run)

def release(arguments):
    try:
        if not arguments['--skip-changelog']:
            changelog(arguments)
        version(arguments)
        test(arguments)
        commit_version_change(arguments)
        install(arguments)
        upload(arguments)
        pypi(arguments)
        tag(arguments)
    except:
        log.exception('Error releasing')

def main():
    commands = ['release', 'changelog', 'test', 'version', 'tag', 'upload',
                'install', 'pypi']

    arguments = docopt(__doc__, version=changes.__version__)
    debug = arguments['--debug']
    logging.basicConfig(level=logging.DEBUG if debug else logging.INFO)

    suppress_version_prompt_for = ['test', 'upload']

    app_name = arguments['<app_name>']

    if arguments['--new-version']:
        arguments['new_version'] = arguments['--new-version']

    log.debug('arguments: %s', arguments)

    for command in commands:
        if arguments[command]:
            if command not in suppress_version_prompt_for:
                arguments['new_version'] = get_new_version(
                    app_name,
                    current_version(app_name),
                    **extract_version_arguments(arguments)
                )
            globals()[command](arguments)
<commit_msg>Cleanup<commit_after>"""
changes.

Usage:
  changes [options] <app_name> changelog
  changes [options] <app_name> release
  changes [options] <app_name> version
  changes [options] <app_name> test
  changes [options] <app_name> install
  changes [options] <app_name> upload
  changes [options] <app_name> pypi
  changes [options] <app_name> tag

  changes -h | --help

Options:
  --new-version=<ver>   Specify version.
  -p --patch            Patch-level version increment.
  -m --minor            Minor-level version increment.
  -M --major            Minor-level version increment.

  -h --help             Show this screen.

  --pypi=<pypi>         Use alternative package index
  --dry-run             Prints the commands that would have been executed.
  --skip-changelog      For the release task: should the changelog be generated
                        and committed?
  --tox                 Use tox instead of nosetests
  --test-command=<cmd>  Command to use to test the newly installed package
  --debug               Debug output.

The commands do the following:
   changelog   Generates an automatic changelog from your commit messages
   version     Increments the __version__ attribute of your module's __init__
   test        Runs your tests with nosetests
   install     Attempts to install the sdist
   tag         Tags your git repo with the new version number
   upload      Uploads your project with setup.py clean sdist upload
   pypi        Attempts to install your package from pypi
   release     Runs all the previous commands
"""

import ast
import re
import subprocess
import tempfile

from docopt import docopt
from path import path
import semantic_version
import logging
import virtualenv

import changes

log = logging.getLogger(__name__)
CHANGELOG = 'CHANGELOG.md'

def extract(dictionary, keys):
    """
    Extract only the specified keys from a dict

    :param dictionary: source dictionary
    :param keys: list of keys to extract
    :return dict: extracted dictionary
    """
    return dict(
        (k, dictionary[k]) for k in keys if k in dictionary
    )

def extract_version_arguments(arguments):
    version_arguments = extract(arguments, ['--major', '--minor', '--patch'])
    return dict([
        (key[2:], value) for key, value in version_arguments.items()
    ])

def increment(version, major=False, minor=False, patch=True):
    """
    Increment a semantic version

    :param version: str of the version to increment
    :param major: bool specifying major level version increment
    :param minor: bool specifying minor level version increment
    :param patch: bool specifying patch level version increment
    :return: str of the incremented version
    """
    version = semantic_version.Version(version)
    if major:
        version.major += 1
        version.minor = 0
        version.patch = 0
    elif minor:
        version.minor += 1
        version.patch = 0
    elif patch:
        version.patch += 1

    return str(version)

def get_new_version(app_name, current_version,
                    major=False, minor=False, patch=True):

    guess_new_version = increment(
        current_version,
        major=major,
        minor=minor,
        patch=patch
    )

    new_version = raw_input(
        'What is the release version for "%s" '
        '[Default: %s]: ' % (
            app_name, guess_new_version
        )
    )
    if not new_version:
        new_version = guess_new_version
    return new_version.strip()

def extract_attribute(app_name, attribute_name):
    """Extract metatdata property from a module"""
    with open('%s/__init__.py' % app_name) as input_file:
        for line in input_file:
            if line.startswith(attribute_name):
                return ast.literal_eval(line.split('=')[1].strip())

def replace_attribute(app_name, attribute_name, new_value, dry_run=True):
    init_file = '%s/__init__.py' % app_name
    _, tmp_file = tempfile.mkstemp()

    with open(init_file) as input_file:
        with open(tmp_file, 'w') as output_file:
            for line in input_file:
                if line.startswith(attribute_name):
                    line = "%s = '%s'\n" % (attribute_name, new_value)

                output_file.write(line)

    if not dry_run:
        path(tmp_file).move(init_file)
    else:
        log.debug(execute(['diff', tmp_file, init_file], dry_run=False))

def current_version(app_name):
    return extract_attribute(app_name, '__version__')

def execute(commands, dry_run=True):
    log.debug('executing %s', commands)
    if not dry_run:
        try:
            return subprocess.check_output(commands).split('\n')
        except subprocess.CalledProcessError, e:
            log.debug('return code: %s, output: %s', e.returncode, e.output)
            return False
    else:
        return True

def write_new_changelog(app_name, filename, content_lines, dry_run=True):
    heading_and_newline = (
        '# [Changelog](%s/releases)\n' %
        extract_attribute(app_name, '__url__')
    )

    with open(filename, 'r+') as f:
        existing = f.readlines()

    output = existing[2:]
    output.insert(0, '\n')

    for index, line in enumerate(content_lines):
        output.insert(0, content_lines[len(content_lines) - index - 1])

    output.insert(0, heading_and_newline)

    output = ''.join(output)

    if not dry_run:
        with open(filename, 'w+') as f:
            f.write(output)
    else:
        log.info('New changelog:\n%s' % output)

def changelog(arguments):
    dry_run = arguments['--dry-run']
    app_name = arguments['<app_name>']
    new_version = arguments['new_version']

    changelog_content = [
        '\n## [%s](%s/compare/%s...%s)\n\n' % (
            new_version, extract_attribute(app_name, '__url__'),
            current_version(app_name), new_version,
        )
    ]

    git_log_commands = [
        'git', 'log',  '--oneline', '--no-merges',
        '%s..master' % current_version(app_name),
    ]

    git_log_content = execute(git_log_commands, dry_run=False)

    if not git_log_content:
        git_log_commands.pop()
        log.debug('sniffing initial release, drop tags: %s', git_log_commands)
        git_log_content = execute(git_log_commands, dry_run=False)

    for index, line in enumerate(git_log_content):
        # http://stackoverflow.com/a/468378/5549
        sha1_re = re.match(r'^[0-9a-f]{5,40}\b', line)
        if sha1_re:
            sha1 = sha1_re.group()

            new_line = line.replace(
                sha1,
                '[%s](%s/commit/%s)' % (
                    sha1,
                    extract_attribute(app_name, '__url__'),
                    sha1
                )
            )
            log.debug('old line: %s\nnew line: %s' % (line, new_line))
            git_log_content[index] = new_line

    if git_log_content:
        [
            changelog_content.append('* %s\n' % line)
            if line else line
            for line in git_log_content[:-1]
        ]

    write_new_changelog(
        app_name,
        CHANGELOG,
        changelog_content,
        dry_run=dry_run
    )
    log.info('Added content to CHANGELOG.md')

def version(arguments):
    dry_run = arguments['--dry-run']
    app_name = arguments['<app_name>']
    new_version = arguments['new_version']

    replace_attribute(
        app_name,
        '__version__',
        new_version,
        dry_run=dry_run)

def commit_version_change(arguments):
    dry_run = arguments['--dry-run']
    app_name = arguments['<app_name>']
    new_version = arguments['new_version']

    commands = [
        'git', 'ci', '-m', new_version,
        '%s/__init__.py' % app_name, CHANGELOG
    ]

    if not (execute(commands, dry_run=dry_run) and
            execute(['git', 'push'], dry_run=dry_run)):
        raise Exception('Version change commit failed')

def test(arguments):
    dry_run = arguments['--dry-run']
    command = 'nosetests'
    if arguments['--tox']:
        command = 'tox'

    if not execute([command], dry_run=False):
        raise Exception('Test command failed')

def make_virtualenv():
    tmp_dir = tempfile.mkdtemp()
    virtualenv.create_environment(tmp_dir, site_packages=False)
    return tmp_dir

def run_test_command(arguments):
    if arguments['--test-command']:
        test_command = arguments['--test-command'].split(' ')
        result = execute(test_command, dry_run=arguments['--dry-run'])
        log.info('Test command "%s" result: %s', test_command, result)

def install(arguments):
    dry_run = arguments['--dry-run']
    app_name = arguments['<app_name>']
    new_version = arguments['new_version']

    result = execute(
        ['python', 'setup.py', 'clean', 'sdist'],
        dry_run=dry_run
    )
    if result:
        tmp_dir = make_virtualenv()
        try:
            virtualenv.install_sdist(
                arguments['<app_name>'],
                'dist/%s-%s.tar.gz' % (app_name, new_version),
                '%s/bin/python' % tmp_dir
            )
            log.info('Successfully installed %s sdist', app_name)
            run_test_command(arguments)
        except:
            log.info('Error installing %s sdist', app_name)

        path(tmp_dir).rmtree(path(tmp_dir))

def upload(arguments):
    dry_run = arguments['--dry-run']
    pypi = arguments['--pypi']

    upload = ['python', 'setup.py', 'clean', 'sdist', 'upload']
    if pypi:
        upload.append('-r')
        upload.append(pypi)

    execute(upload, dry_run=dry_run)

def pypi(arguments):
    dry_run = arguments['--dry-run']
    app_name = arguments['<app_name>']
    pypi = arguments['--pypi']
    package_index = 'pypi'

    tmp_dir = make_virtualenv()

    install = ['%s/bin/pip' % tmp_dir, 'install', app_name]
    if pypi:
        install.append('-i')
        install.append(pypi)
        package_index = pypi

    try:
        result = execute(install, dry_run=dry_run)
        if result:
            log.info('Successfully installed %s from %s',
                     app_name, package_index)
        else:
            log.error('Failed to install %s from %s',
                      app_name, package_index)

        run_test_command(arguments)
    except:
        log.info('Error installing %s from %s', app_name, package_index)

    path(tmp_dir).rmtree(path(tmp_dir))

def tag(arguments):
    dry_run = arguments['--dry-run']
    new_version = arguments['new_version']

    execute(
        ['git', 'tag', '-a', new_version, '-m', '"%s"' % new_version],
        dry_run=dry_run
    )
    execute(['git', 'push', '--tags'], dry_run=dry_run)

def release(arguments):
    try:
        if not arguments['--skip-changelog']:
            changelog(arguments)
        version(arguments)
        test(arguments)
        commit_version_change(arguments)
        install(arguments)
        upload(arguments)
        pypi(arguments)
        tag(arguments)
    except:
        log.exception('Error releasing')

def main():
    commands = ['release', 'changelog', 'test', 'version', 'tag', 'upload',
                'install', 'pypi']

    arguments = docopt(__doc__, version=changes.__version__)
    debug = arguments['--debug']
    logging.basicConfig(level=logging.DEBUG if debug else logging.INFO)

    suppress_version_prompt_for = ['test', 'upload']

    app_name = arguments['<app_name>']

    if arguments['--new-version']:
        arguments['new_version'] = arguments['--new-version']

    log.debug('arguments: %s', arguments)

    for command in commands:
        if arguments[command]:
            if command not in suppress_version_prompt_for:
                arguments['new_version'] = get_new_version(
                    app_name,
                    current_version(app_name),
                    **extract_version_arguments(arguments)
                )
            globals()[command](arguments)
<|endoftext|>


<commit_before># Copyright 2012-2014 Jussi Pakkanen

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pickle, os, uuid

version = '0.16.0'

# This class contains all data that must persist over multiple
# invocations of Meson. It is roughly the same thing as
# cmakecache.

class CoreData():

    def __init__(self, options):
        self.guid = str(uuid.uuid4()).upper()
        self.test_guid = str(uuid.uuid4()).upper()
        self.target_guids = {}
        self.version = version
        self.prefix = options.prefix
        self.libdir = options.libdir
        self.bindir = options.bindir
        self.includedir = options.includedir
        self.datadir = options.datadir
        self.mandir = options.mandir
        self.localedir = options.localedir
        self.backend = options.backend
        self.buildtype = options.buildtype
        self.strip = options.strip
        self.use_pch = options.use_pch
        self.unity = options.unity
        self.coverage = options.coverage
        self.user_options = {}
        if options.cross_file is not None:
            self.cross_file = os.path.join(os.getcwd(), options.cross_file)
        else:
            self.cross_file = None

        self.compilers = {}
        self.cross_compilers = {}
        self.deps = {}
        self.ext_progs = {}
        self.ext_libs = {}

def load(filename):
    obj = pickle.load(open(filename, 'rb'))
    if not isinstance(obj, CoreData):
        raise RuntimeError('Core data file is corrupted.')
    if obj.version != version:
        raise RuntimeError('Build tree has been generated with Meson version %s, which is incompatible with current version %s.'%
                           (obj.version, version))
    return obj

def save(obj, filename):
    if obj.version != version:
        raise RuntimeError('Fatal version mismatch corruption.')
    pickle.dump(obj, open(filename, 'wb'))

forbidden_target_names = {'clean': None,
                          'clean-gcno': None,
                          'clean-gcda': None,
                          'coverage-text': None,
                          'coverage-xml': None,
                          'coverage-html': None,
                          'phony': None,
                          'all': None,
                          'test': None,
                          'test-valgrind': None,
                          'install': None,
                          'build.ninja': None,
                          'cppcheck': None,
                          }

class MesonException(Exception):
    def __init__(self, *args, **kwargs):
        Exception.__init__(self, *args, **kwargs)
<commit_msg>Bump version for new development.<commit_after># Copyright 2012-2014 Jussi Pakkanen

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pickle, os, uuid

version = '0.17.0-research'

# This class contains all data that must persist over multiple
# invocations of Meson. It is roughly the same thing as
# cmakecache.

class CoreData():

    def __init__(self, options):
        self.guid = str(uuid.uuid4()).upper()
        self.test_guid = str(uuid.uuid4()).upper()
        self.target_guids = {}
        self.version = version
        self.prefix = options.prefix
        self.libdir = options.libdir
        self.bindir = options.bindir
        self.includedir = options.includedir
        self.datadir = options.datadir
        self.mandir = options.mandir
        self.localedir = options.localedir
        self.backend = options.backend
        self.buildtype = options.buildtype
        self.strip = options.strip
        self.use_pch = options.use_pch
        self.unity = options.unity
        self.coverage = options.coverage
        self.user_options = {}
        if options.cross_file is not None:
            self.cross_file = os.path.join(os.getcwd(), options.cross_file)
        else:
            self.cross_file = None

        self.compilers = {}
        self.cross_compilers = {}
        self.deps = {}
        self.ext_progs = {}
        self.ext_libs = {}

def load(filename):
    obj = pickle.load(open(filename, 'rb'))
    if not isinstance(obj, CoreData):
        raise RuntimeError('Core data file is corrupted.')
    if obj.version != version:
        raise RuntimeError('Build tree has been generated with Meson version %s, which is incompatible with current version %s.'%
                           (obj.version, version))
    return obj

def save(obj, filename):
    if obj.version != version:
        raise RuntimeError('Fatal version mismatch corruption.')
    pickle.dump(obj, open(filename, 'wb'))

forbidden_target_names = {'clean': None,
                          'clean-gcno': None,
                          'clean-gcda': None,
                          'coverage-text': None,
                          'coverage-xml': None,
                          'coverage-html': None,
                          'phony': None,
                          'all': None,
                          'test': None,
                          'test-valgrind': None,
                          'install': None,
                          'build.ninja': None,
                          'cppcheck': None,
                          }

class MesonException(Exception):
    def __init__(self, *args, **kwargs):
        Exception.__init__(self, *args, **kwargs)
<|endoftext|>