<commit_before># -*- coding: utf-8 -*-

# Copyright 2010-2011 OpenStack Foundation
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import os
import shutil
import stat
import uuid

from alembic import command
import fixtures
from oslo_concurrency.fixture import lockutils as lockutils_fixture
from oslo_config import cfg
from oslo_log import log as logging
import pecan
import pecan.testing
import six
import sqlalchemy
import testtools

import storyboard.common.working_dir as working_dir
from storyboard.db.api import base as db_api_base
from storyboard.db.migration.cli import get_alembic_config
import storyboard.tests.mock_data as mock_data

CONF = cfg.CONF
_TRUE_VALUES = ('true', '1', 'yes')

_DB_CACHE = None

logging.register_options(CONF)
logging.setup(CONF, 'storyboard')

class TestCase(testtools.TestCase):

    """Test case base class for all unit tests."""

    def setUp(self):
        """Run before each test method to initialize test environment."""

        super(TestCase, self).setUp()
        test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
        try:
            test_timeout = int(test_timeout)
        except ValueError:
            # If timeout value is invalid do not set a timeout.
            test_timeout = 0
        if test_timeout > 0:
            self.useFixture(fixtures.Timeout(test_timeout, gentle=True))

        env_test_db = os.environ.get('STORYBOARD_TEST_DB')
        if env_test_db is not None:
            self.test_connection = env_test_db
        else:
            self.test_connection = ("mysql+pymysql://openstack_citest:"
                                    "openstack_citest@127.0.0.1:3306")

        self.useFixture(fixtures.NestedTempfile())
        self.useFixture(fixtures.TempHomeDir())
        self.useFixture(lockutils_fixture.ExternalLockFixture())

        self.addCleanup(CONF.reset)

        if os.environ.get('OS_STDOUT_CAPTURE') in _TRUE_VALUES:
            stdout = self.useFixture(fixtures.StringStream('stdout')).stream
            self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
        if os.environ.get('OS_STDERR_CAPTURE') in _TRUE_VALUES:
            stderr = self.useFixture(fixtures.StringStream('stderr')).stream
            self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))

        self.log_fixture = self.useFixture(fixtures.FakeLogger())

    def config(self, **kw):
        """Override some configuration values.

        The keyword arguments are the names of configuration options to
        override and their values.

        If a group argument is supplied, the overrides are applied to
        the specified configuration option group.

        All overrides are automatically cleared at the end of the current
        test by the fixtures cleanup process.
        """
        group = kw.pop('group', None)
        for k, v in six.iteritems(kw):
            CONF.set_override(k, v, group)

class WorkingDirTestCase(TestCase):
    """A test case that ensures that the configured working directory,
    if ever invoked, does not pollute the global working-directory space.
    """
    def setUp(self):
        super(WorkingDirTestCase, self).setUp()

        # First register the teardown.
        self.addCleanup(self._delete_working_dir)

        workdir = fixtures.TempDir()
        self.useFixture(workdir)
        temp_path = workdir.path
        CONF.set_override('working_directory', temp_path)

        # Create the directory
        path = working_dir.get_working_directory()
        self.assertTrue(os.path.exists(path))

    def _delete_working_dir(self):
        path = working_dir.get_working_directory()
        perms = stat.S_IRWXU + stat.S_IRWXG + stat.S_IRWXO

        if os.path.exists(path):
            os.chmod(path, perms)
            shutil.rmtree(path)

        self.assertFalse(os.path.exists(path))

        working_dir.WORKING_DIRECTORY = None

class DbTestCase(WorkingDirTestCase):

    def setUp(self):
        super(DbTestCase, self).setUp()
        self.setup_db()

    def setup_db(self):
        self.db_name = "storyboard_test_db_%s" % uuid.uuid4()
        self.db_name = self.db_name.replace("-", "_")

        # The engine w/o db name
        engine = sqlalchemy.create_engine(
            self.test_connection)
        engine.execute("CREATE DATABASE %s" % self.db_name)

        alembic_config = get_alembic_config()
        alembic_config.storyboard_config = CONF
        CONF.set_override(
            "connection",
            self.test_connection + "/%s"
            % self.db_name,
            group="database")

        command.upgrade(alembic_config, "head")
        self.addCleanup(self._drop_db)

    def _drop_db(self):
        engine = sqlalchemy.create_engine(
            self.test_connection)
        engine.execute("DROP DATABASE %s" % self.db_name)
        db_api_base.cleanup()

PATH_PREFIX = '/v1'

class HybridSessionManager(object):

    def _mock_get_session(self, autocommit=True, expire_on_commit=False,
                          in_request=True, **kwargs):
        return self.original_get_session(autocommit=autocommit,
                                         expire_on_commit=expire_on_commit,
                                         in_request=False, **kwargs)

    def __enter__(self):
        self.original_get_session = db_api_base.get_session
        db_api_base.get_session = self._mock_get_session

    def __exit__(self, exc_type, exc_val, exc_tb):
        db_api_base.get_session = self.original_get_session

class FunctionalTest(DbTestCase):
    """Used for functional tests of Pecan controllers where you need to
    test your literal application and its integration with the
    framework.
    """

    def setUp(self):
        super(FunctionalTest, self).setUp()

        self.default_headers = {}
        self.app = self._make_app()
        self.addCleanup(self._reset_pecan)
        self.addCleanup(self._clear_headers)
        mock_data.load()

    def _make_app(self):
        config = {
            'app': {
                'root': 'storyboard.api.root_controller.RootController',
                'modules': ['storyboard.api']
            }
        }
        return pecan.testing.load_test_app(config=config)

    def _reset_pecan(self):
        pecan.set_config({}, overwrite=True)

    def _clear_headers(self):
        self.default_headers = {}

    def _request_json(self, path, params, expect_errors=False, headers=None,
                      method="post", extra_environ=None, status=None,
                      path_prefix=PATH_PREFIX):
        """Sends simulated HTTP request to Pecan test app.

        :param path: url path of target service
        :param params: content for wsgi.input of request
        :param expect_errors: Boolean value; whether an error is expected based
                              on request
        :param headers: a dictionary of headers to send along with the request
        :param method: Request method type. Appropriate method function call
                       should be used rather than passing attribute in.
        :param extra_environ: a dictionary of environ variables to send along
                              with the request
        :param status: expected status code of response
        :param path_prefix: prefix of the url path
        """

        merged_headers = self.default_headers.copy()
        if headers:
            merged_headers.update(headers)

        full_path = path_prefix + path
        response = getattr(self.app, "%s_json" % method)(
            six.text_type(full_path),
            params=params,
            headers=merged_headers,
            status=status,
            extra_environ=extra_environ,
            expect_errors=expect_errors
        )
        return response

    def put_json(self, path, params, expect_errors=False, headers=None,
                 extra_environ=None, status=None):
        """Sends simulated HTTP PUT request to Pecan test app.

        :param path: url path of target service
        :param params: content for wsgi.input of request
        :param expect_errors: Boolean value; whether an error is expected based
                              on request
        :param headers: a dictionary of headers to send along with the request
        :param extra_environ: a dictionary of environ variables to send along
                              with the request
        :param status: expected status code of response
        """
        return self._request_json(path=path, params=params,
                                  expect_errors=expect_errors,
                                  headers=headers, extra_environ=extra_environ,
                                  status=status, method="put")

    def post_json(self, path, params, expect_errors=False, headers=None,
                  extra_environ=None, status=None):
        """Sends simulated HTTP POST request to Pecan test app.

        :param path: url path of target service
        :param params: content for wsgi.input of request
        :param expect_errors: Boolean value; whether an error is expected based
                              on request
        :param headers: a dictionary of headers to send along with the request
        :param extra_environ: a dictionary of environ variables to send along
                              with the request
        :param status: expected status code of response
        """
        return self._request_json(path=path, params=params,
                                  expect_errors=expect_errors,
                                  headers=headers, extra_environ=extra_environ,
                                  status=status, method="post")

    def patch_json(self, path, params, expect_errors=False, headers=None,
                   extra_environ=None, status=None):
        """Sends simulated HTTP PATCH request to Pecan test app.

        :param path: url path of target service
        :param params: content for wsgi.input of request
        :param expect_errors: Boolean value; whether an error is expected based
                              on request
        :param headers: a dictionary of headers to send along with the request
        :param extra_environ: a dictionary of environ variables to send along
                              with the request
        :param status: expected status code of response
        """
        return self._request_json(path=path, params=params,
                                  expect_errors=expect_errors,
                                  headers=headers, extra_environ=extra_environ,
                                  status=status, method="patch")

    def delete(self, path, expect_errors=False, headers=None,
               extra_environ=None, status=None, path_prefix=PATH_PREFIX):
        """Sends simulated HTTP DELETE request to Pecan test app.

        :param path: url path of target service
        :param expect_errors: Boolean value; whether an error is expected based
                              on request
        :param headers: a dictionary of headers to send along with the request
        :param extra_environ: a dictionary of environ variables to send along
                              with the request
        :param status: expected status code of response
        :param path_prefix: prefix of the url path
        """
        merged_headers = self.default_headers.copy()
        if headers:
            merged_headers.update(headers)

        full_path = path_prefix + path
        response = self.app.delete(six.text_type(full_path),
                                   headers=merged_headers,
                                   status=status,
                                   extra_environ=extra_environ,
                                   expect_errors=expect_errors)
        return response

    def get_json(self, path, expect_errors=False, headers=None,
                 extra_environ=None, q=[], path_prefix=PATH_PREFIX, **params):
        """Sends simulated HTTP GET request to Pecan test app.

        :param path: url path of target service
        :param expect_errors: Boolean value;whether an error is expected based
                              on request
        :param headers: a dictionary of headers to send along with the request
        :param extra_environ: a dictionary of environ variables to send along
                              with the request
        :param q: list of queries consisting of: field, value, op, and type
                  keys
        :param path_prefix: prefix of the url path
        :param params: content for wsgi.input of request
        """
        merged_headers = self.default_headers.copy()
        if headers:
            merged_headers.update(headers)

        full_path = path_prefix + path
        query_params = {'q.field': [],
                        'q.value': [],
                        'q.op': [],
                        }
        for query in q:
            for name in ['field', 'op', 'value']:
                query_params['q.%s' % name].append(query.get(name, ''))
        all_params = {}
        all_params.update(params)
        if q:
            all_params.update(query_params)
        response = self.app.get(full_path,
                                params=all_params,
                                headers=merged_headers,
                                extra_environ=extra_environ,
                                expect_errors=expect_errors)
        if not expect_errors:
            response = response.json
        return response

    def validate_link(self, link):
        """Checks if the given link can get correct data."""

        # removes 'http://loicalhost' part
        full_path = link.split('localhost', 1)[1]
        try:
            self.get_json(full_path, path_prefix='')
            return True
        except Exception:
            return False
<commit_msg>log database management actions in tests<commit_after># -*- coding: utf-8 -*-

# Copyright 2010-2011 OpenStack Foundation
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import os
import shutil
import stat
import uuid

from alembic import command
import fixtures
from oslo_concurrency.fixture import lockutils as lockutils_fixture
from oslo_config import cfg
from oslo_log import log as logging
import pecan
import pecan.testing
import six
import sqlalchemy
import testtools

import storyboard.common.working_dir as working_dir
from storyboard.db.api import base as db_api_base
from storyboard.db.migration.cli import get_alembic_config
import storyboard.tests.mock_data as mock_data

CONF = cfg.CONF
_TRUE_VALUES = ('true', '1', 'yes')

_DB_CACHE = None

logging.register_options(CONF)
logging.setup(CONF, 'storyboard')
LOG = logging.getLogger(__name__)

class TestCase(testtools.TestCase):

    """Test case base class for all unit tests."""

    def setUp(self):
        """Run before each test method to initialize test environment."""

        super(TestCase, self).setUp()
        test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
        try:
            test_timeout = int(test_timeout)
        except ValueError:
            # If timeout value is invalid do not set a timeout.
            test_timeout = 0
        if test_timeout > 0:
            self.useFixture(fixtures.Timeout(test_timeout, gentle=True))

        env_test_db = os.environ.get('STORYBOARD_TEST_DB')
        if env_test_db is not None:
            self.test_connection = env_test_db
        else:
            self.test_connection = ("mysql+pymysql://openstack_citest:"
                                    "openstack_citest@127.0.0.1:3306")

        self.useFixture(fixtures.NestedTempfile())
        self.useFixture(fixtures.TempHomeDir())
        self.useFixture(lockutils_fixture.ExternalLockFixture())

        self.addCleanup(CONF.reset)

        if os.environ.get('OS_STDOUT_CAPTURE') in _TRUE_VALUES:
            stdout = self.useFixture(fixtures.StringStream('stdout')).stream
            self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
        if os.environ.get('OS_STDERR_CAPTURE') in _TRUE_VALUES:
            stderr = self.useFixture(fixtures.StringStream('stderr')).stream
            self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))

        self.log_fixture = self.useFixture(fixtures.FakeLogger())

    def config(self, **kw):
        """Override some configuration values.

        The keyword arguments are the names of configuration options to
        override and their values.

        If a group argument is supplied, the overrides are applied to
        the specified configuration option group.

        All overrides are automatically cleared at the end of the current
        test by the fixtures cleanup process.
        """
        group = kw.pop('group', None)
        for k, v in six.iteritems(kw):
            CONF.set_override(k, v, group)

class WorkingDirTestCase(TestCase):
    """A test case that ensures that the configured working directory,
    if ever invoked, does not pollute the global working-directory space.
    """
    def setUp(self):
        super(WorkingDirTestCase, self).setUp()

        # First register the teardown.
        self.addCleanup(self._delete_working_dir)

        workdir = fixtures.TempDir()
        self.useFixture(workdir)
        temp_path = workdir.path
        CONF.set_override('working_directory', temp_path)

        # Create the directory
        path = working_dir.get_working_directory()
        self.assertTrue(os.path.exists(path))

    def _delete_working_dir(self):
        path = working_dir.get_working_directory()
        perms = stat.S_IRWXU + stat.S_IRWXG + stat.S_IRWXO

        if os.path.exists(path):
            os.chmod(path, perms)
            shutil.rmtree(path)

        self.assertFalse(os.path.exists(path))

        working_dir.WORKING_DIRECTORY = None

class DbTestCase(WorkingDirTestCase):

    def setUp(self):
        super(DbTestCase, self).setUp()
        self.setup_db()

    def setup_db(self):
        self.db_name = "storyboard_test_db_%s" % uuid.uuid4()
        self.db_name = self.db_name.replace("-", "_")
        LOG.info('creating database %s', self.db_name)

        # The engine w/o db name
        engine = sqlalchemy.create_engine(
            self.test_connection)
        engine.execute("CREATE DATABASE %s" % self.db_name)

        alembic_config = get_alembic_config()
        alembic_config.storyboard_config = CONF
        CONF.set_override(
            "connection",
            self.test_connection + "/%s"
            % self.db_name,
            group="database")

        command.upgrade(alembic_config, "head")
        self.addCleanup(self._drop_db)

    def _drop_db(self):
        engine = sqlalchemy.create_engine(
            self.test_connection)
        try:
            engine.execute("DROP DATABASE %s" % self.db_name)
        except Exception as err:
            LOG.error('failed to drop database %s: %s',
                      self.db_name, err)
        db_api_base.cleanup()

PATH_PREFIX = '/v1'

class HybridSessionManager(object):

    def _mock_get_session(self, autocommit=True, expire_on_commit=False,
                          in_request=True, **kwargs):
        return self.original_get_session(autocommit=autocommit,
                                         expire_on_commit=expire_on_commit,
                                         in_request=False, **kwargs)

    def __enter__(self):
        self.original_get_session = db_api_base.get_session
        db_api_base.get_session = self._mock_get_session

    def __exit__(self, exc_type, exc_val, exc_tb):
        db_api_base.get_session = self.original_get_session

class FunctionalTest(DbTestCase):
    """Used for functional tests of Pecan controllers where you need to
    test your literal application and its integration with the
    framework.
    """

    def setUp(self):
        super(FunctionalTest, self).setUp()

        self.default_headers = {}
        self.app = self._make_app()
        self.addCleanup(self._reset_pecan)
        self.addCleanup(self._clear_headers)
        mock_data.load()

    def _make_app(self):
        config = {
            'app': {
                'root': 'storyboard.api.root_controller.RootController',
                'modules': ['storyboard.api']
            }
        }
        return pecan.testing.load_test_app(config=config)

    def _reset_pecan(self):
        pecan.set_config({}, overwrite=True)

    def _clear_headers(self):
        self.default_headers = {}

    def _request_json(self, path, params, expect_errors=False, headers=None,
                      method="post", extra_environ=None, status=None,
                      path_prefix=PATH_PREFIX):
        """Sends simulated HTTP request to Pecan test app.

        :param path: url path of target service
        :param params: content for wsgi.input of request
        :param expect_errors: Boolean value; whether an error is expected based
                              on request
        :param headers: a dictionary of headers to send along with the request
        :param method: Request method type. Appropriate method function call
                       should be used rather than passing attribute in.
        :param extra_environ: a dictionary of environ variables to send along
                              with the request
        :param status: expected status code of response
        :param path_prefix: prefix of the url path
        """

        merged_headers = self.default_headers.copy()
        if headers:
            merged_headers.update(headers)

        full_path = path_prefix + path
        response = getattr(self.app, "%s_json" % method)(
            six.text_type(full_path),
            params=params,
            headers=merged_headers,
            status=status,
            extra_environ=extra_environ,
            expect_errors=expect_errors
        )
        return response

    def put_json(self, path, params, expect_errors=False, headers=None,
                 extra_environ=None, status=None):
        """Sends simulated HTTP PUT request to Pecan test app.

        :param path: url path of target service
        :param params: content for wsgi.input of request
        :param expect_errors: Boolean value; whether an error is expected based
                              on request
        :param headers: a dictionary of headers to send along with the request
        :param extra_environ: a dictionary of environ variables to send along
                              with the request
        :param status: expected status code of response
        """
        return self._request_json(path=path, params=params,
                                  expect_errors=expect_errors,
                                  headers=headers, extra_environ=extra_environ,
                                  status=status, method="put")

    def post_json(self, path, params, expect_errors=False, headers=None,
                  extra_environ=None, status=None):
        """Sends simulated HTTP POST request to Pecan test app.

        :param path: url path of target service
        :param params: content for wsgi.input of request
        :param expect_errors: Boolean value; whether an error is expected based
                              on request
        :param headers: a dictionary of headers to send along with the request
        :param extra_environ: a dictionary of environ variables to send along
                              with the request
        :param status: expected status code of response
        """
        return self._request_json(path=path, params=params,
                                  expect_errors=expect_errors,
                                  headers=headers, extra_environ=extra_environ,
                                  status=status, method="post")

    def patch_json(self, path, params, expect_errors=False, headers=None,
                   extra_environ=None, status=None):
        """Sends simulated HTTP PATCH request to Pecan test app.

        :param path: url path of target service
        :param params: content for wsgi.input of request
        :param expect_errors: Boolean value; whether an error is expected based
                              on request
        :param headers: a dictionary of headers to send along with the request
        :param extra_environ: a dictionary of environ variables to send along
                              with the request
        :param status: expected status code of response
        """
        return self._request_json(path=path, params=params,
                                  expect_errors=expect_errors,
                                  headers=headers, extra_environ=extra_environ,
                                  status=status, method="patch")

    def delete(self, path, expect_errors=False, headers=None,
               extra_environ=None, status=None, path_prefix=PATH_PREFIX):
        """Sends simulated HTTP DELETE request to Pecan test app.

        :param path: url path of target service
        :param expect_errors: Boolean value; whether an error is expected based
                              on request
        :param headers: a dictionary of headers to send along with the request
        :param extra_environ: a dictionary of environ variables to send along
                              with the request
        :param status: expected status code of response
        :param path_prefix: prefix of the url path
        """
        merged_headers = self.default_headers.copy()
        if headers:
            merged_headers.update(headers)

        full_path = path_prefix + path
        response = self.app.delete(six.text_type(full_path),
                                   headers=merged_headers,
                                   status=status,
                                   extra_environ=extra_environ,
                                   expect_errors=expect_errors)
        return response

    def get_json(self, path, expect_errors=False, headers=None,
                 extra_environ=None, q=[], path_prefix=PATH_PREFIX, **params):
        """Sends simulated HTTP GET request to Pecan test app.

        :param path: url path of target service
        :param expect_errors: Boolean value;whether an error is expected based
                              on request
        :param headers: a dictionary of headers to send along with the request
        :param extra_environ: a dictionary of environ variables to send along
                              with the request
        :param q: list of queries consisting of: field, value, op, and type
                  keys
        :param path_prefix: prefix of the url path
        :param params: content for wsgi.input of request
        """
        merged_headers = self.default_headers.copy()
        if headers:
            merged_headers.update(headers)

        full_path = path_prefix + path
        query_params = {'q.field': [],
                        'q.value': [],
                        'q.op': [],
                        }
        for query in q:
            for name in ['field', 'op', 'value']:
                query_params['q.%s' % name].append(query.get(name, ''))
        all_params = {}
        all_params.update(params)
        if q:
            all_params.update(query_params)
        response = self.app.get(full_path,
                                params=all_params,
                                headers=merged_headers,
                                extra_environ=extra_environ,
                                expect_errors=expect_errors)
        if not expect_errors:
            response = response.json
        return response

    def validate_link(self, link):
        """Checks if the given link can get correct data."""

        # removes 'http://loicalhost' part
        full_path = link.split('localhost', 1)[1]
        try:
            self.get_json(full_path, path_prefix='')
            return True
        except Exception:
            return False
<|endoftext|>


<commit_before>#!/usr/bin/env python2
from __future__ import print_function

from stravalib import Client, exc
from sys import stderr, stdin
from tempfile import NamedTemporaryFile
import webbrowser
import os.path
import gzip
import argparse
#from cStringIO import StringIO
from io import StringIO
import requests
import gpxpy

try:
    import configparser
    cp = configparser.ConfigParser()
except:
    import ConfigParser
    cp = ConfigParser.ConfigParser()
try:
    from lxml import etree
except ImportError:
    import xml.etree.ElementTree as etree

#####

def main(arguments=""):

    allowed_exts = {'.tcx': lambda v: '<TrainingCenterDatabase' in v[:200],
                    '.gpx': lambda v: '<gpx' in v[:200],
                    '.fit': lambda v: v[8:12]=='.FIT'}

    p = argparse.ArgumentParser(description='''Uploads activities to Strava.''')
    p.add_argument('activities', nargs='*', type=argparse.FileType("rb"), default=(stdin,),
                   help="Activity files to upload (plain or gzipped {})".format(', '.join(allowed_exts)))
    p.add_argument('-P', '--no-popup', action='store_true', help="Don't browse to activities after upload.")
    p.add_argument('-E', '--env', help='Look for ACCESS_TOKEN in environment variable rather than ~/.stravacli')
    p.add_argument('-o', '--overwrite', action='store_true', help="In case of duplicate, delete and replace original.")
    g = p.add_argument_group('Activity file details')
    g.add_argument('-p', '--private', action='store_true', help='Make activities private')
    g.add_argument('-t', '--type', choices=allowed_exts, default=None,
                   help='Force files to be interpreted as being of given type (default is to autodetect based on name, or contents for stdin)')
    g.add_argument('-x', '--xml-desc', action='store_true', help="Parse name/description fields from GPX and TCX files.")
    g.add_argument('-T', '--title', help='Activity title')
    g.add_argument('-D', '--desc', dest='description', help='Activity description')
    g.add_argument('-A', '--activity-type', default=None, help='''Type of activity. If not specified, the default value is taken
                                                                  from user profile. Supported values:
                                                                  ride, run, swim, workout, hike, walk, nordicski, alpineski,
                                                                  backcountryski, iceskate, inlineskate, kitesurf, rollerski,
                                                                  windsurf, workout, snowboard, snowshoe''')

    if arguments == "" :
        args = p.parse_args()
    else :
        args = p.parse_args(arguments.split(" "))

    if args.xml_desc:
        if args.title:
            p.error('argument -T/--title not allowed with argument -x/--xml-desc')
        if args.description:
            p.error('argument -D/--desc not allowed with argument -x/--xml-desc')

    #####

    # Authorize Strava

    cid = 3163 # CLIENT_ID
    if args.env:
        cat = os.environ.get('ACCESS_TOKEN')
    else:
        cp.read(os.path.expanduser('~/.stravacli'))
        cat = None
        if cp.has_section('API'):
            cat = cp.get('API', 'ACCESS_TOKEN') if 'access_token' in cp.options('API') else None

    while True:
        client = Client(cat)
        try:
            athlete = client.get_athlete()
        except requests.exceptions.ConnectionError:
            p.error("Could not connect to Strava API")
        except Exception as e:
            print("NOT AUTHORIZED", file=stderr)
            print("Need Strava API access token. Launching web browser to obtain one.", file=stderr)
            client = Client()
            authorize_url = client.authorization_url(client_id=cid, redirect_uri='http://stravacli-dlenski.rhcloud.com/auth', scope='view_private,write')
            webbrowser.open_new_tab(authorize_url)
            client.access_token = cat = raw_input("Enter access token: ")
        else:
            if not cp.has_section('API'):
                cp.add_section('API')
            if not 'ACCESS_TOKEN' in cp.options('API') or cp.get('API', 'ACCESS_TOKEN', None)!=cat:
                cp.set('API', 'ACCESS_TOKEN', cat)
                cp.write(open(os.path.expanduser('~/.stravacli'),"w"))
            break

    print("Authorized to access account of {} {} (id {:d}).".format(athlete.firstname, athlete.lastname, athlete.id))

    #####

    for ii,f in enumerate(args.activities):
        if f is stdin:
            fn = 'stdin'
            contents = f.read()
            f = StringIO(contents)
            if args.type is None:
                # autodetect gzip and extension based on content
                if contents.startswith('\x1f\x8b'):
                    gz, cf, uf = '.gz', f, gzip.GzipFile(fileobj=f, mode='rb')
                    contents = uf.read()
                else:
                    gz, uf, cf = '', f, NamedTemporaryFile(suffix='.gz')
                    gzip.GzipFile(fileobj=cf, mode='w+b').writelines(f)
                for ext, checker in allowed_exts.items():
                    if checker(contents):
                        print("Uploading {} activity from stdin...".format(ext+gz))
                        break
                else:
                    p.error("Could not determine file type of stdin")
            else:
                base, ext = 'activity', args.type
        else:
            base, ext = os.path.splitext(f.name if args.type is None else 'activity.'+args.type)
            # autodetect based on extensions
            if ext.lower()=='.gz':
                base, ext = os.path.splitext(base)
                # un-gzip it in order to parse it
                gz, cf, uf = '.gz', f, None if args.no_parse else gzip.GzipFile(fileobj=f, mode='rb')
            else:
                gz, uf, cf = '', f, NamedTemporaryFile(suffix='.gz')
                gzip.GzipFile(fileobj=cf, mode='w+b').writelines(f)
            if ext.lower() not in allowed_exts:
                p.error("Don't know how to handle extension {} (allowed are {}).".format(ext, ', '.join(allowed_exts)))
            print("Uploading {} activity from {}...".format(ext+gz, f.name))

        # try to parse activity name, description from file if requested
        if args.xml_desc:
            uf.seek(0, 0)
            if ext.lower()=='.gpx':

                gpx_file = open(f.name, 'r')
                gpx = gpxpy.parse(gpx_file)

                for track in gpx.tracks :
                    title = track.name
                    desc = track.description

            elif ext.lower()=='.tcx':
                x = etree.parse(uf)
                notestag = x.find("{*}Activities/{*}Activity/{*}Notes")
                if notestag is not None:
                    title, desc = (notestag.text.split('\n',1)+[None])[:2]
        else:
            title = args.title
            desc = args.description

        # upload activity
        duplicate = False
        try:
            cf.seek(0, 0)
            upstat = client.upload_activity(cf, ext[1:] + '.gz', title, desc, private=args.private, activity_type=args.activity_type)
            activity = upstat.wait()
        except exc.ActivityUploadFailed as e:
            words = e.args[0].split()
            if words[-4:-1]==['duplicate','of','activity']:
                activity = client.get_activity(words[-1])
                if args.overwrite :
                    print("Uploading duplicate, overwriting original")
                    activity_id = activity.id
                    client.delete_activity(activity_id)
                    try:
                        cf.seek(0, 0)
                        upstat = client.upload_activity(cf, ext[1:] + '.gz', title, desc, private=args.private, activity_type=args.activity_type)
                        activity = upstat.wait()
                    except exc.ActivityUploadFailed as e:
                        print("Upload failed")

                else:
                    duplicate = True
            else:
                raise

        # show results
        uri = "http://strava.com/activities/{:d}".format(activity.id)
        print("  {}{}".format(uri, " (duplicate)" if duplicate else ''), file=stderr)
        if not args.no_popup:
            webbrowser.open_new_tab(uri)

if __name__ == "__main__":
    main()
<commit_msg>Delete stravaup.py<commit_after><|endoftext|>


<commit_before># coding: utf-8

""" General utilities. """

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import os
import sys

# Third-party
from astropy import log as logger
import numpy as np
from gary.util import get_pool

__all__ = ['main', 'get_parser']

def main(worker, path, cache_filename, cache_dtype, callback=None,
         mpi=False, overwrite=False, seed=42, **kwargs):
    np.random.seed(seed)

    # get a pool object for multiprocessing / MPI
    pool = get_pool(mpi=mpi)
    if mpi:
        logger.info("|----------- Using MPI -----------|")
    else:
        logger.info("|----------- Running in serial -----------|")

    # file that will hold all results
    cache_path = os.path.join(path, cache_filename)

    # path to initial conditions cache
    w0_filename = os.path.join(path, 'w0.npy')
    w0 = np.load(w0_filename)

    # path to potential name file
    pot_filename = os.path.join(path, 'potential.yml')

    norbits = len(w0)
    logger.info("Number of orbits: {}".format(norbits))

    if os.path.exists(cache_path) and overwrite:
        os.remove(cache_path)

    if not os.path.exists(cache_path):
        # make sure memmap file exists
        d = np.memmap(cache_path, mode='w+', dtype=cache_dtype, shape=(norbits,))
        d[:] = np.zeros(shape=(norbits), dtype=cache_dtype)

    tasks = [dict(index=i, w0_filename=w0_filename,
                  cache_filename=cache_path,
                  potential_filename=pot_filename, **kwargs) for i in range(norbits)]

    pool.map(worker, tasks, callback=callback)
    pool.close()

    sys.exit(0)

def get_parser():
    from argparse import ArgumentParser

    # Define parser object
    parser = ArgumentParser()
    parser.add_argument("-v", "--verbose", action="store_true", dest="verbose",
                        default=False, help="Be chatty! (default = False)")
    parser.add_argument("-q", "--quiet", action="store_true", dest="quiet",
                        default=False, help="Be quiet! (default = False)")
    parser.add_argument("-o", "--overwrite", action="store_true", dest="overwrite",
                        default=False, help="DESTROY. DESTROY. (default = False)")
    parser.add_argument("--seed", dest="seed", default=42, type=int,
                        help="Seed for random number generators.")

    parser.add_argument("--mpi", dest="mpi", default=False, action="store_true",
                        help="Use an MPI pool.")
    parser.add_argument("--path", dest="path", type=str, required=True,
                        help="Path to cache everything to (e.g., where to save the "
                             "initial conditions grid).")

    return parser

def callback(result):
    if result is None:
        return

    memmap = np.memmap(result['mmap_filename'], mode='r+',
                       shape=(result['norbits'],), dtype=result['dtype'])

    logger.debug("Flushing {0} to output array...".format(result['index']))
    if result['error_code'] != 0.:
        # error happened
        for key in memmap.dtype.names:
            if key in result:
                memmap[key][result['index']] = result[key]

    else:
        # all is well
        for key in memmap.dtype.names:
            memmap[key][result['index']] = result[key]

    # flush to output array
    memmap.flush()
    logger.debug("...flushed, washing hands.")
<commit_msg>log dtype<commit_after># coding: utf-8

""" General utilities. """

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import os
import sys

# Third-party
from astropy import log as logger
import numpy as np
from gary.util import get_pool

__all__ = ['main', 'get_parser']

def main(worker, path, cache_filename, cache_dtype, callback=None,
         mpi=False, overwrite=False, seed=42, **kwargs):
    np.random.seed(seed)

    # get a pool object for multiprocessing / MPI
    pool = get_pool(mpi=mpi)
    if mpi:
        logger.info("|----------- Using MPI -----------|")
    else:
        logger.info("|----------- Running in serial -----------|")

    # file that will hold all results
    cache_path = os.path.join(path, cache_filename)

    # path to initial conditions cache
    w0_filename = os.path.join(path, 'w0.npy')
    w0 = np.load(w0_filename)

    # path to potential name file
    pot_filename = os.path.join(path, 'potential.yml')

    norbits = len(w0)
    logger.info("Number of orbits: {}".format(norbits))

    if os.path.exists(cache_path) and overwrite:
        os.remove(cache_path)

    logger.info("Cache dtype: {0}".format(cache_dtype))
    logger.info("Cache norbits: {0}".format(norbits))

    if not os.path.exists(cache_path):
        # make sure memmap file exists
        d = np.memmap(cache_path, mode='w+', dtype=cache_dtype, shape=(norbits,))
        d[:] = np.zeros(shape=(norbits), dtype=cache_dtype)

    tasks = [dict(index=i, w0_filename=w0_filename,
                  cache_filename=cache_path,
                  potential_filename=pot_filename, **kwargs) for i in range(norbits)]

    pool.map(worker, tasks, callback=callback)
    pool.close()

    sys.exit(0)

def get_parser():
    from argparse import ArgumentParser

    # Define parser object
    parser = ArgumentParser()
    parser.add_argument("-v", "--verbose", action="store_true", dest="verbose",
                        default=False, help="Be chatty! (default = False)")
    parser.add_argument("-q", "--quiet", action="store_true", dest="quiet",
                        default=False, help="Be quiet! (default = False)")
    parser.add_argument("-o", "--overwrite", action="store_true", dest="overwrite",
                        default=False, help="DESTROY. DESTROY. (default = False)")
    parser.add_argument("--seed", dest="seed", default=42, type=int,
                        help="Seed for random number generators.")

    parser.add_argument("--mpi", dest="mpi", default=False, action="store_true",
                        help="Use an MPI pool.")
    parser.add_argument("--path", dest="path", type=str, required=True,
                        help="Path to cache everything to (e.g., where to save the "
                             "initial conditions grid).")

    return parser

def callback(result):
    if result is None:
        return

    memmap = np.memmap(result['mmap_filename'], mode='r+',
                       shape=(result['norbits'],), dtype=result['dtype'])

    logger.debug("Flushing {0} to output array...".format(result['index']))
    if result['error_code'] != 0.:
        # error happened
        for key in memmap.dtype.names:
            if key in result:
                memmap[key][result['index']] = result[key]

    else:
        # all is well
        for key in memmap.dtype.names:
            memmap[key][result['index']] = result[key]

    # flush to output array
    memmap.flush()
    logger.debug("...flushed, washing hands.")
<|endoftext|>


<commit_before><commit_msg>remove extraneous __all__<commit_after><|endoftext|>


<commit_before># Copyright (c) 2016 Intracom S.A. Telecom Solutions. All rights reserved.
#
# This program and the accompanying materials are made available under the
# terms of the Eclipse Public License v1.0 which accompanies this distribution,
# and is available at http://www.eclipse.org/legal/epl-v10.html

""" Controller Class- All controller-related functionality is here"""

import json
import logging
import os
import time
import stress_test.controller
import stress_test.emulator
import sys
import util.file_ops
import util.netutil
import util.process
import queue

class TestType:

    def __init__(self, args):

        """
        """
        self.nstat_test_type = args.test_type

    def load_test_conf(self, args):
        """ Loading test configuration for NSTAT experiment. Parsing
        configuration options from JSON input file

        :param args: ArgumentParser object containing user specified
        parameters (i.e test type, controller base directory, generator base
        directory) when running NSTAT
        :returns: test_config:
        :rtype: test_config:  python object resulting from a deserialized file
        like object containing a json document
        :type args: ArgumentParser object
        """
        json_conf = {}
        with open(args.json_config) as conf_file:
            json_conf = json.load(conf_file)

            ctrl_base_dir = args.ctrl_base_dir
            sb_emu_base_dir = args.sb_emu_base_dir

            controller = stress_test.controller.Controller.new(ctrl_base_dir,
                                                               json_conf)
            sb_emulator = stress_test.emulator.SBEmu.new(sb_emu_base_dir,
                                                         json_conf)

            if hasattr(args, 'nb_emu_base_dir'):
                nb_emu_base_dir = args.nb_emu_base_dir
                nb_emulator = stress_test.nb_generator.NBgen(nb_emu_base_dir,
                                                             json_conf,
                                                             controller,
                                                             sb_emulator)

    def set_test_log_level(self, args):
        """Setting log level for NSTAT experiment

        :param args: ArgumentParser object containing user specified
        parameters (i.e test type, controller base directory, generator base
        directory) when running NSTAT
        :type args: ArgumentParser object
        """
        logging_format = '[%(asctime)s %(levelname)7s ] %(message)s'
        if args.logging_level == 'INFO':
            logging.basicConfig(level=logging.INFO, stream=sys.stdout,
                            format=logging_format)
        elif args.logging_level == 'ERROR':
            logging.basicConfig(level=logging.ERROR, stream=sys.stdout,
                            format=logging_format)
        else:
            logging.basicConfig(level=logging.DEBUG, stream=sys.stdout,
                            format=logging_format)

        if args.log_file:
            open(args.log_file, 'a').close()
            file_logging_handler = logging.FileHandler(filename=args.log_file,
                                                       mode='w')
            if args.logging_level == 'INFO':
                file_logging_handler.setLevel(level=logging.INFO)
            elif args.logging_level == 'ERROR':
                file_logging_handler.setLevel(level=logging.ERROR)
            else:
                file_logging_handler.setLevel(level=logging.DEBUG)

    def test_selector(self, args):
        """
        """
        self.set_test_log_level(args)
        self.load_test_conf(args)
        exit()
        # compose full NSTAT test type depending on cli argument test_type and
        # emulator type
        emulator_name
        nstat_test_type = args.test_type + args.emulator_name

        # Run the test
        if nstat_test_type == 'sb_active_scalability_mtcbench':
            if not args.bypass_test:
                logging.info('[nstat_orchestrator] Running test {0}'.
                             format(nstat_test_type))
                exit()
                sb_active_scalability_cbench.sb_active_scalability_cbench_run(
                    args.json_output,
                    args.ctrl_base_dir,
                    args.sb_gen_base_dir,
                    test_config,
                    args.output_dir)

        # sb_active_stability_mtcbench
        elif nstat_test_type == 'sb_active_stability_mtcbench':
            if not args.bypass_test:
                logging.info('[nstat_orchestrator] Running test {0}'.
                             format(nstat_test_type))
                exit()
                sb_active_stability_cbench.sb_active_stability_cbench_run(
                    args.json_output,
                    args.ctrl_base_dir,
                    args.sb_gen_base_dir,
                    test_config,
                    args.output_dir)

        elif nstat_test_type == 'sb_active_scalability_multinet':
            if not args.bypass_test:
                logging.info('[nstat_orchestrator] Running test {0}'.
                             format(nstat_test_type))
                exit()
                oftraf_path = get_oftraf_path()
                sb_active_scalability_multinet.sb_active_scalability_multinet_run(
                    args.json_output,
                    args.ctrl_base_dir,
                    args.sb_gen_base_dir,
                    test_config,
                    args.output_dir,
                    oftraf_path)

        elif nstat_test_type == 'sb_idle_scalability_mtcbench':
            if not args.bypass_test:
                logging.info('[nstat_orchestrator] Running test {0}'.
                             format(nstat_test_type))
                exit()
                sb_idle_scalability_cbench.sb_idle_scalability_cbench_run(
                    args.json_output,
                    args.ctrl_base_dir,
                    args.sb_gen_base_dir,
                    test_config,
                    args.output_dir)

        elif nstat_test_type == 'sb_idle_scalability_multinet':
            if not args.bypass_test:
                logging.info('[nstat_orchestrator] Running test {0}'.
                             format(nstat_test_type))
                exit()
                sb_idle_scalability_multinet.sb_idle_scalability_multinet_run(
                    args.json_output,
                    args.ctrl_base_dir,
                    args.sb_gen_base_dir,
                    test_config,
                    args.output_dir)

        elif nstat_test_type == 'sb_idle_stability_multinet':
            if not args.bypass_test:
                logging.info('[nstat_orchestrator] Running test {0}'.
                             format(nstat_test_type))
                exit()
                oftraf_path = get_oftraf_path()
                sb_idle_stability_multinet.sb_idle_stability_multinet_run(
                    args.json_output,
                    args.ctrl_base_dir,
                    args.sb_gen_base_dir,
                    test_config,
                    args.output_dir,
                    oftraf_path)

        elif nstat_test_type == 'nb_active_scalability_multinet':
            if not args.bypass_test:
                logging.info('[nstat_orchestrator] Running test {0}'.
                             format(nstat_test_type))
                exit()
                nb_active_scalability_multinet.nb_active_scalability_multinet_run(
                    args.json_output,
                    args.ctrl_base_dir,
                    args.nb_gen_base_dir,
                    args.sb_gen_base_dir,
                    test_config,
                    args.output_dir,
                    args.logging_level)

        else:
            logging.error('[nstat_orchestrator] not valid test configuration')
            exit(0)

<commit_msg>Add missing import of nb_generator<commit_after># Copyright (c) 2016 Intracom S.A. Telecom Solutions. All rights reserved.
#
# This program and the accompanying materials are made available under the
# terms of the Eclipse Public License v1.0 which accompanies this distribution,
# and is available at http://www.eclipse.org/legal/epl-v10.html

""" Controller Class- All controller-related functionality is here"""

import json
import logging
import os
import time
import stress_test.controller
import stress_test.emulator
import stress_test.nb_generator
import sys
import util.file_ops
import util.netutil
import util.process
import queue

class TestType:

    def __init__(self, args):

        """
        """
        self.nstat_test_type = args.test_type

    def load_test_conf(self, args):
        """ Loading test configuration for NSTAT experiment. Parsing
        configuration options from JSON input file

        :param args: ArgumentParser object containing user specified
        parameters (i.e test type, controller base directory, generator base
        directory) when running NSTAT
        :returns: test_config:
        :rtype: test_config:  python object resulting from a deserialized file
        like object containing a json document
        :type args: ArgumentParser object
        """
        json_conf = {}
        with open(args.json_config) as conf_file:
            json_conf = json.load(conf_file)

            ctrl_base_dir = args.ctrl_base_dir
            sb_emu_base_dir = args.sb_emu_base_dir

            controller = stress_test.controller.Controller.new(ctrl_base_dir,
                                                               json_conf)
            sb_emulator = stress_test.emulator.SBEmu.new(sb_emu_base_dir,
                                                         json_conf)

            if hasattr(args, 'nb_emu_base_dir'):
                nb_emu_base_dir = args.nb_emu_base_dir
                nb_emulator = stress_test.nb_generator.NBgen(nb_emu_base_dir,
                                                             json_conf,
                                                             controller,
                                                             sb_emulator)

    def set_test_log_level(self, args):
        """Setting log level for NSTAT experiment

        :param args: ArgumentParser object containing user specified
        parameters (i.e test type, controller base directory, generator base
        directory) when running NSTAT
        :type args: ArgumentParser object
        """
        logging_format = '[%(asctime)s %(levelname)7s ] %(message)s'
        if args.logging_level == 'INFO':
            logging.basicConfig(level=logging.INFO, stream=sys.stdout,
                            format=logging_format)
        elif args.logging_level == 'ERROR':
            logging.basicConfig(level=logging.ERROR, stream=sys.stdout,
                            format=logging_format)
        else:
            logging.basicConfig(level=logging.DEBUG, stream=sys.stdout,
                            format=logging_format)

        if args.log_file:
            open(args.log_file, 'a').close()
            file_logging_handler = logging.FileHandler(filename=args.log_file,
                                                       mode='w')
            if args.logging_level == 'INFO':
                file_logging_handler.setLevel(level=logging.INFO)
            elif args.logging_level == 'ERROR':
                file_logging_handler.setLevel(level=logging.ERROR)
            else:
                file_logging_handler.setLevel(level=logging.DEBUG)

    def test_selector(self, args):
        """
        """
        self.set_test_log_level(args)
        self.load_test_conf(args)
        exit()
        # compose full NSTAT test type depending on cli argument test_type and
        # emulator type
        emulator_name
        nstat_test_type = args.test_type + args.emulator_name

        # Run the test
        if nstat_test_type == 'sb_active_scalability_mtcbench':
            if not args.bypass_test:
                logging.info('[nstat_orchestrator] Running test {0}'.
                             format(nstat_test_type))
                exit()
                sb_active_scalability_cbench.sb_active_scalability_cbench_run(
                    args.json_output,
                    args.ctrl_base_dir,
                    args.sb_gen_base_dir,
                    test_config,
                    args.output_dir)

        # sb_active_stability_mtcbench
        elif nstat_test_type == 'sb_active_stability_mtcbench':
            if not args.bypass_test:
                logging.info('[nstat_orchestrator] Running test {0}'.
                             format(nstat_test_type))
                exit()
                sb_active_stability_cbench.sb_active_stability_cbench_run(
                    args.json_output,
                    args.ctrl_base_dir,
                    args.sb_gen_base_dir,
                    test_config,
                    args.output_dir)

        elif nstat_test_type == 'sb_active_scalability_multinet':
            if not args.bypass_test:
                logging.info('[nstat_orchestrator] Running test {0}'.
                             format(nstat_test_type))
                exit()
                oftraf_path = get_oftraf_path()
                sb_active_scalability_multinet.sb_active_scalability_multinet_run(
                    args.json_output,
                    args.ctrl_base_dir,
                    args.sb_gen_base_dir,
                    test_config,
                    args.output_dir,
                    oftraf_path)

        elif nstat_test_type == 'sb_idle_scalability_mtcbench':
            if not args.bypass_test:
                logging.info('[nstat_orchestrator] Running test {0}'.
                             format(nstat_test_type))
                exit()
                sb_idle_scalability_cbench.sb_idle_scalability_cbench_run(
                    args.json_output,
                    args.ctrl_base_dir,
                    args.sb_gen_base_dir,
                    test_config,
                    args.output_dir)

        elif nstat_test_type == 'sb_idle_scalability_multinet':
            if not args.bypass_test:
                logging.info('[nstat_orchestrator] Running test {0}'.
                             format(nstat_test_type))
                exit()
                sb_idle_scalability_multinet.sb_idle_scalability_multinet_run(
                    args.json_output,
                    args.ctrl_base_dir,
                    args.sb_gen_base_dir,
                    test_config,
                    args.output_dir)

        elif nstat_test_type == 'sb_idle_stability_multinet':
            if not args.bypass_test:
                logging.info('[nstat_orchestrator] Running test {0}'.
                             format(nstat_test_type))
                exit()
                oftraf_path = get_oftraf_path()
                sb_idle_stability_multinet.sb_idle_stability_multinet_run(
                    args.json_output,
                    args.ctrl_base_dir,
                    args.sb_gen_base_dir,
                    test_config,
                    args.output_dir,
                    oftraf_path)

        elif nstat_test_type == 'nb_active_scalability_multinet':
            if not args.bypass_test:
                logging.info('[nstat_orchestrator] Running test {0}'.
                             format(nstat_test_type))
                exit()
                nb_active_scalability_multinet.nb_active_scalability_multinet_run(
                    args.json_output,
                    args.ctrl_base_dir,
                    args.nb_gen_base_dir,
                    args.sb_gen_base_dir,
                    test_config,
                    args.output_dir,
                    args.logging_level)

        else:
            logging.error('[nstat_orchestrator] not valid test configuration')
            exit(0)

<|endoftext|>


<commit_before><commit_msg>aceita retirar de rua válida, se endereço for inválido<commit_after><|endoftext|>


<commit_before>import unittest

import numpy
import six

import chainer
from chainer import cuda
from chainer import gradient_check
from chainer import links
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition

@testing.parameterize(*testing.product({
    'test': [True, False],
    'volatile': ['on', 'off'],
    'ndim': [0, 1, 2, 3],
}))
class BatchNormalizationTest(unittest.TestCase):

    def setUp(self):
        self.expander = (None, Ellipsis) + (None,) * (self.ndim)
        self.aggr_axes = (0,) + tuple(six.moves.range(2, self.ndim + 2))

        self.link = links.BatchNormalization(3)
        gamma = self.link.gamma.data
        gamma[...] = numpy.random.uniform(.5, 1, gamma.shape)
        beta = self.link.beta.data
        beta[...] = numpy.random.uniform(-1, 1, beta.shape)
        self.link.zerograds()

        self.gamma = gamma.copy()[self.expander]  # fixed on CPU
        self.beta = beta.copy()[self.expander]   # fixed on CPU

        shape = (7, 3) + (2,) * self.ndim
        self.x = numpy.random.uniform(-1, 1, shape).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, shape).astype(numpy.float32)

        if self.test:
            self.mean = numpy.random.uniform(-1, 1, (3,)).astype(numpy.float32)
            self.var = numpy.random.uniform(0.5, 1, (3,)).astype(numpy.float32)
            self.link.avg_mean[...] = self.mean
            self.link.avg_var[...] = self.var
        else:
            self.mean = self.x.mean(axis=self.aggr_axes)
            self.var = self.x.var(axis=self.aggr_axes)

    def check_forward(self, x_data):
        x = chainer.Variable(x_data, volatile=self.volatile)
        y = self.link(x, test=self.test)
        self.assertEqual(y.data.dtype, numpy.float32)

        mean = self.mean[self.expander]
        if self.test:
            std = numpy.sqrt(self.var[self.expander])
        else:
            std = numpy.sqrt(self.var[self.expander] + self.link.eps)
        y_expect = self.gamma * (self.x - mean) / std + self.beta

        gradient_check.assert_allclose(y_expect, y.data, rtol=1e-3, atol=1e-4)
        self.assertEqual(numpy.float32, y.data.dtype)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.link.to_gpu()
        self.check_forward(cuda.to_gpu(self.x))

    def check_backward(self, x_data, y_grad):
        def f(x):
            self.link(x, test=self.test)
        gradient_check.check_backward(
            self.link, x_data, y_grad, (self.link.gamma, self.link.beta),
            eps=1e-2, rtol=1e-3, atol=1e-4)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.link.to_gpu()
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

testing.run_module(__name__, __file__)
<commit_msg>Extract forward function<commit_after>import unittest

import numpy
import six

import chainer
from chainer import cuda
from chainer import gradient_check
from chainer import links
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition

def _batch_normalization(expander, gamma, beta, x, mean, var, eps, test):
    mean = mean[expander]
    if test:
        std = numpy.sqrt(var[expander])
    else:
        std = numpy.sqrt(var[expander] + eps)
    y_expect = gamma * (x - mean) / std + beta
    return y_expect

@testing.parameterize(*testing.product({
    'test': [True, False],
    'volatile': ['on', 'off'],
    'ndim': [0, 1, 2, 3],
}))
class BatchNormalizationTest(unittest.TestCase):

    def setUp(self):
        self.expander = (None, Ellipsis) + (None,) * (self.ndim)
        self.aggr_axes = (0,) + tuple(six.moves.range(2, self.ndim + 2))

        self.link = links.BatchNormalization(3)
        gamma = self.link.gamma.data
        gamma[...] = numpy.random.uniform(.5, 1, gamma.shape)
        beta = self.link.beta.data
        beta[...] = numpy.random.uniform(-1, 1, beta.shape)
        self.link.zerograds()

        self.gamma = gamma.copy()[self.expander]  # fixed on CPU
        self.beta = beta.copy()[self.expander]   # fixed on CPU

        shape = (7, 3) + (2,) * self.ndim
        self.x = numpy.random.uniform(-1, 1, shape).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, shape).astype(numpy.float32)

        if self.test:
            self.mean = numpy.random.uniform(-1, 1, (3,)).astype(numpy.float32)
            self.var = numpy.random.uniform(0.5, 1, (3,)).astype(numpy.float32)
            self.link.avg_mean[...] = self.mean
            self.link.avg_var[...] = self.var
        else:
            self.mean = self.x.mean(axis=self.aggr_axes)
            self.var = self.x.var(axis=self.aggr_axes)

    def check_forward(self, x_data):
        x = chainer.Variable(x_data, volatile=self.volatile)
        y = self.link(x, test=self.test)
        self.assertEqual(y.data.dtype, numpy.float32)

        y_expect = _batch_normalization(
            self.expander, self.gamma, self.beta, self.x, self.mean,
            self.var, self.link.eps, self.test)

        gradient_check.assert_allclose(y_expect, y.data, rtol=1e-3, atol=1e-4)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.link.to_gpu()
        self.check_forward(cuda.to_gpu(self.x))

    def check_backward(self, x_data, y_grad):
        def f(x):
            self.link(x, test=self.test)
        gradient_check.check_backward(
            self.link, x_data, y_grad, (self.link.gamma, self.link.beta),
            eps=1e-2, rtol=1e-3, atol=1e-4)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.link.to_gpu()
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>
import urllib2

from pox.openflow.libopenflow_01 import *
from entities import *
import sts.headerspace.topology_loader.topology_loader as hsa_topo
import sts.headerspace.headerspace.applications as hsa
from sts.headerspace.config_parser.openflow_parser import get_uniq_port_id
import pickle
import logging
import collections
from sts.util.console import msg

log = logging.getLogger("invariant_checker")

class InvariantChecker(object):
  def __init__(self, snapshotService):
    self.snapshotService = snapshotService

  # --------------------------------------------------------------#
  #                    Invariant checks                           #
  # --------------------------------------------------------------#
  @staticmethod
  def check_loops(simulation):
    # Warning! depends on python Hassell -- may be really slow!
    NTF = hsa_topo.generate_NTF(simulation.topology.live_switches)
    TTF = hsa_topo.generate_TTF(simulation.topology.live_links)
    loops = hsa.detect_loop(NTF, TTF, simulation.topology.access_links)
    return loops

  @staticmethod
  def check_connectivity(simulation):
    ''' Return any pairs that couldn't reach each other '''
    # Effectively, run compute physical omega, ignore concrete values of headers, and
    # check that all pairs can reach eachother
    physical_omega = InvariantChecker.compute_physical_omega(simulation.topology.live_switches,
                                                             simulation.topology.live_links,
                                                             simulation.topology.access_links)
    connected_pairs = set()
    # Omegas are { original port -> [(final hs1, final port1), (final hs2, final port2)...] }
    for start_port, final_location_list in physical_omega.iteritems():
      for _, final_port in final_location_list:
        connected_pairs.add((start_port, final_port))

    # TODO(cs): translate HSA port numbers to ofp_phy_ports in the
    # headerspace/ module instead of computing uniq_port_id here
    access_links = simulation.topology.access_links
    all_pairs = [ (get_uniq_port_id(l1.switch, l1.switch_port),get_uniq_port_id(l2.switch, l2.switch_port))
                  for l1 in access_links
                  for l2 in access_links if l1 != l2 ]
    all_pairs = set(all_pairs)
    remaining_pairs = all_pairs - connected_pairs
    # TODO(cs): don't print results here
    if len(remaining_pairs) > 0:
      msg.fail("Not all %d pairs are connected! (%d missing)" %
               (len(all_pairs),len(remaining_pairs)))
      log.info("remaining_pairs: %s" % (str(remaining_pairs)))
    else:
      msg.success("Fully connected!")
    return remaining_pairs

  @staticmethod
  def check_correspondence(simulation):
    ''' Return if there were any policy-violations '''
    log.debug("Snapshotting controller...")
    controllers_with_violations = []
    for controller in simulation.controller_manager.controllers:
      controller_snapshot = controller.snapshot_service.fetchSnapshot(controller)
      for c in controller_snapshot.switches:
        print "SNAPSHOT s %d : %s" % (c.dpid, c.flow_table.entries)
      log.debug("Computing physical omega...")
      physical_omega = InvariantChecker.compute_physical_omega(simulation.topology.live_switches,
                                                               simulation.topology.live_links,
                                                               simulation.topology.access_links)
      log.debug("Computing controller omega...")
      # note: using all_switches to compute the controller omega. The controller might still
      # reference switches in his omega that are currently dead, which should result in a
      # policy violation, not sts crashing
      controller_omega = InvariantChecker.compute_controller_omega(controller_snapshot,
                                                                   simulation.topology.switches,
                                                                   simulation.topology.live_links,
                                                                   simulation.topology.access_links)
      violations = InvariantChecker.infer_policy_violations(physical_omega, controller_omega)
      if violations:
        controllers_with_violations.append(controller)
    return controllers_with_violations

  # --------------------------------------------------------------#
  #                    HSA utilities                              #
  # --------------------------------------------------------------#
  @staticmethod
  def compute_physical_omega(live_switches, live_links, edge_links):
    (name_tf_pairs, TTF) = InvariantChecker._get_transfer_functions(live_switches, live_links)
    physical_omega = hsa.compute_omega(name_tf_pairs, TTF, edge_links)
    return physical_omega

  @staticmethod
  def compute_controller_omega(controller_snapshot, live_switches, live_links, edge_links):
    name_tf_pairs = hsa_topo.tf_pairs_from_snapshot(controller_snapshot, live_switches)
    # Frenetic doesn't store any link or host information.
    # No virtualization though, so we can assume the same TTF. TODO(cs): for now...
    TTF = hsa_topo.generate_TTF(live_links)
    return hsa.compute_omega(name_tf_pairs, TTF, edge_links)

  @staticmethod
  def _get_transfer_functions(live_switches, live_links):
    name_tf_pairs = hsa_topo.generate_tf_pairs(live_switches)
    TTF = hsa_topo.generate_TTF(live_links)
    return (name_tf_pairs, TTF)

  @staticmethod
  def infer_policy_violations(physical_omega, controller_omega):
    ''' Return if there were any missing entries '''
    log.info("# entries in physical omega: %d" % len(physical_omega))
    log.info("# entries in controller omega: %d" % len(controller_omega))

    def get_simple_dict(omega):
      # TODO(cs): ignoring original hs means that we don't account for
      # field modifications, e.g. TTL decreases
      #
      # Omegas are { original port -> [(final hs1, final port1), (final hs2, final port2)...] }
      # Want to turn them into port -> [(final hs1, final port1), (final hs2, final port2)...]
      simple_dict = collections.defaultdict(lambda: set())
      for key, tuples in omega.iteritems():
        port = key
        for tup in tuples:
          printable_tup = (str(tup[0]), tup[1])
          simple_dict[port].add(printable_tup)
      return simple_dict

    physical_omega = get_simple_dict(physical_omega)
    controller_omega = get_simple_dict(controller_omega)

    def print_missing_entries(print_string, omega1, omega2):
      any_missing_entries = False
      for origin_port, final_locations in omega1.iteritems():
        for final_location in final_locations:
          if origin_port not in omega2 or final_location not in omega2[origin_port]:
            any_missing_entries = True
            log.info(": %s: %s" % (print_string,  str(final_location)))
      if not any_missing_entries:
        log.info("No %s!" % print_string)
      return any_missing_entries

    # (physical - controller) = missing routing policies
    missing_routing_entries = print_missing_entries("final locations in physical missing from virtual",
                                                    physical_omega, controller_omega)
    # (controller - physical) = missing ACL policies.
    missing_acl_entries = print_missing_entries("final locations in virtual missing from physical",
                                                controller_omega, physical_omega)
    return missing_routing_entries or missing_acl_entries
<commit_msg>Clarifying invariant check API<commit_after>
import urllib2

from pox.openflow.libopenflow_01 import *
from entities import *
import sts.headerspace.topology_loader.topology_loader as hsa_topo
import sts.headerspace.headerspace.applications as hsa
from sts.headerspace.config_parser.openflow_parser import get_uniq_port_id
import pickle
import logging
import collections
from sts.util.console import msg

log = logging.getLogger("invariant_checker")

class InvariantChecker(object):
  def __init__(self, snapshotService):
    self.snapshotService = snapshotService

  # --------------------------------------------------------------#
  #                    Invariant checks                           #
  # --------------------------------------------------------------#

  # All invariant check methods must return a list, which is empty
  # for no violations, and non-empty for violations

  @staticmethod
  def check_loops(simulation):
    # Warning! depends on python Hassell -- may be really slow!
    NTF = hsa_topo.generate_NTF(simulation.topology.live_switches)
    TTF = hsa_topo.generate_TTF(simulation.topology.live_links)
    loops = hsa.detect_loop(NTF, TTF, simulation.topology.access_links)
    return loops

  @staticmethod
  def check_connectivity(simulation):
    ''' Return any pairs that couldn't reach each other '''
    # Effectively, run compute physical omega, ignore concrete values of headers, and
    # check that all pairs can reach eachother
    physical_omega = InvariantChecker.compute_physical_omega(simulation.topology.live_switches,
                                                             simulation.topology.live_links,
                                                             simulation.topology.access_links)
    connected_pairs = set()
    # Omegas are { original port -> [(final hs1, final port1), (final hs2, final port2)...] }
    for start_port, final_location_list in physical_omega.iteritems():
      for _, final_port in final_location_list:
        connected_pairs.add((start_port, final_port))

    # TODO(cs): translate HSA port numbers to ofp_phy_ports in the
    # headerspace/ module instead of computing uniq_port_id here
    access_links = simulation.topology.access_links
    all_pairs = [ (get_uniq_port_id(l1.switch, l1.switch_port),get_uniq_port_id(l2.switch, l2.switch_port))
                  for l1 in access_links
                  for l2 in access_links if l1 != l2 ]
    all_pairs = set(all_pairs)
    remaining_pairs = all_pairs - connected_pairs
    # TODO(cs): don't print results here
    if len(remaining_pairs) > 0:
      msg.fail("Not all %d pairs are connected! (%d missing)" %
               (len(all_pairs),len(remaining_pairs)))
      log.info("remaining_pairs: %s" % (str(remaining_pairs)))
    else:
      msg.success("Fully connected!")
    return list(remaining_pairs)

  @staticmethod
  def check_correspondence(simulation):
    ''' Return if there were any policy-violations '''
    log.debug("Snapshotting controller...")
    controllers_with_violations = []
    for controller in simulation.controller_manager.controllers:
      controller_snapshot = controller.snapshot_service.fetchSnapshot(controller)
      for c in controller_snapshot.switches:
        print "SNAPSHOT s %d : %s" % (c.dpid, c.flow_table.entries)
      log.debug("Computing physical omega...")
      physical_omega = InvariantChecker.compute_physical_omega(simulation.topology.live_switches,
                                                               simulation.topology.live_links,
                                                               simulation.topology.access_links)
      log.debug("Computing controller omega...")
      # note: using all_switches to compute the controller omega. The controller might still
      # reference switches in his omega that are currently dead, which should result in a
      # policy violation, not sts crashing
      controller_omega = InvariantChecker.compute_controller_omega(controller_snapshot,
                                                                   simulation.topology.switches,
                                                                   simulation.topology.live_links,
                                                                   simulation.topology.access_links)
      violations = InvariantChecker.infer_policy_violations(physical_omega, controller_omega)
      if violations:
        controllers_with_violations.append(controller)
    return controllers_with_violations

  # --------------------------------------------------------------#
  #                    HSA utilities                              #
  # --------------------------------------------------------------#
  @staticmethod
  def compute_physical_omega(live_switches, live_links, edge_links):
    (name_tf_pairs, TTF) = InvariantChecker._get_transfer_functions(live_switches, live_links)
    physical_omega = hsa.compute_omega(name_tf_pairs, TTF, edge_links)
    return physical_omega

  @staticmethod
  def compute_controller_omega(controller_snapshot, live_switches, live_links, edge_links):
    name_tf_pairs = hsa_topo.tf_pairs_from_snapshot(controller_snapshot, live_switches)
    # Frenetic doesn't store any link or host information.
    # No virtualization though, so we can assume the same TTF. TODO(cs): for now...
    TTF = hsa_topo.generate_TTF(live_links)
    return hsa.compute_omega(name_tf_pairs, TTF, edge_links)

  @staticmethod
  def _get_transfer_functions(live_switches, live_links):
    name_tf_pairs = hsa_topo.generate_tf_pairs(live_switches)
    TTF = hsa_topo.generate_TTF(live_links)
    return (name_tf_pairs, TTF)

  @staticmethod
  def infer_policy_violations(physical_omega, controller_omega):
    ''' Return if there were any missing entries '''
    log.info("# entries in physical omega: %d" % len(physical_omega))
    log.info("# entries in controller omega: %d" % len(controller_omega))

    def get_simple_dict(omega):
      # TODO(cs): ignoring original hs means that we don't account for
      # field modifications, e.g. TTL decreases
      #
      # Omegas are { original port -> [(final hs1, final port1), (final hs2, final port2)...] }
      # Want to turn them into port -> [(final hs1, final port1), (final hs2, final port2)...]
      simple_dict = collections.defaultdict(lambda: set())
      for key, tuples in omega.iteritems():
        port = key
        for tup in tuples:
          printable_tup = (str(tup[0]), tup[1])
          simple_dict[port].add(printable_tup)
      return simple_dict

    physical_omega = get_simple_dict(physical_omega)
    controller_omega = get_simple_dict(controller_omega)

    def print_missing_entries(print_string, omega1, omega2):
      any_missing_entries = False
      for origin_port, final_locations in omega1.iteritems():
        for final_location in final_locations:
          if origin_port not in omega2 or final_location not in omega2[origin_port]:
            any_missing_entries = True
            log.info(": %s: %s" % (print_string,  str(final_location)))
      if not any_missing_entries:
        log.info("No %s!" % print_string)
      return any_missing_entries

    # (physical - controller) = missing routing policies
    missing_routing_entries = print_missing_entries("final locations in physical missing from virtual",
                                                    physical_omega, controller_omega)
    # (controller - physical) = missing ACL policies.
    missing_acl_entries = print_missing_entries("final locations in virtual missing from physical",
                                                controller_omega, physical_omega)
    return missing_routing_entries or missing_acl_entries
<|endoftext|>