<commit_before># -*- coding: utf-8 -*-

# Copyright 2018 Juca Crispim <juca@poraodojuca.net>

# This file is part of toxicbuild.

# toxicbuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# toxicbuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with toxicbuild. If not, see <http://www.gnu.org/licenses/>.

from asyncio import ensure_future
import base64
import json
from pyrocumulus.web.applications import PyroApplication
from pyrocumulus.web.decorators import post, get
from pyrocumulus.web.handlers import BasePyroHandler
from pyrocumulus.web.urlmappers import URLSpec
from tornado import gen
from tornado.web import HTTPError
from toxicbuild.core.utils import LoggerMixin, validate_string
from toxicbuild.master.users import User
from toxicbuild.integrations import settings
from toxicbuild.integrations.github import (GithubInstallation, GithubApp,
                                            BadSignature)
from toxicbuild.integrations.gitlab import GitLabInstallation

class BaseWebhookReceiver(LoggerMixin, BasePyroHandler):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.event_type = None
        self.body = None
        self.events = {}

    async def _get_user_from_cookie(self):
        cookie = self.get_secure_cookie(settings.TOXICUI_COOKIE)
        if not cookie:
            self.log('No cookie found.', level='debug')
            return

        user_dict = json.loads(base64.decodebytes(cookie).decode('utf-8'))
        user = await User.objects.get(id=user_dict['id'])
        return user

    def _parse_body(self):
        if self.request.body:
            self.body = json.loads(self.request.body.decode())

    def check_event_type(self):
        raise NotImplementedError

    def validate_webhook(self):
        raise NotImplementedError

    async def get_install(self):
        raise NotImplementedError

    def prepare(self):
        self._parse_body()
        self.event_type = self.check_event_type()

    @get('hello')
    def hello(self):
        return {'code': 200, 'msg': 'Hi there!'}

    @post('webhooks')
    async def receive_webhook(self):

        await self.validate_webhook()

        async def default_call():
            raise HTTPError(400, 'What was that? {}'.format(self.event_type))

        call = self.events.get(self.event_type, default_call)
        self.log('event_type {} received'.format(self.event_type))
        await call()
        msg = '{} handled successfully'.format(self.event_type)
        return {'code': 200, 'msg': msg}

    def create_installation(self, user):
        raise NotImplementedError

    def get_repo_external_id(self):
        raise NotImplementedError

    async def handle_push(self):
        external_id = self.get_repo_external_id()
        install = await self.get_install()
        ensure_future(install.update_repository(external_id))
        return 'updating repo'

    def get_pull_request_source(self):
        raise NotImplementedError

    def get_pull_request_target(self):
        raise NotImplementedError

    async def handle_pull_request(self):
        install = await self.get_install()
        source = self.get_pull_request_source()
        target = self.get_pull_request_target()

        repo_branches = {source['branch']: {
            'notify_only_latest': True,
            'builders_fallback': target['branch']}
        }
        if source['id'] != target['id']:
            external = {'url': source['url'],
                        'name': source['name'],
                        'branch': source['branch'],
                        'into': target['branch']}

            await install.update_repository(target['id'], external=external,
                                            repo_branches=repo_branches,
                                            wait_for_lock=True)
        else:
            await install.update_repository(target['id'],
                                            repo_branches=repo_branches,
                                            wait_for_lock=True)

    @get('setup')
    @gen.coroutine
    def setup(self):
        user = yield from self._get_user_from_cookie()
        if not user:
            url = '{}?redirect={}'.format(
                settings.TOXICUI_LOGIN_URL, self.request.full_url())
        else:
            self.create_installation(user)
            url = settings.TOXICUI_URL

        return self.redirect(url)

class GithubWebhookReceiver(BaseWebhookReceiver):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        handle_repo_removed = self._handle_install_repo_removed
        handle_repo_added = self._handle_install_repo_added
        self.events = {
            'ping': self._handle_ping,
            'push': self.handle_push,
            'repository-create': self._handle_install_repo_added,
            'pull_request-opened': self.handle_pull_request,
            'pull_request-synchronize': self.handle_pull_request,
            'check_run-rerequested': self._handle_check_run_rerequested,
            'installation-deleted': self._handle_install_deleted,
            'installation_repositories-removed': handle_repo_removed,
            'installation_repositories-added': handle_repo_added}

    def create_installation(self, user):
        githubapp_id = self.params.get('installation_id')
        if not githubapp_id:
            raise HTTPError(400)

        ensure_future(GithubInstallation.create(
            user, github_id=githubapp_id))

    async def _handle_ping(self):  # pragma no cover
        msg = 'Ping received. App id {}\n'.format(self.body['app_id'])
        msg += 'zen: {}'.format(self.body['zen'])
        self.log(msg, level='debug')
        return 'Got it.'

    async def get_install(self):
        install_id = self.body['installation']['id']
        install = await GithubInstallation.objects.get(github_id=install_id)
        return install

    def get_repo_external_id(self):
        repo_github_id = self.body['repository']['id']
        return repo_github_id

    async def _handle_install_repo_added(self):
        install = await self.get_install()
        tasks = []
        for repo_info in self.body['repositories_added']:
            t = ensure_future(self._get_and_import_repo(
                install, repo_info['full_name']))
            tasks.append(t)

        return tasks

    async def _get_and_import_repo(self, install, repo_full_name):
        repo_full_info = await install.get_repo(repo_full_name)
        await install.import_repository(repo_full_info)

    async def _handle_install_repo_removed(self):
        install = await self.get_install()
        for repo_info in self.body['repositories_removed']:
            ensure_future(install.remove_repository(repo_info['id']))

    def get_pull_request_source(self):
        head = self.body['pull_request']['head']
        source = {'id': head['repo']['id'],
                  'url': head['clone_url'],
                  'name': head['label'],
                  'branch': head['ref']}
        return source

    def get_pull_request_target(self):
        base = self.body['pull_request']['base']
        source = {'id': base['repo']['id'],
                  'branch': base['ref']}
        return source

    async def _handle_check_run_rerequested(self):
        install = await self.get_install()
        check_suite = self.body['check_run']['check_suite']
        repo_id = self.body['repository']['id']
        branch = check_suite['head_branch']
        named_tree = check_suite['head_sha']
        ensure_future(install.repo_request_build(repo_id, branch, named_tree))

    async def _handle_install_deleted(self):
        install = await self.get_install()
        await install.delete()

    async def validate_webhook(self):
        signature = self.request.headers.get('X-Hub-Signature')
        app = await GithubApp.get_app()
        try:
            app.validate_token(signature, self.request.body)
        except BadSignature:
            raise HTTPError(403, 'Bad signature')

    def check_event_type(self):
        event_type = self.request.headers.get('X-GitHub-Event')

        if not event_type:
            msg = 'No event type\n{}'.format(self.body)
            self.log(msg, level='warning')

        action = self.body.get('action') if self.body else None
        if action:
            event_type = '{}-{}'.format(event_type, action)
        return event_type

class GitlabWebhookReceiver(BaseWebhookReceiver):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.events = {'push': self.handle_push,
                       'merge_request': self.handle_pull_request}

    def check_event_type(self):
        body = self.body or {}
        return body.get('object_kind')

    def state_is_valid(self):
        """Checks if the state hash sent by gitlab is valid.
        """

        state = self.params.get('state')
        if not state:
            raise HTTPError(400)

        secret = settings.TORNADO_OPTS['cookie_secret']

        return validate_string(state, secret)

    def create_installation(self, user):
        code = self.params.get('code')
        if not code:
            raise HTTPError(400)

        if not self.state_is_valid():
            raise HTTPError(400)

        ensure_future(GitLabInstallation.create(user, code=code))

    async def get_install(self):
        install_id = self.params.get('installation_id')
        install = await GitLabInstallation.objects.get(id=install_id)
        return install

    def get_repo_external_id(self):
        return self.body['project']['id']

    def validate_webhook(self):
        secret = self.request.headers.get('X-Gitlab-Token')

        if secret != settings.GITLAB_WEBHOOK_TOKEN:
            raise HTTPError(403)
        return True

    def get_pull_request_source(self):
        attrs = self.body['object_attributes']
        return {'name': attrs['source']['name'],
                'id': attrs['source_project_id'],
                'branch': attrs['source_branch'],
                'url': attrs['source']['git_http_url']}

    def get_pull_request_target(self):
        attrs = self.body['object_attributes']
        return {'name': attrs['target']['name'],
                'id': attrs['target_project_id'],
                'branch': attrs['target_branch'],
                'url': attrs['target']['git_http_url']}

gh_url = URLSpec('/github/(.*)', GithubWebhookReceiver)
gl_url = URLSpec('/gitlab/(.*)', GitlabWebhookReceiver)
app = PyroApplication([gh_url, gl_url])
<commit_msg>[integrations] Prepare webhook request params<commit_after># -*- coding: utf-8 -*-

# Copyright 2018 Juca Crispim <juca@poraodojuca.net>

# This file is part of toxicbuild.

# toxicbuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# toxicbuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with toxicbuild. If not, see <http://www.gnu.org/licenses/>.

from asyncio import ensure_future
import base64
import json
from pyrocumulus.web.applications import PyroApplication
from pyrocumulus.web.decorators import post, get
from pyrocumulus.web.handlers import BasePyroHandler, PyroRequest
from pyrocumulus.web.urlmappers import URLSpec
from tornado import gen
from tornado.web import HTTPError
from toxicbuild.core.utils import LoggerMixin, validate_string
from toxicbuild.master.users import User
from toxicbuild.integrations import settings
from toxicbuild.integrations.github import (GithubInstallation, GithubApp,
                                            BadSignature)
from toxicbuild.integrations.gitlab import GitLabInstallation

class BaseWebhookReceiver(LoggerMixin, BasePyroHandler):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.event_type = None
        self.body = None
        self.events = {}

    async def _get_user_from_cookie(self):
        cookie = self.get_secure_cookie(settings.TOXICUI_COOKIE)
        if not cookie:
            self.log('No cookie found.', level='debug')
            return

        user_dict = json.loads(base64.decodebytes(cookie).decode('utf-8'))
        user = await User.objects.get(id=user_dict['id'])
        return user

    def _parse_body(self):
        if self.request.body:
            self.body = json.loads(self.request.body.decode())

    def check_event_type(self):
        raise NotImplementedError

    def validate_webhook(self):
        raise NotImplementedError

    async def get_install(self):
        raise NotImplementedError

    def prepare(self):
        self.params = PyroRequest(self.request.arguments)
        self._parse_body()
        self.event_type = self.check_event_type()

    @get('hello')
    def hello(self):
        return {'code': 200, 'msg': 'Hi there!'}

    @post('webhooks')
    async def receive_webhook(self):

        await self.validate_webhook()

        async def default_call():
            raise HTTPError(400, 'What was that? {}'.format(self.event_type))

        call = self.events.get(self.event_type, default_call)
        self.log('event_type {} received'.format(self.event_type))
        await call()
        msg = '{} handled successfully'.format(self.event_type)
        return {'code': 200, 'msg': msg}

    def create_installation(self, user):
        raise NotImplementedError

    def get_repo_external_id(self):
        raise NotImplementedError

    async def handle_push(self):
        external_id = self.get_repo_external_id()
        install = await self.get_install()
        ensure_future(install.update_repository(external_id))
        return 'updating repo'

    def get_pull_request_source(self):
        raise NotImplementedError

    def get_pull_request_target(self):
        raise NotImplementedError

    async def handle_pull_request(self):
        install = await self.get_install()
        source = self.get_pull_request_source()
        target = self.get_pull_request_target()

        repo_branches = {source['branch']: {
            'notify_only_latest': True,
            'builders_fallback': target['branch']}
        }
        if source['id'] != target['id']:
            external = {'url': source['url'],
                        'name': source['name'],
                        'branch': source['branch'],
                        'into': target['branch']}

            await install.update_repository(target['id'], external=external,
                                            repo_branches=repo_branches,
                                            wait_for_lock=True)
        else:
            await install.update_repository(target['id'],
                                            repo_branches=repo_branches,
                                            wait_for_lock=True)

    @get('setup')
    @gen.coroutine
    def setup(self):
        user = yield from self._get_user_from_cookie()
        if not user:
            url = '{}?redirect={}'.format(
                settings.TOXICUI_LOGIN_URL, self.request.full_url())
        else:
            self.create_installation(user)
            url = settings.TOXICUI_URL

        return self.redirect(url)

class GithubWebhookReceiver(BaseWebhookReceiver):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        handle_repo_removed = self._handle_install_repo_removed
        handle_repo_added = self._handle_install_repo_added
        self.events = {
            'ping': self._handle_ping,
            'push': self.handle_push,
            'repository-create': self._handle_install_repo_added,
            'pull_request-opened': self.handle_pull_request,
            'pull_request-synchronize': self.handle_pull_request,
            'check_run-rerequested': self._handle_check_run_rerequested,
            'installation-deleted': self._handle_install_deleted,
            'installation_repositories-removed': handle_repo_removed,
            'installation_repositories-added': handle_repo_added}

    def create_installation(self, user):
        githubapp_id = self.params.get('installation_id')
        if not githubapp_id:
            raise HTTPError(400)

        ensure_future(GithubInstallation.create(
            user, github_id=githubapp_id))

    async def _handle_ping(self):  # pragma no cover
        msg = 'Ping received. App id {}\n'.format(self.body['app_id'])
        msg += 'zen: {}'.format(self.body['zen'])
        self.log(msg, level='debug')
        return 'Got it.'

    async def get_install(self):
        install_id = self.body['installation']['id']
        install = await GithubInstallation.objects.get(github_id=install_id)
        return install

    def get_repo_external_id(self):
        repo_github_id = self.body['repository']['id']
        return repo_github_id

    async def _handle_install_repo_added(self):
        install = await self.get_install()
        tasks = []
        for repo_info in self.body['repositories_added']:
            t = ensure_future(self._get_and_import_repo(
                install, repo_info['full_name']))
            tasks.append(t)

        return tasks

    async def _get_and_import_repo(self, install, repo_full_name):
        repo_full_info = await install.get_repo(repo_full_name)
        await install.import_repository(repo_full_info)

    async def _handle_install_repo_removed(self):
        install = await self.get_install()
        for repo_info in self.body['repositories_removed']:
            ensure_future(install.remove_repository(repo_info['id']))

    def get_pull_request_source(self):
        head = self.body['pull_request']['head']
        source = {'id': head['repo']['id'],
                  'url': head['clone_url'],
                  'name': head['label'],
                  'branch': head['ref']}
        return source

    def get_pull_request_target(self):
        base = self.body['pull_request']['base']
        source = {'id': base['repo']['id'],
                  'branch': base['ref']}
        return source

    async def _handle_check_run_rerequested(self):
        install = await self.get_install()
        check_suite = self.body['check_run']['check_suite']
        repo_id = self.body['repository']['id']
        branch = check_suite['head_branch']
        named_tree = check_suite['head_sha']
        ensure_future(install.repo_request_build(repo_id, branch, named_tree))

    async def _handle_install_deleted(self):
        install = await self.get_install()
        await install.delete()

    async def validate_webhook(self):
        signature = self.request.headers.get('X-Hub-Signature')
        app = await GithubApp.get_app()
        try:
            app.validate_token(signature, self.request.body)
        except BadSignature:
            raise HTTPError(403, 'Bad signature')

    def check_event_type(self):
        event_type = self.request.headers.get('X-GitHub-Event')

        if not event_type:
            msg = 'No event type\n{}'.format(self.body)
            self.log(msg, level='warning')

        action = self.body.get('action') if self.body else None
        if action:
            event_type = '{}-{}'.format(event_type, action)
        return event_type

class GitlabWebhookReceiver(BaseWebhookReceiver):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.events = {'push': self.handle_push,
                       'merge_request': self.handle_pull_request}

    def check_event_type(self):
        body = self.body or {}
        return body.get('object_kind')

    def state_is_valid(self):
        """Checks if the state hash sent by gitlab is valid.
        """

        state = self.params.get('state')
        if not state:
            raise HTTPError(400)

        secret = settings.TORNADO_OPTS['cookie_secret']

        return validate_string(state, secret)

    def create_installation(self, user):
        code = self.params.get('code')
        if not code:
            raise HTTPError(400)

        if not self.state_is_valid():
            raise HTTPError(400)

        ensure_future(GitLabInstallation.create(user, code=code))

    async def get_install(self):
        install_id = self.params.get('installation_id')
        install = await GitLabInstallation.objects.get(id=install_id)
        return install

    def get_repo_external_id(self):
        return self.body['project']['id']

    def validate_webhook(self):
        secret = self.request.headers.get('X-Gitlab-Token')

        if secret != settings.GITLAB_WEBHOOK_TOKEN:
            raise HTTPError(403)
        return True

    def get_pull_request_source(self):
        attrs = self.body['object_attributes']
        return {'name': attrs['source']['name'],
                'id': attrs['source_project_id'],
                'branch': attrs['source_branch'],
                'url': attrs['source']['git_http_url']}

    def get_pull_request_target(self):
        attrs = self.body['object_attributes']
        return {'name': attrs['target']['name'],
                'id': attrs['target_project_id'],
                'branch': attrs['target_branch'],
                'url': attrs['target']['git_http_url']}

gh_url = URLSpec('/github/(.*)', GithubWebhookReceiver)
gl_url = URLSpec('/gitlab/(.*)', GitlabWebhookReceiver)
app = PyroApplication([gh_url, gl_url])
<|endoftext|>


<commit_before>#  Copyright (c) 2022, Manfred Moitzi
#  License: MIT License
from pathlib import Path
import ezdxf
from ezdxf import colors
from ezdxf.gfxattribs import GfxAttribs
from ezdxf.math import intersect_polylines_2d, Vec2

DIR = Path("~/Desktop/Outbox").expanduser()
ENTITIES = "ENTITIES"
INTERSECTION_POINTS = "INTERSECTION_POINTS"
CURVE_APPROXIMATIONS = "CURVE_APPROXIMATIONS"

def main():
    doc = ezdxf.new()
    doc.layers.add(ENTITIES, color=colors.YELLOW)
    doc.layers.add(INTERSECTION_POINTS, color=colors.RED)
    doc.layers.add(CURVE_APPROXIMATIONS, color=colors.CYAN)
    msp = doc.modelspace()
    ellipse = msp.add_ellipse(
        center=(0, 0),
        major_axis=(3, 0),
        ratio=0.5,
        dxfattribs=GfxAttribs(layer=ENTITIES),
    )
    fit_points = [(-4, -4), (-2, -1), (2, 1), (4, 4)]
    spline = msp.add_spline_control_frame(
        fit_points, dxfattribs=GfxAttribs(layer=ENTITIES)
    )

    p1 = Vec2.list(ellipse.flattening(distance=0.001))
    p2 = Vec2.list(spline.flattening(distance=0.001))
    msp.add_lwpolyline(p1, dxfattribs=GfxAttribs(layer=CURVE_APPROXIMATIONS))
    msp.add_lwpolyline(p2, dxfattribs=GfxAttribs(layer=CURVE_APPROXIMATIONS))
    res = intersect_polylines_2d(p1, p2)
    for point in res:
        msp.add_circle(
            center=point,
            radius=0.1,
            dxfattribs=GfxAttribs(layer=INTERSECTION_POINTS),
        )
    doc.set_modelspace_vport(height=10)
    doc.saveas(DIR / "intersect_ellipse_and_spline.dxf")

if __name__ == "__main__":
    main()
<commit_msg>refactor example intersect_ellipse_and_spline.py<commit_after>#  Copyright (c) 2022, Manfred Moitzi
#  License: MIT License
import pathlib
import ezdxf
from ezdxf import colors
from ezdxf.gfxattribs import GfxAttribs
from ezdxf.math import intersect_polylines_2d, Vec2

CWD = pathlib.Path("~/Desktop/Outbox").expanduser()
if not CWD.exists():
    CWD = pathlib.Path(".")

# ------------------------------------------------------------------------------
# This example shows how to calculate intersection points of arbitrary 2d
# polylines, an ellipse and a spline in this example.
# ------------------------------------------------------------------------------

ENTITIES = "ENTITIES"
INTERSECTION_POINTS = "INTERSECTION_POINTS"
CURVE_APPROXIMATIONS = "CURVE_APPROXIMATIONS"

def main():
    doc = ezdxf.new()
    doc.layers.add(ENTITIES, color=colors.YELLOW)
    doc.layers.add(INTERSECTION_POINTS, color=colors.RED)
    doc.layers.add(CURVE_APPROXIMATIONS, color=colors.CYAN)
    msp = doc.modelspace()
    ellipse = msp.add_ellipse(
        center=(0, 0),
        major_axis=(3, 0),
        ratio=0.5,
        dxfattribs=GfxAttribs(layer=ENTITIES),
    )
    fit_points = [(-4, -4), (-2, -1), (2, 1), (4, 4)]
    spline = msp.add_spline_control_frame(
        fit_points, dxfattribs=GfxAttribs(layer=ENTITIES)
    )
    # create 2D polylines from the DXF entities:
    p1 = Vec2.list(ellipse.flattening(distance=0.001))
    p2 = Vec2.list(spline.flattening(distance=0.001))

    # add the 2D polylines as reference graphics:
    msp.add_lwpolyline(p1, dxfattribs=GfxAttribs(layer=CURVE_APPROXIMATIONS))
    msp.add_lwpolyline(p2, dxfattribs=GfxAttribs(layer=CURVE_APPROXIMATIONS))

    # the intersection calculation between these polylines:
    res = intersect_polylines_2d(p1, p2)

    # add intersection points as CIRCLE entities:
    for point in res:
        msp.add_circle(
            center=point,
            radius=0.1,
            dxfattribs=GfxAttribs(layer=INTERSECTION_POINTS),
        )
    doc.set_modelspace_vport(height=10)
    doc.saveas(CWD / "intersect_ellipse_and_spline.dxf")

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>r'''
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import subprocess
import re

def get_interface_addresses():
    """Get all ip addresses assigned to interfaces.

    Returns a tuple of (v4 addresses, v6 addresses)
    """
    try:
        output = subprocess.check_output("ifconfig")
    except subprocess.CalledProcessError:
        # Couldn't call ifconfig. Best guess it.
        return (["127.0.0.1"], [])
    # Parse out the results.
    v4 = re.findall("inet addr:([^ ]*)", output)
    v6 = re.findall("inet6 addr: ([^ ]*)", output)
    return v4, v6
<commit_msg>Fix local interface addr parsing<commit_after>r'''
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import subprocess
import re

def get_interface_addresses():
    """Get all ip addresses assigned to interfaces.

    Returns a tuple of (v4 addresses, v6 addresses)
    """
    try:
        output = subprocess.check_output("ifconfig")
    except subprocess.CalledProcessError:
        # Couldn't call ifconfig. Best guess it.
        return (["127.0.0.1"], [])
    # Parse out the results.
    v4 = re.findall("inet (addr:)?([^ ]*)", output)
    v6 = re.findall("inet6 (addr: )?([^ ]*)", output)
    v4 = [e[1] for e in v4]
    v6 = [e[1] for e in v6]
    return v4, v6
<|endoftext|>


<commit_before># -*- coding: iso-8859-1 -*-
#
# Copyright (C) 2004-2005 Edgewall Software
# Copyright (C) 2004-2005 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.com/license.html.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://projects.edgewall.com/trac/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

from trac.util import escape, Markup

from difflib import SequenceMatcher
import re

__all__ = ['get_diff_options', 'hdf_diff', 'unified_diff']

def _get_change_extent(str1, str2):
    """
    Determines the extent of differences between two strings. Returns a tuple
    containing the offset at which the changes start, and the negative offset
    at which the changes end. If the two strings have neither a common prefix
    nor a common suffix, (0, 0) is returned.
    """
    start = 0
    limit = min(len(str1), len(str2))
    while start < limit and str1[start] == str2[start]:
        start += 1
    end = -1
    limit = limit - start
    while -end <= limit and str1[end] == str2[end]:
        end -= 1
    return (start, end + 1)

def _get_opcodes(fromlines, tolines, ignore_blank_lines=False,
                 ignore_case=False, ignore_space_changes=False):
    """
    Generator built on top of SequenceMatcher.get_opcodes().

    This function detects line changes that should be ignored and emits them
    as tagged as 'equal', possibly joined with the preceding and/or following
    'equal' block.
    """

    def is_ignorable(tag, fromlines, tolines):
        if tag == 'delete' and ignore_blank_lines:
            if ''.join(fromlines) == '':
                return True
        elif tag == 'insert' and ignore_blank_lines:
            if ''.join(tolines) == '':
                return True
        elif tag == 'replace' and (ignore_case or ignore_space_changes):
            if len(fromlines) != len(tolines):
                return False
            def f(str):
                if ignore_case:
                    str = str.lower()
                if ignore_space_changes:
                    str = ' '.join(str.split())
                return str
            for i in range(len(fromlines)):
                if f(fromlines[i]) != f(tolines[i]):
                    return False
            return True

    matcher = SequenceMatcher(None, fromlines, tolines)
    previous = None
    for tag, i1, i2, j1, j2 in matcher.get_opcodes():
        if tag == 'equal':
            if previous:
                previous = (tag, previous[1], i2, previous[3], j2)
            else:
                previous = (tag, i1, i2, j1, j2)
        else:
            if is_ignorable(tag, fromlines[i1:i2], tolines[j1:j2]):
                if previous:
                    previous = 'equal', previous[1], i2, previous[3], j2
                else:
                    previous = 'equal', i1, i2, j1, j2
                continue
            if previous:
                yield previous
            yield tag, i1, i2, j1, j2
            previous = None

    if previous:
        yield previous

def _group_opcodes(opcodes, n=3):
    """
    Python 2.2 doesn't have SequenceMatcher.get_grouped_opcodes(), so let's
    provide equivalent here. The opcodes parameter can be any iterable or
    sequence.

    This function can also be used to generate full-context diffs by passing
    None for the parameter n.
    """
    # Full context produces all the opcodes
    if n is None:
        yield opcodes
        return

    # Otherwise we leave at most n lines with the tag 'equal' before and after
    # every change
    nn = n + n
    group = []
    for idx, (tag, i1, i2, j1, j2) in enumerate(opcodes):
        if idx == 0 and tag == 'equal': # Fixup leading unchanged block
            i1, j1 = max(i1, i2 - n), max(j1, j2 - n)
        elif tag == 'equal' and i2 - i1 > nn:
            group.append((tag, i1, min(i2, i1 + n), j1, min(j2, j1 + n)))
            yield group
            group = []
            i1, j1 = max(i1, i2 - n), max(j1, j2 - n)
        group.append((tag, i1, i2, j1 ,j2))

    if group and not (len(group) == 1 and group[0][0] == 'equal'):
        if group[-1][0] == 'equal': # Fixup trailing unchanged block
            tag, i1, i2, j1, j2 = group[-1]
            group[-1] = tag, i1, min(i2, i1 + n), j1, min(j2, j1 + n)
        yield group

def hdf_diff(fromlines, tolines, context=None, tabwidth=8,
             ignore_blank_lines=0, ignore_case=0, ignore_space_changes=0):
    """
    Return an array that is adequate for adding the the HDF data set for HTML
    rendering of the differences.
    """

    type_map = {'replace': 'mod', 'delete': 'rem', 'insert': 'add',
                'equal': 'unmod'}

    space_re = re.compile(' ( +)|^ ')
    def htmlify(match):
        div, mod = divmod(len(match.group(0)), 2)
        return div * '  ' + mod * ' '

    def markup_intraline_changes(opcodes):
        for tag, i1, i2, j1, j2 in opcodes:
            if tag == 'replace' and i2 - i1 == j2 - j1:
                for i in range(i2 - i1):
                    fromline, toline = fromlines[i1 + i], tolines[j1 + i]
                    (start, end) = _get_change_extent(fromline, toline)

                    if start == 0 and end < 0:
                        # Change at start of line
                        fromlines[i1 + i] = '\0' + fromline[:end] + '\1' + \
                                            fromline[end:]
                        tolines[j1 + i] = '\0' + toline[:end] + '\1' + \
                                          toline[end:]
                    elif start > 0 and end == 0:
                        # Change at end of line
                        fromlines[i1 + i] = fromline[:start] + '\0' + \
                                            fromline[start:] + '\1'
                        tolines[j1 + i] = toline[:start] + '\0' + \
                                          toline[start:] + '\1'
                    elif start > 0 and end < 0:
                        # Change somewhere in the middle
                        fromlines[i1 + i] = fromline[:start] + '\0' + \
                                            fromline[start:end] + '\1' + \
                                            fromline[end:]
                        tolines[j1 + i] = toline[:start] + '\0' + \
                                          toline[start:end] + '\1' + \
                                          toline[end:]
            yield tag, i1, i2, j1, j2

    changes = []
    opcodes = _get_opcodes(fromlines, tolines, ignore_blank_lines, ignore_case,
                           ignore_space_changes)
    for group in _group_opcodes(opcodes, context):
        blocks = []
        last_tag = None
        for tag, i1, i2, j1, j2 in markup_intraline_changes(group):
            if tag != last_tag:
                blocks.append({'type': type_map[tag], 'base.offset': i1,
                               'base.lines': [], 'changed.offset': j1,
                               'changed.lines': []})
            if tag == 'equal':
                for line in fromlines[i1:i2]:
                    line = line.expandtabs(tabwidth)
                    line = space_re.sub(htmlify, escape(line, quotes=False))
                    blocks[-1]['base.lines'].append(Markup(line))
                for line in tolines[j1:j2]:
                    line = line.expandtabs(tabwidth)
                    line = space_re.sub(htmlify, escape(line, quotes=False))
                    blocks[-1]['changed.lines'].append(Markup(line))
            else:
                if tag in ('replace', 'delete'):
                    for line in fromlines[i1:i2]:
                        line = line.expandtabs(tabwidth)
                        line = escape(line, quotes=False).replace('\0', '') \
                                                         .replace('\1', '')
                        blocks[-1]['base.lines'].append(Markup(space_re.sub(htmlify,
                                                                            line)))
                if tag in ('replace', 'insert'):
                    for line in tolines[j1:j2]:
                        line = line.expandtabs(tabwidth)
                        line = escape(line, quotes=False).replace('\0', '') \
                                                         .replace('\1', '')
                        blocks[-1]['changed.lines'].append(Markup(space_re.sub(htmlify,
                                                                               line)))
        changes.append(blocks)
    return changes

def unified_diff(fromlines, tolines, context=None, ignore_blank_lines=0,
                 ignore_case=0, ignore_space_changes=0):
    opcodes = _get_opcodes(fromlines, tolines, ignore_blank_lines, ignore_case,
                           ignore_space_changes)
    for group in _group_opcodes(opcodes, context):
        i1, i2, j1, j2 = group[0][1], group[-1][2], group[0][3], group[-1][4]
        yield '@@ -%d,%d +%d,%d @@' % (i1 + 1, i2 - i1, j1 + 1, j2 - j1)
        for tag, i1, i2, j1, j2 in group:
            if tag == 'equal':
                for line in fromlines[i1:i2]:
                    yield ' ' + line
            else:
                if tag in ('replace', 'delete'):
                    for line in fromlines[i1:i2]:
                        yield '-' + line
                if tag in ('replace', 'insert'):
                    for line in tolines[j1:j2]:
                        yield '+' + line

def get_diff_options(req):

    def get_bool_option(name, default=0):
        pref = int(req.session.get('diff_' + name, default))
        arg = int(req.args.has_key(name))
        if req.args.has_key('update') and arg != pref:
            req.session['diff_' + name] = arg
        else:
            arg = pref
        return arg

    pref = req.session.get('diff_style', 'inline')
    style = req.args.get('style', pref)
    if req.args.has_key('update') and style != pref:
        req.session['diff_style'] = style
    req.hdf['diff.style'] = style

    pref = int(req.session.get('diff_contextlines', 2))
    try:
        arg = int(req.args.get('contextlines', pref))
    except ValueError:
        arg = -1
    if req.args.has_key('update') and arg != pref:
        req.session['diff_contextlines'] = arg
    options = ['-U%d' % arg]
    if arg >= 0:
        req.hdf['diff.options.contextlines'] = arg
    else:
        req.hdf['diff.options.contextlines'] = 'all'

    arg = get_bool_option('ignoreblanklines')
    if arg:
        options.append('-B')
    req.hdf['diff.options.ignoreblanklines'] = arg

    arg = get_bool_option('ignorecase')
    if arg:
        options.append('-i')
    req.hdf['diff.options.ignorecase'] = arg

    arg = get_bool_option('ignorewhitespace')
    if arg:
        options.append('-b')
    req.hdf['diff.options.ignorewhitespace'] = arg

    return (style, options)
<commit_msg>Update copyright in `diff.py`.<commit_after># -*- coding: iso-8859-1 -*-
#
# Copyright (C) 2004-2006 Edgewall Software
# Copyright (C) 2004-2006 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.com/license.html.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://projects.edgewall.com/trac/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

from trac.util import escape, Markup

from difflib import SequenceMatcher
import re

__all__ = ['get_diff_options', 'hdf_diff', 'unified_diff']

def _get_change_extent(str1, str2):
    """
    Determines the extent of differences between two strings. Returns a tuple
    containing the offset at which the changes start, and the negative offset
    at which the changes end. If the two strings have neither a common prefix
    nor a common suffix, (0, 0) is returned.
    """
    start = 0
    limit = min(len(str1), len(str2))
    while start < limit and str1[start] == str2[start]:
        start += 1
    end = -1
    limit = limit - start
    while -end <= limit and str1[end] == str2[end]:
        end -= 1
    return (start, end + 1)

def _get_opcodes(fromlines, tolines, ignore_blank_lines=False,
                 ignore_case=False, ignore_space_changes=False):
    """
    Generator built on top of SequenceMatcher.get_opcodes().

    This function detects line changes that should be ignored and emits them
    as tagged as 'equal', possibly joined with the preceding and/or following
    'equal' block.
    """

    def is_ignorable(tag, fromlines, tolines):
        if tag == 'delete' and ignore_blank_lines:
            if ''.join(fromlines) == '':
                return True
        elif tag == 'insert' and ignore_blank_lines:
            if ''.join(tolines) == '':
                return True
        elif tag == 'replace' and (ignore_case or ignore_space_changes):
            if len(fromlines) != len(tolines):
                return False
            def f(str):
                if ignore_case:
                    str = str.lower()
                if ignore_space_changes:
                    str = ' '.join(str.split())
                return str
            for i in range(len(fromlines)):
                if f(fromlines[i]) != f(tolines[i]):
                    return False
            return True

    matcher = SequenceMatcher(None, fromlines, tolines)
    previous = None
    for tag, i1, i2, j1, j2 in matcher.get_opcodes():
        if tag == 'equal':
            if previous:
                previous = (tag, previous[1], i2, previous[3], j2)
            else:
                previous = (tag, i1, i2, j1, j2)
        else:
            if is_ignorable(tag, fromlines[i1:i2], tolines[j1:j2]):
                if previous:
                    previous = 'equal', previous[1], i2, previous[3], j2
                else:
                    previous = 'equal', i1, i2, j1, j2
                continue
            if previous:
                yield previous
            yield tag, i1, i2, j1, j2
            previous = None

    if previous:
        yield previous

def _group_opcodes(opcodes, n=3):
    """
    Python 2.2 doesn't have SequenceMatcher.get_grouped_opcodes(), so let's
    provide equivalent here. The opcodes parameter can be any iterable or
    sequence.

    This function can also be used to generate full-context diffs by passing
    None for the parameter n.
    """
    # Full context produces all the opcodes
    if n is None:
        yield opcodes
        return

    # Otherwise we leave at most n lines with the tag 'equal' before and after
    # every change
    nn = n + n
    group = []
    for idx, (tag, i1, i2, j1, j2) in enumerate(opcodes):
        if idx == 0 and tag == 'equal': # Fixup leading unchanged block
            i1, j1 = max(i1, i2 - n), max(j1, j2 - n)
        elif tag == 'equal' and i2 - i1 > nn:
            group.append((tag, i1, min(i2, i1 + n), j1, min(j2, j1 + n)))
            yield group
            group = []
            i1, j1 = max(i1, i2 - n), max(j1, j2 - n)
        group.append((tag, i1, i2, j1 ,j2))

    if group and not (len(group) == 1 and group[0][0] == 'equal'):
        if group[-1][0] == 'equal': # Fixup trailing unchanged block
            tag, i1, i2, j1, j2 = group[-1]
            group[-1] = tag, i1, min(i2, i1 + n), j1, min(j2, j1 + n)
        yield group

def hdf_diff(fromlines, tolines, context=None, tabwidth=8,
             ignore_blank_lines=0, ignore_case=0, ignore_space_changes=0):
    """
    Return an array that is adequate for adding the the HDF data set for HTML
    rendering of the differences.
    """

    type_map = {'replace': 'mod', 'delete': 'rem', 'insert': 'add',
                'equal': 'unmod'}

    space_re = re.compile(' ( +)|^ ')
    def htmlify(match):
        div, mod = divmod(len(match.group(0)), 2)
        return div * '  ' + mod * ' '

    def markup_intraline_changes(opcodes):
        for tag, i1, i2, j1, j2 in opcodes:
            if tag == 'replace' and i2 - i1 == j2 - j1:
                for i in range(i2 - i1):
                    fromline, toline = fromlines[i1 + i], tolines[j1 + i]
                    (start, end) = _get_change_extent(fromline, toline)

                    if start == 0 and end < 0:
                        # Change at start of line
                        fromlines[i1 + i] = '\0' + fromline[:end] + '\1' + \
                                            fromline[end:]
                        tolines[j1 + i] = '\0' + toline[:end] + '\1' + \
                                          toline[end:]
                    elif start > 0 and end == 0:
                        # Change at end of line
                        fromlines[i1 + i] = fromline[:start] + '\0' + \
                                            fromline[start:] + '\1'
                        tolines[j1 + i] = toline[:start] + '\0' + \
                                          toline[start:] + '\1'
                    elif start > 0 and end < 0:
                        # Change somewhere in the middle
                        fromlines[i1 + i] = fromline[:start] + '\0' + \
                                            fromline[start:end] + '\1' + \
                                            fromline[end:]
                        tolines[j1 + i] = toline[:start] + '\0' + \
                                          toline[start:end] + '\1' + \
                                          toline[end:]
            yield tag, i1, i2, j1, j2

    changes = []
    opcodes = _get_opcodes(fromlines, tolines, ignore_blank_lines, ignore_case,
                           ignore_space_changes)
    for group in _group_opcodes(opcodes, context):
        blocks = []
        last_tag = None
        for tag, i1, i2, j1, j2 in markup_intraline_changes(group):
            if tag != last_tag:
                blocks.append({'type': type_map[tag], 'base.offset': i1,
                               'base.lines': [], 'changed.offset': j1,
                               'changed.lines': []})
            if tag == 'equal':
                for line in fromlines[i1:i2]:
                    line = line.expandtabs(tabwidth)
                    line = space_re.sub(htmlify, escape(line, quotes=False))
                    blocks[-1]['base.lines'].append(Markup(line))
                for line in tolines[j1:j2]:
                    line = line.expandtabs(tabwidth)
                    line = space_re.sub(htmlify, escape(line, quotes=False))
                    blocks[-1]['changed.lines'].append(Markup(line))
            else:
                if tag in ('replace', 'delete'):
                    for line in fromlines[i1:i2]:
                        line = line.expandtabs(tabwidth)
                        line = escape(line, quotes=False).replace('\0', '') \
                                                         .replace('\1', '')
                        blocks[-1]['base.lines'].append(Markup(space_re.sub(htmlify,
                                                                            line)))
                if tag in ('replace', 'insert'):
                    for line in tolines[j1:j2]:
                        line = line.expandtabs(tabwidth)
                        line = escape(line, quotes=False).replace('\0', '') \
                                                         .replace('\1', '')
                        blocks[-1]['changed.lines'].append(Markup(space_re.sub(htmlify,
                                                                               line)))
        changes.append(blocks)
    return changes

def unified_diff(fromlines, tolines, context=None, ignore_blank_lines=0,
                 ignore_case=0, ignore_space_changes=0):
    opcodes = _get_opcodes(fromlines, tolines, ignore_blank_lines, ignore_case,
                           ignore_space_changes)
    for group in _group_opcodes(opcodes, context):
        i1, i2, j1, j2 = group[0][1], group[-1][2], group[0][3], group[-1][4]
        yield '@@ -%d,%d +%d,%d @@' % (i1 + 1, i2 - i1, j1 + 1, j2 - j1)
        for tag, i1, i2, j1, j2 in group:
            if tag == 'equal':
                for line in fromlines[i1:i2]:
                    yield ' ' + line
            else:
                if tag in ('replace', 'delete'):
                    for line in fromlines[i1:i2]:
                        yield '-' + line
                if tag in ('replace', 'insert'):
                    for line in tolines[j1:j2]:
                        yield '+' + line

def get_diff_options(req):

    def get_bool_option(name, default=0):
        pref = int(req.session.get('diff_' + name, default))
        arg = int(req.args.has_key(name))
        if req.args.has_key('update') and arg != pref:
            req.session['diff_' + name] = arg
        else:
            arg = pref
        return arg

    pref = req.session.get('diff_style', 'inline')
    style = req.args.get('style', pref)
    if req.args.has_key('update') and style != pref:
        req.session['diff_style'] = style
    req.hdf['diff.style'] = style

    pref = int(req.session.get('diff_contextlines', 2))
    try:
        arg = int(req.args.get('contextlines', pref))
    except ValueError:
        arg = -1
    if req.args.has_key('update') and arg != pref:
        req.session['diff_contextlines'] = arg
    options = ['-U%d' % arg]
    if arg >= 0:
        req.hdf['diff.options.contextlines'] = arg
    else:
        req.hdf['diff.options.contextlines'] = 'all'

    arg = get_bool_option('ignoreblanklines')
    if arg:
        options.append('-B')
    req.hdf['diff.options.ignoreblanklines'] = arg

    arg = get_bool_option('ignorecase')
    if arg:
        options.append('-i')
    req.hdf['diff.options.ignorecase'] = arg

    arg = get_bool_option('ignorewhitespace')
    if arg:
        options.append('-b')
    req.hdf['diff.options.ignorewhitespace'] = arg

    return (style, options)
<|endoftext|>


<commit_before>from django.core.management.base import BaseCommand, CommandError
from fossevents.events.models import Event
from datetime import datetime
from django.utils import timezone

class Command(BaseCommand):

	def handle(self, *args, **options):
		print(' Creating events')
		self.events = self.create_events()
                print(' events created')

	def create_events(self):
		for count in range(0,20):
			Event.objects.create(**{
			'name': "name %d" % count,
			'description': "%d description" % count,
			'start_date': timezone.now(),
			'end_date': timezone.now(),
			'homepage': "www.google.com",
			'is_published': True,
			'auth_token':"12345",
			'owner_email':"kushagra43@gmail.com",
			})
		return
<commit_msg>bug fixes 1<commit_after>from django.core.management.base import BaseCommand, CommandError
from fossevents.events.modelsimport Event
from django.utils import timezone

class Command(BaseCommand):

    def handle(self, * args, * * options):
    print(' Creating events')
    self.events = self.create_events()
    print(' events created')

def create_events(self):
    for count in range(0, 20):
    Event.objects.create( * * {
        'name': "name %d" % count,
        'description': "%d description" % count,
        'start_date': timezone.now(),
        'end_date': timezone.now(),
        'homepage': "www.google.com",
        'is_published': True,
        'auth_token': "12345",
        'owner_email': "kushagra43@gmail.com",
    })
    return
<|endoftext|>


<commit_before><commit_msg>aptpkg will specify --install-recommends if enabled by the SLS<commit_after><|endoftext|>


<commit_before>import os
import glob
import pandas as pd
from datetime import date
from tempfile import mkdtemp
from shutil import rmtree
from unittest import main, TestCase

from serenata_toolbox.chamber_of_deputies.dataset import Dataset

class TestChamberOfDeputiesDataset(TestCase):

    def setUp(self):
        self.path = mkdtemp(prefix='serenata-')
        print(self.path)
        self.subject = Dataset(self.path)
        self.years = [year for year in range(2009, date.today().year + 1)]

    def tearDown(self):
        rmtree(self.path, ignore_errors=True)

    def test_fetch_translate_clean_integration(self):
        self.subject.fetch()
        files = ["Ano-{}.csv".format(n) for n in self.years]
        files.append('datasets-format.html')

        for name in files:
            file_path = os.path.join(self.path, name)
            assert(os.path.exists(file_path))

        self.subject.translate()
        for name in ["reimbursements-{}.xz".format(n) for n in self.years]:
            file_path = os.path.join(self.path, name)
            assert(os.path.exists(file_path))

        self.subject.clean()
        file_path = os.path.join(self.path, 'reimbursements.xz')
        assert(os.path.exists(file_path))

        # test for subquota translation
        dataset = pd.read_csv(file_path, compression='xz')
        all_subquotas = ['Maintenance of office supporting parliamentary activity',
                     'Locomotion, meal and lodging',
                     'Fuels and lubricants',
                     'Consultancy, research and technical work',
                     'Publicity of parliamentary activity',
                     'Purchase of office supplies',
                     'Software purchase or renting; Postal services; Subscriptions',
                     'Security service provided by specialized company',
                     'Flight tickets',
                     'Telecommunication',
                     'Postal services',
                     'Publication subscriptions',
                     'Congressperson meal',
                     'Lodging, except for congressperson from Distrito Federal',
                     'Automotive vehicle renting or watercraft charter',
                     'Aircraft renting or charter of aircraft',
                     'Automotive vehicle renting or charter',
                     'Watercraft renting or charter',
                     'Taxi, toll and parking',
                     'Terrestrial, maritime and fluvial tickets',
                     'Participation in course, talk or similar event',
                     'Flight ticket issue']

        present_subquotas = pd.unique(dataset['subquota_description'])
        for subquota in present_subquotas:
            assert(subquota in all_subquotas)

if __name__ == '__main__':
    main()
<commit_msg>Skip a journey that takes too long to run<commit_after>import os
import glob
import pandas as pd
from datetime import date
from tempfile import mkdtemp
from shutil import rmtree
from unittest import main, TestCase, skip

from serenata_toolbox.chamber_of_deputies.dataset import Dataset

class TestChamberOfDeputiesDataset(TestCase):

    def setUp(self):
        self.path = mkdtemp(prefix='serenata-')
        print(self.path)
        self.subject = Dataset(self.path)
        self.years = [year for year in range(2009, date.today().year + 1)]

    def tearDown(self):
        rmtree(self.path, ignore_errors=True)

    @skip('Takes to long to run')
    def test_fetch_translate_clean_integration(self):
        self.subject.fetch()
        files = ["Ano-{}.csv".format(n) for n in self.years]
        files.append('datasets-format.html')

        for name in files:
            file_path = os.path.join(self.path, name)
            assert(os.path.exists(file_path))

        self.subject.translate()
        for name in ["reimbursements-{}.xz".format(n) for n in self.years]:
            file_path = os.path.join(self.path, name)
            assert(os.path.exists(file_path))

        self.subject.clean()
        file_path = os.path.join(self.path, 'reimbursements.xz')
        assert(os.path.exists(file_path))

        # test for subquota translation
        dataset = pd.read_csv(file_path, compression='xz')
        all_subquotas = ['Maintenance of office supporting parliamentary activity',
                     'Locomotion, meal and lodging',
                     'Fuels and lubricants',
                     'Consultancy, research and technical work',
                     'Publicity of parliamentary activity',
                     'Purchase of office supplies',
                     'Software purchase or renting; Postal services; Subscriptions',
                     'Security service provided by specialized company',
                     'Flight tickets',
                     'Telecommunication',
                     'Postal services',
                     'Publication subscriptions',
                     'Congressperson meal',
                     'Lodging, except for congressperson from Distrito Federal',
                     'Automotive vehicle renting or watercraft charter',
                     'Aircraft renting or charter of aircraft',
                     'Automotive vehicle renting or charter',
                     'Watercraft renting or charter',
                     'Taxi, toll and parking',
                     'Terrestrial, maritime and fluvial tickets',
                     'Participation in course, talk or similar event',
                     'Flight ticket issue']

        present_subquotas = pd.unique(dataset['subquota_description'])
        for subquota in present_subquotas:
            assert(subquota in all_subquotas)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>__author__ = "Sepehr Hamzehlouy"
__copyright__ = "Copyright 2016, Sepehr Hamzehlouy"
__version__ = "1.1.7"
__maintainer__ = "Sepehr Hamzehlouy"
__email__ = "s.hamzelooy@gmail.com"
__status__ = "Development"

from demonhunter.nodes.master import Master
from demonhunter.dh import DemonHunter, get_current_hunter
from demonhunter.nodes.honeypots import Agent<commit_msg>move to version 2<commit_after>__author__ = "Sepehr Hamzehlouy"
__copyright__ = "Copyright 2016, Sepehr Hamzehlouy"
__version__ = "2.0.0"
__maintainer__ = "Sepehr Hamzehlouy"
__email__ = "s.hamzelooy@gmail.com"
__status__ = "Development"

from demonhunter.nodes.master import Master
from demonhunter.dh import DemonHunter, get_current_hunter
from demonhunter.nodes.honeypots import Agent<|endoftext|>


<commit_before># This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
#
from os import path
from machina import MACHINA_MAIN_TEMPLATE_DIR
from .paths import PROJECT_DIR

DEBUG = True

ADMINS = [
    # ('Your Name', 'your_email@domain.com'),
]

MANAGERS = ADMINS

CONTACT_EMAIL = 'fundacja@nowoczesnapolska.org.pl'

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',  # 'postgresql_psycopg2'
        'NAME': path.join(PROJECT_DIR, 'dev.db'),
        'USER': '',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
    }
}

SOLR_TEST = "http://localhost:8983/solr/wl_test/"
SOLR_STOPWORDS = "/path/to/solr/data/conf/lang/stopwords_pl.txt"

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'Europe/Warsaw'
USE_TZ = True

SITE_ID = 1

TEMPLATES = [{
    'BACKEND': 'django.template.backends.django.DjangoTemplates',
    'DIRS': (
        path.join(PROJECT_DIR, 'templates'),  # Duplicate, because of Machina<1 weird configuration.
        MACHINA_MAIN_TEMPLATE_DIR,
    ),
    'OPTIONS': {
        'loaders': (
            ('django.template.loaders.cached.Loader', (
                'django.template.loaders.filesystem.Loader',
                'django.template.loaders.app_directories.Loader',
            )),
        ),
        'context_processors': (
            'django.contrib.auth.context_processors.auth',
            'django.contrib.messages.context_processors.messages',
            'django.template.context_processors.debug',
            'django.template.context_processors.i18n',
            'django.template.context_processors.media',
            'django.template.context_processors.request',
            'wolnelektury.context_processors.extra_settings',
            'search.context_processors.search_form',
            'machina.core.context_processors.metadata',
        ),
    },
}]

MIDDLEWARE = [
    'debug_toolbar.middleware.DebugToolbarMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.middleware.cache.UpdateCacheMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.admindocs.middleware.XViewMiddleware',
    'fnp_django_pagination.middleware.PaginationMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'fnpdjango.middleware.SetRemoteAddrFromXRealIP',
    'django.middleware.cache.FetchFromCacheMiddleware',
    'machina.apps.forum_permission.middleware.ForumPermissionMiddleware',
    'experiments.middleware.experiments_middleware',
]

ROOT_URLCONF = 'wolnelektury.urls'

FILE_UPLOAD_PERMISSIONS = 0o640
<commit_msg>Overdue fix.<commit_after># This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
#
from os import path
from machina import MACHINA_MAIN_TEMPLATE_DIR
from .paths import PROJECT_DIR

DEBUG = True

ADMINS = [
    # ('Your Name', 'your_email@domain.com'),
]

MANAGERS = ADMINS

CONTACT_EMAIL = 'fundacja@nowoczesnapolska.org.pl'

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',  # 'postgresql_psycopg2'
        'NAME': path.join(PROJECT_DIR, 'dev.db'),
        'USER': '',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
    }
}

SOLR_TEST = "http://localhost:8983/solr/wl_test/"
SOLR_STOPWORDS = "/path/to/solr/data/conf/lang/stopwords_pl.txt"

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'Europe/Warsaw'
USE_TZ = True

SITE_ID = 1

TEMPLATES = [{
    'BACKEND': 'django.template.backends.django.DjangoTemplates',
    'APP_DIRS': True,
    'DIRS': (
        path.join(PROJECT_DIR, 'templates'),  # Duplicate, because of Machina<1 weird configuration.
        MACHINA_MAIN_TEMPLATE_DIR,
    ),
    'OPTIONS': {
        'context_processors': (
            'django.contrib.auth.context_processors.auth',
            'django.contrib.messages.context_processors.messages',
            'django.template.context_processors.debug',
            'django.template.context_processors.i18n',
            'django.template.context_processors.media',
            'django.template.context_processors.request',
            'wolnelektury.context_processors.extra_settings',
            'search.context_processors.search_form',
            'machina.core.context_processors.metadata',
        ),
    },
}]

MIDDLEWARE = [
    'debug_toolbar.middleware.DebugToolbarMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.middleware.cache.UpdateCacheMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.admindocs.middleware.XViewMiddleware',
    'fnp_django_pagination.middleware.PaginationMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'fnpdjango.middleware.SetRemoteAddrFromXRealIP',
    'django.middleware.cache.FetchFromCacheMiddleware',
    'machina.apps.forum_permission.middleware.ForumPermissionMiddleware',
    'experiments.middleware.experiments_middleware',
]

ROOT_URLCONF = 'wolnelektury.urls'

FILE_UPLOAD_PERMISSIONS = 0o640
<|endoftext|>


<commit_before>import os
import sys
import json
import time
import bottle
import socket
import requests
import threading

import Queue

from socket import gethostname
from bottle import Bottle, response, request

app = Bottle()

class RequestDispatcher:
  def __init__(self):
    self.working = True
    self.requests = Queue.Queue()

  def stop(self):
    print 'Stoping request distpatcher thread...'
    self.working = False

  def send(self, request):
    #s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
      #s.connect((request['master.ip'], int(request['master.scheduler.port'])))
      if 'wombat' in request['destination'] and not '.doc.res.ic.ac.uk' in request['destination']:
        request['destination'] += '.doc.res.ic.ac.uk'
      #s.sendall('migrate,' + request['id'] + ',' + request['destination'])
      print 'Dispached request to move W' + request['id'] + ' from ' + request['source'] + ' to ' + request['destination']
    except:
      print 'Request', request, 'failed!'
      print sys.exc_info()
    finally:
      #s.close()
      pass

  def add(self, request):
    self.requests.put(request)
    print 'Received request to move W' + request['id'] + ' from ' + request['source'] + ' to ' + request['destination']

  def run(self):
    while self.working:
      while not self.requests.empty():
        request = self.requests.get()
        self.send(request)
      time.sleep(1)

class Scheduler:
  def __init__(self):
    self.report = {}
    self.updates = []
    self.working = True

  def sleep(self, seconds):
    while self.working and seconds > 0:
      time.sleep(1)
      seconds -= 1

  def stop(self):
    print 'Stoping scheduler thread...'
    self.working = False

  def reportResources(self, report):
    self.updates.append(report)

  def estimatePotential(self, cpu):
    return max(0, (cpu - 60))

  def schedule(self):
    # Assign some greedy resource consumtion scores
    hosts = []
    for hostName in self.report:
      # This is reserved for scheduler, zookeeper, analytics
      if 'wombat07' in hostName:
        continue

      host = self.report[hostName]
      host['potential'] = self.estimatePotential(host['avg_cpu'])
      host['cpu_score'] = sum(float(x['cpu_percent'] + host['potential']) / len(host['cpu']) for x in host['workers'])
      nonSeepCpu = host['avg_cpu'] - sum(float(x['cpu_percent']) / len(host['cpu']) for x in host['workers'])
      host['cpu_score'] += (0 if nonSeepCpu > 5 else host['potential']) + nonSeepCpu
      hosts.append(host)

    workersToMove = []
    for host in hosts:
      print host['host'], 'avg_cpu:', host['avg_cpu'], 'cpu_score:', host['cpu_score'], map(lambda x: {x['data.port']: x['cpu_percent']}, host['workers'])

      # cpu score represent actual plus extra estimation utilization
      if host['cpu_score'] < 100:
        continue
      worker = max(host['workers'], key=lambda x: x['cpu_percent'])
      worker['cpu_score'] = worker['cpu_percent'] + host['potential']
      worker['source'] = host['host']
      workersToMove.append(worker)

      print 'selected', worker['data.port'] + ':' + str(worker['cpu_percent']), worker['cpu_score']

    hosts = sorted(hosts, key=lambda x: x['cpu_score'], reverse=True)
    workersToMove = sorted(workersToMove, key=lambda x: x['cpu_score'])

    for host in hosts:
      if host['cpu_score'] > 95 or not len(workersToMove):
        break
      worker = workersToMove.pop()
      request = {'id': worker['data.port'],
        'master.ip': worker['master.ip'],
        'master.scheduler.port': worker['master.scheduler.port'],
        'destination': host['host'], 'source': worker['source']}
      dispatcher.add(request)

  def run(self):
    while self.working:
      self.report = (self.updates[-1] if len(self.updates) else [])
      self.schedule()
      self.updates = []
      self.sleep(5)

@app.route('/scheduler/event', method='post')
def scheduler_evet():
  data = json.loads(request.forms.get('data'))
  scheduler.reportResources(data['hosts'])

def computeBusyScore(data):
  return data['avg_cpu']

@app.route('/scheduler/host')
def scheduler_host():
  '''preferredNodes = []
  cInfo = Globals.clusterInfo['hosts']
  if len(Globals.clusterInfo['hosts']) == 0:
    return '*'

  for host in cInfo:
    # If we have at least 3 hosts don't allocate on the current node
    if len(cInfo) > 3 and os.uname()[1] in host:
      continue
    preferredNodes.append([cInfo[host]['score'], host])
  preferredNodes.sort()

  node = preferredNodes[0][1]
  # Assume we will allocate a container already
  cInfo[node]['allocations'] += 1
  cInfo[node]['score'] = computeBusyScore(cInfo[node])
  return node + ('.doc.res.ic.ac.uk' if 'wombat' in node else '')'''
  return 'wombat01.doc.res.ic.ac.uk'

scheduler = Scheduler()
dispatcher = RequestDispatcher()

if __name__ == "__main__":
  t = threading.Thread(target=scheduler.run)
  t.deamon = True
  t.start()
  t = threading.Thread(target=dispatcher.run)
  t.deamon = True
  t.start()

app.run(host=gethostname(), port=7009, reloader=False)
scheduler.stop()
dispatcher.stop()<commit_msg>Make scheduler aware of pending requests.<commit_after>import os
import sys
import json
import time
import bottle
import socket
import requests
import threading

import Queue

from socket import gethostname
from bottle import Bottle, response, request

app = Bottle()

class RequestDispatcher:
  def __init__(self):
    self.working = True
    self.requests = Queue.Queue()
    self.pending = {}
    self.estimate = {}

  def stop(self):
    print 'Stoping request distpatcher thread...'
    self.working = False

  def setEstimation(self, src, dest, value):
    self.estimate[src] = (self.estimate[src] if src in self.estimate else 0) - value
    self.estimate[dest] = (self.estimate[dest] if dest in self.estimate else 0) + value

  def getEstimation(self, host):
    return (self.estimate[host] if host in self.estimate else 0)

  def printRequest(self, request):
    return 'request W' + request['id'] + ' from ' + request['source'] + ' to ' + request['destination']

  def send(self, request):
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
      s.connect((request['master.ip'], int(request['master.scheduler.port'])))
      if 'wombat' in request['destination'] and not '.doc.res.ic.ac.uk' in request['destination']:
        request['destination'] += '.doc.res.ic.ac.uk'
      s.sendall('migrate,' + request['id'] + ',' + request['destination'])
      self.setEstimation(request['source'], request['destination'], request['value'])
      print 'Dispached', self.printRequest(request)
      request['time'] = time.time()
      self.pending[request['id']] = request
    except:
      print 'Request', request, 'failed!'
      print sys.exc_info()
    finally:
      s.close()
      pass

  def add(self, request):
    if request['id'] in self.pending:
      return
    self.requests.put(request)
    print 'Received', self.printRequest(request)

  def checkPending(self, report):
    widToHostMap = {}
    for host in report.values():
      for worker in host['workers']:
        widToHostMap[worker['data.port']] = host['host']

    for wid in self.pending.keys():
      request = self.pending[wid]
      dest = request['destination']
      # If we see the worker running on the destination host, remove the request
      if wid in widToHostMap[dest]:
        timeDelta = int(time.time() - request['time'])
        self.setEstimation(request['source'], request['destination'], -request['value'])
        print 'Completed', self.printRequest(request), 'after', timeDelta, 'seconds'
        del self.pending[wid]

  def run(self):
    while self.working:
      while not self.requests.empty():
        request = self.requests.get()
        self.send(request)
      time.sleep(1)

class Scheduler:
  def __init__(self):
    self.report = {}
    self.updates = []
    self.working = True

  def sleep(self, seconds):
    while self.working and seconds > 0:
      time.sleep(1)
      seconds -= 1

  def stop(self):
    print 'Stoping scheduler thread...'
    self.working = False

  def reportResources(self, report):
    dispatcher.checkPending(report)
    self.updates.append(report)

  def estimatePotential(self, host):
    return max(0, (host['avg_cpu'] + dispatcher.getEstimation(host['host']) / len(host['cpu']) - 60))

  def schedule(self):
    # Assign some resource consumtion scores based on a greedy algorithm
    hosts = []
    for host in self.report.values():
      # This is reserved for scheduler, zookeeper, analytics
      if 'wombat07' in host['host']:
        continue
      host['potential'] = self.estimatePotential(host)
      host['cpu_score'] = sum(float(x['cpu_percent'] + host['potential']) / len(host['cpu']) for x in host['workers'])
      nonSeepCpu = host['avg_cpu'] - sum(float(x['cpu_percent']) / len(host['cpu']) for x in host['workers'])
      host['cpu_score'] += (0 if nonSeepCpu > 5 else host['potential']) + nonSeepCpu
      hosts.append(host)

    workersToMove = []
    for host in hosts:
      print host['host'], 'avg_cpu:', host['avg_cpu'], 'cpu_score:', host['cpu_score'], map(lambda x: {x['data.port']: x['cpu_percent']}, host['workers'])

      # cpu score represent actual plus extra estimation utilization
      if host['cpu_score'] < 100:
        continue
      worker = max(host['workers'], key=lambda x: x['cpu_percent'])
      worker['cpu_score'] = worker['cpu_percent'] + host['potential']
      worker['source'] = host['host']
      workersToMove.append(worker)

      print 'selected', worker['data.port'] + ':' + str(worker['cpu_percent']), worker['cpu_score']

    hosts = sorted(hosts, key=lambda x: x['cpu_score'], reverse=True)
    workersToMove = sorted(workersToMove, key=lambda x: x['cpu_score'])

    for host in hosts:
      if host['cpu_score'] > 95 or not len(workersToMove):
        break
      worker = workersToMove.pop()
      request = {'id': worker['data.port'],
        'master.ip': worker['master.ip'], 'value': worker['cpu_percent'],
        'master.scheduler.port': worker['master.scheduler.port'],
        'destination': host['host'], 'source': worker['source']}
      dispatcher.add(request)

  def run(self):
    while self.working:
      self.report = (self.updates[-1] if len(self.updates) else {})
      self.schedule()
      self.updates = []
      self.sleep(2)

@app.route('/scheduler/event', method='post')
def scheduler_evet():
  data = json.loads(request.forms.get('data'))
  scheduler.reportResources(data['hosts'])

def computeBusyScore(data):
  return data['avg_cpu']

@app.route('/scheduler/host')
def scheduler_host():
  '''preferredNodes = []
  cInfo = Globals.clusterInfo['hosts']
  if len(Globals.clusterInfo['hosts']) == 0:
    return '*'

  for host in cInfo:
    # If we have at least 3 hosts don't allocate on the current node
    if len(cInfo) > 3 and os.uname()[1] in host:
      continue
    preferredNodes.append([cInfo[host]['score'], host])
  preferredNodes.sort()

  node = preferredNodes[0][1]
  # Assume we will allocate a container already
  cInfo[node]['allocations'] += 1
  cInfo[node]['score'] = computeBusyScore(cInfo[node])
  return node + ('.doc.res.ic.ac.uk' if 'wombat' in node else '')'''
  return 'wombat01.doc.res.ic.ac.uk'

scheduler = Scheduler()
dispatcher = RequestDispatcher()

if __name__ == "__main__":
  t = threading.Thread(target=scheduler.run)
  t.deamon = True
  t.start()
  t = threading.Thread(target=dispatcher.run)
  t.deamon = True
  t.start()

app.run(host=gethostname(), port=7009, reloader=False)
scheduler.stop()
dispatcher.stop()<|endoftext|>


<commit_before>from wodpy import wod
import glob, time
import numpy as np
import sys, os, json, data.ds
import util.main as main
import pandas, psycopg2
from multiprocessing import Pool

def run(test, profiles):
  '''
  run <test> on a list of <profiles>, return an array summarizing when exceptions were raised
  '''
  qcResults = []
  verbose = []
  exec('from qctests import ' + test)
  for profile in profiles:
    exec('result = ' + test + '.test(profile)')

    #demand tests returned bools:
    for i in result:
      assert isinstance(i, np.bool_), str(i) + ' in test result list is of type ' + str(type(i))

    qcResults.append(np.any(result))
    verbose.append(result)
  return [qcResults, verbose]

def processFile(fName):
  # run each test on each profile, and record its summary & verbose performance
  testResults  = []
  testVerbose  = []
  trueResults  = []
  trueVerbose  = []
  profileIDs   = []
  firstProfile = True
  currentFile  = ''
  f = None

  # keep a list of only the profiles in this thread
  data.ds.threadProfiles = main.extractProfiles([fName])
  data.ds.threadFile     = fName

  for iprofile, pinfo in enumerate(data.ds.threadProfiles):
    # Load the profile data.
    p, currentFile, f = main.profileData(pinfo, currentFile, f)
    # Check that there are temperature data in the profile, otherwise skip.
    if p.var_index() is None:
      continue
    main.catchFlags(p)
    if np.sum(p.t().mask == False) == 0:
      continue
    # Run each test.
    for itest, test in enumerate(testNames):
      result = run(test, [p])
      if firstProfile:
        testResults.append(result[0])
        testVerbose.append(result[1])
      else:
        testResults[itest].append(result[0][0])
        testVerbose[itest].append(result[1][0])
    firstProfile = False
    # Read the reference result.
    truth = main.referenceResults([p])
    trueResults.append(truth[0][0])
    trueVerbose.append(truth[1][0])
    profileIDs.append(p.uid())
  # testResults[i][j] now contains a flag indicating the exception raised by test i on profile j

  return trueResults, testResults, profileIDs

########################################
# main
########################################

if len(sys.argv)>2:
  # Identify and import tests
  testNames = main.importQC('qctests')
  testNames.sort()
  print('{} quality control checks have been found'.format(len(testNames)))
  testNames = main.checkQCTestRequirements(testNames)
  print('{} quality control checks are able to be run:'.format(len(testNames)))
  for testName in testNames:
    print('  {}'.format(testName))

  # Parallel processing.
  print('\nPlease wait while QC is performed\n')

  def process_row(i):
    '''run all tests on the ith database row'''

    # extract profile
    cur.execute('SELECT * FROM demo LIMIT 1 OFFSET ' + str(i))
    row = cur.fetchall()
    profile = main.mock_wodpy(row[0])

    # data pre-validation
    # ---tbd---

    # run tests
    results = [profile.qcflag()]
    for itest, test in enumerate(testNames):
      if test[0:5] != 'CSIRO':  # testing on CSIRO suite for now
        continue
      result = run(test, [profile])
      query = "UPDATE demo SET " + test.lower() + " = " + str(result[0][0]) + " WHERE uid = " + str(profile.uid()) + ";"
      cur.execute(query)

    print profile.uid()

  # connect to database & determine how many profiles are present
  conn = psycopg2.connect("dbname='root' user='root'")
  cur = conn.cursor()
  cur.execute('SELECT COUNT(*) FROM demo')
  nRows = cur.fetchall()[0][0]

  # launch async processes
  pool = Pool(processes=int(sys.argv[2]))
  for i in range(nRows):  # <-- this is not quite right; examine the list of uids printed in process_row, duplicates found.
    pool.apply_async(process_row, (i,))
  pool.close()
  pool.join()

  conn.commit()

  # -------------------------------------------

  # # Recombine results
  # truth, results, profileIDs = main.combineArrays(parallel_result)

  # # Print summary statistics and write output file.
  # main.printSummary(truth, results, testNames)
  # main.generateCSV(truth, results, testNames, profileIDs, sys.argv[1])
else:
  print 'Please add command line arguments to name your output file and set parallelization:'
  print 'python AutoQC myFile 4'
  print 'will result in output written to results-myFile.csv, and will run the calculation parallelized across 4 cores.'
<commit_msg>fixed loop over postgres rows<commit_after>from wodpy import wod
import glob, time
import numpy as np
import sys, os, json, data.ds
import util.main as main
import pandas, psycopg2
from multiprocessing import Pool

def run(test, profiles):
  '''
  run <test> on a list of <profiles>, return an array summarizing when exceptions were raised
  '''
  qcResults = []
  verbose = []
  exec('from qctests import ' + test)
  for profile in profiles:
    exec('result = ' + test + '.test(profile)')

    #demand tests returned bools:
    for i in result:
      assert isinstance(i, np.bool_), str(i) + ' in test result list is of type ' + str(type(i))

    qcResults.append(np.any(result))
    verbose.append(result)
  return [qcResults, verbose]

def processFile(fName):
  # run each test on each profile, and record its summary & verbose performance
  testResults  = []
  testVerbose  = []
  trueResults  = []
  trueVerbose  = []
  profileIDs   = []
  firstProfile = True
  currentFile  = ''
  f = None

  # keep a list of only the profiles in this thread
  data.ds.threadProfiles = main.extractProfiles([fName])
  data.ds.threadFile     = fName

  for iprofile, pinfo in enumerate(data.ds.threadProfiles):
    # Load the profile data.
    p, currentFile, f = main.profileData(pinfo, currentFile, f)
    # Check that there are temperature data in the profile, otherwise skip.
    if p.var_index() is None:
      continue
    main.catchFlags(p)
    if np.sum(p.t().mask == False) == 0:
      continue
    # Run each test.
    for itest, test in enumerate(testNames):
      result = run(test, [p])
      if firstProfile:
        testResults.append(result[0])
        testVerbose.append(result[1])
      else:
        testResults[itest].append(result[0][0])
        testVerbose[itest].append(result[1][0])
    firstProfile = False
    # Read the reference result.
    truth = main.referenceResults([p])
    trueResults.append(truth[0][0])
    trueVerbose.append(truth[1][0])
    profileIDs.append(p.uid())
  # testResults[i][j] now contains a flag indicating the exception raised by test i on profile j

  return trueResults, testResults, profileIDs

########################################
# main
########################################

if len(sys.argv)>2:
  # Identify and import tests
  testNames = main.importQC('qctests')
  testNames.sort()
  print('{} quality control checks have been found'.format(len(testNames)))
  testNames = main.checkQCTestRequirements(testNames)
  print('{} quality control checks are able to be run:'.format(len(testNames)))
  for testName in testNames:
    print('  {}'.format(testName))

  # Parallel processing.
  print('\nPlease wait while QC is performed\n')

  def process_row(uid):
    '''run all tests on the ith database row'''

    # extract profile
    cur.execute('SELECT * FROM demo WHERE uid = ' + str(uid) )
    row = cur.fetchall()
    profile = main.mock_wodpy(row[0])

    # data pre-validation
    # ---tbd---

    # run tests
    results = [profile.qcflag()]
    for itest, test in enumerate(testNames):
      if test[0:5] != 'CSIRO':  # testing on CSIRO suite for now
        continue
      result = run(test, [profile])
      query = "UPDATE demo SET " + test.lower() + " = " + str(result[0][0]) + " WHERE uid = " + str(profile.uid()) + ";"
      cur.execute(query)

    print profile.uid()

  # connect to database & fetch list of all uids
  conn = psycopg2.connect("dbname='root' user='root'")
  cur = conn.cursor()
  cur.execute('SELECT uid FROM demo')
  uids = cur.fetchall()

  # launch async processes
  pool = Pool(processes=int(sys.argv[2]))
  for i in range(len(uids)):
    pool.apply_async(process_row, (uids[i][0],))
  pool.close()
  pool.join()

  conn.commit()

  # -------------------------------------------

  # # Recombine results
  # truth, results, profileIDs = main.combineArrays(parallel_result)

  # # Print summary statistics and write output file.
  # main.printSummary(truth, results, testNames)
  # main.generateCSV(truth, results, testNames, profileIDs, sys.argv[1])
else:
  print 'Please add command line arguments to name your output file and set parallelization:'
  print 'python AutoQC myFile 4'
  print 'will result in output written to results-myFile.csv, and will run the calculation parallelized across 4 cores.'
<|endoftext|>