<commit_before># Copyright 2012 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# So we can name this module "jenkins" and still load the "jenkins"
# system module
from __future__ import absolute_import

import threading
from webob import Request
from paste import httpserver
from uuid import uuid1
import jenkins
import json
import urllib2  # for extending jenkins lib
import logging
import pprint

from zuul.model import Build

class JenkinsCallback(threading.Thread):
    log = logging.getLogger("zuul.JenkinsCallback")

    def __init__(self, jenkins):
        threading.Thread.__init__(self)
        self.jenkins = jenkins

    def run(self):
        httpserver.serve(self.app, host='0.0.0.0', port='8090')

    def app(self, environ, start_response):
        request = Request(environ)
        if request.path == '/jenkins_endpoint':
            self.jenkins_endpoint(request)
        start_response('200 OK', [('content-type', 'text/html')])
        return ['Zuul good.']

    def jenkins_endpoint(self, request):
        data = json.loads(request.body)
        if data:
            self.log.debug("Received data from Jenkins: \n%s" % (
                    pprint.pformat(data)))
            build = data.get('build')
            if build:
                phase = build.get('phase')
                status = build.get('status')
                url = build.get('full_url')
                number = build.get('number')
                params = build.get('parameters')
                if params:
                    uuid = params.get('UUID')
                    if (status and url and uuid and phase
                        and phase == 'COMPLETED'):
                        self.jenkins.onBuildCompleted(uuid, status, url, number)
                    if (phase and phase == 'STARTED'):
                        self.jenkins.onBuildStarted(uuid, url, number)

STOP_BUILD = 'job/%(name)s/%(number)s/stop'
CANCEL_QUEUE = 'queue/item/%(number)s/cancelQueue'
BUILD_INFO = 'job/%(name)s/%(number)s/api/json?depth=0'

class ExtendedJenkins(jenkins.Jenkins):
    def jenkins_open(self, req):
        '''
        Utility routine for opening an HTTP request to a Jenkins server.
        '''
        try:
            if self.auth:
                req.add_header('Authorization', self.auth)
            return urllib2.urlopen(req).read()
        except urllib2.HTTPError, e:
            print e.msg
            print e.fp.read()
            raise

    def stop_build(self, name, number):
        '''
        Stop a running Jenkins build.

        @param name: Name of Jenkins job
        @type  name: str
        @param number: Jenkins build number for the job
        @type  number: int
        '''
        self.jenkins_open(urllib2.Request(self.server + STOP_BUILD%locals()))

    def cancel_queue(self, number):
        '''
        Cancel a queued build.

        @param number: Jenkins queue number for the build
        @type  number: int
        '''
        # Jenkins returns a 302 from this URL, unless Referer is not set,
        # then you get a 404.
        self.jenkins_open(urllib2.Request(self.server + CANCEL_QUEUE%locals(),
                                          headers={'Referer': self.server}))

    def get_build_info(self, name, number):
        '''
        Get information for a build.

        @param name: Name of Jenkins job
        @type  name: str
        @param number: Jenkins build number for the job
        @type  number: int
        @return: dictionary
        '''
        return json.loads(self.jenkins_open(urllib2.Request(self.server + BUILD_INFO%locals())))

class Jenkins(object):
    log = logging.getLogger("zuul.Jenkins")

    def __init__(self, config, sched):
        self.sched = sched
        self.builds = {}
        server = config.get('jenkins', 'server')
        user = config.get('jenkins', 'user')
        apikey = config.get('jenkins', 'apikey')
        self.jenkins = ExtendedJenkins(server, user, apikey)
        self.callback_thread = JenkinsCallback(self)
        self.callback_thread.start()

    def launch(self, job, change, dependent_changes = []):
        self.log.info("Launch job %s for change %s with dependent changes %s" % (
                job, change, dependent_changes))
        uuid = str(uuid1())
        params = dict(UUID=uuid)
        build = Build(job, uuid)
        self.builds[uuid] = build
        # We can get the started notification on another thread before this is done
        # so we add the build even before we trigger the job on Jenkins.  We should
        # be careful to clean it up if it doesn't actually kick off.
        try:
            self.jenkins.build_job(job.name, parameters=params)
        except:
            self.log.exception("Exception launching build %s for job %s for change %s:" % (
                    build, job, change))
            # Whoops.  Remove that build we added.
            del self.builds[uuid]
            raise
        return build

    def cancel(self, build):
        self.log.info("Cancel build %s for job %s" % (build, build.job))
        if build.number:
            self.log.debug("Build %s has already started" % build)
            self.jenkins.stop_build(build.job.name, build.number)
            self.log.debug("Canceled running build %s" % build)
            return
        else:
            self.log.debug("Build %s has not started yet" % build)

        self.log.debug("Looking for build %s in queue" % build)
        for item in self.jenkins.get_queue_info():
            if not item.has_key('actions'):
                continue
            for action in item['actions']:
                if not action.has_key('parameters'):
                    continue
                parameters = action['parameters']
                for param in parameters:
                    if (param['name'] == 'UUID' and build.uuid == param['value']):
                        self.log.debug("Found queue item %s for build %s" % (
                                item['id'], build))
                        try:
                            self.jenkins.cancel_queue(item['id'])
                            self.log.debug("Canceled queue item %s for build %s" % (
                                    item['id'], build))
                            return
                        except:
                            self.log.exception("Exception canceling queue item %s for build %s" % (
                                    item['id'], build))

        self.log.debug("Still unable to find build %s to cancel" % build)
        if build.number:
            self.log.debug("Build %s has just started" % build)
            self.jenkins.stop_build(build.job.name, build.number)
            self.log.debug("Canceled just running build %s" % build)
        else:
            self.log.error("Build %s has not started but was not found in queue" % build)

    def onBuildCompleted(self, uuid, status, url, number):
        self.log.info("Build %s #%s complete, status %s" % (
                uuid, number, status))
        build = self.builds.get(uuid)
        if build:
            self.log.debug("Found build %s" % build)
            del self.builds[uuid]
            build.result = status
            build.url = url
            build.number = number
            self.sched.onBuildCompleted(build)
        else:
            self.log.error("Unable to find build %s" % uuid)

    def onBuildStarted(self, uuid, url, number):
        self.log.info("Build %s #%s started, url: %s" % (uuid, number, url))
        build = self.builds.get(uuid)
        if build:
            self.log.debug("Found build %s" % build)
            build.url = url
            build.number = number
        else:
            self.log.error("Unable to find build %s" % uuid)
<commit_msg>Change default jenkins port<commit_after># Copyright 2012 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# So we can name this module "jenkins" and still load the "jenkins"
# system module
from __future__ import absolute_import

import threading
from webob import Request
from paste import httpserver
from uuid import uuid1
import jenkins
import json
import urllib2  # for extending jenkins lib
import logging
import pprint

from zuul.model import Build

class JenkinsCallback(threading.Thread):
    log = logging.getLogger("zuul.JenkinsCallback")

    def __init__(self, jenkins):
        threading.Thread.__init__(self)
        self.jenkins = jenkins

    def run(self):
        httpserver.serve(self.app, host='0.0.0.0', port='8001')

    def app(self, environ, start_response):
        request = Request(environ)
        if request.path == '/jenkins_endpoint':
            self.jenkins_endpoint(request)
        start_response('200 OK', [('content-type', 'text/html')])
        return ['Zuul good.']

    def jenkins_endpoint(self, request):
        data = json.loads(request.body)
        if data:
            self.log.debug("Received data from Jenkins: \n%s" % (
                    pprint.pformat(data)))
            build = data.get('build')
            if build:
                phase = build.get('phase')
                status = build.get('status')
                url = build.get('full_url')
                number = build.get('number')
                params = build.get('parameters')
                if params:
                    uuid = params.get('UUID')
                    if (status and url and uuid and phase
                        and phase == 'COMPLETED'):
                        self.jenkins.onBuildCompleted(uuid, status, url, number)
                    if (phase and phase == 'STARTED'):
                        self.jenkins.onBuildStarted(uuid, url, number)

STOP_BUILD = 'job/%(name)s/%(number)s/stop'
CANCEL_QUEUE = 'queue/item/%(number)s/cancelQueue'
BUILD_INFO = 'job/%(name)s/%(number)s/api/json?depth=0'

class ExtendedJenkins(jenkins.Jenkins):
    def jenkins_open(self, req):
        '''
        Utility routine for opening an HTTP request to a Jenkins server.
        '''
        try:
            if self.auth:
                req.add_header('Authorization', self.auth)
            return urllib2.urlopen(req).read()
        except urllib2.HTTPError, e:
            print e.msg
            print e.fp.read()
            raise

    def stop_build(self, name, number):
        '''
        Stop a running Jenkins build.

        @param name: Name of Jenkins job
        @type  name: str
        @param number: Jenkins build number for the job
        @type  number: int
        '''
        self.jenkins_open(urllib2.Request(self.server + STOP_BUILD%locals()))

    def cancel_queue(self, number):
        '''
        Cancel a queued build.

        @param number: Jenkins queue number for the build
        @type  number: int
        '''
        # Jenkins returns a 302 from this URL, unless Referer is not set,
        # then you get a 404.
        self.jenkins_open(urllib2.Request(self.server + CANCEL_QUEUE%locals(),
                                          headers={'Referer': self.server}))

    def get_build_info(self, name, number):
        '''
        Get information for a build.

        @param name: Name of Jenkins job
        @type  name: str
        @param number: Jenkins build number for the job
        @type  number: int
        @return: dictionary
        '''
        return json.loads(self.jenkins_open(urllib2.Request(self.server + BUILD_INFO%locals())))

class Jenkins(object):
    log = logging.getLogger("zuul.Jenkins")

    def __init__(self, config, sched):
        self.sched = sched
        self.builds = {}
        server = config.get('jenkins', 'server')
        user = config.get('jenkins', 'user')
        apikey = config.get('jenkins', 'apikey')
        self.jenkins = ExtendedJenkins(server, user, apikey)
        self.callback_thread = JenkinsCallback(self)
        self.callback_thread.start()

    def launch(self, job, change, dependent_changes = []):
        self.log.info("Launch job %s for change %s with dependent changes %s" % (
                job, change, dependent_changes))
        uuid = str(uuid1())
        params = dict(UUID=uuid)
        build = Build(job, uuid)
        self.builds[uuid] = build
        # We can get the started notification on another thread before this is done
        # so we add the build even before we trigger the job on Jenkins.  We should
        # be careful to clean it up if it doesn't actually kick off.
        try:
            self.jenkins.build_job(job.name, parameters=params)
        except:
            self.log.exception("Exception launching build %s for job %s for change %s:" % (
                    build, job, change))
            # Whoops.  Remove that build we added.
            del self.builds[uuid]
            raise
        return build

    def cancel(self, build):
        self.log.info("Cancel build %s for job %s" % (build, build.job))
        if build.number:
            self.log.debug("Build %s has already started" % build)
            self.jenkins.stop_build(build.job.name, build.number)
            self.log.debug("Canceled running build %s" % build)
            return
        else:
            self.log.debug("Build %s has not started yet" % build)

        self.log.debug("Looking for build %s in queue" % build)
        for item in self.jenkins.get_queue_info():
            if not item.has_key('actions'):
                continue
            for action in item['actions']:
                if not action.has_key('parameters'):
                    continue
                parameters = action['parameters']
                for param in parameters:
                    if (param['name'] == 'UUID' and build.uuid == param['value']):
                        self.log.debug("Found queue item %s for build %s" % (
                                item['id'], build))
                        try:
                            self.jenkins.cancel_queue(item['id'])
                            self.log.debug("Canceled queue item %s for build %s" % (
                                    item['id'], build))
                            return
                        except:
                            self.log.exception("Exception canceling queue item %s for build %s" % (
                                    item['id'], build))

        self.log.debug("Still unable to find build %s to cancel" % build)
        if build.number:
            self.log.debug("Build %s has just started" % build)
            self.jenkins.stop_build(build.job.name, build.number)
            self.log.debug("Canceled just running build %s" % build)
        else:
            self.log.error("Build %s has not started but was not found in queue" % build)

    def onBuildCompleted(self, uuid, status, url, number):
        self.log.info("Build %s #%s complete, status %s" % (
                uuid, number, status))
        build = self.builds.get(uuid)
        if build:
            self.log.debug("Found build %s" % build)
            del self.builds[uuid]
            build.result = status
            build.url = url
            build.number = number
            self.sched.onBuildCompleted(build)
        else:
            self.log.error("Unable to find build %s" % uuid)

    def onBuildStarted(self, uuid, url, number):
        self.log.info("Build %s #%s started, url: %s" % (uuid, number, url))
        build = self.builds.get(uuid)
        if build:
            self.log.debug("Found build %s" % build)
            build.url = url
            build.number = number
        else:
            self.log.error("Unable to find build %s" % uuid)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.11.8 on 2018-01-17 17:12
from __future__ import unicode_literals

from django.conf import settings
import django.core.validators
from django.db import migrations, models
import django.db.models.deletion

class Migration(migrations.Migration):

    dependencies = [
        ('main', '0002_auto_20180117_2354'),
    ]

    operations = [
        migrations.AlterField(
            model_name='extendsuser',
            name='nickname',
            field=models.CharField(error_messages={'unique': '이미 사용중인 닉네임입니다.'}, help_text='필수 항목. 10자 이하. 문자와 숫자만 가능.', max_length=10, null=True, unique=True, validators=[django.core.validators.RegexValidator('^[\\w]+$', '유효한 닉네임을 입력해주세요. 이 항목은® 문자와 숫자만 사용 가능합니다.')], verbose_name='닉네임'),
        ),
        migrations.AlterField(
            model_name='extendsuser',
            name='user',
            field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='extendsuser', to=settings.AUTH_USER_MODEL),
        ),
    ]
<commit_msg>refactor<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.11.8 on 2018-01-17 17:12
from __future__ import unicode_literals

from django.conf import settings
import django.core.validators
from django.db import migrations, models
import django.db.models.deletion

class Migration(migrations.Migration):

    dependencies = [
        ('main', '0002_auto_20180117_2354'),
    ]

    operations = [
        migrations.AlterField(
            model_name='extendsuser',
            name='nickname',
            field=models.CharField(
                error_messages={'unique': '이미 사용중인 닉네임입니다.'}, help_text='필수 항목. 10자 이하. 문자와 숫자만 가능.',
                max_length=10, null=True, unique=True, validators=[django.core.validators.RegexValidator(
                    '^[\\w]+$', '유효한 닉네임을 입력해주세요. 이 항목은® 문자와 숫자만 사용 가능합니다.')],
                verbose_name='닉네임'),
        ),
        migrations.AlterField(
            model_name='extendsuser',
            name='user',
            field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='extendsuser',
                                       to=settings.AUTH_USER_MODEL),
        ),
    ]
<|endoftext|>


<commit_before>import datetime
import pytz
from django.test import SimpleTestCase
from mock import patch

from corehq.apps.domain.models import Domain
from corehq.apps.export.forms import FilterFormESExportDownloadForm

class TestFilterFormESExportDownloadForm(SimpleTestCase):

    @patch('corehq.apps.reports.util.get_first_form_submission_received', lambda x: datetime.datetime(2015, 1, 1))
    def test_get_datespan_filter(self):
        form_data = {'date_range': '2015-06-25 to 2016-02-19'}
        form = FilterFormESExportDownloadForm(Domain(name="foo"), pytz.utc, form_data)
        self.assertTrue(form.is_valid())
        datespan_filter = form._get_datespan_filter()
        self.assertEqual(datespan_filter.lt, datetime.datetime(2016, 2, 20, tzinfo=pytz.utc))
        self.assertEqual(datespan_filter.gte, datetime.datetime(2015, 6, 25, tzinfo=pytz.utc))
        self.assertEqual(datespan_filter.lte, None)
        self.assertEqual(datespan_filter.gt, None)

    def test_get_group_filter(self):
        raise NotImplementedError

    @patch('corehq.apps.reports.util.get_first_form_submission_received', lambda x: datetime.datetime(2015, 1, 1))
    def test_get_user_filter(self):
        form_data = {'type_or_group': 'type', 'group': '', 'user_types': ['mobile', 'unknown']}
        form = FilterFormESExportDownloadForm(Domain(name="foo"), pytz.utc, form_data)
        self.assertTrue(form.is_valid())
        raise NotImplementedError

<commit_msg>Implement simple tests for ES form export filter form<commit_after>import datetime
import pytz
from django.test import SimpleTestCase
from mock import patch

from corehq.apps.domain.models import Domain
from corehq.apps.export.filters import FormSubmittedByFilter
from corehq.apps.export.forms import FilterFormESExportDownloadForm

@patch('corehq.apps.reports.util.get_first_form_submission_received', lambda x: datetime.datetime(2015, 1, 1))
class TestFilterFormESExportDownloadForm(SimpleTestCase):

    def test_get_datespan_filter(self):
        form_data = {'date_range': '2015-06-25 to 2016-02-19'}
        form = FilterFormESExportDownloadForm(Domain(name="foo"), pytz.utc, form_data)
        self.assertTrue(form.is_valid())
        datespan_filter = form._get_datespan_filter()
        self.assertEqual(datespan_filter.lt, datetime.datetime(2016, 2, 20, tzinfo=pytz.utc))
        self.assertEqual(datespan_filter.gte, datetime.datetime(2015, 6, 25, tzinfo=pytz.utc))
        self.assertEqual(datespan_filter.lte, None)
        self.assertEqual(datespan_filter.gt, None)

    def test_get_group_filter(self):
        """
        Confirm that FilterFormESExportDownloadForm._get_group_filter() returns
        a filter with the correct group_id and correct base_filter.
        """
        form_data = {
            'type_or_group': 'group',
            'group': 'some_group_id',
            'date_range': '2015-06-25 to 2016-02-19',
        }
        form = FilterFormESExportDownloadForm(Domain(name="foo"), pytz.utc, form_data)
        self.assertTrue(form.is_valid(), "Form had the following errors: {}".format(form.errors))
        group_filter = form._get_group_filter()
        self.assertEqual(group_filter.group_id, 'some_group_id')
        self.assertEqual(group_filter.base_filter, FormSubmittedByFilter)

    def test_get_user_filter(self):
        """
        Confirm that FilterFormESExportDownloadForm._get_user_filter()
        returns a FormSubmittedByFilter with the correct user ids. This test
        DOES NOT verify that BaseFilterExportDownloadForm._get_filtered_users()
        retrieves the correct user ids from the database.
        """
        form_data = {
            'type_or_group': 'type',
            'user_types': ['mobile', 'unknown'],
            'group': '',
            'date_range': '2015-06-25 to 2016-02-19',
        }
        def mock_users_matching_filter(domain, user_filters):
            return [None, "some_user_id", "some_other_user_id"]

        form = FilterFormESExportDownloadForm(Domain(name="foo"), pytz.utc, form_data)

        self.assertTrue(form.is_valid(), "Form had the following errors: {}".format(form.errors))
        with patch("corehq.apps.export.forms.users_matching_filter", mock_users_matching_filter):
            user_filter = form._get_user_filter()
        self.assertEqual(user_filter.submitted_by, [None, "some_user_id", "some_other_user_id"])

<|endoftext|>


<commit_before># vim: set expandtab sw=4 ts=4:
'''
Interface to Keen IO.

Copyright (C) 2014 Dieter Adriaenssens <ruleant@users.sourceforge.net>

This file is part of buildtimetrend/python-lib
<https://github.com/buildtimetrend/python-lib/>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
'''

import os
from buildtimetrend.tools import get_logger
import copy
import keen
import requests
from keen import scoped_keys
from buildtimetrend.settings import Settings
from buildtimetrend.tools import check_file
from buildtimetrend.tools import check_dict
from buildtimetrend.tools import check_list

TIME_INTERVALS = {
    'week': {'name': 'week', 'timeframe': 'this_7_days'},
    'month': {'name': 'month', 'timeframe': 'this_30_days'},
    'year': {'name': 'year', 'timeframe': 'this_52_weeks'}
}

def keen_has_project_id():
    '''
    Check if Keen.io project ID is set.
    '''
    if "KEEN_PROJECT_ID" in os.environ or keen.project_id is not None:
        return True

    get_logger().warning("Keen.io Project ID is not set")
    return False

def keen_is_writable():
    '''
    Check if login keys for Keen IO API are set, to allow writing.
    '''
    if (keen_has_project_id() and
            "KEEN_WRITE_KEY" in os.environ or keen.write_key is not None):
        return True

    get_logger().warning("Keen.io Write Key is not set")
    return False

def keen_is_readable():
    '''
    Check if login keys for Keen IO API are set, to allow reading.
    '''
    if (keen_has_project_id() and
            "KEEN_READ_KEY" in os.environ or keen.read_key is not None):
        return True

    get_logger().warning("Keen.io Read Key is not set")
    return False

def keen_io_generate_read_key(repo):
    '''
    Create scoped key for reading only the build-stages related data.
    Param repo : github repository slug (fe. buildtimetrend/python-lib)
    '''
    logger = get_logger()

    if "KEEN_MASTER_KEY" in os.environ:
        master_key = os.getenv("KEEN_MASTER_KEY")
        privileges = {
            "filters": [{
                "property_name": "build.repo",
                "operator": "eq",
                "property_value": repo
            }],
            "allowed_operations": ["read"]
        }

        logger.info("Keen.io Read Key is created for %s", repo)
        return scoped_keys.encrypt(master_key, privileges)

    logger.warning("Keen.io Read Key was not created.")
    return None

def log_build_keen(build):
    '''Send build data to keen.io'''
    if keen_is_writable():
        get_logger().info("Sending data to Keen.io")
        keen_add_event("builds", {"build": build.to_dict()})
        keen_add_events("build_stages", build.stages_to_list())

def keen_add_event(event_collection, payload):
    '''
    Wrapper for keen.add_event(), adds project info
    Param event_collection : collection event data is submitted to
    Param payload : data that is submitted
    '''
    # add project info to this event
    payload = add_project_info_dict(payload)

    # submit list of events to Keen.io
    keen.add_event(event_collection, payload)

def keen_add_events(event_collection, payload):
    '''
    Wrapper for keen.add_events(), adds project info to each event
    Param event_collection : collection event data is submitted to
    Param payload : array of events that is submitted
    '''
    # add project info to each event
    payload = add_project_info_list(payload)

    # submit list of events to Keen.io
    keen.add_events({event_collection: payload})

def add_project_info_dict(payload):
    '''
    Adds project info to a dictonary
    Param payload: dictonary payload
    '''
    if not check_dict(payload, "payload"):
        return None

    payload_as_dict = copy.deepcopy(payload)

    payload_as_dict["buildtime_trend"] = Settings().get_project_info()

    # override timestamp, set to finished_at timestamp
    if "build" in payload and "finished_at" in payload["build"]:
        payload_as_dict["keen"] = {
            "timestamp": payload["build"]["finished_at"]["isotimestamp"]
        }

    return payload_as_dict

def add_project_info_list(payload):
    '''
    Adds project info to a list of dictionaries
    Param payload: list of dictionaries
    '''
    if not check_list(payload, "payload"):
        return None

    payload_as_list = []

    # loop over dicts in payload and add project info to each one
    for event_dict in payload:
        payload_as_list.append(add_project_info_dict(event_dict))

    return payload_as_list

def generate_overview_config_file(repo):
    '''
    Generates a config file for the overview HTML file that contains the
    graphs generated by Keen.io
    '''
    logger = get_logger()

    if not ("BUILD_TREND_SAMPLE_CONFIGFILE" in os.environ
            and "BUILD_TREND_CONFIGFILE" in os.environ
            and "KEEN_PROJECT_ID" in os.environ):
        logger.error("Trends overview config file was not created")
        return

    sample_filename = os.getenv("BUILD_TREND_SAMPLE_CONFIGFILE")
    config_file = os.getenv("BUILD_TREND_CONFIGFILE")
    keen_project_id = os.getenv("KEEN_PROJECT_ID")

    # check if sample config file exists
    if not check_file(sample_filename):
        return

    # generate read key
    read_key = keen_io_generate_read_key(repo)

    # list of string pairs to be replaced (search string : new string)
    replacements = {
        'keen_project_id': str(keen_project_id),
        'keen_read_key': str(read_key),
        'project_name': str(repo)
    }

    with open(sample_filename, 'rb') as infile, \
            open(config_file, 'w') as outfile:
        for line in infile:
            for src, target in replacements.iteritems():
                line = line.replace(src, target)
            outfile.write(line)

    logger.info("Created trends overview config file")

def get_avg_buildtime(repo=None, interval=None):
    '''
    Query Keen.io database and retrieve average buildtime

    Parameters :
    - repo : repo name (fe. buildtimetrend/service)
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    '''
    if repo is None or not keen_is_readable():
        return -1

    timeframe = check_time_interval(interval)['timeframe']

    try:
        return keen.average(
            "builds",
            target_property="build.duration",
            timeframe=timeframe,
            filters=[get_repo_filter(repo)]
        )
    except requests.ConnectionError:
        get_logger().error("Connection to Keen.io API failed")
        return -1

def get_total_build_jobs(repo=None, interval=None):
    '''
    Query Keen.io database and retrieve total number of builds

    Parameters :
    - repo : repo name (fe. buildtimetrend/service)
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    '''
    if repo is None or not keen_is_readable():
        return -1

    timeframe = check_time_interval(interval)['timeframe']

    try:
        return keen.count_unique(
            "builds",
            target_property="build.job",
            timeframe=timeframe,
            filters=[get_repo_filter(repo)]
        )
    except requests.ConnectionError:
        get_logger().error("Connection to Keen.io API failed")
        return -1

def get_latest_buildtime(repo=None):
    '''
    Query Keen.io database and retrieve buildtime duration of last build

    Parameters :
    - repo : repo name (fe. buildtimetrend/python-lib)
    '''
    if repo is None or not keen_is_readable():
        return -1

    try:
        result = keen.extraction(
            "builds",
            property_names="build.duration",
            latest=1,
            filters=[get_repo_filter(repo)]
        )
    except requests.ConnectionError:
        get_logger().error("Connection to Keen.io API failed")
        return -1

    if result is not None and len(result) > 0:
        return result[0]['build']['duration']

    return -1

def get_repo_filter(repo=None):
    '''
    Return filter for analysis request
    Parameters
    - repo : repo slug name, fe. buildtimetrend/python-lib
    '''
    if repo is None:
        return None

    return {
        "property_name": "build.repo",
        "operator": "eq",
        "property_value": str(repo)
    }

def check_time_interval(interval=None):
    '''
    Check time interval and returns corresponding parameters

    Parameters :
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    '''
    if type(interval) is str:
        # convert to lowercase
        interval = interval.lower()

        if interval in TIME_INTERVALS:
            return TIME_INTERVALS[interval]

    return TIME_INTERVALS['week']
<commit_msg>get total builds, issue #17<commit_after># vim: set expandtab sw=4 ts=4:
'''
Interface to Keen IO.

Copyright (C) 2014 Dieter Adriaenssens <ruleant@users.sourceforge.net>

This file is part of buildtimetrend/python-lib
<https://github.com/buildtimetrend/python-lib/>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
'''

import os
from buildtimetrend.tools import get_logger
import copy
import keen
import requests
from keen import scoped_keys
from buildtimetrend.settings import Settings
from buildtimetrend.tools import check_file
from buildtimetrend.tools import check_dict
from buildtimetrend.tools import check_list

TIME_INTERVALS = {
    'week': {'name': 'week', 'timeframe': 'this_7_days'},
    'month': {'name': 'month', 'timeframe': 'this_30_days'},
    'year': {'name': 'year', 'timeframe': 'this_52_weeks'}
}

def keen_has_project_id():
    '''
    Check if Keen.io project ID is set.
    '''
    if "KEEN_PROJECT_ID" in os.environ or keen.project_id is not None:
        return True

    get_logger().warning("Keen.io Project ID is not set")
    return False

def keen_is_writable():
    '''
    Check if login keys for Keen IO API are set, to allow writing.
    '''
    if (keen_has_project_id() and
            "KEEN_WRITE_KEY" in os.environ or keen.write_key is not None):
        return True

    get_logger().warning("Keen.io Write Key is not set")
    return False

def keen_is_readable():
    '''
    Check if login keys for Keen IO API are set, to allow reading.
    '''
    if (keen_has_project_id() and
            "KEEN_READ_KEY" in os.environ or keen.read_key is not None):
        return True

    get_logger().warning("Keen.io Read Key is not set")
    return False

def keen_io_generate_read_key(repo):
    '''
    Create scoped key for reading only the build-stages related data.
    Param repo : github repository slug (fe. buildtimetrend/python-lib)
    '''
    logger = get_logger()

    if "KEEN_MASTER_KEY" in os.environ:
        master_key = os.getenv("KEEN_MASTER_KEY")
        privileges = {
            "filters": [{
                "property_name": "build.repo",
                "operator": "eq",
                "property_value": repo
            }],
            "allowed_operations": ["read"]
        }

        logger.info("Keen.io Read Key is created for %s", repo)
        return scoped_keys.encrypt(master_key, privileges)

    logger.warning("Keen.io Read Key was not created.")
    return None

def log_build_keen(build):
    '''Send build data to keen.io'''
    if keen_is_writable():
        get_logger().info("Sending data to Keen.io")
        keen_add_event("builds", {"build": build.to_dict()})
        keen_add_events("build_stages", build.stages_to_list())

def keen_add_event(event_collection, payload):
    '''
    Wrapper for keen.add_event(), adds project info
    Param event_collection : collection event data is submitted to
    Param payload : data that is submitted
    '''
    # add project info to this event
    payload = add_project_info_dict(payload)

    # submit list of events to Keen.io
    keen.add_event(event_collection, payload)

def keen_add_events(event_collection, payload):
    '''
    Wrapper for keen.add_events(), adds project info to each event
    Param event_collection : collection event data is submitted to
    Param payload : array of events that is submitted
    '''
    # add project info to each event
    payload = add_project_info_list(payload)

    # submit list of events to Keen.io
    keen.add_events({event_collection: payload})

def add_project_info_dict(payload):
    '''
    Adds project info to a dictonary
    Param payload: dictonary payload
    '''
    if not check_dict(payload, "payload"):
        return None

    payload_as_dict = copy.deepcopy(payload)

    payload_as_dict["buildtime_trend"] = Settings().get_project_info()

    # override timestamp, set to finished_at timestamp
    if "build" in payload and "finished_at" in payload["build"]:
        payload_as_dict["keen"] = {
            "timestamp": payload["build"]["finished_at"]["isotimestamp"]
        }

    return payload_as_dict

def add_project_info_list(payload):
    '''
    Adds project info to a list of dictionaries
    Param payload: list of dictionaries
    '''
    if not check_list(payload, "payload"):
        return None

    payload_as_list = []

    # loop over dicts in payload and add project info to each one
    for event_dict in payload:
        payload_as_list.append(add_project_info_dict(event_dict))

    return payload_as_list

def generate_overview_config_file(repo):
    '''
    Generates a config file for the overview HTML file that contains the
    graphs generated by Keen.io
    '''
    logger = get_logger()

    if not ("BUILD_TREND_SAMPLE_CONFIGFILE" in os.environ
            and "BUILD_TREND_CONFIGFILE" in os.environ
            and "KEEN_PROJECT_ID" in os.environ):
        logger.error("Trends overview config file was not created")
        return

    sample_filename = os.getenv("BUILD_TREND_SAMPLE_CONFIGFILE")
    config_file = os.getenv("BUILD_TREND_CONFIGFILE")
    keen_project_id = os.getenv("KEEN_PROJECT_ID")

    # check if sample config file exists
    if not check_file(sample_filename):
        return

    # generate read key
    read_key = keen_io_generate_read_key(repo)

    # list of string pairs to be replaced (search string : new string)
    replacements = {
        'keen_project_id': str(keen_project_id),
        'keen_read_key': str(read_key),
        'project_name': str(repo)
    }

    with open(sample_filename, 'rb') as infile, \
            open(config_file, 'w') as outfile:
        for line in infile:
            for src, target in replacements.iteritems():
                line = line.replace(src, target)
            outfile.write(line)

    logger.info("Created trends overview config file")

def get_avg_buildtime(repo=None, interval=None):
    '''
    Query Keen.io database and retrieve average buildtime

    Parameters :
    - repo : repo name (fe. buildtimetrend/service)
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    '''
    if repo is None or not keen_is_readable():
        return -1

    timeframe = check_time_interval(interval)['timeframe']

    try:
        return keen.average(
            "builds",
            target_property="build.duration",
            timeframe=timeframe,
            filters=[get_repo_filter(repo)]
        )
    except requests.ConnectionError:
        get_logger().error("Connection to Keen.io API failed")
        return -1

def get_total_build_jobs(repo=None, interval=None):
    '''
    Query Keen.io database and retrieve total number of build jobs

    Parameters :
    - repo : repo name (fe. buildtimetrend/service)
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    '''
    if repo is None or not keen_is_readable():
        return -1

    timeframe = check_time_interval(interval)['timeframe']

    try:
        return keen.count_unique(
            "builds",
            target_property="build.job",
            timeframe=timeframe,
            filters=[get_repo_filter(repo)]
        )
    except requests.ConnectionError:
        get_logger().error("Connection to Keen.io API failed")
        return -1

def get_total_builds(repo=None, interval=None):
    '''
    Query Keen.io database and retrieve total number of builds

    Parameters :
    - repo : repo name (fe. buildtimetrend/service)
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    '''
    if repo is None or not keen_is_readable():
        return -1

    timeframe = check_time_interval(interval)['timeframe']

    try:
        return keen.count_unique(
            "builds",
            target_property="build.build",
            timeframe=timeframe,
            filters=[get_repo_filter(repo)]
        )
    except requests.ConnectionError:
        get_logger().error("Connection to Keen.io API failed")
        return -1

def get_latest_buildtime(repo=None):
    '''
    Query Keen.io database and retrieve buildtime duration of last build

    Parameters :
    - repo : repo name (fe. buildtimetrend/python-lib)
    '''
    if repo is None or not keen_is_readable():
        return -1

    try:
        result = keen.extraction(
            "builds",
            property_names="build.duration",
            latest=1,
            filters=[get_repo_filter(repo)]
        )
    except requests.ConnectionError:
        get_logger().error("Connection to Keen.io API failed")
        return -1

    if result is not None and len(result) > 0:
        return result[0]['build']['duration']

    return -1

def get_repo_filter(repo=None):
    '''
    Return filter for analysis request
    Parameters
    - repo : repo slug name, fe. buildtimetrend/python-lib
    '''
    if repo is None:
        return None

    return {
        "property_name": "build.repo",
        "operator": "eq",
        "property_value": str(repo)
    }

def check_time_interval(interval=None):
    '''
    Check time interval and returns corresponding parameters

    Parameters :
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    '''
    if type(interval) is str:
        # convert to lowercase
        interval = interval.lower()

        if interval in TIME_INTERVALS:
            return TIME_INTERVALS[interval]

    return TIME_INTERVALS['week']
<|endoftext|>


<commit_before>"""
python-social-auth application, allows OpenId or OAuth user
registration/authentication just adding a few configurations.
"""
version = (0, 2, 0)
extra = '-dev'
__version__ = '.'.join(map(str, version)) + extra
<commit_msg>v0.1.25<commit_after>"""
python-social-auth application, allows OpenId or OAuth user
registration/authentication just adding a few configurations.
"""
version = (0, 1, 25)
extra = ''
__version__ = '.'.join(map(str, version)) + extra
<|endoftext|>


<commit_before># Copyright 2019 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file is autogenerated by copybara, please do not edit.

bazel_version = "4843f29a445761c31761152ab4e7d4cad9c346d2"
bazel_sha256 = "e35883ecdbd06841cfa048e7260827b35ef1cb190b870a2b8d18addbee423fb1"
<commit_msg>Update dependency bazelbuild/bazel to latest version<commit_after># Copyright 2019 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file is autogenerated by copybara, please do not edit.

bazel_version = "843bf6e8f6a4d521280b7ab02ce685dce761702e"
bazel_sha256 = "664ba56a61d12f109d862fb30b520c62ef46cdbd480486041f0c255c73248810"
<|endoftext|>


<commit_before>"""
<Program Name>
  update_software.py

<Started>
  December 1, 2008

<Author>
  Carter Butaud

<Purpose>
  Populates the folder that the software updater checks with
  the latest version of the code.
"""

import sys
import make_base_installers

UPDATE_SITE = "/home/couvb/public_html/updatesite"
DEBUG = False

def update(trunk_location):
  """
  <Purpose>
    Populates the update directory (set by a constant) with the
    program files from the current repository.

  <Arguments>
    trunk_location:
      The location of the repository's trunk directory, used to
      find the program files.

  <Exceptions>
    IOError on bad filepath.

  <Side Effects>
    None.

  <Returns>
    None.
  """
  update_dir = UPDATE_SITE + "/real"
  # If we're in DEBUG mode, don't use the real update
  # directory
  if DEBUG:
      update_dir = UPDATE_SITE + "/test"
  make_base_installers.prepare_initial_files(trunk_location, update_dir)

def main():
    if len(sys.argv) < 2:
        print "usage: python update_software.py trunk/location/ [-d]"
    else:
        if sys.argv[2] == "-d":
            DEBUG = True
        update(sys.argv[1])

if __name__ == "__main__":
    main()
<commit_msg>Fixed debug mode<commit_after>"""
<Program Name>
  update_software.py

<Started>
  December 1, 2008

<Author>
  Carter Butaud

<Purpose>
  Populates the folder that the software updater checks with
  the latest version of the code.
"""

import sys
import make_base_installers

UPDATE_SITE = "/homes/iws/butaud/public_html/updatesite"
DEBUG = False

def update(trunk_location):
  """
  <Purpose>
    Populates the update directory (set by a constant) with the
    program files from the current repository.

  <Arguments>
    trunk_location:
      The location of the repository's trunk directory, used to
      find the program files.

  <Exceptions>
    IOError on bad filepath.

  <Side Effects>
    None.

  <Returns>
    None.
  """
  update_dir = UPDATE_SITE + "/real"
  # If we're in DEBUG mode, don't use the real update
  # directory
  if DEBUG:
    print "Debug mode..."
    update_dir = UPDATE_SITE + "/test"
    make_base_installers.prepare_initial_files(trunk_location, update_dir)

def main():
  global DEBUG
  if len(sys.argv) < 2:
    print "usage: python update_software.py trunk/location/ [-d]"
  else:
    if sys.argv[2] == "-d":
      DEBUG = True
      update(sys.argv[1])

if __name__ == "__main__":
  main()
<|endoftext|>


<commit_before>from abstract import layer, _activate, _dropout
import numpy
import theano
import theano.tensor as T
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams

class rotate_layer (layer):
    """
    This is a rotate layer. This takes a layer and an angle (rotation normalized in [0,1]) as input
    and rotates the batch of images by the specified rotation parameter.

    Args:
        input: An input ``theano.tensor`` variable. Even ``theano.shared`` will work as long as they
               are in the following shape ``mini_batch_size, height, width, channels``
        verbose: similar to the rest of the toolbox.
        input_shape: ``(mini_batch_size, input_size)``
        angle: value from [0,1]
        borrow: ``theano`` borrow, typicall ``True``.
        input_params: Supply params or initializations from a pre-trained system.
    """

    def __init__ (self,
                  input,
                  input_shape,
                  id,
                  angle = None,
                  borrow = True,
                  verbose = 2 ):
        super(rotate_layer,self).__init__(id = id, type = 'rotate', verbose = verbose)
        srng = RandomStreams(rng.randint(1,2147462468), use_cuda=None)
        if verbose >= 3:
            print "... Creating rotate layer"

        if len(input) == 4:
            if verbose >= 3:
                print "... Creating the rotate layer"

            if angle is None:
                angle = srng.uniform(size = input_shape[0], low =limits[0], high =limits[1],
                                                dtype = theano.config.floatX )

            theta = numpy.zeros((input_shape[0],2,3),dtype='float32')
            theta[:,0,0] = numpy.cos(angle[:,0]*180)
            theta[:,0,1] = -numpy.sin(angle[:,0]*180)
            theta[:,1,0] = numpy.sin(angle[:,0]*180)
            theta[:,1,1] = numpy.cos(angle[:,0]*180)
            theta = theta.reshape((input_shape[0], 6))

            self.output = self._transform_affine(theta, input)
            self.output_shape = self.output.shape
            self.angle = angle

    def _transform_affine(self, theta, input):
        num_batch, num_channels, height, width = input.shape
        theta = T.reshape(theta, (-1, 2, 3))

        # grid of (x_t, y_t, 1)
        out_height = T.cast(height, 'int64')
        out_width = T.cast(width, 'int64')
        grid = _meshgrid(out_height, out_width)

        # Transform A x (x_t, y_t, 1)^T -> (x_s, y_s)
        T_g = T.dot(theta, grid)
        x_s = T_g[:, 0]
        y_s = T_g[:, 1]
        x_s_flat = x_s.flatten()
        y_s_flat = y_s.flatten()

        # dimshuffle input to  (bs, height, width, channels)
        input_dim = input.dimshuffle(0, 2, 3, 1)
        input_transformed = _interpolate(
            input_dim, x_s_flat, y_s_flat,
            out_height, out_width)

        output = T.reshape(
            input_transformed, (num_batch, out_height, out_width, num_channels))
        output = output.dimshuffle(0, 3, 1, 2)  # dimshuffle to conv format
        return output

    def _interpolate(self, im, x, y, out_height, out_width):
        # *_f are floats
        num_batch, height, width, channels = im.shape
        height_f = T.cast(height, theano.config.floatX)
        width_f = T.cast(width, theano.config.floatX)

        # clip coordinates to [-1, 1]
        x = T.clip(x, -1, 1)
        y = T.clip(y, -1, 1)

        # scale coordinates from [-1, 1] to [0, width/height - 1]
        x = (x + 1) / 2 * (width_f - 1)
        y = (y + 1) / 2 * (height_f - 1)

        # obtain indices of the 2x2 pixel neighborhood surrounding the coordinates;
        # we need those in floatX for interpolation and in int64 for indexing. for
        # indexing, we need to take care they do not extend past the image.
        x0_f = T.floor(x)
        y0_f = T.floor(y)
        x1_f = x0_f + 1
        y1_f = y0_f + 1
        x0 = T.cast(x0_f, 'int64')
        y0 = T.cast(y0_f, 'int64')
        x1 = T.cast(T.minimum(x1_f, width_f - 1), 'int64')
        y1 = T.cast(T.minimum(y1_f, height_f - 1), 'int64')

        # The input is [num_batch, height, width, channels]. We do the lookup in
        # the flattened input, i.e [num_batch*height*width, channels]. We need
        # to offset all indices to match the flat version
        dim2 = width
        dim1 = width*height
        base = T.repeat(
            T.arange(num_batch, dtype='int64')*dim1, out_height*out_width)
        base_y0 = base + y0*dim2
        base_y1 = base + y1*dim2
        idx_a = base_y0 + x0
        idx_b = base_y1 + x0
        idx_c = base_y0 + x1
        idx_d = base_y1 + x1

        # use indices to lookup pixels for all samples
        im_flat = im.reshape((-1, channels))
        Ia = im_flat[idx_a]
        Ib = im_flat[idx_b]
        Ic = im_flat[idx_c]
        Id = im_flat[idx_d]

        # calculate interpolated values
        wa = ((x1_f-x) * (y1_f-y)).dimshuffle(0, 'x')
        wb = ((x1_f-x) * (y-y0_f)).dimshuffle(0, 'x')
        wc = ((x-x0_f) * (y1_f-y)).dimshuffle(0, 'x')
        wd = ((x-x0_f) * (y-y0_f)).dimshuffle(0, 'x')
        output = T.sum([wa*Ia, wb*Ib, wc*Ic, wd*Id], axis=0)
        return output

    def _linspace(self, start, stop, num):
        # Theano linspace. Behaves similar to np.linspace
        start = T.cast(start, theano.config.floatX)
        stop = T.cast(stop, theano.config.floatX)
        num = T.cast(num, theano.config.floatX)
        step = (stop-start)/(num-1)
        return T.arange(num, dtype=theano.config.floatX)*step+start

    def _meshgrid(self, height, width):
        # This function is the grid generator.
        # It is equivalent to the following numpy code:
        #  x_t, y_t = np.meshgrid(np.linspace(-1, 1, width),
        #                         np.linspace(-1, 1, height))
        #  ones = np.ones(np.prod(x_t.shape))
        #  grid = np.vstack([x_t.flatten(), y_t.flatten(), ones])
        # It is implemented in Theano instead to support symbolic grid sizes.
        # Note: If the image size is known at layer construction time, we could
        # compute the meshgrid offline in numpy instead of doing it dynamically
        # in Theano. However, it hardly affected performance when we tried.
        x_t = T.dot(T.ones((height, 1)),
                    _linspace(-1.0, 1.0, width).dimshuffle('x', 0))
        y_t = T.dot(_linspace(-1.0, 1.0, height).dimshuffle(0, 'x'),
                    T.ones((1, width)))

        x_t_flat = x_t.reshape((1, -1))
        y_t_flat = y_t.reshape((1, -1))
        ones = T.ones_like(x_t_flat)
        grid = T.concatenate([x_t_flat, y_t_flat, ones], axis=0)
        return grid

class dropout_rotate_layer (rotate_layer):
    """
    This class is the typical dropout neural hidden layer and batch normalization layer. Called
    by the ``add_layer`` method in network class.

    Args:
        input: An input ``theano.tensor`` variable. Even ``theano.shared`` will work as long as they
               are in the following shape ``mini_batch_size, height, width, channels``
        verbose: similar to the rest of the toolbox.
        num_neurons: number of neurons in the layer
        input_shape: ``(mini_batch_size, input_size)``
        batch_norm: If provided will be used, default is ``False``.
        rng: typically ``numpy.random``.
        borrow: ``theano`` borrow, typicall ``True``.
        activation: String, takes options that are listed in :mod:`activations` Needed for
                    layers that use activations.
                    Some activations also take support parameters, for instance ``maxout``
                    takes maxout type and size, ``softmax`` takes an option temperature.
                    Refer to the module :mod:`activations` to know more.
        input_params: Supply params or initializations from a pre-trained system.
        dropout_rate: ``0.5`` is the default.

    Notes:
        Use ``dropout_dot_product_layer.output`` and ``dropout_dot_product_layer.output_shape`` from
        this class. ``L1`` and ``L2`` are also public and can also can be used for regularization.
        The class also has in public ``w``, ``b`` and ``alpha``
        which are also a list in ``params``, another property of this class.
    """
    def __init__ (self,
                  input,
                  input_shape,
                  id,
                  dropout_rate = 0.5,
                  angle = None,
                  borrow = True,
                  verbose = 2):

        if verbose >= 3:
            print "... set up the dropout rotate layer"
        if rng is None:
            rng = numpy.random
        super(dropout_rotate_layer, self).__init__ (
                                        input = input,
                                        input_shape = input_shape,
                                        id = id,
                                        borrow = borrow,
                                        verbose = verbose
                                        )
        if not dropout_rate == 0:
            self.output = _dropout(rng = rng,
                                params = self.output,
                                dropout_rate = dropout_rate)
        self.dropout_rate = dropout_rate
        if verbose >=3:
            print "... Dropped out"

if __name__ == '__main__':
    pass  <commit_msg>network.py<commit_after>from abstract import layer, _activate, _dropout
import numpy
import theano
import theano.tensor as T
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams

class rotate_layer (layer):
    """
    This is a rotate layer. This takes a layer and an angle (rotation normalized in [0,1]) as input
    and rotates the batch of images by the specified rotation parameter.

    Args:
        input: An input ``theano.tensor`` variable. Even ``theano.shared`` will work as long as they
               are in the following shape ``mini_batch_size, height, width, channels``
        verbose: similar to the rest of the toolbox.
        input_shape: ``(mini_batch_size, input_size)``
        angle: value from [0,1]
        borrow: ``theano`` borrow, typicall ``True``.
        input_params: Supply params or initializations from a pre-trained system.
    """

    def __init__ (self,
                  input,
                  input_shape,
                  id,
                  angle = None,
                  borrow = True,
                  verbose = 2 ):
        super(rotate_layer,self).__init__(id = id, type = 'rotate', verbose = verbose)
        srng = RandomStreams(rng.randint(1,2147462468), use_cuda=None)
        if verbose >= 3:
            print "... Creating rotate layer"

        if len(input) == 4:
            if verbose >= 3:
                print "... Creating the rotate layer"

            if angle is None:
                angle = srng.uniform(size = input_shape[0], low =limits[0], high =limits[1],
                                                dtype = theano.config.floatX )

            theta = numpy.zeros((input_shape[0],2,3),dtype='float32')
            theta[:,0,0] = numpy.cos(angle[:,0]*180)
            theta[:,0,1] = -numpy.sin(angle[:,0]*180)
            theta[:,1,0] = numpy.sin(angle[:,0]*180)
            theta[:,1,1] = numpy.cos(angle[:,0]*180)
            theta = theta.reshape((input_shape[0], 6))

            self.output = self._transform_affine(theta, input)
            self.output_shape = self.output.shape
            self.angle = angle

    def _transform_affine(self, theta, input):
        num_batch, num_channels, height, width = input.shape
        theta = T.reshape(theta, (-1, 2, 3))

        # grid of (x_t, y_t, 1)
        out_height = T.cast(height, 'int64')
        out_width = T.cast(width, 'int64')
        grid = _meshgrid(out_height, out_width)

        # Transform A x (x_t, y_t, 1)^T -> (x_s, y_s)
        T_g = T.dot(theta, grid)
        x_s = T_g[:, 0]
        y_s = T_g[:, 1]
        x_s_flat = x_s.flatten()
        y_s_flat = y_s.flatten()

        # dimshuffle input to  (bs, height, width, channels)
        input_dim = input.dimshuffle(0, 2, 3, 1)
        input_transformed = _interpolate(
            input_dim, x_s_flat, y_s_flat,
            out_height, out_width)

        output = T.reshape(
            input_transformed, (num_batch, out_height, out_width, num_channels))
        output = output.dimshuffle(0, 3, 1, 2)  # dimshuffle to conv format
        return output

    def _interpolate(self, im, x, y, out_height, out_width):
        # *_f are floats
        num_batch, height, width, channels = im.shape
        height_f = T.cast(height, theano.config.floatX)
        width_f = T.cast(width, theano.config.floatX)

        # clip coordinates to [-1, 1]
        x = T.clip(x, -1, 1)
        y = T.clip(y, -1, 1)

        # scale coordinates from [-1, 1] to [0, width/height - 1]
        x = (x + 1) / 2 * (width_f - 1)
        y = (y + 1) / 2 * (height_f - 1)

        # obtain indices of the 2x2 pixel neighborhood surrounding the coordinates;
        # we need those in floatX for interpolation and in int64 for indexing. for
        # indexing, we need to take care they do not extend past the image.
        x0_f = T.floor(x)
        y0_f = T.floor(y)
        x1_f = x0_f + 1
        y1_f = y0_f + 1
        x0 = T.cast(x0_f, 'int64')
        y0 = T.cast(y0_f, 'int64')
        x1 = T.cast(T.minimum(x1_f, width_f - 1), 'int64')
        y1 = T.cast(T.minimum(y1_f, height_f - 1), 'int64')

        # The input is [num_batch, height, width, channels]. We do the lookup in
        # the flattened input, i.e [num_batch*height*width, channels]. We need
        # to offset all indices to match the flat version
        dim2 = width
        dim1 = width*height
        base = T.repeat(
            T.arange(num_batch, dtype='int64')*dim1, out_height*out_width)
        base_y0 = base + y0*dim2
        base_y1 = base + y1*dim2
        idx_a = base_y0 + x0
        idx_b = base_y1 + x0
        idx_c = base_y0 + x1
        idx_d = base_y1 + x1

        # use indices to lookup pixels for all samples
        im_flat = im.reshape((-1, channels))
        Ia = im_flat[idx_a]
        Ib = im_flat[idx_b]
        Ic = im_flat[idx_c]
        Id = im_flat[idx_d]

        # calculate interpolated values
        wa = ((x1_f-x) * (y1_f-y)).dimshuffle(0, 'x')
        wb = ((x1_f-x) * (y-y0_f)).dimshuffle(0, 'x')
        wc = ((x-x0_f) * (y1_f-y)).dimshuffle(0, 'x')
        wd = ((x-x0_f) * (y-y0_f)).dimshuffle(0, 'x')
        output = T.sum([wa*Ia, wb*Ib, wc*Ic, wd*Id], axis=0)
        return output

    def _linspace(self, start, stop, num):
        # Theano linspace. Behaves similar to np.linspace
        start = T.cast(start, theano.config.floatX)
        stop = T.cast(stop, theano.config.floatX)
        num = T.cast(num, theano.config.floatX)
        step = (stop-start)/(num-1)
        return T.arange(num, dtype=theano.config.floatX)*step+start

    def _meshgrid(self, height, width):
        # This function is the grid generator.
        # It is equivalent to the following numpy code:
        #  x_t, y_t = np.meshgrid(np.linspace(-1, 1, width),
        #                         np.linspace(-1, 1, height))
        #  ones = np.ones(np.prod(x_t.shape))
        #  grid = np.vstack([x_t.flatten(), y_t.flatten(), ones])
        # It is implemented in Theano instead to support symbolic grid sizes.
        # Note: If the image size is known at layer construction time, we could
        # compute the meshgrid offline in numpy instead of doing it dynamically
        # in Theano. However, it hardly affected performance when we tried.
        x_t = T.dot(T.ones((height, 1)),
                    _linspace(-1.0, 1.0, width).dimshuffle('x', 0))
        y_t = T.dot(_linspace(-1.0, 1.0, height).dimshuffle(0, 'x'),
                    T.ones((1, width)))

        x_t_flat = x_t.reshape((1, -1))
        y_t_flat = y_t.reshape((1, -1))
        ones = T.ones_like(x_t_flat)
        grid = T.concatenate([x_t_flat, y_t_flat, ones], axis=0)
        return grid

class dropout_rotate_layer (rotate_layer):
    """
    This class is the typical dropout neural hidden layer and batch normalization layer. Called
    by the ``add_layer`` method in network class.

    Args:
        input: An input ``theano.tensor`` variable. Even ``theano.shared`` will work as long as they
               are in the following shape ``mini_batch_size, height, width, channels``
        verbose: similar to the rest of the toolbox.
        num_neurons: number of neurons in the layer
        input_shape: ``(mini_batch_size, input_size)``
        batch_norm: If provided will be used, default is ``False``.
        rng: typically ``numpy.random``.
        borrow: ``theano`` borrow, typicall ``True``.
        activation: String, takes options that are listed in :mod:`activations` Needed for
                    layers that use activations.
                    Some activations also take support parameters, for instance ``maxout``
                    takes maxout type and size, ``softmax`` takes an option temperature.
                    Refer to the module :mod:`activations` to know more.
        input_params: Supply params or initializations from a pre-trained system.
        dropout_rate: ``0.5`` is the default.

    Notes:
        Use ``dropout_dot_product_layer.output`` and ``dropout_dot_product_layer.output_shape`` from
        this class. ``L1`` and ``L2`` are also public and can also can be used for regularization.
        The class also has in public ``w``, ``b`` and ``alpha``
        which are also a list in ``params``, another property of this class.
    """
    def __init__ (self,
                  input,
                  input_shape,
                  id,
                  rng = None,
                  dropout_rate = 0.5,
                  angle = None,
                  borrow = True,
                  verbose = 2):

        if verbose >= 3:
            print "... set up the dropout rotate layer"
        if rng is None:
            rng = numpy.random
        super(dropout_rotate_layer, self).__init__ (
                                        input = input,
                                        input_shape = input_shape,
                                        id = id,
                                        borrow = borrow,
                                        verbose = verbose
                                        )
        if not dropout_rate == 0:
            self.output = _dropout(rng = rng,
                                params = self.output,
                                dropout_rate = dropout_rate)
        self.dropout_rate = dropout_rate
        if verbose >=3:
            print "... Dropped out"

if __name__ == '__main__':
    pass  <|endoftext|>


<commit_before>from corehq.apps.groups.tests.test_groups import WrapGroupTestMixin
from corehq.apps.locations.models import Location, LocationType, SQLLocation
from corehq.apps.locations.tests.util import make_loc
from corehq.apps.commtrack.helpers import make_supply_point, make_product
from corehq.apps.commtrack.tests.util import bootstrap_location_types
from corehq.apps.users.models import CommCareUser
from django.test import TestCase, SimpleTestCase
from corehq.apps.products.models import SQLProduct
from corehq.apps.domain.shortcuts import create_domain

class LocationProducts(TestCase):
    def setUp(self):
        self.domain = create_domain('locations-test')
        self.domain.save()

        LocationType.objects.get_or_create(
            domain=self.domain.name,
            name='outlet',
        )

        make_product(self.domain.name, 'apple', 'apple')
        make_product(self.domain.name, 'orange', 'orange')
        make_product(self.domain.name, 'banana', 'banana')
        make_product(self.domain.name, 'pear', 'pear')

        couch_loc = make_loc('loc', type='outlet', domain=self.domain.name)
        self.loc = couch_loc.sql_location

    def tearDown(self):
        # domain delete cascades to everything else
        self.domain.delete()
        LocationType.objects.filter(domain=self.domain.name).delete()

    def test_start_state(self):
        self.assertTrue(self.loc.stocks_all_products)
        self.assertEqual(
            set(SQLProduct.objects.filter(domain=self.domain.name)),
            set(self.loc.products),
        )

    def test_specify_products(self):
        products = [
            SQLProduct.objects.get(name='apple'),
            SQLProduct.objects.get(name='orange'),
        ]
        self.loc.products = products
        self.loc.save()
        self.assertFalse(self.loc.stocks_all_products)
        self.assertEqual(
            set(products),
            set(self.loc.products),
        )

    def test_setting_all_products(self):
        # If all products are set for the location,
        # set stocks_all_products to True
        products = [
            SQLProduct.objects.get(name='apple'),
            SQLProduct.objects.get(name='orange'),
            SQLProduct.objects.get(name='banana'),
            SQLProduct.objects.get(name='pear'),
        ]
        self.loc.products = products
        self.loc.save()
        self.assertTrue(self.loc.stocks_all_products)
        self.assertEqual(
            set(SQLProduct.objects.filter(domain=self.domain.name)),
            set(self.loc.products),
        )

class LocationTestBase(TestCase):
    def setUp(self):
        self.domain = create_domain('locations-test')
        self.domain.convert_to_commtrack()
        bootstrap_location_types(self.domain.name)

        self.loc = make_loc('loc', type='outlet', domain=self.domain.name)
        self.sp = make_supply_point(self.domain.name, self.loc)

        self.user = CommCareUser.create(
            self.domain.name,
            'username',
            'password',
            first_name='Bob',
            last_name='Builder',
        )
        self.user.set_location(self.loc)

    def tearDown(self):
        self.user.delete()
        # domain delete cascades to everything else
        self.domain.delete()

class LocationsTest(LocationTestBase):
    def test_storage_types(self):
        # make sure we can go between sql/couch locs
        sql_loc = SQLLocation.objects.get(name=self.loc.name)
        self.assertEqual(
            sql_loc.couch_location._id,
            self.loc.location_id
        )

        self.assertEqual(
            sql_loc.id,
            self.loc.sql_location.id
        )

    def test_location_queries(self):
        test_state1 = make_loc(
            'teststate1',
            type='state',
            parent=self.user.location,
            domain=self.domain.name
        )
        test_state2 = make_loc(
            'teststate2',
            type='state',
            parent=self.user.location,
            domain=self.domain.name
        )
        test_village1 = make_loc(
            'testvillage1',
            type='village',
            parent=test_state1,
            domain=self.domain.name
        )
        test_village1.site_code = 'tv1'
        test_village1.save()
        test_village2 = make_loc(
            'testvillage2',
            type='village',
            parent=test_state2,
            domain=self.domain.name
        )

        def compare(list1, list2):
            self.assertEqual(
                set([l._id for l in list1]),
                set([l._id for l in list2])
            )

        # descendants
        compare(
            [test_state1, test_state2, test_village1, test_village2],
            self.user.location.descendants
        )

        # children
        compare(
            [test_state1, test_state2],
            self.user.location.children
        )

        # siblings
        compare(
            [test_state2],
            test_state1.siblings()
        )

        # parent and parent_id
        self.assertEqual(
            self.user.location.location_id,
            test_state1.parent_id
        )
        self.assertEqual(
            self.user.location.location_id,
            test_state1.parent._id
        )

        # is_root
        self.assertTrue(self.user.location.is_root)
        self.assertFalse(test_state1.is_root)

        # Location.root_locations
        compare(
            [self.user.location],
            Location.root_locations(self.domain.name)
        )

        # Location.filter_by_type
        compare(
            [test_village1, test_village2],
            Location.filter_by_type(self.domain.name, 'village')
        )
        compare(
            [test_village1],
            Location.filter_by_type(self.domain.name, 'village', test_state1)
        )

        # Location.get_in_domain
        test_village2.domain = 'rejected'
        bootstrap_location_types('rejected')
        test_village2.save()
        self.assertEqual(
            Location.get_in_domain(self.domain.name, test_village1._id)._id,
            test_village1._id
        )
        self.assertIsNone(
            Location.get_in_domain(self.domain.name, test_village2._id),
        )
        self.assertIsNone(
            Location.get_in_domain(self.domain.name, 'not-a-real-id'),
        )

        self.assertEqual(
            {loc.location_id for loc in [self.user.location, test_state1, test_state2,
                                 test_village1]},
            set(SQLLocation.objects.filter(domain=self.domain.name).location_ids()),
        )

        # Location.by_site_code
        self.assertEqual(
            test_village1._id,
            Location.by_site_code(self.domain.name, 'tv1')._id
        )
        self.assertIsNone(
            None,
            Location.by_site_code(self.domain.name, 'notreal')
        )

        # Location.by_domain
        compare(
            [self.user.location, test_state1, test_state2, test_village1],
            Location.by_domain(self.domain.name)
        )

class WrapLocationTest(WrapGroupTestMixin, SimpleTestCase):
    document_class = Location
<commit_msg>add dependencies to base test<commit_after>from corehq.apps.groups.tests.test_groups import WrapGroupTestMixin
from corehq.apps.locations.models import Location, LocationType, SQLLocation
from corehq.apps.locations.tests.util import make_loc
from corehq.apps.commtrack.helpers import make_supply_point, make_product
from corehq.apps.commtrack.tests.util import bootstrap_location_types
from corehq.apps.users.models import CommCareUser
from django.test import TestCase, SimpleTestCase
from corehq.apps.products.models import SQLProduct
from corehq.apps.domain.shortcuts import create_domain

class LocationProducts(TestCase):
    def setUp(self):
        self.domain = create_domain('locations-test')
        self.domain.save()

        LocationType.objects.get_or_create(
            domain=self.domain.name,
            name='outlet',
        )

        make_product(self.domain.name, 'apple', 'apple')
        make_product(self.domain.name, 'orange', 'orange')
        make_product(self.domain.name, 'banana', 'banana')
        make_product(self.domain.name, 'pear', 'pear')

        couch_loc = make_loc('loc', type='outlet', domain=self.domain.name)
        self.loc = couch_loc.sql_location

    def tearDown(self):
        # domain delete cascades to everything else
        self.domain.delete()
        LocationType.objects.filter(domain=self.domain.name).delete()

    def test_start_state(self):
        self.assertTrue(self.loc.stocks_all_products)
        self.assertEqual(
            set(SQLProduct.objects.filter(domain=self.domain.name)),
            set(self.loc.products),
        )

    def test_specify_products(self):
        products = [
            SQLProduct.objects.get(name='apple'),
            SQLProduct.objects.get(name='orange'),
        ]
        self.loc.products = products
        self.loc.save()
        self.assertFalse(self.loc.stocks_all_products)
        self.assertEqual(
            set(products),
            set(self.loc.products),
        )

    def test_setting_all_products(self):
        # If all products are set for the location,
        # set stocks_all_products to True
        products = [
            SQLProduct.objects.get(name='apple'),
            SQLProduct.objects.get(name='orange'),
            SQLProduct.objects.get(name='banana'),
            SQLProduct.objects.get(name='pear'),
        ]
        self.loc.products = products
        self.loc.save()
        self.assertTrue(self.loc.stocks_all_products)
        self.assertEqual(
            set(SQLProduct.objects.filter(domain=self.domain.name)),
            set(self.loc.products),
        )

class LocationTestBase(TestCase):
    dependent_apps = [
        'auditcare',
        'casexml.apps.case',
        'casexml.apps.phone',
        'casexml.apps.stock',
        'corehq.apps.accounting',
        'corehq.apps.commtrack',
        'corehq.apps.domain',
        'corehq.apps.dropbox',
        'corehq.apps.fixtures',
        'corehq.apps.hqcase',
        'corehq.apps.products',
        'corehq.apps.reminders',
        'corehq.apps.sms',
        'corehq.apps.smsforms',
        'corehq.apps.tzmigration',
        'corehq.apps.users',
        'corehq.couchapps',
        'couchforms',
        'custom.logistics',
        'custom.ilsgateway',
        'custom.ewsghana',
        'django.contrib.admin',
        'django_digest',
        'django_prbac',
        'tastypie',
        'touchforms.formplayer',
    ]

    def setUp(self):
        self.domain = create_domain('locations-test')
        self.domain.convert_to_commtrack()
        bootstrap_location_types(self.domain.name)

        self.loc = make_loc('loc', type='outlet', domain=self.domain.name)
        self.sp = make_supply_point(self.domain.name, self.loc)

        self.user = CommCareUser.create(
            self.domain.name,
            'username',
            'password',
            first_name='Bob',
            last_name='Builder',
        )
        self.user.set_location(self.loc)

    def tearDown(self):
        self.user.delete()
        # domain delete cascades to everything else
        self.domain.delete()

class LocationsTest(LocationTestBase):
    def test_storage_types(self):
        # make sure we can go between sql/couch locs
        sql_loc = SQLLocation.objects.get(name=self.loc.name)
        self.assertEqual(
            sql_loc.couch_location._id,
            self.loc.location_id
        )

        self.assertEqual(
            sql_loc.id,
            self.loc.sql_location.id
        )

    def test_location_queries(self):
        test_state1 = make_loc(
            'teststate1',
            type='state',
            parent=self.user.location,
            domain=self.domain.name
        )
        test_state2 = make_loc(
            'teststate2',
            type='state',
            parent=self.user.location,
            domain=self.domain.name
        )
        test_village1 = make_loc(
            'testvillage1',
            type='village',
            parent=test_state1,
            domain=self.domain.name
        )
        test_village1.site_code = 'tv1'
        test_village1.save()
        test_village2 = make_loc(
            'testvillage2',
            type='village',
            parent=test_state2,
            domain=self.domain.name
        )

        def compare(list1, list2):
            self.assertEqual(
                set([l._id for l in list1]),
                set([l._id for l in list2])
            )

        # descendants
        compare(
            [test_state1, test_state2, test_village1, test_village2],
            self.user.location.descendants
        )

        # children
        compare(
            [test_state1, test_state2],
            self.user.location.children
        )

        # siblings
        compare(
            [test_state2],
            test_state1.siblings()
        )

        # parent and parent_id
        self.assertEqual(
            self.user.location.location_id,
            test_state1.parent_id
        )
        self.assertEqual(
            self.user.location.location_id,
            test_state1.parent._id
        )

        # is_root
        self.assertTrue(self.user.location.is_root)
        self.assertFalse(test_state1.is_root)

        # Location.root_locations
        compare(
            [self.user.location],
            Location.root_locations(self.domain.name)
        )

        # Location.filter_by_type
        compare(
            [test_village1, test_village2],
            Location.filter_by_type(self.domain.name, 'village')
        )
        compare(
            [test_village1],
            Location.filter_by_type(self.domain.name, 'village', test_state1)
        )

        # Location.get_in_domain
        test_village2.domain = 'rejected'
        bootstrap_location_types('rejected')
        test_village2.save()
        self.assertEqual(
            Location.get_in_domain(self.domain.name, test_village1._id)._id,
            test_village1._id
        )
        self.assertIsNone(
            Location.get_in_domain(self.domain.name, test_village2._id),
        )
        self.assertIsNone(
            Location.get_in_domain(self.domain.name, 'not-a-real-id'),
        )

        self.assertEqual(
            {loc.location_id for loc in [self.user.location, test_state1, test_state2,
                                 test_village1]},
            set(SQLLocation.objects.filter(domain=self.domain.name).location_ids()),
        )

        # Location.by_site_code
        self.assertEqual(
            test_village1._id,
            Location.by_site_code(self.domain.name, 'tv1')._id
        )
        self.assertIsNone(
            None,
            Location.by_site_code(self.domain.name, 'notreal')
        )

        # Location.by_domain
        compare(
            [self.user.location, test_state1, test_state2, test_village1],
            Location.by_domain(self.domain.name)
        )

class WrapLocationTest(WrapGroupTestMixin, SimpleTestCase):
    document_class = Location
<|endoftext|>


<commit_before>'''
vim: set expandtab sw=4 ts=4:

Interface to Travis CI API.

Copyright (C) 2014 Dieter Adriaenssens <ruleant@users.sourceforge.net>

This file is part of buildtime-trend
<https://github.com/ruleant/buildtime-trend/>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
'''

class TravisData(object):
    '''
    Gather data from Travis CI using the API
    '''

    def __init__(self, repo, build_id):
        '''
        Retrieve Travis CI build data using the API.
        Param repo : github repository slug (fe. ruleant/buildtime-trend)
        Param build_id : Travis CI build id (fe. 158)
        '''
        self.build_data = {}
        self.repo = repo
        self.build_id = str(build_id)

    def get_build_data(self):
        '''
        Retrieve Travis CI build data using the API.
        '''
        import urllib2
        import json

        req = urllib2.Request(
            'https://api.travis-ci.org/repos/' + self.repo
            + '/builds?number=' + self.build_id,
            None,
            {
                # get version from Config class
                'user-agent': 'buildtime-trend/0.2-dev',
                'accept': 'application/vnd.travis-ci.2+json'
            }
        )
        opener = urllib2.build_opener()
        result = opener.open(req)

        self.build_data = json.load(result)

    def get_started_at(self):
        '''
        Retrieve timestamp when build was started.
        '''
        if len(self.build_data) > 0:
            return self.build_data['builds'][0]['started_at']
        else:
            return None
<commit_msg>import libs for entire file<commit_after>'''
vim: set expandtab sw=4 ts=4:

Interface to Travis CI API.

Copyright (C) 2014 Dieter Adriaenssens <ruleant@users.sourceforge.net>

This file is part of buildtime-trend
<https://github.com/ruleant/buildtime-trend/>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
'''
import urllib2
import json

class TravisData(object):
    '''
    Gather data from Travis CI using the API
    '''

    def __init__(self, repo, build_id):
        '''
        Retrieve Travis CI build data using the API.
        Param repo : github repository slug (fe. ruleant/buildtime-trend)
        Param build_id : Travis CI build id (fe. 158)
        '''
        self.build_data = {}
        self.repo = repo
        self.build_id = str(build_id)

    def get_build_data(self):
        '''
        Retrieve Travis CI build data using the API.
        '''
        req = urllib2.Request(
            'https://api.travis-ci.org/repos/' + self.repo
            + '/builds?number=' + self.build_id,
            None,
            {
                # get version from Config class
                'user-agent': 'buildtime-trend/0.2-dev',
                'accept': 'application/vnd.travis-ci.2+json'
            }
        )
        opener = urllib2.build_opener()
        result = opener.open(req)

        self.build_data = json.load(result)

    def get_started_at(self):
        '''
        Retrieve timestamp when build was started.
        '''
        if len(self.build_data) > 0:
            return self.build_data['builds'][0]['started_at']
        else:
            return None
<|endoftext|>


<commit_before>import os
import yaml

from utils import slugify

class File(object):

    def __init__(self, path, context):

        self.context = context # site wide config
        self.path = path

        self.filename = os.path.basename(path)
        self.contents, self.config = self.get_contents()
        self.title = self.config['title']

    def get_contents(self):
        """
        Read file contents. Extract yaml config. Return post contents and
        parsed config.
        """
        f = open(self.path, 'r')

        c = ""
        start = False
        end = False
        conf = ""
        for x in f:
            if x.startswith('-'*80):
                if not start:
                    start = True
                else:
                    end = True
                continue

            if start and not end:
                conf += x
            else:
                c += x
        f.close()

        config = yaml.load(conf)
        c = self._process_contents(c)
        return c, config

    def _process_contents(self, text):
        """
        Run contents through a text processor.
        Options:
            - markdown
            - textile
            - restructuredtext
            - html
        """
        p = self.context['text_processor']
        p = p.lower()

        if p == 'markdown':
            from markdown import markdown
            html = markdown(text)
        elif p == 'textile':
            from textile import textile
            html = textile(text)
        elif p == 'restructuredtext':
            from docutils.core import publish_parts
            parts = publish_parts(source=text, writer_name="html4css1")
            html = parts.get('fragment')
        elif p == 'pygmentedmarkdown':
            from pygmentedmarkdown import markdown
            html = markdown(text)
        elif p == 'html':
            html = text
        else:
            raise Exception("Unknown text processor")

        return html

class Post(File):
    """
    Post object. Exposes the following attributes:
        - filename (e.g. 2010-cool-post.md)
        - path (e.g. /home/user/2010-cool-post.md)
        - contents (post content)
        - config (post front config)
        - year, month, date (in the format specified in config.yaml)
        - title
        - url (e.g. /2010/02/cool-post/)
        - categories (list)
        - author (from config.yaml or from front config)

    All attributes are calculated on init.
    """

    def __init__(self, path, context):
        super(Post, self).__init__(path, context)

        self.year = str(self.config['date'].year)
        self.month = self.config['date'].strftime("%m")
        self.date = self.config['date'].strftime(self.context['date_format'])
        self.atom_date = self._get_atom_date(self.config['date'])

        self.slug = slugify(self.config['title'])
        self.url = "%s/%s/%s/" % (self.year, self.month, self.slug,)

        categories = self.config['categories']
        self.categories = []
        for c in categories:
            v = {
                'name': c,
                'slug': slugify(c)
            }
            self.categories.append(v)

        if 'author' not in self.config:
            self.author = self.context['author']
        else:
            self.author = self.config['author']

    def _get_atom_date(self, date):
        d = date.strftime('%Y-%m-%dT%H:%M:%S%z')
        return d[:-2] + ':' + d[-2:]

class Page(File):

    def __init__(self, path, context):
        super(Page, self).__init__(path, context)
<commit_msg>Catch import errors when doing text processing.<commit_after>import os
import yaml

from utils import slugify

class File(object):

    def __init__(self, path, context):

        self.context = context # site wide config
        self.path = path

        self.filename = os.path.basename(path)
        self.contents, self.config = self.get_contents()
        self.title = self.config['title']

    def get_contents(self):
        """
        Read file contents. Extract yaml config. Return post contents and
        parsed config.
        """
        f = open(self.path, 'r')

        c = ""
        start = False
        end = False
        conf = ""
        for x in f:
            if x.startswith('-'*80):
                if not start:
                    start = True
                else:
                    end = True
                continue

            if start and not end:
                conf += x
            else:
                c += x
        f.close()

        config = yaml.load(conf)
        c = self._process_contents(c)
        return c, config

    def _process_contents(self, text):
        """
        Run contents through a text processor.
        Options:
            - markdown
            - textile
            - restructuredtext
            - html
        """
        p = self.context['text_processor']
        p = p.lower()

        try:
            if p == 'markdown':
                from markdown import markdown
                html = markdown(text)
            elif p == 'textile':
                from textile import textile
                html = textile(text)
            elif p == 'restructuredtext':
                from docutils.core import publish_parts
                parts = publish_parts(source=text, writer_name="html4css1")
                html = parts.get('fragment')
            elif p == 'pygmentedmarkdown':
                from pygmentedmarkdown import markdown
                html = markdown(text)
            elif p == 'html':
                html = text
            else:
                raise Exception("Unknown text processor")
        except ImportError:
            raise Exception("Unknown text processor.")

        return html

class Post(File):
    """
    Post object. Exposes the following attributes:
        - filename (e.g. 2010-cool-post.md)
        - path (e.g. /home/user/2010-cool-post.md)
        - contents (post content)
        - config (post front config)
        - year, month, date (in the format specified in config.yaml)
        - title
        - url (e.g. /2010/02/cool-post/)
        - categories (list)
        - author (from config.yaml or from front config)

    All attributes are calculated on init.
    """

    def __init__(self, path, context):
        super(Post, self).__init__(path, context)

        self.year = str(self.config['date'].year)
        self.month = self.config['date'].strftime("%m")
        self.date = self.config['date'].strftime(self.context['date_format'])
        self.atom_date = self._get_atom_date(self.config['date'])

        self.slug = slugify(self.config['title'])
        self.url = "%s/%s/%s/" % (self.year, self.month, self.slug,)

        categories = self.config['categories']
        self.categories = []
        for c in categories:
            v = {
                'name': c,
                'slug': slugify(c)
            }
            self.categories.append(v)

        if 'author' not in self.config:
            self.author = self.context['author']
        else:
            self.author = self.config['author']

    def _get_atom_date(self, date):
        d = date.strftime('%Y-%m-%dT%H:%M:%S%z')
        return d[:-2] + ':' + d[-2:]

class Page(File):

    def __init__(self, path, context):
        super(Page, self).__init__(path, context)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
Extract the opcode from the objdump of a binary
'''

import os
import sys
import getopt
import string
from subprocess import check_output

__version__ = "$Id: bin2op.py,v 1.0 2015/12/27 15:47:42 dhn Exp $"
__license__ = "BSD"

def getopts():
    try:
        usage = ["file=", "short", "large", "intel", "att", "version", "help"]
        opts, args = getopt.getopt(sys.argv[1:], "f:sliavh", usage)

    except getopt.GetoptError as err:
        print(str(err))
        usage()
        sys.exit(2)
    objfile = None
    shellcode = None
    code = None
    syntax = "intel"
    for opt, arg in opts:
        if opt in ("-i", "--intel"):
            syntax = "intel"
        elif opt in ("-a", "--att"):
            syntax = "att"
        elif opt in ("-f", "--file"):
            objfile = arg
            if os.path.exists(objfile):
                shellcode, code = parse(objfile, syntax)
            else:
                print("[!] file does not exist")
                sys.exit(1)
        elif opt in ("-s", "--short"):
            if objfile is not None:
                print(shellcode)
        elif opt in ("-l", "--large"):
            if objfile is not None:
                for line in code:
                    print("", line)
        elif opt in ("-h", "--help"):
            usage()
        elif opt in ("-v", "--version"):
            print("%s - %s" % (__version__, __license__))
        else:
            assert False, "unhandled option"

def usage():
    print("usage: %s [options]" % __file__)
    print("   -f, --file     The assembly code filename")
    print("   -s, --short    Show less version of opcode")
    print("   -l, --large    Show verbose version of opcode")
    print("   -i, --intel    Use the intel assembly syntax")
    print("   -a, --att      Use the AT&T assembly syntax")
    print("")
    print("Example: %s -a -f bindshell/build/bindshell.o -l" % __file__)
    print("         %s -f bindshell/build/bindshell.o -s" % __file__)

# thanks zerosum0x0
def parse(obj, syntax):
    objdump = ['objdump', '-d', '-M', syntax, obj]

    lines = check_output(objdump)
    lines = lines.split(b'Disassembly of section')[1]
    lines = lines.split(b'\n')[3:]

    shellcode = ""
    code = []

    for line in lines:
        line = line.strip()

        tabs = line.split(b'\t')
        if (len(tabs) < 2):
            continue
        bytes = tabs[1].strip()

        instruction = "."
        if (len(tabs) == 3):
            instruction = tabs[2].strip().decode("utf-8")

        bytes = bytes.split(b' ')
        shellcodeline = ""
        for byte in bytes:
            shellcodeline += "\\x" + byte.decode("utf-8")

        shellcode += shellcodeline
        c = '%-*s/* %s */' % (32, '"'+shellcodeline+'"', instruction)
        code.append(c)

    return shellcode, code

if __name__ == "__main__":
    if len(sys.argv) <= 1:
        usage()
    else:
        getopts()
<commit_msg>Insert a newline character every 64 characters (only in the small output)<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
Extract the opcode from the objdump of a binary
'''

import re
import os
import sys
import getopt
import string
from subprocess import check_output

__version__ = "$Id: bin2op.py,v 1.0 2015/12/27 15:47:42 dhn Exp $"
__license__ = "BSD"

def getopts():
    try:
        usage = ["file=", "short", "large", "intel", "att", "version", "help"]
        opts, args = getopt.getopt(sys.argv[1:], "f:sliavh", usage)

    except getopt.GetoptError as err:
        print(str(err))
        usage()
        sys.exit(2)
    objfile = None
    shellcode = None
    code = None
    syntax = "intel"
    for opt, arg in opts:
        if opt in ("-i", "--intel"):
            syntax = "intel"
        elif opt in ("-a", "--att"):
            syntax = "att"
        elif opt in ("-f", "--file"):
            objfile = arg
            if os.path.exists(objfile):
                shellcode, code = parse(objfile, syntax)
            else:
                print("[!] file does not exist")
                sys.exit(1)
        elif opt in ("-s", "--short"):
            if objfile is not None:
                print(re.sub("(.{64})", "\\1\n", shellcode, 0, re.DOTALL))
        elif opt in ("-l", "--large"):
            if objfile is not None:
                for line in code:
                    print("", line)
        elif opt in ("-h", "--help"):
            usage()
        elif opt in ("-v", "--version"):
            print("%s - %s" % (__version__, __license__))
        else:
            assert False, "unhandled option"

def usage():
    print("usage: %s [options]" % __file__)
    print("   -f, --file     The assembly code filename")
    print("   -s, --short    Show less version of opcode")
    print("   -l, --large    Show verbose version of opcode")
    print("   -i, --intel    Use the intel assembly syntax")
    print("   -a, --att      Use the AT&T assembly syntax")
    print("")
    print("Example: %s -a -f bindshell/build/bindshell.o -l" % __file__)
    print("         %s -f bindshell/build/bindshell.o -s" % __file__)

# thanks zerosum0x0
def parse(obj, syntax):
    objdump = ['objdump', '-d', '-M', syntax, obj]

    lines = check_output(objdump)
    lines = lines.split(b'Disassembly of section')[1]
    lines = lines.split(b'\n')[3:]

    shellcode = ""
    code = []

    for line in lines:
        line = line.strip()

        tabs = line.split(b'\t')
        if (len(tabs) < 2):
            continue
        bytes = tabs[1].strip()

        instruction = "."
        if (len(tabs) == 3):
            instruction = tabs[2].strip().decode("utf-8")

        bytes = bytes.split(b' ')
        shellcodeline = ""
        for byte in bytes:
            shellcodeline += "\\x" + byte.decode("utf-8")

        shellcode += shellcodeline
        c = '%-*s/* %s */' % (32, '"'+shellcodeline+'"', instruction)
        code.append(c)

    return shellcode, code

if __name__ == "__main__":
    if len(sys.argv) <= 1:
        usage()
    else:
        getopts()
<|endoftext|>