<commit_before>#!/usr/bin/env python
# coding=utf-8
# Filename: __version__.py
# pylint: disable=C0103
"""
Pep 386 compliant version info.

    (major, minor, micro, alpha/beta/rc/final, #)
    (1, 1, 2, 'alpha', 0) => "1.1.2.dev"
    (1, 2, 0, 'beta', 2) => "1.2b2"

"""
import json
try:
    from urllib import urlopen
except ImportError:
    from urllib.request import urlopen

from km3pipe.logger import logging

__author__ = 'tamasgal'

log = logging.getLogger(__name__)  # pylint: disable=C0103

version_info = (1, 3, 0, 'final', 0)

def _get_version(version_info):
    """Return a PEP 386-compliant version number."""
    assert len(version_info) == 5
    assert version_info[3] in ('alpha', 'beta', 'rc', 'final')

    parts = 2 if version_info[2] == 0 else 3
    main = '.'.join(map(str, version_info[:parts]))

    sub = ''
    if version_info[3] == 'alpha' and version_info[4] == 0:
        sub = '.dev'
    elif version_info[3] != 'final':
        mapping = {'alpha': 'a', 'beta': 'b', 'rc': 'c'}
        sub = mapping[version_info[3]] + str(version_info[4])

    return str(main + sub)

def _get_latest_version():
    response = urlopen('https://pypi.python.org/pypi/km3pipe/json')
    content = response.read()
    latest_version = json.loads(content.decode())['info']['version']
    return str(latest_version)

def check_for_update():
    try:
        latest_version = _get_latest_version()
    except IOError:
        pass
    else:
        version = _get_version(version_info)
        if latest_version != version:
            log.warn("There is an update for km3pipe available.\n" +
                     "    Installed: {0}\n"
                     "    Latest: {1}\n".format(version, latest_version) +
                     "Please run `pip install --upgrade km3pipe` to update.")

version = _get_version(version_info)
check_for_update()
<commit_msg>Changes version<commit_after>#!/usr/bin/env python
# coding=utf-8
# Filename: __version__.py
# pylint: disable=C0103
"""
Pep 386 compliant version info.

    (major, minor, micro, alpha/beta/rc/final, #)
    (1, 1, 2, 'alpha', 0) => "1.1.2.dev"
    (1, 2, 0, 'beta', 2) => "1.2b2"

"""
import json
try:
    from urllib import urlopen
except ImportError:
    from urllib.request import urlopen

from km3pipe.logger import logging

__author__ = 'tamasgal'

log = logging.getLogger(__name__)  # pylint: disable=C0103

version_info = (1, 3, 1, 'final', 0)

def _get_version(version_info):
    """Return a PEP 386-compliant version number."""
    assert len(version_info) == 5
    assert version_info[3] in ('alpha', 'beta', 'rc', 'final')

    parts = 2 if version_info[2] == 0 else 3
    main = '.'.join(map(str, version_info[:parts]))

    sub = ''
    if version_info[3] == 'alpha' and version_info[4] == 0:
        sub = '.dev'
    elif version_info[3] != 'final':
        mapping = {'alpha': 'a', 'beta': 'b', 'rc': 'c'}
        sub = mapping[version_info[3]] + str(version_info[4])

    return str(main + sub)

def _get_latest_version():
    response = urlopen('https://pypi.python.org/pypi/km3pipe/json')
    content = response.read()
    latest_version = json.loads(content.decode())['info']['version']
    return str(latest_version)

def check_for_update():
    try:
        latest_version = _get_latest_version()
    except IOError:
        pass
    else:
        version = _get_version(version_info)
        if latest_version != version:
            log.warn("There is an update for km3pipe available.\n" +
                     "    Installed: {0}\n"
                     "    Latest: {1}\n".format(version, latest_version) +
                     "Please run `pip install --upgrade km3pipe` to update.")

version = _get_version(version_info)
check_for_update()
<|endoftext|>


<commit_before># coding: utf-8
# author: wie@ppi.co.jp
from PySide import QtGui, QtCore

class Plotter(QtGui.QWidget):
    """Plotter and Double buffering examples"""

    def __init__(self, *args, **kwargs):
        super(Plotter, self).__init__(*args, **kwargs)

        self.zoomInButton
        self.zoomOutButton
        self.curveMap
        self.zoomStack
        self.curZoom
        self.rubberBandIsShown
        self.rubberBandRect
        self.pixmap

    def setPlotSettings(self, settings):
        pass

    def setCurveData(self, _id, data):
        pass

    def clearCurve(self, _id):
        pass

    def minimumSizeHint(self, *args, **kwargs):
        size = None
        return size

    def sizeHint(self, *args, **kwargs):
        size = None
        return size

    @QtCore.Slot()
    def zoomIn(self):
        pass

    @QtCore.Slot()
    def zoomOut(self):
        pass

    def paintEvent(self, event):
        pass

    def resizeEvent(self, event):
        pass

    def mousePressEvent(self, event):
        pass

    def mouseMoveEvent(self, event):
        pass

    def mouseReleaseEvent(self, event):
        pass

    def keyPressEvent(self, event):
        pass

    def wheelEvent(self, event):
        pass

    def updateRubberBendRegion(self):
        pass

    def refreshPixmap(self):
        pass

    def drawGrid(self, painter):
        pass

    def drawCurves(self, painter):
        pass

class PlotSettings:

    def __init__(self):
        self.minY = None
        self.maxY = None
        self.minX = None
        self.maxX = None
        self.numXTicks = None
        self.numYTicks = None

    def scroll(self, dx, dy):
        pass

    def adjust(self):
        pass

    def spanX(self):
        return self.maxX - self.minX

    def spanY(self):
        return self.maxY - self.minY

    def adjustAxis(self, _min, _max, numTicks):
        pass

if __name__ == '__main__':
    import sys

    app = QtGui.QApplication(sys.argv)

    win = Plotter()

    win.show()

    sys.exit(app.exec_())<commit_msg>branch commit test<commit_after># coding: utf-8
# author: wie@ppi.co.jp

from PySide import QtGui, QtCore

class Plotter(QtGui.QWidget):
    """Plotter and Double buffering examples"""

    def __init__(self, parent=None):
        super(Plotter, self).__init__(parent)

        self.zoomInButton
        self.zoomOutButton
        self.curveMap
        self.zoomStack
        self.curZoom
        self.rubberBandIsShown
        self.rubberBandRect
        self.pixmap

    def setPlotSettings(self, settings):
        pass

    def setCurveData(self, _id, data):
        pass

    def clearCurve(self, _id):
        pass

    def minimumSizeHint(self, *args, **kwargs):
        size = None
        return size

    def sizeHint(self, *args, **kwargs):
        size = None
        return size

    @QtCore.Slot()
    def zoomIn(self):
        pass

    @QtCore.Slot()
    def zoomOut(self):
        pass

    def paintEvent(self, event):
        pass

    def resizeEvent(self, event):
        pass

    def mousePressEvent(self, event):
        pass

    def mouseMoveEvent(self, event):
        pass

    def mouseReleaseEvent(self, event):
        pass

    def keyPressEvent(self, event):
        pass

    def wheelEvent(self, event):
        pass

    def updateRubberBendRegion(self):
        pass

    def refreshPixmap(self):
        pass

    def drawGrid(self, painter):
        pass

    def drawCurves(self, painter):
        pass

class PlotSettings:

    def __init__(self):
        self.minY = None
        self.maxY = None
        self.minX = None
        self.maxX = None
        self.numXTicks = None
        self.numYTicks = None

    def scroll(self, dx, dy):
        pass

    def adjust(self):
        pass

    def spanX(self):
        return self.maxX - self.minX

    def spanY(self):
        return self.maxY - self.minY

    def adjustAxis(self, _min, _max, numTicks):
        pass

if __name__ == '__main__':
    import sys

    app = QtGui.QApplication(sys.argv)

    win = Plotter()

    win.show()

    sys.exit(app.exec_())<|endoftext|>


<commit_before>from __future__ import absolute_import

import urllib
import urllib2
import base64

from celery.task import task
from celery.utils.log import get_task_logger
log = get_task_logger(__name__)

from django.utils import simplejson

from .conf import settings as mp_settings

@task(name="mixpanel.tasks.PeopleTracker", max_retries=mp_settings.MIXPANEL_MAX_RETRIES)
def people_tracker(distinct_id, properties=None, token=None):
    """
    Track an event occurrence to mixpanel through the API.

    ``event_name`` is the string for the event/category you'd like to log
    this event under
    ``properties`` is (optionally) a dictionary of key/value pairs
    describing the event.
    ``token`` is (optionally) your Mixpanel api token. Not required if
    you've already configured your MIXPANEL_API_TOKEN setting.
    """
    log.info("Recording people datapoint: <%s>" % distinct_id)

    url_params = _build_people_params(distinct_id, properties)

    try:
        result = _send_request(url_params, mp_settings.MIXPANEL_PEOPLE_TRACKING_ENDPOINT)
    except FailedEventRequest as e:
        log.info("Event failed. Retrying: user <%s>" % distinct_id)
        raise people_tracker.retry(exc=e, countdown=mp_settings.MIXPANEL_RETRY_DELAY)
    return result

@task(name="mixpanel.tasks.EventTracker", max_retries=mp_settings.MIXPANEL_MAX_RETRIES)
def event_tracker(event_name, properties=None, token=None):
    """
    Track an event occurrence to mixpanel through the API.

    ``event_name`` is the string for the event/category you'd like to log
    this event under
    ``properties`` is (optionally) a dictionary of key/value pairs
    describing the event.
    ``token`` is (optionally) your Mixpanel api token. Not required if
    you've already configured your MIXPANEL_API_TOKEN setting.
    """
    log.info("Recording event: <%s>" % event_name)

    generated_properties = _handle_properties(properties, token)

    url_params = _build_params(event_name, generated_properties)

    try:
        result = _send_request(url_params)
    except FailedEventRequest as e:
        log.info("Event failed. Retrying: <%s>" % event_name)
        raise event_tracker.retry(exc=e, countdown=mp_settings.MIXPANEL_RETRY_DELAY)
    return result

@task(name="mixpanel.tasks.FunnelEventTracker", max_retries=mp_settings.MIXPANEL_MAX_RETRIES)
def funnel_event_tracker(funnel, step, goal, properties, token=None):
    """
    Track an event occurrence to mixpanel through the API.

    ``funnel`` is the string for the funnel you'd like to log
    this event under
    ``step`` the step in the funnel you're registering
    ``goal`` the end goal of this funnel
    ``properties`` is a dictionary of key/value pairs
    describing the funnel event. A ``distinct_id`` is required.
    ``token`` is (optionally) your Mixpanel api token. Not required if
    you've already configured your MIXPANEL_API_TOKEN setting.
    """
    log.info("Recording funnel: <%s>-<%s>" % (funnel, step))
    properties = _handle_properties(properties, token)

    properties = _add_funnel_properties(properties, funnel, step, goal)

    url_params = _build_params(mp_settings.MIXPANEL_FUNNEL_EVENT_ID, properties)

    try:
        result = _send_request(url_params)
    except FailedEventRequest as e:
        log.info("Funnel failed. Retrying: <%s>-<%s>" % (funnel, step))
        raise funnel_event_tracker.retry(exc=e, countdown=mp_settings.MIXPANEL_RETRY_DELAY)
    return result

class FailedEventRequest(Exception):
    """The attempted recording event failed because of a non-200 HTTP return code"""
    pass

class InvalidFunnelProperties(Exception):
    """Required properties were missing from the funnel-tracking call"""
    pass

def _handle_properties(properties, token):
    """
    Build a properties dictionary, accounting for the token.
    """
    if properties == None:
        properties = {}

    if not properties.get('token', None):
        if token is None:
            token = mp_settings.MIXPANEL_API_TOKEN
        properties['token'] = token

    return properties

def _build_people_params(distinct_id, properties):
    """
    Build HTTP params to record the given event and properties.
    """
    params = {'$distinct_id': distinct_id,'$token': mp_settings.MIXPANEL_API_TOKEN}
    if 'set' in properties:
        #adding $ to any reserved mixpanel vars
        for special_prop in mp_settings.MIXPANEL_RESERVED_PEOPLE_PROPERTIES:
            if special_prop in properties['set']:
                properties['set']['${}'.format(special_prop)] = properties['set'][special_prop]
                del properties['set'][special_prop]
        params['$set'] = properties['set']
    if 'increment' in properties:
        params['$add'] = properties['increment']
    data = base64.b64encode(simplejson.dumps(params))

    data_var = mp_settings.MIXPANEL_DATA_VARIABLE
    url_params = urllib.urlencode({data_var: data})

    return url_params

def _build_params(event, properties):
    """
    Build HTTP params to record the given event and properties.
    """
    params = {'event': event, 'properties': properties}
    data = base64.b64encode(simplejson.dumps(params))

    data_var = mp_settings.MIXPANEL_DATA_VARIABLE
    url_params = urllib.urlencode({data_var: data})

    return url_params

def _send_request(params, endpoint=mp_settings.MIXPANEL_TRACKING_ENDPOINT):
    """
    Send a an event with its properties to the api server.

    Returns ``true`` if the event was logged by Mixpanel.
    """
    url = 'https://%s%s?%s' % (mp_settings.MIXPANEL_API_SERVER, endpoint, params)
    try:
        response = urllib2.urlopen(url, None, mp_settings.MIXPANEL_API_TIMEOUT)
    except urllib2.URLError as e:
        raise FailedEventRequest("Tracking request failed: %s" % e)

    # Successful request gets a single-byte response of "1" from mixpanel
    return response.read() == '1'

def _add_funnel_properties(properties, funnel, step, goal):
    if not 'distinct_id' in properties:
        error_msg = "A ``distinct_id`` must be given to record a funnel event"
        raise InvalidFunnelProperties(error_msg)
    properties['funnel'] = funnel
    properties['step'] = step
    properties['goal'] = goal

    return properties

<commit_msg>Don't hardcode task names; celery gets this right without help.<commit_after>from __future__ import absolute_import

import urllib
import urllib2
import base64

from celery.task import task
from celery.utils.log import get_task_logger
log = get_task_logger(__name__)

from django.utils import simplejson

from .conf import settings as mp_settings

@task(max_retries=mp_settings.MIXPANEL_MAX_RETRIES)
def people_tracker(distinct_id, properties=None, token=None):
    """
    Track an event occurrence to mixpanel through the API.

    ``event_name`` is the string for the event/category you'd like to log
    this event under
    ``properties`` is (optionally) a dictionary of key/value pairs
    describing the event.
    ``token`` is (optionally) your Mixpanel api token. Not required if
    you've already configured your MIXPANEL_API_TOKEN setting.
    """
    log.info("Recording people datapoint: <%s>" % distinct_id)

    url_params = _build_people_params(distinct_id, properties)

    try:
        result = _send_request(url_params, mp_settings.MIXPANEL_PEOPLE_TRACKING_ENDPOINT)
    except FailedEventRequest as e:
        log.info("Event failed. Retrying: user <%s>" % distinct_id)
        raise people_tracker.retry(exc=e, countdown=mp_settings.MIXPANEL_RETRY_DELAY)
    return result

@task(max_retries=mp_settings.MIXPANEL_MAX_RETRIES)
def event_tracker(event_name, properties=None, token=None):
    """
    Track an event occurrence to mixpanel through the API.

    ``event_name`` is the string for the event/category you'd like to log
    this event under
    ``properties`` is (optionally) a dictionary of key/value pairs
    describing the event.
    ``token`` is (optionally) your Mixpanel api token. Not required if
    you've already configured your MIXPANEL_API_TOKEN setting.
    """
    log.info("Recording event: <%s>" % event_name)

    generated_properties = _handle_properties(properties, token)

    url_params = _build_params(event_name, generated_properties)

    try:
        result = _send_request(url_params)
    except FailedEventRequest as e:
        log.info("Event failed. Retrying: <%s>" % event_name)
        raise event_tracker.retry(exc=e, countdown=mp_settings.MIXPANEL_RETRY_DELAY)
    return result

@task(max_retries=mp_settings.MIXPANEL_MAX_RETRIES)
def funnel_event_tracker(funnel, step, goal, properties, token=None):
    """
    Track an event occurrence to mixpanel through the API.

    ``funnel`` is the string for the funnel you'd like to log
    this event under
    ``step`` the step in the funnel you're registering
    ``goal`` the end goal of this funnel
    ``properties`` is a dictionary of key/value pairs
    describing the funnel event. A ``distinct_id`` is required.
    ``token`` is (optionally) your Mixpanel api token. Not required if
    you've already configured your MIXPANEL_API_TOKEN setting.
    """
    log.info("Recording funnel: <%s>-<%s>" % (funnel, step))
    properties = _handle_properties(properties, token)

    properties = _add_funnel_properties(properties, funnel, step, goal)

    url_params = _build_params(mp_settings.MIXPANEL_FUNNEL_EVENT_ID, properties)

    try:
        result = _send_request(url_params)
    except FailedEventRequest as e:
        log.info("Funnel failed. Retrying: <%s>-<%s>" % (funnel, step))
        raise funnel_event_tracker.retry(exc=e, countdown=mp_settings.MIXPANEL_RETRY_DELAY)
    return result

class FailedEventRequest(Exception):
    """The attempted recording event failed because of a non-200 HTTP return code"""
    pass

class InvalidFunnelProperties(Exception):
    """Required properties were missing from the funnel-tracking call"""
    pass

def _handle_properties(properties, token):
    """
    Build a properties dictionary, accounting for the token.
    """
    if properties == None:
        properties = {}

    if not properties.get('token', None):
        if token is None:
            token = mp_settings.MIXPANEL_API_TOKEN
        properties['token'] = token

    return properties

def _build_people_params(distinct_id, properties):
    """
    Build HTTP params to record the given event and properties.
    """
    params = {'$distinct_id': distinct_id,'$token': mp_settings.MIXPANEL_API_TOKEN}
    if 'set' in properties:
        #adding $ to any reserved mixpanel vars
        for special_prop in mp_settings.MIXPANEL_RESERVED_PEOPLE_PROPERTIES:
            if special_prop in properties['set']:
                properties['set']['${}'.format(special_prop)] = properties['set'][special_prop]
                del properties['set'][special_prop]
        params['$set'] = properties['set']
    if 'increment' in properties:
        params['$add'] = properties['increment']
    data = base64.b64encode(simplejson.dumps(params))

    data_var = mp_settings.MIXPANEL_DATA_VARIABLE
    url_params = urllib.urlencode({data_var: data})

    return url_params

def _build_params(event, properties):
    """
    Build HTTP params to record the given event and properties.
    """
    params = {'event': event, 'properties': properties}
    data = base64.b64encode(simplejson.dumps(params))

    data_var = mp_settings.MIXPANEL_DATA_VARIABLE
    url_params = urllib.urlencode({data_var: data})

    return url_params

def _send_request(params, endpoint=mp_settings.MIXPANEL_TRACKING_ENDPOINT):
    """
    Send a an event with its properties to the api server.

    Returns ``true`` if the event was logged by Mixpanel.
    """
    url = 'https://%s%s?%s' % (mp_settings.MIXPANEL_API_SERVER, endpoint, params)
    try:
        response = urllib2.urlopen(url, None, mp_settings.MIXPANEL_API_TIMEOUT)
    except urllib2.URLError as e:
        raise FailedEventRequest("Tracking request failed: %s" % e)

    # Successful request gets a single-byte response of "1" from mixpanel
    return response.read() == '1'

def _add_funnel_properties(properties, funnel, step, goal):
    if not 'distinct_id' in properties:
        error_msg = "A ``distinct_id`` must be given to record a funnel event"
        raise InvalidFunnelProperties(error_msg)
    properties['funnel'] = funnel
    properties['step'] = step
    properties['goal'] = goal

    return properties

<|endoftext|>


<commit_before>import logging
import os
import subprocess
import sys

from vdist import defaults

class BuildMachine(object):

    def __init__(self, machine_logs=True, image=None, insecure_registry=False,
                 docker_cli='docker', docker_args=None):
        self.logger = logging.getLogger('BuildMachine')

        self.machine_logs = machine_logs
        self.image = image

        self.container_id = None

        self.docker_cli = docker_cli

        self.insecure_registry = insecure_registry

    def _run_cli(self, cmd):
        self.logger.info('Running command: "%s"' % cmd)
        p = subprocess.Popen(
            cmd,
            shell=True,
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE
        )
        first_line = None
        for line in iter(p.stdout.readline, b''):
            line = line.strip()
            if not first_line:
                first_line = line

            self.logger.info(line)

        p.stdout.close()
        p.wait()

        return first_line

    def _binds_to_shell_volumes(self, binds):
        vol_list = ['-v %s:%s' % (k, v) for k, v in binds.iteritems()]
        return ' '.join(vol_list)

    def launch(self, build_dir, extra_binds=None):
        binds = {build_dir: defaults.SHARED_DIR}
        if extra_binds:
            binds = binds.items() + extra_binds.items()

        path_to_command = os.path.join(
            defaults.SHARED_DIR,
            defaults.SCRATCH_DIR,
            defaults.SCRATCH_BUILDSCRIPT_NAME
        )
        self.logger.info('Starting container: %s' % self.image)
        self.container_id = self._run_cli('%s run -d -ti %s %s bash' % \
                (self.docker_cli,
                 self._binds_to_shell_volumes(binds),
                 self.image))

        self._run_cli('%s exec %s %s' % \
                (self.docker_cli, self.container_id, path_to_command))

    def shutdown(self):
        self.logger.info('Stopping container: %s' % self.container_id)
        self._run_cli('%s stop %s' % (self.docker_cli, self.container_id))

        self.logger.info('Removing container: %s' % self.container_id)
        self._run_cli('%s rm -f %s' % (self.docker_cli, self.container_id))
<commit_msg>log subprocess stderr on error log level<commit_after>import logging
import os
import subprocess
import sys

from vdist import defaults

class BuildMachine(object):

    def __init__(self, machine_logs=True, image=None, insecure_registry=False,
                 docker_cli='docker', docker_args=None):
        self.logger = logging.getLogger('BuildMachine')

        self.machine_logs = machine_logs
        self.image = image

        self.container_id = None

        self.docker_cli = docker_cli

        self.insecure_registry = insecure_registry

    def _run_cli(self, cmd):
        self.logger.info('Running command: "%s"' % cmd)
        p = subprocess.Popen(
            cmd,
            shell=True,
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE
        )
        first_line = None
        for line in iter(p.stdout.readline, b''):
            line = line.strip()
            if not first_line:
                first_line = line

            self.logger.info(line)

        for line in iter(p.stderr.readline, b''):
            line = line.strip()
            if not first_line:
                first_line = line

            self.logger.error(line)

        p.stdout.close()
        p.stderr.close()
        p.wait()

        return first_line

    def _binds_to_shell_volumes(self, binds):
        vol_list = ['-v %s:%s' % (k, v) for k, v in binds.iteritems()]
        return ' '.join(vol_list)

    def launch(self, build_dir, extra_binds=None):
        binds = {build_dir: defaults.SHARED_DIR}
        if extra_binds:
            binds = binds.items() + extra_binds.items()

        path_to_command = os.path.join(
            defaults.SHARED_DIR,
            defaults.SCRATCH_DIR,
            defaults.SCRATCH_BUILDSCRIPT_NAME
        )
        self.logger.info('Starting container: %s' % self.image)
        self.container_id = self._run_cli('%s run -d -ti %s %s bash' % \
                (self.docker_cli,
                 self._binds_to_shell_volumes(binds),
                 self.image))

        self._run_cli('%s exec %s %s' % \
                (self.docker_cli, self.container_id, path_to_command))

    def shutdown(self):
        self.logger.info('Stopping container: %s' % self.container_id)
        self._run_cli('%s stop %s' % (self.docker_cli, self.container_id))

        self.logger.info('Removing container: %s' % self.container_id)
        self._run_cli('%s rm -f %s' % (self.docker_cli, self.container_id))
<|endoftext|>


<commit_before>#!/usr/bin/env python

import numpy
import pylab

from shogun.Features import RealFeatures, SparseRealFeatures, Labels
from shogun.Classifier import SVMOcas
from shogun.Evaluation import *

QUITKEY='q'

def quit (event):
	if event.key==QUITKEY or event.key==QUITKEY.upper():
		pylab.close()

def classify (true_labels):
	num_feats=2
	num_vec=true_labels.get_num_labels()

	data_train=numpy.concatenate(
		(numpy.random.randn(num_feats, num_vec/2)-1,
			numpy.random.randn(num_feats, num_vec/2)+1),
		axis=1)
	realfeat=RealFeatures(data_train)
	feats_train=SparseRealFeatures()
	feats_train.obtain_from_simple(realfeat)
	C=3.
	svm=SVMOcas(C, feats_train, true_labels)
	svm.train()

	# prepare test data
	multiplier=1.2

	# yields perfect curve without permutation, but random guess curve when
	# shuffled
#	data_test=[]
#	for i in xrange(len(data_train)):
#		data_test.append(numpy.linspace(
#			multiplier*numpy.min(data_train[i]),
#			multiplier*numpy.max(data_train[i]), num_vec))
#		data_test[i]=numpy.random.permutation(data_test[i])
#	data_test=numpy.array(data_test)

	# taken from matplotlib/multiple_svms.py, got issues due to
	# rounding problems. works nicely when round(sqrt(x))^2==x
#	print 'num_vec', num_vec
	num=numpy.sqrt(num_vec)
#	print num
	x1=numpy.linspace(
			multiplier*numpy.min(data_train[0]),
			multiplier*numpy.max(data_train[0]), num)
#	print x1
	x2=numpy.linspace(
			multiplier*numpy.min(data_train[1]),
			multiplier*numpy.max(data_train[1]), num)
	x,y=numpy.meshgrid(x1,x2)
	data_test=numpy.array((numpy.ravel(x), numpy.ravel(y)))
#	print numpy.ravel(x1)
#	print len(data_test[0])

	realfeat=RealFeatures(data_test)
	feats_test=SparseRealFeatures()
	feats_test.obtain_from_simple(realfeat)
	svm.set_features(feats_test)
	return numpy.array(svm.classify().get_labels())

###########################################################################
# performance measures
###########################################################################

def roc():
	num_points=100

	# why does it have to be sorted?
#	true_labels=Labels(numpy.random.rand(num_points).round()*2-1)
#	true_labels=Labels(numpy.sort(numpy.random.rand(num_points).round()*2-1))
	true_labels=Labels(numpy.concatenate(
		(-numpy.ones(num_points/2), numpy.ones(num_points/2))))
	output=Labels(classify(true_labels))
	#print "### true labels\n", true_labels.get_labels()
	#print "### output\n", output.get_labels()

	pm=PerformanceMeasures(true_labels, output)
	points=pm.compute_ROC()
	#print "### ROC points"
	#print points
	points=numpy.array(points).T # for pylab.plot

	pylab.plot(points[0], points[1], 'bo', linewidth=2.)
	pylab.plot([0, 1], [0, 1], 'r-s')
	pylab.axis([0, 1, 0, 1])
	ticks=numpy.arange(0., 1., .1, dtype=numpy.float64)
	pylab.xticks(ticks)
	pylab.yticks(ticks)
	pylab.title('ROC of SVMOcas with random examples')
	pylab.xlabel('false positive rate')
	pylab.ylabel('true positive rate')

	pylab.connect('key_press_event', quit)
	pylab.show()

###########################################################################
# call functions
###########################################################################

if __name__=='__main__':
	numpy.random.seed(42)

	roc()
<commit_msg>evaluation: - Fixed example for ROC on SVMOcas<commit_after>#!/usr/bin/env python

import numpy
import pylab

from shogun.Features import RealFeatures, SparseRealFeatures, Labels
from shogun.Classifier import SVMOcas
from shogun.Evaluation import *

QUITKEY='q'

def quit (event):
	if event.key==QUITKEY or event.key==QUITKEY.upper():
		pylab.close()

def classify (true_labels):
	num_feats=2
	num_vec=true_labels.get_num_labels()

	data_train=numpy.concatenate(
		(numpy.random.randn(num_feats, num_vec/2)-1,
			numpy.random.randn(num_feats, num_vec/2)+1),
		axis=1)
	realfeat=RealFeatures(data_train)
	feats_train=SparseRealFeatures()
	feats_train.obtain_from_simple(realfeat)
	C=3.
	svm=SVMOcas(C, feats_train, true_labels)
	svm.train()

	data_test=numpy.concatenate(
		(numpy.random.randn(num_feats, num_vec/2)-1,
			numpy.random.randn(num_feats, num_vec/2)+1),
		axis=1)
	realfeat=RealFeatures(data_test)
	feats_test=SparseRealFeatures()
	feats_test.obtain_from_simple(realfeat)
	svm.set_features(feats_test)

	return numpy.array(svm.classify().get_labels())

###########################################################################
# performance measures
###########################################################################

def roc():
	num_points=500

	true_labels=Labels(numpy.concatenate(
		(-numpy.ones(num_points/2), numpy.ones(num_points/2))))
	output=Labels(classify(true_labels))

	pm=PerformanceMeasures(true_labels, output)
	points=pm.compute_ROC()
	points=numpy.array(points).T # for pylab.plot

	pylab.plot(points[0], points[1], 'b.', linewidth=2.)
	pylab.plot([0, 1], [0, 1], 'r-s')
	pylab.axis([0, 1, 0, 1])
	ticks=numpy.arange(0., 1., .1, dtype=numpy.float64)
	pylab.xticks(ticks)
	pylab.yticks(ticks)
	pylab.title('ROC of SVMOcas with random examples/true labels')
	pylab.xlabel('false positive rate')
	pylab.ylabel('true positive rate')

	pylab.connect('key_press_event', quit)
	pylab.show()

###########################################################################
# call functions
###########################################################################

if __name__=='__main__':
	# fixate 'random' values
	numpy.random.seed(42)

	roc()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
#
# MPL5.py
#
# Created by José Sánchez-Gallego on 8 Aug 2017.

from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

from astropy import units as u

from .base import Bintype, Template, DAPDataModel, Property, MultiChannelProperty, spaxel, Channel
from .MPL4 import MPL4_emline_channels

GAU_MILESHC = Template('GAU-MILESHC',
                       description='Set of stellar templates derived from the MILES library.')

ALL = Bintype('ALL', description='Sum of all spectra in each datacube.')
NRE = Bintype('NRE', description='Two radial bins, binning all spectra from 0-1 '
                                 'and 1-2 (elliptical Petrosian) effective radii.')
SPX = Bintype('SPX', binned=False, description='Unbinned spaxels.')
VOR10 = Bintype('VOR10', description='Spectra binned to S/N~10 using the Voronoi '
                                     'binning algorithm (Cappellari & Copin 2003).')

last_idx = len(MPL4_emline_channels)
MPL5_emline_channels = MPL4_emline_channels + [
    Channel('oii_3727', formats={'string': 'OII 3727',
                                 'latex': r'$\forb{O\,II}\;\lambda 3727$'}, idx=last_idx + 1),
    Channel('oii_3729', formats={'string': 'OII 3729',
                                 'latex': r'$\forb{O\,II}\;\lambda 3729$'}, idx=last_idx + 2),
    Channel('heps_3971', formats={'string': 'H-epsilon 3971',
                                  'latex': r'H$\epsilon\;\lambda 3971$'}, idx=last_idx + 3),
    Channel('hdel_4102', formats={'string': 'H-delta 4102',
                                  'latex': r'H$\delta\;\lambda 4102$'}, idx=last_idx + 4),
    Channel('hgam_4341', formats={'string': 'H-gamma 4341',
                                  'latex': r'H$\gamma\;\lambda 4341$'}, idx=last_idx + 5),
    Channel('heii_4687', formats={'string': 'HeII 4681',
                                  'latex': r'He\,II$\;\lambda 4687$'}, idx=last_idx + 6),
    Channel('hei_5877', formats={'string': 'HeI 5877',
                                 'latex': r'He\,I$\;\lambda 5877$'}, idx=last_idx + 7),
    Channel('siii_8831', formats={'string': 'SIII 8831',
                                  'latex': r'$\forb{S\,III}\;\lambda 8831$'}, idx=last_idx + 8),
    Channel('siii_9071', formats={'string': 'SIII 9071',
                                  'latex': r'$\forb{S\,III}\;\lambda 9071$'}, idx=last_idx + 9),
    Channel('siii_9533', formats={'string': 'SIII 9533',
                                  'latex': r'$\forb{S\,III}\;\lambda 9533$'}, idx=last_idx + 10)
]

MPL5_specindex_channels = [
    Channel('d4000', formats={'string': 'D4000'}, unit=u.Angstrom, idx=0),
    Channel('dn4000', formats={'string': 'Dn4000'}, unit=u.Angstrom, idx=1)
]

MPL5_maps = [
    MultiChannelProperty('spx_skycoo', ivar=False, mask=False,
                         channels=[Channel('on_sky_x', formats={'string': 'On-sky X'}, idx=0),
                                   Channel('on_sky_y', formats={'string': 'On-sky Y'}, idx=1)],
                         unit=u.arcsec,
                         formats={'string': 'Sky coordinates'},
                         description='Offsets of each spaxel from the galaxy center.'),
    MultiChannelProperty('spx_ellcoo', ivar=False, mask=False,
                         channels=[Channel('elliptical_radius',
                                           formats={'string': 'Elliptical radius'},
                                           idx=0, unit=u.arcsec),
                                   Channel('elliptical_azimuth',
                                           formats={'string': 'Elliptical azimuth'},
                                           idx=1, unit=u.deg)],
                         formats={'string': 'Elliptical coordinates'},
                         description='Elliptical polar coordinates of each spaxel from '
                                     'the galaxy center.'),
    Property('spx_mflux', ivar=True, mask=False,
             unit=u.erg / u.s / (u.cm ** 2) / spaxel, scale=1e-17,
             formats={'string': 'r-band mean flux'},
             description='Mean flux in r-band (5600.1-6750.0 ang).'),
    Property('spx_snr', ivar=False, mask=False,
             formats={'string': 'r-band SNR'},
             description='r-band signal-to-noise ratio per pixel.'),
    Property('binid', ivar=False, mask=False,
             formats={'string': 'Bin ID'},
             description='Numerical ID for spatial bins.'),
    MultiChannelProperty('bin_lwskycoo', ivar=False, mask=False,
                         channels=[Channel('lum_weighted_on_sky_x',
                                           formats={'string': 'Light-weighted offset X'},
                                           idx=0, unit=u.arcsec),
                                   Channel('lum_weighted_on_sky_y',
                                           formats={'string': 'Light-weighted offset Y'},
                                           idx=1, unit=u.arcsec)],
                         description='Light-weighted offset of each bin from the galaxy center.'),
    MultiChannelProperty('bin_lwellcoo', ivar=False, mask=False,
                         channels=[Channel('lum_weighted_elliptical_radius',
                                           formats={'string': 'Light-weighted radial offset'},
                                           idx=0, unit=u.arcsec),
                                   Channel('lum_weighted_elliptical_azimuth',
                                           formats={'string': 'Light-weighted azimuthal offset'},
                                           idx=1, unit=u.deg)],
                         description='Light-weighted elliptical polar coordinates of each bin '
                                     'from the galaxy center.'),
    Property('bin_area', ivar=False, mask=False,
             unit=u.arcsec ** 2,
             formats={'string': 'Bin area'},
             description='Area of each bin.'),
    Property('bin_farea', ivar=False, mask=False,
             formats={'string': 'Bin fractional area'},
             description='Fractional area that the bin covers for the expected bin '
                         'shape (only relevant for radial binning).'),
    Property('bin_mflux', ivar=True, mask=True,
             unit=u.erg / u.s / (u.cm ** 2) / spaxel, scale=1e-17,
             formats={'string': 'r-band binned spectra mean flux'},
             description='Mean flux in the r-band for the binned spectra.'),
    Property('bin_snr', ivar=False, mask=False,
             formats={'string': 'Bin SNR'},
             description='r-band signal-to-noise ratio per pixel in the binned spectra.'),
    Property('stellar_vel', ivar=True, mask=True,
             unit=u.km / u.s,
             formats={'string': 'Stellar velocity'},
             description='Stellar velocity relative to NSA redshift.'),
    Property('stellar_sigma', ivar=True, mask=True,
             unit=u.km / u.s,
             formats={'string': 'Stellar velocity dispersion', 'latex': r'Stellar $\sigma$'},
             description='Stellar velocity dispersion (must be corrected using '
                         'STELLAR_SIGMACORR)'),
    Property('stellar_sigmacorr', ivar=False, mask=False,
             unit=u.km / u.s,
             formats={'string': 'Stellar sigma correction', 'latex': r'Stellar $\sigma$ correction'},
             description='Quadrature correction for STELLAR_SIGMA to obtain the '
                         'astrophysical velocity dispersion.)'),
    MultiChannelProperty('stellar_cont_fresid', ivar=False, mask=False,
                         channels=[Channel('68th_percentile',
                                           formats={'string': '68th percentile',
                                                    'latex': r'68^{th} percentile'}, idx=0),
                                   Channel('99th_percentile',
                                           formats={'string': '99th percentile',
                                                    'latex': r'99^{th} percentile'}, idx=1)],
                         formats={'string': 'Fractional residual growth'},
                         description='68%% and 99%% growth of the fractional residuals between '
                                     'the model and data'),
    Property('stellar_cont_rchi2', ivar=False, mask=False,
             formats={'string': 'Stellar continuum reduced chi-square',
                      'latex': r'Stellar\ continuum\ reduced\ \chi^2'},
             description='Reduced chi-square of the stellar continuum fit.'),
    MultiChannelProperty('emline_sflux', ivar=True, mask=True,
                         channels=MPL5_emline_channels,
                         formats={'string': 'Emission line summed flux'},
                         unit=u.erg / u.s / (u.cm ** 2) / spaxel, scale=1e-17,
                         description='Non-parametric summed flux for emission lines.'),
    MultiChannelProperty('emline_sew', ivar=True, mask=True,
                         channels=MPL5_emline_channels,
                         formats={'string': 'Emission line EW'},
                         unit=u.Angstrom,
                         description='Emission line non-parametric equivalent '
                                     'widths measurements.'),
    MultiChannelProperty('emline_gflux', ivar=True, mask=True,
                         channels=MPL5_emline_channels,
                         formats={'string': 'Emission line Gaussian flux'},
                         unit=u.erg / u.s / (u.cm ** 2) / spaxel, scale=1e-17,
                         description='Gaussian profile integrated flux for emission lines.'),
    MultiChannelProperty('emline_gvel', ivar=True, mask=True,
                         channels=MPL5_emline_channels,
                         formats={'string': 'Emission line Gaussian velocity'},
                         unit=u.km / u.s,
                         description='Gaussian profile velocity for emission lines.'),
    MultiChannelProperty('emline_gsigma', ivar=True, mask=True,
                         channels=MPL5_emline_channels,
                         formats={'string': 'Emission line Gaussian sigma',
                                  'latex': r'Emission line Gaussian $\sigma$'},
                         unit=u.km / u.s,
                         description='Gaussian profile velocity dispersion for emission lines; '
                                     'must be corrected using EMLINE_INSTSIGMA'),
    MultiChannelProperty('emline_instsigma', ivar=False, mask=False,
                         channels=MPL5_emline_channels,
                         formats={'string': 'Emission line instrumental sigma',
                                  'latex': r'Emission line instrumental $\sigma$'},
                         unit=u.km / u.s,
                         description='Instrumental dispersion at the fitted line center.'),
    MultiChannelProperty('specindex', ivar=True, mask=True,
                         channels=MPL5_specindex_channels,
                         formats={'string': 'Spectral index'},
                         description='Measurements of spectral indices.'),
    MultiChannelProperty('specindex_corr', ivar=False, mask=False,
                         channels=MPL5_specindex_channels,
                         formats={'string': 'Spectral index sigma correction',
                                  'latex': r'Spectral index $\sigma$ correction'},
                         description='Velocity dispersion corrections for the '
                                     'spectral index measurements '
                                     '(can be ignored for D4000, Dn4000).')
]

MPL5 = DAPDataModel('2.0.2', aliases=['MPL-5', 'MPL5'], bintypes=[ALL, NRE, VOR10, SPX],
                    templates=[GAU_MILESHC], properties=MPL5_maps,
                    default_bintype='SPX', default_template='GAU-MILESHC')
<commit_msg>line too long<commit_after>#!/usr/bin/env python
# encoding: utf-8
#
# MPL5.py
#
# Created by José Sánchez-Gallego on 8 Aug 2017.

from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

from astropy import units as u

from .base import Bintype, Template, DAPDataModel, Property, MultiChannelProperty, spaxel, Channel
from .MPL4 import MPL4_emline_channels

GAU_MILESHC = Template('GAU-MILESHC',
                       description='Set of stellar templates derived from the MILES library.')

ALL = Bintype('ALL', description='Sum of all spectra in each datacube.')
NRE = Bintype('NRE', description='Two radial bins, binning all spectra from 0-1 '
                                 'and 1-2 (elliptical Petrosian) effective radii.')
SPX = Bintype('SPX', binned=False, description='Unbinned spaxels.')
VOR10 = Bintype('VOR10', description='Spectra binned to S/N~10 using the Voronoi '
                                     'binning algorithm (Cappellari & Copin 2003).')

last_idx = len(MPL4_emline_channels)
MPL5_emline_channels = MPL4_emline_channels + [
    Channel('oii_3727', formats={'string': 'OII 3727',
                                 'latex': r'$\forb{O\,II}\;\lambda 3727$'}, idx=last_idx + 1),
    Channel('oii_3729', formats={'string': 'OII 3729',
                                 'latex': r'$\forb{O\,II}\;\lambda 3729$'}, idx=last_idx + 2),
    Channel('heps_3971', formats={'string': 'H-epsilon 3971',
                                  'latex': r'H$\epsilon\;\lambda 3971$'}, idx=last_idx + 3),
    Channel('hdel_4102', formats={'string': 'H-delta 4102',
                                  'latex': r'H$\delta\;\lambda 4102$'}, idx=last_idx + 4),
    Channel('hgam_4341', formats={'string': 'H-gamma 4341',
                                  'latex': r'H$\gamma\;\lambda 4341$'}, idx=last_idx + 5),
    Channel('heii_4687', formats={'string': 'HeII 4681',
                                  'latex': r'He\,II$\;\lambda 4687$'}, idx=last_idx + 6),
    Channel('hei_5877', formats={'string': 'HeI 5877',
                                 'latex': r'He\,I$\;\lambda 5877$'}, idx=last_idx + 7),
    Channel('siii_8831', formats={'string': 'SIII 8831',
                                  'latex': r'$\forb{S\,III}\;\lambda 8831$'}, idx=last_idx + 8),
    Channel('siii_9071', formats={'string': 'SIII 9071',
                                  'latex': r'$\forb{S\,III}\;\lambda 9071$'}, idx=last_idx + 9),
    Channel('siii_9533', formats={'string': 'SIII 9533',
                                  'latex': r'$\forb{S\,III}\;\lambda 9533$'}, idx=last_idx + 10)
]

MPL5_specindex_channels = [
    Channel('d4000', formats={'string': 'D4000'}, unit=u.Angstrom, idx=0),
    Channel('dn4000', formats={'string': 'Dn4000'}, unit=u.Angstrom, idx=1)
]

MPL5_maps = [
    MultiChannelProperty('spx_skycoo', ivar=False, mask=False,
                         channels=[Channel('on_sky_x', formats={'string': 'On-sky X'}, idx=0),
                                   Channel('on_sky_y', formats={'string': 'On-sky Y'}, idx=1)],
                         unit=u.arcsec,
                         formats={'string': 'Sky coordinates'},
                         description='Offsets of each spaxel from the galaxy center.'),
    MultiChannelProperty('spx_ellcoo', ivar=False, mask=False,
                         channels=[Channel('elliptical_radius',
                                           formats={'string': 'Elliptical radius'},
                                           idx=0, unit=u.arcsec),
                                   Channel('elliptical_azimuth',
                                           formats={'string': 'Elliptical azimuth'},
                                           idx=1, unit=u.deg)],
                         formats={'string': 'Elliptical coordinates'},
                         description='Elliptical polar coordinates of each spaxel from '
                                     'the galaxy center.'),
    Property('spx_mflux', ivar=True, mask=False,
             unit=u.erg / u.s / (u.cm ** 2) / spaxel, scale=1e-17,
             formats={'string': 'r-band mean flux'},
             description='Mean flux in r-band (5600.1-6750.0 ang).'),
    Property('spx_snr', ivar=False, mask=False,
             formats={'string': 'r-band SNR'},
             description='r-band signal-to-noise ratio per pixel.'),
    Property('binid', ivar=False, mask=False,
             formats={'string': 'Bin ID'},
             description='Numerical ID for spatial bins.'),
    MultiChannelProperty('bin_lwskycoo', ivar=False, mask=False,
                         channels=[Channel('lum_weighted_on_sky_x',
                                           formats={'string': 'Light-weighted offset X'},
                                           idx=0, unit=u.arcsec),
                                   Channel('lum_weighted_on_sky_y',
                                           formats={'string': 'Light-weighted offset Y'},
                                           idx=1, unit=u.arcsec)],
                         description='Light-weighted offset of each bin from the galaxy center.'),
    MultiChannelProperty('bin_lwellcoo', ivar=False, mask=False,
                         channels=[Channel('lum_weighted_elliptical_radius',
                                           formats={'string': 'Light-weighted radial offset'},
                                           idx=0, unit=u.arcsec),
                                   Channel('lum_weighted_elliptical_azimuth',
                                           formats={'string': 'Light-weighted azimuthal offset'},
                                           idx=1, unit=u.deg)],
                         description='Light-weighted elliptical polar coordinates of each bin '
                                     'from the galaxy center.'),
    Property('bin_area', ivar=False, mask=False,
             unit=u.arcsec ** 2,
             formats={'string': 'Bin area'},
             description='Area of each bin.'),
    Property('bin_farea', ivar=False, mask=False,
             formats={'string': 'Bin fractional area'},
             description='Fractional area that the bin covers for the expected bin '
                         'shape (only relevant for radial binning).'),
    Property('bin_mflux', ivar=True, mask=True,
             unit=u.erg / u.s / (u.cm ** 2) / spaxel, scale=1e-17,
             formats={'string': 'r-band binned spectra mean flux'},
             description='Mean flux in the r-band for the binned spectra.'),
    Property('bin_snr', ivar=False, mask=False,
             formats={'string': 'Bin SNR'},
             description='r-band signal-to-noise ratio per pixel in the binned spectra.'),
    Property('stellar_vel', ivar=True, mask=True,
             unit=u.km / u.s,
             formats={'string': 'Stellar velocity'},
             description='Stellar velocity relative to NSA redshift.'),
    Property('stellar_sigma', ivar=True, mask=True,
             unit=u.km / u.s,
             formats={'string': 'Stellar velocity dispersion', 'latex': r'Stellar $\sigma$'},
             description='Stellar velocity dispersion (must be corrected using '
                         'STELLAR_SIGMACORR)'),
    Property('stellar_sigmacorr', ivar=False, mask=False,
             unit=u.km / u.s,
             formats={'string': 'Stellar sigma correction',
                      'latex': r'Stellar $\sigma$ correction'},
             description='Quadrature correction for STELLAR_SIGMA to obtain the '
                         'astrophysical velocity dispersion.)'),
    MultiChannelProperty('stellar_cont_fresid', ivar=False, mask=False,
                         channels=[Channel('68th_percentile',
                                           formats={'string': '68th percentile',
                                                    'latex': r'68^{th} percentile'}, idx=0),
                                   Channel('99th_percentile',
                                           formats={'string': '99th percentile',
                                                    'latex': r'99^{th} percentile'}, idx=1)],
                         formats={'string': 'Fractional residual growth'},
                         description='68%% and 99%% growth of the fractional residuals between '
                                     'the model and data'),
    Property('stellar_cont_rchi2', ivar=False, mask=False,
             formats={'string': 'Stellar continuum reduced chi-square',
                      'latex': r'Stellar\ continuum\ reduced\ \chi^2'},
             description='Reduced chi-square of the stellar continuum fit.'),
    MultiChannelProperty('emline_sflux', ivar=True, mask=True,
                         channels=MPL5_emline_channels,
                         formats={'string': 'Emission line summed flux'},
                         unit=u.erg / u.s / (u.cm ** 2) / spaxel, scale=1e-17,
                         description='Non-parametric summed flux for emission lines.'),
    MultiChannelProperty('emline_sew', ivar=True, mask=True,
                         channels=MPL5_emline_channels,
                         formats={'string': 'Emission line EW'},
                         unit=u.Angstrom,
                         description='Emission line non-parametric equivalent '
                                     'widths measurements.'),
    MultiChannelProperty('emline_gflux', ivar=True, mask=True,
                         channels=MPL5_emline_channels,
                         formats={'string': 'Emission line Gaussian flux'},
                         unit=u.erg / u.s / (u.cm ** 2) / spaxel, scale=1e-17,
                         description='Gaussian profile integrated flux for emission lines.'),
    MultiChannelProperty('emline_gvel', ivar=True, mask=True,
                         channels=MPL5_emline_channels,
                         formats={'string': 'Emission line Gaussian velocity'},
                         unit=u.km / u.s,
                         description='Gaussian profile velocity for emission lines.'),
    MultiChannelProperty('emline_gsigma', ivar=True, mask=True,
                         channels=MPL5_emline_channels,
                         formats={'string': 'Emission line Gaussian sigma',
                                  'latex': r'Emission line Gaussian $\sigma$'},
                         unit=u.km / u.s,
                         description='Gaussian profile velocity dispersion for emission lines; '
                                     'must be corrected using EMLINE_INSTSIGMA'),
    MultiChannelProperty('emline_instsigma', ivar=False, mask=False,
                         channels=MPL5_emline_channels,
                         formats={'string': 'Emission line instrumental sigma',
                                  'latex': r'Emission line instrumental $\sigma$'},
                         unit=u.km / u.s,
                         description='Instrumental dispersion at the fitted line center.'),
    MultiChannelProperty('specindex', ivar=True, mask=True,
                         channels=MPL5_specindex_channels,
                         formats={'string': 'Spectral index'},
                         description='Measurements of spectral indices.'),
    MultiChannelProperty('specindex_corr', ivar=False, mask=False,
                         channels=MPL5_specindex_channels,
                         formats={'string': 'Spectral index sigma correction',
                                  'latex': r'Spectral index $\sigma$ correction'},
                         description='Velocity dispersion corrections for the '
                                     'spectral index measurements '
                                     '(can be ignored for D4000, Dn4000).')
]

MPL5 = DAPDataModel('2.0.2', aliases=['MPL-5', 'MPL5'], bintypes=[ALL, NRE, VOR10, SPX],
                    templates=[GAU_MILESHC], properties=MPL5_maps,
                    default_bintype='SPX', default_template='GAU-MILESHC')
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
from scapy.all import *

def packetCallBack(packet):
    '''scans the payload of the TCP packet for
       ftp traffic for usernames and passwords.
    '''
    effteepea_packet = str(packet[TCP].payload)
    if "USER" in effteepea_packet or "PASS" in effteepea_packet:
        print "[=>] Server: %s" % packet[IP].dst
        print "[=>] %s" % packet[TCP].payload

if __name__ == '__main__':
    try:
        if os.getuid() == 0:
            print "[>] Sniffing traffic on port 21 for credentials....\n"
            sniff(filter='tcp port 21', prn=packetCallBack, store=0)
        else:
            print "[!] Please run this script as root."
    except Exception, e:
        print '[!] Error! %s' % e
<commit_msg>Cleaned this up a bit. Also fixed the output to show source and destination.<commit_after>#!/usr/bin/env python
# -----------------------------------------------------------
# Filename      : ftpSneef.py
# Description   :
# Created By    : Joe Pistone
# Date Created  : 28-Nov-2015 18:57
# Date Modified :
#
# License       : Development
#
# Description   : Sniff packets for credentials on port 21
#
# (c) Copyright 2015, TheKillingTime all rights reserved.
#-----------------------------------------------------------

__author__  = "Joe Pistone"
__version__ = "0.1"

import os
from scapy.all import *

def packetCallBack(packet):
    '''scans the payload of the TCP packet for
       ftp traffic for usernames and passwords.
    '''
    effteepea_packet = str(packet[TCP].payload)
    if "USER" in effteepea_packet or "PASS" in effteepea_packet:
        print "--" * 25
        print "\n[=>] Server: %s" % packet[IP].dst
        print "[=>] Source: %s" % packet[IP].src
        print "[=>] %s" % packet[TCP].payload
        print "--" * 25

if __name__ == '__main__':
    try:
        if os.getuid() == 0:
            print "[*] Sniffing traffic on port 21 for credentials....\n"
            sniff(filter='tcp port 21', prn=packetCallBack, store=0)
        else:
            print "[!] Please run this script as root."
    except Exception, e:
        print '[!] Error! %s' % e
<|endoftext|>


<commit_before># select_corpus.py
#
# This module imports metadata about reviewed volumes; selects a set of
# volumes in the "reviewed" class, and then uses a larger generic
# metadata corpus to select a matching chronological distribution of
# random volumes.
#
# It differs from the original version of the script in using
# a newer metadata set.

import csv
import SonicScrewdriver as utils
import random

selecteddates = dict()
selected = set()

reviews = '/Users/tunder/Dropbox/ted/reception/reviewed/lists/ReviewedTitles1860-1879_200.csv'
with open(reviews) as f:
    reader = csv.reader(f)
    for fields in reader:
        htid = fields[0]
        if htid == "HTid":
            continue
        jgenre = fields[13]
        date = int(fields[1])

        if jgenre == 'poe':
            selecteddates[htid] = date
            selected.add(htid)

rows, columns, table = utils.readtsv('/Users/tunder/Dropbox/GenreProject/metadata/filteredpoetry.tsv')

bydate = dict()
authors = dict()
titles = dict()
datesbyhtid = dict()

with open('/Users/tunder/work/genre/metadata/poemeta.csv', encoding = 'utf-8') as f:
    reader = csv.DictReader(f)
    for row in reader:
        htid = row['htid']
        authors[htid] = row['author']
        titles[htid] = row['title']

        date = utils.date_row(row)
        datesbyhtid[htid] = date
        if htid in selected:
            continue
        if date in bydate:
            bydate[date].append(htid)
        else:
            bydate[date] = [htid]

controlset = set()

for theid, date in selecteddates.items():
    found = False
    while not found:
        candidates = bydate[date]
        choice = random.sample(candidates, 1)[0]
        print(authors[choice])
        print(titles[choice])
        acceptable = input("ACCEPT? (y/n): ")
        if acceptable == "y":
            controlset.add(choice)
            found = True

ficmetadata = list()
for line in selected:
    htid = utils.clean_pairtree(line.rstrip())
    if htid not in datesbyhtid:
        print(htid)
        continue
    date = str(datesbyhtid[htid])
    author = authors[htid]
    title = titles[htid]
    outline = htid + '\t' + 'elite' + '\t' + date + '\t' + author + '\t' + title + '\n'
    ficmetadata.append(outline)
for line in controlset:
    htid = utils.clean_pairtree(line.rstrip())
    if htid not in datesbyhtid:
        print(htid)
        continue
    date = str(datesbyhtid[htid])
    author = authors[htid]
    title = titles[htid]
    outline = htid + '\t' + 'vulgar' + '\t' + date + '\t' + author + '\t' + title + '\n'
    ficmetadata.append(outline)

metapath = '/Users/tunder/Dropbox/GenreProject/metadata/richpoemeta1879.tsv'
with open(metapath, mode = 'a', encoding = 'utf-8') as f:
    for line in ficmetadata:
        f.write(line)

<commit_msg>Fixing dates, allowing quit.<commit_after># select_corpus.py
#
# This module imports metadata about reviewed volumes; selects a set of
# volumes in the "reviewed" class, and then uses a larger generic
# metadata corpus to select a matching chronological distribution of
# random volumes.
#
# It differs from the original version of the script in using
# a newer metadata set.

import csv
import SonicScrewdriver as utils
import random

selecteddates = dict()
selected = set()

reviews = '/Users/tunder/Dropbox/ted/reception/reviewed/lists/ReviewedTitles1840-1859_200.csv'
with open(reviews) as f:
    reader = csv.DictReader(f)

    for fields in reader:

        htid = row['HTid']
        pubdate = int(row['date'])
        firstpub = int(row['firstpub'])
        yrrev = int(row['yrrev'])

        if pubdate > yrrev + 5:
            date = yrrev
            print(str(pubdate) + " => " + str(yrrev))
        else:
            date = pubdate

        jgenre = fields[13]

        if jgenre == 'poe':
            selecteddates[htid] = date
            selected.add(htid)

bydate = dict()
authors = dict()
titles = dict()
datesbyhtid = dict()

with open('/Users/tunder/work/genre/metadata/poemeta.csv', encoding = 'utf-8') as f:
    reader = csv.DictReader(f)
    for row in reader:
        htid = row['htid']
        authors[htid] = row['author']
        titles[htid] = row['title']

        date = utils.date_row(row)
        datesbyhtid[htid] = date
        if htid in selected:
            continue
        if date in bydate:
            bydate[date].append(htid)
        else:
            bydate[date] = [htid]

controlset = set()

for theid in selected:
    date = datesbyhtid[theid]
    print(theid)
    print(date)
    found = False
    while not found:
        candidates = bydate[date]
        choice = random.sample(candidates, 1)[0]
        print(authors[choice])
        print(titles[choice])
        acceptable = input("ACCEPT? (y/n): ")
        if acceptable == "y":
            controlset.add(choice)
            found = True
        if acceptable == 'quit':
            break
    if acceptable == 'quit':
        break

ficmetadata = list()
for line in selected:
    htid = utils.clean_pairtree(line.rstrip())
    if htid not in datesbyhtid:
        print(htid)
        continue
    date = str(datesbyhtid[htid])
    author = authors[htid]
    title = titles[htid]
    outline = htid + '\t' + 'elite' + '\t' + date + '\t' + author + '\t' + title + '\n'
    ficmetadata.append(outline)
for line in controlset:
    htid = utils.clean_pairtree(line.rstrip())
    if htid not in datesbyhtid:
        print(htid)
        continue
    date = str(datesbyhtid[htid])
    author = authors[htid]
    title = titles[htid]
    outline = htid + '\t' + 'vulgar' + '\t' + date + '\t' + author + '\t' + title + '\n'
    ficmetadata.append(outline)

metapath = '/Users/tunder/Dropbox/GenreProject/metadata/richpoemeta1879.tsv'
with open(metapath, mode = 'a', encoding = 'utf-8') as f:
    for line in ficmetadata:
        f.write(line)

<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
from configmgr.configmanager import ConfigManager
from plistextractor.plistextractor import indexPListExtr
import helper
import time
import os
from dl_task.dl_task import download_file_by_curl
from datetime import datetime

def downloadSegments(plist_data, file_list, dest_dir):
    if not( plist_data.get('plist_array') and plist_data.get('plist_url')):
        return

    segments = plist_data['plist_array'].segments
    media_seq = int(plist_data['plist_array'].media_sequence)
    latest_datetime_str = datetime.now().strftime("%Y%m%d-%H-%M-%S-%f")

    # Download segments
    for item in segments:
        if not file_list.get( item.uri ):
            ts_url = helper.validateReturnURL( plist_data['plist_url'], item.uri )
            dest_file = str(media_seq) + '.ts'

            # print(ts_url)
            # print(os.path.join(dest_dir, dest_file))

            download_file_by_curl.apply_async((ts_url, os.path.join(dest_dir, dest_file)), expires=120)

            file_list[item.uri] = media_seq

        media_seq += 1

    # Download keys
    for key in plist_data['plist_array'].keys:
        if key:  # First one could be None
            key_uri = helper.validateReturnURL(plist_data['plist_url'], key.uri)

            key_filepath = os.path.join(dest_dir, latest_datetime_str) + '_key'
            download_file_by_curl.apply_async((key_uri, key_filepath), expires=120)

def main():
    if len(sys.argv) == 1:
        print 'No source URLs given.'
        sys.exit()

    # Get channel
    channel = sys.argv[1]
    cfgMgr = ConfigManager(channel)

    if not cfgMgr.load():
        sys.exit()

    # dest_url: this url can change
    dest_url = cfgMgr.url
    file_list = {}

    while True:
        plist_extr = indexPListExtr(dest_url, cfgMgr.logger)
        ret_plist = plist_extr.retrieveAndProcess()

        if not ret_plist:
            time.sleep(cfgMgr.fileduration)
            continue

        if ret_plist['plist_array']:
            downloadSegments(ret_plist, file_list, cfgMgr.destination)
        else:
            dest_url = ret_plist['plist_url']

        time.sleep(cfgMgr.fileduration)

if __name__ == '__main__':
    main()<commit_msg>Fix bug where ret_plist is a bool<commit_after>#!/usr/bin/env python
import sys
from configmgr.configmanager import ConfigManager
from plistextractor.plistextractor import indexPListExtr
import helper
import time
import os
from dl_task.dl_task import download_file_by_curl
from datetime import datetime

def downloadSegments(plist_data, file_list, dest_dir):
    if not( plist_data.get('plist_array') and plist_data.get('plist_url')):
        return

    segments = plist_data['plist_array'].segments
    media_seq = int(plist_data['plist_array'].media_sequence)
    latest_datetime_str = datetime.now().strftime("%Y%m%d-%H-%M-%S-%f")

    # Download segments
    for item in segments:
        if not file_list.get( item.uri ):
            ts_url = helper.validateReturnURL( plist_data['plist_url'], item.uri )
            dest_file = str(media_seq) + '.ts'

            # print(ts_url)
            # print(os.path.join(dest_dir, dest_file))

            download_file_by_curl.apply_async((ts_url, os.path.join(dest_dir, dest_file)), expires=120)

            file_list[item.uri] = media_seq

        media_seq += 1

    # Download keys
    for key in plist_data['plist_array'].keys:
        if key:  # First one could be None
            key_uri = helper.validateReturnURL(plist_data['plist_url'], key.uri)

            key_filepath = os.path.join(dest_dir, latest_datetime_str) + '_key'
            download_file_by_curl.apply_async((key_uri, key_filepath), expires=120)

def main():
    if len(sys.argv) == 1:
        print 'No source URLs given.'
        sys.exit()

    # Get channel
    channel = sys.argv[1]
    cfgMgr = ConfigManager(channel)

    if not cfgMgr.load():
        sys.exit()

    # dest_url: this url can change
    dest_url = cfgMgr.url
    file_list = {}

    while True:
        plist_extr = indexPListExtr(dest_url, cfgMgr.logger)
        ret_plist = plist_extr.retrieveAndProcess()

        if not isinstance(ret_plist, dict):
            time.sleep(cfgMgr.fileduration)
            continue

        if ret_plist['plist_array']:
            downloadSegments(ret_plist, file_list, cfgMgr.destination)
        else:
            dest_url = ret_plist['plist_url']

        time.sleep(cfgMgr.fileduration)

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>#!/usr/bin/env python

import logging
import os

from collections import namedtuple
from subprocess import Popen, PIPE

from xml.etree.ElementTree import Element, SubElement, ElementTree

from penchy import __version__ as penchy_version
from penchy.util import memoized, tree_pp, dict2tree

@memoized
def get_classpath():
    """
    Returns the Java classpath with the help of Maven

    :returns: java classpath
    :rtype: string
    """
    if not os.path.exists('pom.xml'):
        raise OSError("No pom.xml found in the current directory!")

    proc2 = Popen(['mvn', 'dependency:build-classpath'], stdout=PIPE)
    stdout, _ = proc2.communicate()
    for line in stdout.split("\n"):
        if not line.startswith("["):
            return line

class MavenDependency(object):
    """
    This class represents a Maven Dependency.

    A sample Maven Dependency might look like::

        dep = MavenDependency('de.tu_darmstadt.penchy',
                              'pia', '2.0.0.0', 'http://mvn.0x0b.de')
    """
    def __init__(self, groupId, artifactId, version, repo=None,
            classifier=None, artifact_type=None, packaging=None):
        """
        :param groupId: the maven group id.
        :type groupId: string
        :param artifactId: the maven artifact id.
        :type artifactId: string
        :param version: the version of the artifact.
        :type version: string
        :param repo: the maven repository to use.
        :type repo: string
        :param classifier: the classifier of the artifact.
        :type classifier: string
        :param artifact_type: the type of the artifact.
        :type artifact_type: string
        :param packaging: the packaging of the artifact.
        :type packaging: string
        """
        self.groupId = groupId
        self.artifactId = artifactId
        self.version = version
        self.repo = repo
        self.classifier = classifier
        self.type = artifact_type
        self.packaging = packaging

    def __eq__(self, other):
        return self.__dict__ == other.__dict__

class POM(object):
    """
    This class represents a basic Maven POM.

    Duplicates are discarded, so no repository or dependency will
    be defined twice in the POM.

    Keywords are directly translated into children of the <project>
    node::

        POM(groupId='de.tu_darmstadt.penchy').write('pom.xml')

    would result in something like::

        <project>
            <groupId>de.tu_darmstadt.penchy</groupId>
        </project>
    """

    ATTRIBS = {
            'modelVersion': '4.0.0',
    }

    def __init__(self, **kwargs):
        self.repository_list = set()
        self.dependency_list = set()

        self.root = Element('project')
        self.tree = ElementTree(self.root)
        self.dependency_tree = SubElement(self.root, 'dependencies')
        self.repository_tree = SubElement(self.root, 'repositories')

        attribs = POM.ATTRIBS
        attribs.update(kwargs)
        dict2tree(self.root, attribs)

    def add_dependency(self, dep):
        """
        Adds a given dependency to the POM.

        :param dep: the dependency
        :type dep: :class:`MavenDependency`
        """
        if dep in self.dependency_list:
            return

        if dep.repo:
            self.add_repository(dep.repo)

        clean_dep = dep.__dict__.copy()
        clean_dep.pop('repo')

        e = SubElement(self.dependency_tree, 'dependency')
        dict2tree(e, clean_dep)

        self.dependency_list.add(dep)

    def add_repository(self, url, identifier=None):
        """
        Adds a repository to the POM.

        The identifier of the repository will be equal to
        the url by default.

        :param url: the URL of the repository
        :type url: string
        """
        if url in self.repository_list:
            return

        if not identifier:
            identifier = url

        e = SubElement(self.repository_tree, 'repository')
        dict2tree(e, {'url': url, 'id': identifier})

        self.repository_list.add(url)

    def write(self, filename, pretty=True):
        """
        Writes the POM to a file.

        :param filename: the filename to write to
        :type filename: string
        :param pretty: pretty-print resulting file
        :type pretty: bool
        """
        if pretty:
            tree_pp(self.root)

        self.tree.write(filename)

class BootstrapPOM(POM):
    """
    This class represents a bootstrap POM which is used to deploy
    PenchY and its dependencies.
    """
    def __init__(self):
        POM.__init__(self,
                groupId='de.tu_darmstadt.penchy',
                artifactId='penchy-bootstrap',
                name='penchy-bootstrap',
                url='http://www.tu-darmstadt.de',
                version=penchy_version,
                packaging='jar',  # won't work with pom
                )

        self.add_dependency(MavenDependency(
            groupId='de.tu_darmstadt.penchy',
            artifactId='penchy',
            version=penchy_version,
            classifier='py',
            artifact_type='zip'))
<commit_msg>added repr for MavenDep<commit_after>#!/usr/bin/env python

import logging
import os

from collections import namedtuple
from subprocess import Popen, PIPE

from xml.etree.ElementTree import Element, SubElement, ElementTree

from penchy import __version__ as penchy_version
from penchy.util import memoized, tree_pp, dict2tree

@memoized
def get_classpath():
    """
    Returns the Java classpath with the help of Maven

    :returns: java classpath
    :rtype: string
    """
    if not os.path.exists('pom.xml'):
        raise OSError("No pom.xml found in the current directory!")

    proc2 = Popen(['mvn', 'dependency:build-classpath'], stdout=PIPE)
    stdout, _ = proc2.communicate()
    for line in stdout.split("\n"):
        if not line.startswith("["):
            return line

class MavenDependency(object):
    """
    This class represents a Maven Dependency.

    A sample Maven Dependency might look like::

        dep = MavenDependency('de.tu_darmstadt.penchy',
                              'pia', '2.0.0.0', 'http://mvn.0x0b.de')
    """
    def __init__(self, groupId, artifactId, version, repo=None,
            classifier=None, artifact_type=None, packaging=None):
        """
        :param groupId: the maven group id.
        :type groupId: string
        :param artifactId: the maven artifact id.
        :type artifactId: string
        :param version: the version of the artifact.
        :type version: string
        :param repo: the maven repository to use.
        :type repo: string
        :param classifier: the classifier of the artifact.
        :type classifier: string
        :param artifact_type: the type of the artifact.
        :type artifact_type: string
        :param packaging: the packaging of the artifact.
        :type packaging: string
        """
        self.groupId = groupId
        self.artifactId = artifactId
        self.version = version
        self.repo = repo
        self.classifier = classifier
        self.type = artifact_type
        self.packaging = packaging

    def __eq__(self, other):
        return self.__dict__ == other.__dict__

    def __repr__(self):
        return "<%s: %s>" % (self.__class__.__name__,
                ":".join(filter(None, (self.groupId, self.artifactId, self.version,
                    self.classifier, self.packaging))))

class POM(object):
    """
    This class represents a basic Maven POM.

    Duplicates are discarded, so no repository or dependency will
    be defined twice in the POM.

    Keywords are directly translated into children of the <project>
    node::

        POM(groupId='de.tu_darmstadt.penchy').write('pom.xml')

    would result in something like::

        <project>
            <groupId>de.tu_darmstadt.penchy</groupId>
        </project>
    """

    ATTRIBS = {
            'modelVersion': '4.0.0',
    }

    def __init__(self, **kwargs):
        self.repository_list = set()
        self.dependency_list = set()

        self.root = Element('project')
        self.tree = ElementTree(self.root)
        self.dependency_tree = SubElement(self.root, 'dependencies')
        self.repository_tree = SubElement(self.root, 'repositories')

        attribs = POM.ATTRIBS
        attribs.update(kwargs)
        dict2tree(self.root, attribs)

    def add_dependency(self, dep):
        """
        Adds a given dependency to the POM.

        :param dep: the dependency
        :type dep: :class:`MavenDependency`
        """
        if dep in self.dependency_list:
            return

        if dep.repo:
            self.add_repository(dep.repo)

        clean_dep = dep.__dict__.copy()
        clean_dep.pop('repo')

        e = SubElement(self.dependency_tree, 'dependency')
        dict2tree(e, clean_dep)

        self.dependency_list.add(dep)

    def add_repository(self, url, identifier=None):
        """
        Adds a repository to the POM.

        The identifier of the repository will be equal to
        the url by default.

        :param url: the URL of the repository
        :type url: string
        """
        if url in self.repository_list:
            return

        if not identifier:
            identifier = url

        e = SubElement(self.repository_tree, 'repository')
        dict2tree(e, {'url': url, 'id': identifier})

        self.repository_list.add(url)

    def write(self, filename, pretty=True):
        """
        Writes the POM to a file.

        :param filename: the filename to write to
        :type filename: string
        :param pretty: pretty-print resulting file
        :type pretty: bool
        """
        if pretty:
            tree_pp(self.root)

        self.tree.write(filename)

class BootstrapPOM(POM):
    """
    This class represents a bootstrap POM which is used to deploy
    PenchY and its dependencies.
    """
    def __init__(self):
        POM.__init__(self,
                groupId='de.tu_darmstadt.penchy',
                artifactId='penchy-bootstrap',
                name='penchy-bootstrap',
                url='http://www.tu-darmstadt.de',
                version=penchy_version,
                packaging='jar',  # won't work with pom
                )

        self.add_dependency(MavenDependency(
            groupId='de.tu_darmstadt.penchy',
            artifactId='penchy',
            version=penchy_version,
            classifier='py',
            artifact_type='zip'))
<|endoftext|>