<commit_before>"""OANDA API wrapper for OANDA's REST-V20 API."""

import json
import requests
from .exceptions import V20Error

TRADING_ENVIRONMENTS = {
    "practice": {
        "stream": 'https://stream-fxpractice.oanda.com',
        "api": 'https://api-fxpractice.oanda.com'
    },
    "live": {
        "stream": 'https://stream-fxtrade.oanda.com',
        "api": 'https://api-fxtrade.oanda.com'
    }
}

class API(object):
    r"""API - class to handle APIRequests objects to access API endpoints.

    Examples
    --------

    ::

        # get a list of trades
        from oandapyV20 import API
        import oandapyV20.endpoints.trades as trades

        api = API(access_token="xxx")
        accountID = "101-305-3091856-001"

        r = trades.Trades(accountID, op=trades.TRADE_LIST)
        # show the endpoint as it is constructed for this call
        print("REQUEST:{}".format(r))
        rv = api.request(r)
        print("RESPONSE:\n{}".format(json.dumps(rv, indent=2)))

    Output::

        REQUEST:v3/accounts/101-305-3091856-001/trades
        RESPONSE:
        "trades": [
            {
              "financing": "0.0000",
              "openTime": "2016-07-21T15:47:05.170212014Z",
              "price": "10133.9",
              "unrealizedPL": "8.0000",
              "realizedPL": "0.0000",
              "instrument": "DE30_EUR",
              "state": "OPEN",
              "initialUnits": "-10",
              "currentUnits": "-10",
              "id": "1032"
            },
            {
              "financing": "0.0000",
              "openTime": "2016-07-21T15:47:04.963590941Z",
              "price": "10134.4",
              "unrealizedPL": "13.0000",
              "realizedPL": "0.0000",
              "instrument": "DE30_EUR",
              "state": "OPEN",
              "initialUnits": "-10",
              "currentUnits": "-10",
              "id": "1030"
            }
          ],
          "lastTransactionID": "1040"
        }

    ::

        # reduce a trade by it's id
        from oandapyV20 import API
        import oandapyV20.endpoints.trades as trades

        headers = {"Content-Type": "application/json"}
        api = API(access_token="...", headers=headers)

        accountID = "101-305-3091856-001"
        tradeID = "1030"
        cfg = { "units": 5 }
        r = trades.Trades(accountID, tradeID=tradeID,
                          op=trades.TRADE_CLOSE, data=cfg)
        # show the endpoint as it is constructed for this call
        print("REQUEST:{}".format(r))
        rv = api.request(r)
        print("RESPONSE\n{}".format(json.dumps(rv, indent=2)))

    Output::

        REQUEST:v3/accounts/101-305-3091856-001/trades/1030/close
        RESPONSE: {
          "orderFillTransaction": {
            "orderID": "1041",
            "financing": "-0.1519",
            "instrument": "DE30_EUR",
            "userID": 1435156,
            "price": "10131.6",
            "tradeReduced": {
              "units": "5",
              "financing": "-0.1519",
              "realizedPL": "14.0000",
              "tradeID": "1030"
            },
            "batchID": "1041",
            "accountBalance": "44876.2548",
            "reason": "MARKET_ORDER_TRADE_CLOSE",
            "time": "2016-07-21T17:32:51.361464739Z",
            "units": "5",
            "type": "ORDER_FILL",
            "id": "1042",
            "pl": "14.0000",
            "accountID": "101-305-3091856-001"
          },
          "orderCreateTransaction": {
            "timeInForce": "FOK",
            "positionFill": "REDUCE_ONLY",
            "userID": 1435156,
            "batchID": "1041",
            "instrument": "DE30_EUR",
            "reason": "TRADE_CLOSE",
            "tradeClose": {
              "units": "5",
              "tradeID": "1030"
            },
            "time": "2016-07-21T17:32:51.361464739Z",
            "units": "5",
            "type": "MARKET_ORDER",
            "id": "1041",
            "accountID": "101-305-3091856-001"
          },
          "relatedTransactionIDs": [
            "1041",
            "1042"
          ],
          "lastTransactionID": "1042"
        }
    """

    def __init__(self, access_token, environment="practice", headers=None):
        """Instantiate an instance of OandaPy's API wrapper.

        Parameters
        ----------
        access_token : string
            Provide a valid access token.

        environment : string
            Provide the environment for OANDA's REST api. Valid values:
            'practice' or 'live'. Default: 'practice'.

        headers : dict (optional)
            Provide request headers to be set for a request. Several API
            endpoints need data in a JSON format. These calls require the
            header: 'Content-Type: application/json'.

        """
        try:
            TRADING_ENVIRONMENTS[environment]
        except:
            raise KeyError("Unknown environment: {}".format(environment))
        else:
            self.environment = environment

        self.access_token = access_token
        self.client = requests.Session()
        self._connected = False
        self.client.stream = False

        # personal token authentication
        if self.access_token:
            self.client.headers['Authorization'] = 'Bearer '+self.access_token

        if headers:
            self.client.headers.update(headers)

    def disconnect(self):
        """disconnect.

        disconnect a streaming connection. The _stream_request generator
        wil terminate.
        """
        self._connected = False

    def _request(self, method, url, request_args):
        func = getattr(self.client, method)

        response = None
        try:
            response = func(url, **request_args)
        except requests.RequestException as e:
            # log it ?
            raise e
        else:
            self._connected = True

        # Handle error responses
        if response.status_code >= 400:
            raise V20Error(response.status_code,
                           response.content.decode('utf-8'))
        return response

    def _api_request(self, method, url, request_args):
        """_api_request.

        make a 'regular' request. This method is called by
        the 'request' method after it has determined which
        call applies: regular or streaming.
        """
        content = None
        response = None
        try:
            response = self._request(method, url, request_args)
        except requests.RequestException as e:
            raise e
        else:
            content = response.content.decode('utf-8')
            content = json.loads(content)
            #endpoint.response(content)

        return content

    def _stream_request(self, method, url, request_args):
        """_stream_request.

        make a 'stream' request. This method is called by
        the 'request' method after it has determined which
        call applies: regular or streaming.
        """
        response = self._request(method, url, request_args)
        for line in response.iter_lines(90):
            if not self._connected:
                break

            if line:
                data = json.loads(line.decode("utf-8"))
                yield data

    def request(self, endpoint):
        """Perform a request for the APIRequest instance 'endpoint'.

        Parameters
        ----------
        endpoint : APIRequest
            The endpoint parameter contains an instance of an APIRequest
            containing the endpoint, method and optionally other parameters
            or body data.

        Raises
        ------
            V20Error in case of HTTP response code >= 400
        """
        at = "api"
        if hasattr(endpoint, "STREAM") and getattr(endpoint, "STREAM") is True:
            self.client.stream = True
            at = "stream"

        url = "{}/{}".format(TRADING_ENVIRONMENTS[self.environment][at],
                             endpoint)

        method = endpoint.method
        method = method.lower()
        params = None
        try:
            params = getattr(endpoint, "params")
        except AttributeError:
            # request does not have params
            params = {}

        request_args = {}
        if method == 'get':
            request_args['params'] = params
        elif hasattr(endpoint, "data") and endpoint.data:
            request_args['data'] = json.dumps(endpoint.data)

        if at == "api":
            content = self._api_request(method, url, request_args)
            endpoint.response(content)
            return content
        else:
            return self._stream_request(method, url, request_args)
<commit_msg>redundant code removed<commit_after>"""OANDA API wrapper for OANDA's REST-V20 API."""

import json
import requests
from .exceptions import V20Error

TRADING_ENVIRONMENTS = {
    "practice": {
        "stream": 'https://stream-fxpractice.oanda.com',
        "api": 'https://api-fxpractice.oanda.com'
    },
    "live": {
        "stream": 'https://stream-fxtrade.oanda.com',
        "api": 'https://api-fxtrade.oanda.com'
    }
}

class API(object):
    r"""API - class to handle APIRequests objects to access API endpoints.

    Examples
    --------

    ::

        # get a list of trades
        from oandapyV20 import API
        import oandapyV20.endpoints.trades as trades

        api = API(access_token="xxx")
        accountID = "101-305-3091856-001"

        r = trades.Trades(accountID, op=trades.TRADE_LIST)
        # show the endpoint as it is constructed for this call
        print("REQUEST:{}".format(r))
        rv = api.request(r)
        print("RESPONSE:\n{}".format(json.dumps(rv, indent=2)))

    Output::

        REQUEST:v3/accounts/101-305-3091856-001/trades
        RESPONSE:
        "trades": [
            {
              "financing": "0.0000",
              "openTime": "2016-07-21T15:47:05.170212014Z",
              "price": "10133.9",
              "unrealizedPL": "8.0000",
              "realizedPL": "0.0000",
              "instrument": "DE30_EUR",
              "state": "OPEN",
              "initialUnits": "-10",
              "currentUnits": "-10",
              "id": "1032"
            },
            {
              "financing": "0.0000",
              "openTime": "2016-07-21T15:47:04.963590941Z",
              "price": "10134.4",
              "unrealizedPL": "13.0000",
              "realizedPL": "0.0000",
              "instrument": "DE30_EUR",
              "state": "OPEN",
              "initialUnits": "-10",
              "currentUnits": "-10",
              "id": "1030"
            }
          ],
          "lastTransactionID": "1040"
        }

    ::

        # reduce a trade by it's id
        from oandapyV20 import API
        import oandapyV20.endpoints.trades as trades

        headers = {"Content-Type": "application/json"}
        api = API(access_token="...", headers=headers)

        accountID = "101-305-3091856-001"
        tradeID = "1030"
        cfg = { "units": 5 }
        r = trades.Trades(accountID, tradeID=tradeID,
                          op=trades.TRADE_CLOSE, data=cfg)
        # show the endpoint as it is constructed for this call
        print("REQUEST:{}".format(r))
        rv = api.request(r)
        print("RESPONSE\n{}".format(json.dumps(rv, indent=2)))

    Output::

        REQUEST:v3/accounts/101-305-3091856-001/trades/1030/close
        RESPONSE: {
          "orderFillTransaction": {
            "orderID": "1041",
            "financing": "-0.1519",
            "instrument": "DE30_EUR",
            "userID": 1435156,
            "price": "10131.6",
            "tradeReduced": {
              "units": "5",
              "financing": "-0.1519",
              "realizedPL": "14.0000",
              "tradeID": "1030"
            },
            "batchID": "1041",
            "accountBalance": "44876.2548",
            "reason": "MARKET_ORDER_TRADE_CLOSE",
            "time": "2016-07-21T17:32:51.361464739Z",
            "units": "5",
            "type": "ORDER_FILL",
            "id": "1042",
            "pl": "14.0000",
            "accountID": "101-305-3091856-001"
          },
          "orderCreateTransaction": {
            "timeInForce": "FOK",
            "positionFill": "REDUCE_ONLY",
            "userID": 1435156,
            "batchID": "1041",
            "instrument": "DE30_EUR",
            "reason": "TRADE_CLOSE",
            "tradeClose": {
              "units": "5",
              "tradeID": "1030"
            },
            "time": "2016-07-21T17:32:51.361464739Z",
            "units": "5",
            "type": "MARKET_ORDER",
            "id": "1041",
            "accountID": "101-305-3091856-001"
          },
          "relatedTransactionIDs": [
            "1041",
            "1042"
          ],
          "lastTransactionID": "1042"
        }
    """

    def __init__(self, access_token, environment="practice", headers=None):
        """Instantiate an instance of OandaPy's API wrapper.

        Parameters
        ----------
        access_token : string
            Provide a valid access token.

        environment : string
            Provide the environment for OANDA's REST api. Valid values:
            'practice' or 'live'. Default: 'practice'.

        headers : dict (optional)
            Provide request headers to be set for a request. Several API
            endpoints need data in a JSON format. These calls require the
            header: 'Content-Type: application/json'.

        """
        try:
            TRADING_ENVIRONMENTS[environment]
        except:
            raise KeyError("Unknown environment: {}".format(environment))
        else:
            self.environment = environment

        self.access_token = access_token
        self.client = requests.Session()
        self._connected = False
        self.client.stream = False

        # personal token authentication
        if self.access_token:
            self.client.headers['Authorization'] = 'Bearer '+self.access_token

        if headers:
            self.client.headers.update(headers)

    def disconnect(self):
        """disconnect.

        disconnect a streaming connection. The _stream_request generator
        wil terminate.
        """
        self._connected = False

    def _request(self, method, url, request_args):
        func = getattr(self.client, method)

        response = None
        try:
            response = func(url, **request_args)
        except requests.RequestException as e:
            # log it ?
            raise e
        else:
            self._connected = True

        # Handle error responses
        if response.status_code >= 400:
            raise V20Error(response.status_code,
                           response.content.decode('utf-8'))
        return response

    def _api_request(self, method, url, request_args):
        """_api_request.

        make a 'regular' request. This method is called by
        the 'request' method after it has determined which
        call applies: regular or streaming.
        """
        response = self._request(method, url, request_args)
        content = response.content.decode('utf-8')
        content = json.loads(content)

        return content

    def _stream_request(self, method, url, request_args):
        """_stream_request.

        make a 'stream' request. This method is called by
        the 'request' method after it has determined which
        call applies: regular or streaming.
        """
        response = self._request(method, url, request_args)
        for line in response.iter_lines(90):
            if not self._connected:
                break

            if line:
                data = json.loads(line.decode("utf-8"))
                yield data

    def request(self, endpoint):
        """Perform a request for the APIRequest instance 'endpoint'.

        Parameters
        ----------
        endpoint : APIRequest
            The endpoint parameter contains an instance of an APIRequest
            containing the endpoint, method and optionally other parameters
            or body data.

        Raises
        ------
            V20Error in case of HTTP response code >= 400
        """
        at = "api"
        if hasattr(endpoint, "STREAM") and getattr(endpoint, "STREAM") is True:
            self.client.stream = True
            at = "stream"

        url = "{}/{}".format(TRADING_ENVIRONMENTS[self.environment][at],
                             endpoint)

        method = endpoint.method
        method = method.lower()
        params = None
        try:
            params = getattr(endpoint, "params")
        except AttributeError:
            # request does not have params
            params = {}

        request_args = {}
        if method == 'get':
            request_args['params'] = params
        elif hasattr(endpoint, "data") and endpoint.data:
            request_args['data'] = json.dumps(endpoint.data)

        if at == "api":
            content = self._api_request(method, url, request_args)
            endpoint.response(content)
            return content
        else:
            return self._stream_request(method, url, request_args)
<|endoftext|>


<commit_before>from django import forms

from .models import get_application_model

class AllowForm(forms.Form):
    allow = forms.BooleanField(required=False)
    redirect_uri = forms.CharField(widget=forms.HiddenInput())
    scope = forms.CharField(required=False, widget=forms.HiddenInput())
    client_id = forms.CharField(widget=forms.HiddenInput())
    state = forms.CharField(required=False, widget=forms.HiddenInput())
    response_type = forms.CharField(widget=forms.HiddenInput())

    def __init__(self, *args, **kwargs):
        data = kwargs.get('data')
        # backwards compatible support for plural `scopes` query parameter
        if data and 'scopes' in data:
            data['scope'] = data['scopes']
        return super(AllowForm, self).__init__(*args, **kwargs)

class RegistrationForm(forms.ModelForm):
    """
    TODO: add docstring
    """
    class Meta:
        model = get_application_model()
        fields = ('name', 'client_id', 'client_secret', 'client_type', 'authorization_grant_type', 'redirect_uris')
<commit_msg>Remove name field from form<commit_after>from django import forms

from .models import get_application_model

class AllowForm(forms.Form):
    allow = forms.BooleanField(required=False)
    redirect_uri = forms.CharField(widget=forms.HiddenInput())
    scope = forms.CharField(required=False, widget=forms.HiddenInput())
    client_id = forms.CharField(widget=forms.HiddenInput())
    state = forms.CharField(required=False, widget=forms.HiddenInput())
    response_type = forms.CharField(widget=forms.HiddenInput())

    def __init__(self, *args, **kwargs):
        data = kwargs.get('data')
        # backwards compatible support for plural `scopes` query parameter
        if data and 'scopes' in data:
            data['scope'] = data['scopes']
        return super(AllowForm, self).__init__(*args, **kwargs)

class RegistrationForm(forms.ModelForm):
    """
    TODO: add docstring
    """
    class Meta:
        model = get_application_model()
        fields = ('client_id', 'client_secret', 'client_type', 'authorization_grant_type', 'redirect_uris')
<|endoftext|>


<commit_before>"""
core.py
-------

Main decu classes and decorators.

"""

import os
import logging
from .config import config
from .logging import DecuLogger
from .io import write
from functools import wraps
from datetime import datetime
from collections import defaultdict
from multiprocessing import Pool, Value, Lock
if 'DISPLAY' not in os.environ:
    import matplotlib
    matplotlib.use('Agg')
import matplotlib.pyplot as plt

__all__ = ['Script', 'experiment', 'figure', 'run_parallel', 'DecuException']

lock = Lock()
runs = defaultdict(lambda: Value('i', 0))

class DecuException(Exception):
    pass

class Script():
    """Base class for experimental computation scripts."""

    data_dir = config['Script']['data_dir']
    results_dir = config['Script']['results_dir']
    figures_dir = config['Script']['figures_dir']
    scripts_dir = config['Script']['scripts_dir']
    gendata_dir = config['Script']['gendata_dir']
    figure_fmt = config['Script']['figure_fmt']

    def __init__(self, project_dir="", module=None):
        self.start_time = datetime.now()
        self.project_dir = os.getcwd() if project_dir is None else project_dir
        self.module = self.__module__ if module is None else module
        self.log = DecuLogger(self.start_time, project_dir, self.module)

    def make_result_basename(self, exp_name, run):
        return os.path.join(self.results_dir, config['Script'].subs(
            'result_file', time=self.start_time, module_name=self.module,
            exp_name=exp_name, run=run))

    def make_figure_basename(self, fig_name, suffix=None):
        opt = 'figure_wo_suffix_file' if suffix is None \
              else 'figure_w_suffix_file'
        outfile = config['Script'].subs(
            opt, time=self.start_time, module_name=self.module,
            fig_name=fig_name, suffix=suffix, ext=self.figure_fmt)
        return os.path.join(self.figures_dir, outfile)

def run_parallel(exp, params):
    """Run an experiment in parallel.

    For each element `p` in `params`, call `exp(*p)`. These calls are made
    in parallel using multiprocessing.

    Args:
        exp (method): A @experiment-decorated method.
        params (list): Each element is a set of arguments to call `exp` with.

    Returns:
        dict: A dictionary where the key `pi` is an element of `params` and
        the value is the result of calling `exp(*pi)`.

    """
    def init(*args):
        global lock, runs
        lock, runs = args

    with Pool(initializer=init, initargs=(lock, runs),
              maxtasksperchild=100) as pool:
        results = pool.starmap(exp, params)
    return results

def _get_parameters(method, param_name, args, kwargs):
    """Return the arguments passed to all experimental parameters.

    All method arguments that are not param_name are treated as
    experimental parameter is method is assumed to have been called as
    method(*args, **kwargs).

    """
    from inspect import getfullargspec
    arg_values = kwargs.copy()
    arg_values.update(dict(zip(getfullargspec(method).args, args)))
    if param_name in arg_values:
        del arg_values[param_name]
    if 'self' in arg_values:
        del arg_values['self']
    return arg_values

def experiment(data_param=None):
    """Decorator that adds logging functionality to experiment methods.

    Args:

        data_param (str): Parameter treated by the method as data
    input. All other parameters are treated as experimental parameters.

    Returns:
        func: A decorator that adds bookkeeping functionality to its
        argument.

    """
    def _experiment(method):
        """Decorator that adds bookkeeping functionality to its argument.

        Parameters
        ----------

        method (function): A experimental computation function.

        Returns
        -------

        The method function, with added logging and bookkeeping
        functionality.

        """
        exp_name = method.__name__
        cfg = config['experiment']

        def exp_start_msg(run, params):
            return cfg.subs('start_msg', exp_name=exp_name, run=run,
                            params=params)

        def exp_end_msg(run, params, elapsed):
            return cfg.subs('end_msg', exp_name=exp_name, params=params,
                            elapsed=round(elapsed, 5), run=run)

        def wrote_results_msg(run, outfile, params):
            return cfg.subs('write_msg', exp_name=exp_name, params=params,
                            outfile=outfile, run=run)

        from time import time

        @wraps(method)
        def decorated(*args, **kwargs):
            obj = args[0]
            with lock:
                decorated.run = runs[decorated].value
                runs[decorated].value += 1

            # Make sure the output dir exists
            os.makedirs(obj.results_dir, exist_ok=True)

            values = _get_parameters(method, data_param, args, kwargs)
            obj.log.info(exp_start_msg(decorated.run, values))

            start = time()
            result = method(*args, **kwargs)
            end = time()
            obj.log.info(exp_end_msg(decorated.run, values, end - start))
            basename = obj.make_result_basename(exp_name, decorated.run)
            write(result, basename)
            obj.log.info(wrote_results_msg(decorated.run, basename, values))

            return result

        return decorated

    return _experiment

def figure(show=False, save=True):
    """Create the figure decorator.

    Args:
        show (bool): Whether or not the figure should be shown.
        save (bool): Whether or not the figure should be saved to disk.

    Returns:
        func: A decorator that adds figure logging functionality to its
        argument.

    """
    def _figure(method):
        """Decorator that adds logging functionality to figure methods.

        The method must return while the figure is still in memory, i.e.,
        neither plt.show or plt.savefig must be called. The resulting
        decorated method has a 'suffix' parameter that accepts a string
        that will be appended to the output file name. The 'suffix'
        parameter must be passed as a keyword argument.

        Parameters
        ----------

        show (bool): Whether or not the figure should be shown.

        save (bool): Whether or not the figure should be saved to disk.

        Returns
        -------

        The figure method, with added logging functionality, and a new
        'suffix' parameter.

        """
        fig_name = method.__name__

        def wrote_fig_msg(outfile):
            return config['figure'].subs('write', fig_name=fig_name,
                                         outfile=outfile)

        @wraps(method)
        def decorated(*args, suffix=None, **kwargs):
            obj = args[0]
            # Make sure the output dir exists
            os.makedirs(obj.figures_dir, exist_ok=True)

            method(*args, **kwargs)
            fig = plt.gcf()
            if save:
                outfile = obj.make_figure_basename(fig_name, suffix)
                fig.savefig(outfile)
                obj.log.info(wrote_fig_msg(outfile))
            if show:
                plt.show()

        return decorated

    return _figure
<commit_msg>[refactor] return decorated methods that accept self, instead of just getting args[0]. Fixes #49.<commit_after>"""
core.py
-------

Main decu classes and decorators.

"""

import os
import logging
from .config import config
from .logging import DecuLogger
from .io import write
from functools import wraps
from datetime import datetime
from collections import defaultdict
from multiprocessing import Pool, Value, Lock
if 'DISPLAY' not in os.environ:
    import matplotlib
    matplotlib.use('Agg')
import matplotlib.pyplot as plt

__all__ = ['Script', 'experiment', 'figure', 'run_parallel', 'DecuException']

lock = Lock()
runs = defaultdict(lambda: Value('i', 0))

class DecuException(Exception):
    pass

class Script():
    """Base class for experimental computation scripts."""

    data_dir = config['Script']['data_dir']
    results_dir = config['Script']['results_dir']
    figures_dir = config['Script']['figures_dir']
    scripts_dir = config['Script']['scripts_dir']
    gendata_dir = config['Script']['gendata_dir']
    figure_fmt = config['Script']['figure_fmt']

    def __init__(self, project_dir="", module=None):
        self.start_time = datetime.now()
        self.project_dir = os.getcwd() if project_dir is None else project_dir
        self.module = self.__module__ if module is None else module
        self.log = DecuLogger(self.start_time, project_dir, self.module)

    def make_result_basename(self, exp_name, run):
        return os.path.join(self.results_dir, config['Script'].subs(
            'result_file', time=self.start_time, module_name=self.module,
            exp_name=exp_name, run=run))

    def make_figure_basename(self, fig_name, suffix=None):
        opt = 'figure_wo_suffix_file' if suffix is None \
              else 'figure_w_suffix_file'
        outfile = config['Script'].subs(
            opt, time=self.start_time, module_name=self.module,
            fig_name=fig_name, suffix=suffix, ext=self.figure_fmt)
        return os.path.join(self.figures_dir, outfile)

def run_parallel(exp, params):
    """Run an experiment in parallel.

    For each element `p` in `params`, call `exp(*p)`. These calls are made
    in parallel using multiprocessing.

    Args:
        exp (method): A @experiment-decorated method.
        params (list): Each element is a set of arguments to call `exp` with.

    Returns:
        dict: A dictionary where the key `pi` is an element of `params` and
        the value is the result of calling `exp(*pi)`.

    """
    def init(*args):
        global lock, runs
        lock, runs = args

    with Pool(initializer=init, initargs=(lock, runs),
              maxtasksperchild=100) as pool:
        results = pool.starmap(exp, params)
    return results

def _get_parameters(method, param_name, args, kwargs):
    """Return the arguments passed to all experimental parameters.

    All method arguments that are not param_name are treated as
    experimental parameter is method is assumed to have been called as
    method(*args, **kwargs).

    """
    from inspect import getfullargspec
    arg_values = kwargs.copy()
    arg_values.update(dict(zip(getfullargspec(method).args, args)))
    if param_name in arg_values:
        del arg_values[param_name]
    if 'self' in arg_values:
        del arg_values['self']
    return arg_values

def experiment(data_param=None):
    """Decorator that adds logging functionality to experiment methods.

    Args:

        data_param (str): Parameter treated by the method as data
    input. All other parameters are treated as experimental parameters.

    Returns:
        func: A decorator that adds bookkeeping functionality to its
        argument.

    """
    def _experiment(method):
        """Decorator that adds bookkeeping functionality to its argument.

        Parameters
        ----------

        method (function): A experimental computation function.

        Returns
        -------

        The method function, with added logging and bookkeeping
        functionality.

        """
        exp_name = method.__name__
        cfg = config['experiment']

        def exp_start_msg(run, params):
            return cfg.subs('start_msg', exp_name=exp_name, run=run,
                            params=params)

        def exp_end_msg(run, params, elapsed):
            return cfg.subs('end_msg', exp_name=exp_name, params=params,
                            elapsed=round(elapsed, 5), run=run)

        def wrote_results_msg(run, outfile, params):
            return cfg.subs('write_msg', exp_name=exp_name, params=params,
                            outfile=outfile, run=run)

        from time import time

        @wraps(method)
        def decorated(self, *args, **kwargs):
            with lock:
                decorated.run = runs[decorated].value
                runs[decorated].value += 1

            # Make sure the output dir exists
            os.makedirs(self.results_dir, exist_ok=True)

            values = _get_parameters(method, data_param, args, kwargs)
            self.log.info(exp_start_msg(decorated.run, values))

            start = time()
            result = method(self, *args, **kwargs)
            end = time()
            self.log.info(exp_end_msg(decorated.run, values, end - start))
            basename = self.make_result_basename(exp_name, decorated.run)
            write(result, basename)
            self.log.info(wrote_results_msg(decorated.run, basename, values))

            return result

        return decorated

    return _experiment

def figure(show=False, save=True):
    """Create the figure decorator.

    Args:
        show (bool): Whether or not the figure should be shown.
        save (bool): Whether or not the figure should be saved to disk.

    Returns:
        func: A decorator that adds figure logging functionality to its
        argument.

    """
    def _figure(method):
        """Decorator that adds logging functionality to figure methods.

        The method must return while the figure is still in memory, i.e.,
        neither plt.show or plt.savefig must be called. The resulting
        decorated method has a 'suffix' parameter that accepts a string
        that will be appended to the output file name. The 'suffix'
        parameter must be passed as a keyword argument.

        Parameters
        ----------

        show (bool): Whether or not the figure should be shown.

        save (bool): Whether or not the figure should be saved to disk.

        Returns
        -------

        The figure method, with added logging functionality, and a new
        'suffix' parameter.

        """
        fig_name = method.__name__

        def wrote_fig_msg(outfile):
            return config['figure'].subs('write', fig_name=fig_name,
                                         outfile=outfile)

        @wraps(method)
        def decorated(self, *args, suffix=None, **kwargs):
            # Make sure the output dir exists
            os.makedirs(self.figures_dir, exist_ok=True)

            method(self, *args, **kwargs)
            fig = plt.gcf()
            if save:
                outfile = self.make_figure_basename(fig_name, suffix)
                fig.savefig(outfile)
                self.log.info(wrote_fig_msg(outfile))
            if show:
                plt.show()

        return decorated

    return _figure
<|endoftext|>


<commit_before>"""
Custom Authenticator to use Google OAuth with JupyterHub.

Derived from the GitHub OAuth authenticator.
"""

import os
import json

from base64 import b64decode, b64encode, urlsafe_b64decode
import re

from tornado             import gen, escape
from tornado.auth        import GoogleOAuth2Mixin
from tornado.web         import HTTPError

from traitlets           import Unicode

from jupyterhub.auth     import LocalAuthenticator
from jupyterhub.utils    import url_path_join

from .oauth2 import OAuthLoginHandler, OAuthCallbackHandler, OAuthenticator

from tornado.util import PY3
import functools

if PY3:
    import urllib.parse as urlparse
    import urllib.parse as urllib_parse
    long = int
else:
    import urlparse
    import urllib as urllib_parse

class AuthError(Exception):
    pass

class OpenIDOAuth2Mixin(GoogleOAuth2Mixin):
    """ An OpenID OAuth2 mixin to use GoogleLoginHandler with
    different Identity Providers using the OpenID standard. The current
    setup should work with MITREid Connect servers. In addtion to the usual
    parameters client ID and secret, the environment variable OPENID_HOST
    should be set to the URL of the OpenID provider. The API endpoints
    might have to be changed, depending on the ID provider."""

    CONNECTORS = os.environ.get('CONNECTOR_LIST')

    _OPENID_ENDPOINT = os.environ.get('OPENID_HOST')
    if _OPENID_ENDPOINT.startswith('http'):
        _OAUTH_AUTHORIZE_URL = "%s/auth" % _OPENID_ENDPOINT
        _OAUTH_ACCESS_TOKEN_URL = "%s/token" % _OPENID_ENDPOINT
        _OAUTH_USERINFO_URL = "%s/auth" % _OPENID_ENDPOINT
    else:
        _OAUTH_AUTHORIZE_URL = "https://%s/auth" % _OPENID_ENDPOINT
        _OAUTH_ACCESS_TOKEN_URL = "https://%s/token" % _OPENID_ENDPOINT
        _OAUTH_USERINFO_URL = "https://%s/auth" % _OPENID_ENDPOINT

    # def get_authenticated_user(self, redirect_uri, code, callback):
    #     """Handles the login for the Google user, returning an access token.
    #     """
    #     http = self.get_auth_http_client()
    #     body = urllib_parse.urlencode({
    #         "redirect_uri": redirect_uri,
    #         "code": code,
    #         "client_id": self.settings[self._OAUTH_SETTINGS_KEY]['key'],
    #         "client_secret": self.settings[self._OAUTH_SETTINGS_KEY]['secret'],
    #         "grant_type": "authorization_code",
    #     })
    #     #self.log.info('http req body: %r', body)
    #     #self.log.info('acc tok url: %r', self._OAUTH_ACCESS_TOKEN_URL)
    #     #self.log.info('callback url: %r', callback)
    #     http.fetch(self._OAUTH_ACCESS_TOKEN_URL,
    #                functools.partial(self._on_access_token, callback),
    #                method="POST", headers={'Content-Type': 'application/x-www-form-urlencoded'}, body=body, validate_cert = False)
    #
    #
    # def _on_access_token(self, future, response):
    #     """Callback function for the exchange to the access token."""
    #     #self.log.info('response body: %r', response)
    #     if response.error:
    #         future.set_exception(AuthError('OpenID auth error: %s' % str(response)))
    #         return
    #
    #     args = escape.json_decode(response.body)
    #     future.set_result(args)

class OpenIDLoginHandler(OAuthLoginHandler, OpenIDOAuth2Mixin):
    '''An OAuthLoginHandler that provides scope to GoogleOAuth2Mixin's
       authorize_redirect.'''
    scope=['openid','profile', 'email','offline_access','groups']
    def get(self):
        redirect_uri = self.authenticator.get_callback_url(self)
        self.log.info('redirect_uri: %r', redirect_uri)
        self.authorize_redirect(
            redirect_uri=redirect_uri,
            client_id=self.authenticator.client_id,
            scope=['openid','profile', 'email','offline_access','groups'],
            response_type='code')

class OpenIDOAuthHandler(OAuthCallbackHandler, OpenIDOAuth2Mixin):
    pass

class OpenIDOAuthenticator(OAuthenticator, OpenIDOAuth2Mixin):

    login_handler = OpenIDLoginHandler
    callback_handler = OpenIDOAuthHandler

    # hosted_domain = Unicode(
    #     os.environ.get('HOSTED_DOMAIN', ''),
    #     config=True,
    #     help="""Hosted domain used to restrict sign-in, e.g. mycollege.edu"""
    # )
    login_service = Unicode(
        os.environ.get('LOGIN_SERVICE', 'Dex'),
        config=True,
        help="""Google Apps hosted domain string, e.g. My College"""
    )

    @gen.coroutine
    def authenticate(self, handler, data=None):
        code = handler.get_argument('code')
        if not code:
            raise HTTPError(400, "oauth callback made without a token")
        handler.settings['google_oauth'] = {
            'key': self.client_id,
            'secret': self.client_secret,
            'scope': ['openid','profile', 'email','offline_access','groups'],
            'response_type': 'code'
        }

        self.log.debug('openid: settings: "%s"', str(handler.settings['google_oauth']))
        self.log.debug('code is: {}'.format(code))
        user = yield handler.get_authenticated_user(
            redirect_uri=self.get_callback_url(handler),
            code=code)
        access_token = str(user['access_token'])
        self.log.debug('token is: {}'.format(access_token))
        self.log.debug('full user json is: {}'.format(user))

        http_client = handler.get_auth_http_client()

        #response = yield http_client.fetch(
        #    self._OAUTH_USERINFO_URL + '?access_token=' + access_token
        #)

        #if not response:
        #    self.clear_all_cookies()
        #    raise HTTPError(500, 'Google authentication failed')

        #body = response.body.decode()
        #self.log.debug('response.body.decode(): {}'.format(body))
        #bodyjs = json.loads(body)
        payload_encoded = user['id_token'].split('.')[1]
        payload = urlsafe_b64decode(payload_encoded + '=' * (4 - len(payload_encoded) % 4)).decode('utf8')
        self.log.debug('urlsafe decoded payload is: {}'.format(payload))
        userstring = re.findall('(?<=sub":").+?(?=",)',payload)[0]
        substring = urlsafe_b64decode(userstring + '=' * (4 - len(userstring) % 4)).decode('utf8')

        substring_print = ''.join([i for i in substring if i.isprintable()])
        self.log.debug('urlsafe decoded, printable substring is: {}'.format(substring_print))

        username = ''

        for connector in self.CONNECTORS.split(','):
            if connector == 'github':
                if re.findall('(?<=name":").+?(?=")', payload):
                    returned_name = re.findall('(?<=name":").+?(?=")', payload)[0]
                    username = re.sub(' ','',returned_name) + '_' + 'github'
                else:
                    pass
            elif re.findall(connector,substring_print):
                username = re.sub(connector,'_' + connector,substring_print)
            else:
                pass

        if not username:
            raise Exception('Connector error: Could not extract username from id_token, sub or name entry.')

        #if self.hosted_domain:
        #    if not username.endswith('@'+self.hosted_domain) or \
        #        bodyjs['hd'] != self.hosted_domain:
        #        raise HTTPError(403,
        #            "You are not signed in to your {} account.".format(
        #                self.hosted_domain)
        #        )
        #    else:
        #        username = username.split('@')[0]

        return username

class LocalOpenIDOAuthenticator(LocalAuthenticator, OpenIDOAuthenticator):
    """A version that mixes in local system user creation"""
    pass
<commit_msg>increase logging, v2<commit_after>"""
Custom Authenticator to use Google OAuth with JupyterHub.

Derived from the GitHub OAuth authenticator.
"""

import os
import json

from base64 import b64decode, b64encode, urlsafe_b64decode
import re

from tornado             import gen, escape
from tornado.auth        import GoogleOAuth2Mixin
from tornado.web         import HTTPError

from traitlets           import Unicode

from jupyterhub.auth     import LocalAuthenticator
from jupyterhub.utils    import url_path_join

from .oauth2 import OAuthLoginHandler, OAuthCallbackHandler, OAuthenticator

from tornado.util import PY3
import functools

if PY3:
    import urllib.parse as urlparse
    import urllib.parse as urllib_parse
    long = int
else:
    import urlparse
    import urllib as urllib_parse

class AuthError(Exception):
    pass

class OpenIDOAuth2Mixin(GoogleOAuth2Mixin):
    """ An OpenID OAuth2 mixin to use GoogleLoginHandler with
    different Identity Providers using the OpenID standard. The current
    setup should work with MITREid Connect servers. In addtion to the usual
    parameters client ID and secret, the environment variable OPENID_HOST
    should be set to the URL of the OpenID provider. The API endpoints
    might have to be changed, depending on the ID provider."""

    CONNECTORS = os.environ.get('CONNECTOR_LIST')

    _OPENID_ENDPOINT = os.environ.get('OPENID_HOST')
    if _OPENID_ENDPOINT.startswith('http'):
        _OAUTH_AUTHORIZE_URL = "%s/auth" % _OPENID_ENDPOINT
        _OAUTH_ACCESS_TOKEN_URL = "%s/token" % _OPENID_ENDPOINT
        _OAUTH_USERINFO_URL = "%s/auth" % _OPENID_ENDPOINT
    else:
        _OAUTH_AUTHORIZE_URL = "https://%s/auth" % _OPENID_ENDPOINT
        _OAUTH_ACCESS_TOKEN_URL = "https://%s/token" % _OPENID_ENDPOINT
        _OAUTH_USERINFO_URL = "https://%s/auth" % _OPENID_ENDPOINT

    def get_authenticated_user(self, redirect_uri, code, callback):
        """Handles the login for the Google user, returning an access token.
        """
        http = self.get_auth_http_client()
        body = urllib_parse.urlencode({
            "redirect_uri": redirect_uri,
            "code": code,
            "client_id": self.settings[self._OAUTH_SETTINGS_KEY]['key'],
            "client_secret": self.settings[self._OAUTH_SETTINGS_KEY]['secret'],
            "grant_type": "authorization_code",
        })
        self.log.info('http req body: %r', body)
        self.log.info('acc tok url: %r', self._OAUTH_ACCESS_TOKEN_URL)
        self.log.info('callback url: %r', callback)
        http.fetch(self._OAUTH_ACCESS_TOKEN_URL,
                   functools.partial(self._on_access_token, callback),
                   method="POST", headers={'Content-Type': 'application/x-www-form-urlencoded'}, body=body, validate_cert = False)

    def _on_access_token(self, future, response):
        """Callback function for the exchange to the access token."""
        self.log.info('response body: %r', response)
        if response.error:
            future.set_exception(AuthError('OpenID auth error: %s' % str(response)))
            return

        args = escape.json_decode(response.body)
        future.set_result(args)

class OpenIDLoginHandler(OAuthLoginHandler, OpenIDOAuth2Mixin):
    '''An OAuthLoginHandler that provides scope to GoogleOAuth2Mixin's
       authorize_redirect.'''
    scope=['openid','profile', 'email','offline_access','groups']
    def get(self):
        redirect_uri = self.authenticator.get_callback_url(self)
        self.log.info('redirect_uri: %r', redirect_uri)
        self.authorize_redirect(
            redirect_uri=redirect_uri,
            client_id=self.authenticator.client_id,
            scope=['openid','profile', 'email','offline_access','groups'],
            response_type='code')

class OpenIDOAuthHandler(OAuthCallbackHandler, OpenIDOAuth2Mixin):
    pass

class OpenIDOAuthenticator(OAuthenticator, OpenIDOAuth2Mixin):

    login_handler = OpenIDLoginHandler
    callback_handler = OpenIDOAuthHandler

    # hosted_domain = Unicode(
    #     os.environ.get('HOSTED_DOMAIN', ''),
    #     config=True,
    #     help="""Hosted domain used to restrict sign-in, e.g. mycollege.edu"""
    # )
    login_service = Unicode(
        os.environ.get('LOGIN_SERVICE', 'Dex'),
        config=True,
        help="""Google Apps hosted domain string, e.g. My College"""
    )

    @gen.coroutine
    def authenticate(self, handler, data=None):
        code = handler.get_argument('code')
        if not code:
            raise HTTPError(400, "oauth callback made without a token")
        handler.settings['google_oauth'] = {
            'key': self.client_id,
            'secret': self.client_secret,
            'scope': ['openid','profile', 'email','offline_access','groups'],
            'response_type': 'code'
        }

        self.log.info('openid: settings: "%s"', str(handler.settings['google_oauth']))
        self.log.info('code is: {}'.format(code))
        user = yield handler.get_authenticated_user(
            redirect_uri=self.get_callback_url(handler),
            code=code)
        access_token = str(user['access_token'])
        self.log.info('token is: {}'.format(access_token))
        self.log.info('full user json is: {}'.format(user))

        http_client = handler.get_auth_http_client()

        #response = yield http_client.fetch(
        #    self._OAUTH_USERINFO_URL + '?access_token=' + access_token
        #)

        #if not response:
        #    self.clear_all_cookies()
        #    raise HTTPError(500, 'Google authentication failed')

        #body = response.body.decode()
        #self.log.debug('response.body.decode(): {}'.format(body))
        #bodyjs = json.loads(body)
        payload_encoded = user['id_token'].split('.')[1]
        payload = urlsafe_b64decode(payload_encoded + '=' * (4 - len(payload_encoded) % 4)).decode('utf8')
        self.log.info('urlsafe decoded payload is: {}'.format(payload))
        userstring = re.findall('(?<=sub":").+?(?=",)',payload)[0]
        substring = urlsafe_b64decode(userstring + '=' * (4 - len(userstring) % 4)).decode('utf8')

        substring_print = ''.join([i for i in substring if i.isprintable()])
        self.log.info('urlsafe decoded, printable substring is: {}'.format(substring_print))

        username = ''

        for connector in self.CONNECTORS.split(','):
            if connector == 'github':
                if re.findall('(?<=name":").+?(?=")', payload):
                    returned_name = re.findall('(?<=name":").+?(?=")', payload)[0]
                    username = re.sub(' ','',returned_name) + '_' + 'github'
                else:
                    pass
            elif re.findall(connector,substring_print):
                username = re.sub(connector,'_' + connector,substring_print)
            else:
                pass

        if not username:
            raise Exception('Connector error: Could not extract username from id_token, sub or name entry.')

        #if self.hosted_domain:
        #    if not username.endswith('@'+self.hosted_domain) or \
        #        bodyjs['hd'] != self.hosted_domain:
        #        raise HTTPError(403,
        #            "You are not signed in to your {} account.".format(
        #                self.hosted_domain)
        #        )
        #    else:
        #        username = username.split('@')[0]

        return username

class LocalOpenIDOAuthenticator(LocalAuthenticator, OpenIDOAuthenticator):
    """A version that mixes in local system user creation"""
    pass
<|endoftext|>


<commit_before># An implementation of Logistic Regression algorithm in python.
# The class use sparse representation of features.
# Author: Xiance Si (sixiance<at>gmail.com)

import math

class LogisticRegression:
    # Initialize member variables. We have two member variables
    # 1) weight: a dict obejct storing the weight of all features.
    # 2) bias: a float value of the bias value.
    def __init__(self):
        self.rate = 0.01
        self.weight = {}
        self.bias = 0
        self.alpha = 0.1
        return
    # data is a list of [label, feature]. label is an integer,
    # 1 for positive instance, 0 for negative instance. feature is
    # a dict object, the key is feature name, the value is feature
    # weight.
    #
    # n is the number of training iterations.
    #
    # We use online update formula to train the model.
    def train(self, data, n):
        old_update = 0
        for i in range(n):
            max_update = 0
            for [label, feature] in data:
                predicted = self.classify(feature)
                rate_n = self.rate - (self.rate * i)/float(n)

                for f,v in feature.iteritems():
                    if f not in self.weight:
                        self.weight[f] = 0
                    update = (label - predicted) * v - self.alpha * self.weight[f]
                    self.weight[f] += rate_n * update
                    if abs(update * self.rate) > max_update :
                        max_update = abs(update * rate_n)
                bias_update = (label - predicted)
                self.bias += rate_n * bias_update
            #print 'iteration', i, 'done. Max update:', max_update
            if max_update < .0001 : return
            #else : old_update = max_update
        return
    # feature is a dict object, the key is feature name, the value
    # is feature weight. Return value is the probability of being
    # a positive instance.
    def classify(self, feature):
        logit = self.bias
        for f,v in feature.iteritems():
            coef = 0
            if f in self.weight:
                coef = self.weight[f]
            logit += coef * v
        return 1.0 / (1.0 + math.exp(-logit))
<commit_msg>fiddled with regularization term<commit_after># An implementation of Logistic Regression algorithm in python.
# The class use sparse representation of features.
# Author: Xiance Si (sixiance<at>gmail.com)

import math

class LogisticRegression:
    # Initialize member variables. We have two member variables
    # 1) weight: a dict obejct storing the weight of all features.
    # 2) bias: a float value of the bias value.
    def __init__(self):
        self.rate = 0.01
        self.weight = {}
        self.bias = 0
        self.alpha = 0.001
        return
    # data is a list of [label, feature]. label is an integer,
    # 1 for positive instance, 0 for negative instance. feature is
    # a dict object, the key is feature name, the value is feature
    # weight.
    #
    # n is the number of training iterations.
    #
    # We use online update formula to train the model.
    def train(self, data, n):
        old_update = 0
        for i in range(n):
            max_update = 0
            for [label, feature] in data:
                predicted = self.classify(feature)
                rate_n = self.rate - (self.rate * i)/float(n)

                for f,v in feature.iteritems():
                    if f not in self.weight:
                        self.weight[f] = 0
                    update = (label - predicted) * v - self.alpha * self.weight[f]
                    self.weight[f] += rate_n * update
                    if abs(update * self.rate) > max_update :
                        max_update = abs(update * rate_n)
                bias_update = (label - predicted)
                self.bias += rate_n * bias_update
            #print 'iteration', i, 'done. Max update:', max_update
            if max_update < .0001 : return
            #else : old_update = max_update
        return
    # feature is a dict object, the key is feature name, the value
    # is feature weight. Return value is the probability of being
    # a positive instance.
    def classify(self, feature):
        logit = self.bias
        for f,v in feature.iteritems():
            coef = 0
            if f in self.weight:
                coef = self.weight[f]
            logit += coef * v
        return 1.0 / (1.0 + math.exp(-logit))
<|endoftext|>


<commit_before>import sys
import logging
import os.path
import argparse

from deen.loader import DeenPluginLoader
from deen import constants

LOGGER = logging.getLogger()

class DeenHelpFormatter(argparse.RawDescriptionHelpFormatter):
    """This formatter is a hack for printing the default
    metavar for the plugin_cmd subparsers. It will prevent
    argparse from printing additional lists of all cmds."""
    def _metavar_formatter(self, action, default_metavar):
        if action.metavar is not None:
            result = action.metavar
        else:
            result = default_metavar

        def format_metavar(tuple_size):
            if isinstance(result, tuple):
                return result
            else:
                return (result, ) * tuple_size
        return format_metavar

ARGS = argparse.ArgumentParser(formatter_class=DeenHelpFormatter,
                               description=constants.cli_description,
                               epilog=constants.cli_epilog)
ARGS.add_argument('-f', '--file', dest='infile', default=None, metavar='filename',
                  help='file name or - for STDIN')
ARGS.add_argument('-l', '--list', action='store_true', dest='list',
                  default=False, help='list available plugins')
ARGS.add_argument('-p', '--plugin', action='store', dest='plugin',
                  metavar='plugin', default=None, help='deen plugin to use')
ARGS.add_argument('-r', '--revert', action='store_true', dest='revert',
                  default=False, help='revert plugin process (e.g. decode or uncompress)')
ARGS.add_argument('-d', '--data', action='store', dest='data', metavar='data',
                  default=None, help='instead of a file, provide an input string')
ARGS.add_argument('-n', '--no-new-line', action='store_true', dest='nonewline',
                  default=False, help='omit new line character at the end of the output')
ARGS.add_argument('--version', action='store_true', dest='version',
                  default=False, help='print the current version')
ARGS.add_argument('-v', '--verbose', action='count', dest='level',
                  default=0, help='verbose logging (repeat for more verbosity)')

def main():
    pl = DeenPluginLoader(argparser=ARGS)
    args = ARGS.parse_args()
    content = pl.read_content_from_args()
    if args.list:
        print('Loaded plugins:')
        print(pl.pprint_available_plugins())
        if pl.invalid_plugins:
            print('\nNot loaded plugins:')
            print(pl.pprint_invalid_plugins())
    elif args.version:
        print(constants.__version__)
    elif any([args.plugin_cmd, args.plugin]):
        # We are in command line mode
        log_format = constants.verbose_log_format if args.level > 0 else '%(message)s'
        levels = [logging.WARN, logging.DEBUG]
        logging.basicConfig(level=levels[min(args.level, len(levels) - 1)], format=log_format)
        if args.plugin_cmd:
            # Run subcommands
            if not pl.get_plugin_cmd_available(args.plugin_cmd):
                LOGGER.error('Plugin cmd not available')
                sys.exit(1)
            plugin = pl.get_plugin_cmd_name_instance(args.plugin_cmd)
            plugin.content = content
            processed = plugin.process_cli(args)
        else:
            # Use plugins via -p/--plugin
            if not pl.plugin_available(args.plugin):
                LOGGER.error('Plugin not available')
                sys.exit(1)
            plugin = pl.get_plugin_instance(args.plugin)
            if not args.revert:
                processed = plugin.process(content)
            else:
                unprocess_func = getattr(plugin, 'unprocess', None)
                if not unprocess_func or not callable(unprocess_func):
                    LOGGER.error('Plugin cannot unprocess data')
                    sys.exit(1)
                processed = plugin.unprocess(content)
        if not processed:
            if plugin.error:
                LOGGER.error(plugin.error)
            else:
                LOGGER.debug('Plugin {} did not return any data'.format(plugin.cmd_name))
            sys.exit(1)
        plugin.write_to_stdout(processed, nonewline=args.nonewline)
    else:
        # We are in GUI mode
        # Import GUI related modules only in GUI
        # mode to speed up CLI mode.
        from PyQt5.QtWidgets import QApplication
        from PyQt5.QtGui import QIcon
        from deen.gui.core import DeenGui
        logging.basicConfig(format=constants.verbose_log_format)
        app = QApplication(sys.argv)
        ex = DeenGui(plugins=pl)
        if content:
            # GUI mode also supports input files and
            # content via STDIN.
            ex.set_root_content(content)
        ex.setWindowIcon(QIcon(os.path.dirname(os.path.abspath(__file__)) +
                               constants.icon_path))
        LOGGER.addHandler(ex.log)
        return app.exec_()
<commit_msg>Log not loaded plugins in GUI<commit_after>import sys
import logging
import os.path
import argparse

from deen.loader import DeenPluginLoader
from deen import constants

LOGGER = logging.getLogger()

class DeenHelpFormatter(argparse.RawDescriptionHelpFormatter):
    """This formatter is a hack for printing the default
    metavar for the plugin_cmd subparsers. It will prevent
    argparse from printing additional lists of all cmds."""
    def _metavar_formatter(self, action, default_metavar):
        if action.metavar is not None:
            result = action.metavar
        else:
            result = default_metavar

        def format_metavar(tuple_size):
            if isinstance(result, tuple):
                return result
            else:
                return (result, ) * tuple_size
        return format_metavar

ARGS = argparse.ArgumentParser(formatter_class=DeenHelpFormatter,
                               description=constants.cli_description,
                               epilog=constants.cli_epilog)
ARGS.add_argument('-f', '--file', dest='infile', default=None, metavar='filename',
                  help='file name or - for STDIN')
ARGS.add_argument('-l', '--list', action='store_true', dest='list',
                  default=False, help='list available plugins')
ARGS.add_argument('-p', '--plugin', action='store', dest='plugin',
                  metavar='plugin', default=None, help='deen plugin to use')
ARGS.add_argument('-r', '--revert', action='store_true', dest='revert',
                  default=False, help='revert plugin process (e.g. decode or uncompress)')
ARGS.add_argument('-d', '--data', action='store', dest='data', metavar='data',
                  default=None, help='instead of a file, provide an input string')
ARGS.add_argument('-n', '--no-new-line', action='store_true', dest='nonewline',
                  default=False, help='omit new line character at the end of the output')
ARGS.add_argument('--version', action='store_true', dest='version',
                  default=False, help='print the current version')
ARGS.add_argument('-v', '--verbose', action='count', dest='level',
                  default=0, help='verbose logging (repeat for more verbosity)')

def main():
    pl = DeenPluginLoader(argparser=ARGS)
    args = ARGS.parse_args()
    content = pl.read_content_from_args()
    if args.list:
        print('Loaded plugins:')
        print(pl.pprint_available_plugins())
        if pl.invalid_plugins:
            print('\nNot loaded plugins:')
            print(pl.pprint_invalid_plugins())
    elif args.version:
        print(constants.__version__)
    elif any([args.plugin_cmd, args.plugin]):
        # We are in command line mode
        log_format = constants.verbose_log_format if args.level > 0 else '%(message)s'
        levels = [logging.WARN, logging.DEBUG]
        logging.basicConfig(level=levels[min(args.level, len(levels) - 1)], format=log_format)
        if args.plugin_cmd:
            # Run subcommands
            if not pl.get_plugin_cmd_available(args.plugin_cmd):
                LOGGER.error('Plugin cmd not available')
                sys.exit(1)
            plugin = pl.get_plugin_cmd_name_instance(args.plugin_cmd)
            plugin.content = content
            processed = plugin.process_cli(args)
        else:
            # Use plugins via -p/--plugin
            if not pl.plugin_available(args.plugin):
                LOGGER.error('Plugin not available')
                sys.exit(1)
            plugin = pl.get_plugin_instance(args.plugin)
            if not args.revert:
                processed = plugin.process(content)
            else:
                unprocess_func = getattr(plugin, 'unprocess', None)
                if not unprocess_func or not callable(unprocess_func):
                    LOGGER.error('Plugin cannot unprocess data')
                    sys.exit(1)
                processed = plugin.unprocess(content)
        if not processed:
            if plugin.error:
                LOGGER.error(plugin.error)
            else:
                LOGGER.debug('Plugin {} did not return any data'.format(plugin.cmd_name))
            sys.exit(1)
        plugin.write_to_stdout(processed, nonewline=args.nonewline)
    else:
        # We are in GUI mode
        # Import GUI related modules only in GUI
        # mode to speed up CLI mode.
        from PyQt5.QtWidgets import QApplication
        from PyQt5.QtGui import QIcon
        from deen.gui.core import DeenGui
        logging.basicConfig(format=constants.verbose_log_format)
        app = QApplication(sys.argv)
        ex = DeenGui(plugins=pl)
        if content:
            # GUI mode also supports input files and
            # content via STDIN.
            ex.set_root_content(content)
        ex.setWindowIcon(QIcon(os.path.dirname(os.path.abspath(__file__)) +
                               constants.icon_path))
        LOGGER.addHandler(ex.log)
        if pl.invalid_plugins:
            LOGGER.warning('Not loaded plugins:\n' + pl.pprint_invalid_plugins())
        return app.exec_()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Last modified: Wang Tai (i@wangtai.me)

"""
url注解
"""

__revision__ = '0.1'

from url_refactor import Type, RequestMethod, M
from url_refactor import login_required, get, post, url<commit_msg>update 请求默认是get or post<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Last modified: Wang Tai (i@wangtai.me)

"""
url注解
"""

__revision__ = '0.1'

from url_refactor import Type, RequestMethod, M
from url_refactor import login_required, get, post, url, param<|endoftext|>


<commit_before>#!/usr/bin/env python
from django.core.management import execute_manager
try:
    import settings # Assumed to be in the same directory.
except ImportError:
    import sys
    sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
    sys.exit(1)

if __name__ == "__main__":
    execute_manager(settings)
<commit_msg>Executable manage.py<commit_after>#!/usr/bin/env python
from django.core.management import execute_manager
try:
    import settings # Assumed to be in the same directory.
except ImportError:
    import sys
    sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
    sys.exit(1)

if __name__ == "__main__":
    execute_manager(settings)
<|endoftext|>


<commit_before>#!/usr/bin/env python2

__author__="The Font Bakery Authors"

import os, sys, argparse, glob, logging

from fontTools import ttLib

def main():
    # set up a basic logging config
    # to include timestamps
    # log_format = '%(asctime)s %(levelname)-8s %(message)s'
    log_format = '%(levelname)-8s %(message)s  '
    logger = logging.getLogger()
    handler = logging.StreamHandler()
    formatter = logging.Formatter(log_format)
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    # to show all log events
    # logger.setLevel(logging.DEBUG)
    logger.setLevel(logging.INFO)

    # set up some command line argument processing
    parser = argparse.ArgumentParser(description="Check TTF files for common issues.")
    parser.add_argument('filenames', nargs='+',
      help='file name(s) of fonts to check. Wildcards like *.ttf are allowed.')
    args = parser.parse_args()

    logging.debug("Checking each file is a ttfs")
    fonts_to_check = []
    for filename in sorted(args.filenames):
      # use glob.glob to accept *.ttf
      for font_file in glob.glob(filename):
        if font_file.endswith('.ttf'):
          fonts_to_check.append(font_file)
        else:
          file_path, filename = os.path.split(font_file)
          logging.error("Skipping " + filename + " as not a ttf")
    fonts_to_check.sort()

    logging.debug("Checking files are named canonically")
    not_canonical = []
    for font_file in fonts_to_check:
      file_path, filename = os.path.split(font_file)
      filename_base, filename_extention = os.path.splitext(filename)
      family, style = filename_base.split('-')
      style_names = ["Thin", "ExtraLight", "Light", "Regular", "Medium",
               "SemiBold", "Bold", "ExtraBold", "Black",
               "Thin Italic", "ExtraLight Italic", "Light Italic",
               "Italic", "Medium Italic", "SemiBold Italic",
               "Bold Italic", "ExtraBold Italic", "Black Italic"]
      # remove spaces
      style_file_names = [name.replace(' ', '') for name in style_names]
      if style in style_file_names:
        logging.info(font_file + " is named canonically")
      else:
          logging.critical(font_file + " is not named canonically")
          not_canonical.append(font_file)
    if not_canonical:
      print '\nAborted, critical errors. Please rename these files canonically and try again:\n ',
      print '\n  '.join(not_canonical)
      print '\nCanonical names are defined in',
      print 'https://github.com/googlefonts/gf-docs/blob/master/ProjectChecklist.md#instance-and-file-naming'
      sys.exit(1)

    for font_file in fonts_to_check:
      check(font_file)

def check(font_file):
    logging.debug("Opening " + font_file)
    font = ttLib.TTFont(font_file)
    logging.info(font_file + " opened")

    # OS/2 Table Checks
    logging.debug("Checking OS/2 fsType")
    if font['OS/2'].fsType != 0:
      logging.warning("fsType is not 0")
      font['OS/2'].fsType = 0
      logging.info("HOTFIX: fsType is now 0")
    else:
      logging.info("fsType is 0")

    logging.info("TODO: Check fsSelection")
    logging.info("TODO: Check head table")
    logging.info("TODO: Check name table")

    logging.debug("Checking name table for items without platformID=1")
    new_names = []
    for name in font['name'].names:
      if name.platformID != 1:
        non_pid1 = True
        new_names.append(name)
    if non_pid1:
      logging.error("name table for items without platformID=1 exist")
      font['name'].names = new_names
      logging.error("HOTFIX: name table items with platformID=1 removed")
    else:
      logging.info("No name table for items without platformID=1")

    # more checks go above this line

    font_file_output = font_file + '.fix'
    font.save(font_file_output)
    font.close()
    logging.info(font_file_output + " saved")

if __name__=='__main__':
    main()<commit_msg>Minor: Use log items instead of code comments<commit_after>#!/usr/bin/env python2

__author__="The Font Bakery Authors"

import os, sys, argparse, glob, logging

from fontTools import ttLib

def main():
    # set up a basic logging config
    # to include timestamps
    # log_format = '%(asctime)s %(levelname)-8s %(message)s'
    log_format = '%(levelname)-8s %(message)s  '
    logger = logging.getLogger()
    handler = logging.StreamHandler()
    formatter = logging.Formatter(log_format)
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    # to show all log events
    # logger.setLevel(logging.DEBUG)
    logger.setLevel(logging.INFO)

    # set up some command line argument processing
    parser = argparse.ArgumentParser(description="Check TTF files for common issues.")
    parser.add_argument('filenames', nargs='+',
      help='file name(s) of fonts to check. Wildcards like *.ttf are allowed.')
    args = parser.parse_args()

    logging.debug("Checking each file is a ttfs")
    fonts_to_check = []
    for filename in sorted(args.filenames):
      # use glob.glob to accept *.ttf
      for font_file in glob.glob(filename):
        if font_file.endswith('.ttf'):
          fonts_to_check.append(font_file)
        else:
          file_path, filename = os.path.split(font_file)
          logging.error("Skipping " + filename + " as not a ttf")
    fonts_to_check.sort()

    logging.debug("Checking files are named canonically")
    not_canonical = []
    for font_file in fonts_to_check:
      file_path, filename = os.path.split(font_file)
      filename_base, filename_extention = os.path.splitext(filename)
      family, style = filename_base.split('-')
      style_names = ["Thin", "ExtraLight", "Light", "Regular", "Medium",
               "SemiBold", "Bold", "ExtraBold", "Black",
               "Thin Italic", "ExtraLight Italic", "Light Italic",
               "Italic", "Medium Italic", "SemiBold Italic",
               "Bold Italic", "ExtraBold Italic", "Black Italic"]
      # remove spaces
      style_file_names = [name.replace(' ', '') for name in style_names]
      if style in style_file_names:
        logging.info(font_file + " is named canonically")
      else:
          logging.critical(font_file + " is not named canonically")
          not_canonical.append(font_file)
    if not_canonical:
      print '\nAborted, critical errors. Please rename these files canonically and try again:\n ',
      print '\n  '.join(not_canonical)
      print '\nCanonical names are defined in',
      print 'https://github.com/googlefonts/gf-docs/blob/master/ProjectChecklist.md#instance-and-file-naming'
      sys.exit(1)

    for font_file in fonts_to_check:
      check(font_file)

def check(font_file):
    logging.debug("Opening " + font_file)
    font = ttLib.TTFont(font_file)
    logging.info(font_file + " opened")

    logging.debug("Checking OS/2 table")
    if font['OS/2'].fsType != 0:
      logging.warning("fsType is not 0")
      font['OS/2'].fsType = 0
      logging.info("HOTFIX: fsType is now 0")
    else:
      logging.info("fsType is 0")

    logging.info("TODO: Check fsSelection")
    logging.info("TODO: Check head table")
    logging.info("TODO: Check name table")

    logging.debug("Checking name table for items without platformID=1")
    new_names = []
    for name in font['name'].names:
      if name.platformID != 1:
        non_pid1 = True
        new_names.append(name)
    if non_pid1:
      logging.error("name table for items without platformID=1 exist")
      font['name'].names = new_names
      logging.error("HOTFIX: name table items with platformID=1 removed")
    else:
      logging.info("No name table for items without platformID=1")

    # more checks go above this line

    font_file_output = font_file + '.fix'
    font.save(font_file_output)
    font.close()
    logging.info(font_file_output + " saved")

if __name__=='__main__':
    main()<|endoftext|>


<commit_before>__author__ = 'calvin'

from .tools import *

logger = logging.getLogger('AnonymousUsage')

class Table(object):
    time_fmt = "%d/%m/%Y %H:%M:%S"
    table_args = ("UUID", "INT"), ("Count", "INT"), ("Time", "TEXT")

    def __init__(self, name, tracker, *args, **kwargs):
        self.tracker = tracker
        self.dbcon = self.tracker.dbcon
        self.name = name

        self.count = self.get_number_of_rows()
        if not check_table_exists(self.dbcon, name):
            create_table(self.dbcon, name, self.table_args)

    def get_number_of_rows(self):
        """
        Attempt to load the statistic from the database.
        :return: Number of entries for the statistic
        """
        rows = []
        if check_table_exists(self.tracker.dbcon_master, self.name):
            rows.extend(get_rows(self.tracker.dbcon_master, self.name))
        if check_table_exists(self.tracker.dbcon_part, self.name):
            rows.extend(get_rows(self.tracker.dbcon_part, self.name))

        logger.debug("{name}: {n} table entries found".format(name=self.name,
                                                                  n=len(rows),
                                                                  rows='\n\t'.join(map(str, rows))))
        return len(rows)

    def insert(self, value):
        """
        Contains the functionally of assigning a value to a statistic in the AnonymousUsageTracker. Usually this will
        involve inserting some data into the database table for the statistic.
        :param value: assignment value to the tracker, ie. `tracker[stat_name] = some_value`
        """
        pass

    def get_last(self, n):
        """
        Retrieve the last n rows from the table
        :param n: number of rows to return
        :return: list of rows
        """
        rows = []
        # Get values from the partial db first
        if check_table_exists(self.tracker.dbcon_part, self.name):
            cur = self.tracker.dbcon_part.cursor()
            # cur.execute("SELECT * FROM %s" % self.name)
            cur.execute("SELECT * FROM %s ORDER BY Count DESC LIMIT %d;" % (self.name, n))
            rows.extend(cur.fetchall())
        # Then add rows from the master if required
        if len(rows) < n and check_table_exists(self.tracker.dbcon_master, self.name):
            cur = self.tracker.dbcon_master.cursor()
            # cur.execute("SELECT * FROM %s" % self.name)
            cur.execute("SELECT * FROM %s ORDER BY Count DESC LIMIT %d;" % (self.name, n))
            rows.extend(cur.fetchall())

        return rows
<commit_msg>fix to get_last_n()<commit_after>__author__ = 'calvin'

from .tools import *

logger = logging.getLogger('AnonymousUsage')

class Table(object):
    time_fmt = "%d/%m/%Y %H:%M:%S"
    table_args = ("UUID", "INT"), ("Count", "INT"), ("Time", "TEXT")

    def __init__(self, name, tracker, *args, **kwargs):
        self.tracker = tracker
        self.dbcon = self.tracker.dbcon
        self.name = name

        self.count = self.get_number_of_rows()
        if not check_table_exists(self.dbcon, name):
            create_table(self.dbcon, name, self.table_args)

    def get_number_of_rows(self):
        """
        Attempt to load the statistic from the database.
        :return: Number of entries for the statistic
        """
        rows = []
        if check_table_exists(self.tracker.dbcon_master, self.name):
            rows.extend(get_rows(self.tracker.dbcon_master, self.name))
        if check_table_exists(self.tracker.dbcon_part, self.name):
            rows.extend(get_rows(self.tracker.dbcon_part, self.name))

        logger.debug("{name}: {n} table entries found".format(name=self.name,
                                                                  n=len(rows),
                                                                  rows='\n\t'.join(map(str, rows))))
        return len(rows)

    def insert(self, value):
        """
        Contains the functionally of assigning a value to a statistic in the AnonymousUsageTracker. Usually this will
        involve inserting some data into the database table for the statistic.
        :param value: assignment value to the tracker, ie. `tracker[stat_name] = some_value`
        """
        pass

    def get_last(self, n):
        """
        Retrieve the last n rows from the table
        :param n: number of rows to return
        :return: list of rows
        """
        rows = []
        # Get values from the partial db first
        if check_table_exists(self.tracker.dbcon_part, self.name):
            cur = self.tracker.dbcon_part.cursor()
            cur.execute("SELECT * FROM %s ORDER BY Count DESC LIMIT %d;" % (self.name, n))
            rows.extend(cur.fetchall())
        # Then add rows from the master if required
        if len(rows) < n and check_table_exists(self.tracker.dbcon_master, self.name):
            cur = self.tracker.dbcon_master.cursor()
            # cur.execute("SELECT * FROM %s" % self.name)
            cur.execute("SELECT * FROM %s ORDER BY Count DESC LIMIT %d;" % (self.name, n))
            rows.extend(cur.fetchall())

        return rows[-n:]
<|endoftext|>


<commit_before>from __future__ import absolute_import

import numpy as np
import warnings
from .base import DifferentiableModel
from .. import utils

class CaffeModel(DifferentiableModel):
    def __init__(self,
                 net,
                 bounds,
                 channel_axis=1,
                 preprocessing=(0, 1),
                 data_blob_name="data",
                 label_blob_name="label",
                 output_blob_name="output"):
        super(CaffeModel, self).__init__(bounds=bounds,
                                         channel_axis=channel_axis,
                                         preprocessing=preprocessing)

        warnings.warn('Caffe was superseeded by Caffe2 and now PyTorch 1.0,'
                      ' thus Caffe support in Foolbox will be removed',
                      DeprecationWarning)

        import caffe
        self.net = net
        assert isinstance(net, caffe.Net)
        assert data_blob_name in self.net.blobs
        assert label_blob_name in self.net.blobs
        self.data_blob_name = data_blob_name
        self.label_blob_name = label_blob_name
        self.output_blob_name = output_blob_name

    def num_classes(self):
        return self.net.blobs[self.output_blob_name].data.shape[-1]

    def batch_predictions(self, images):
        images, _ = self._process_input(images)
        self.net.blobs[self.data_blob_name].reshape(*images.shape)
        self.net.blobs[self.label_blob_name].reshape(images.shape[0])
        self.net.blobs[self.data_blob_name].data[:] = images
        self.net.forward()
        return self.net.blobs[self.output_blob_name].data

    def predictions_and_gradient(self, image, label):
        input_shape = image.shape

        image, dpdx = self._process_input(image)
        self.net.blobs[self.data_blob_name].data[0, :] = image
        self.net.blobs[self.label_blob_name].data[0] = label

        self.net.forward()
        predictions = self.net.blobs[self.output_blob_name].data[0]

        grad_data = self.net.backward(diffs=[self.data_blob_name])
        grad = grad_data[self.data_blob_name][0]
        grad = self._process_gradient(dpdx, grad)
        assert grad.shape == input_shape

        return predictions, grad

    def batch_gradients(self, images, labels):
        if images.shape[0] == labels.shape[0] == 1:
            return self.gradient(images[0], labels[0])[np.newaxis]
        raise NotImplementedError

    def _loss_fn(self, image, label):
        logits = self.batch_predictions(image[None])
        return utils.batch_crossentropy([label], logits)

    def backward(self, gradient, image):
        input_shape = image.shape
        image, dpdx = self._process_input(image)
        self.net.blobs[self.data_blob_name].data[:] = image
        self.net.forward()
        self.net.blobs[self.output_blob_name].diff[...] = gradient
        grad_data = self.net.backward(start=self.output_blob_name,
                                      diffs=[self.data_blob_name])
        grad = grad_data[self.data_blob_name][0]
        grad = self._process_gradient(dpdx, grad)
        assert grad.shape == input_shape

        return grad

    def batch_backward(self, gradients, images):
        if images.shape[0] == gradients.shape[0] == 1:
            return self.backward(gradients[0], images[0])[np.newaxis]
        raise NotImplementedError
<commit_msg>avoid infinite recursion in caffe model<commit_after>from __future__ import absolute_import

import numpy as np
import warnings
from .base import DifferentiableModel
from .. import utils

class CaffeModel(DifferentiableModel):
    def __init__(self,
                 net,
                 bounds,
                 channel_axis=1,
                 preprocessing=(0, 1),
                 data_blob_name="data",
                 label_blob_name="label",
                 output_blob_name="output"):
        super(CaffeModel, self).__init__(bounds=bounds,
                                         channel_axis=channel_axis,
                                         preprocessing=preprocessing)

        warnings.warn('Caffe was superseeded by Caffe2 and now PyTorch 1.0,'
                      ' thus Caffe support in Foolbox will be removed',
                      DeprecationWarning)

        import caffe
        self.net = net
        assert isinstance(net, caffe.Net)
        assert data_blob_name in self.net.blobs
        assert label_blob_name in self.net.blobs
        self.data_blob_name = data_blob_name
        self.label_blob_name = label_blob_name
        self.output_blob_name = output_blob_name

    def num_classes(self):
        return self.net.blobs[self.output_blob_name].data.shape[-1]

    def batch_predictions(self, images):
        images, _ = self._process_input(images)
        self.net.blobs[self.data_blob_name].reshape(*images.shape)
        self.net.blobs[self.label_blob_name].reshape(images.shape[0])
        self.net.blobs[self.data_blob_name].data[:] = images
        self.net.forward()
        return self.net.blobs[self.output_blob_name].data

    def predictions_and_gradient(self, image, label):
        input_shape = image.shape

        image, dpdx = self._process_input(image)
        self.net.blobs[self.data_blob_name].data[0, :] = image
        self.net.blobs[self.label_blob_name].data[0] = label

        self.net.forward()
        predictions = self.net.blobs[self.output_blob_name].data[0]

        grad_data = self.net.backward(diffs=[self.data_blob_name])
        grad = grad_data[self.data_blob_name][0]
        grad = self._process_gradient(dpdx, grad)
        assert grad.shape == input_shape

        return predictions, grad

    def batch_gradients(self, images, labels):
        if images.shape[0] == labels.shape[0] == 1:
            _, g = self.predictions_and_gradient(images[0], labels[0])
            return g[np.newaxis]
        raise NotImplementedError

    def _loss_fn(self, image, label):
        logits = self.batch_predictions(image[None])
        return utils.batch_crossentropy([label], logits)

    def backward(self, gradient, image):
        input_shape = image.shape
        image, dpdx = self._process_input(image)
        self.net.blobs[self.data_blob_name].data[:] = image
        self.net.forward()
        self.net.blobs[self.output_blob_name].diff[...] = gradient
        grad_data = self.net.backward(start=self.output_blob_name,
                                      diffs=[self.data_blob_name])
        grad = grad_data[self.data_blob_name][0]
        grad = self._process_gradient(dpdx, grad)
        assert grad.shape == input_shape

        return grad

    def batch_backward(self, gradients, images):
        if images.shape[0] == gradients.shape[0] == 1:
            return self.backward(gradients[0], images[0])[np.newaxis]
        raise NotImplementedError
<|endoftext|>


<commit_before>import cv2
import numpy as np

import segment

VIDEO = "data/fish_video.mp4"

class Fish(object):

    def __init__(self, control_pts, contour):
        self.control_pts = control_pts
        self.contour = contour

    def in_region(self, contour):
        hits = 0
        for (x, y) in self.control_pts:
            if cv2.pointPolygonTest(contour, (x, y), False) >= 0:
                hits += 1

        # do majority vote of points
        return hits > len(self.control_pts) / 2

class MergedFish(Fish):

    def __init__(self, control_pts, contour, children):
        super(MergedFish, self).__init__(control_pts, contour)
        self.children = children

def draw_flow(im,flow,step=16):
    """ Plot optical flow at sample points spaced step pixels apart. """
    h,w = im.shape[:2]
    y,x = np.mgrid[step/2:h:step,step/2:w:step].reshape(2,-1)
    fx,fy = flow[y,x].T

    # create line endpoints
    lines = np.vstack([x,y,x+fx,y+fy]).T.reshape(-1,2,2)
    lines = np.int32(lines)

    lines = [((x1,y1),(x2,y2)) for ((x1,y1),(x2,y2)) in lines if abs(x1-x2) + abs(y1-y2) > 1 ]

    # create image and draw
    vis = cv2.cvtColor(im,cv2.COLOR_GRAY2BGR)
    for (x1,y1),(x2,y2) in lines:
        cv2.line(vis,(x1,y1),(x2,y2),(0,255,0),1)
        cv2.circle(vis,(x1,y1),1,(0,255,0), -1)

    return vis

def segment_by_velocity(flow, l_thresh=1.5, min_frame_portion=0.025):
    mag = np.sum(np.fabs(flow), axis=2)
    _, magbin = cv2.threshold(mag, l_thresh, 255, cv2.THRESH_BINARY)
    magbin = magbin.astype(np.uint8)

    # Apply morphological closing to combine pieces of same fish
    kernel = cv2.getStructuringElement(cv2.MORPH_CROSS, (35, 35))
    magbin = cv2.morphologyEx(magbin, cv2.MORPH_CLOSE, kernel)

    contours, _ = cv2.findContours(magbin, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

    frameshape = np.shape(flow)
    framesize = frameshape[0] * frameshape[1]

    large_contours = []
    for contour in contours:
        area = cv2.contourArea(contour)
        if area > min_frame_portion * framesize:
            large_contours.append(contour)

    return large_contours

def get_control_pts(img, contour):
    x, y, width, height = cv2.boundingRect(contour)

    sub_img = img[y:y+height, x:x+width]
    features = cv2.goodFeaturesToTrack(sub_img, 5, 0.5, 1)

    if features is None:
        return None

    # Get the points back in the coordinates of the original image
    # while also filtering out those which are not within the contour
    norm_pts = []
    for pt in features:
        norm_pt = (int(pt[0][0]) + x, int(pt[0][1]) + y)
        if cv2.pointPolygonTest(contour, norm_pt, False) >= 0:
            norm_pts.append(norm_pt)

    if len(norm_pts) == 0:
        return None

    return np.array(norm_pts, dtype=np.float32)

def draw_debug_frame(img, fishes, current_contours):
    debug_img = img.copy()

    for fish in fishes:
        for (x, y) in fish.control_pts:
            cv2.circle(debug_img, (int(x), int(y)), 7, (0, 0, 255))

    cv2.drawContours(debug_img, contours, -1, 255)

    cv2.imshow("Debug", debug_img)

# setup video capture
cap = cv2.VideoCapture(VIDEO)
ret, img = cap.read()

prev_gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

skip = 1150

fishes = []

while True:
    ret, img = cap.read()
    skip -= 1

    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

    # Update fish control points
#    import pdb; pdb.set_trace()
    for fish in fishes:
        fish.control_pts, _, _ = cv2.calcOpticalFlowPyrLK(prev_gray, gray, fish.control_pts, None)

    # compute flow
    flow = cv2.calcOpticalFlowFarneback(prev_gray, gray, None,
                    0.5, # pyramid scale - each layer is half previous size
                    3, # number of pyramid levels
                    35, # averaging window size
                    3, # iterations at each pyramid level
                    5, # size of the pixel neighbourhood used to find
                       # polynomial expansion in each pixel
                    1.1, # standard deviation of the Gaussian that is used to
                         # smooth derivatives used as a basis for the
                         # polynomial expansion
                    0 # additional flags
                    )

    contours = segment_by_velocity(flow)
#    contours = segment.segment_by_velocity(flow, 0.5, 1.5)

    # Find contours that are new fishes
    untracked_contours = []

    fishes_to_add = []
    fishes_to_delete = []
    for contour in contours:
        fishes_in_contour = []
        for fish in fishes:
            if fish.in_region(contour):
                # Update fish contour
                fish.contour = contour
                fishes_in_contour.append(fish)

        # Fish has not been tracked
        if not fishes_in_contour:
            control_pts = get_control_pts(gray, contour)
            if control_pts is not None:
                fishes_to_add.append(Fish(control_pts, contour))

        # Multiple fish have overlapped
        elif len(fishes_in_contour) >= 1:
            control_pts = get_control_pts(gray, contour)
            if control_pts is not None:
                fishes_to_delete.extend(fishes_in_contour)
                merged_fish = MergedFish(control_pts, contour, fishes_in_contour)
                fishes_to_add.append(merged_fish)

    fishes = [f for f in fishes if f not in fishes_to_delete]
    fishes.extend(fishes_to_add)

    prev_gray = gray

    print skip
    if skip < 0:
        draw_debug_frame(img, fishes, contours)
        print "Number of fish: %d" % len(fishes)

    if cv2.waitKey(10) == 27:
        break

<commit_msg>Split previously merged fish<commit_after>import cv2
import numpy as np

import segment

VIDEO = "data/fish_video.mp4"

class Fish(object):

    def __init__(self, control_pts, contour):
        self.control_pts = control_pts
        self.contour = contour
        self.prev_contour = None

    def update_contour(self, new_contour):
        self.prev_contour = self.contour
        self.contour = new_contour

    def change_in_area(self):
        if self.prev_contour is None:
            return cv2.contourArea(self.contour)
        else:
            return cv2.contourArea(self.contour) - cv2.contourArea(self.prev_contour)

    def in_region(self, contour):
        hits = 0
        for (x, y) in self.control_pts:
            if cv2.pointPolygonTest(contour, (x, y), False) >= 0:
                hits += 1

        # do majority vote of points
        return hits > len(self.control_pts) / 2

class MergedFish(Fish):

    def __init__(self, control_pts, contour, children):
        super(MergedFish, self).__init__(control_pts, contour)
        self.children = children

def draw_flow(im,flow,step=16):
    """ Plot optical flow at sample points spaced step pixels apart. """
    h,w = im.shape[:2]
    y,x = np.mgrid[step/2:h:step,step/2:w:step].reshape(2,-1)
    fx,fy = flow[y,x].T

    # create line endpoints
    lines = np.vstack([x,y,x+fx,y+fy]).T.reshape(-1,2,2)
    lines = np.int32(lines)

    lines = [((x1,y1),(x2,y2)) for ((x1,y1),(x2,y2)) in lines if abs(x1-x2) + abs(y1-y2) > 1 ]

    # create image and draw
    vis = cv2.cvtColor(im,cv2.COLOR_GRAY2BGR)
    for (x1,y1),(x2,y2) in lines:
        cv2.line(vis,(x1,y1),(x2,y2),(0,255,0),1)
        cv2.circle(vis,(x1,y1),1,(0,255,0), -1)

    return vis

def segment_by_velocity(flow, l_thresh=1.5, min_frame_portion=0.025):
    mag = np.sum(np.fabs(flow), axis=2)
    _, magbin = cv2.threshold(mag, l_thresh, 255, cv2.THRESH_BINARY)
    magbin = magbin.astype(np.uint8)

    # Apply morphological closing to combine pieces of same fish
    kernel = cv2.getStructuringElement(cv2.MORPH_CROSS, (35, 35))
    magbin = cv2.morphologyEx(magbin, cv2.MORPH_CLOSE, kernel)

    contours, _ = cv2.findContours(magbin, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

    frameshape = np.shape(flow)
    framesize = frameshape[0] * frameshape[1]

    large_contours = []
    for contour in contours:
        area = cv2.contourArea(contour)
        if area > min_frame_portion * framesize:
            large_contours.append(contour)

    return large_contours

def get_control_pts(img, contour):
    x, y, width, height = cv2.boundingRect(contour)

    sub_img = img[y:y+height, x:x+width]
    features = cv2.goodFeaturesToTrack(sub_img, 5, 0.5, 1)

    if features is None:
        return None

    # Get the points back in the coordinates of the original image
    # while also filtering out those which are not within the contour
    norm_pts = []
    for pt in features:
        norm_pt = (int(pt[0][0]) + x, int(pt[0][1]) + y)
        if cv2.pointPolygonTest(contour, norm_pt, False) >= 0:
            norm_pts.append(norm_pt)

    if len(norm_pts) == 0:
        return None

    return np.array(norm_pts, dtype=np.float32)

def draw_debug_frame(img, fishes, current_contours):
    debug_img = img.copy()

    for fish in fishes:
        for (x, y) in fish.control_pts:
            cv2.circle(debug_img, (int(x), int(y)), 7, (0, 0, 255))

    cv2.drawContours(debug_img, contours, -1, 255)

    cv2.imshow("Debug", debug_img)

# setup video capture
cap = cv2.VideoCapture(VIDEO)
ret, img = cap.read()

prev_gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

skip = 1090

fishes = []

while True:
    ret, img = cap.read()
    if skip > 0:
        skip -= 1
        continue
    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

    # Update fish control points
#    import pdb; pdb.set_trace()
    for fish in fishes:
        fish.control_pts, _, _ = cv2.calcOpticalFlowPyrLK(prev_gray, gray, fish.control_pts, None)

    # compute flow
    flow = cv2.calcOpticalFlowFarneback(prev_gray, gray, None,
                    0.5, # pyramid scale - each layer is half previous size
                    3, # number of pyramid levels
                    35, # averaging window size
                    3, # iterations at each pyramid level
                    5, # size of the pixel neighbourhood used to find
                       # polynomial expansion in each pixel
                    1.1, # standard deviation of the Gaussian that is used to
                         # smooth derivatives used as a basis for the
                         # polynomial expansion
                    0 # additional flags
                    )

    contours = segment_by_velocity(flow)
#    contours = segment.segment_by_velocity(flow, 0.5, 1.5)

    # Find contours that are new fishes
    untracked_contours = []

    fishes_to_add = []
    fishes_to_delete = []
    untracked_contours = []

    # Find untracked contours
    for contour in contours:
        fishes_in_contour = []
        for fish in fishes:
            if fish.in_region(contour):
                # Update fish contour
                fish.update_contour(contour)
                fishes_in_contour.append(fish)

        # Fish has not been tracked
        if not fishes_in_contour:
            untracked_contours.append(contour)
#            control_pts = get_control_pts(gray, contour)
#            if control_pts is not None:
#                fishes_to_add.append(Fish(control_pts, contour))
#                untracked_contours.append(contour)

        # Multiple fish have overlapped
        elif len(fishes_in_contour) >= 1:
            control_pts = get_control_pts(gray, contour)
            if control_pts is not None:
                fishes_to_delete.extend(fishes_in_contour)
                merged_fish = MergedFish(control_pts, contour, fishes_in_contour)
                fishes_to_add.append(merged_fish)

    for contour in untracked_contours:
        merged_fishes = [f for f in fishes if isinstance(f, MergedFish)]
        for fish in merged_fishes:
            if fish.change_in_area() < -1000:
                # Split
                previous_merged_pts = get_control_pts(gray, fish.contour)
                if previous_merged_pts is not None:
                    fishes_to_add.append(Fish(previous_merged_pts, fish.control_pts))
                    fishes_to_delete.append(fish)
                    break
        # new fish
        new_fish_pts = get_control_pts(gray, contour)
        if new_fish_pts is not None:
            fishes_to_add.append(Fish(new_fish_pts, contour))

    fishes = [f for f in fishes if f not in fishes_to_delete]
    fishes.extend(fishes_to_add)

    prev_gray = gray

    draw_debug_frame(img, fishes, contours)

    print "Number of fish: %d" % len(fishes)

    if cv2.waitKey(10) == 27:
        break

<|endoftext|>