<commit_before>from django.conf import settings

from django_statsd.clients import statsd
from django_statsd.middleware import GraphiteRequestTimingMiddleware

class GraphiteViewHitCountMiddleware(GraphiteRequestTimingMiddleware):
    """add hit counting to statsd's request timer."""

    def process_view(self, request, view_func, view_args, view_kwargs):
        super(GraphiteViewHitCountMiddleware, self).process_view(
            request, view_func, view_args, view_kwargs)
        if hasattr(request, '_view_name'):
            data = dict(module=request._view_module, name=request._view_name,
                        method=request.method)
            statsd.incr('view.count.{module}.{name}.{method}'.format(**data))
            statsd.incr('view.count.{module}.{method}'.format(**data))
            statsd.incr('view.count.{method}'.format(**data))

class HostnameMiddleware(object):
    def __init__(self):
        values = [getattr(settings, x) for x in ['HOSTNAME', 'DEIS_APP',
                                                 'DEIS_RELEASE', 'DEIS_DOMAIN']]
        self.backend_server = '.'.join(x for x in values if x)

    def process_response(self, request, response):
        response['X-Backend-Server'] = self.backend_server
        return response
<commit_msg>Add statsd data for (in)secure requests<commit_after>from django.conf import settings

from django_statsd.clients import statsd
from django_statsd.middleware import GraphiteRequestTimingMiddleware

class GraphiteViewHitCountMiddleware(GraphiteRequestTimingMiddleware):
    """add hit counting to statsd's request timer."""

    def process_view(self, request, view_func, view_args, view_kwargs):
        super(GraphiteViewHitCountMiddleware, self).process_view(
            request, view_func, view_args, view_kwargs)
        if hasattr(request, '_view_name'):
            secure = 'secure' if request.is_secure() else 'insecure'
            data = dict(module=request._view_module, name=request._view_name,
                        method=request.method, secure=secure)
            statsd.incr('view.count.{module}.{name}.{method}.{secure}'.format(**data))
            statsd.incr('view.count.{module}.{name}.{method}'.format(**data))
            statsd.incr('view.count.{module}.{method}.{secure}'.format(**data))
            statsd.incr('view.count.{module}.{method}'.format(**data))
            statsd.incr('view.count.{method}.{secure}'.format(**data))
            statsd.incr('view.count.{method}'.format(**data))

class HostnameMiddleware(object):
    def __init__(self):
        values = [getattr(settings, x) for x in ['HOSTNAME', 'DEIS_APP',
                                                 'DEIS_RELEASE', 'DEIS_DOMAIN']]
        self.backend_server = '.'.join(x for x in values if x)

    def process_response(self, request, response):
        response['X-Backend-Server'] = self.backend_server
        return response
<|endoftext|>


<commit_before>#!/usr/bin/python
# newspaperSearch.py
#
# python script that searches the Chronicling America historic newspaper database for a
# keyword and tallies its apperance each year for all provided states.
# input: startDate, endDate, searchText, state(s)
# output: The script will print a table of counts to stdout
# usage: ./newspaperSearch.py [startYear] [endYear] [searchWord] [state]...
#
# Azhar Hussain, July 18, 2017

import re
import sys
import requests

def searchDatabase(startDate, endDate, searchText, stateArray):

    # Database imposed year range
    STARTYEAR = 1789
    ENDYEAR = 1922

    # check if all arguments are present
    if len(sys.argv) >= 5:

        if startDate >= STARTYEAR and endDate <= ENDYEAR:
            header = '{:>45}| {:>12}| {:>6}| {:>14}|'.format("Historic Newspaper Usage of " + searchText,
                                                             "STATE","YEAR","MATCHES")
            print(header)

            for state in stateArray:

                for year in range(startDate, endDate + 1):

                    # create url and return html from results page
                    url = "http://chroniclingamerica.loc.gov/search/pages/results/?state=" + state + "&date1=" + \
                          str(year) + "&date2=" + str(year) + "&proxtext=" + searchText \
                          + "&x=17&y=16&dateFilterType=yearRange&rows=20&searchType=basic"
                    page = requests.get(url)

                    # search for results pattern
                    matchObj = re.search('
(.*) results', page.text)

                    # if matches
                    if matchObj:
                        matches = matchObj.group(1) + " matches"
                        result = '{:>45}| {:>12}| {:>6}| {:>14}|'.format("",state, year, matches)
                        print(result)
                    else:
                        matches = "0 matches"
                        result = '{:>45}| {:>12}| {:>6}| {:>14}|'.format("",state, year, matches)
                        print(result)

        else:
            sys.stderr.write("error: start date must be >= " + STARTYEAR + " AND end date must be <= " + ENDYEAR)
    else:
        sys.stderr.write("usage: newspaperSearch.py startDate endDate searchWord state...")

# state argumentes provided past index 4
stateArray = sys.argv[4:]
searchDatabase(int(sys.argv[1]), int(sys.argv[2]), sys.argv[3], stateArray)
<commit_msg>cleaned up script<commit_after>#!/usr/bin/python
# newspaperSearch.py
#
# python script that searches the Chronicling America historic newspaper database for a
# keyword and tallies its apperance each year for all provided states.
# input: startDate, endDate, searchText, state(s)
# output: The script will print a table of counts to stdout
# usage: ./newspaperSearch.py [startYear] [endYear] [searchWord] [state]...
#
# Azhar Hussain, July 18, 2017

import re
import sys
import requests

def searchDatabase(startDate, endDate, searchText, stateArray):

    # Database imposed year range
    STARTYEAR = 1789
    ENDYEAR = 1922

    # check if all arguments are present
    if len(sys.argv) >= 5:

        if startDate >= STARTYEAR and endDate <= ENDYEAR:
            header = '{:>45}| {:>12}| {:>6}| {:>14}|'.format("Historic Newspaper Usage of " + searchText,
                                                             "STATE","YEAR","MATCHES")
            print(header)

            for state in stateArray:

                for year in range(startDate, endDate + 1):

                    # create url and return html from results page
                    url = "http://chroniclingamerica.loc.gov/search/pages/results/?state=" + state + "&date1=" + \
                          str(year) + "&date2=" + str(year) + "&proxtext=" + searchText \
                          + "&x=17&y=16&dateFilterType=yearRange&rows=20&searchType=basic"
                    page = requests.get(url)

                    # search for results pattern
                    matchObj = re.search('
(.*) results', page.text)

                    # if matches
                    if matchObj:
                        matches = matchObj.group(1) + " matches"
                        result = '{:>45}| {:>12}| {:>6}| {:>14}|'.format("",state, year, matches)
                        print(result)
                    else:
                        matches = "0 matches"
                        result = '{:>45}| {:>12}| {:>6}| {:>14}|'.format("",state, year, matches)
                        print(result)

        else:
            sys.stderr.write("error: start date must be >= " + str(STARTYEAR) + " AND end date must be <= " + str(ENDYEAR))
    else:
        sys.stderr.write("usage: newspaperSearch.py startDate endDate searchWord state...")

# state argumentes provided past index 4
stateArray = sys.argv[4:]
searchDatabase(int(sys.argv[1]), int(sys.argv[2]), sys.argv[3], stateArray)
<|endoftext|>


<commit_before>import jax.numpy as np
import numpy as onp
import jax.scipy as sp
import scipy.optimize as optim
import pickle
import numpy.random as np_rnd

from jax import jit, grad, vmap, jacfwd, jacrev
from ngboost.distns import Normal

from ngboost.scores import MLE, MLE_SURV, CRPS_SURV, CRPS
from ngboost.learners import default_tree_learner, default_linear_learner
from ngboost.distns.normal import Normal

class NGBoost(object):

    def __init__(self, Dist=Normal, Score=MLE(),
                 Base=default_tree_learner, natural_gradient=False,
                 n_estimators=100, learning_rate=0.1, minibatch_frac=1.0,
                 verbose=True, tol=1e-4):
        self.Dist = Dist
        self.Score = Score
        self.Base = Base
        self.natural_gradient = natural_gradient
        self.n_estimators = n_estimators
        self.learning_rate = learning_rate
        self.minibatch_frac = minibatch_frac
        self.verbose = verbose
        self.init_params = None
        self.base_models = []
        self.scalings = []
        self.tol = tol
        self.loss_fn = lambda P, Y: self.Score(self.Dist(P.T), Y).sum()
        self.grad_fn = grad(self.loss_fn)
        #self.grad_fn = jit(vmap(grad(self.loss_fn)))
        self.hessian_fn = jit(vmap(jacrev(grad(self.loss_fn))))
        #self.loss_fn = jit(vmap(self.loss_fn))
        self.Score.setup_distn(self.Dist)
        if isinstance(self.Score, CRPS_SURV):
            self.marginal_score = MLE_SURV()
        elif isinstance(self.Score, CRPS):
            self.marginal_score = MLE()
        else:
            self.marginal_score = self.Score
        self.marginal_loss = lambda P, Y: self.marginal_score(self.Dist(P), Y)
        self.marginal_grad = jit(vmap(grad(self.marginal_loss)))
        self.marginal_loss = jit(vmap(self.marginal_loss))
        self.matmul_inv_fn = jit(vmap(lambda A, b: np.linalg.solve(A, b)))

    def pred_param(self, X, max_iter=None):
        m, n = X.shape
        params = np.ones((m, self.Dist.n_params)) * self.init_params
        for i, (models, s) in enumerate(zip(self.base_models, self.scalings)):
            if max_iter and i == max_iter:
                break
            resids = np.array([model.predict(X) for model in models]).T
            params -= self.learning_rate * resids * s
        return params

    def sample(self, X, Y, params):
        sample_size = int(self.minibatch_frac * len(Y))
        idxs = np_rnd.choice(np.arange(len(Y)), sample_size, replace=False)
        return idxs, X[idxs,:], Y[idxs], params[idxs, :]

    def fit_base(self, X, grads):
        models = [self.Base().fit(X, g) for g in grads.T]
        fitted = np.array([m.predict(X) for m in models]).T
        self.base_models.append(models)
        return fitted

    def line_search(self, resids, start, Y, scale_init=1):
        loss_init = self.loss_fn(start, Y).mean()
        scale = scale_init
        while True:
            scaled_resids = resids * scale
            loss = self.loss_fn(start - scaled_resids, Y).mean()
            norm = np.mean(np.linalg.norm(scaled_resids, axis=1))
            if not np.isnan(loss) and (loss < loss_init or norm < self.tol) and\
               np.linalg.norm(scaled_resids, axis=1).mean() < 5.0:
                break
            scale = scale * 0.5
        self.scalings.append(scale)
        return scale

    def fit(self, X, Y, X_val = None, Y_val = None):

        loss_list = []
        val_loss_list = []
        self.fit_init_params_to_marginal(Y)

        params = self.pred_param(X)
        if X_val is not None and Y_val is not None:
            val_params = self.pred_param(X_val)

        for itr in range(self.n_estimators):
            _, X_batch, Y_batch, P_batch = self.sample(X, Y, params)

            losses = self.loss_fn(P_batch, Y_batch)
            loss = losses.mean()
            if np.isinf(loss) or np.isnan(loss):
                for i, (p, l, y) in enumerate(zip(P_batch, losses, Y_batch)):
                    if np.isinf(l) or np.isnan(l):
                        print('[%d] Params=[%.4f,%.4f], loss=%.4f, y=%.4f' % (i, p[0], p[1], l, y))
                        print('Loss=%.4f' % self.Dist(p).crps_debug(y))
                breakpoint()

            grads = self.grad_fn(P_batch, Y_batch)

            if self.natural_gradient:
                grads = self.Score.naturalize(P_batch, grads)

            #scale = self.line_search(grads, P_batch, Y_batch, scale_init=1)
            #grads = grads * scale

            if np.any(np.isnan(grads)) or np.any(np.isinf(grads)):
                print(grads)
                grads = self.grad_fn(P_batch, Y_batch)
                print('recalculated')
                print(grads)
                print('params')
                print(grads)
                pass

            proj_grad = self.fit_base(X_batch, grads)
            scale = self.line_search(proj_grad, P_batch, Y_batch)

            loss_list += [loss]

            params -= self.learning_rate * scale * np.array([m.predict(X) for m in self.base_models[-1]]).T

            val_loss = 0
            if X_val is not None and Y_val is not None:
                val_params -= self.learning_rate * scale * np.array([m.predict(X_val) for m in self.base_models[-1]]).T
                val_loss = self.loss_fn(val_params, Y_val).mean()
                val_loss_list += [val_loss]
                if np.mean(np.array(val_loss_list[-5:])) > \
                   np.mean(np.array(val_loss_list[-10:-5])):
                    if self.verbose:
                        print(f"== Quitting at iteration / VAL {itr} (val_loss={val_loss:.4f})")
                    pass

            if self.verbose:
                grad_norm = np.linalg.norm(grads, axis=1).mean() * scale
                print(f"[iter {itr}] loss={loss:.4f} val_loss={val_loss:.4f} scale={scale:.4f} "
                      f"norm={grad_norm:.4f}")

            if np.linalg.norm(proj_grad, axis=1).mean() < self.tol:
                if self.verbose:
                    print(f"== Quitting at iteration / GRAD {itr}")
                break

        return loss_list, val_loss_list

    def fit_init_params_to_marginal(self, Y, iters=1000):
        try:
            E = Y['Event']
            T = Y['Time'].reshape((-1, 1))[E == 1]
        except:
            T = Y
        self.init_params = self.Dist.fit(T)
        return

    def pred_dist(self, X, max_iter=None):
        params = onp.asarray(self.pred_param(X, max_iter))
        dist = self.Dist(params.T)
        return dist

    def predict(self, X):
        dists = self.pred_dist(X)
        return list(dist.loc.flatten())

    def staged_predict(self, X, max_iter=None):
        predictions = []
        m, n = X.shape
        params = np.ones((m, self.Dist.n_params)) * self.init_params
        for i, (models, s) in enumerate(zip(self.base_models, self.scalings)):
            if max_iter and i == max_iter:
                break
            resids = np.array([model.predict(X) for model in models]).T
            params -= self.learning_rate * resids * s
            dists = self.Dist(onp.asarray(params).T)
            predictions.append(dists.loc.flatten())
        return predictions

    def staged_pred_dist(self, X, max_iter=None):
        predictions = []
        m, n = X.shape
        params = np.ones((m, self.Dist.n_params)) * self.init_params
        for i, (models, s) in enumerate(zip(self.base_models, self.scalings)):
            if max_iter and i == max_iter:
                break
            resids = np.array([model.predict(X) for model in models]).T
            params -= self.learning_rate * resids * s
            dists = self.Dist(onp.asarray(params).T)
            predictions.append(dists)
        return predictions
<commit_msg>ngboost: fix predict()<commit_after>import jax.numpy as np
import numpy as onp
import jax.scipy as sp
import scipy.optimize as optim
import pickle
import numpy.random as np_rnd

from jax import jit, grad, vmap, jacfwd, jacrev
from ngboost.distns import Normal

from ngboost.scores import MLE, MLE_SURV, CRPS_SURV, CRPS
from ngboost.learners import default_tree_learner, default_linear_learner
from ngboost.distns.normal import Normal

class NGBoost(object):

    def __init__(self, Dist=Normal, Score=MLE(),
                 Base=default_tree_learner, natural_gradient=False,
                 n_estimators=100, learning_rate=0.1, minibatch_frac=1.0,
                 verbose=True, tol=1e-4):
        self.Dist = Dist
        self.Score = Score
        self.Base = Base
        self.natural_gradient = natural_gradient
        self.n_estimators = n_estimators
        self.learning_rate = learning_rate
        self.minibatch_frac = minibatch_frac
        self.verbose = verbose
        self.init_params = None
        self.base_models = []
        self.scalings = []
        self.tol = tol
        self.loss_fn = lambda P, Y: self.Score(self.Dist(P.T), Y).sum()
        self.grad_fn = grad(self.loss_fn)
        #self.grad_fn = jit(vmap(grad(self.loss_fn)))
        self.hessian_fn = jit(vmap(jacrev(grad(self.loss_fn))))
        #self.loss_fn = jit(vmap(self.loss_fn))
        self.Score.setup_distn(self.Dist)
        if isinstance(self.Score, CRPS_SURV):
            self.marginal_score = MLE_SURV()
        elif isinstance(self.Score, CRPS):
            self.marginal_score = MLE()
        else:
            self.marginal_score = self.Score
        self.marginal_loss = lambda P, Y: self.marginal_score(self.Dist(P), Y)
        self.marginal_grad = jit(vmap(grad(self.marginal_loss)))
        self.marginal_loss = jit(vmap(self.marginal_loss))
        self.matmul_inv_fn = jit(vmap(lambda A, b: np.linalg.solve(A, b)))

    def pred_param(self, X, max_iter=None):
        m, n = X.shape
        params = np.ones((m, self.Dist.n_params)) * self.init_params
        for i, (models, s) in enumerate(zip(self.base_models, self.scalings)):
            if max_iter and i == max_iter:
                break
            resids = np.array([model.predict(X) for model in models]).T
            params -= self.learning_rate * resids * s
        return params

    def sample(self, X, Y, params):
        sample_size = int(self.minibatch_frac * len(Y))
        idxs = np_rnd.choice(np.arange(len(Y)), sample_size, replace=False)
        return idxs, X[idxs,:], Y[idxs], params[idxs, :]

    def fit_base(self, X, grads):
        models = [self.Base().fit(X, g) for g in grads.T]
        fitted = np.array([m.predict(X) for m in models]).T
        self.base_models.append(models)
        return fitted

    def line_search(self, resids, start, Y, scale_init=1):
        loss_init = self.loss_fn(start, Y).mean()
        scale = scale_init
        while True:
            scaled_resids = resids * scale
            loss = self.loss_fn(start - scaled_resids, Y).mean()
            norm = np.mean(np.linalg.norm(scaled_resids, axis=1))
            if not np.isnan(loss) and (loss < loss_init or norm < self.tol) and\
               np.linalg.norm(scaled_resids, axis=1).mean() < 5.0:
                break
            scale = scale * 0.5
        self.scalings.append(scale)
        return scale

    def fit(self, X, Y, X_val = None, Y_val = None):

        loss_list = []
        val_loss_list = []
        self.fit_init_params_to_marginal(Y)

        params = self.pred_param(X)
        if X_val is not None and Y_val is not None:
            val_params = self.pred_param(X_val)

        for itr in range(self.n_estimators):
            _, X_batch, Y_batch, P_batch = self.sample(X, Y, params)

            losses = self.loss_fn(P_batch, Y_batch)
            loss = losses.mean()
            if np.isinf(loss) or np.isnan(loss):
                for i, (p, l, y) in enumerate(zip(P_batch, losses, Y_batch)):
                    if np.isinf(l) or np.isnan(l):
                        print('[%d] Params=[%.4f,%.4f], loss=%.4f, y=%.4f' % (i, p[0], p[1], l, y))
                        print('Loss=%.4f' % self.Dist(p).crps_debug(y))
                breakpoint()

            grads = self.grad_fn(P_batch, Y_batch)

            if self.natural_gradient:
                grads = self.Score.naturalize(P_batch, grads)

            #scale = self.line_search(grads, P_batch, Y_batch, scale_init=1)
            #grads = grads * scale

            if np.any(np.isnan(grads)) or np.any(np.isinf(grads)):
                print(grads)
                grads = self.grad_fn(P_batch, Y_batch)
                print('recalculated')
                print(grads)
                print('params')
                print(grads)
                pass

            proj_grad = self.fit_base(X_batch, grads)
            scale = self.line_search(proj_grad, P_batch, Y_batch)

            loss_list += [loss]

            params -= self.learning_rate * scale * np.array([m.predict(X) for m in self.base_models[-1]]).T

            val_loss = 0
            if X_val is not None and Y_val is not None:
                val_params -= self.learning_rate * scale * np.array([m.predict(X_val) for m in self.base_models[-1]]).T
                val_loss = self.loss_fn(val_params, Y_val).mean()
                val_loss_list += [val_loss]
                if np.mean(np.array(val_loss_list[-5:])) > \
                   np.mean(np.array(val_loss_list[-10:-5])):
                    if self.verbose:
                        print(f"== Quitting at iteration / VAL {itr} (val_loss={val_loss:.4f})")
                    pass

            if self.verbose:
                grad_norm = np.linalg.norm(grads, axis=1).mean() * scale
                print(f"[iter {itr}] loss={loss:.4f} val_loss={val_loss:.4f} scale={scale:.4f} "
                      f"norm={grad_norm:.4f}")

            if np.linalg.norm(proj_grad, axis=1).mean() < self.tol:
                if self.verbose:
                    print(f"== Quitting at iteration / GRAD {itr}")
                break

        return loss_list, val_loss_list

    def fit_init_params_to_marginal(self, Y, iters=1000):
        try:
            E = Y['Event']
            T = Y['Time'].reshape((-1, 1))[E == 1]
        except:
            T = Y
        self.init_params = self.Dist.fit(T)
        return

    def pred_dist(self, X, max_iter=None):
        params = onp.asarray(self.pred_param(X, max_iter))
        dist = self.Dist(params.T)
        return dist

    def predict(self, X):
        dist = self.pred_dist(X)
        return list(dist.loc.flatten())

    def staged_predict(self, X, max_iter=None):
        predictions = []
        m, n = X.shape
        params = np.ones((m, self.Dist.n_params)) * self.init_params
        for i, (models, s) in enumerate(zip(self.base_models, self.scalings)):
            if max_iter and i == max_iter:
                break
            resids = np.array([model.predict(X) for model in models]).T
            params -= self.learning_rate * resids * s
            dists = self.Dist(onp.asarray(params).T)
            predictions.append(dists.loc.flatten())
        return predictions

    def staged_pred_dist(self, X, max_iter=None):
        predictions = []
        m, n = X.shape
        params = np.ones((m, self.Dist.n_params)) * self.init_params
        for i, (models, s) in enumerate(zip(self.base_models, self.scalings)):
            if max_iter and i == max_iter:
                break
            resids = np.array([model.predict(X) for model in models]).T
            params -= self.learning_rate * resids * s
            dists = self.Dist(onp.asarray(params).T)
            predictions.append(dists)
        return predictions
<|endoftext|>


<commit_before>from Bio import SeqIO
import yaml
import sys
import os

def replace_deletions(word, seq, idxs, del_letter='d'):
    """
    Replace any '-' in word with del_letter if the nucleotides next to it in
    seq are not '-'.

    """
    new_word = [c for c in word]

    for i, letter in enumerate(word):
        # assume we're not looking at the start or end of sequence
        idx = idxs[i]
        assert idx > 0 and idx < len(seq)

        if letter == '-':
            if seq[idx-1] != '-' and seq[idx+1] != '-':
                new_word[i] = del_letter

    return ''.join(new_word)

def get_positions(f_name, sites, keep_dashes=True, mark_deletions=True):
    """
    Reads in a fasta file of sequences (usually produced by nextgen_main.py)
    at location f_name, and pulls out the bases at the (0-start) indices in
    sites.

    Input:
        - f_name: str
        - sites: list (ints)
        - keep_dashes: bool
        - mark_deletions: bool
    Output:
        - words: list

    Options:
    keep_dashes: if this is false, get_positions will discard any words with a
        dash in them (generally denoting a deletion)
    mark_deletions: if this is true, deletions (dashes flanked by non-dashes on
        both sides) will be marked (with a 'd', but this should be
        customizable?)
    """

    words = []
    with open(f_name) as f:
        seq_iter = SeqIO.parse(f, 'fasta')
        for s in seq_iter:
            selected_letters = ''.join([str(s.seq[i]) for i in sites])
            if '-' not in selected_letters:
                words.append(selected_letters)
            elif keep_dashes:
                if mark_deletions:
                    words.append(replace_deletions(selected_letters, s, sites))
                else:
                    words.append(selected_letters)

    return words

if __name__ == '__main__':
    in_name = sys.argv[1]
    out_name = sys.argv[2]
    sites = [int(x) for x in sys.argv[3:]]

    words = get_positions(in_name, sites, keep_dashes=True)

    with open(out_name, 'w') as of:
        for word in words:
            of.write('%s\n' % word)<commit_msg>changed default deletions behavior<commit_after>from Bio import SeqIO
import yaml
import sys
import os

def replace_deletions(word, seq, idxs, del_letter='d'):
    """
    Replace any '-' in word with del_letter if the nucleotides next to it in
    seq are not '-'.

    """
    new_word = [c for c in word]

    for i, letter in enumerate(word):
        # assume we're not looking at the start or end of sequence
        idx = idxs[i]
        assert idx > 0 and idx < len(seq)

        if letter == '-':
            if seq[idx-1] != '-' and seq[idx+1] != '-':
                new_word[i] = del_letter

    return ''.join(new_word)

def get_positions(f_name, sites, keep_dashes=True, mark_deletions=False):
    """
    Reads in a fasta file of sequences (usually produced by nextgen_main.py)
    at location f_name, and pulls out the bases at the (0-start) indices in
    sites.

    Input:
        - f_name: str
        - sites: list (ints)
        - keep_dashes: bool
        - mark_deletions: bool
    Output:
        - words: list

    Options:
    keep_dashes: if this is false, get_positions will discard any words with a
        dash in them (generally denoting a deletion)
    mark_deletions: if this is true, deletions (dashes flanked by non-dashes on
        both sides) will be marked (with a 'd', but this should be
        customizable?)
    """

    words = []
    with open(f_name) as f:
        seq_iter = SeqIO.parse(f, 'fasta')
        for s in seq_iter:
            selected_letters = ''.join([str(s.seq[i]) for i in sites])
            if '-' not in selected_letters:
                words.append(selected_letters)
            elif keep_dashes:
                if mark_deletions:
                    words.append(replace_deletions(selected_letters, s, sites))
                else:
                    words.append(selected_letters)

    return words

if __name__ == '__main__':
    in_name = sys.argv[1]
    out_name = sys.argv[2]
    sites = [int(x) for x in sys.argv[3:]]

    words = get_positions(in_name, sites, keep_dashes=True)

    with open(out_name, 'w') as of:
        for word in words:
            of.write('%s\n' % word)<|endoftext|>


<commit_before># Copyright (c) 2016, German Neuroinformatics Node (G-Node)
# -*- coding: utf-8 -*-
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted under the terms of the BSD License. See
# LICENSE file in the root of the Project.

class LinkType(object):
    Tagged = "tagged"
    Untagged = "untagged"
    Indexed = "indexed"

<commit_msg>Uppercase first letter for LinkType members<commit_after># Copyright (c) 2016, German Neuroinformatics Node (G-Node)
# -*- coding: utf-8 -*-
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted under the terms of the BSD License. See
# LICENSE file in the root of the Project.

class LinkType(object):
    Tagged = "Tagged"
    Untagged = "Untagged"
    Indexed = "Indexed"

<|endoftext|>


<commit_before>class NumberToWords(object):
    """
    Class for converting positive integer values to a textual representation
    of the submitted number for value of 0 up to 999999999.
    """

    MAX = 999999999
    SMALL_NUMBERS = ['', 'one', 'two', 'three', 'four', 'five', 'six',
                     'seven', 'eight', 'nine', 'ten', 'eleven',
                     'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen',
                     'seventeen', 'eighteen', 'nineteen']
    TENS = ['', '', 'twenty', 'thirty', 'fourty', 'fifty', 'sixty', 'seventy',
            'eighty', 'ninety']
    LARGE_NUMBERS = ['', 'thousand', 'million']
    EXCEPTION_STRING = "This method expects positive integer values between " \
        + "0 and {0}".format(MAX)

    def convert(self, number):
        """
        Take an integer and return it converted to a textual representation.

        Args:
            number (int): The number to be converted.

        Returns:
            sentence (string): The textual representation of `number`.

        Raises:
            ValueError: If `number` is not a positive integer or is greater
                        than `MAX`.
        """

        if not isinstance(number, (int, long)):
            raise ValueError(self.EXCEPTION_STRING)
        try:
            sentence = ""
            if number is 0:
                sentence = "zero"
            else:
                # split number into a list of strings where each list item is
                # at most 3 character in length.
                groups = format(number, ',').split(',')

                # make sure each list item is exactly 3 characters long by
                # zero filling
                zero_filled_groups = []
                for group in groups:
                    zero_filled_groups.append(group.zfill(3))

                # reverse the list of strings so that the list indexes of the
                # string representation of hundreds, thousands and million
                # match those of `LARGE_NUMBERS`
                zero_filled_groups.reverse()
                for group in zero_filled_groups:
                    index = zero_filled_groups.index(group)
                    suffix = self.LARGE_NUMBERS[index]
                    number_as_words = " ".join(
                        self._number_to_word_list(group, suffix))
                    if len(number_as_words) > 0:
                        sentence = "{0} {1}".format(number_as_words, sentence)
                    # set this group to None so as to not set a false `index`
                    # for subsequent groups where `number` has multiple
                    # identical groups
                    zero_filled_groups[index] = None
            return sentence.rstrip()
        except (IndexError, ValueError):
            raise ValueError(self.EXCEPTION_STRING)

    def _number_to_word_list(self, number_string, suffix=None):
        """
        Take a 3 digit string representation of an integer and convert it to a
        textual representation with an optional suffix.
        """

        words = []
        hundreds, tens, units = [int(n) for n in list(number_string)]

        if hundreds is not 0:
            string = self.SMALL_NUMBERS[hundreds]
            words.append("{0} hundred".format(string))
            if tens is not 0 or units is not 0:
                words.append("and")

        if tens is 1:
            string = self.SMALL_NUMBERS[int("{0}{1}".format(tens, units))]
            words.append("{0}".format(string))
        else:
            if tens is not 0:
                string = self.TENS[tens]
                words.append("{0}".format(string))
            if units is not 0:
                string = self.SMALL_NUMBERS[units]
                words.append("{0}".format(string))

        if suffix and (sum([hundreds, tens, units]) is not 0):
            words.append(suffix)

        return words
<commit_msg>Update class docstring to add example usage<commit_after>class NumberToWords(object):
    """
    Class for converting positive integer values to a textual representation
    of the submitted number for value of 0 up to 999999999.

    Example:
        >>> from number_to_words import NumberToWords
        >>> n2w = NumberToWords()
        >>> n2w.convert(123)
        'one hundreed and twenty three'
    """

    MAX = 999999999
    SMALL_NUMBERS = ['', 'one', 'two', 'three', 'four', 'five', 'six',
                     'seven', 'eight', 'nine', 'ten', 'eleven',
                     'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen',
                     'seventeen', 'eighteen', 'nineteen']
    TENS = ['', '', 'twenty', 'thirty', 'fourty', 'fifty', 'sixty', 'seventy',
            'eighty', 'ninety']
    LARGE_NUMBERS = ['', 'thousand', 'million']
    EXCEPTION_STRING = "This method expects positive integer values between " \
        + "0 and {0}".format(MAX)

    def convert(self, number):
        """
        Take an integer and return it converted to a textual representation.

        Args:
            number (int): The number to be converted.

        Returns:
            sentence (string): The textual representation of `number`.

        Raises:
            ValueError: If `number` is not a positive integer or is greater
                        than `MAX`.
        """

        if not isinstance(number, (int, long)):
            raise ValueError(self.EXCEPTION_STRING)
        try:
            sentence = ""
            if number is 0:
                sentence = "zero"
            else:
                # split number into a list of strings where each list item is
                # at most 3 character in length.
                groups = format(number, ',').split(',')

                # make sure each list item is exactly 3 characters long by
                # zero filling
                zero_filled_groups = []
                for group in groups:
                    zero_filled_groups.append(group.zfill(3))

                # reverse the list of strings so that the list indexes of the
                # string representation of hundreds, thousands and million
                # match those of `LARGE_NUMBERS`
                zero_filled_groups.reverse()
                for group in zero_filled_groups:
                    index = zero_filled_groups.index(group)
                    suffix = self.LARGE_NUMBERS[index]
                    number_as_words = " ".join(
                        self._number_to_word_list(group, suffix))
                    if len(number_as_words) > 0:
                        sentence = "{0} {1}".format(number_as_words, sentence)
                    # set this group to None so as to not set a false `index`
                    # for subsequent groups where `number` has multiple
                    # identical groups
                    zero_filled_groups[index] = None
            return sentence.rstrip()
        except (IndexError, ValueError):
            raise ValueError(self.EXCEPTION_STRING)

    def _number_to_word_list(self, number_string, suffix=None):
        """
        Take a 3 digit string representation of an integer and convert it to a
        textual representation with an optional suffix.
        """

        words = []
        hundreds, tens, units = [int(n) for n in list(number_string)]

        if hundreds is not 0:
            string = self.SMALL_NUMBERS[hundreds]
            words.append("{0} hundred".format(string))
            if tens is not 0 or units is not 0:
                words.append("and")

        if tens is 1:
            string = self.SMALL_NUMBERS[int("{0}{1}".format(tens, units))]
            words.append("{0}".format(string))
        else:
            if tens is not 0:
                string = self.TENS[tens]
                words.append("{0}".format(string))
            if units is not 0:
                string = self.SMALL_NUMBERS[units]
                words.append("{0}".format(string))

        if suffix and (sum([hundreds, tens, units]) is not 0):
            words.append(suffix)

        return words
<|endoftext|>


<commit_before># vim: se sw=2:
# MIT licensed
# Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.

import sys
import argparse
import structlog
import json

from . import core

logger = structlog.get_logger(logger_name=__name__)

def take() -> None:
  parser = argparse.ArgumentParser(description='update version records of nvchecker')
  core.add_common_arguments(parser)
  parser.add_argument('--all', action='store_true',
                      help='take all updates')
  parser.add_argument('--ignore-nonexistent', action='store_true',
                      help='ignore nonexistent names')
  parser.add_argument('names', metavar='NAME', nargs='*',
                      help='software name to be updated. use NAME=VERSION to update '
                           'to a specific version instead of the new version.')
  args = parser.parse_args()
  if core.process_common_arguments(args):
    return

  opt = core.load_file(args.file, use_keymanager=False)[1]
  if opt.ver_files is None:
    logger.critical(
      "doesn't have 'oldver' and 'newver' set.",
      source=args.file,
    )
    sys.exit(2)
  else:
    oldverf = opt.ver_files[0]
    newverf = opt.ver_files[1]

  oldvers = core.read_verfile(oldverf)
  newvers = core.read_verfile(newverf)

  if args.all:
    oldvers.update(newvers)
  else:
    for name in args.names:
      if "=" in name:
        name, newver = name.split("=")
        oldvers[name] = newver
      else:
        try:
          oldvers[name] = newvers[name]
        except KeyError:
          if args.ignore_nonexistent:
            logger.warning('nonexistent in newver, ignored', name=name)
            continue

          logger.critical(
            "doesn't exist in 'newver' set.", name=name,
          )
          sys.exit(2)

  try:
    oldverf.rename(
      oldverf.with_name(oldverf.name + '~'),
    )
  except FileNotFoundError:
    pass
  core.write_verfile(oldverf, oldvers)

def cmp() -> None:
  parser = argparse.ArgumentParser(description='compare version records of nvchecker')
  core.add_common_arguments(parser)
  parser.add_argument('-j', '--json', action='store_true',
                      help='Output JSON array of dictionaries with {name, newver, oldver, [delta]} '
                           '(or array of names if --quiet)')
  parser.add_argument('-q', '--quiet', action='store_true',
                      help="Quiet mode, output only the names.")
  parser.add_argument('-a', '--all', action='store_true',
                      help="Include unchanged versions.")
  parser.add_argument('-s', '--sort',
                      choices=('parse_version', 'vercmp', 'none'), default='parse_version',
                      help='Version compare method to backwards the arrow '
                           '(default: parse_version)')
  parser.add_argument('-n', '--newer', action='store_true',
                      help='Shows only the newer ones according to --sort.')
  parser.add_argument('--exit-status', action='store_true',
                      help="exit with status 4 if there are updates")
  args = parser.parse_args()
  if core.process_common_arguments(args):
    return

  opt = core.load_file(args.file, use_keymanager=False)[1]
  if opt.ver_files is None:
    logger.critical(
      "doesn't have 'oldver' and 'newver' set.",
      source=args.file,
    )
    sys.exit(2)
  else:
    oldverf = opt.ver_files[0]
    newverf = opt.ver_files[1]

  oldvers = core.read_verfile(oldverf)
  newvers = core.read_verfile(newverf)

  differences = []

  for name, newver in sorted(newvers.items()):  # accumulate differences
    oldver = oldvers.get(name, None)

    diff = {
      'name': name,
      'oldver': oldver,
      'newver': newver
    }

    if oldver is not None and newver is not None:
      if oldver == newver:
        diff['delta'] = 'equal'

      elif args.sort == "none":
        diff['delta'] = 'new'  # assume it's a new version if we're not comparing

      else:
        from .sortversion import sort_version_keys
        version = sort_version_keys[args.sort]

        if version(oldver) > version(newver):  # type: ignore
          if args.newer:
            continue  # don't store this diff
          diff['delta'] = 'old'
        else:
          diff['delta'] = 'new'

    elif oldver is None:
      diff['delta'] = 'added'

    elif newver is None:
      if args.newer:
        continue  # don't store this diff
      diff['delta'] = 'gone'

    if args.all or diff['delta'] != 'equal':
      differences.append(diff)

  if args.json:
    if args.quiet:
      print(json.dumps([diff['name'] for diff in differences], separators=(',', ':')))
    else:
      print(json.dumps(differences, sort_keys=True, separators=(',', ':')))

  elif args.quiet:
    for diff in differences:
      print(diff['name'])

  else:
    from .lib.nicelogger import Colors, support_color
    c = Colors(support_color(sys.stdout))

    diffstyles = {
      'new': {
        'symbol': '->',
        'oldc': c.red
      },
      'old': {
        'symbol': f'{c.red}<-{c.normal}',
        'oldc': c.red
      },
      'added': {
        'symbol': '++',
        'oldc': c.red
      },
      'gone': {
        'symbol': f'{c.red}--{c.normal}',
        'oldc': c.green
      },
      'equal': {
        'symbol': '==',
        'oldc': c.green
      }
    }

    for diff in differences:
      style = diffstyles[diff.get('delta', 'equal')] # type: ignore # mypy has issues with this line

      print(f'{diff["name"]} {style["oldc"]}{diff["oldver"]}{c.normal} {style["symbol"]} {c.green}{diff["newver"]}{c.normal}')

  if args.exit_status and any(
    diff.get('delta') != 'equal' for diff in differences
  ):
    sys.exit(4)
<commit_msg>make mypy happy<commit_after># vim: se sw=2:
# MIT licensed
# Copyright (c) 2013-2020 lilydjwg <lilydjwg@gmail.com>, et al.

import sys
import argparse
import structlog
import json

from . import core

logger = structlog.get_logger(logger_name=__name__)

def take() -> None:
  parser = argparse.ArgumentParser(description='update version records of nvchecker')
  core.add_common_arguments(parser)
  parser.add_argument('--all', action='store_true',
                      help='take all updates')
  parser.add_argument('--ignore-nonexistent', action='store_true',
                      help='ignore nonexistent names')
  parser.add_argument('names', metavar='NAME', nargs='*',
                      help='software name to be updated. use NAME=VERSION to update '
                           'to a specific version instead of the new version.')
  args = parser.parse_args()
  if core.process_common_arguments(args):
    return

  opt = core.load_file(args.file, use_keymanager=False)[1]
  if opt.ver_files is None:
    logger.critical(
      "doesn't have 'oldver' and 'newver' set.",
      source=args.file,
    )
    sys.exit(2)
  else:
    oldverf = opt.ver_files[0]
    newverf = opt.ver_files[1]

  oldvers = core.read_verfile(oldverf)
  newvers = core.read_verfile(newverf)

  if args.all:
    oldvers.update(newvers)
  else:
    for name in args.names:
      if "=" in name:
        name, newver = name.split("=")
        oldvers[name] = newver
      else:
        try:
          oldvers[name] = newvers[name]
        except KeyError:
          if args.ignore_nonexistent:
            logger.warning('nonexistent in newver, ignored', name=name)
            continue

          logger.critical(
            "doesn't exist in 'newver' set.", name=name,
          )
          sys.exit(2)

  try:
    oldverf.rename(
      oldverf.with_name(oldverf.name + '~'),
    )
  except FileNotFoundError:
    pass
  core.write_verfile(oldverf, oldvers)

def cmp() -> None:
  parser = argparse.ArgumentParser(description='compare version records of nvchecker')
  core.add_common_arguments(parser)
  parser.add_argument('-j', '--json', action='store_true',
                      help='Output JSON array of dictionaries with {name, newver, oldver, [delta]} '
                           '(or array of names if --quiet)')
  parser.add_argument('-q', '--quiet', action='store_true',
                      help="Quiet mode, output only the names.")
  parser.add_argument('-a', '--all', action='store_true',
                      help="Include unchanged versions.")
  parser.add_argument('-s', '--sort',
                      choices=('parse_version', 'vercmp', 'none'), default='parse_version',
                      help='Version compare method to backwards the arrow '
                           '(default: parse_version)')
  parser.add_argument('-n', '--newer', action='store_true',
                      help='Shows only the newer ones according to --sort.')
  parser.add_argument('--exit-status', action='store_true',
                      help="exit with status 4 if there are updates")
  args = parser.parse_args()
  if core.process_common_arguments(args):
    return

  opt = core.load_file(args.file, use_keymanager=False)[1]
  if opt.ver_files is None:
    logger.critical(
      "doesn't have 'oldver' and 'newver' set.",
      source=args.file,
    )
    sys.exit(2)
  else:
    oldverf = opt.ver_files[0]
    newverf = opt.ver_files[1]

  oldvers = core.read_verfile(oldverf)
  newvers = core.read_verfile(newverf)

  differences = []

  for name, newver in sorted(newvers.items()):  # accumulate differences
    oldver = oldvers.get(name, None)

    diff = {
      'name': name,
      'oldver': oldver,
      'newver': newver
    }

    if oldver is not None and newver is not None:
      if oldver == newver:
        diff['delta'] = 'equal'

      elif args.sort == "none":
        diff['delta'] = 'new'  # assume it's a new version if we're not comparing

      else:
        from .sortversion import sort_version_keys
        version = sort_version_keys[args.sort]

        if version(oldver) > version(newver):
          if args.newer:
            continue  # don't store this diff
          diff['delta'] = 'old'
        else:
          diff['delta'] = 'new'

    elif oldver is None:
      diff['delta'] = 'added'

    elif newver is None:
      if args.newer:
        continue  # don't store this diff
      diff['delta'] = 'gone'

    if args.all or diff['delta'] != 'equal':
      differences.append(diff)

  if args.json:
    if args.quiet:
      print(json.dumps([diff['name'] for diff in differences], separators=(',', ':')))
    else:
      print(json.dumps(differences, sort_keys=True, separators=(',', ':')))

  elif args.quiet:
    for diff in differences:
      print(diff['name'])

  else:
    from .lib.nicelogger import Colors, support_color
    c = Colors(support_color(sys.stdout))

    diffstyles = {
      'new': {
        'symbol': '->',
        'oldc': c.red
      },
      'old': {
        'symbol': f'{c.red}<-{c.normal}',
        'oldc': c.red
      },
      'added': {
        'symbol': '++',
        'oldc': c.red
      },
      'gone': {
        'symbol': f'{c.red}--{c.normal}',
        'oldc': c.green
      },
      'equal': {
        'symbol': '==',
        'oldc': c.green
      }
    }

    for diff in differences:
      style = diffstyles[diff.get('delta', 'equal')] # type: ignore # mypy has issues with this line

      print(f'{diff["name"]} {style["oldc"]}{diff["oldver"]}{c.normal} {style["symbol"]} {c.green}{diff["newver"]}{c.normal}')

  if args.exit_status and any(
    diff.get('delta') != 'equal' for diff in differences
  ):
    sys.exit(4)
<|endoftext|>


<commit_before>"""FokaBot related functions"""
import re

from common import generalUtils
from common.constants import actions
from common.ripple import userUtils
from constants import fokabotCommands
from constants import serverPackets
from objects import glob

# Tillerino np regex, compiled only once to increase performance
npRegex = re.compile("^https?:\\/\\/osu\\.ppy\\.sh\\/b\\/(\\d*)")

def connect():
	"""
	Connect FokaBot to Bancho

	:return:
	"""
	token = glob.tokens.addToken(999)
	token.actionID = actions.IDLE
	glob.streams.broadcast("main", serverPackets.userPanel(999))
	glob.streams.broadcast("main", serverPackets.userStats(999))

def disconnect():
	"""
	Disconnect FokaBot from Bancho

	:return:
	"""
	glob.tokens.deleteToken(glob.tokens.getTokenFromUserID(999))

def fokabotResponse(fro, chan, message):
	"""
	Check if a message has triggered FokaBot

	:param fro: sender username
	:param chan: channel name (or receiver username)
	:param message: chat mesage
	:return: FokaBot's response or False if no response
	"""
	for i in fokabotCommands.commands:
		# Loop though all commands
		if message.strip().startswith(i["trigger"]):
			# message has triggered a command

			# Make sure the user has right permissions
			if i["privileges"] is not None:
				# Rank = x
				if userUtils.getPrivileges(userUtils.getID(fro)) & i["privileges"] == 0:
					return False

			# Check argument number
			message = message.split(" ")
			if i["syntax"] != "" and len(message) <= len(i["syntax"].split(" ")):
				return "Wrong syntax: {} {}".format(i["trigger"], i["syntax"])

			# Return response or execute callback
			if i["callback"] is None:
				return i["response"]
			else:
				return i["callback"](fro, chan, message[1:])

	# No commands triggered
	return False<commit_msg>Fix wrong commands being triggered if they have the same letters at the beginning<commit_after>"""FokaBot related functions"""
import re

from common import generalUtils
from common.constants import actions
from common.ripple import userUtils
from constants import fokabotCommands
from constants import serverPackets
from objects import glob

# Tillerino np regex, compiled only once to increase performance
npRegex = re.compile("^https?:\\/\\/osu\\.ppy\\.sh\\/b\\/(\\d*)")

def connect():
	"""
	Connect FokaBot to Bancho

	:return:
	"""
	token = glob.tokens.addToken(999)
	token.actionID = actions.IDLE
	glob.streams.broadcast("main", serverPackets.userPanel(999))
	glob.streams.broadcast("main", serverPackets.userStats(999))

def disconnect():
	"""
	Disconnect FokaBot from Bancho

	:return:
	"""
	glob.tokens.deleteToken(glob.tokens.getTokenFromUserID(999))

def fokabotResponse(fro, chan, message):
	"""
	Check if a message has triggered FokaBot

	:param fro: sender username
	:param chan: channel name (or receiver username)
	:param message: chat mesage
	:return: FokaBot's response or False if no response
	"""
	for i in fokabotCommands.commands:
		# Loop though all commands
		if message.strip().startswith("{} ".format(i["trigger"])):
			# message has triggered a command

			# Make sure the user has right permissions
			if i["privileges"] is not None:
				# Rank = x
				if userUtils.getPrivileges(userUtils.getID(fro)) & i["privileges"] == 0:
					return False

			# Check argument number
			message = message.split(" ")
			if i["syntax"] != "" and len(message) <= len(i["syntax"].split(" ")):
				return "Wrong syntax: {} {}".format(i["trigger"], i["syntax"])

			# Return response or execute callback
			if i["callback"] is None:
				return i["response"]
			else:
				return i["callback"](fro, chan, message[1:])

	# No commands triggered
	return False<|endoftext|>


<commit_before>from ocdsmerge.flatten import flatten, unflatten
from ocdsmerge.rules import get_merge_rules
from ocdsmerge.util import sorted_releases

class Merger:
    def __init__(self, schema=None, merge_rules=None, rule_overrides=None):
        """
        Initializes a reusable ``Merger`` instance for creating merged releases.

        :param schema: the release schema (if not provided, will default to the latest version of OCDS)
        :param dict merge_rules: the merge rules (if not provided, will determine the rules from the ``schema``)
        :param dict rule_overrides: any rule overrides, in which keys are field paths as tuples, and values are either
            ``ocdsmerge.APPEND`` or ``ocdsmerge.MERGE_BY_POSITION``
        :type schema: dict or str
        """
        if merge_rules is None:
            merge_rules = get_merge_rules(schema)
        if rule_overrides is None:
            rule_overrides = {}

        self.merge_rules = merge_rules
        self.rule_overrides = rule_overrides

    def create_compiled_release(self, releases):
        """
        Merges a list of releases into a compiled release.
        """
        return self._create_merged_release(CompiledRelease, releases)

    def create_versioned_release(self, releases):
        """
        Merges a list of releases into a versioned release.
        """
        return self._create_merged_release(VersionedRelease, releases)

    def _create_merged_release(self, cls, releases):
        merged_release = cls(merge_rules=self.merge_rules, rule_overrides=self.rule_overrides)
        merged_release.extend(releases)
        return merged_release.asdict()

class MergedRelease:
    """
    Whether the class is for merging versioned releases.
    """
    versioned = None

    def __init__(self, data=None, schema=None, merge_rules=None, rule_overrides=None):
        """
        Initializes a merged release.

        :param dict data: the latest copy of the merged release, if any
        :param schema: the release schema (if not provided, will default to the latest version of OCDS)
        :param dict merge_rules: the merge rules (if not provided, will determine the rules from the ``schema``)
        :param dict rule_overrides: any rule overrides, in which keys are field paths as tuples, and values are either
            ``ocdsmerge.APPEND`` or ``ocdsmerge.MERGE_BY_POSITION``
        :type schema: dict or str
        """
        if merge_rules is None:
            merge_rules = get_merge_rules(schema)
        if rule_overrides is None:
            rule_overrides = {}

        self.merge_rules = merge_rules
        self.rule_overrides = rule_overrides

        if data is None:
            self.data = {}
        else:
            self.data = flatten(data, self.merge_rules, self.rule_overrides, flattened={}, versioned=self.versioned)

    def asdict(self):
        """
        Returns the merged release as a dictionary.
        """
        return unflatten(self.data)

    def extend(self, releases):
        """
        Sorts and merges many releases into the merged release.
        """
        for release in sorted_releases(releases):
            self.append(release)

    def append(self, release):
        """
        Merges one release into the merged release.
        """
        release = release.copy()

        # Store the values of fields that set "omitWhenMerged": true.
        ocid = release.get('ocid')
        release_id = release.get('id')
        date = release.get('date')
        # Prior to OCDS 1.1.4, `tag` didn't set "omitWhenMerged": true.
        tag = release.pop('tag', None)

        flat = flatten(release, self.merge_rules, self.rule_overrides, flattened={})
        self.flat_append(flat, ocid, release_id, date, tag)

    def flat_append(self, flat, ocid, release_id, date, tag):
        raise NotImplementedError('subclasses must implement flat_append()')

class CompiledRelease(MergedRelease):
    versioned = False

    def __init__(self, data=None, **kwargs):
        super().__init__(data, **kwargs)
        self.data[('tag',)] = ['compiled']

    def flat_append(self, flat, ocid, release_id, date, tag):
        # Add an `id` and `date`.
        self.data[('id',)] = '{}-{}'.format(ocid, date)
        self.data[('date',)] = date
        # In OCDS 1.0, `ocid` incorrectly sets "mergeStrategy": "ocdsOmit".
        self.data[('ocid',)] = ocid

        self.data.update(flat)

class VersionedRelease(MergedRelease):
    versioned = True

    def flat_append(self, flat, ocid, release_id, date, tag):
        # Don't version the OCID.
        flat.pop(('ocid',), None)
        self.data[('ocid',)] = ocid

        for key, value in flat.items():
            # If key is not versioned, continue. If the value is unchanged, don't add it to the history.
            if key in self.data and (type(self.data[key]) is not list or value == self.data[key][-1]['value']):
                continue

            if key not in self.data:
                self.data[key] = []

            self.data[key].append({
                'releaseID': release_id,
                'releaseDate': date,
                'releaseTag': tag,
                'value': value,
            })
<commit_msg>Use setdefault<commit_after>from ocdsmerge.flatten import flatten, unflatten
from ocdsmerge.rules import get_merge_rules
from ocdsmerge.util import sorted_releases

class Merger:
    def __init__(self, schema=None, merge_rules=None, rule_overrides=None):
        """
        Initializes a reusable ``Merger`` instance for creating merged releases.

        :param schema: the release schema (if not provided, will default to the latest version of OCDS)
        :param dict merge_rules: the merge rules (if not provided, will determine the rules from the ``schema``)
        :param dict rule_overrides: any rule overrides, in which keys are field paths as tuples, and values are either
            ``ocdsmerge.APPEND`` or ``ocdsmerge.MERGE_BY_POSITION``
        :type schema: dict or str
        """
        if merge_rules is None:
            merge_rules = get_merge_rules(schema)
        if rule_overrides is None:
            rule_overrides = {}

        self.merge_rules = merge_rules
        self.rule_overrides = rule_overrides

    def create_compiled_release(self, releases):
        """
        Merges a list of releases into a compiled release.
        """
        return self._create_merged_release(CompiledRelease, releases)

    def create_versioned_release(self, releases):
        """
        Merges a list of releases into a versioned release.
        """
        return self._create_merged_release(VersionedRelease, releases)

    def _create_merged_release(self, cls, releases):
        merged_release = cls(merge_rules=self.merge_rules, rule_overrides=self.rule_overrides)
        merged_release.extend(releases)
        return merged_release.asdict()

class MergedRelease:
    """
    Whether the class is for merging versioned releases.
    """
    versioned = None

    def __init__(self, data=None, schema=None, merge_rules=None, rule_overrides=None):
        """
        Initializes a merged release.

        :param dict data: the latest copy of the merged release, if any
        :param schema: the release schema (if not provided, will default to the latest version of OCDS)
        :param dict merge_rules: the merge rules (if not provided, will determine the rules from the ``schema``)
        :param dict rule_overrides: any rule overrides, in which keys are field paths as tuples, and values are either
            ``ocdsmerge.APPEND`` or ``ocdsmerge.MERGE_BY_POSITION``
        :type schema: dict or str
        """
        if merge_rules is None:
            merge_rules = get_merge_rules(schema)
        if rule_overrides is None:
            rule_overrides = {}

        self.merge_rules = merge_rules
        self.rule_overrides = rule_overrides

        if data is None:
            self.data = {}
        else:
            self.data = flatten(data, self.merge_rules, self.rule_overrides, flattened={}, versioned=self.versioned)

    def asdict(self):
        """
        Returns the merged release as a dictionary.
        """
        return unflatten(self.data)

    def extend(self, releases):
        """
        Sorts and merges many releases into the merged release.
        """
        for release in sorted_releases(releases):
            self.append(release)

    def append(self, release):
        """
        Merges one release into the merged release.
        """
        release = release.copy()

        # Store the values of fields that set "omitWhenMerged": true.
        ocid = release.get('ocid')
        release_id = release.get('id')
        date = release.get('date')
        # Prior to OCDS 1.1.4, `tag` didn't set "omitWhenMerged": true.
        tag = release.pop('tag', None)

        flat = flatten(release, self.merge_rules, self.rule_overrides, flattened={})
        self.flat_append(flat, ocid, release_id, date, tag)

    def flat_append(self, flat, ocid, release_id, date, tag):
        raise NotImplementedError('subclasses must implement flat_append()')

class CompiledRelease(MergedRelease):
    versioned = False

    def __init__(self, data=None, **kwargs):
        super().__init__(data, **kwargs)
        self.data[('tag',)] = ['compiled']

    def flat_append(self, flat, ocid, release_id, date, tag):
        # Add an `id` and `date`.
        self.data[('id',)] = '{}-{}'.format(ocid, date)
        self.data[('date',)] = date
        # In OCDS 1.0, `ocid` incorrectly sets "mergeStrategy": "ocdsOmit".
        self.data[('ocid',)] = ocid

        self.data.update(flat)

class VersionedRelease(MergedRelease):
    versioned = True

    def flat_append(self, flat, ocid, release_id, date, tag):
        # Don't version the OCID.
        flat.pop(('ocid',), None)
        self.data[('ocid',)] = ocid

        for key, value in flat.items():
            # If key is not versioned, continue. If the value is unchanged, don't add it to the history.
            if key in self.data and (type(self.data[key]) is not list or value == self.data[key][-1]['value']):
                continue

            self.data.setdefault(key, []).append({
                'releaseID': release_id,
                'releaseDate': date,
                'releaseTag': tag,
                'value': value,
            })
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Oct2Py is a means to seamlessly call M-files and GNU Octave functions from Python.
It manages the Octave session for you, sharing data behind the scenes using
MAT files.  Usage is as simple as:

.. code-block:: python

    >>> import oct2py
    >>> oc = oct2py.Oct2Py()
    >>> x = oc.zeros(3,3)
    >>> print(x, x.dtype.str)  # doctest: +SKIP
    [[ 0.  0.  0.]
     [ 0.  0.  0.]
     [ 0.  0.  0.]] <f8

If you want to run legacy m-files, do not have MATLAB(TM), and do not fully
trust a code translator, this is your library.
"""

__title__ = 'oct2py'
__version__ = '3.3.1'
__author__ = 'Steven Silvester'
__license__ = 'MIT'
__copyright__ = 'Copyright 2014 Steven Silvester'
__all__ = ['Oct2Py', 'Oct2PyError', 'octave', 'Struct', 'demo', 'speed_test',
           'thread_test', '__version__', 'get_log']

import imp
import functools
import os
import ctypes

try:
    import thread
except ImportError:
    import _thread as thread

if os.name == 'nt':
    """
    Allow Windows to intecept KeyboardInterrupt
    http://stackoverflow.com/questions/15457786/ctrl-c-crashes-python-after-importing-scipy-stats
    """
    basepath = imp.find_module('numpy')[1]
    try:
        lib1 = ctypes.CDLL(os.path.join(basepath, 'core', 'libmmd.dll'))
        lib2 = ctypes.CDLL(os.path.join(basepath, 'core', 'libifcoremd.dll'))

        def handler(sig, hook=thread.interrupt_main):
            hook()
            return 1

        routine = ctypes.WINFUNCTYPE(ctypes.c_int, ctypes.c_uint)(handler)
        ctypes.windll.kernel32.SetConsoleCtrlHandler(routine, 1)
    except Exception:
        pass

from .core import Oct2Py, Oct2PyError
from .utils import Struct, get_log
from .demo import demo
from .speed_check import speed_check
from .thread_check import thread_check

try:
    octave = Oct2Py()
except Oct2PyError as e:
    print(e)

def kill_octave():
    """Kill all octave instances (cross-platform).

    This will restart the "octave" instance.  If you have instantiated
    Any other Oct2Py objects, you must restart them.
    """
    import os
    if os.name == 'nt':
        os.system('taskkill /im octave /f')
    else:
        os.system('killall -9 octave')
        os.system('killall -9 octave-cli')
    octave.restart()

# clean up namespace
del functools, imp, os, ctypes, thread
<commit_msg>Keep examples in sync<commit_after># -*- coding: utf-8 -*-
"""
Oct2Py is a means to seamlessly call M-files and GNU Octave functions from Python.
It manages the Octave session for you, sharing data behind the scenes using
MAT files.  Usage is as simple as:

.. code-block:: python

    >>> import oct2py
    >>> oc = oct2py.Oct2Py()
    >>> x = oc.zeros(3,3)
    >>> print(x, x.dtype.str)  # doctest: +SKIP
    [[ 0.  0.  0.]
     [ 0.  0.  0.]
     [ 0.  0.  0.]] <f8

If you want to run legacy m-files, do not have MATLAB(TM), and do not fully
trust a code translator, this is your library.
"""

__title__ = 'oct2py'
__version__ = '3.3.2'
__author__ = 'Steven Silvester'
__license__ = 'MIT'
__copyright__ = 'Copyright 2014-2015 Steven Silvester'
__all__ = ['Oct2Py', 'Oct2PyError', 'octave', 'Struct', 'demo', 'speed_test',
           'thread_test', '__version__', 'get_log']

import imp
import functools
import os
import ctypes

try:
    import thread
except ImportError:
    import _thread as thread

if os.name == 'nt':
    """
    Allow Windows to intecept KeyboardInterrupt
    http://stackoverflow.com/questions/15457786/ctrl-c-crashes-python-after-importing-scipy-stats
    """
    basepath = imp.find_module('numpy')[1]
    try:
        lib1 = ctypes.CDLL(os.path.join(basepath, 'core', 'libmmd.dll'))
        lib2 = ctypes.CDLL(os.path.join(basepath, 'core', 'libifcoremd.dll'))

        def handler(sig, hook=thread.interrupt_main):
            hook()
            return 1

        routine = ctypes.WINFUNCTYPE(ctypes.c_int, ctypes.c_uint)(handler)
        ctypes.windll.kernel32.SetConsoleCtrlHandler(routine, 1)
    except Exception:
        pass

from .core import Oct2Py, Oct2PyError
from .utils import Struct, get_log
from .demo import demo
from .speed_check import speed_check
from .thread_check import thread_check

try:
    octave = Oct2Py()
except Oct2PyError as e:
    print(e)

def kill_octave():
    """Kill all octave instances (cross-platform).

    This will restart the "octave" instance.  If you have instantiated
    Any other Oct2Py objects, you must restart them.
    """
    import os
    if os.name == 'nt':
        os.system('taskkill /im octave /f')
    else:
        os.system('killall -9 octave')
        os.system('killall -9 octave-cli')
    octave.restart()

# clean up namespace
del functools, imp, os, ctypes, thread
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Swagger Support
~~~~~~~~~~~~~~~

Built in support for generating a swagger spec, along with built in Swagger UI.

To enable add the :py:class:`SwaggerSpec` resource API into your API::

    >>> from odinweb import api
    >>> from odinweb.swagger import SwaggerSpec
    >>> my_api = api.ApiCollection(
    ...    SwaggerSpec("Title of my Swagger spec", enable_ui=True),
    ... )

"""
import collections
import os

# Imported for typing support
from typing import List, Dict, Any, Union, Tuple  # noqa

from odin.fields.virtual import VirtualField
from .data_structures import PathParam  # noqa

from odin import fields
from odin.utils import getmeta, lazy_property, force_tuple

from . import doc
from . import resources
from ._compat import binary_type
from .constants import HTTPStatus, Type
from .containers import ResourceApi, CODECS
from .data_structures import UrlPath, Param, HttpResponse, NoPath
from .decorators import Operation
from .exceptions import HttpError
from .utils import dict_filter

SWAGGER_SPEC_TYPE_MAPPING = [
    (fields.IntegerField, Type.Long),
    (fields.FloatField, Type.Float),
    (fields.EmailField, Type.Email),
    (fields.StringField, Type.String),
    (fields.TimeField, Type.Time),
    (fields.DateField, Type.Date),
    (fields.DateTimeField, Type.DateTime),
    (fields.BooleanField, Type.Boolean),
]
"""
Mapping of fields to Swagger types.
"""

def map_field_to_type(field):
    # type: (Any) -> Type
    for field_type, type_ in SWAGGER_SPEC_TYPE_MAPPING:
        if isinstance(field, field_type):
            return type_

def resource_definition(resource):
    """
    Generate a `Swagger Definitions Object <http://swagger.io/specification/#definitionsObject>`_
    from a resource.

    """
    meta = getmeta(resource)

    definition = {
        'type': "object",
        'properties': {}
    }

    for field in meta.all_fields:
        field_definition = {}

        type_def = map_field_to_type(field)
        if type_def:
            field_definition['type'] = str(type_def)
            if type_def.format:
                field_definition['format'] = type_def.format

        if field.doc_text:
            field_definition['description'] = field.doc_text

        if isinstance(field, VirtualField):
            field_definition['readOnly'] = True

        # Use getattr to support calculated fields
        if getattr(field, 'choices', None):
            field_definition['enum'] = [c[0] for c in field.choices]

        definition['properties'][field.name] = field_definition

    return definition

class SwaggerSpec(ResourceApi):
    """
    Resource API instance that generates a Swagger spec of the current API.
    """
    api_name = 'swagger'
    SWAGGER_TAG = 'swagger'
    tags = (SWAGGER_TAG, )

    static_path = os.path.join(os.path.dirname(__file__), 'static')

    def __init__(self, title, enabled=True, enable_ui=False, host=None, schemes=None):
        # type: (str, bool, bool, str, Union[str, Tuple[str]]) -> None
        # Register operations
        if enabled:
            self._operations.append(Operation(SwaggerSpec.get_swagger)
)
            if enable_ui:
                self._operations.append(Operation(SwaggerSpec.get_ui, UrlPath.parse('ui')))
                self._operations.append(Operation(SwaggerSpec.get_static, UrlPath.parse('ui/{file_name:String}')))

        super(SwaggerSpec, self).__init__()
        self.title = title
        self.enabled = enabled
        self.enable_ui = enabled and enable_ui
        self.host = host
        self.schemes = set(force_tuple(schemes or ()))

        self._ui_cache = None

    @lazy_property
    def base_path(self):
        """
        Calculate the APIs base path
        """
        path = UrlPath()

        # Walk up the API to find the base object
        parent = self.parent
        while parent:
            path_prefix = getattr(parent, 'path_prefix', NoPath)
            path = path_prefix + path
            parent = getattr(parent, 'parent', None)

        return path

    @property
    def swagger_path(self):
        return self.base_path + 'swagger'

    @staticmethod
    def generate_parameters(path):
        # type: (UrlPath) -> List[Dict[str, Any]]
        return [Param.path(node.name, node.type).to_swagger() for node in path.path_nodes]

    @staticmethod
    def swagger_node_formatter(path_node):
        # type: (PathParam) -> str
        """
        Format a node for swagger spec (default formatter for the format method).
        """
        return "{{{}}}".format(path_node.name)

    def parse_operations(self):
        """
        Flatten routes into a path -> method -> route structure
        """
        resource_defs = {
            getmeta(resources.Error).resource_name: resource_definition(resources.Error),
            getmeta(resources.Listing).resource_name: resource_definition(resources.Listing),
        }

        paths = collections.OrderedDict()
        for path, operation in self.parent.op_paths():
            # Cut of first item (will be the parents path)
            path = '/' + path[1:]  # type: UrlPath

            # Filter out swagger endpoints
            if self.SWAGGER_TAG in operation.tags:
                continue

            # Add to resource definitions
            if operation.resource:
                resource_defs[getmeta(operation.resource).resource_name] = resource_definition(operation.resource)

            # Add path parameters
            path_spec = paths.setdefault(path.format(self.swagger_node_formatter), {})

            # Add parameters
            parameters = self.generate_parameters(path)
            if parameters:
                path_spec['parameters'] = parameters

            # Add methods
            for method in operation.methods:
                path_spec[method.value.lower()] = operation.to_swagger()

        return paths, resource_defs

    @doc.response(HTTPStatus.OK, "Swagger JSON of this API")
    def get_swagger(self, request):
        """
        Generate this document.
        """
        api_base = self.parent
        paths, definitions = self.parse_operations()
        return dict_filter({
            'swagger': '2.0',
            'info': {
                'title': self.title,
                'version': str(getattr(api_base, 'version', 0))
            },
            'host': self.host or request.host,
            'schemes': list(self.schemes) or None,
            'basePath': str(self.base_path),
            'consumes': list(CODECS.keys()),
            'produces': list(CODECS.keys()),
            'paths': paths,
            'definitions': definitions,
        })

    def load_static(self, file_name):
        file_path = os.path.abspath(os.path.join(self.static_path, file_name))
        # This is a security check to ensure this is not abused to
        # read files outside of the static folder.
        if not file_path.startswith(self.static_path):
            raise HttpError(HTTPStatus.NOT_FOUND, 42)

        try:
            return open(file_path, 'rb').read()
        except IOError:
            raise HttpError(HTTPStatus.NOT_FOUND, 42)

    @doc.response(HTTPStatus.OK, "HTML content")
    @doc.produces('text/html')
    def get_ui(self, _):
        """
        Load the Swagger UI interface
        """
        if not self._ui_cache:
            content = self.load_static('ui.html')
            if isinstance(content, binary_type):
                content = content.decode('UTF-8')
            self._ui_cache = content.replace(u"{{SWAGGER_PATH}}", str(self.swagger_path))
        return HttpResponse(self._ui_cache, headers={
            'Content-Type': 'text/html'
        })

    @doc.response(HTTPStatus.OK, "HTML content")
    def get_static(self, _, file_name=None):
        """
        Get static content for UI.
        """
        content_type = {
            'ss': 'text/css',
            'js': 'application/javascript',
        }.get(file_name[-2:])
        if not content_type:
            raise HttpError(HTTPStatus.NOT_FOUND, 42)

        return HttpResponse(self.load_static(file_name), headers={
            'Content-Type': content_type,
            'Content-Encoding': 'gzip',
            'Cache-Control': 'public, max-age=300',
        })
<commit_msg>Better handling of readonly<commit_after># -*- coding: utf-8 -*-
"""
Swagger Support
~~~~~~~~~~~~~~~

Built in support for generating a swagger spec, along with built in Swagger UI.

To enable add the :py:class:`SwaggerSpec` resource API into your API::

    >>> from odinweb import api
    >>> from odinweb.swagger import SwaggerSpec
    >>> my_api = api.ApiCollection(
    ...    SwaggerSpec("Title of my Swagger spec", enable_ui=True),
    ... )

"""
import collections
import os

# Imported for typing support
from typing import List, Dict, Any, Union, Tuple  # noqa

from odin.fields.virtual import VirtualField
from .data_structures import PathParam  # noqa

from odin import fields
from odin.utils import getmeta, lazy_property, force_tuple

from . import doc
from . import resources
from ._compat import binary_type
from .constants import HTTPStatus, Type
from .containers import ResourceApi, CODECS
from .data_structures import UrlPath, Param, HttpResponse, NoPath
from .decorators import Operation
from .exceptions import HttpError
from .utils import dict_filter

SWAGGER_SPEC_TYPE_MAPPING = [
    (fields.IntegerField, Type.Long),
    (fields.FloatField, Type.Float),
    (fields.EmailField, Type.Email),
    (fields.StringField, Type.String),
    (fields.TimeField, Type.Time),
    (fields.DateField, Type.Date),
    (fields.DateTimeField, Type.DateTime),
    (fields.BooleanField, Type.Boolean),
]
"""
Mapping of fields to Swagger types.
"""

def map_field_to_type(field):
    # type: (Any) -> Type
    for field_type, type_ in SWAGGER_SPEC_TYPE_MAPPING:
        if isinstance(field, field_type):
            return type_

def resource_definition(resource):
    """
    Generate a `Swagger Definitions Object <http://swagger.io/specification/#definitionsObject>`_
    from a resource.

    """
    meta = getmeta(resource)

    definition = {
        'type': "object",
        'properties': {}
    }

    for field in meta.all_fields:
        field_definition = {}

        type_def = map_field_to_type(field)
        if type_def:
            field_definition['type'] = str(type_def)
            if type_def.format:
                field_definition['format'] = type_def.format

        if field.doc_text:
            field_definition['description'] = field.doc_text

        if isinstance(field, VirtualField) or field in meta.readonly_fields:
            field_definition['readOnly'] = True

        # Use getattr to support calculated fields
        if getattr(field, 'choices', None):
            field_definition['enum'] = [c[0] for c in field.choices]

        definition['properties'][field.name] = field_definition

    return definition

class SwaggerSpec(ResourceApi):
    """
    Resource API instance that generates a Swagger spec of the current API.
    """
    api_name = 'swagger'
    SWAGGER_TAG = 'swagger'
    tags = (SWAGGER_TAG, )

    static_path = os.path.join(os.path.dirname(__file__), 'static')

    def __init__(self, title, enabled=True, enable_ui=False, host=None, schemes=None):
        # type: (str, bool, bool, str, Union[str, Tuple[str]]) -> None
        # Register operations
        if enabled:
            self._operations.append(Operation(SwaggerSpec.get_swagger)
)
            if enable_ui:
                self._operations.append(Operation(SwaggerSpec.get_ui, UrlPath.parse('ui')))
                self._operations.append(Operation(SwaggerSpec.get_static, UrlPath.parse('ui/{file_name:String}')))

        super(SwaggerSpec, self).__init__()
        self.title = title
        self.enabled = enabled
        self.enable_ui = enabled and enable_ui
        self.host = host
        self.schemes = set(force_tuple(schemes or ()))

        self._ui_cache = None

    @lazy_property
    def base_path(self):
        """
        Calculate the APIs base path
        """
        path = UrlPath()

        # Walk up the API to find the base object
        parent = self.parent
        while parent:
            path_prefix = getattr(parent, 'path_prefix', NoPath)
            path = path_prefix + path
            parent = getattr(parent, 'parent', None)

        return path

    @property
    def swagger_path(self):
        return self.base_path + 'swagger'

    @staticmethod
    def generate_parameters(path):
        # type: (UrlPath) -> List[Dict[str, Any]]
        return [Param.path(node.name, node.type).to_swagger() for node in path.path_nodes]

    @staticmethod
    def swagger_node_formatter(path_node):
        # type: (PathParam) -> str
        """
        Format a node for swagger spec (default formatter for the format method).
        """
        return "{{{}}}".format(path_node.name)

    def parse_operations(self):
        """
        Flatten routes into a path -> method -> route structure
        """
        resource_defs = {
            getmeta(resources.Error).resource_name: resource_definition(resources.Error),
            getmeta(resources.Listing).resource_name: resource_definition(resources.Listing),
        }

        paths = collections.OrderedDict()
        for path, operation in self.parent.op_paths():
            # Cut of first item (will be the parents path)
            path = '/' + path[1:]  # type: UrlPath

            # Filter out swagger endpoints
            if self.SWAGGER_TAG in operation.tags:
                continue

            # Add to resource definitions
            if operation.resource:
                resource_defs[getmeta(operation.resource).resource_name] = resource_definition(operation.resource)

            # Add path parameters
            path_spec = paths.setdefault(path.format(self.swagger_node_formatter), {})

            # Add parameters
            parameters = self.generate_parameters(path)
            if parameters:
                path_spec['parameters'] = parameters

            # Add methods
            for method in operation.methods:
                path_spec[method.value.lower()] = operation.to_swagger()

        return paths, resource_defs

    @doc.response(HTTPStatus.OK, "Swagger JSON of this API")
    def get_swagger(self, request):
        """
        Generate this document.
        """
        api_base = self.parent
        paths, definitions = self.parse_operations()
        return dict_filter({
            'swagger': '2.0',
            'info': {
                'title': self.title,
                'version': str(getattr(api_base, 'version', 0))
            },
            'host': self.host or request.host,
            'schemes': list(self.schemes) or None,
            'basePath': str(self.base_path),
            'consumes': list(CODECS.keys()),
            'produces': list(CODECS.keys()),
            'paths': paths,
            'definitions': definitions,
        })

    def load_static(self, file_name):
        file_path = os.path.abspath(os.path.join(self.static_path, file_name))
        # This is a security check to ensure this is not abused to
        # read files outside of the static folder.
        if not file_path.startswith(self.static_path):
            raise HttpError(HTTPStatus.NOT_FOUND, 42)

        try:
            return open(file_path, 'rb').read()
        except IOError:
            raise HttpError(HTTPStatus.NOT_FOUND, 42)

    @doc.response(HTTPStatus.OK, "HTML content")
    @doc.produces('text/html')
    def get_ui(self, _):
        """
        Load the Swagger UI interface
        """
        if not self._ui_cache:
            content = self.load_static('ui.html')
            if isinstance(content, binary_type):
                content = content.decode('UTF-8')
            self._ui_cache = content.replace(u"{{SWAGGER_PATH}}", str(self.swagger_path))
        return HttpResponse(self._ui_cache, headers={
            'Content-Type': 'text/html'
        })

    @doc.response(HTTPStatus.OK, "HTML content")
    def get_static(self, _, file_name=None):
        """
        Get static content for UI.
        """
        content_type = {
            'ss': 'text/css',
            'js': 'application/javascript',
        }.get(file_name[-2:])
        if not content_type:
            raise HttpError(HTTPStatus.NOT_FOUND, 42)

        return HttpResponse(self.load_static(file_name), headers={
            'Content-Type': content_type,
            'Content-Encoding': 'gzip',
            'Cache-Control': 'public, max-age=300',
        })
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Sat Aug 12 18:08:52 2017

@author: Shaun Werkhoven
@purpose: To create a image classification system for the Image Intelligence
TakeHome Assignment

Licensed under the Apache License, Version 2.0 (the "License");

Simple image classification with TensorFlow and the Inception model.

Run image classification with Inception trained on ImageNet 2012 Challenge data
set.

This program creates a graph from a saved GraphDef protocol buffer,
and runs inference on an input JPEG, GIF or PNG image. It outputs human readable
strings of the top 5 predictions along with their probabilities.

"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import os.path
import re
import sys
import tarfile

import numpy as np
import tensorflow as tf

import json
import urllib
from flask import Flask, jsonify, render_template, abort, make_response, request, Markup
from flask_httpauth import HTTPBasicAuth
from PIL import Image

auth = HTTPBasicAuth()
app = Flask(__name__)

FLAGS = None

DATA_URL = 'http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz'

images = [
    {
        'id': 1,
        'title': u'Nikes',
        'url': 'http://imgdirect.s3-website-us-west-2.amazonaws.com/nike.jpg',
        'results': '',
        'resize': False,
        'size': ""
    },
    {
        'id': 2,
        'title': u'Altra',
        'url': 'https://s3-us-west-2.amazonaws.com/imgdirect/altra.jpg',
        'results': '',
        'resize': False,
        'size': ""
    }
]

@auth.error_handler
def unauthorized():
    return make_response(jsonify({'error': 'unauthorized access'}), 403)

@app.errorhandler(404)
def not_found(error):
    return make_response(jsonify({'error': 'not found'}), 404)

@app.errorhandler(400)
def bad_request(error):
    return make_response(jsonify({'error': 'missing json data'}), 400)

@app.errorhandler(410)
def missing_URL(error):
    return make_response(jsonify({'error': 'missing URL field'}), 410)

@app.route('/')
@app.route('/index')
def index():
    return "Hello, World!"

### test string
### curl -i http://127.0.0.1:5000/img/api/v1.0/images
@app.route('/img/api/v1.0/images', methods=['GET'])
#@auth.login_required
def get_imgs():
    return jsonify({'images': images})

### test String
### curl -i http://127.0.0.1:5000/img/api/v1.0/images/2
@app.route('/img/api/v1.0/images/<int:img_id>', methods = ['GET'])
#@auth.login_required
def get_img(img_id):
    img = [img for img in images if img['id'] == img_id]
    if len(img) == 0:
        abort(404)
    return jsonify({'img': img[0]})

### test String
### curl -i -H "Content-Type: application/json" -X POST -d '{"url":"http://imgdirect.s3-website-us-west-2.amazonaws.com/neither.jpg"}' http://127.0.0.1:5000/img/api/v1.0/images
@app.route('/img/api/v1.0/images', methods = ['POST'])
#@auth.login_required
def add_imgs():
    if not request.json:
        abort(400)

    missing_url = False
    json_str = request.json
    img_data = json_str['new_imgs']
    new_images = []

    for img in img_data:
        if img.get('url') == None:
            missing_url = True
            continue

        if img.get('title') == None:
            new_title = ""
        else:
            new_title = img.get('title')
        if img.get('results') == None:
            new_results = ""
        else:
            new_results = img.get('results')

        image = {
            # simple way to ensure a unique id
            'id' : images[-1]['id'] + 1,
            'title': new_title,
            # url is required, otherwise return error
            'url': img['url'],
            'results': new_results,
            'resize': False,
            'size': ""
        }
        images.append(image)
        new_images.append(image)

    if missing_url:
        return_val = jsonify(new_images), 410
    else:
        return_val = jsonify(new_images), 201

    return return_val

### test String
### curl -i -H "Content-Type: application/json" -X DELETE http://127.0.0.1:5000/img/api/v1.0/images/3
@app.route('/img/api/v1.0/images/<int:img_id>', methods=['DELETE'])
#@auth.login_required
def delete_img(img_id):
    img = [img for img in images if img['id'] == img_id]
    if len(img) == 0:
        abort(404)
    images.remove(img[0])
    return jsonify({'result': True})

def main():
    app.run(host = '0.0.0.0')

if __name__ == '__main__':
    main()
    <commit_msg>start to build in tensorflow recognition engine. This adds argument parsing for tensorflow<commit_after># -*- coding: utf-8 -*-
"""
Created on Sat Aug 12 18:08:52 2017

@author: Shaun Werkhoven
@purpose: To create a image classification system for the Image Intelligence
TakeHome Assignment

Licensed under the Apache License, Version 2.0 (the "License");

Simple image classification with TensorFlow and the Inception model.

Run image classification with Inception trained on ImageNet 2012 Challenge data
set.

This program creates a graph from a saved GraphDef protocol buffer,
and runs inference on an input JPEG, GIF or PNG image. It outputs human readable
strings of the top 5 predictions along with their probabilities.

"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import os.path
import re
import sys
import tarfile

import numpy as np
import tensorflow as tf

import json
import urllib
from flask import Flask, jsonify, render_template, abort, make_response, request, Markup
from flask_httpauth import HTTPBasicAuth
from PIL import Image

auth = HTTPBasicAuth()
app = Flask(__name__)

FLAGS = None

DATA_URL = 'http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz'

images = [
    {
        'id': 1,
        'title': u'Nikes',
        'url': 'http://imgdirect.s3-website-us-west-2.amazonaws.com/nike.jpg',
        'results': '',
        'resize': False,
        'size': ""
    },
    {
        'id': 2,
        'title': u'Altra',
        'url': 'https://s3-us-west-2.amazonaws.com/imgdirect/altra.jpg',
        'results': '',
        'resize': False,
        'size': ""
    }
]

@auth.error_handler
def unauthorized():
    return make_response(jsonify({'error': 'unauthorized access'}), 403)

@app.errorhandler(404)
def not_found(error):
    return make_response(jsonify({'error': 'not found'}), 404)

@app.errorhandler(400)
def bad_request(error):
    return make_response(jsonify({'error': 'missing json data'}), 400)

@app.errorhandler(410)
def missing_URL(error):
    return make_response(jsonify({'error': 'missing URL field'}), 410)

@app.route('/')
@app.route('/index')
def index():
    return "Hello, World!"

### test string
### curl -i http://127.0.0.1:5000/img/api/v1.0/images
@app.route('/img/api/v1.0/images', methods=['GET'])
#@auth.login_required
def get_imgs():
    return jsonify({'images': images})

### test String
### curl -i http://127.0.0.1:5000/img/api/v1.0/images/2
@app.route('/img/api/v1.0/images/<int:img_id>', methods = ['GET'])
#@auth.login_required
def get_img(img_id):
    img = [img for img in images if img['id'] == img_id]
    if len(img) == 0:
        abort(404)
    return jsonify({'img': img[0]})

### test String
### curl -i -H "Content-Type: application/json" -X POST -d '{"url":"http://imgdirect.s3-website-us-west-2.amazonaws.com/neither.jpg"}' http://127.0.0.1:5000/img/api/v1.0/images
@app.route('/img/api/v1.0/images', methods = ['POST'])
#@auth.login_required
def add_imgs():
    if not request.json:
        abort(400)

    missing_url = False
    json_str = request.json
    img_data = json_str['new_imgs']
    new_images = []

    for img in img_data:
        if img.get('url') == None:
            missing_url = True
            continue

        if img.get('title') == None:
            new_title = ""
        else:
            new_title = img.get('title')
        if img.get('results') == None:
            new_results = ""
        else:
            new_results = img.get('results')

        image = {
            # simple way to ensure a unique id
            'id' : images[-1]['id'] + 1,
            'title': new_title,
            # url is required, otherwise return error
            'url': img['url'],
            'results': new_results,
            'resize': False,
            'size': ""
        }
        images.append(image)
        new_images.append(image)

    if missing_url:
        return_val = jsonify(new_images), 410
    else:
        return_val = jsonify(new_images), 201

    return return_val

### test String
### curl -i -H "Content-Type: application/json" -X DELETE http://127.0.0.1:5000/img/api/v1.0/images/3
@app.route('/img/api/v1.0/images/<int:img_id>', methods=['DELETE'])
#@auth.login_required
def delete_img(img_id):
    img = [img for img in images if img['id'] == img_id]
    if len(img) == 0:
        abort(404)
    images.remove(img[0])
    return jsonify({'result': True})

def main(_):
    app.run(host = '0.0.0.0')

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
  # classify_image_graph_def.pb:
  #   Binary representation of the GraphDef protocol buffer.
  # imagenet_synset_to_human_label_map.txt:
  #   Map from synset ID to a human readable string.
  # imagenet_2012_challenge_label_map_proto.pbtxt:
  #   Text representation of a protocol buffer mapping a label to synset ID.
    parser.add_argument(
      '--model_dir',
      type = str,
      default = '/tmp/imagenet',
      help = """\
      Path to classify_image_graph_def.pb,
      imagenet_synset_to_human_label_map.txt, and
      imagenet_2012_challenge_label_map_proto.pbtxt.\
      """
  )
    parser.add_argument(
      '--image_file',
      type = str,
      default = '',
      help = 'Absolute path to image file.'
  )
    parser.add_argument(
      '--num_top_predictions',
      type = int,
      default = 5,
      help = 'Display this many predictions.'
  )
    FLAGS, unparsed = parser.parse_known_args()
    tf.app.run(main = main, argv = [sys.argv[0]] + unparsed)
    <|endoftext|>