<commit_before>#!/usr/bin/env python
import numpy as np
import scipy.optimize
import scipy.linalg
import scipy.spatial
import rbf.basis
import rbf.poly
import rbf.geometry
import modest
import matplotlib.pyplot as plt

def product_trace(A,B):
  '''
  efficiently returns the trace of a matrix product. this is used for
  general cross validation
  '''
  return np.sum(A*B.T)

def laplacian_diff_op(dim):
  '''
  returns the coefficients and derivatives for each term in a
  Laplacian differential operator for the given number of dimensions.
  For example, calling this function with dim=2 will return

    array([[1.0,(2,0)],
           [1.0,(0,2)]], dtype=object),

  which is represents the differential operator

    1.0*d^2/dx_1^2 + 1.0*d^2/dx_2^2

  '''
  I = np.eye(dim,dtype=int)
  out = [[1.0,tuple(2*i)] for i in I]
  return np.array(out,dtype=object)

def coefficient_matrix(x,eps=None,basis=rbf.basis.phs3,order=0):
  '''
  returns the matrix that maps the coefficients to the function values
  at the observation points.

  Parameters
  ----------
    x: (N,D) array of observation points
    diff: (D,) tuple of derivatives
    basis (default=phs3): radial basis function
    order (default=0): additional polynomial order
  '''
  # number of observation points and spatial dimensions
  N,D = x.shape

  # powers for the additional polynomials
  powers = rbf.poly.monomial_powers(order,D)
  # number of polynomial terms
  P = powers.shape[0]

  # allocate array
  A = np.zeros((N+P,N+P))
  A[:N,:N] = basis(x,x,eps=eps)
  Ap = rbf.poly.mvmonos(x,powers)
  A[N:,:N] = Ap.T
  A[:N,N:] = Ap
  return A

def regularization_matrix(x,eps=None,basis=rbf.basis.phs3,order=0,diff=None):
  # number of observation points and spatial dimensions
  N,D = x.shape
  # powers for the additional polynomials
  powers = rbf.poly.monomial_powers(order,D)

  # number of polynomial terms
  P = powers.shape[0]

  # have the regularization minimize the size of the RBF coefficients
  A = np.zeros((N+P,N+P))
  A[:N,:N] = np.eye(N)

  #A[:,:N] = basis(x,x,eps=eps,diff=diff)
  #A[:,N:] = rbf.poly.mvmonos(x,powers,diff=diff)
  return A

def interpolation_matrix(xitp,x,diff=None,eps=None,basis=rbf.basis.phs3,order=0):
  # number of interpolation points and spatial dimensions
  I,D = xitp.shape

  # number of observation points
  N = x.shape[0]

  # powers for the additional polynomials
  powers = rbf.poly.monomial_powers(order,D)
  # number of polynomial terms
  P = powers.shape[0]

  A = np.zeros((I,N+P))
  A[:,:N] = basis(xitp,x,eps=eps,diff=diff)
  A[:,N:] = rbf.poly.mvmonos(xitp,powers,diff=diff)
  return A

def find_coeff(A,L,value,damping):
  '''
  Parameters
  ----------
    A: (N+P,N+P) coefficient matrix
    L: (K,K) regularization matrix
    value: (N,...) observations
    damping: scalar damping parameter
  '''
  # number of data points
  N = value.shape[0]
  # number of added polynomials
  P = A.shape[0] - N

  # number of model parameters
  M = P + N

  # extend values to have a consistent size
  value = np.concatenate((value,np.zeros(P)))

  # Make Gramian matrices
  ATA = A.T.dot(A)
  LTL = L.T.dot(L)

  if damping == 'gcv':
    # define function to be minimized
    def predictive_error(d):
      # map d to a entirely positive domain
      d = np.exp(d)
      A_ginv = scipy.linalg.inv(ATA + d**2*LTL).dot(A.T)
      coeff = A_ginv.dot(value)
      predicted = A.dot(coeff)
      residual = predicted - value
      misfit = residual.dot(residual)
      numerator = M*misfit
      denominator = (M - product_trace(A,A_ginv))**2
      return numerator/denominator

    damping = scipy.optimize.fmin(predictive_error,0.0)
    damping = np.exp(damping)

  A_ginv = scipy.linalg.inv(ATA + damping**2*LTL).dot(A.T)
  coeff = A_ginv.dot(value)
  return coeff

def in_hull(p, hull):
  '''
  Tests if points in p are in the convex hull made up by hull
  '''
  dim = p.shape[1]
  if dim >= 2:
    hull = scipy.spatial.Delaunay(hull)
    return hull.find_simplex(p)>=0
  else:
    # one dimensional points
    min = np.min(hull)
    max = np.max(hull)
    return (p[:,0] >= min) & (p[:,0] <= max)

class RBFInterpolant(object):
  '''
  Regularized Radial Basis Function Interpolant

  Description
  -----------

    returns an interpolant consisting of a linear combination of
    radial basis functions and a polynomial with the specified order.

    We use A to denote the RBF alternant matrix with the additional
    polynomial constraints. m is a vector consisting of the unknown
    coefficients.  The coefficients are chosen to minimize

      ||W(Am - d)||_2^2 + ||p*Lm||_2^2

    where L is a regularization matrix which approximates the
    Laplacian of the interpolant, p is the penalty parameter, and W is
    a weight matrix consisting of the data uncertainty.
  '''
  def __init__(self,
               x,
               value,
               weight=None,
               eps=None,
               basis=rbf.basis.phs1,
               order=0,
               extrapolate=True,
               fill=np.nan,
               penalty=0.0):
    '''
    Initiates the RBF interpolant

    Parameters
    ----------
      x: (N,D) array of interpolation points which make up the rbf
        centers

      value: (N,) function values at the interpolation points.

      weight (default=None): (N,) array of weights to put on each
        interpolation point

      eps (default=None): (N,) array of shape parameters for each RBF.
        this has no effect for the default basis function, which is
        scale invariant.

      basis (default=phs1): type of basis function to use. phs1 creates
        a piecewise linear interpolant. Higher order RBFs are smoother
        but are not as well conditioned

      extrapolate (default=True): whether to allows points to be
        extrapolated outside of a convex hull formed by x.

      fill (default=np.nan): if extrapolate is False then points
        outside of the convex hull will be assigned this value

    '''
    x = np.asarray(x)
    # number of observation points
    N = x.shape[0]
    if eps is None:
      eps = np.ones(N)
    else:
      eps = np.asarray(eps)

    if weight is None:
      weight = np.ones(N)
    else:
      weight = np.asarray(weight)

    # form matrix for the LHS
    A = coefficient_matrix(x,eps=eps,basis=basis,order=order)

    # scale RHS and LHS by weight
    A[:N,:] *= weight[:,None]
    value *= weight

    # form regularization matrix
    laplacian = laplacian_diff_op(x.shape[1])
    L = np.sum(c*regularization_matrix(x,eps=eps,basis=basis,diff=d,order=order)
               for c,d in laplacian)

    coeff = find_coeff(A,L,value,penalty)

    self.x = x
    self.coeff = coeff
    self.basis = basis
    self.order = order
    self.eps = eps
    self.extrapolate = extrapolate
    self.fill = fill

  def __call__(self,xitp,diff=None):
    '''Returns the interpolant evaluated at xitp

    Parameters
    ----------
      xitp: ((N,) or (N,D) array) points where the interpolant is to
        be evaluated

      diff: ((D,) tuple, default=(0,)*dim) a tuple whos length is
        equal to the number of spatial dimensions.  Each value in the
        tfuple must be an integer indicating the order of the
        derivative in that spatial dimension.  For example, if the the
        spatial dimensions of the problem are 3 then diff=(2,0,1)
        would compute the second derivative in the first dimension and
        the first derivative in the third dimension.

    '''
    max_chunk = 100000
    n = 0
    xitp = np.asarray(xitp,dtype=float)
    Nitp = xitp.shape[0]
    # allocate output array
    out = np.zeros(Nitp)
    while n < Nitp:
      # xitp indices for this chunk
      idx = range(n,min(n+max_chunk,Nitp))
      A = interpolation_matrix(xitp[idx],self.x,
                               diff=diff,eps=self.eps,
                               basis=self.basis,order=self.order)
      out[idx] = np.einsum('ij,j...->i...',A,self.coeff)
      n += max_chunk

    # return zero for points outside of the convex hull if
    # extrapolation is not allowed
    if not self.extrapolate:
      out[~in_hull(xitp,self.x)] = self.fill

    return out

<commit_msg>fixed bug where data was not copied in the interpolant<commit_after>#!/usr/bin/env python
import numpy as np
import scipy.optimize
from scipy.linalg import inv
import scipy.spatial
import rbf.basis
import rbf.poly
import rbf.geometry
import modest
import matplotlib.pyplot as plt

def product_trace(A,B):
  '''
  efficiently returns the trace of a matrix product. this is used for
  general cross validation
  '''
  return np.sum(A*B.T)

def predictive_error(log_alpha,A,L,data):
  '''
  solves

    A*m = data
    alpha*L*m = 0

  for m, where A is the system matrix and L is the regularization
  matrix.  Then this function returns the predictive error of the
  solution using generalized cross validation
  '''
  # map alpha to an entirely positive domain
  alpha = 10**log_alpha

  # compute generalized inverse
  try:
    A_ginv = inv(A.T.dot(A) + alpha**2*L.T.dot(L)).dot(A.T)
  except np.linalg.linalg.LinAlgError:
    return np.inf

  # estimate m
  m = A_ginv.dot(data)

  # compute misfit
  predicted = A.dot(m)
  residual = predicted - data
  misfit = residual.dot(residual)

  # compute predictive error
  numerator = len(data)*misfit
  denominator = (len(data) - product_trace(A,A_ginv))**2
  return numerator/denominator

def laplacian_diff_op(dim):
  '''
  returns the coefficients and derivatives for each term in a
  Laplacian differential operator for the given number of dimensions.
  For example, calling this function with dim=2 will return

    array([[1.0,(2,0)],
           [1.0,(0,2)]], dtype=object),

  which is represents the differential operator

    1.0*d^2/dx_1^2 + 1.0*d^2/dx_2^2

  '''
  I = np.eye(dim,dtype=int)
  out = [[1.0,tuple(2*i)] for i in I]
  return np.array(out,dtype=object)

def coefficient_matrix(x,eps=None,basis=rbf.basis.phs3,order=0):
  '''
  returns the matrix that maps the coefficients to the function values
  at the observation points.

  Parameters
  ----------
    x: (N,D) array of observation points
    diff: (D,) tuple of derivatives
    basis (default=phs3): radial basis function
    order (default=0): additional polynomial order
  '''
  # number of observation points and spatial dimensions
  N,D = x.shape

  # powers for the additional polynomials
  powers = rbf.poly.monomial_powers(order,D)
  # number of polynomial terms
  P = powers.shape[0]

  # allocate array
  A = np.zeros((N+P,N+P))
  A[:N,:N] = basis(x,x,eps=eps)
  Ap = rbf.poly.mvmonos(x,powers)
  A[N:,:N] = Ap.T
  A[:N,N:] = Ap
  return A

def regularization_matrix(x,eps=None,basis=rbf.basis.phs3,order=0,diff=None):
  # number of observation points and spatial dimensions
  N,D = x.shape
  # powers for the additional polynomials
  powers = rbf.poly.monomial_powers(order,D)

  # number of polynomial terms
  P = powers.shape[0]

  # have the regularization minimize the size of the RBF coefficients
  A = np.zeros((N+P,N+P))
  # have the first N rows and columns be an identity matrix
  A[range(N),range(N)] = 1.0

  #A[:,:N] = basis(x,x,eps=eps,diff=diff)
  #A[:,N:] = rbf.poly.mvmonos(x,powers,diff=diff)
  return A

def interpolation_matrix(xitp,x,diff=None,eps=None,basis=rbf.basis.phs3,order=0):
  # number of interpolation points and spatial dimensions
  I,D = xitp.shape

  # number of observation points
  N = x.shape[0]

  # powers for the additional polynomials
  powers = rbf.poly.monomial_powers(order,D)
  # number of polynomial terms
  P = powers.shape[0]

  A = np.zeros((I,N+P))
  A[:,:N] = basis(xitp,x,eps=eps,diff=diff)
  A[:,N:] = rbf.poly.mvmonos(xitp,powers,diff=diff)
  return A

def find_coeff(A,L,value,damping):
  '''
  Parameters
  ----------
    A: (N+P,N+P) coefficient matrix
    L: (K,K) regularization matrix
    value: (N,...) observations
    damping: scalar damping parameter
  '''
  # number of data points
  N = value.shape[0]

  # number of added polynomials
  P = A.shape[0] - N

  # extend values to have a consistent size
  value = np.concatenate((value,np.zeros(P)))

  if damping == 'gcv':
    # define function to be minimized
    damping = scipy.optimize.minimize_scalar(predictive_error,args=(A,L,value)).x
    #damping = scipy.optimize.fmin(predictive_error,0.0,args=(A,L,value))
    damping = 10**damping
    print('optimal damping parameter from GCV: %s' % damping)
    # plot the soluton and make sure its a minimum
    damping_logmin = np.log10(damping) - 3
    damping_logmax = np.log10(damping) + 3
    damping_logrange = np.linspace(damping_logmin,damping_logmax,100)
    err = [predictive_error(d,A,L,value) for d in damping_logrange]
    current_ax = plt.gca()
    fig,ax = plt.subplots()
    ax.set_title('GCV curve')
    ax.loglog(10**damping_logrange,err)
    ax.vlines(damping,ax.get_ylim()[0],ax.get_ylim()[1])
    plt.sca(current_ax)

  A_ginv = inv(A.T.dot(A) + damping**2*L.T.dot(L)).dot(A.T)
  coeff = A_ginv.dot(value)
  return coeff

def in_hull(p, hull):
  '''
  Tests if points in p are in the convex hull made up by hull
  '''
  dim = p.shape[1]
  if dim >= 2:
    hull = scipy.spatial.Delaunay(hull)
    return hull.find_simplex(p)>=0
  else:
    # one dimensional points
    min = np.min(hull)
    max = np.max(hull)
    return (p[:,0] >= min) & (p[:,0] <= max)

class RBFInterpolant(object):
  '''
  Regularized Radial Basis Function Interpolant

  Description
  -----------

    returns an interpolant consisting of a linear combination of
    radial basis functions and a polynomial with the specified order.

    We use A to denote the RBF alternant matrix with the additional
    polynomial constraints. m is a vector consisting of the unknown
    coefficients.  The coefficients are chosen to minimize

      ||W(Am - d)||_2^2 + ||p*Lm||_2^2

    where L is a regularization matrix which approximates the
    Laplacian of the interpolant, p is the penalty parameter, and W is
    a weight matrix consisting of the data uncertainty.
  '''
  def __init__(self,
               x,
               value,
               weight=None,
               eps=None,
               basis=rbf.basis.phs1,
               order=0,
               extrapolate=True,
               fill=np.nan,
               penalty=0.0):
    '''
    Initiates the RBF interpolant

    Parameters
    ----------
      x: (N,D) array of interpolation points which make up the rbf
        centers

      value: (N,) function values at the interpolation points.

      weight (default=None): (N,) array of weights to put on each
        interpolation point

      eps (default=None): (N,) array of shape parameters for each RBF.
        this has no effect for the default basis function, which is
        scale invariant.

      basis (default=phs1): type of basis function to use. phs1 creates
        a piecewise linear interpolant. Higher order RBFs are smoother
        but are not as well conditioned

      extrapolate (default=True): whether to allows points to be
        extrapolated outside of a convex hull formed by x.

      fill (default=np.nan): if extrapolate is False then points
        outside of the convex hull will be assigned this value

    '''
    # copy data
    x = np.array(x,copy=True)
    value = np.array(value,copy=True)

    # number of observation points
    N = x.shape[0]
    if eps is None:
      eps = np.ones(N)
    else:
      eps = np.array(eps,copy=True)

    if weight is None:
      weight = np.ones(N)
    else:
      weight = np.array(weight,copy=True)

    # form matrix for the LHS
    A = coefficient_matrix(x,eps=eps,basis=basis,order=order)

    # scale RHS and LHS by weight
    A[:N,:] *= weight[:,None]
    value *= weight

    # form regularization matrix
    laplacian = laplacian_diff_op(x.shape[1])
    L = np.sum(c*regularization_matrix(x,eps=eps,basis=basis,diff=d,order=order)
               for c,d in laplacian)

    coeff = find_coeff(A,L,value,penalty)

    self.x = x
    self.coeff = coeff
    self.basis = basis
    self.order = order
    self.eps = eps
    self.extrapolate = extrapolate
    self.fill = fill

  def __call__(self,xitp,diff=None):
    '''Returns the interpolant evaluated at xitp

    Parameters
    ----------
      xitp: ((N,) or (N,D) array) points where the interpolant is to
        be evaluated

      diff: ((D,) tuple, default=(0,)*dim) a tuple whos length is
        equal to the number of spatial dimensions.  Each value in the
        tfuple must be an integer indicating the order of the
        derivative in that spatial dimension.  For example, if the the
        spatial dimensions of the problem are 3 then diff=(2,0,1)
        would compute the second derivative in the first dimension and
        the first derivative in the third dimension.

    '''
    max_chunk = 100000
    n = 0
    xitp = np.asarray(xitp,dtype=float)
    Nitp = xitp.shape[0]
    # allocate output array
    out = np.zeros(Nitp)
    while n < Nitp:
      # xitp indices for this chunk
      idx = range(n,min(n+max_chunk,Nitp))
      A = interpolation_matrix(xitp[idx],self.x,
                               diff=diff,eps=self.eps,
                               basis=self.basis,order=self.order)
      out[idx] = np.einsum('ij,j...->i...',A,self.coeff)
      n += max_chunk

    # return zero for points outside of the convex hull if
    # extrapolation is not allowed
    if not self.extrapolate:
      out[~in_hull(xitp,self.x)] = self.fill

    return out

<|endoftext|>


<commit_before>sec = lambda t: t
min = lambda t: 60*t
hr = lambda t: 60*min(t)
days = lambda t: 24*hr(t)
months = lambda t: 30.5*days(t)
yr = lambda t: 365.25 * days(t)
Kyr = lambda t: 1_000 * yr(t)
Myr = lambda t: 1_000_000 * yr(t)

class Time(object):

    _units = { 'sec' : sec,
               'min' : min,
               'hr' : hr,
               'days' : days,
               'months' : months,
               'yr' : yr,
               'Kyr' : Kyr,
               'Myr' : Myr,
               }

    def __init__(self, t=0):
        if type(t) == str:
            t = t.strip()
            t2 = t.split(' ')
            if len(t2) == 1:
                t = t2[0]
            elif len(t2) > 2:
                raise ValueError
            elif t2[1] not in self._units:
                raise ValueError
            else:
                f = self._units[t2[1]]
                t = f(float(t2[0]))
        self._sec = float(t)

    def __bool__(self):
        return True

    def __lt__ (self, other):
        if type(other) == Time:
            return self._sec < other._sec
        else:
            return self._sec < float(other)

    def __eq__ (self, other):
        if type(other) == Time:
            return self._sec == other._sec
        else:
            return self._sec == float(other)

    def __add__(self, other):
        if type(other) == Time:
            return Time(self._sec + other._sec)
        else:
            return Time(self._sec + float(other))

    def __str__(self):
        for unit in self._units:
            if unit == 'months':
                continue
            f = self._units[unit]
            t = self._sec /f(1)
            if len(str(int(t)))>3:
                continue
            break
        return f"{t:4.7} {unit}"
<commit_msg>Add inf outpu to largetime.py<commit_after>sec = lambda t: t
min = lambda t: 60*t
hr = lambda t: 60*min(t)
days = lambda t: 24*hr(t)
months = lambda t: 30.5*days(t)
yr = lambda t: 365.25 * days(t)
Kyr = lambda t: 1_000 * yr(t)
Myr = lambda t: 1_000_000 * yr(t)

class Time(object):

    _units = { 'sec' : sec,
               'min' : min,
               'hr' : hr,
               'days' : days,
               'months' : months,
               'yr' : yr,
               'Kyr' : Kyr,
               'Myr' : Myr,
               }

    def __init__(self, t=0):
        if type(t) == str:
            t = t.strip()
            t2 = t.split(' ')
            if len(t2) == 1:
                t = t2[0]
            elif len(t2) > 2:
                raise ValueError
            elif t2[1] not in self._units:
                raise ValueError
            else:
                f = self._units[t2[1]]
                t = f(float(t2[0]))
        self._sec = float(t)

    def __bool__(self):
        return True

    def __lt__ (self, other):
        if type(other) == Time:
            return self._sec < other._sec
        else:
            return self._sec < float(other)

    def __eq__ (self, other):
        if type(other) == Time:
            return self._sec == other._sec
        else:
            return self._sec == float(other)

    def __add__(self, other):
        if type(other) == Time:
            return Time(self._sec + other._sec)
        else:
            return Time(self._sec + float(other))

    def __str__(self):
        for unit in self._units:
            if unit == 'months':
                continue
            f = self._units[unit]
            t = self._sec /f(1)
            try:
                if len(str(int(t)))>3:
                    continue
            except ValueError:
                t = "N"
                continue
            break
        if t == "N":
            return 'inf '
        return f"{t:4.7} {unit}"
<|endoftext|>


<commit_before>"""A pure Python package providing the core RDF constructs.

The packages is intended to provide the core RDF types and interfaces
for working with RDF. The package defines a plugin interface for
parsers, stores, and serializers that other packages can use to
implement parsers, stores, and serializers that will plug into the
rdflib package.

The primary interface `rdflib` exposes to work with RDF is
`rdflib.graph.Graph`.

A tiny example:

    >>> from rdflib import Graph, URIRef, Literal

    >>> g = Graph()
    >>> result = g.parse("http://www.w3.org/2000/10/swap/test/meet/blue.rdf")

    >>> print("graph has %s statements." % len(g))
    graph has 4 statements.
    >>>
    >>> for s, p, o in g:
    ...     if (s, p, o) not in g:
    ...         raise Exception("It better be!")

    >>> s = g.serialize(format='nt')
    >>>
    >>> sorted(g) == [
    ...  (URIRef("http://meetings.example.com/cal#m1"),
    ...   URIRef("http://www.example.org/meeting_organization#homePage"),
    ...   URIRef("http://meetings.example.com/m1/hp")),
    ...  (URIRef("http://www.example.org/people#fred"),
    ...   URIRef("http://www.example.org/meeting_organization#attending"),
    ...   URIRef("http://meetings.example.com/cal#m1")),
    ...  (URIRef("http://www.example.org/people#fred"),
    ...   URIRef("http://www.example.org/personal_details#GivenName"),
    ...   Literal("Fred")),
    ...  (URIRef("http://www.example.org/people#fred"),
    ...   URIRef("http://www.example.org/personal_details#hasEmail"),
    ...   URIRef("mailto:fred@example.com"))
    ... ]
    True

"""
__docformat__ = "restructuredtext en"

# The format of the __version__ line is matched by a regex in setup.py
__version__ = "6.2.0-alpha"
__date__ = "2021-12-20"

__all__ = [
    "URIRef",
    "BNode",
    "IdentifiedNode",
    "Literal",
    "Variable",
    "Namespace",
    "Dataset",
    "Graph",
    "ConjunctiveGraph",
    "BRICK",
    "CSVW",
    "DC",
    "DCAT",
    "DCMITYPE",
    "DCTERMS",
    "DOAP",
    "FOAF",
    "ODRL2",
    "ORG",
    "OWL",
    "PROF",
    "PROV",
    "QB",
    "RDF",
    "RDFS",
    "SDO",
    "SH",
    "SKOS",
    "SOSA",
    "SSN",
    "TIME",
    "VANN",
    "VOID",
    "XSD",
    "util",
]

import sys
import logging

logger = logging.getLogger(__name__)

try:
    import __main__

    if (
        not hasattr(__main__, "__file__")
        and sys.stdout is not None
        and sys.stderr.isatty()
    ):
        # show log messages in interactive mode
        logger.setLevel(logging.INFO)
        logger.addHandler(logging.StreamHandler())
    del __main__
except ImportError:
    # Main already imported from elsewhere
    import warnings

    warnings.warn("__main__ already imported", ImportWarning)
    del warnings

del sys

NORMALIZE_LITERALS = True
"""
If True - Literals lexical forms are normalized when created.
I.e. the lexical forms is parsed according to data-type, then the
stored lexical form is the re-serialized value that was parsed.

Illegal values for a datatype are simply kept.  The normalized keyword
for Literal.__new__ can override this.

For example:

>>> from rdflib import Literal,XSD
>>> Literal("01", datatype=XSD.int)
rdflib.term.Literal("1", datatype=rdflib.term.URIRef("http://www.w3.org/2001/XMLSchema#integer"))

This flag may be changed at any time, but will only affect literals
created after that time, previously created literals will remain
(un)normalized.

"""

DAWG_LITERAL_COLLATION = False
"""
DAWG_LITERAL_COLLATION determines how literals are ordered or compared
to each other.

In SPARQL, applying the >,<,>=,<= operators to literals of
incompatible data-types is an error, i.e:

Literal(2)>Literal('cake') is neither true nor false, but an error.

This is a problem in PY3, where lists of Literals of incompatible
types can no longer be sorted.

Setting this flag to True gives you strict DAWG/SPARQL compliance,
setting it to False will order Literals with incompatible datatypes by
datatype URI

In particular, this determines how the rich comparison operators for
Literal work, eq, __neq__, __lt__, etc.
"""

from rdflib.term import URIRef, BNode, IdentifiedNode, Literal, Variable

from rdflib.graph import Dataset, Graph, ConjunctiveGraph

from rdflib import plugin
from rdflib import query

from rdflib.namespace import (
    BRICK,
    CSVW,
    DC,
    DCAT,
    DCMITYPE,
    DCTERMS,
    DOAP,
    FOAF,
    ODRL2,
    ORG,
    OWL,
    PROF,
    PROV,
    QB,
    RDF,
    RDFS,
    SDO,
    SH,
    SKOS,
    SOSA,
    SSN,
    TIME,
    VANN,
    VOID,
    XMLNS,
    XSD,
    Namespace,
)

# tedious sop to flake8
assert plugin
assert query

from rdflib import util

from rdflib.container import *
<commit_msg>Check if sys.stderr has isatty (fix: #1760)<commit_after>"""A pure Python package providing the core RDF constructs.

The packages is intended to provide the core RDF types and interfaces
for working with RDF. The package defines a plugin interface for
parsers, stores, and serializers that other packages can use to
implement parsers, stores, and serializers that will plug into the
rdflib package.

The primary interface `rdflib` exposes to work with RDF is
`rdflib.graph.Graph`.

A tiny example:

    >>> from rdflib import Graph, URIRef, Literal

    >>> g = Graph()
    >>> result = g.parse("http://www.w3.org/2000/10/swap/test/meet/blue.rdf")

    >>> print("graph has %s statements." % len(g))
    graph has 4 statements.
    >>>
    >>> for s, p, o in g:
    ...     if (s, p, o) not in g:
    ...         raise Exception("It better be!")

    >>> s = g.serialize(format='nt')
    >>>
    >>> sorted(g) == [
    ...  (URIRef("http://meetings.example.com/cal#m1"),
    ...   URIRef("http://www.example.org/meeting_organization#homePage"),
    ...   URIRef("http://meetings.example.com/m1/hp")),
    ...  (URIRef("http://www.example.org/people#fred"),
    ...   URIRef("http://www.example.org/meeting_organization#attending"),
    ...   URIRef("http://meetings.example.com/cal#m1")),
    ...  (URIRef("http://www.example.org/people#fred"),
    ...   URIRef("http://www.example.org/personal_details#GivenName"),
    ...   Literal("Fred")),
    ...  (URIRef("http://www.example.org/people#fred"),
    ...   URIRef("http://www.example.org/personal_details#hasEmail"),
    ...   URIRef("mailto:fred@example.com"))
    ... ]
    True

"""
__docformat__ = "restructuredtext en"

# The format of the __version__ line is matched by a regex in setup.py
__version__ = "6.2.0-alpha"
__date__ = "2021-12-20"

__all__ = [
    "URIRef",
    "BNode",
    "IdentifiedNode",
    "Literal",
    "Variable",
    "Namespace",
    "Dataset",
    "Graph",
    "ConjunctiveGraph",
    "BRICK",
    "CSVW",
    "DC",
    "DCAT",
    "DCMITYPE",
    "DCTERMS",
    "DOAP",
    "FOAF",
    "ODRL2",
    "ORG",
    "OWL",
    "PROF",
    "PROV",
    "QB",
    "RDF",
    "RDFS",
    "SDO",
    "SH",
    "SKOS",
    "SOSA",
    "SSN",
    "TIME",
    "VANN",
    "VOID",
    "XSD",
    "util",
]

import sys
import logging

logger = logging.getLogger(__name__)

try:
    import __main__

    if (
        not hasattr(__main__, "__file__")
        and sys.stdout is not None
        and hasattr(sys.stderr, "isatty")
        and sys.stderr.isatty()
    ):
        # show log messages in interactive mode
        logger.setLevel(logging.INFO)
        logger.addHandler(logging.StreamHandler())
    del __main__
except ImportError:
    # Main already imported from elsewhere
    import warnings

    warnings.warn("__main__ already imported", ImportWarning)
    del warnings

del sys

NORMALIZE_LITERALS = True
"""
If True - Literals lexical forms are normalized when created.
I.e. the lexical forms is parsed according to data-type, then the
stored lexical form is the re-serialized value that was parsed.

Illegal values for a datatype are simply kept.  The normalized keyword
for Literal.__new__ can override this.

For example:

>>> from rdflib import Literal,XSD
>>> Literal("01", datatype=XSD.int)
rdflib.term.Literal("1", datatype=rdflib.term.URIRef("http://www.w3.org/2001/XMLSchema#integer"))

This flag may be changed at any time, but will only affect literals
created after that time, previously created literals will remain
(un)normalized.

"""

DAWG_LITERAL_COLLATION = False
"""
DAWG_LITERAL_COLLATION determines how literals are ordered or compared
to each other.

In SPARQL, applying the >,<,>=,<= operators to literals of
incompatible data-types is an error, i.e:

Literal(2)>Literal('cake') is neither true nor false, but an error.

This is a problem in PY3, where lists of Literals of incompatible
types can no longer be sorted.

Setting this flag to True gives you strict DAWG/SPARQL compliance,
setting it to False will order Literals with incompatible datatypes by
datatype URI

In particular, this determines how the rich comparison operators for
Literal work, eq, __neq__, __lt__, etc.
"""

from rdflib.term import URIRef, BNode, IdentifiedNode, Literal, Variable

from rdflib.graph import Dataset, Graph, ConjunctiveGraph

from rdflib import plugin
from rdflib import query

from rdflib.namespace import (
    BRICK,
    CSVW,
    DC,
    DCAT,
    DCMITYPE,
    DCTERMS,
    DOAP,
    FOAF,
    ODRL2,
    ORG,
    OWL,
    PROF,
    PROV,
    QB,
    RDF,
    RDFS,
    SDO,
    SH,
    SKOS,
    SOSA,
    SSN,
    TIME,
    VANN,
    VOID,
    XMLNS,
    XSD,
    Namespace,
)

# tedious sop to flake8
assert plugin
assert query

from rdflib import util

from rdflib.container import *
<|endoftext|>


<commit_before>"""
Ricker Snow
this is a short script that makes the map by calling the mk_map() function from CV_Map.py, flips the map, makes the waypoints, and pickles the map and waypoints.  map2txt can also be called to make text files of the map to verify the map visually.
"""
import argparse
import CV_Map
from map_class import MapClass
import map2txt
import flip_map
import pickler
import mk_waypoints

def main():

  	parser = argparse.ArgumentParser(description='Map generator')
	parser.add_argument('-r', '--res', help='Map resolution', type=int, default='16' )
  	args = parser.parse_args()	#args.res has units inches/tile

	#print(args.res)
	"""
	Make dictonaries of variables that need, or may need, to be passed to another function
	"""
	map_real_vars = {}	#raw map variables with units inches
	map_prop_vars= {}	#descriptive map variables used in mk_map() that may need to be passed to vision or localization process

	map_real_vars['height'] = 73	#Upper platform width
	map_real_vars['width'] = 97	#width of entire course
	map_real_vars['upPltW'] = 24	#Upper platform width
	map_real_vars['upPltH'] = 24	#Upper platform height
	map_real_vars['upRmpW'] = 49	#upper ramp width
	map_real_vars['upRmpH'] = 24	#upper ramp height
	map_real_vars['loPltW'] = 24	#lower platform width
	map_real_vars['loPltH'] = 24	#lower platform height
	map_real_vars['loRmpW'] = 24	#lower ramp width
	map_real_vars['loRmpH'] = 24	#lower ramp height
	map_real_vars['wall'] = 0.75 	#wall thickness
	map_real_vars['whiteLine'] = 0.5	#thickness of white lines
	map_real_vars['startW'] = 12	#start square width
	map_real_vars['startH'] = 12	#start square height
	map_real_vars['upPlt_2_Air'] = 8.75
	map_real_vars['zone_short'] = 2.5	#the short dimension of each individual loading slot
	map_real_vars['air_long'] = 3	#the long dimension of each individual loading slot
	map_real_vars['sea_long'] = 4
	map_real_vars['land_long'] = 5
	map_real_vars['stor_long'] = 6
	map_real_vars['EdgetoSea'] = 8.5  #from edge of upper platform to start of sea area
	map_real_vars['Start2Sea'] = 8.5
	map_real_vars['edge2storage'] = 14.875
	map_real_vars['start2land'] = 31.75 #distance b/w start and land
	map_real_vars['cargoL'] = 42.5
	map_real_vars['seaH'] = 18.5
	map_real_vars['landW'] = 18.5
	map_real_vars['bot_width'] = 12
	map_real_vars['bot_length'] = 12

	#map property variables
	#status vars
	map_prop_vars['filled'] = 0
	map_prop_vars['empty'] = 1
	#color vars
	map_prop_vars['unk'] = 0
	map_prop_vars['blue'] = 1
	map_prop_vars['black'] = 2
	map_prop_vars['green'] = 3
	map_prop_vars['yellow'] = 4
	map_prop_vars['red'] = 5
	map_prop_vars['brown'] = 6
	map_prop_vars['white'] = 7
	#level vars
	map_prop_vars['ground'] = 0
	map_prop_vars['ramp'] = 1
	map_prop_vars['lwr_plat'] = 2
	map_prop_vars['upp_plat'] = 3
	#path vars
	map_prop_vars['path'] = 0
	map_prop_vars['not_path'] = 1
	#desc variables
	map_prop_vars['driv_srfc'] = 0
	map_prop_vars['start'] = 1
	map_prop_vars['storage'] = 2
	map_prop_vars['land'] = 3
	map_prop_vars['sea'] = 4
	map_prop_vars['air'] = 5
	map_prop_vars['line'] = 7
	map_prop_vars['wall'] = 8
	map_prop_vars['edge'] = 9

	map_grid_vars = map_real_vars.copy()	#map variables that will have units 'tiles'
	for key in map_grid_vars:		#multiply all of the variables by the resolution to get units of 'tiles'
		map_grid_vars[key] = int(args.res * map_grid_vars[key])

	#print(map_grid_vars);

	#Make map
	Map = CV_Map.mk_map(args.res, map_grid_vars, map_prop_vars)
	#Flip the map
	Map = flip_map.flip_map(Map)
	#Make the waypoints
	waypoints = mk_waypoints.mk_waypoints(args.res, map_grid_vars)
	#Output map to text
	map2txt.map2txt(Map)
	#pickle the map
	pickler.pickle_map(Map)
	#pickle the waypoints
	pickler.pickle_waypoints(waypoints)
	#pickle the map_prop_vars
	pickler.pickle_map_prop_vars(map_prop_vars)
	#unpickle the waypoints - to test function
	#Map_result = pickler.unpickle_map('map.pkl')
	#unpickle the waypoints - to test function
	#waypoints_result = pickler.unpickle_waypoints('waypoints.pkl')
	#unpickle the map_prop_vars - to test function
	#map_prop_vars_result = pickler.unpickle_map_prop_vars('map_prop_vars.pkl')

	#print(Map_result.grid[0][0]['desc'])
	#print(waypoints_result['land'])
	#print(map_prop_vars_result['blue'])

	return 0

if __name__ == "__main__":
	main()
<commit_msg>now the txt files and map in memory are the same, they were mirror images before<commit_after>"""
Ricker Snow
this is a short script that makes the map by calling the mk_map() function from CV_Map.py, flips the map, makes the waypoints, and pickles the map and waypoints.  map2txt can also be called to make text files of the map to verify the map visually.
"""
import argparse
import CV_Map
from map_class import MapClass
import map2txt
import flip_map
import pickler
import mk_waypoints

def main():

  	parser = argparse.ArgumentParser(description='Map generator')
	parser.add_argument('-r', '--res', help='Map resolution', type=int, default='16' )
  	args = parser.parse_args()	#args.res has units inches/tile

	#print(args.res)
	"""
	Make dictonaries of variables that need, or may need, to be passed to another function
	"""
	map_real_vars = {}	#raw map variables with units inches
	map_prop_vars= {}	#descriptive map variables used in mk_map() that may need to be passed to vision or localization process

	map_real_vars['height'] = 73	#Upper platform width
	map_real_vars['width'] = 97	#width of entire course
	map_real_vars['upPltW'] = 24	#Upper platform width
	map_real_vars['upPltH'] = 24	#Upper platform height
	map_real_vars['upRmpW'] = 49	#upper ramp width
	map_real_vars['upRmpH'] = 24	#upper ramp height
	map_real_vars['loPltW'] = 24	#lower platform width
	map_real_vars['loPltH'] = 24	#lower platform height
	map_real_vars['loRmpW'] = 24	#lower ramp width
	map_real_vars['loRmpH'] = 24	#lower ramp height
	map_real_vars['wall'] = 0.75 	#wall thickness
	map_real_vars['whiteLine'] = 0.5	#thickness of white lines
	map_real_vars['startW'] = 12	#start square width
	map_real_vars['startH'] = 12	#start square height
	map_real_vars['upPlt_2_Air'] = 8.75
	map_real_vars['zone_short'] = 2.5	#the short dimension of each individual loading slot
	map_real_vars['air_long'] = 3	#the long dimension of each individual loading slot
	map_real_vars['sea_long'] = 4
	map_real_vars['land_long'] = 5
	map_real_vars['stor_long'] = 6
	map_real_vars['EdgetoSea'] = 8.5  #from edge of upper platform to start of sea area
	map_real_vars['Start2Sea'] = 8.5
	map_real_vars['edge2storage'] = 14.875
	map_real_vars['start2land'] = 31.75 #distance b/w start and land
	map_real_vars['cargoL'] = 42.5
	map_real_vars['seaH'] = 18.5
	map_real_vars['landW'] = 18.5
	map_real_vars['bot_width'] = 12
	map_real_vars['bot_length'] = 12

	#map property variables
	#status vars
	map_prop_vars['filled'] = 0
	map_prop_vars['empty'] = 1
	#color vars
	map_prop_vars['unk'] = 0
	map_prop_vars['blue'] = 1
	map_prop_vars['black'] = 2
	map_prop_vars['green'] = 3
	map_prop_vars['yellow'] = 4
	map_prop_vars['red'] = 5
	map_prop_vars['brown'] = 6
	map_prop_vars['white'] = 7
	#level vars
	map_prop_vars['ground'] = 0
	map_prop_vars['ramp'] = 1
	map_prop_vars['lwr_plat'] = 2
	map_prop_vars['upp_plat'] = 3
	#path vars
	map_prop_vars['path'] = 0
	map_prop_vars['not_path'] = 1
	#desc variables
	map_prop_vars['driv_srfc'] = 0
	map_prop_vars['start'] = 1
	map_prop_vars['storage'] = 2
	map_prop_vars['land'] = 3
	map_prop_vars['sea'] = 4
	map_prop_vars['air'] = 5
	map_prop_vars['line'] = 7
	map_prop_vars['wall'] = 8
	map_prop_vars['edge'] = 9

	map_grid_vars = map_real_vars.copy()	#map variables that will have units 'tiles'
	for key in map_grid_vars:		#multiply all of the variables by the resolution to get units of 'tiles'
		map_grid_vars[key] = int(args.res * map_grid_vars[key])

	#print(map_grid_vars);

	#Make map
	Map = CV_Map.mk_map(args.res, map_grid_vars, map_prop_vars)
	#Output map to text - the function map2txt flips the map during storing so don't need to call flip map first
	map2txt.map2txt(Map)
	#Flip the map
	Map = flip_map.flip_map(Map)
	#Make the waypoints
	waypoints = mk_waypoints.mk_waypoints(args.res, map_grid_vars)
	#pickle the map
	pickler.pickle_map(Map)
	#pickle the waypoints
	pickler.pickle_waypoints(waypoints)
	#pickle the map_prop_vars
	pickler.pickle_map_prop_vars(map_prop_vars)
	#unpickle the waypoints - to test function
	#Map_result = pickler.unpickle_map('map.pkl')
	#unpickle the waypoints - to test function
	#waypoints_result = pickler.unpickle_waypoints('waypoints.pkl')
	#unpickle the map_prop_vars - to test function
	#map_prop_vars_result = pickler.unpickle_map_prop_vars('map_prop_vars.pkl')

	#print(Map_result.grid[0][0]['desc'])
	#print(waypoints_result['land'])
	#print(map_prop_vars_result['blue'])

	return 0

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
legit.scm
~~~~~~~~~

This module provides the main interface to Git.
"""

import os
import sys
import subprocess
from collections import namedtuple
from exceptions import ValueError
from operator import attrgetter

from git import Repo
from git.exc import GitCommandError

from .settings import settings

LEGIT_TEMPLATE = 'Legit: stashing before {0}.'

git = os.environ.get("GIT_PYTHON_GIT_EXECUTABLE", 'git')

Branch = namedtuple('Branch', ['name', 'is_published'])

class Aborted(object):

    def __init__(self):
        self.message = None
        self.log = None

def abort(message, log=None):

    a = Aborted()
    a.message = message
    a.log = log

    settings.abort_handler(a)

def repo_check(require_remote=False):
    if repo is None:
        print 'Not a git repository.'
        sys.exit(128)

    # TODO: no remote fail
    if not repo.remotes and require_remote:
        print 'No git remotes configured. Please add one.'
        sys.exit(128)

    # TODO: You're in a merge state.

def stash_it(sync=False):
    repo_check()
    msg = 'syncing branch' if sync else 'switching branches'

    return repo.git.execute([git,
        'stash', 'save', '--include-untracked',
        LEGIT_TEMPLATE.format(msg)])

def unstash_index(sync=False, branch=None):
    """Returns an unstash index if one is available."""

    repo_check()

    stash_list = repo.git.execute([git,
        'stash', 'list', '--date=default'])

    if branch is None:
        branch = repo.head.ref.name

    for stash in stash_list.splitlines():

        verb = 'syncing' if sync else 'switching'

        if (
            (('Legit' in stash) and
                ('On {0}:'.format(branch) in stash) and
                (verb in stash))
            or (('GitHub' in stash) and
                ('On {0}:'.format(branch) in stash) and
                (verb in stash))
        ):
            return stash[7]

def unstash_it(sync=False, branch=None):
    """Unstashes changes from current branch for branch sync."""

    repo_check()

    stash_index = unstash_index(sync=sync, branch=branch)

    if stash_index is not None:
        return repo.git.execute([git,
            'stash', 'pop', 'stash@{{{0}}}'.format(stash_index)])

def fetch():

    repo_check()

    return repo.git.execute([git, 'fetch', remote.name])

def smart_pull():
    'git log --merges origin/master..master'

    repo_check()

    branch = repo.head.ref.name

    fetch()

    return smart_merge('{0}/{1}'.format(remote.name, branch))

def smart_merge(branch, allow_rebase=True):

    repo_check()

    from_branch = repo.head.ref.name

    merges = repo.git.execute([git,
        'log', '--merges', '{0}..{1}'.format(branch, from_branch)])

    if allow_rebase:
        verb = 'merge' if merges.count('commit') else 'rebase'
    else:
        verb = 'merge'

    try:
        return repo.git.execute([git, verb, branch])
    except GitCommandError, why:
        log = repo.git.execute([git,'merge', '--abort'])
        abort('Merge failed. Reverting.', log=why)

def push(branch=None):

    repo_check()

    if branch is None:
        return repo.git.execute([git, 'push'])
    else:
        return repo.git.execute([git, 'push', remote.name, branch])

def checkout_branch(branch):
    """Checks out given branch."""

    repo_check()

    return repo.git.execute([git, 'checkout', branch])

def sprout_branch(off_branch, branch):
    """Checks out given branch."""

    repo_check()

    return repo.git.execute([git, 'checkout', off_branch, '-b', branch])

def graft_branch(branch):
    """Merges branch into current branch, and deletes it."""

    repo_check()

    log = []

    try:
        msg = repo.git.execute([git, 'merge', '--no-ff', branch])
        log.append(msg)
    except GitCommandError, why:
        log = repo.git.execute([git,'merge', '--abort'])
        abort('Merge failed. Reverting.', log='{0}\n{1}'.format(why, log))

    out = repo.git.execute([git, 'branch', '-D', branch])
    log.append(out)
    return '\n'.join(log)

def unpublish_branch(branch):
    """Unpublishes given branch."""

    repo_check()

    return repo.git.execute([git,
        'push', remote.name, ':{0}'.format(branch)])

def publish_branch(branch):
    """Publishes given branch."""

    repo_check()

    return repo.git.execute([git,
        'push', '-u', remote.name, branch])

def get_repo():
    """Returns the current Repo, based on path."""

    work_path = subprocess.Popen([git, 'rev-parse', '--show-toplevel'],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE).communicate()[0].rstrip('\n')

    if work_path:
        return Repo(work_path)
    else:
        return None

def get_remote():

    repo_check(require_remote=True)

    reader = repo.config_reader()

    # If there is no legit section return the default remote.
    if not reader.has_section('legit'):
        return repo.remotes[0]

    # If there is no remote option in the legit section return the default.
    if not any('legit' in s and 'remote' in s for s in reader.sections()):
        return repo.remotes[0]

    remote_name = reader.get('legit', 'remote')
    if not remote_name in [r.name for r in repo.remotes]:
        raise ValueError('Remote "{0}" does not exist! Please update your git '
                         'configuration.'.format(remote_name))

    return repo.remote(remote_name)

def get_branches(local=True, remote_branches=True):
    """Returns a list of local and remote branches."""

    repo_check()

    # print local
    branches = []

    if remote_branches:

        # Remote refs.
        try:
            for b in remote.refs:
                name = '/'.join(b.name.split('/')[1:])

                if name not in settings.forbidden_branches:
                    branches.append(Branch(name, True))
        except (IndexError, AssertionError):
            pass

    if local:

        # Local refs.
        for b in [h.name for h in repo.heads]:

            if b not in [br.name for br in branches] or not remote_branches:
                if b not in settings.forbidden_branches:
                    branches.append(Branch(b, False))

    return sorted(branches, key=attrgetter('name'))

def get_branch_names(local=True, remote_branches=True):

    repo_check()

    branches = get_branches(local=local, remote_branches=remote_branches)

    return [b.name for b in branches]

repo = get_repo()
remote = get_remote()
<commit_msg>Fix broken stash index by remove date info<commit_after># -*- coding: utf-8 -*-

"""
legit.scm
~~~~~~~~~

This module provides the main interface to Git.
"""

import os
import sys
import subprocess
from collections import namedtuple
from exceptions import ValueError
from operator import attrgetter

from git import Repo
from git.exc import GitCommandError

from .settings import settings

LEGIT_TEMPLATE = 'Legit: stashing before {0}.'

git = os.environ.get("GIT_PYTHON_GIT_EXECUTABLE", 'git')

Branch = namedtuple('Branch', ['name', 'is_published'])

class Aborted(object):

    def __init__(self):
        self.message = None
        self.log = None

def abort(message, log=None):

    a = Aborted()
    a.message = message
    a.log = log

    settings.abort_handler(a)

def repo_check(require_remote=False):
    if repo is None:
        print 'Not a git repository.'
        sys.exit(128)

    # TODO: no remote fail
    if not repo.remotes and require_remote:
        print 'No git remotes configured. Please add one.'
        sys.exit(128)

    # TODO: You're in a merge state.

def stash_it(sync=False):
    repo_check()
    msg = 'syncing branch' if sync else 'switching branches'

    return repo.git.execute([git,
        'stash', 'save', '--include-untracked',
        LEGIT_TEMPLATE.format(msg)])

def unstash_index(sync=False, branch=None):
    """Returns an unstash index if one is available."""

    repo_check()

    stash_list = repo.git.execute([git,
        'stash', 'list'])

    if branch is None:
        branch = repo.head.ref.name

    for stash in stash_list.splitlines():

        verb = 'syncing' if sync else 'switching'

        if (
            (('Legit' in stash) and
                ('On {0}:'.format(branch) in stash) and
                (verb in stash))
            or (('GitHub' in stash) and
                ('On {0}:'.format(branch) in stash) and
                (verb in stash))
        ):
            return stash[7]

def unstash_it(sync=False, branch=None):
    """Unstashes changes from current branch for branch sync."""

    repo_check()

    stash_index = unstash_index(sync=sync, branch=branch)

    if stash_index is not None:
        return repo.git.execute([git,
            'stash', 'pop', 'stash@{{{0}}}'.format(stash_index)])

def fetch():

    repo_check()

    return repo.git.execute([git, 'fetch', remote.name])

def smart_pull():
    'git log --merges origin/master..master'

    repo_check()

    branch = repo.head.ref.name

    fetch()

    return smart_merge('{0}/{1}'.format(remote.name, branch))

def smart_merge(branch, allow_rebase=True):

    repo_check()

    from_branch = repo.head.ref.name

    merges = repo.git.execute([git,
        'log', '--merges', '{0}..{1}'.format(branch, from_branch)])

    if allow_rebase:
        verb = 'merge' if merges.count('commit') else 'rebase'
    else:
        verb = 'merge'

    try:
        return repo.git.execute([git, verb, branch])
    except GitCommandError, why:
        log = repo.git.execute([git,'merge', '--abort'])
        abort('Merge failed. Reverting.', log=why)

def push(branch=None):

    repo_check()

    if branch is None:
        return repo.git.execute([git, 'push'])
    else:
        return repo.git.execute([git, 'push', remote.name, branch])

def checkout_branch(branch):
    """Checks out given branch."""

    repo_check()

    return repo.git.execute([git, 'checkout', branch])

def sprout_branch(off_branch, branch):
    """Checks out given branch."""

    repo_check()

    return repo.git.execute([git, 'checkout', off_branch, '-b', branch])

def graft_branch(branch):
    """Merges branch into current branch, and deletes it."""

    repo_check()

    log = []

    try:
        msg = repo.git.execute([git, 'merge', '--no-ff', branch])
        log.append(msg)
    except GitCommandError, why:
        log = repo.git.execute([git,'merge', '--abort'])
        abort('Merge failed. Reverting.', log='{0}\n{1}'.format(why, log))

    out = repo.git.execute([git, 'branch', '-D', branch])
    log.append(out)
    return '\n'.join(log)

def unpublish_branch(branch):
    """Unpublishes given branch."""

    repo_check()

    return repo.git.execute([git,
        'push', remote.name, ':{0}'.format(branch)])

def publish_branch(branch):
    """Publishes given branch."""

    repo_check()

    return repo.git.execute([git,
        'push', '-u', remote.name, branch])

def get_repo():
    """Returns the current Repo, based on path."""

    work_path = subprocess.Popen([git, 'rev-parse', '--show-toplevel'],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE).communicate()[0].rstrip('\n')

    if work_path:
        return Repo(work_path)
    else:
        return None

def get_remote():

    repo_check(require_remote=True)

    reader = repo.config_reader()

    # If there is no legit section return the default remote.
    if not reader.has_section('legit'):
        return repo.remotes[0]

    # If there is no remote option in the legit section return the default.
    if not any('legit' in s and 'remote' in s for s in reader.sections()):
        return repo.remotes[0]

    remote_name = reader.get('legit', 'remote')
    if not remote_name in [r.name for r in repo.remotes]:
        raise ValueError('Remote "{0}" does not exist! Please update your git '
                         'configuration.'.format(remote_name))

    return repo.remote(remote_name)

def get_branches(local=True, remote_branches=True):
    """Returns a list of local and remote branches."""

    repo_check()

    # print local
    branches = []

    if remote_branches:

        # Remote refs.
        try:
            for b in remote.refs:
                name = '/'.join(b.name.split('/')[1:])

                if name not in settings.forbidden_branches:
                    branches.append(Branch(name, True))
        except (IndexError, AssertionError):
            pass

    if local:

        # Local refs.
        for b in [h.name for h in repo.heads]:

            if b not in [br.name for br in branches] or not remote_branches:
                if b not in settings.forbidden_branches:
                    branches.append(Branch(b, False))

    return sorted(branches, key=attrgetter('name'))

def get_branch_names(local=True, remote_branches=True):

    repo_check()

    branches = get_branches(local=local, remote_branches=remote_branches)

    return [b.name for b in branches]

repo = get_repo()
remote = get_remote()
<|endoftext|>


<commit_before>import os
from subprocess import check_call

from charmhelpers.core import hookenv

from charms import reactive
from charms.reactive import hook

@hook('install')
def install():
    hookenv.status_set('maintenance', 'Installing Docker and AUFS')
    # Using getenv will return '' if CHARM_DIR is not an environment variable.
    charm_path = path(os.getenv('CHARM_DIR'), '')
    install_script_path = os.path.join(charm_path, 'scripts/install_docker.sh')
    check_call([install_script_path])
    # Install pip.
    check_call(['apt-get', 'install', '-y', 'python-pip'])
    # Pip install docker-compose.
    check_call(['pip', 'install', '-U', 'docker-compose'])
    # Leave a status message that Docker is installed.
    hookenv.status_set('active', 'Docker Installed')
    reactive.set_state('docker.available')
<commit_msg>Removed reference to path.py<commit_after>import os
from subprocess import check_call

from charmhelpers.core import hookenv

from charms import reactive
from charms.reactive import hook

@hook('install')
def install():
    hookenv.status_set('maintenance', 'Installing Docker and AUFS')
    # Using getenv will return '' if CHARM_DIR is not an environment variable.
    charm_path = os.getenv('CHARM_DIR', '')
    install_script_path = os.path.join(charm_path, 'scripts/install_docker.sh')
    check_call([install_script_path])
    # Install pip.
    check_call(['apt-get', 'install', '-y', 'python-pip'])
    # Pip install docker-compose.
    check_call(['pip', 'install', '-U', 'docker-compose'])
    # Leave a status message that Docker is installed.
    hookenv.status_set('active', 'Docker Installed')
    reactive.set_state('docker.available')
<|endoftext|>


<commit_before>#-------------------------------------------------------------------------------
#
# The MIT License (MIT)
#
# Copyright (c) 2015 William De Freitas
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
#-------------------------------------------------------------------------------

#!/usr/bin/env python

'''Counts ticks from the reed switch of a rain gauge via an interrupt driven
    callback function. This count is stored in the precipiattion rate variable
    and is reset every loop.
    Script loops until stopped by the user.
    Data is stored to an RRD file at the update time pulled from the RRD file.
    Current precipitation accumulated value is pulled from the RRD file and
    incremented by the counted ticks from the rain gauge.
    At midnight, the precipitation accumulated value is reset.
    If there is no RRD file or its set up is different from requirement, the
    script will abort.'''

#===============================================================================
# Import modules
#===============================================================================

# Standard Library
import sys
import threading
import time
import datetime
import collections

# Third party modules
import pigpio

# Application modules
import log
import logging
import settings as s
import rrd_tools

#===============================================================================
# GLOBAL VARIABLES
#===============================================================================
last_rising_edge = None

#===============================================================================
# EDGE CALLBACK FUNCTION TO COUNT RAIN TICKS
#===============================================================================
def count_rain_ticks(gpio, level, tick):

    '''Count the ticks from a reed switch'''

    global precip_tick_count
    global last_rising_edge

    logger = logging.getLogger('root')

    pulse = False

    if last_rising_edge is not None:
        #check tick in microseconds
        if pigpio.tickDiff(last_rising_edge, tick) > s.DEBOUNCE_MICROS * 1000000:
            pulse = True

    else:
        pulse = True

    if pulse:
        last_rising_edge = tick
        precip_tick_count += 1
        logger.debug('Precip tick count : {tick}'.format(tick= precip_tick_count))

#===============================================================================
# MAIN
#===============================================================================
def main():

    '''Entry point for script'''

    global precip_tick_count
    global precip_accu

    precip_tick_count = 0
    precip_accu       = 0

    #---------------------------------------------------------------------------
    # SET UP LOGGER
    #---------------------------------------------------------------------------
    logger = log.setup('root', '/home/pi/weather/logs/read_rain_gauge.log')

    logger.info('--- Read Rain Gauge Script Started ---')

    #---------------------------------------------------------------------------
    # LOAD DRIVERS
    #---------------------------------------------------------------------------
    try:
        pi = pigpio.pi()

    except ValueError:
        logger.critical('Failed to connect to PIGPIO ({error_v}). Exiting...'.format(
            error_v=ValueError))
        sys.exit()

    #---------------------------------------------------------------------------
    # CHECK RRD FILE
    #---------------------------------------------------------------------------
    try:
        rrd = rrd_tools.RrdFile(s.RRDTOOL_RRD_FILE)

        if sorted(rrd.ds_list()) != sorted(list(s.SENSOR_SET.keys())):
            logger.error('Data sources in RRD file does not match set up.')
            logger.error(rrd.ds_list())
            logger.error(list(s.SENSOR_SET.keys()))
            logger.error('Exiting...')
            sys.exit()
        else:
            logger.info('RRD fetch successful.')

    except ValueError:
        logger.critical('RRD fetch failed. Exiting...')
        sys.exit()

    #---------------------------------------------------------------------------
    # SET UP SENSOR VARIABLES
    #---------------------------------------------------------------------------
    sensor_value = {x: 'U' for x in s.SENSOR_SET}

    ss = collections.namedtuple('ss', 'enable ref unit min max type')
    sensor = {k: ss(*s.SENSOR_SET[k]) for k in s.SENSOR_SET}

    logger.debug(sensor_value)

    #---------------------------------------------------------------------------
    # SET UP RAIN SENSOR HARDWARE
    #---------------------------------------------------------------------------
    pi.set_mode(sensor['precip_acc'].ref, pigpio.INPUT)
    rain_gauge = pi.callback(sensor['precip_acc'].ref, pigpio.FALLING_EDGE,
                                count_rain_ticks)

    #---------------------------------------------------------------------------
    # TIMED LOOP
    #---------------------------------------------------------------------------
    try:
        while True:

            #-------------------------------------------------------------------
            # Delay to give update rate
            #-------------------------------------------------------------------
            next_reading  = rrd.next_update()
            logger.debug('Next sensor reading at {time}'.format(
                time=time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(next_reading))))

            sleep_length = next_reading - time.time()
            if sleep_length > 0:
                time.sleep(sleep_length)

            #-------------------------------------------------------------------
            # Get loop start time
            #-------------------------------------------------------------------
            loop_start = datetime.datetime.now()
            logger.info('Loop start time: {start_time}'.format(
                start_time=loop_start.strftime('%Y-%m-%d %H:%M:%S')))

            #-------------------------------------------------------------------
            # Get rain fall measurement
            #-------------------------------------------------------------------
            sensor_value['precip_acc'] = 0.000000
            sensor_value['precip_rate'] = precip_tick_count * s.PRECIP_TICK_MEASURE
            precip_tick_count = 0.000000
            logger.debug('Precip tick counter RESET')

            #If last entry was before midnight this moninng do not use accumulated
            # precipitation value taken from round robin database
            last_entry_time = rrd.last_update()

            last_reset = loop_start.replace(hour=0, minute=0, second=0, microsecond=0)
            tdelta = datetime.datetime.fromtimestamp(last_entry_time) - last_reset
            tdelta = tdelta.total_seconds()
            logger.debug('last entry = {last_entry}'.format(last_entry= last_entry_time))
            logger.debug('last_reset = {last_reset_t}'.format(last_reset_t= last_reset))
            logger.debug('tdelta = {delta_t}'.format(delta_t= tdelta))

            if tdelta >= 0.00:
                #Fetch data from round robin database
                try:
                    rrd_data = []
                    rrd_data = rrd.fetch(start=last_entry_time-300, end=last_entry_time)
                    rt = collections.namedtuple( 'rt', 'start end step ds value')
                    rrd_data = rt(rrd_data[0][0], rrd_data[0][1], rrd_data[0][2],
                                            rrd_data[1], rrd_data[2])

                    sensor_value['precip_acc'] = float(
                        rrd_data.value[len(rrd_data.value)-2][rrd_data.ds.index('precip_acc')] or 0)

                    logger.info('Data fetched from RRD file')

                except ValueError, e:
                    logger.error('Could not fetch data from RRD file!')

            #Add previous precip. acc'ed value to current precip. rate
            sensor_value['precip_acc'] += sensor_value['precip_rate']

            #Log values
            logger.info('Precip_acc:  {precip_acc}'.format(
                                        precip_acc= sensor_value['precip_acc']))
            logger.info('Precip_rate: {precip_rate}'.format(
                                        precip_rate= sensor_value['precip_rate']))

            #-------------------------------------------------------------------
            # Add data to RRD
            #-------------------------------------------------------------------
            logger.debug('Update time = {update_time}'.format(update_time= 'N'))#rrd.next_update()))
            logger.debug([v for (k, v) in sorted(sensor_value.items()) if v != 'U'])

            result = rrd.update_file(timestamp= 'N',
                ds_name= [k for (k, v) in sorted(sensor_value.items()) if v!='U'],
                values= [v for (k, v) in sorted(sensor_value.items()) if v != 'U'])

            if result == 'OK':
                logger.info('Update RRD file OK')
            else:
                logger.error('Failed to update RRD file ({value_error})'.format(
                    value_error=result))
                logger.error(sensor_value)

    #---------------------------------------------------------------------------
    # User exit command
    #---------------------------------------------------------------------------
    except KeyboardInterrupt:
        logger.info('USER ACTION: End command')

    #---------------------------------------------------------------------------
    # Other error captured
    #---------------------------------------------------------------------------
    except Exception, e:
        logger.error('Script Error', exc_info=True)

    #---------------------------------------------------------------------------
    # Stop processes
    #---------------------------------------------------------------------------
    finally:
        rain_gauge.cancel()
        logger.info('--- Read Rain Gauge Finished ---')

#===============================================================================
# BOILER PLATE
#===============================================================================
if __name__=='__main__':
    sys.exit(main())
<commit_msg>Fix precip accu timestamps<commit_after>#-------------------------------------------------------------------------------
#
# The MIT License (MIT)
#
# Copyright (c) 2015 William De Freitas
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
#-------------------------------------------------------------------------------

#!/usr/bin/env python

'''Counts ticks from the reed switch of a rain gauge via an interrupt driven
    callback function. This count is stored in the precipiattion rate variable
    and is reset every loop.
    Script loops until stopped by the user.
    Data is stored to an RRD file at the update time pulled from the RRD file.
    Current precipitation accumulated value is pulled from the RRD file and
    incremented by the counted ticks from the rain gauge.
    At midnight, the precipitation accumulated value is reset.
    If there is no RRD file or its set up is different from requirement, the
    script will abort.'''

#===============================================================================
# Import modules
#===============================================================================

# Standard Library
import sys
import threading
import time
import datetime
import collections

# Third party modules
import pigpio

# Application modules
import log
import logging
import settings as s
import rrd_tools

#===============================================================================
# GLOBAL VARIABLES
#===============================================================================
last_rising_edge = None

#===============================================================================
# EDGE CALLBACK FUNCTION TO COUNT RAIN TICKS
#===============================================================================
def count_rain_ticks(gpio, level, tick):

    '''Count the ticks from a reed switch'''

    global precip_tick_count
    global last_rising_edge

    logger = logging.getLogger('root')

    pulse = False

    if last_rising_edge is not None:
        #check tick in microseconds
        if pigpio.tickDiff(last_rising_edge, tick) > s.DEBOUNCE_MICROS * 1000000:
            pulse = True

    else:
        pulse = True

    if pulse:
        last_rising_edge = tick
        precip_tick_count += 1
        logger.debug('Precip tick count : {tick}'.format(tick= precip_tick_count))

#===============================================================================
# MAIN
#===============================================================================
def main():

    '''Entry point for script'''

    global precip_tick_count
    global precip_accu

    precip_tick_count = 0
    precip_accu       = 0

    #---------------------------------------------------------------------------
    # SET UP LOGGER
    #---------------------------------------------------------------------------
    logger = log.setup('root', '/home/pi/weather/logs/read_rain_gauge.log')

    logger.info('--- Read Rain Gauge Script Started ---')

    #---------------------------------------------------------------------------
    # LOAD DRIVERS
    #---------------------------------------------------------------------------
    try:
        pi = pigpio.pi()

    except ValueError:
        logger.critical('Failed to connect to PIGPIO ({error_v}). Exiting...'.format(
            error_v=ValueError))
        sys.exit()

    #---------------------------------------------------------------------------
    # CHECK RRD FILE
    #---------------------------------------------------------------------------
    try:
        rrd = rrd_tools.RrdFile(s.RRDTOOL_RRD_FILE)

        if sorted(rrd.ds_list()) != sorted(list(s.SENSOR_SET.keys())):
            logger.error('Data sources in RRD file does not match set up.')
            logger.error(rrd.ds_list())
            logger.error(list(s.SENSOR_SET.keys()))
            logger.error('Exiting...')
            sys.exit()
        else:
            logger.info('RRD fetch successful.')

    except ValueError:
        logger.critical('RRD fetch failed. Exiting...')
        sys.exit()

    #---------------------------------------------------------------------------
    # SET UP SENSOR VARIABLES
    #---------------------------------------------------------------------------
    sensor_value = {x: 'U' for x in s.SENSOR_SET}

    ss = collections.namedtuple('ss', 'enable ref unit min max type')
    sensor = {k: ss(*s.SENSOR_SET[k]) for k in s.SENSOR_SET}

    logger.debug(sensor_value)

    #---------------------------------------------------------------------------
    # SET UP RAIN SENSOR HARDWARE
    #---------------------------------------------------------------------------
    pi.set_mode(sensor['precip_acc'].ref, pigpio.INPUT)
    rain_gauge = pi.callback(sensor['precip_acc'].ref, pigpio.FALLING_EDGE,
                                count_rain_ticks)

    #---------------------------------------------------------------------------
    # TIMED LOOP
    #---------------------------------------------------------------------------
    try:
        while True:

            #-------------------------------------------------------------------
            # Delay to give update rate
            #-------------------------------------------------------------------
            next_reading  = rrd.next_update()
            logger.debug('Next sensor reading at {time}'.format(
                time=time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(next_reading))))

            sleep_length = next_reading - time.time()
            if sleep_length > 0:
                time.sleep(sleep_length)

            #-------------------------------------------------------------------
            # Get loop start time
            #-------------------------------------------------------------------
            loop_start = datetime.datetime.now()
            logger.info('Loop start time: {start_time}'.format(
                start_time=loop_start.strftime('%Y-%m-%d %H:%M:%S')))

            #-------------------------------------------------------------------
            # Get rain fall measurement
            #-------------------------------------------------------------------
            sensor_value['precip_acc'] = 0.000000
            sensor_value['precip_rate'] = precip_tick_count * s.PRECIP_TICK_MEASURE
            precip_tick_count = 0.000000
            logger.debug('Precip tick counter RESET')

            #If last entry was before midnight this moninng do not use accumulated
            # precipitation value taken from round robin database
            last_entry_time = rrd.last_update()

            last_reset = loop_start.replace(hour=0, minute=0, second=0, microsecond=0)
            tdelta = datetime.datetime.utcfromtimestamp(last_entry_time) - last_reset
            tdelta = tdelta.total_seconds()
            logger.debug('last entry = {last_entry}'.format(last_entry= datetime.datetime.utcfromtimestamp(last_entry_time)))
            logger.debug('last_reset = {last_reset_t}'.format(last_reset_t= last_reset))
            logger.debug('tdelta = {delta_t}'.format(delta_t= tdelta))

            if tdelta >= 0.00:
                #Fetch data from round robin database
                try:
                    rrd_data = []
                    rrd_data = rrd.fetch(start=last_entry_time-300, end=last_entry_time)
                    rt = collections.namedtuple( 'rt', 'start end step ds value')
                    rrd_data = rt(rrd_data[0][0], rrd_data[0][1], rrd_data[0][2],
                                            rrd_data[1], rrd_data[2])

                    sensor_value['precip_acc'] = float(
                        rrd_data.value[len(rrd_data.value)-2][rrd_data.ds.index('precip_acc')] or 0)

                    logger.info('Data fetched from RRD file')

                except ValueError, e:
                    logger.error('Could not fetch data from RRD file!')

            #Add previous precip. acc'ed value to current precip. rate
            sensor_value['precip_acc'] += sensor_value['precip_rate']

            #Log values
            logger.info('Precip_acc:  {precip_acc}'.format(
                                        precip_acc= sensor_value['precip_acc']))
            logger.info('Precip_rate: {precip_rate}'.format(
                                        precip_rate= sensor_value['precip_rate']))

            #-------------------------------------------------------------------
            # Add data to RRD
            #-------------------------------------------------------------------
            logger.debug('Update time = {update_time}'.format(update_time= 'N'))#rrd.next_update()))
            logger.debug([v for (k, v) in sorted(sensor_value.items()) if v != 'U'])

            result = rrd.update_file(timestamp= 'N',
                ds_name= [k for (k, v) in sorted(sensor_value.items()) if v!='U'],
                values= [v for (k, v) in sorted(sensor_value.items()) if v != 'U'])

            if result == 'OK':
                logger.info('Update RRD file OK')
            else:
                logger.error('Failed to update RRD file ({value_error})'.format(
                    value_error=result))
                logger.error(sensor_value)

    #---------------------------------------------------------------------------
    # User exit command
    #---------------------------------------------------------------------------
    except KeyboardInterrupt:
        logger.info('USER ACTION: End command')

    #---------------------------------------------------------------------------
    # Other error captured
    #---------------------------------------------------------------------------
    except Exception, e:
        logger.error('Script Error', exc_info=True)

    #---------------------------------------------------------------------------
    # Stop processes
    #---------------------------------------------------------------------------
    finally:
        rain_gauge.cancel()
        logger.info('--- Read Rain Gauge Finished ---')

#===============================================================================
# BOILER PLATE
#===============================================================================
if __name__=='__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>import os
import sys
import bz2
import unittest
from functools import partial

import dlib
import numpy as np
from PIL import Image
from tqdm import tqdm

SHAPE_PREDICTOR_FNAME = 'shape_predictor_68_face_landmarks.dat'
SHAPE_PREDICTOR_BZ2_FNAME = SHAPE_PREDICTOR_FNAME + '.bz2'
SHAPE_PREDICTOR_URL = 'http://dlib.net/files/{}'.format(SHAPE_PREDICTOR_BZ2_FNAME)

def _download_file(url, out_path):
    try:
        from urllib import urlretrieve          # Python 2
    except ImportError:
        from urllib.request import urlretrieve  # Python 3

    # Wrap tqdm instance with urlretrieve compatible function
    # Abuse mutable [] argument to give function 'memory'
    # First argument will be supplied using partial (an instance of tqdm)
    def reporthook(t, b=1, bsize=1, tsize=None, last_b=[0]):
        if tsize is not None:
            t.total = tsize
        t.update((b - last_b[0]) * bsize)
        last_b[0] = b

    with tqdm(unit='B', unit_scale=True, miniters=1, desc=out_path) as t:
        urlretrieve(url, filename=out_path, reporthook=partial(reporthook, t))

def _bz2_decompress_inplace(path, out_path):
    with open(path, 'rb') as source, open(out_path, 'wb') as dest:
        dest.write(bz2.decompress(source.read()))

def _load_image_using_pillow(path):
    return np.array(Image.open(path))

class TestDlib(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        # Get paths to test data
        cls.script_path = os.path.dirname(os.path.abspath(__file__))
        cls.test_dir_path = os.path.join(cls.script_path, 'test_data')
        cls.face_jpg_path = os.path.join(cls.test_dir_path, 'face.jpg')
        cls.face_png_path = os.path.join(cls.test_dir_path, 'face.png')
        cls.images_xml_path = os.path.join(cls.test_dir_path, 'images.xml')

        # Download shape_predictor model
        print('Downloading {} to ./{}'.format(SHAPE_PREDICTOR_URL,
                                              SHAPE_PREDICTOR_BZ2_FNAME))
        _download_file(SHAPE_PREDICTOR_URL, SHAPE_PREDICTOR_BZ2_FNAME)
        _bz2_decompress_inplace(SHAPE_PREDICTOR_BZ2_FNAME,
                                SHAPE_PREDICTOR_FNAME)

    def test_builtin_frontal_face_detection(self):
        detector = dlib.get_frontal_face_detector()
        image = _load_image_using_pillow(self.face_jpg_path)
        results = detector(image)
        self.assertEqual(len(results), 1)

    def test_shape_predictor(self):
        predictor = dlib.shape_predictor(SHAPE_PREDICTOR_FNAME)
        image = _load_image_using_pillow(self.face_jpg_path)

        # This is the output of the detector, hardcoded
        detection = dlib.rectangle(left=125, top=56, right=434, bottom=365)
        shape = predictor(image, detection)
        self.assertEqual(len(shape.parts()), 68)
        for p in shape.parts():
            self.assertGreater(p.x, 0)
            self.assertGreater(p.y, 0)

    def test_train_xml_detector(self):
        # This effectively tests that we can successfully load images
        options = dlib.simple_object_detector_training_options()
        options.add_left_right_image_flips = True
        options.C = 1
        options.num_threads = 1

        # Change the working directory so that Windows can find the images
        os.chdir(self.test_dir_path)
        dlib.train_simple_object_detector(self.images_xml_path, './test.svm',
                                          options)
        self.assertTrue(os.path.exists('./test.svm'))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Remove Windows hack - no longer needed<commit_after>import os
import sys
import bz2
import unittest
from functools import partial

import dlib
import numpy as np
from PIL import Image
from tqdm import tqdm

SHAPE_PREDICTOR_FNAME = 'shape_predictor_68_face_landmarks.dat'
SHAPE_PREDICTOR_BZ2_FNAME = SHAPE_PREDICTOR_FNAME + '.bz2'
SHAPE_PREDICTOR_URL = 'http://dlib.net/files/{}'.format(SHAPE_PREDICTOR_BZ2_FNAME)

def _download_file(url, out_path):
    try:
        from urllib import urlretrieve          # Python 2
    except ImportError:
        from urllib.request import urlretrieve  # Python 3

    # Wrap tqdm instance with urlretrieve compatible function
    # Abuse mutable [] argument to give function 'memory'
    # First argument will be supplied using partial (an instance of tqdm)
    def reporthook(t, b=1, bsize=1, tsize=None, last_b=[0]):
        if tsize is not None:
            t.total = tsize
        t.update((b - last_b[0]) * bsize)
        last_b[0] = b

    with tqdm(unit='B', unit_scale=True, miniters=1, desc=out_path) as t:
        urlretrieve(url, filename=out_path, reporthook=partial(reporthook, t))

def _bz2_decompress_inplace(path, out_path):
    with open(path, 'rb') as source, open(out_path, 'wb') as dest:
        dest.write(bz2.decompress(source.read()))

def _load_image_using_pillow(path):
    return np.array(Image.open(path))

class TestDlib(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        # Get paths to test data
        cls.script_path = os.path.dirname(os.path.abspath(__file__))
        cls.test_dir_path = os.path.join(cls.script_path, 'test_data')
        cls.face_jpg_path = os.path.join(cls.test_dir_path, 'face.jpg')
        cls.face_png_path = os.path.join(cls.test_dir_path, 'face.png')
        cls.images_xml_path = os.path.join(cls.test_dir_path, 'images.xml')

        # Download shape_predictor model
        print('Downloading {} to ./{}'.format(SHAPE_PREDICTOR_URL,
                                              SHAPE_PREDICTOR_BZ2_FNAME))
        _download_file(SHAPE_PREDICTOR_URL, SHAPE_PREDICTOR_BZ2_FNAME)
        _bz2_decompress_inplace(SHAPE_PREDICTOR_BZ2_FNAME,
                                SHAPE_PREDICTOR_FNAME)

    def test_builtin_frontal_face_detection(self):
        detector = dlib.get_frontal_face_detector()
        image = _load_image_using_pillow(self.face_jpg_path)
        results = detector(image)
        self.assertEqual(len(results), 1)

    def test_shape_predictor(self):
        predictor = dlib.shape_predictor(SHAPE_PREDICTOR_FNAME)
        image = _load_image_using_pillow(self.face_jpg_path)

        # This is the output of the detector, hardcoded
        detection = dlib.rectangle(left=125, top=56, right=434, bottom=365)
        shape = predictor(image, detection)
        self.assertEqual(len(shape.parts()), 68)
        for p in shape.parts():
            self.assertGreater(p.x, 0)
            self.assertGreater(p.y, 0)

    def test_train_xml_detector(self):
        # This effectively tests that we can successfully load images
        options = dlib.simple_object_detector_training_options()
        options.add_left_right_image_flips = True
        options.C = 1
        options.num_threads = 1

        dlib.train_simple_object_detector(self.images_xml_path, './test.svm',
                                          options)
        self.assertTrue(os.path.exists('./test.svm'))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import os

from osgeo import gdal
from osgeo import ogr
from osgeo import osr

# Set GDAL_DATA. This is done normally done by the activate script,
# but this doesn't happen in the testing environment
if 'LIBRARY_PREFIX' in os.environ:
    # Windows.
    gdalData = os.path.join(os.environ['LIBRARY_PREFIX'], 'share', 'gdal')
else:
    # Linux/OS X.
    gdalData = os.path.join(os.environ['PREFIX'], 'share', 'gdal')

os.environ['GDAL_DATA'] = gdalData

driver = gdal.GetDriverByName("netCDF")
assert driver is not None

driver = gdal.GetDriverByName("HDF4")
assert driver is not None

driver = gdal.GetDriverByName("HDF5")
assert driver is not None

driver = gdal.GetDriverByName("GTiff")
assert driver is not None

driver = gdal.GetDriverByName("PNG")
assert driver is not None

driver = gdal.GetDriverByName("JPEG")
assert driver is not None

# only available when libkea successfully linked in.
driver = gdal.GetDriverByName("KEA")
assert driver is not None

# only available when xerces-c++ successfully linked in.
driver = ogr.GetDriverByName("GML")
assert driver is not None

# only available when openjpeg successfully linked in.
driver = gdal.GetDriverByName("JP2OpenJPEG")
assert driver is not None

# only available when curl successfully linked in.
driver = gdal.GetDriverByName("WCS")
assert driver is not None

# only available when freexl successfully linked in.
driver = ogr.GetDriverByName("XLS")
assert driver is not None

# only available when expat successfully linked in.
driver = ogr.GetDriverByName("KML")
assert driver is not None

# only available when SQLite successfully linked in.
driver = ogr.GetDriverByName("SQLite")
assert driver is not None

def has_geos():
    pnt1 = ogr.CreateGeometryFromWkt( 'POINT(10 20)' )
    pnt2 = ogr.CreateGeometryFromWkt( 'POINT(30 20)' )
    ogrex = ogr.GetUseExceptions()
    ogr.DontUseExceptions()
    hasgeos = pnt1.Union( pnt2 ) is not None
    if ogrex:
        ogr.UseExceptions()
    return hasgeos

assert has_geos(), 'GEOS not available within GDAL'

def has_proj():
    sr1 = osr.SpatialReference()
    sr1.ImportFromEPSG(4326) # lat, lon.
    sr2 = osr.SpatialReference()
    sr2.ImportFromEPSG(28355) # GDA94/MGA zone 55.
    osrex = osr.GetUseExceptions()
    osr.UseExceptions()
    hasproj = True
    # Use exceptions to determine if we have proj and epsg files
    # otherwise we can't reliably determine if it has failed.
    try:
        trans = osr.CoordinateTransformation(sr1, sr2)
    except RuntimeError:
        hasproj = False
    return hasproj

assert has_proj(), 'PROJ not available within GDAL'

# This module does some additional tests.
import extra_tests
<commit_msg>new test<commit_after>import os

from osgeo import gdal
from osgeo import ogr
from osgeo import osr

# Set GDAL_DATA. This is done normally done by the activate script,
# but this doesn't happen in the testing environment
if 'LIBRARY_PREFIX' in os.environ:
    # Windows.
    gdalData = os.path.join(os.environ['LIBRARY_PREFIX'], 'share', 'gdal')
else:
    # Linux/OS X.
    gdalData = os.path.join(os.environ['PREFIX'], 'share', 'gdal')

os.environ['GDAL_DATA'] = gdalData

driver = gdal.GetDriverByName("netCDF")
assert driver is not None

driver = gdal.GetDriverByName("HDF4")
assert driver is not None

driver = gdal.GetDriverByName("HDF5")
assert driver is not None

driver = gdal.GetDriverByName("GTiff")
assert driver is not None

driver = gdal.GetDriverByName("PNG")
assert driver is not None

driver = gdal.GetDriverByName("JPEG")
assert driver is not None

# only available when libkea successfully linked in.
driver = gdal.GetDriverByName("KEA")
assert driver is not None

# only available when xerces-c++ successfully linked in.
driver = ogr.GetDriverByName("GML")
assert driver is not None

# only available when openjpeg successfully linked in.
driver = gdal.GetDriverByName("JP2OpenJPEG")
assert driver is not None

# only available when curl successfully linked in.
driver = gdal.GetDriverByName("WCS")
assert driver is not None

# only available when freexl successfully linked in.
driver = ogr.GetDriverByName("XLS")
assert driver is not None

# only available when expat successfully linked in.
driver = ogr.GetDriverByName("KML")
assert driver is not None

# only available when SQLite successfully linked in.
driver = ogr.GetDriverByName("SQLite")
assert driver is not None

# only available when PostgreSQL successfully linked in.
driver = ogr.GetDriverByName("PostgreSQL")
assert driver is not None

def has_geos():
    pnt1 = ogr.CreateGeometryFromWkt( 'POINT(10 20)' )
    pnt2 = ogr.CreateGeometryFromWkt( 'POINT(30 20)' )
    ogrex = ogr.GetUseExceptions()
    ogr.DontUseExceptions()
    hasgeos = pnt1.Union( pnt2 ) is not None
    if ogrex:
        ogr.UseExceptions()
    return hasgeos

assert has_geos(), 'GEOS not available within GDAL'

def has_proj():
    sr1 = osr.SpatialReference()
    sr1.ImportFromEPSG(4326) # lat, lon.
    sr2 = osr.SpatialReference()
    sr2.ImportFromEPSG(28355) # GDA94/MGA zone 55.
    osrex = osr.GetUseExceptions()
    osr.UseExceptions()
    hasproj = True
    # Use exceptions to determine if we have proj and epsg files
    # otherwise we can't reliably determine if it has failed.
    try:
        trans = osr.CoordinateTransformation(sr1, sr2)
    except RuntimeError:
        hasproj = False
    return hasproj

assert has_proj(), 'PROJ not available within GDAL'

# This module does some additional tests.
import extra_tests
<|endoftext|>


<commit_before>from .helper import IntegrationHelper

class TestRelease(IntegrationHelper):
    def test_delete(self):
        """Test the ability to delete a release"""
        self.token_login()
        cassette_name = self.cassette_name('delete')
        with self.recorder.use_cassette(cassette_name):
            repository = self.gh.repository('github3py', 'github3.py')
            release = repository.create_release(
                '0.8.0.pre', 'develop', '0.8.0 fake release',
                'To be deleted'
                )
            assert release is not None
            assert release.delete() is True

    def test_edit(self):
        """Test the ability to edit a release on a repository."""
        self.token_login()
        cassette_name = self.cassette_name('edit')
        with self.recorder.use_cassette(cassette_name):
            repository = self.gh.repository('sigmavirus24', 'charade')
            release = repository.release(85783)
            assert release.edit(body='Test editing a release') is True
            assert release.body == 'Test editing a release'

    def test_upload_asset(self):
        """Test the ability to upload an asset to a release."""
        self.token_login()
        cassette_name = self.cassette_name('upload_asset')
        with self.recorder.use_cassette(cassette_name):
            repository = self.gh.repository('github3py', 'github3.py')
            release = repository.create_release(
                '0.8.0.pre', 'develop', '0.8.0 fake release with upload',
                'To be deleted'
                )
            with open(__file__) as fd:
                asset = release.upload_asset(
                    'text/plain', 'test_repos_release.py', fd.read()
                    )
            assert isinstance(asset, github3.repos.release.Asset)
            release.delete()
<commit_msg>Add period to keep consistency<commit_after>import github3

from .helper import IntegrationHelper

class TestRelease(IntegrationHelper):
    def test_delete(self):
        """Test the ability to delete a release."""
        self.token_login()
        cassette_name = self.cassette_name('delete')
        with self.recorder.use_cassette(cassette_name):
            repository = self.gh.repository('github3py', 'github3.py')
            release = repository.create_release(
                '0.8.0.pre', 'develop', '0.8.0 fake release',
                'To be deleted'
                )
            assert release is not None
            assert release.delete() is True

    def test_edit(self):
        """Test the ability to edit a release on a repository."""
        self.token_login()
        cassette_name = self.cassette_name('edit')
        with self.recorder.use_cassette(cassette_name):
            repository = self.gh.repository('sigmavirus24', 'charade')
            release = repository.release(85783)
            assert release.edit(body='Test editing a release') is True
            assert release.body == 'Test editing a release'

    def test_upload_asset(self):
        """Test the ability to upload an asset to a release."""
        self.token_login()
        cassette_name = self.cassette_name('upload_asset')
        with self.recorder.use_cassette(cassette_name):
            repository = self.gh.repository('github3py', 'github3.py')
            release = repository.create_release(
                '0.8.0.pre', 'develop', '0.8.0 fake release with upload',
                'To be deleted'
                )
            with open(__file__) as fd:
                asset = release.upload_asset(
                    'text/plain', 'test_repos_release.py', fd.read()
                    )
            assert isinstance(asset, github3.repos.release.Asset)
            release.delete()
<|endoftext|>