<commit_before>from itertools import count
from types import *
from converters import sqlrepr

creationOrder = count()

class SODatabaseIndex(object):

    def __init__(self,
                 soClass,
                 name,
                 columns,
                 creationOrder,
                 unique=False):
        self.soClass = soClass
        self.name = name
        self.descriptions = self.convertColumns(columns)
        self.creationOrder = creationOrder
        self.unique = unique

    def get(self, *args, **kw):
        if not self.unique:
            raise AttributeError(
                "'%s' object has no attribute 'get' "
                "(index is not unique)" % self.name)
        connection = kw.pop('connection', None)
        if args and kw:
            raise TypeError("You cannot mix named and unnamed arguments")
        columns = [d['column'] for d in self.descriptions if 'column' in d]
        if kw and len(kw) != len(columns) or \
                args and len(args) != len(columns):
            raise TypeError(
                "get() takes exactly %d argument and an optional "
                "named argument 'connection' (%d given)" % (
                    len(columns), len(args)+len(kw)))
        if args:
            kw = {}
            for i in range(len(args)):
                if columns[i].foreignName is not None:
                    kw[columns[i].foreignName] = args[i]
                else:
                    kw[columns[i].name] = args[i]
        return self.soClass.selectBy(connection=connection, **kw).getOne()

    def convertColumns(self, columns):
        """
        Converts all the columns to dictionary descriptors;
        dereferences string column names.
        """
        new = []
        for desc in columns:
            if not isinstance(desc, dict):
                desc = {'column': desc}
            if 'expression' in desc:
                assert 'column' not in desc, (
                    'You cannot provide both an expression and a column '
                    '(for %s in index %s in %s)' %
                    (desc, self.name, self.soClass))
                assert 'length' not in desc, (
                    'length does not apply to expressions (for %s in '
                    'index %s in %s)' %
                    (desc, self.name, self.soClass))
                new.append(desc)
                continue
            columnName = desc['column']
            if not isinstance(columnName, str):
                columnName = columnName.name
            colDict = self.soClass.sqlmeta.columns
            if columnName not in colDict:
                for possible in colDict.values():
                    if possible.origName == columnName:
                        column = possible
                        break
                else:
                    # None found
                    raise ValueError(
                        "The column by the name %r was not found "
                        "in the class %r" % (columnName, self.soClass))
            else:
                column = colDict[columnName]
            desc['column'] = column
            new.append(desc)
        return new

    def getExpression(self, desc, db):
        if isinstance(desc['expression'], str):
            return desc['expression']
        else:
            return sqlrepr(desc['expression'], db)

    def sqliteCreateIndexSQL(self, soClass):
        if self.unique:
            uniqueOrIndex = 'UNIQUE INDEX'
        else:
            uniqueOrIndex = 'INDEX'
        spec = []
        for desc in self.descriptions:
            if 'expression' in desc:
                spec.append(self.getExpression(desc, 'sqlite'))
            else:
                spec.append(desc['column'].dbName)
        ret = 'CREATE %s %s_%s ON %s (%s)' % \
              (uniqueOrIndex,
               self.soClass.sqlmeta.table,
               self.name,
               self.soClass.sqlmeta.table,
               ', '.join(spec))
        return ret

    postgresCreateIndexSQL = maxdbCreateIndexSQL = mssqlCreateIndexSQL = \
        sybaseCreateIndexSQL = firebirdCreateIndexSQL = sqliteCreateIndexSQL

    def mysqlCreateIndexSQL(self, soClass):
        if self.unique:
            uniqueOrIndex = 'UNIQUE'
        else:
            uniqueOrIndex = 'INDEX'
        spec = []
        for desc in self.descriptions:
            if 'expression' in desc:
                spec.append(self.getExpression(desc, 'mysql'))
            elif 'length' in desc:
                spec.append('%s(%d)' % (desc['column'].dbName, desc['length']))
            else:
                spec.append(desc['column'].dbName)

        return 'ALTER TABLE %s ADD %s %s (%s)' % \
               (soClass.sqlmeta.table, uniqueOrIndex,
                self.name,
                ', '.join(spec))

class DatabaseIndex(object):
    """
    This takes a variable number of parameters, each of which is a
    column for indexing.  Each column may be a column object or the
    string name of the column (*not* the database name).  You may also
    use dictionaries, to further customize the indexing of the column.
    The dictionary may have certain keys:

    'column':
        The column object or string identifier.
    'length':
        MySQL will only index the first N characters if this is
        given.  For other databases this is ignored.
    'expression':
        You can create an index based on an expression, e.g.,
        'lower(column)'.  This can either be a string or a sqlbuilder
        expression.

    Further keys may be added to the column specs in the future.

    The class also take the keyword argument `unique`; if true then
    a UNIQUE index is created.
    """

    baseClass = SODatabaseIndex

    def __init__(self, *columns, **kw):
        kw['columns'] = columns
        self.kw = kw
        self.creationOrder = creationOrder.next()

    def setName(self, value):
        assert self.kw.get('name') is None, \
            "You cannot change a name after it has already been set " \
            "(from %s to %s)" % (self.kw['name'], value)
        self.kw['name'] = value

    def _get_name(self):
        return self.kw['name']

    def _set_name(self, value):
        self.setName(value)

    name = property(_get_name, _set_name)

    def withClass(self, soClass):
        return self.baseClass(soClass=soClass,
                              creationOrder=self.creationOrder, **self.kw)

    def __repr__(self):
        return '<%s %s %s>' % (
            self.__class__.__name__,
            hex(abs(id(self)))[2:],
            self.kw)

__all__ = ['DatabaseIndex']
<commit_msg>Change to relative import<commit_after>from itertools import count
from types import *
from .converters import sqlrepr

creationOrder = count()

class SODatabaseIndex(object):

    def __init__(self,
                 soClass,
                 name,
                 columns,
                 creationOrder,
                 unique=False):
        self.soClass = soClass
        self.name = name
        self.descriptions = self.convertColumns(columns)
        self.creationOrder = creationOrder
        self.unique = unique

    def get(self, *args, **kw):
        if not self.unique:
            raise AttributeError(
                "'%s' object has no attribute 'get' "
                "(index is not unique)" % self.name)
        connection = kw.pop('connection', None)
        if args and kw:
            raise TypeError("You cannot mix named and unnamed arguments")
        columns = [d['column'] for d in self.descriptions if 'column' in d]
        if kw and len(kw) != len(columns) or \
                args and len(args) != len(columns):
            raise TypeError(
                "get() takes exactly %d argument and an optional "
                "named argument 'connection' (%d given)" % (
                    len(columns), len(args)+len(kw)))
        if args:
            kw = {}
            for i in range(len(args)):
                if columns[i].foreignName is not None:
                    kw[columns[i].foreignName] = args[i]
                else:
                    kw[columns[i].name] = args[i]
        return self.soClass.selectBy(connection=connection, **kw).getOne()

    def convertColumns(self, columns):
        """
        Converts all the columns to dictionary descriptors;
        dereferences string column names.
        """
        new = []
        for desc in columns:
            if not isinstance(desc, dict):
                desc = {'column': desc}
            if 'expression' in desc:
                assert 'column' not in desc, (
                    'You cannot provide both an expression and a column '
                    '(for %s in index %s in %s)' %
                    (desc, self.name, self.soClass))
                assert 'length' not in desc, (
                    'length does not apply to expressions (for %s in '
                    'index %s in %s)' %
                    (desc, self.name, self.soClass))
                new.append(desc)
                continue
            columnName = desc['column']
            if not isinstance(columnName, str):
                columnName = columnName.name
            colDict = self.soClass.sqlmeta.columns
            if columnName not in colDict:
                for possible in colDict.values():
                    if possible.origName == columnName:
                        column = possible
                        break
                else:
                    # None found
                    raise ValueError(
                        "The column by the name %r was not found "
                        "in the class %r" % (columnName, self.soClass))
            else:
                column = colDict[columnName]
            desc['column'] = column
            new.append(desc)
        return new

    def getExpression(self, desc, db):
        if isinstance(desc['expression'], str):
            return desc['expression']
        else:
            return sqlrepr(desc['expression'], db)

    def sqliteCreateIndexSQL(self, soClass):
        if self.unique:
            uniqueOrIndex = 'UNIQUE INDEX'
        else:
            uniqueOrIndex = 'INDEX'
        spec = []
        for desc in self.descriptions:
            if 'expression' in desc:
                spec.append(self.getExpression(desc, 'sqlite'))
            else:
                spec.append(desc['column'].dbName)
        ret = 'CREATE %s %s_%s ON %s (%s)' % \
              (uniqueOrIndex,
               self.soClass.sqlmeta.table,
               self.name,
               self.soClass.sqlmeta.table,
               ', '.join(spec))
        return ret

    postgresCreateIndexSQL = maxdbCreateIndexSQL = mssqlCreateIndexSQL = \
        sybaseCreateIndexSQL = firebirdCreateIndexSQL = sqliteCreateIndexSQL

    def mysqlCreateIndexSQL(self, soClass):
        if self.unique:
            uniqueOrIndex = 'UNIQUE'
        else:
            uniqueOrIndex = 'INDEX'
        spec = []
        for desc in self.descriptions:
            if 'expression' in desc:
                spec.append(self.getExpression(desc, 'mysql'))
            elif 'length' in desc:
                spec.append('%s(%d)' % (desc['column'].dbName, desc['length']))
            else:
                spec.append(desc['column'].dbName)

        return 'ALTER TABLE %s ADD %s %s (%s)' % \
               (soClass.sqlmeta.table, uniqueOrIndex,
                self.name,
                ', '.join(spec))

class DatabaseIndex(object):
    """
    This takes a variable number of parameters, each of which is a
    column for indexing.  Each column may be a column object or the
    string name of the column (*not* the database name).  You may also
    use dictionaries, to further customize the indexing of the column.
    The dictionary may have certain keys:

    'column':
        The column object or string identifier.
    'length':
        MySQL will only index the first N characters if this is
        given.  For other databases this is ignored.
    'expression':
        You can create an index based on an expression, e.g.,
        'lower(column)'.  This can either be a string or a sqlbuilder
        expression.

    Further keys may be added to the column specs in the future.

    The class also take the keyword argument `unique`; if true then
    a UNIQUE index is created.
    """

    baseClass = SODatabaseIndex

    def __init__(self, *columns, **kw):
        kw['columns'] = columns
        self.kw = kw
        self.creationOrder = creationOrder.next()

    def setName(self, value):
        assert self.kw.get('name') is None, \
            "You cannot change a name after it has already been set " \
            "(from %s to %s)" % (self.kw['name'], value)
        self.kw['name'] = value

    def _get_name(self):
        return self.kw['name']

    def _set_name(self, value):
        self.setName(value)

    name = property(_get_name, _set_name)

    def withClass(self, soClass):
        return self.baseClass(soClass=soClass,
                              creationOrder=self.creationOrder, **self.kw)

    def __repr__(self):
        return '<%s %s %s>' % (
            self.__class__.__name__,
            hex(abs(id(self)))[2:],
            self.kw)

__all__ = ['DatabaseIndex']
<|endoftext|>


<commit_before>import functools32
import logging
_log = logging.getLogger(__name__)

def ascii(s):
  return s

class RedisIDAssigner(object):
  """
  assigns ids to object using a redis database
  """
  def __init__(self):
    import redis
    import caleydo_server.config

    c = caleydo_server.config.view('caleydo_data_redis.assigner')

    #print c.hostname, c.port, c.db
    self._db = redis.Redis(host=c.hostname, port=c.port, db=c.db)

    from werkzeug.contrib.cache import SimpleCache
    self._cache = SimpleCache(threshold=16384)

  @staticmethod
  def to_forward_key(idtype, identifier):
    return ascii(idtype + '2id.' + str(identifier))

  @staticmethod
  def to_backward_key(idtype, id):
    return ascii('id2' + idtype + '.' + str(id))

  def _get_entry(self, key):
    r = self._cache.get(key)
    if r is not None: #cached
      return r
    r = self._db.get(key) #lookup in db
    if r is not None: #cache if found
      self._cache.set(key, r)
    return r

  def unmap(self, uids, idtype):
    idtype = ascii(idtype)
    def lookup(id):
      key = self.to_backward_key(idtype, id)
      return self._get_entry(key)
    return map(lookup, uids)

  def load(self, idtype, mapping):
    """
    resets and loads the given mapping
    :param idtype:
    :param mapping: array of tuples (id, uid)
    :return:
    """
    idtype = ascii(idtype)

    #assuming incremental ids
    if idtype in self._db:
      _log.info('clearing %s',idtype)
      self._cache.clear()
      forward_keys = self._db.keys(idtype + '2id.*')
      _log.debug('found %d forward keys', len(forward_keys))
      self._db.delete(forward_keys)
      backward_keys = self._db.keys('id2' + idtype + '.*')
      _log.debug('found %d backwards keys', len(backward_keys))
      self._db.delete(backward_keys)
      _log.debug('deleted %d keys', len(backward_keys)+len(forward_keys))
      del self._db[idtype]

    if len(mapping) == 0:
      return #no mappings to load

    max_uid = None
    pipe = self._db.pipeline()

    _log.info('%s loading %s keys', idtype, len(mapping))
    for id,uid in mapping:
      key = self.to_forward_key(idtype, id)
      max_uid = uid if max_uid is None else max(uid, max_uid)
      pipe.set(key, uid)
      pipe.set(self.to_backward_key(idtype, uid), str(id))

    pipe.set(idtype, max_uid)
    _log.debug('flushing')

    pipe.execute()
    _log.debug('flushed')

  def __call__(self, ids, idtype):
    """
     return the integer index ids for the given ids in the given idtype
    """
    idtype = ascii(idtype)

    before = int(self._db.get(idtype) if idtype in self._db else self._db.decr(idtype)) #initialize with -1
    def lookup(id):
      key = self.to_forward_key(idtype, id)
      i = self._get_entry(key)
      if i is not None:
        return int(i)
      i = self._db.incr(idtype)
      self._db.set(key,i)
      self._db.set(self.to_backward_key(idtype, i),str(id))
      return i
    r = map(lookup, ids)

    after = int(self._db.get(idtype))
    if before != after:
      _log.debug('create %s %d!=%d',idtype,before,after)

    return r

def create():
  _log.info('create redis assigner')
  return RedisIDAssigner()
<commit_msg>remove unneeded import<commit_after>import logging
_log = logging.getLogger(__name__)

def ascii(s):
  return s

class RedisIDAssigner(object):
  """
  assigns ids to object using a redis database
  """
  def __init__(self):
    import redis
    import caleydo_server.config

    c = caleydo_server.config.view('caleydo_data_redis.assigner')

    #print c.hostname, c.port, c.db
    self._db = redis.Redis(host=c.hostname, port=c.port, db=c.db)

    from werkzeug.contrib.cache import SimpleCache
    self._cache = SimpleCache(threshold=16384)

  @staticmethod
  def to_forward_key(idtype, identifier):
    return ascii(idtype + '2id.' + str(identifier))

  @staticmethod
  def to_backward_key(idtype, id):
    return ascii('id2' + idtype + '.' + str(id))

  def _get_entry(self, key):
    r = self._cache.get(key)
    if r is not None: #cached
      return r
    r = self._db.get(key) #lookup in db
    if r is not None: #cache if found
      self._cache.set(key, r)
    return r

  def unmap(self, uids, idtype):
    idtype = ascii(idtype)
    def lookup(id):
      key = self.to_backward_key(idtype, id)
      return self._get_entry(key)
    return map(lookup, uids)

  def load(self, idtype, mapping):
    """
    resets and loads the given mapping
    :param idtype:
    :param mapping: array of tuples (id, uid)
    :return:
    """
    idtype = ascii(idtype)

    #assuming incremental ids
    if idtype in self._db:
      _log.info('clearing %s',idtype)
      self._cache.clear()
      forward_keys = self._db.keys(idtype + '2id.*')
      _log.debug('found %d forward keys', len(forward_keys))
      self._db.delete(forward_keys)
      backward_keys = self._db.keys('id2' + idtype + '.*')
      _log.debug('found %d backwards keys', len(backward_keys))
      self._db.delete(backward_keys)
      _log.debug('deleted %d keys', len(backward_keys)+len(forward_keys))
      del self._db[idtype]

    if len(mapping) == 0:
      return #no mappings to load

    max_uid = None
    pipe = self._db.pipeline()

    _log.info('%s loading %s keys', idtype, len(mapping))
    for id,uid in mapping:
      key = self.to_forward_key(idtype, id)
      max_uid = uid if max_uid is None else max(uid, max_uid)
      pipe.set(key, uid)
      pipe.set(self.to_backward_key(idtype, uid), str(id))

    pipe.set(idtype, max_uid)
    _log.debug('flushing')

    pipe.execute()
    _log.debug('flushed')

  def __call__(self, ids, idtype):
    """
     return the integer index ids for the given ids in the given idtype
    """
    idtype = ascii(idtype)

    before = int(self._db.get(idtype) if idtype in self._db else self._db.decr(idtype)) #initialize with -1
    def lookup(id):
      key = self.to_forward_key(idtype, id)
      i = self._get_entry(key)
      if i is not None:
        return int(i)
      i = self._db.incr(idtype)
      self._db.set(key,i)
      self._db.set(self.to_backward_key(idtype, i),str(id))
      return i
    r = map(lookup, ids)

    after = int(self._db.get(idtype))
    if before != after:
      _log.debug('create %s %d!=%d',idtype,before,after)

    return r

def create():
  _log.info('create redis assigner')
  return RedisIDAssigner()
<|endoftext|>


<commit_before>import enum
import http
from http import client
import operator
import typing as t
from urllib import parse

import aiohttp
from aiohttp import hdrs
from aiohttp import web

from . import abc

LABEL_PREFIX = 'CLA: '
CLA_OK = LABEL_PREFIX + '☑'
NO_CLA = LABEL_PREFIX + '☐'

@enum.unique
class PullRequestEvent(enum.Enum):
    # https://developer.github.com/v3/activity/events/types/#pullrequestevent
    assigned = "assigned"
    unassigned = "unassigned"
    labeled = "labeled"
    unlabeled = "unlabeled"
    opened = "opened"
    closed = "closed"
    reopened = "reopened"
    synchronize = "synchronize"

JSONType = t.Union[str, int, float, bool, None, t.Dict[str, t.Any],
                   t.List[t.Any]]

class Host(abc.ContribHost):

    """Implement a webhook for GitHub pull requests."""

    route = 'POST', '/github'

    _useful_actions =  {PullRequestEvent.opened.value,
                        PullRequestEvent.unlabeled.value,
                        PullRequestEvent.synchronize.value}

    def __init__(self, event: PullRequestEvent, request: JSONType):
        """Represent a contribution."""
        self.event = event
        self.request = request

    @classmethod
    async def process(cls, request):
        """Process the pull request."""
        # https://developer.github.com/webhooks/creating/#content-type
        if request.content_type != 'application/json':
            msg = ('can only accept application/json, '
                   'not {}').format(request.content_type)
            raise abc.ResponseExit(
                    status=http.HTTPStatus.UNSUPPORTED_MEDIA_TYPE, text=msg)

        payload = await request.json()
        if 'zen' in payload:
            # A ping event; nothing to do.
            # https://developer.github.com/webhooks/#ping-event
            raise abc.ResponseExit(status=http.HTTPStatus.NO_CONTENT)
        elif payload['action'] not in cls._useful_actions:
            raise abc.ResponseExit(status=http.HTTPStatus.NO_CONTENT)
        elif payload['action'] == PullRequestEvent.opened.value:
            return cls(PullRequestEvent.opened, payload)
        elif payload['action'] == PullRequestEvent.unlabeled.value:
            label = payload['label']['name']
            if not label.startswith(LABEL_PREFIX):
                raise abc.ResponseExit(status=http.HTTPStatus.NO_CONTENT)
            return cls(PullRequestEvent.unlabeled, payload)
        elif payload['action'] == PullRequestEvent.synchronize.value:
            return cls(PullRequestEvent.synchronize, payload)
        else:  # pragma: no cover
            # Should never happen.
            msg = "don't know how to handle a {!r} event".format(
                payload['action'])
            raise TypeError(msg)

    @staticmethod
    def check_response(response: web.Response):
        if response.status >= 300:
            msg = 'unexpected response: {}'.format(response.status)
            raise client.HTTPException(msg)

    async def get(self, url: str) -> JSONType:
        """Make a GET request for some JSON data.

        Abstracted out for easy testing w/o requiring internet access.
        """
        async with abc.session().get(url) as response:
            self.check_response(response)
            return (await response.json())

    async def post(self, url: str, payload: JSONType) -> None:
        """Make a POST request with JSON data to a URL."""
        encoded_json = json.dumps(payload).encode('utf-8')
        header = {hdrs.CONTENT_TYPE: 'application/json; charset=utf-8'}
        async with abc.session().post(url, data=encoded_json, headers=header) as response:
            self.check_response(response)

    async def delete(self, url: str) -> None:
        """Make a DELETE request to a URL."""
        async with abc.session().delete(url) as response:
            self.check_response(response)

    async def usernames(self):
        """Return an iterable with all of the contributors' usernames."""
        pull_request = self.request['pull_request']
        # Start with the author of the pull request.
        logins = {pull_request['user']['login']}
        # Fetch the commit data for the pull request.
        commits = await self.get(pull_request['commits_url'])
        # For each commit, get the author and committer.
        for commit in commits:
            logins.add(commit['author']['login'])
            logins.add(commit['committer']['login'])
        return frozenset(logins)

    async def labels_url(self, label=None):
        """Construct the URL to the label."""
        if not hasattr(self, '_labels_url'):
            issue_url = self.request['pull_request']['issue_url']
            issue_data = await self.get(issue_url)
            self._labels_url = issue_data['labels_url']
        quoted_label = ''
        if label is not None:
            quoted_label = '/' + parse.quote(label)
        mapping = {'/name': quoted_label}
        return self._labels_url.format_map(mapping)

    async def set_label(self, status: abc.Status) -> str:
        """Set the label on the pull request based on the status of the CLA."""
        labels_url = await self.labels_url()
        if status == abc.Status.signed:
            await self.post(labels_url, [CLA_OK])
            return CLA_OK
        else:
            await self.post(labels_url, [NO_CLA])
            return NO_CLA

    async def remove_labels(self) -> t.Container[str]:
        """Remove any CLA-related labels from the pull request."""
        removed = set()
        labels_url = await self.labels_url()
        all_labels = map(operator.itemgetter('name'),
                         await self.get(labels_url))
        for cla_label in (x for x in all_labels if x.startswith(LABEL_PREFIX)):
            deletion_url = await self.labels_url(cla_label)
            await self.delete(deletion_url)
            removed.add(cla_label)
        return frozenset(removed)

    async def comment(self, status: abc.Status):
        """Add an appropriate comment relating to the CLA status."""
        if status == abc.Status.signed:
            return
        # XXX not_signed
        # XXX username_not_found

    async def update(self, status):
        if self.event == PullRequestEvent.opened:
            await self.set_label(status)
            await self.comment(status)
        elif self.event == PullRequestEvent.unlabeled:
            # The assumption is that a PR will almost always go from no CLA to
            # being cleared, so don't bug the user with what will probably
            # amount to a repeated message about lacking a CLA.
            await self.set_label(status)
        elif self.event == PullRequestEvent.synchronize:
            current_label = await self.current_label()
            if status == abc.Status.signed:
                if current_label != CLA_OK:
                    await self.remove_labels()
            elif current_label != NO_CLA:
                    await self.remove_labels()
                    # Since there is a chance a new person was added to a PR
                    # which caused the change in status, a comment on how to
                    # resolve the CLA issue is probably called for.
                    await self.comment(status)
        else:  # pragma: no cover
            # Should never be reached.
            msg = 'do not know how to update a PR for {}'.format(self.event)
            raise RunimeError(msg)
<commit_msg>First try and writing comment text for missing details<commit_after>import enum
import http
from http import client
import operator
import typing as t
from urllib import parse

import aiohttp
from aiohttp import hdrs
from aiohttp import web

from . import abc

LABEL_PREFIX = 'CLA: '
CLA_OK = LABEL_PREFIX + '☑'
NO_CLA = LABEL_PREFIX + '☐'

NO_CLA_TEMPLATE = """Hello, and thanks for your contribution!

{body}

Once you have done everything that's needed, please reply here and someone will
verify everything is in order.

Also, please be aware that all interactions involving this repository fall
under the
[PSF Code of Conduct](https://www.python.org/psf/codeofconduct/).
"""

NO_CLA_BODY = """Unfortunately our records indicate you have not signed a
[PSF contributor agreement](https://www.python.org/psf/contrib/contrib-form/)
(CLA). For legal reasons we need you to sign this before we can look at your
contribution."""

NO_USERNAME_BODY = """Unfortunately we couldn't find an account corresponding
to your GitHub username at [bugs.python.org](http://bugs.python.org/) (b.p.o).
If you don't already have an account at b.p.o, please
[create one](http://bugs.python.org/user?@template=register) and make sure to
add your GitHub username. If you do already have an account at b.p.o then
please go there and under "Your Details" add your GitHub username.

And in case you haven't already, please make sure to sign the
[PSF contributor agreement](https://www.python.org/psf/contrib/contrib-form/)
(CLA); we can't legally look at your contribution until you have signed the
CLA."""

@enum.unique
class PullRequestEvent(enum.Enum):
    # https://developer.github.com/v3/activity/events/types/#pullrequestevent
    assigned = "assigned"
    unassigned = "unassigned"
    labeled = "labeled"
    unlabeled = "unlabeled"
    opened = "opened"
    closed = "closed"
    reopened = "reopened"
    synchronize = "synchronize"

JSONType = t.Union[str, int, float, bool, None, t.Dict[str, t.Any],
                   t.List[t.Any]]

class Host(abc.ContribHost):

    """Implement a webhook for GitHub pull requests."""

    route = 'POST', '/github'

    _useful_actions =  {PullRequestEvent.opened.value,
                        PullRequestEvent.unlabeled.value,
                        PullRequestEvent.synchronize.value}

    def __init__(self, event: PullRequestEvent, request: JSONType):
        """Represent a contribution."""
        self.event = event
        self.request = request

    @classmethod
    async def process(cls, request):
        """Process the pull request."""
        # https://developer.github.com/webhooks/creating/#content-type
        if request.content_type != 'application/json':
            msg = ('can only accept application/json, '
                   'not {}').format(request.content_type)
            raise abc.ResponseExit(
                    status=http.HTTPStatus.UNSUPPORTED_MEDIA_TYPE, text=msg)

        payload = await request.json()
        if 'zen' in payload:
            # A ping event; nothing to do.
            # https://developer.github.com/webhooks/#ping-event
            raise abc.ResponseExit(status=http.HTTPStatus.NO_CONTENT)
        elif payload['action'] not in cls._useful_actions:
            raise abc.ResponseExit(status=http.HTTPStatus.NO_CONTENT)
        elif payload['action'] == PullRequestEvent.opened.value:
            return cls(PullRequestEvent.opened, payload)
        elif payload['action'] == PullRequestEvent.unlabeled.value:
            label = payload['label']['name']
            if not label.startswith(LABEL_PREFIX):
                raise abc.ResponseExit(status=http.HTTPStatus.NO_CONTENT)
            return cls(PullRequestEvent.unlabeled, payload)
        elif payload['action'] == PullRequestEvent.synchronize.value:
            return cls(PullRequestEvent.synchronize, payload)
        else:  # pragma: no cover
            # Should never happen.
            msg = "don't know how to handle a {!r} event".format(
                payload['action'])
            raise TypeError(msg)

    @staticmethod
    def check_response(response: web.Response):
        if response.status >= 300:
            msg = 'unexpected response: {}'.format(response.status)
            raise client.HTTPException(msg)

    async def get(self, url: str) -> JSONType:
        """Make a GET request for some JSON data.

        Abstracted out for easy testing w/o requiring internet access.
        """
        async with abc.session().get(url) as response:
            self.check_response(response)
            return (await response.json())

    async def post(self, url: str, payload: JSONType) -> None:
        """Make a POST request with JSON data to a URL."""
        encoded_json = json.dumps(payload).encode('utf-8')
        header = {hdrs.CONTENT_TYPE: 'application/json; charset=utf-8'}
        async with abc.session().post(url, data=encoded_json, headers=header) as response:
            self.check_response(response)

    async def delete(self, url: str) -> None:
        """Make a DELETE request to a URL."""
        async with abc.session().delete(url) as response:
            self.check_response(response)

    async def usernames(self):
        """Return an iterable with all of the contributors' usernames."""
        pull_request = self.request['pull_request']
        # Start with the author of the pull request.
        logins = {pull_request['user']['login']}
        # Fetch the commit data for the pull request.
        commits = await self.get(pull_request['commits_url'])
        # For each commit, get the author and committer.
        for commit in commits:
            logins.add(commit['author']['login'])
            logins.add(commit['committer']['login'])
        return frozenset(logins)

    async def labels_url(self, label=None):
        """Construct the URL to the label."""
        if not hasattr(self, '_labels_url'):
            issue_url = self.request['pull_request']['issue_url']
            issue_data = await self.get(issue_url)
            self._labels_url = issue_data['labels_url']
        quoted_label = ''
        if label is not None:
            quoted_label = '/' + parse.quote(label)
        mapping = {'/name': quoted_label}
        return self._labels_url.format_map(mapping)

    async def set_label(self, status: abc.Status) -> str:
        """Set the label on the pull request based on the status of the CLA."""
        labels_url = await self.labels_url()
        if status == abc.Status.signed:
            await self.post(labels_url, [CLA_OK])
            return CLA_OK
        else:
            await self.post(labels_url, [NO_CLA])
            return NO_CLA

    async def remove_labels(self) -> t.Container[str]:
        """Remove any CLA-related labels from the pull request."""
        removed = set()
        labels_url = await self.labels_url()
        all_labels = map(operator.itemgetter('name'),
                         await self.get(labels_url))
        for cla_label in (x for x in all_labels if x.startswith(LABEL_PREFIX)):
            deletion_url = await self.labels_url(cla_label)
            await self.delete(deletion_url)
            removed.add(cla_label)
        return frozenset(removed)

    async def comment(self, status: abc.Status):
        """Add an appropriate comment relating to the CLA status."""
        if status == abc.Status.signed:
            return
        # XXX not_signed
        # XXX username_not_found

    async def update(self, status):
        if self.event == PullRequestEvent.opened:
            await self.set_label(status)
            await self.comment(status)
        elif self.event == PullRequestEvent.unlabeled:
            # The assumption is that a PR will almost always go from no CLA to
            # being cleared, so don't bug the user with what will probably
            # amount to a repeated message about lacking a CLA.
            await self.set_label(status)
        elif self.event == PullRequestEvent.synchronize:
            current_label = await self.current_label()
            if status == abc.Status.signed:
                if current_label != CLA_OK:
                    await self.remove_labels()
            elif current_label != NO_CLA:
                    await self.remove_labels()
                    # Since there is a chance a new person was added to a PR
                    # which caused the change in status, a comment on how to
                    # resolve the CLA issue is probably called for.
                    await self.comment(status)
        else:  # pragma: no cover
            # Should never be reached.
            msg = 'do not know how to update a PR for {}'.format(self.event)
            raise RunimeError(msg)
<|endoftext|>


<commit_before>import logging
import time
import json

from boto.sqs.message import Message

from nymms import resources
from nymms.config import config
from nymms.tasks import Task
from nymms.scheduler.Scheduler import Scheduler

logger = logging.getLogger(__name__)

class AWSScheduler(Scheduler):
    def __init__(self, node_backend, conn_mgr, task_queue):
        self._node_backend = node_backend
        self._conn = conn_mgr
        self._queue_name = task_queue
        self._default_queue = None
        self._realm_queues = {}
        logger.debug(self.__class__.__name__ + " initialized.")

    def _setup_realm(self, realm):
        if realm in self._realm_queues:
            return
        queue_name = self._queue_name + '_REALM_' + realm
        logger.debug("setting up realm queue %s", queue_name)
        self._realm_queues[realm] = self._conn.sqs.create_queue(queue_name)

    def _setup_queue(self):
        if self._default_queue:
            return
        logger.debug("setting up queue %s", self._queue_name)
        self._default_queue = self._conn.sqs.create_queue(self._queue_name)

    def _choose_queue(self, task):
        realm = task.context['realm']
        print "REALM: %s" % (realm)
        if realm:
            self._setup_realm(realm)
            queue = self._realm_queues[realm]
        else:
            self._setup_queue()
            queue = self._default_queue
        return queue

    def submit_task(self, task, **kwargs):
        queue = self._choose_queue(task)
        logger.debug("Sending task '%s' to queue '%s'.", task.id,
                     queue.name)
        m = Message()
        m.set_body(json.dumps(task.serialize()))
        return queue.write(m)
<commit_msg>remove debug print statement<commit_after>import logging
import time
import json

from boto.sqs.message import Message

from nymms import resources
from nymms.config import config
from nymms.tasks import Task
from nymms.scheduler.Scheduler import Scheduler

logger = logging.getLogger(__name__)

class AWSScheduler(Scheduler):
    def __init__(self, node_backend, conn_mgr, task_queue):
        self._node_backend = node_backend
        self._conn = conn_mgr
        self._queue_name = task_queue
        self._default_queue = None
        self._realm_queues = {}
        logger.debug(self.__class__.__name__ + " initialized.")

    def _setup_realm(self, realm):
        if realm in self._realm_queues:
            return
        queue_name = self._queue_name + '_REALM_' + realm
        logger.debug("setting up realm queue %s", queue_name)
        self._realm_queues[realm] = self._conn.sqs.create_queue(queue_name)

    def _setup_queue(self):
        if self._default_queue:
            return
        logger.debug("setting up queue %s", self._queue_name)
        self._default_queue = self._conn.sqs.create_queue(self._queue_name)

    def _choose_queue(self, task):
        realm = task.context['realm']
        if realm:
            self._setup_realm(realm)
            queue = self._realm_queues[realm]
        else:
            self._setup_queue()
            queue = self._default_queue
        return queue

    def submit_task(self, task, **kwargs):
        queue = self._choose_queue(task)
        logger.debug("Sending task '%s' to queue '%s'.", task.id,
                     queue.name)
        m = Message()
        m.set_body(json.dumps(task.serialize()))
        return queue.write(m)
<|endoftext|>


<commit_before>import os
import misaka as m
from flask import Flask, render_template, abort

def preview(working_directory=None, host=None, port=None):
    """Runs a local server to preview the working directory of a repository."""

    # Flask application
    app = Flask('gitpress')
    app.config.from_pyfile('default_config.py')
    app.config.from_pyfile('local_config.py', silent=True)

    # Override default configuration
    config = {'HOST': host, 'PORT': port, 'WORKING_DIRECTORY': working_directory}
    app.config.update((k, v) for k, v in config.iteritems() if v is not None)

    # Shared values
    working_directory = os.path.abspath(app.config['WORKING_DIRECTORY'])
    site_title = os.path.basename(working_directory).title()
    pages = _pages(working_directory)

    # Views
    @app.route('/')
    def index(slug=None):
        return render_template('index.html', site_title=site_title, pages=pages)

    @app.route('/<path:slug>')
    def page(slug):
        page = _page_for(pages, slug)
        # TODO: use Gitpress rules and Git for validation
        if not page:
            abort(404)
        return render_template('page.html', site_title=site_title, page=page)

    # Run local server
    app.run(app.config['HOST'], app.config['PORT'], debug=app.debug, use_reloader=app.config['DEBUG_GITPRESS'])

class _Page:
    """Represents a named or unordered page."""
    def __init__(self, root, path):
        self.root = root
        self.path = path
        self.fullpath = os.path.join(root, path)
        self.filename = os.path.basename(path)
        # TODO: use Gitpress with settings to transform paths into URLs
        self.url = _slugify(self.filename)
        self.title = self.url.replace('-', ' ').title()
        self.content = _read_file(self.fullpath)
        # TODO: Use smarter content cutoff
        self.summary = m.html(self.content[:300])
        self.content = m.html(self.content)

def _page_for(pages, slug):
    """Gets a page from the specified slug."""
    for page in pages:
        if page.url == slug:
            return page
    return None

def _read_file(path):
    """Reads a file and closes it."""
    with open(path) as f:
        return f.read()

def _slugify(path):
    """Returns a slug from the specified path."""
    path = os.path.normpath(path)
    path = os.path.splitext(path)[0]
    return '-'.join(path.split('-')[1:])

def _ordered_pages(path):
    """Returns all the ordered pages from the specified site."""
    # TODO: use Gitpress rules for page iteration
    posts = []
    for root, dirs, files in os.walk(path):
        if not _is_post_path(os.path.relpath(root, path)):
            continue
        posts += [os.path.join(root, filename) for filename in files if _starts_with_digit(filename)]
    return posts

def _named_pages(path):
    """Returns all the named pages from the specified site."""
    # TODO: get named pages
    return []

def _pages(path):
    """Gets a list of available named and ordered pages."""
    page_paths = _ordered_pages(path) + _named_pages(path)
    return map(lambda page_path: _Page(path, page_path), page_paths)

def _starts_with_digit(s):
    """Returns whether the specified string starts with a digit."""
    return len(s) > 1 and s[0].isdigit()

def _is_post_path(path):
    """Returns whether the specified path is valid for a post."""
    while path:
        path, directory = os.path.split(path)
        if not _starts_with_digit(directory):
            return False
    return True
<commit_msg>Throw exception when working directory not found.<commit_after>import os
import misaka as m
from flask import Flask, render_template, abort

def preview(working_directory=None, host=None, port=None):
    """Runs a local server to preview the working directory of a repository."""

    # Check for working directory
    if working_directory and not os.path.isdir(working_directory):
        raise ValueError('Directory not found: ' + repr(working_directory))

    # Flask application
    app = Flask('gitpress')
    app.config.from_pyfile('default_config.py')
    app.config.from_pyfile('local_config.py', silent=True)

    # Override default configuration
    config = {'HOST': host, 'PORT': port, 'WORKING_DIRECTORY': working_directory}
    app.config.update((k, v) for k, v in config.iteritems() if v is not None)

    # Shared values
    working_directory = os.path.abspath(app.config['WORKING_DIRECTORY'])
    site_title = os.path.basename(working_directory).title()
    pages = _pages(working_directory)

    # Views
    @app.route('/')
    def index(slug=None):
        return render_template('index.html', site_title=site_title, pages=pages)

    @app.route('/<path:slug>')
    def page(slug):
        page = _page_for(pages, slug)
        # TODO: use Gitpress rules and Git for validation
        if not page:
            abort(404)
        return render_template('page.html', site_title=site_title, page=page)

    # Run local server
    app.run(app.config['HOST'], app.config['PORT'], debug=app.debug, use_reloader=app.config['DEBUG_GITPRESS'])

class _Page:
    """Represents a named or unordered page."""
    def __init__(self, root, path):
        self.root = root
        self.path = path
        self.fullpath = os.path.join(root, path)
        self.filename = os.path.basename(path)
        # TODO: use Gitpress with settings to transform paths into URLs
        self.url = _slugify(self.filename)
        self.title = self.url.replace('-', ' ').title()
        self.content = _read_file(self.fullpath)
        # TODO: Use smarter content cutoff
        self.summary = m.html(self.content[:300])
        self.content = m.html(self.content)

def _page_for(pages, slug):
    """Gets a page from the specified slug."""
    for page in pages:
        if page.url == slug:
            return page
    return None

def _read_file(path):
    """Reads a file and closes it."""
    with open(path) as f:
        return f.read()

def _slugify(path):
    """Returns a slug from the specified path."""
    path = os.path.normpath(path)
    path = os.path.splitext(path)[0]
    return '-'.join(path.split('-')[1:])

def _ordered_pages(path):
    """Returns all the ordered pages from the specified site."""
    # TODO: use Gitpress rules for page iteration
    posts = []
    for root, dirs, files in os.walk(path):
        if not _is_post_path(os.path.relpath(root, path)):
            continue
        posts += [os.path.join(root, filename) for filename in files if _starts_with_digit(filename)]
    return posts

def _named_pages(path):
    """Returns all the named pages from the specified site."""
    # TODO: get named pages
    return []

def _pages(path):
    """Gets a list of available named and ordered pages."""
    page_paths = _ordered_pages(path) + _named_pages(path)
    return map(lambda page_path: _Page(path, page_path), page_paths)

def _starts_with_digit(s):
    """Returns whether the specified string starts with a digit."""
    return len(s) > 1 and s[0].isdigit()

def _is_post_path(path):
    """Returns whether the specified path is valid for a post."""
    while path:
        path, directory = os.path.split(path)
        if not _starts_with_digit(directory):
            return False
    return True
<|endoftext|>


<commit_before>from tastypie.resources import ModelResource, ALL
from tastypie import fields

from astrobin.models import Image, ImageRevision, Subject, SubjectIdentifier, \
                            Comment

class SubjectResource(ModelResource):
    identifiers = fields.ToManyField('astrobin.api.SubjectIdentifierResource', 'idlist')

    class Meta:
        queryset = Subject.objects.all()
        allowed_methods = ['get']

class SubjectIdentifierResource(ModelResource):
    subject = fields.ForeignKey(SubjectResource, 'subject')

    class Meta:
        queryset = SubjectIdentifier.objects.all()
        allowed_methods = ['get']

class CommentResource(ModelResource):
    image = fields.ForeignKey('astrobin.api.ImageResource', 'image')
    author = fields.CharField()
    replies = fields.ToManyField('self', 'children')

    class Meta:
        queryset = Comment.objects.all()
        fields = ['comment', 'added']

    def dehydrate_comment(self, bundle):
        if bundle.obj.is_deleted:
            return '(comment deleted)'
        return bundle.data['comment']

    def dehydrate_author(self, bundle):
        return bundle.obj.author.username

class ImageRevisionResource(ModelResource):
    image = fields.ForeignKey('astrobin.api.ImageResource', 'image')

    class Meta:
        queryset = ImageRevision.objects.all()
        fields = [
            'uploaded',
            'w',
            'h',
            'is_solved',
            'is_final',
        ]
        allowed_methods = ['get']

class ImageResource(ModelResource):
    author = fields.CharField()
    revisions = fields.ToManyField(ImageRevisionResource, 'imagerevision_set')

    subjects = fields.ToManyField(SubjectResource, 'subjects')

    imaging_telescopes = fields.ListField()
    imaging_cameras = fields.ListField()

    comments = fields.ToManyField(CommentResource, 'comment_set')

    class Meta:
        queryset = Image.objects.filter(is_stored = True, is_wip = False)
        fields = [
            'id',
            'author',
            'title',
            'uploaded',
            'updated',
            'description',
            'h',
            'w',
            'animated',
            'link',
            'link_to_fits',
            'rating_score',
            'rating_votes',
            'is_solved',
            'license',

            'is_final',

            'ra_center_hms',
            'dec_center_dms',
            'orientation',
            'pixel_scale',
            'fieldw',
            'fieldh',
            'fieldunits',
        ]
        allowed_methods = ['get']
        filtering = {
            'title': ALL,
            'is_solved': ('exact',),
            'author': ALL,
            'uploaded': ALL,
            'fieldw': ALL,
            'fieldh': ALL,
            'fieldunits': ALL,
        }

    def dehydrate_imaging_telescopes(self, bundle):
        telescopes = bundle.obj.imaging_telescopes.all()
        return [x.name for x in telescopes]

    def dehydrate_imaging_cameras(self, bundle):
        cameras = bundle.obj.imaging_cameras.all()
        return [x.name for x in cameras]

    def dehydrate_author(self, bundle):
        return bundle.obj.user.username

    def build_filters(self, filters = None):
        if filters is None:
            filters = {}

        orm_filters = super(ImageResource, self).build_filters(filters)

        #if 'author' in filters:
        #   qs = Image.objects.filter(user__username = filters['author'])
        #   orm_filters['pk__in'] = [i.pk for i in qs]

        return orm_filters
<commit_msg>Allow various filtering and ordering on the API.<commit_after>from django.db.models import Q

from tastypie.resources import ModelResource, ALL
from tastypie import fields

from astrobin.models import Image, ImageRevision, Subject, SubjectIdentifier, \
                            Comment

class SubjectResource(ModelResource):
    identifiers = fields.ToManyField('astrobin.api.SubjectIdentifierResource', 'idlist')

    class Meta:
        queryset = Subject.objects.all()
        allowed_methods = ['get']

class SubjectIdentifierResource(ModelResource):
    subject = fields.ForeignKey(SubjectResource, 'subject')

    class Meta:
        queryset = SubjectIdentifier.objects.all()
        allowed_methods = ['get']

class CommentResource(ModelResource):
    image = fields.ForeignKey('astrobin.api.ImageResource', 'image')
    author = fields.CharField()
    replies = fields.ToManyField('self', 'children')

    class Meta:
        queryset = Comment.objects.all()
        fields = ['comment', 'added']

    def dehydrate_comment(self, bundle):
        if bundle.obj.is_deleted:
            return '(comment deleted)'
        return bundle.data['comment']

    def dehydrate_author(self, bundle):
        return bundle.obj.author.username

class ImageRevisionResource(ModelResource):
    image = fields.ForeignKey('astrobin.api.ImageResource', 'image')

    class Meta:
        queryset = ImageRevision.objects.all()
        fields = [
            'uploaded',
            'w',
            'h',
            'is_solved',
            'is_final',
        ]
        allowed_methods = ['get']

class ImageResource(ModelResource):
    user = fields.CharField('user__username')
    revisions = fields.ToManyField(ImageRevisionResource, 'imagerevision_set')

    subjects = fields.ToManyField(SubjectResource, 'subjects')

    imaging_telescopes = fields.ListField()
    imaging_cameras = fields.ListField()

    uploaded = fields.DateField('uploaded')
    updated = fields.DateField('updated')

    comments = fields.ToManyField(CommentResource, 'comment_set')

    class Meta:
        queryset = Image.objects.filter(is_stored = True, is_wip = False)
        fields = [
            'id',
            'title',

            'filename',
            'original_ext',
            'description',
            'h',
            'w',
            'animated',
            'link',
            'link_to_fits',
            'rating_score',
            'rating_votes',
            'is_solved',
            'license',

            'is_final',

            'ra_center_hms',
            'dec_center_dms',
            'orientation',
            'pixel_scale',
            'fieldw',
            'fieldh',
            'fieldunits',
        ]
        allowed_methods = ['get']
        filtering = {
            'title': ALL,
            'is_solved': ('exact',),
            'user': ALL,
            'uploaded': ALL,
            'fieldw': ALL,
            'fieldh': ALL,
            'fieldunits': ALL,
        }
        ordering = ['rating_score', 'rating_votes']

    def dehydrate_imaging_telescopes(self, bundle):
        telescopes = bundle.obj.imaging_telescopes.all()
        return [x.name for x in telescopes]

    def dehydrate_imaging_cameras(self, bundle):
        cameras = bundle.obj.imaging_cameras.all()
        return [x.name for x in cameras]

    def build_filters(self, filters = None):
        if filters is None:
            filters = {}

        orm_filters = super(ImageResource, self).build_filters(filters)

        if 'subject' in filters:
            qs = Image.objects.filter(
                Q(subjects__mainId = filters['subject']) |
                Q(subjects__idlist__identifier = filters['subject']))
            orm_filters['pk__in'] = [i.pk for i in qs]

        return orm_filters
<|endoftext|>


<commit_before># !/usr/bin/python
# -*- coding: utf-8

import numpy as np

from moviepy.audio.AudioClip import AudioArrayClip
from moviepy.audio.io.AudioFileClip import AudioFileClip
from moviepy.video.VideoClip import TextClip, ImageClip
from moviepy.video.compositing.CompositeVideoClip import CompositeVideoClip

from random import choice
from os import listdir

def speedup_audio(sound_array, factor):
    """ Multiplies the sound's speed by some factor (including pitch shift).
        source: http://goo.gl/MlI9UM """

    indices = np.round(np.arange(0, len(sound_array), factor))
    indices = indices[indices < len(sound_array)].astype(int)
    return sound_array[indices.astype(int)]

def prepare_audio(file_name, rate=44100, speedup=1.3):
    """ Read the audio from the input file, speed it up and return it as an
        AudioArrayClip. """

    audio = AudioFileClip(file_name)
    data = audio.to_soundarray(fps=rate)
    data = speedup_audio(data, speedup)

    return AudioArrayClip(data, fps=rate)

def prepare_image(text, duration, fps=1):
    """ Load a random image from file, add text to it and return it as a
        clip. """

    image_file = choice([file for file in listdir("./images")])

    image = ImageClip("images/" + image_file).set_fps(fps)
    text = TextClip(txt=text, bg_color="rgba(0, 0, 0, 0.7)", color="white",
                    size=(1920, 150), font="Droid-Sans-Fallback")

    text = text.set_pos((0, 100))

    return CompositeVideoClip([image, text]).set_duration(duration)

def nightcorify(source_file, destination_file, text):
    """ Create a nightcore video using the audio from source_file and write it
        to destination_file. Draw the text on top of the video. """

    audio = prepare_audio(source_file)
    image = prepare_image(text, audio.duration)
    video = image.set_audio(audio)

    video.write_videofile(destination_file)

if __name__ == "__main__":
    from sys import argv, exit

    if len(argv) < 4:
        print "usage: nightcore.py <inputfile.wav> <outputfile.avi> <title>"
        exit(1)

    nightcorify(argv[1], argv[2], argv[3])
<commit_msg>me so stupid<commit_after># !/usr/bin/python
# -*- coding: utf-8 -*-

import numpy as np

from moviepy.audio.AudioClip import AudioArrayClip
from moviepy.audio.io.AudioFileClip import AudioFileClip
from moviepy.video.VideoClip import TextClip, ImageClip
from moviepy.video.compositing.CompositeVideoClip import CompositeVideoClip

from random import choice
from os import listdir

def speedup_audio(sound_array, factor):
    """ Multiplies the sound's speed by some factor (including pitch shift).
        source: http://goo.gl/MlI9UM """

    indices = np.round(np.arange(0, len(sound_array), factor))
    indices = indices[indices < len(sound_array)].astype(int)
    return sound_array[indices.astype(int)]

def prepare_audio(file_name, rate=44100, speedup=1.3):
    """ Read the audio from the input file, speed it up and return it as an
        AudioArrayClip. """

    audio = AudioFileClip(file_name)
    data = audio.to_soundarray(fps=rate)
    data = speedup_audio(data, speedup)

    return AudioArrayClip(data, fps=rate)

def prepare_image(text, duration, fps=1):
    """ Load a random image from file, add text to it and return it as a
        clip. """

    image_file = choice([file for file in listdir("./images")])

    image = ImageClip("images/" + image_file).set_fps(fps)
    text = TextClip(txt=text, bg_color="rgba(0, 0, 0, 0.7)", color="white",
                    size=(1920, 150), font="Droid-Sans-Fallback")

    text = text.set_pos((0, 100))

    return CompositeVideoClip([image, text]).set_duration(duration)

def nightcorify(source_file, destination_file, text):
    """ Create a nightcore video using the audio from source_file and write it
        to destination_file. Draw the text on top of the video. """

    audio = prepare_audio(source_file)
    image = prepare_image(text, audio.duration)
    video = image.set_audio(audio)

    video.write_videofile(destination_file)

if __name__ == "__main__":
    from sys import argv, exit

    if len(argv) < 4:
        print "usage: nightcore.py <inputfile.wav> <outputfile.avi> <title>"
        exit(1)

    nightcorify(argv[1], argv[2], argv[3])
<|endoftext|>


<commit_before>'''
MMap Arrays Test Suite

(c) 2014 Farsight Security Inc.

Released under the MIT license.  See license.txt.
'''

import math
import unittest
import tempfile
import os

import mmaparray
import six

def setUp(typecode, min_val=0, max_val=0, size=1024):
    def fn(self):
        self.backing = tempfile.NamedTemporaryFile(prefix='mmapfile_test')
        self.backing.file.close()

        self.min_val = min_val
        self.max_val = max_val
        self.span = 1 + max_val - min_val
        self.size = size

        self.array = mmaparray.array(self.backing.name, typecode, self.size)
    return fn

class TestMMapArrayGeneric:
    def tearDown(self):
        self.array.close()
        self.backing.close()

    def test_reopen(self):
        array2 = type(self.array)(self.array.name)
        self.assertEqual(len(array2), len(self.array))

    def test_out_of_range(self):
        if self.max_val:
            with self.assertRaises(OverflowError):
                self.array[0] = self.min_val-1
                self.array[0] = self.max_val+1

    def test_out_of_bounds(self):
        with self.assertRaises(OverflowError):
            self.array[-1]
            self.array[-1] = 0

        with self.assertRaises(IndexError):
            self.array[len(self.array)]
            self.array[len(self.array)] = 0

    def test_setgetitem(self):
        self.assertEqual(self.array[0], 0)
        self.array[0] = 1
        self.assertEqual(self.array[0], 1)

        for i in range(1,8):
            self.assertEqual(self.array[i], 0)

        for i in range(1,8):
            self.array[i] = 1
        self.array[0] = 0

        self.assertEqual(self.array[0], 0)
        for i in range(1,8):
            self.assertEqual(self.array[i], 1)

    def test_len(self):
        self.assertEqual(len(self.array), self.size)

    def test_name(self):
        self.assertEqual(self.array.name, six.b(self.backing.name))

    def test_iter(self):
        for i in range(0, self.size):
            self.array[i] = (i+1) % self.span + self.min_val
        for i, val in enumerate(self.array):
            self.assertEqual((i+1) % self.span + self.min_val, val)

    def test_populate(self):
        type(self.array)(self.array.name, want_populate=True)

    def test_fallocate(self):
        arrayfile = tempfile.NamedTemporaryFile(prefix='mmapfile_test')
        array2 = type(self.array)(arrayfile.name, 1000, want_fallocate=True)
        self.assertEqual(os.fstat(array2.fd).st_size, array2.bytesize)

class TestMMapBitArray(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('o')

    def test_iter(self):
        for i in range(0, self.size):
            self.array[i] = i % 2
        for i, val in enumerate(self.array):
            self.assertEqual(i % 2, val)

class TestMMapInt8Array(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('b', -2**7, 2**7-1)

class TestMMapUint8Array(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('B', 0, 2**8-1)

class TestMMapInt16Array(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('h', -2**15, 2**15-1, size=2**18)

class TestMMapUint16Array(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('H', 0, 2**16-1, size=2**18)

class TestMMapInt32Array(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('i', -2**31, 2**31-1)

class TestMMapUint32Array(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('I', 0, 2**32-1)

class TestMMapInt64Array(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('l', -2**63, 2**63-1)

class TestMMapUint64Array(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('L', 0, 2**64-1)

class TestMMapFloatArray(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('f')

    def test_iter(self):
        for i in range(0, self.size):
            self.array[i] = math.sqrt(i)
        for i, val in enumerate(self.array):
            assert abs(math.sqrt(i) - val) < 0.0001

class TestMMapDoubleArray(TestMMapFloatArray):
    setUp = setUp('d')
<commit_msg>Add conditions around Linux-specific tests<commit_after>'''
MMap Arrays Test Suite

(c) 2014 Farsight Security Inc.

Released under the MIT license.  See license.txt.
'''

import ctypes
import math
import unittest
import os
import platform
import tempfile

import mmaparray
import pkg_resources
import six

_can_populate = platform.system() == 'Linux' and pkg_resources.parse_version(platform.release()) >= pkg_resources.parse_version('2.5.46')

_can_fallocate = False
libc = ctypes.cdll.LoadLibrary(ctypes.util.find_library('c'))
if libc._name:
    try:
        gnu_get_libc_version = libc.gnu_get_libc_version
        _can_fallocate = True
    except AttributeError:
        pass

def setUp(typecode, min_val=0, max_val=0, size=1024):
    def fn(self):
        self.backing = tempfile.NamedTemporaryFile(prefix='mmapfile_test')
        self.backing.file.close()

        self.min_val = min_val
        self.max_val = max_val
        self.span = 1 + max_val - min_val
        self.size = size

        self.array = mmaparray.array(self.backing.name, typecode, self.size)
    return fn

class TestMMapArrayGeneric:
    def tearDown(self):
        self.array.close()
        self.backing.close()

    def test_reopen(self):
        array2 = type(self.array)(self.array.name)
        self.assertEqual(len(array2), len(self.array))

    def test_out_of_range(self):
        if self.max_val:
            with self.assertRaises(OverflowError):
                self.array[0] = self.min_val-1
                self.array[0] = self.max_val+1

    def test_out_of_bounds(self):
        with self.assertRaises(OverflowError):
            self.array[-1]
            self.array[-1] = 0

        with self.assertRaises(IndexError):
            self.array[len(self.array)]
            self.array[len(self.array)] = 0

    def test_setgetitem(self):
        self.assertEqual(self.array[0], 0)
        self.array[0] = 1
        self.assertEqual(self.array[0], 1)

        for i in range(1,8):
            self.assertEqual(self.array[i], 0)

        for i in range(1,8):
            self.array[i] = 1
        self.array[0] = 0

        self.assertEqual(self.array[0], 0)
        for i in range(1,8):
            self.assertEqual(self.array[i], 1)

    def test_len(self):
        self.assertEqual(len(self.array), self.size)

    def test_name(self):
        self.assertEqual(self.array.name, six.b(self.backing.name))

    def test_iter(self):
        for i in range(0, self.size):
            self.array[i] = (i+1) % self.span + self.min_val
        for i, val in enumerate(self.array):
            self.assertEqual((i+1) % self.span + self.min_val, val)

    @unittest.skipUnless(_can_populate, 'MAP_POPULATE not supported on this platform')
    def test_populate(self):
        type(self.array)(self.array.name, want_populate=True)

    @unittest.skipUnless(_can_fallocate, 'fallocate not supported on this platform')
    def test_fallocate(self):
        arrayfile = tempfile.NamedTemporaryFile(prefix='mmapfile_test')
        array2 = type(self.array)(arrayfile.name, 1000, want_fallocate=True)
        self.assertEqual(os.fstat(array2.fd).st_size, array2.bytesize)

class TestMMapBitArray(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('o')

    def test_iter(self):
        for i in range(0, self.size):
            self.array[i] = i % 2
        for i, val in enumerate(self.array):
            self.assertEqual(i % 2, val)

class TestMMapInt8Array(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('b', -2**7, 2**7-1)

class TestMMapUint8Array(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('B', 0, 2**8-1)

class TestMMapInt16Array(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('h', -2**15, 2**15-1, size=2**18)

class TestMMapUint16Array(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('H', 0, 2**16-1, size=2**18)

class TestMMapInt32Array(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('i', -2**31, 2**31-1)

class TestMMapUint32Array(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('I', 0, 2**32-1)

class TestMMapInt64Array(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('l', -2**63, 2**63-1)

class TestMMapUint64Array(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('L', 0, 2**64-1)

class TestMMapFloatArray(unittest.TestCase, TestMMapArrayGeneric):
    setUp = setUp('f')

    def test_iter(self):
        for i in range(0, self.size):
            self.array[i] = math.sqrt(i)
        for i, val in enumerate(self.array):
            assert abs(math.sqrt(i) - val) < 0.0001

class TestMMapDoubleArray(TestMMapFloatArray):
    setUp = setUp('d')
<|endoftext|>


<commit_before>#! /usr/bin/env python3
#
#	load a cube with the given command line arguments
#
#  Usage:
#       SkyStats.py  file1.fits [file2.fits ...]
#
#  Output:  an ASCII table with the following columns
#       local_time median_sky exp_time moon_phase file_name

import sys
import numpy as np
from astropy.io import fits

try:
    from astropy.time import Time
    from astroplan.moon import moon_illumination
    Qmoon = True
except:
    Qmoon = False

def my_moon(ffile, Qmiddle, box, m):
    hdu = fits.open(ffile)
    h = hdu[0].header
    d = hdu[0].data
    nx = d.shape[1]
    ny = d.shape[0]
    dc = d[ny//2-box:ny//2+box, nx//2-box:nx//2+box]
    moon = -10.0
    if 'EXPTIME' in h:
        exp = float(h['EXPTIME'])
    else:
        exp = -1.0
    if 'TIME-OBS' in h:
        #
        iso_date = h['TIME-OBS']
        hms = iso_date.split(':')
        moon = -1.0
    elif 'DATE-LOC' in h:
        # '2020-02-29T20:13:34.920'
        iso_date = h['DATE-LOC']
        hms = iso_date.split('T')[1].split(':')
        if len(hms) == 1:
            # '2017-11-19T22-43-30.506'
            # there was a time we did it wrong....
            hms = h['DATE-LOC'].split('T')[1].split('-')
            moon = -3.0
        else:
            # Qmoon is true if astroplan succesfully opened
            if Qmoon:
                # Qmiddle is true if currently on an image you want to calc
                # the moon phase for
                if Qmiddle:
                    # example iso_date (in UT):   2021-09-12T19:58:21.025
                    # print("HMS: ",iso_date)
                    t = Time(iso_date)
                    moon = moon_illumination(t)
                # else, use m value passed in, which should be the middle
                # argument's moon phase
                else:
                    moon = m
    else:
        hms = -999.999
        moon = -2.0
    t = float(hms[0]) + float(hms[1])/60  + float(hms[2])/3600
    return (t,np.median(dc),exp,moon,ffile)
    #print("%.4f %g %g %g %s" % (t,np.median(dc),exp,moon,ffile))

if __name__ == '__main__':

    box = 200

    # Sets Qmiddle to true so it only calculates the moon once, that spot
    # being the middle of the arguments sent in to SkyStats.py
    middle_moon = my_moon(sys.argv[(len(sys.argv)+2)//2], True, box, -4.0)[3]

    # for each file name sent in, calculate all stats except moon phase
    for ffile in sys.argv[1:]:
        moon = my_moon(ffile, False, box, middle_moon)
        print("%.4f %g %g %g %s" % (moon[0],moon[1],moon[2],moon[3],moon[4]))

<commit_msg>fix when only one file given<commit_after>#! /usr/bin/env python3
#
#	load a cube with the given command line arguments
#
#  Usage:
#       SkyStats.py  file1.fits [file2.fits ...]
#
#  Output:  an ASCII table with the following columns
#       local_time median_sky exp_time moon_phase file_name

import sys
import numpy as np
from astropy.io import fits

try:
    from astropy.time import Time
    from astroplan.moon import moon_illumination
    Qmoon = True
except:
    Qmoon = False

def my_moon(ffile, Qmiddle, box, m):
    hdu = fits.open(ffile)
    h = hdu[0].header
    d = hdu[0].data
    nx = d.shape[1]
    ny = d.shape[0]
    dc = d[ny//2-box:ny//2+box, nx//2-box:nx//2+box]
    moon = -10.0
    if 'EXPTIME' in h:
        exp = float(h['EXPTIME'])
    else:
        exp = -1.0
    if 'TIME-OBS' in h:
        #
        iso_date = h['TIME-OBS']
        hms = iso_date.split(':')
        moon = -1.0
    elif 'DATE-LOC' in h:
        # '2020-02-29T20:13:34.920'
        iso_date = h['DATE-LOC']
        hms = iso_date.split('T')[1].split(':')
        if len(hms) == 1:
            # '2017-11-19T22-43-30.506'
            # there was a time we did it wrong....
            hms = h['DATE-LOC'].split('T')[1].split('-')
            moon = -3.0
        else:
            # Qmoon is true if astroplan succesfully opened
            if Qmoon:
                # Qmiddle is true if currently on an image you want to calc
                # the moon phase for
                if Qmiddle:
                    # example iso_date (in UT):   2021-09-12T19:58:21.025
                    # print("HMS: ",iso_date)
                    t = Time(iso_date)
                    moon = moon_illumination(t)
                # else, use m value passed in, which should be the middle
                # argument's moon phase
                else:
                    moon = m
    else:
        hms = -999.999
        moon = -2.0
    t = float(hms[0]) + float(hms[1])/60  + float(hms[2])/3600
    return (t,np.median(dc),exp,moon,ffile)
    #print("%.4f %g %g %g %s" % (t,np.median(dc),exp,moon,ffile))

if __name__ == '__main__':

    box = 200

    # Sets Qmiddle to true so it only calculates the moon once, that spot
    # being the middle of the arguments sent in to SkyStats.py
    if len(sys.argv) == 2:
        middle_moon = my_moon(sys.argv[1], True, box, -4.0)[3]
    else:
        middle_moon = my_moon(sys.argv[(len(sys.argv)+2)//2], True, box, -4.0)[3]

    # for each file name sent in, calculate all stats except moon phase
    for ffile in sys.argv[1:]:
        moon = my_moon(ffile, False, box, middle_moon)
        print("%.4f %g %g %g %s" % (moon[0],moon[1],moon[2],moon[3],moon[4]))

<|endoftext|>


<commit_before>import datetime
from decimal import Decimal

BIRTHDAY = datetime.date(2012, 6, 5)
def age():
    age = datetime.date.today() - BIRTHDAY
    return "%d days" % age.days

db = None # This global is wired in wireup. It's an instance of
          # gittip.postgres.PostgresManager.
AMOUNTS= [Decimal(a) for a in ('0.00', '0.08', '0.16', '0.32', '0.64', '1.28')]

__version__ = "3.0.6"

def get_tip(tipper, tippee):
    """Given two user ids, return a Decimal.
    """
    TIP = """\

        SELECT amount
          FROM tips
         WHERE tipper=%s
           AND tippee=%s
      ORDER BY mtime DESC
         LIMIT 1

    """
    rec = db.fetchone(TIP, (tipper, tippee))
    if rec is None:
        tip = Decimal(0.00)
    else:
        tip = rec['amount']
    return tip

def get_backed_amount(participant_id):
    """Given a unicode, return a Decimal.
    """

    BACKED = """\

        SELECT sum(amount) AS backed
          FROM ( SELECT DISTINCT ON (tipper)
                        amount
                      , tipper
                   FROM tips
                   JOIN participants p ON p.id = tipper
                  WHERE tippee=%s
                    AND last_bill_result = ''
               ORDER BY tipper
                      , mtime DESC
                ) AS foo

    """
    rec = db.fetchone(BACKED, (participant_id,))
    if rec is None:
        amount = None
    else:
        amount = rec['backed']  # might be None

    if amount is None:
        amount = Decimal(0.00)

    return amount

def get_tipjar(participant_id, pronoun="their", claimed=False):
    """Given a participant id, return a unicode.
    """

    amount = get_backed_amount(participant_id)

    # Compute a unicode describing the amount.
    # ========================================
    # This is down here because we use it in multiple places in the app,
    # basically in the claimed participant page and the non-claimed GitHub page
    # (hopefully soon to be Facebook and Twitter as well).

    if pronoun not in ('your', 'their'):
        raise Exception("Unknown, um, pronoun: %s" % pronoun)

    if amount == 0:
        if pronoun == "your":
            tipjar = u"have no backed tips."
        elif pronoun == "their":
            tipjar = u"has no backed tips."
    else:
        if pronoun == "your":
            tipjar = u"have $%s in backed tips."
        elif pronoun == "their":
            tipjar = u"has $%s in backed tips."
        tipjar %= amount

    # We're opt-in.
    # =============
    # If the user hasn't claimed the tipjar then the tips are only pledges,
    # we're not going to actually collect money on their behalf.

    if not claimed:
        tipjar = tipjar.replace("backed", "pledged")

    return tipjar

def get_tips_and_total(tipper, for_payday=False):
    """Given a participant id and a date, return a list and a Decimal.

    This function is used to populate a participant's page for their own
    viewing pleasure, and also by the payday function. If for_payday is not
    False it must be a date object.

    """
    if for_payday:

        # For payday we want the oldest relationship to be paid first.
        order_by = "ctime ASC"

        # This is where it gets crash-proof.
        # ==================================
        # We need to account for the fact that we may have crashed during
        # Payday and we're re-running that function. We only want to select
        # tips that existed before Payday started, but haven't been processed
        # as part of this Payday yet.
        #
        # It's a bug if the paydays subselect returns > 1 rows.
        #
        # XXX If we crash during Payday and we rerun it after a timezone
        # change, will we get burned? How?

        ts_filter = """\

               AND mtime < %s
               AND ( SELECT id
                       FROM transfers
                      WHERE tipper=t.tipper
                        AND tippee=t.tippee
                        AND timestamp >= %s
                    ) IS NULL

        """
        args = (tipper, for_payday, for_payday)
    else:
        order_by = "amount DESC"
        ts_filter = ""
        args = (tipper,)

    TIPS = """\

        SELECT * FROM (
            SELECT DISTINCT ON (tippee)
                   amount
                 , tippee
                 , t.ctime
                 , p.claimed_time
              FROM tips t
              JOIN participants p ON p.id = t.tippee
             WHERE tipper = %%s
               %s
          ORDER BY tippee
                 , t.mtime DESC
        ) AS foo
        ORDER BY %s
               , tippee

    """ % (ts_filter, order_by)  # XXX, No injections here, right?!
    tips = list(db.fetchall(TIPS, args))

    # Compute the total.
    # ==================
    # For payday we only want to process payments to tippees who have
    # themselves opted into Gittip. For the tipper's profile page we want to
    # show the total amount they've pledged (so they're not surprised when
    # someone *does* start accepting tips and all of a sudden they're hit with
    # bigger charges.

    if for_payday:
        to_total = [t for t in tips if t['claimed_time'] is not None]
    else:
        to_total = tips
    total = sum([t['amount'] for t in to_total])

    if not total:  # XXX Why is this necessary?
        total = Decimal('0.00')

    return tips, total

# canonizer
# =========
# This is an Aspen hook to ensure that requests are served on a certain root
# URL, even if multiple domains point to the application.

class X: pass
canonical_scheme = None
canonical_host = None

def canonize(request):
    """Enforce a certain scheme and hostname. Store these on request as well.
    """
    scheme = request.headers.get('X-Forwarded-Proto', 'http') # per Heroku
    host = request.headers['Host']
    bad_scheme = scheme != canonical_scheme
    bad_host = bool(canonical_host) and (host != canonical_host)
                # '' and False => ''
    if bad_scheme or bad_host:
        url = '%s://%s/' % (canonical_scheme, canonical_host)
        request.redirect(url, permanent=True)
<commit_msg>Resetting version in gittip/__init__.py.<commit_after>import datetime
from decimal import Decimal

BIRTHDAY = datetime.date(2012, 6, 5)
def age():
    age = datetime.date.today() - BIRTHDAY
    return "%d days" % age.days

db = None # This global is wired in wireup. It's an instance of
          # gittip.postgres.PostgresManager.
AMOUNTS= [Decimal(a) for a in ('0.00', '0.08', '0.16', '0.32', '0.64', '1.28')]

__version__ = "~~VERSION~~"

def get_tip(tipper, tippee):
    """Given two user ids, return a Decimal.
    """
    TIP = """\

        SELECT amount
          FROM tips
         WHERE tipper=%s
           AND tippee=%s
      ORDER BY mtime DESC
         LIMIT 1

    """
    rec = db.fetchone(TIP, (tipper, tippee))
    if rec is None:
        tip = Decimal(0.00)
    else:
        tip = rec['amount']
    return tip

def get_backed_amount(participant_id):
    """Given a unicode, return a Decimal.
    """

    BACKED = """\

        SELECT sum(amount) AS backed
          FROM ( SELECT DISTINCT ON (tipper)
                        amount
                      , tipper
                   FROM tips
                   JOIN participants p ON p.id = tipper
                  WHERE tippee=%s
                    AND last_bill_result = ''
               ORDER BY tipper
                      , mtime DESC
                ) AS foo

    """
    rec = db.fetchone(BACKED, (participant_id,))
    if rec is None:
        amount = None
    else:
        amount = rec['backed']  # might be None

    if amount is None:
        amount = Decimal(0.00)

    return amount

def get_tipjar(participant_id, pronoun="their", claimed=False):
    """Given a participant id, return a unicode.
    """

    amount = get_backed_amount(participant_id)

    # Compute a unicode describing the amount.
    # ========================================
    # This is down here because we use it in multiple places in the app,
    # basically in the claimed participant page and the non-claimed GitHub page
    # (hopefully soon to be Facebook and Twitter as well).

    if pronoun not in ('your', 'their'):
        raise Exception("Unknown, um, pronoun: %s" % pronoun)

    if amount == 0:
        if pronoun == "your":
            tipjar = u"have no backed tips."
        elif pronoun == "their":
            tipjar = u"has no backed tips."
    else:
        if pronoun == "your":
            tipjar = u"have $%s in backed tips."
        elif pronoun == "their":
            tipjar = u"has $%s in backed tips."
        tipjar %= amount

    # We're opt-in.
    # =============
    # If the user hasn't claimed the tipjar then the tips are only pledges,
    # we're not going to actually collect money on their behalf.

    if not claimed:
        tipjar = tipjar.replace("backed", "pledged")

    return tipjar

def get_tips_and_total(tipper, for_payday=False):
    """Given a participant id and a date, return a list and a Decimal.

    This function is used to populate a participant's page for their own
    viewing pleasure, and also by the payday function. If for_payday is not
    False it must be a date object.

    """
    if for_payday:

        # For payday we want the oldest relationship to be paid first.
        order_by = "ctime ASC"

        # This is where it gets crash-proof.
        # ==================================
        # We need to account for the fact that we may have crashed during
        # Payday and we're re-running that function. We only want to select
        # tips that existed before Payday started, but haven't been processed
        # as part of this Payday yet.
        #
        # It's a bug if the paydays subselect returns > 1 rows.
        #
        # XXX If we crash during Payday and we rerun it after a timezone
        # change, will we get burned? How?

        ts_filter = """\

               AND mtime < %s
               AND ( SELECT id
                       FROM transfers
                      WHERE tipper=t.tipper
                        AND tippee=t.tippee
                        AND timestamp >= %s
                    ) IS NULL

        """
        args = (tipper, for_payday, for_payday)
    else:
        order_by = "amount DESC"
        ts_filter = ""
        args = (tipper,)

    TIPS = """\

        SELECT * FROM (
            SELECT DISTINCT ON (tippee)
                   amount
                 , tippee
                 , t.ctime
                 , p.claimed_time
              FROM tips t
              JOIN participants p ON p.id = t.tippee
             WHERE tipper = %%s
               %s
          ORDER BY tippee
                 , t.mtime DESC
        ) AS foo
        ORDER BY %s
               , tippee

    """ % (ts_filter, order_by)  # XXX, No injections here, right?!
    tips = list(db.fetchall(TIPS, args))

    # Compute the total.
    # ==================
    # For payday we only want to process payments to tippees who have
    # themselves opted into Gittip. For the tipper's profile page we want to
    # show the total amount they've pledged (so they're not surprised when
    # someone *does* start accepting tips and all of a sudden they're hit with
    # bigger charges.

    if for_payday:
        to_total = [t for t in tips if t['claimed_time'] is not None]
    else:
        to_total = tips
    total = sum([t['amount'] for t in to_total])

    if not total:  # XXX Why is this necessary?
        total = Decimal('0.00')

    return tips, total

# canonizer
# =========
# This is an Aspen hook to ensure that requests are served on a certain root
# URL, even if multiple domains point to the application.

class X: pass
canonical_scheme = None
canonical_host = None

def canonize(request):
    """Enforce a certain scheme and hostname. Store these on request as well.
    """
    scheme = request.headers.get('X-Forwarded-Proto', 'http') # per Heroku
    host = request.headers['Host']
    bad_scheme = scheme != canonical_scheme
    bad_host = bool(canonical_host) and (host != canonical_host)
                # '' and False => ''
    if bad_scheme or bad_host:
        url = '%s://%s/' % (canonical_scheme, canonical_host)
        request.redirect(url, permanent=True)
<|endoftext|>


<commit_before>from collections import namedtuple, OrderedDict
import re
from datetime import datetime
from wq.io.exceptions import NoData

class BaseMapper(object):
    def get_key_field(self):
        return self.map_field(self.key_field)

    def map_field(self, field):
        return field

    def map_value(self, field, value):
        return value

    def unmap_field(self, field):
        return field

    def unmap_value(self, field, value):
        return value

    def usable_item(self, item):
        uitem = {}
        for key, val in item.iteritems():
            field = self.map_field(key)
            value = self.map_value(field, val)
            uitem[field] = value
        return uitem

    def parse_usable_item(self, uitem):
        item = {}
        for field, value in uitem.iteritems():
            key = self.unmap_field(field)
            val = self.unmap_value(field, value)
            item[key] = val
        return item

class DictMapper(BaseMapper):
    field_map = {}
    value_map = {}

    def map_field(self, field):
        field = self.field_map[field] if field in self.field_map else field
        return field

    def map_value(self, field, value):
        if not isinstance(value, basestring):
            return value
        value = self.value_map[value] if value in self.value_map else value
        return value

    def unmap_field(self, field):
        for f in self.field_map:
            if self.field_map[f] == field:
                return f
        return field

    def unmap_value(self, field, value):
        if not isinstance(value, basestring):
            return value
        for v in self.value_map:
            if self.value_map[v] == value:
                return v
        return value

class TupleMapper(DictMapper):
    @property
    def field_map(self):
        if not getattr(self, 'data', None):
            raise NoData

        #FIXME: check for duplicates
        if not hasattr(self, '_field_map'):
            field_names = self.get_field_names()
            items = [
                (field, self.tuple_field_name(field))
                for field in field_names
            ]
            self._field_map = OrderedDict(items)
        return self._field_map

    def tuple_field_name(self, field):
        field = self.clean_field_name(field)
        return re.sub(r'\W', '', field.lower())

    def clean_field_name(self, field):
        return field

    @property
    def tuple_class(self):
        "Returns a class to use for individual items"

        if not hasattr(self, '_item_class'):
            cls = namedtuple(
                self.__class__.__name__ + 'Tuple',
                self.field_map.values()
            )
            self._tuple_class = cls

        return self._tuple_class

    @property
    def tuple_prototype(self):
        if not hasattr(self, '_tuple_prototype'):
            vals = {field: None for field in self.field_map.values()}
            self._tuple_prototype = self.tuple_class(**vals)
        return self._tuple_prototype

    def usable_item(self, item):
        mapped = super(TupleMapper, self).usable_item(item)
        return self.tuple_prototype._replace(**mapped)

    def parse_usable_item(self, uitem):
        mapped = {key: getattr(uitem, key) for key in self.field_map.values()}
        return super(TupleMapper, self).parse_usable_item(mapped)

    def create(self, **kwargs):
        return self.tuple_prototype._replace(**kwargs)

def make_date_mapper(fmt):
    """
    Generate functions to use for mapping strings to dates
    """
    def mapper(val):
        val = datetime.strptime(val, fmt)
        if '%Y' in fmt or '%y' in fmt:
            return val
        else:
            return val.time()
    return mapper

class TimeSeriesMapper(TupleMapper):
    date_formats = None
    map_floats = True

    def map_value(self, field, value):
        if not isinstance(value, basestring):
            return value

        functions = [make_date_mapper(fmt) for fmt in self.date_formats]
        if self.map_floats:
            functions.insert(0, float)

        value = value.strip()
        for i, fn in enumerate(functions):
            try:
                return fn(value)
            except ValueError:
                pass
        return value

    @property
    def key_fields(self):
        raise NotImplementedError("Key fields must be specified")

    def parameter_fields(self):
        return sorted(set(self.field_map.values()) - set(self.key_fields))
<commit_msg>don't need to pickle tuple prototype<commit_after>from collections import namedtuple, OrderedDict
import re
from datetime import datetime
from wq.io.exceptions import NoData

class BaseMapper(object):
    def get_key_field(self):
        return self.map_field(self.key_field)

    def map_field(self, field):
        return field

    def map_value(self, field, value):
        return value

    def unmap_field(self, field):
        return field

    def unmap_value(self, field, value):
        return value

    def usable_item(self, item):
        uitem = {}
        for key, val in item.iteritems():
            field = self.map_field(key)
            value = self.map_value(field, val)
            uitem[field] = value
        return uitem

    def parse_usable_item(self, uitem):
        item = {}
        for field, value in uitem.iteritems():
            key = self.unmap_field(field)
            val = self.unmap_value(field, value)
            item[key] = val
        return item

class DictMapper(BaseMapper):
    field_map = {}
    value_map = {}

    def map_field(self, field):
        field = self.field_map[field] if field in self.field_map else field
        return field

    def map_value(self, field, value):
        if not isinstance(value, basestring):
            return value
        value = self.value_map[value] if value in self.value_map else value
        return value

    def unmap_field(self, field):
        for f in self.field_map:
            if self.field_map[f] == field:
                return f
        return field

    def unmap_value(self, field, value):
        if not isinstance(value, basestring):
            return value
        for v in self.value_map:
            if self.value_map[v] == value:
                return v
        return value

class TupleMapper(DictMapper):
    @property
    def field_map(self):
        if not getattr(self, 'data', None):
            raise NoData

        #FIXME: check for duplicates
        if not hasattr(self, '_field_map'):
            field_names = self.get_field_names()
            items = [
                (field, self.tuple_field_name(field))
                for field in field_names
            ]
            self._field_map = OrderedDict(items)
        return self._field_map

    def tuple_field_name(self, field):
        field = self.clean_field_name(field)
        return re.sub(r'\W', '', field.lower())

    def clean_field_name(self, field):
        return field

    @property
    def tuple_class(self):
        "Returns a class to use for individual items"

        if not hasattr(self, '_tuple_class'):
            cls = namedtuple(
                self.__class__.__name__ + 'Tuple',
                self.field_map.values()
            )
            self._tuple_class = cls

        return self._tuple_class

    @property
    def tuple_prototype(self):
        if not hasattr(self, '_tuple_prototype'):
            vals = {field: None for field in self.field_map.values()}
            self._tuple_prototype = self.tuple_class(**vals)
        return self._tuple_prototype

    def usable_item(self, item):
        mapped = super(TupleMapper, self).usable_item(item)
        return self.tuple_prototype._replace(**mapped)

    def parse_usable_item(self, uitem):
        mapped = {key: getattr(uitem, key) for key in self.field_map.values()}
        return super(TupleMapper, self).parse_usable_item(mapped)

    def create(self, **kwargs):
        return self.tuple_prototype._replace(**kwargs)

    def __getstate__(self):
        """
        Don't include auto-created properties in pickle state.
        (especially since the class doesn't have a real path)
        """
        state = self.__dict__.copy()
        state.pop('_tuple_class', None)
        state.pop('_tuple_prototype', None)
        return state

def make_date_mapper(fmt):
    """
    Generate functions to use for mapping strings to dates
    """
    def mapper(val):
        val = datetime.strptime(val, fmt)
        if '%Y' in fmt or '%y' in fmt:
            return val
        else:
            return val.time()
    return mapper

class TimeSeriesMapper(TupleMapper):
    date_formats = None
    map_floats = True

    def map_value(self, field, value):
        if not isinstance(value, basestring):
            return value

        functions = [make_date_mapper(fmt) for fmt in self.date_formats]
        if self.map_floats:
            functions.insert(0, float)

        value = value.strip()
        for i, fn in enumerate(functions):
            try:
                return fn(value)
            except ValueError:
                pass
        return value

    @property
    def key_fields(self):
        raise NotImplementedError("Key fields must be specified")

    def parameter_fields(self):
        return sorted(set(self.field_map.values()) - set(self.key_fields))
<|endoftext|>