<commit_before>#!/usr/bin/env python
#
# $Id$
#

"""psutil is a module providing convenience functions for managing
processes in a portable way by using Python.
"""

__all__ = [
    "NoSuchProcess",
    "AccessDenied",
    "NUM_CPUS",
    "ProcessInfo",
    "Process",
    "test",
    "CPUTimes",
    "pid_exists",
    "get_pid_list",
    "process_iter",
    "get_process_list",
    "TOTAL_PHYMEM",
    "avail_phymem",
    "used_phymem",
    "total_virtmem",
    "avail_virtmem",
    "used_virtmem",
    "cpu_times",
    "cpu_percent",
    ]

import sys
import os
import time
try:
    import pwd, grp
except ImportError:
    pwd = grp = None

# exceptions are imported here, but may be overriden by platform
# module implementation later
from error import *

# import the appropriate module for our platform only
if sys.platform.lower().startswith("linux"):
    from _pslinux import *

elif sys.platform.lower().startswith("win32"):
    from _psmswindows import *

elif sys.platform.lower().startswith("darwin"):
    from _psosx import *

elif sys.platform.lower().startswith("freebsd"):
    from _psbsd import *
else:
    raise ImportError('no os specific module found')

# platform-specific modules define an Impl implementation class
_platform_impl = Impl()

class CPUTimes:
    """This class contains information about CPU times.
    It is not used directly but it's returned as an instance by
    psutil.cpu_times() function.

    Every CPU time is accessible in form of an attribute and represents
    the time CPU has spent in the given mode.

    The attributes availability varies depending on the platform.
    Here follows a list of all available attributes:

     - user
     - system
     - idle
     - nice (UNIX)
     - iowait (Linux)
     - irq (Linux)
     - softirq (Linux)
     - interrupt (FreeBSD)
    """

    def __init__(self, **kwargs):
        self.attrs = []
        for name in kwargs:
            setattr(self, name, kwargs[name])
            self.attrs.append(name)

    def __str__(self):
        string = []
        for attr in self.attrs:
            value = getattr(self, attr)
            string.append("%s=%s" %(attr, value))
        return '; '.join(string)

    def __iter__(self):
        for attr in self.attrs:
            yield getattr(self, attr)

class ProcessInfo(object):
    """Class that allows the process information to be passed between
    external code and psutil.  Used directly by the Process class.
    """

    def __init__(self, pid, ppid=None, name=None, path=None, cmdline=None,
                       uid=None, gid=None):
        self.pid = pid
        self.ppid = ppid
        self.name = name
        self.path = path
        self.cmdline = cmdline
        # if we have the cmdline but not the path, figure it out from argv[0]
        if cmdline and not path:
            self.path = os.path.dirname(cmdline[0])
        self.uid = uid
        self.gid = gid
        self.create = None

class Process(object):
    """Represents an OS process."""

    def __init__(self, pid):
        """Create a new Process object, raises NoSuchProcess if the PID
        does not exist, and ValueError if the parameter is not an
        integer PID."""
        if not isinstance(pid, int):
            raise ValueError("An integer is required")
        if not pid_exists(pid):
            raise NoSuchProcess("No process found with PID %s" % pid)
        self._procinfo = ProcessInfo(pid)
        self.is_proxy = True
        # try to init CPU times, if it raises AccessDenied then suppress
        # it so it won't interrupt the constructor. First call to
        # get_cpu_percent() will trigger the AccessDenied exception
        # instead.
        try:
            self._last_sys_time = time.time()
            self._last_user_time, self._last_kern_time = self.get_cpu_times()
        except AccessDenied:
            self._last_user_time, self._last_kern_time = None, None

    def __str__(self):
        return "psutil.Process <PID:%s; PPID:%s; NAME:'%s'; PATH:'%s'; " \
               "CMDLINE:%s; UID:%s; GID:%s;>" \
               %(self.pid, self.ppid, self.name, self.path, self.cmdline, \
                 self.uid, self.gid)

    def __eq__(self, other):
        """Test for equality with another Process object based on PID,
        name etc."""
        if self.pid != other.pid:
            return False

        # make sure both objects are initialized
        self.deproxy()
        other.deproxy()
        # check all non-callable and non-private attributes for equality
        # if the attribute is missing from other then return False
        for attr in dir(self):
            attrobj = getattr(self, attr)
            # skip private attributes
            if attr.startswith("_"):
                continue

            # skip methods and Process objects
            if callable(attrobj) or isinstance(attrobj, Process):
                continue

            # attribute doesn't exist or isn't equal, so return False
            if not hasattr(other, attr):
                return False
            if getattr(self, attr) != getattr(other, attr):
                return False

        return True

    def deproxy(self):
        """Used internally by Process properties. The first call to
        deproxy() initializes the ProcessInfo object in self._procinfo
        with process data read from platform-specific module's
        get_process_info() method.

        This method becomes a NO-OP after the first property is accessed.
        Property data is filled in from the ProcessInfo object created,
        and further calls to deproxy() simply return immediately without
        calling get_process_info().
        """
        if self.is_proxy:
            # get_process_info returns a tuple we use as the arguments
            # to the ProcessInfo constructor
            self._procinfo = ProcessInfo(*_platform_impl.get_process_info(self._procinfo.pid))
            self.is_proxy = False

    @property
    def pid(self):
        """The process pid."""
        return self._procinfo.pid

    @property
    def ppid(self):
        """The process parent pid."""
        self.deproxy()
        return self._procinfo.ppid

    @property
    def parent(self):
        """Return the parent process as a Process object. If no ppid is
        known then return None."""
        if self.ppid is not None:
            return Process(self.ppid)
        return None

    @property
    def name(self):
        """The process name."""
        self.deproxy()
        return self._procinfo.name

    @property
    def path(self):
        """The process path."""
        self.deproxy()
        return self._procinfo.path

    @property
    def cmdline(self):
        """The command line process has been called with."""
        self.deproxy()
        return self._procinfo.cmdline

    @property
    def uid(self):
        """The real user id of the current process."""
        self.deproxy()
        return self._procinfo.uid

    @property
    def gid(self):
        """The real group id of the current process."""
        self.deproxy()
        return self._procinfo.gid

    @property
    def create_time(self):
        """The process creation time as a floating point number
        expressed in seconds since the epoch, in UTC.
        """
        if self._procinfo.create is None:
            self._procinfo.create = _platform_impl.get_process_create_time(self.pid)
        return self._procinfo.create

    def get_cpu_percent(self):
        """Compare process times to system time elapsed since last call
        and calculate CPU utilization as a percentage. It is recommended
        for accuracy that this function be called with at least 1 second
        between calls. The initial delta is calculated from the
        instantiation of the Process object.
        """
        now = time.time()
        # will raise AccessDenied on OS X if not root or in procmod group
        user_t, kern_t = _platform_impl.get_cpu_times(self.pid)

        total_proc_time = float((user_t - self._last_user_time) + \
                                (kern_t - self._last_kern_time))
        try:
            percent = total_proc_time / float((now - self._last_sys_time))
        except ZeroDivisionError:
            percent = 0.000

        # reset the values
        self._last_sys_time = time.time()
        self._last_user_time, self._last_kern_time = self.get_cpu_times()

        return (percent * 100.0) / NUM_CPUS

    def get_cpu_times(self):
        """Return a tuple whose values are process CPU user and system
        time.  These are the same first two values that os.times()
        returns for the current process.
        """
        return _platform_impl.get_cpu_times(self.pid)

    def get_memory_info(self):
        """Return a tuple representing RSS (Resident Set Size) and VMS
        (Virtual Memory Size) in bytes.

        On UNIX RSS and VMS are the same values shown by ps.

        On Windows RSS and VMS refer to "Mem Usage" and "VM Size" columns
        of taskmgr.exe.
        """
        return _platform_impl.get_memory_info(self.pid)

    def get_memory_percent(self):
        """Compare physical system memory to process resident memory and
        calculate process memory utilization as a percentage.
        """
        rss = _platform_impl.get_memory_info(self.pid)[0]
        try:
            return (rss / float(TOTAL_PHYMEM)) * 100
        except ZeroDivisionError:
            return 0.0

    def is_running(self):
        """Return whether the current process is running in the current process
        list."""
        try:
            new_proc = Process(self.pid)
            # calls get_process_info() which may in turn trigger NSP exception
            str(new_proc)
        except NoSuchProcess:
            return False
        return self == new_proc

    def kill(self, sig=None):
        """Kill the current process by using signal sig (defaults to SIGKILL).
        """
        _platform_impl.kill_process(self.pid, sig)

def pid_exists(pid):
    """Check whether the given PID exists in the current process list."""
    return _platform_impl.pid_exists(pid)

def get_pid_list():
    """Return a list of current running PIDs."""
    return _platform_impl.get_pid_list()

def process_iter():
    """Return an iterator yielding a Process class instances for all
    running processes on the local machine.
    """
    pids = _platform_impl.get_pid_list()
    # for each PID, create a proxyied Process object
    # it will lazy init it's name and path later if required
    for pid in pids:
        try:
            yield Process(pid)
        except (NoSuchProcess, AccessDenied):
            continue

def get_process_list():
    """Return a list of Process class instances for all running
    processes on the local machine.
    """
    return list(process_iter())

def test():
    """List info of all currently running processes emulating a
    ps -aux output.
    """
    import datetime
    today_day = datetime.date.today()

    def get_process_info(pid):
        proc = Process(pid)
        uid = proc.uid
        pid = proc.pid
        cpu = round(proc.get_cpu_percent(), 1)
        mem = round(proc.get_memory_percent(), 1)
        rss, vsz = [x / 1024 for x in proc.get_memory_info()]

        # If process has been created today print H:M, else MonthDay
        start = datetime.datetime.fromtimestamp(proc.create_time)
        if start.date() == today_day:
            start = start.strftime("%H:%M")
        else:
            start = start.strftime("%b%d")

        cputime = time.strftime("%M:%S", time.localtime(sum(proc.get_cpu_times())))
        cmd = ' '.join(proc.cmdline)
        # where cmdline is not available UNIX shows process name between
        # [] parentheses
        if not cmd:
            cmd = "[%s]" %proc.name
        return "%-5s %7s %4s %4s %7s %7s %5s %8s %s" \
               %(uid, pid, cpu, mem, vsz, rss, start, cputime, cmd)

    print "%-5s %7s %4s %4s %7s %7s %5s %8s %s" \
       %("UID", "PID", "%CPU", "%MEM", "VSZ", "RSS", "START", "TIME", "COMMAND")
    pids = get_pid_list()
    pids.sort()
    for pid in pids:
        try:
            line = get_process_info(pid)
        except (AccessDenied, NoSuchProcess):
            pass
        else:
            print line

def cpu_times():
    """Return CPU times as a CPUTimes object."""
    values = get_system_cpu_times()
    return CPUTimes(**values)

_last_cpu_idle = cpu_times().idle
_last_system_time = time.time()

def cpu_percent():
    """Return CPU utilization times (user, kernel, idle) as a
    percentage calculated from the last call to the function.
    """
    global _last_cpu_idle
    global _last_system_time

    idle_time = cpu_times().idle
    system_time = time.time()

    total_time_delta = system_time - _last_system_time
    cpu_times_delta = idle_time - _last_cpu_idle

    _last_cpu_idle = idle_time
    _last_system_time = system_time

    try :
        idle_percent = (cpu_times_delta / total_time_delta) * 100.0
        percent = (100 * NUM_CPUS) - idle_percent
    except ZeroDivisionError:
        return 0.00

    return percent / NUM_CPUS

if __name__ == "__main__":
    test()
<commit_msg>updated cpu_percent() doc string and moved functions for more logical placement.<commit_after>#!/usr/bin/env python
#
# $Id$
#

"""psutil is a module providing convenience functions for managing
processes in a portable way by using Python.
"""

__all__ = [
    "NoSuchProcess",
    "AccessDenied",
    "NUM_CPUS",
    "ProcessInfo",
    "Process",
    "test",
    "CPUTimes",
    "pid_exists",
    "get_pid_list",
    "process_iter",
    "get_process_list",
    "TOTAL_PHYMEM",
    "avail_phymem",
    "used_phymem",
    "total_virtmem",
    "avail_virtmem",
    "used_virtmem",
    "cpu_times",
    "cpu_percent",
    ]

import sys
import os
import time
try:
    import pwd, grp
except ImportError:
    pwd = grp = None

# exceptions are imported here, but may be overriden by platform
# module implementation later
from error import *

# import the appropriate module for our platform only
if sys.platform.lower().startswith("linux"):
    from _pslinux import *

elif sys.platform.lower().startswith("win32"):
    from _psmswindows import *

elif sys.platform.lower().startswith("darwin"):
    from _psosx import *

elif sys.platform.lower().startswith("freebsd"):
    from _psbsd import *
else:
    raise ImportError('no os specific module found')

# platform-specific modules define an Impl implementation class
_platform_impl = Impl()

class CPUTimes:
    """This class contains information about CPU times.
    It is not used directly but it's returned as an instance by
    psutil.cpu_times() function.

    Every CPU time is accessible in form of an attribute and represents
    the time CPU has spent in the given mode.

    The attributes availability varies depending on the platform.
    Here follows a list of all available attributes:

     - user
     - system
     - idle
     - nice (UNIX)
     - iowait (Linux)
     - irq (Linux)
     - softirq (Linux)
     - interrupt (FreeBSD)
    """

    def __init__(self, **kwargs):
        self.attrs = []
        for name in kwargs:
            setattr(self, name, kwargs[name])
            self.attrs.append(name)

    def __str__(self):
        string = []
        for attr in self.attrs:
            value = getattr(self, attr)
            string.append("%s=%s" %(attr, value))
        return '; '.join(string)

    def __iter__(self):
        for attr in self.attrs:
            yield getattr(self, attr)

class ProcessInfo(object):
    """Class that allows the process information to be passed between
    external code and psutil.  Used directly by the Process class.
    """

    def __init__(self, pid, ppid=None, name=None, path=None, cmdline=None,
                       uid=None, gid=None):
        self.pid = pid
        self.ppid = ppid
        self.name = name
        self.path = path
        self.cmdline = cmdline
        # if we have the cmdline but not the path, figure it out from argv[0]
        if cmdline and not path:
            self.path = os.path.dirname(cmdline[0])
        self.uid = uid
        self.gid = gid
        self.create = None

class Process(object):
    """Represents an OS process."""

    def __init__(self, pid):
        """Create a new Process object, raises NoSuchProcess if the PID
        does not exist, and ValueError if the parameter is not an
        integer PID."""
        if not isinstance(pid, int):
            raise ValueError("An integer is required")
        if not pid_exists(pid):
            raise NoSuchProcess("No process found with PID %s" % pid)
        self._procinfo = ProcessInfo(pid)
        self.is_proxy = True
        # try to init CPU times, if it raises AccessDenied then suppress
        # it so it won't interrupt the constructor. First call to
        # get_cpu_percent() will trigger the AccessDenied exception
        # instead.
        try:
            self._last_sys_time = time.time()
            self._last_user_time, self._last_kern_time = self.get_cpu_times()
        except AccessDenied:
            self._last_user_time, self._last_kern_time = None, None

    def __str__(self):
        return "psutil.Process <PID:%s; PPID:%s; NAME:'%s'; PATH:'%s'; " \
               "CMDLINE:%s; UID:%s; GID:%s;>" \
               %(self.pid, self.ppid, self.name, self.path, self.cmdline, \
                 self.uid, self.gid)

    def __eq__(self, other):
        """Test for equality with another Process object based on PID,
        name etc."""
        if self.pid != other.pid:
            return False

        # make sure both objects are initialized
        self.deproxy()
        other.deproxy()
        # check all non-callable and non-private attributes for equality
        # if the attribute is missing from other then return False
        for attr in dir(self):
            attrobj = getattr(self, attr)
            # skip private attributes
            if attr.startswith("_"):
                continue

            # skip methods and Process objects
            if callable(attrobj) or isinstance(attrobj, Process):
                continue

            # attribute doesn't exist or isn't equal, so return False
            if not hasattr(other, attr):
                return False
            if getattr(self, attr) != getattr(other, attr):
                return False

        return True

    def deproxy(self):
        """Used internally by Process properties. The first call to
        deproxy() initializes the ProcessInfo object in self._procinfo
        with process data read from platform-specific module's
        get_process_info() method.

        This method becomes a NO-OP after the first property is accessed.
        Property data is filled in from the ProcessInfo object created,
        and further calls to deproxy() simply return immediately without
        calling get_process_info().
        """
        if self.is_proxy:
            # get_process_info returns a tuple we use as the arguments
            # to the ProcessInfo constructor
            self._procinfo = ProcessInfo(*_platform_impl.get_process_info(self._procinfo.pid))
            self.is_proxy = False

    @property
    def pid(self):
        """The process pid."""
        return self._procinfo.pid

    @property
    def ppid(self):
        """The process parent pid."""
        self.deproxy()
        return self._procinfo.ppid

    @property
    def parent(self):
        """Return the parent process as a Process object. If no ppid is
        known then return None."""
        if self.ppid is not None:
            return Process(self.ppid)
        return None

    @property
    def name(self):
        """The process name."""
        self.deproxy()
        return self._procinfo.name

    @property
    def path(self):
        """The process path."""
        self.deproxy()
        return self._procinfo.path

    @property
    def cmdline(self):
        """The command line process has been called with."""
        self.deproxy()
        return self._procinfo.cmdline

    @property
    def uid(self):
        """The real user id of the current process."""
        self.deproxy()
        return self._procinfo.uid

    @property
    def gid(self):
        """The real group id of the current process."""
        self.deproxy()
        return self._procinfo.gid

    @property
    def create_time(self):
        """The process creation time as a floating point number
        expressed in seconds since the epoch, in UTC.
        """
        if self._procinfo.create is None:
            self._procinfo.create = _platform_impl.get_process_create_time(self.pid)
        return self._procinfo.create

    def get_cpu_percent(self):
        """Compare process times to system time elapsed since last call
        and calculate CPU utilization as a percentage. It is recommended
        for accuracy that this function be called with at least 1 second
        between calls. The initial delta is calculated from the
        instantiation of the Process object.
        """
        now = time.time()
        # will raise AccessDenied on OS X if not root or in procmod group
        user_t, kern_t = _platform_impl.get_cpu_times(self.pid)

        total_proc_time = float((user_t - self._last_user_time) + \
                                (kern_t - self._last_kern_time))
        try:
            percent = total_proc_time / float((now - self._last_sys_time))
        except ZeroDivisionError:
            percent = 0.000

        # reset the values
        self._last_sys_time = time.time()
        self._last_user_time, self._last_kern_time = self.get_cpu_times()

        return (percent * 100.0) / NUM_CPUS

    def get_cpu_times(self):
        """Return a tuple whose values are process CPU user and system
        time.  These are the same first two values that os.times()
        returns for the current process.
        """
        return _platform_impl.get_cpu_times(self.pid)

    def get_memory_info(self):
        """Return a tuple representing RSS (Resident Set Size) and VMS
        (Virtual Memory Size) in bytes.

        On UNIX RSS and VMS are the same values shown by ps.

        On Windows RSS and VMS refer to "Mem Usage" and "VM Size" columns
        of taskmgr.exe.
        """
        return _platform_impl.get_memory_info(self.pid)

    def get_memory_percent(self):
        """Compare physical system memory to process resident memory and
        calculate process memory utilization as a percentage.
        """
        rss = _platform_impl.get_memory_info(self.pid)[0]
        try:
            return (rss / float(TOTAL_PHYMEM)) * 100
        except ZeroDivisionError:
            return 0.0

    def is_running(self):
        """Return whether the current process is running in the current process
        list."""
        try:
            new_proc = Process(self.pid)
            # calls get_process_info() which may in turn trigger NSP exception
            str(new_proc)
        except NoSuchProcess:
            return False
        return self == new_proc

    def kill(self, sig=None):
        """Kill the current process by using signal sig (defaults to SIGKILL).
        """
        _platform_impl.kill_process(self.pid, sig)

def pid_exists(pid):
    """Check whether the given PID exists in the current process list."""
    return _platform_impl.pid_exists(pid)

def get_pid_list():
    """Return a list of current running PIDs."""
    return _platform_impl.get_pid_list()

def process_iter():
    """Return an iterator yielding a Process class instances for all
    running processes on the local machine.
    """
    pids = _platform_impl.get_pid_list()
    # for each PID, create a proxyied Process object
    # it will lazy init it's name and path later if required
    for pid in pids:
        try:
            yield Process(pid)
        except (NoSuchProcess, AccessDenied):
            continue

def get_process_list():
    """Return a list of Process class instances for all running
    processes on the local machine.
    """
    return list(process_iter())

def cpu_times():
    """Return system CPU times as a CPUTimes object."""
    values = get_system_cpu_times()
    return CPUTimes(**values)

_last_cpu_idle = cpu_times().idle
_last_system_time = time.time()

def cpu_percent():
    """Return the current system-wide CPU utilization as a percentage. For
    highest accuracy, it is recommended that this be called at least 1/10th
    of a second after importing the module or calling cpu_percent() in a
    previous call, to allow for a larger time delta from which to calculate
    the percentage value.
    """
    global _last_cpu_idle
    global _last_system_time

    idle_time = cpu_times().idle
    system_time = time.time()

    total_time_delta = system_time - _last_system_time
    cpu_times_delta = idle_time - _last_cpu_idle

    _last_cpu_idle = idle_time
    _last_system_time = system_time

    try :
        idle_percent = (cpu_times_delta / total_time_delta) * 100.0
        percent = (100 * NUM_CPUS) - idle_percent
    except ZeroDivisionError:
        return 0.00

    return percent / NUM_CPUS

def test():
    """List info of all currently running processes emulating a
    ps -aux output.
    """
    import datetime
    today_day = datetime.date.today()

    def get_process_info(pid):
        proc = Process(pid)
        uid = proc.uid
        pid = proc.pid
        cpu = round(proc.get_cpu_percent(), 1)
        mem = round(proc.get_memory_percent(), 1)
        rss, vsz = [x / 1024 for x in proc.get_memory_info()]

        # If process has been created today print H:M, else MonthDay
        start = datetime.datetime.fromtimestamp(proc.create_time)
        if start.date() == today_day:
            start = start.strftime("%H:%M")
        else:
            start = start.strftime("%b%d")

        cputime = time.strftime("%M:%S", time.localtime(sum(proc.get_cpu_times())))
        cmd = ' '.join(proc.cmdline)
        # where cmdline is not available UNIX shows process name between
        # [] parentheses
        if not cmd:
            cmd = "[%s]" %proc.name
        return "%-5s %7s %4s %4s %7s %7s %5s %8s %s" \
               %(uid, pid, cpu, mem, vsz, rss, start, cputime, cmd)

    print "%-5s %7s %4s %4s %7s %7s %5s %8s %s" \
       %("UID", "PID", "%CPU", "%MEM", "VSZ", "RSS", "START", "TIME", "COMMAND")
    pids = get_pid_list()
    pids.sort()
    for pid in pids:
        try:
            line = get_process_info(pid)
        except (AccessDenied, NoSuchProcess):
            pass
        else:
            print line

if __name__ == "__main__":
    test()
<|endoftext|>


<commit_before># Python imports
from subprocess import Popen, PIPE
import ConfigParser
import json
import optparse
import os
import re
import sys
import urllib2
import webbrowser

config = ConfigParser.ConfigParser({'url':'http://localhost:8080'})
config.read(os.path.expanduser('~/.pubdiffrc'))

AFTER = 'after'
BEFORE = 'before'
CONTENTS = 'contents'
NAME = 'name'
URL = config.get('core', 'url') + '/api/upload'
VERSION = '0.0.2.dev'

class SourceFile(dict):
    def __init__(self, name=None, contents=None):
        self[NAME] = name
        self[CONTENTS] = contents

class Diff(dict):
    def __init__(self, before, after):
        self[BEFORE] = before
        self[AFTER] = after

class DiffParser(object):
    def __init__(self, stdin):
        self.cwd = os.getcwd()
        self.diff = stdin.split('\n')

    def diffs(self):
        raise NotImplementedError('fetch_diffs() must be implementd')

    def shell(self, cmd):
        output = Popen(cmd, cwd=self.cwd, shell=True, stdout=PIPE, stderr=PIPE)
        return output.communicate()

class GitParser(DiffParser):
    RE_FILE_NAMES = re.compile(r'a/(?P<before>.*) b/(?P<after>.*)')
    RE_PATTERN = re.compile(r'^diff --git')
    RE_BLOBS = re.compile(r'index '
                          r'(?P<before>[a-z0-9]{7,})\.\.'
                          r'(?P<after>[a-z0-9]{7,}) ?'
                          r'(?P<perms>[0-9]{6})?')

    def source(self, name, blob):
        if blob ==  '0000000':
            return SourceFile(name, '')

        cmd = 'git show %s -- %s' % (blob, name)
        stdout, stderr = self.shell(cmd)

        # If the index hash specified in the diff doesn't exist, this
        # might be a unstaged/cached change.  We can validate by
        # comparing the hash in the diff, with the hash of the file on
        # disk, if the same... use that file exactly as is.  Once they
        # add the change to the index, git show will be able to fetch
        # the contents by the hash (avoiding this lookup).
        if 'bad revision' in stderr:
            cmd = 'git hash-object %s' % name
            stdout, stderr_inner = self.shell(cmd)
            if stdout.startswith(blob):
                stdout = open(name).read()
            else:
                print('Error when running:', cmd)
                print(stderr)

        return SourceFile(name, stdout)

    def fetch_source_files(self, names, blobs):
        before = self.source(names.group('before'), blobs.group('before'))
        after = self.source(names.group('after'), blobs.group('after'))
        return Diff(before, after)

    def fetch_diffs(self):
        diffs = []

        for i, line in enumerate(self.diff):
            blobs = GitParser.RE_BLOBS.match(line)
            if blobs:
                for p in xrange(15):
                    previous = self.diff[i - p]
                    names = GitParser.RE_FILE_NAMES.search(self.diff[i - p])
                    if names:
                        diffs.append(self.fetch_source_files(names, blobs))
                        break
                else:
                    print('Failed to find the files names for this diff')

        return diffs

class Client(object):
    def __init__(self):
        self.stdin = sys.stdin.read()
        self.parser = self.fetch_parser(self.stdin)

    def fetch_parser(self, diff):
        for cls in [GitParser]:
            if cls.RE_PATTERN.match(diff):
                return cls(self.stdin)

        msg = 'Your diff appears to be unsupported'
        raise UnsupportedClientError(msg)

    def main(self):
        # Parse command line options
        (options, args) = self.getopts()

        diffs = self.parser.fetch_diffs()
        if diffs:
            if os.environ.get('debug'):
                for diff in diffs:
                    print('Before:')
                    print(diff['before']['contents'])
                    print('After:')
                    print(diff['after']['contents'])

            request = urllib2.Request(URL, json.dumps(diffs))
            response = urllib2.urlopen(request)
            payload = json.loads(response.read())
            if payload['success']:
                url = payload['data']['url']

                if options.browser:
                    webbrowser.open(url)

                print(url)
            else:
                print(payload['exception'])

    def getopts(self):
        p = optparse.OptionParser('Usage: %prog [options]')
        p.add_option('-b', '--browser', dest='browser', action='store_true')
        p.set_defaults(browser=False)

        return p.parse_args()

class UnsupportedClientError(Exception):
    pass

def main():
    client = Client()
    client.main()

if __name__ == '__main__':
    main()
<commit_msg>Set default endpoint to be www.pubdiff.com<commit_after># Python imports
from subprocess import Popen, PIPE
import ConfigParser
import json
import optparse
import os
import re
import sys
import urllib2
import webbrowser

config = ConfigParser.ConfigParser({'url':'http://www.pubdiff.com'})
config.read(os.path.expanduser('~/.pubdiffrc'))

AFTER = 'after'
BEFORE = 'before'
CONTENTS = 'contents'
NAME = 'name'
URL = config.get('core', 'url') + '/api/upload'
VERSION = '0.0.3'

class SourceFile(dict):
    def __init__(self, name=None, contents=None):
        self[NAME] = name
        self[CONTENTS] = contents

class Diff(dict):
    def __init__(self, before, after):
        self[BEFORE] = before
        self[AFTER] = after

class DiffParser(object):
    def __init__(self, stdin):
        self.cwd = os.getcwd()
        self.diff = stdin.split('\n')

    def diffs(self):
        raise NotImplementedError('fetch_diffs() must be implementd')

    def shell(self, cmd):
        output = Popen(cmd, cwd=self.cwd, shell=True, stdout=PIPE, stderr=PIPE)
        return output.communicate()

class GitParser(DiffParser):
    RE_FILE_NAMES = re.compile(r'a/(?P<before>.*) b/(?P<after>.*)')
    RE_PATTERN = re.compile(r'^diff --git')
    RE_BLOBS = re.compile(r'index '
                          r'(?P<before>[a-z0-9]{7,})\.\.'
                          r'(?P<after>[a-z0-9]{7,}) ?'
                          r'(?P<perms>[0-9]{6})?')

    def source(self, name, blob):
        if blob ==  '0000000':
            return SourceFile(name, '')

        cmd = 'git show %s -- %s' % (blob, name)
        stdout, stderr = self.shell(cmd)

        # If the index hash specified in the diff doesn't exist, this
        # might be a unstaged/cached change.  We can validate by
        # comparing the hash in the diff, with the hash of the file on
        # disk, if the same... use that file exactly as is.  Once they
        # add the change to the index, git show will be able to fetch
        # the contents by the hash (avoiding this lookup).
        if 'bad revision' in stderr:
            cmd = 'git hash-object %s' % name
            stdout, stderr_inner = self.shell(cmd)
            if stdout.startswith(blob):
                stdout = open(name).read()
            else:
                print('Error when running:', cmd)
                print(stderr)

        return SourceFile(name, stdout)

    def fetch_source_files(self, names, blobs):
        before = self.source(names.group('before'), blobs.group('before'))
        after = self.source(names.group('after'), blobs.group('after'))
        return Diff(before, after)

    def fetch_diffs(self):
        diffs = []

        for i, line in enumerate(self.diff):
            blobs = GitParser.RE_BLOBS.match(line)
            if blobs:
                for p in xrange(15):
                    previous = self.diff[i - p]
                    names = GitParser.RE_FILE_NAMES.search(self.diff[i - p])
                    if names:
                        diffs.append(self.fetch_source_files(names, blobs))
                        break
                else:
                    print('Failed to find the files names for this diff')

        return diffs

class Client(object):
    def __init__(self):
        self.stdin = sys.stdin.read()
        self.parser = self.fetch_parser(self.stdin)

    def fetch_parser(self, diff):
        for cls in [GitParser]:
            if cls.RE_PATTERN.match(diff):
                return cls(self.stdin)

        msg = 'Your diff appears to be unsupported'
        raise UnsupportedClientError(msg)

    def main(self):
        # Parse command line options
        (options, args) = self.getopts()

        diffs = self.parser.fetch_diffs()
        if diffs:
            if os.environ.get('debug'):
                for diff in diffs:
                    print('Before:')
                    print(diff['before']['contents'])
                    print('After:')
                    print(diff['after']['contents'])

            request = urllib2.Request(URL, json.dumps(diffs))
            response = urllib2.urlopen(request)
            payload = json.loads(response.read())
            if payload['success']:
                url = payload['data']['url']

                if options.browser:
                    webbrowser.open(url)

                print(url)
            else:
                print(payload['exception'])

    def getopts(self):
        p = optparse.OptionParser('Usage: %prog [options]')
        p.add_option('-b', '--browser', dest='browser', action='store_true')
        p.set_defaults(browser=False)

        return p.parse_args()

class UnsupportedClientError(Exception):
    pass

def main():
    client = Client()
    client.main()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>
__version__ = '0.24.0'
NEWEST_CONAN_SUPPORTED = "1.14.0-dev"
<commit_msg>0.25.0<commit_after>
__version__ = '0.25.0'
NEWEST_CONAN_SUPPORTED = "1.15.0-dev"
<|endoftext|>


<commit_before>from PIL import Image

import sys

# run with: python crop_sprites.py `find sprites/ -name *.png -print`
# (on linux, obviously)

for filename in sys.argv[1:]:
    im = Image.open(filename)
    im.convert("RGBA")
    w, h = im.size
    r = 0
    l = w
    t = h
    b = 0

    pix = im.load()
    for x in range(w):
        for y in range(h):
            if pix[x, y][3] != 0:
                if l > x:
                    l = x
                if r < x:
                    r = x
                if t > y:
                    t = y
                if b < y:
                    b = y

    print('"{0}": [{1}, {2}],'.format(filename[8:], w/2-l, h/2-t))
    im2 = im.crop((l, t, r+1, b+1))
    im2.save(filename)
<commit_msg>Made the crop_sprites.py file have a help argument for easier use.<commit_after>from PIL import Image

import sys

# run with: python crop_sprites.py `find sprites/ -name *.png -print`
# (on linux, obviously)

if sys.argv[1] == "help":
    print "run with: python crop_sprites.py `find sprites/ -name *.png -print`"
else:
    for filename in sys.argv[1:]:
        im = Image.open(filename)
        im.convert("RGBA")
        w, h = im.size
        r = 0
        l = w
        t = h
        b = 0

        pix = im.load()
        for x in range(w):
            for y in range(h):
                if pix[x, y][3] != 0:
                    if l > x:
                        l = x
                    if r < x:
                        r = x
                    if t > y:
                        t = y
                    if b < y:
                        b = y

        print('"{0}": [{1}, {2}],'.format(filename[8:], w/2-l, h/2-t))
        im2 = im.crop((l, t, r+1, b+1))
        im2.save(filename)
<|endoftext|>


<commit_before>"""
cryptography.py
Author: Nils Kingston
Credit: Roger

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
from operator import add
associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"
a = ""
while a != "q":
    a = input("Enter e to encrypt, d to decrypt, or q to quit: ")
    if a == "e":
        encripting = input("What do you want to encript? ")
        jumbler = input("What should the key be? ")
        e_list = []
        for x in encripting:
            e_list.append(associations.find(x))
        j_list = []
        for x in jumbler:
            j_list.append(associations.find(x))
        newlist = []
        length = len(e_list)
        for x in range(length):
            newlist.append(e_list[x] + j_list[x])
        print(newlist)

    elif a == "d":
        jumbled_code = input("Message?")
        key = input("Key?")
        jum_list = []
        for x in jumbled_code:
            jum_list.append(associations.find(x))
        k_list = []
        for x in key:
            k_list.append(associations.find(x))

    elif a == "q":
        print("Goodbye!")
    else:
        print("Did not understand try again")

<commit_msg>Updated from Brython Server: 10/8/2015 2:53:48 PM<commit_after>"""
cryptography.py
Author: Nils Kingston
Credit: Roger

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
from operator import add
associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"
a = ""
while a != "q":
    a = input("Enter e to encrypt, d to decrypt, or q to quit: ")
    if a == "e":
        encripting = input("What do you want to encript? ")
        jumbler = input("What should the key be? ")
        e_list = []
        for x in encripting:
            e_list.append(associations.find(x))
        j_list = []
        for x in jumbler:
            j_list.append(associations.find(x))
        newlist = []
        length = len(e_list)
        for x in range(length):
            newlist.append(e_list[x] + j_list[x])
        print(newlist)

    elif a == "d":
        jumbled_code = input("Message?")
        key = input("Key?")
        jum_list = []
        for x in jumbled_code:
            jum_list.append(associations.find(x))
        k_list = []
        for x in key:
            k_list.append(associations.find(x))

    elif a == "q":
        print("Goodbye!")
    else:
        print("Did not understand try again")

<|endoftext|>


<commit_before>"""
cryptography.py
Author: <your name here>
Credit: <list sources used, if any>

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"
<commit_msg>Updated from Brython Server: 4/18/2016 2:42:03 PM<commit_after>"""
cryptography.py
Author: Andreas Moberg
Credit: Payton (for mad math skillz)

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"

keystroke=input("Enter e to encrypt, d to decrypt, or q to quit:")

cryptm=[]

if keystroke=="e":
    mess=input("Message:")
    key=input("Key:")
    k=len(key)
    m=len(mess)

    if m>k:
        count=key*int((m-(m%k))/k)
        trun=key[0:(m%k)]
        newkey=count+trun
    elif k>m:
        newkey=key[0:m]
        print(newkey, message)

    for x in mess:
        for y in key:
            cryptm.append(associations.find(x)+associations.find(y))
    for x in cryptm:
        print(associations[x])

elif keystroke=="d":
    mess=input("Message:")
    key=input("Key:")

elif keystroke=="q":
    ...
    print("Goodbye!")

else:
    print("Did not understand command, try again.")

<|endoftext|>


<commit_before>"""
cryptography.py
Author: Dina
Credit: none

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"
command = input("Enter e to encrypt, d to decrypt, or q to quit: ")

while command != 'q':
    while command != 'e' and command != 'd':
        command = input("Did not understand command, try again. ")
    if command == 'e':
        message = input("Message: ")
        key = input("Key: ")
        while len(key) < len(message):
            key = key+key

        print(key)
        #print(associations[lnum], end = '')
    else:
        message = input("Message: ")
        key = input("Key: ")
        for x in message:
            mnum = associations.find(x)
            for y in key:
                knum = associations.find(y)
            lnum = mnum-knum
            print(associations[lnum], end = '')
    print()
    command = input("Enter e to encrypt, d to decrypt, or q to quit: ")
print("Goodbye! ")<commit_msg>Updated from Brython Server: 11/2/2015 2:30:15 PM<commit_after>"""
cryptography.py
Author: Dina
Credit: none

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"
command = input("Enter e to encrypt, d to decrypt, or q to quit: ")

while command != 'q':
    while command != 'e' and command != 'd':
        command = input("Did not understand command, try again. ")
    if command == 'e':
        message = input("Message: ")
        key = input("Key: ")
        while len(key) < len(message):
            key = key+key
        km = list(zip(message, key))
        for a in km:
            mnum = associations.find(a[0])
            knum = associations.find(a[1])
            b = mnum+knum
            print(associations[b], end = '')
    else:
        message = input("Message: ")
        key = input("Key: ")
        while len(key) < len(message):
            key = key+key
        km = list(zip(message, key))
        for a in km:
            mnum = associations.find(a[0])
            knum = associations.find(a[1])
            b = mnum-knum
            print(associations[b], end = '')
    print()
    command = input("Enter e to encrypt, d to decrypt, or q to quit: ")
print("Goodbye! ")
"""
message = input("Message: ")
        key = input("Key: ")
        for x in message:
            mnum = associations.find(x)
            for y in key:
                knum = associations.find(y)
            lnum = mnum-knum
            print(associations[lnum], end = '')
"""<|endoftext|>


<commit_before>"""
cryptography.py
Author: Dimitri
Credit: http://stackoverflow.com/questions/3391076/repeat-string-to-certain-length, http://stackoverflow.com/questions/14050824/add-sum-of-values-of-two-lists-into-new-list

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"

def rep(s, m):
    a, b = divmod(m, len(s))
    return s * a + s[:b]

def encrypt():
    list1 = []
    list2 = []
    for x in message:
        letter = associations.find(x)
        list1.append(letter)
    for y in key:
        keyletter = associations.find(y)
        list2.append(keyletter)
        list21 = rep(list2, len(list1))
    list3 = [x + y for x, y in zip(list1, list21)]
    list4 = []
    for n in list3:
        if n%86 == 0:
            l = "!"
        else:
            l = associations[n%86]
        list4.append(l)
    for x in list4:
        print(x, end='')

def decrypt():
    dlist1 = []
    dlist2 = []
    for x in dmessage:
        dletter = associations.find(x)
        dlist1.append(dletter)
    for y in dkey:
        dkeyletter = associations.find(y)
        dlist2.append(dkeyletter)
        list22 = rep(dlist2, len(dlist1))
    dlist3 = [x - y for x, y in zip(dlist1, list22)]
    dlist4 = []
    for m in dlist3:
        if m%86 == 0:
            dl = "!"
        else:
            dl = associations[m%86]
        dlist4.append(dl)
    for x in dlist4:
        print(x, end='')

running = 0
while running == 0:
    command = str(input("Enter e to encrypt, d to decrypt, or q to quit: "))
    if command == "e":
        message = str(input("Message: "))
        key = str(input("Key: "))
        encrypt()
        print('\n')
    elif command == "d":
        dmessage = str(input("Message: "))
        dkey = str(input("Key: "))
        decrypt()
        print('\n')
    elif command == "q":
        print("Goodbye!")
        running += 1
    else:
        print("Did not understand command, try again.")
        print('\n')

"""
if command == "e":
    message = str(input("Message: "))
    key = str(input("Key: "))
    encrypt()
elif command == "d":
    dmessage = str(input("Message: "))
    dkey = str(input("Key: "))
    decrypt()
elif command == "q":
    print("Goodbye!")
else:
    print("Did not understand command, try again.")
"""<commit_msg>Updated from Brython Server: 11/3/2015 5:48:39 PM<commit_after>"""
cryptography.py
Author: Dimitri
Credit: http://stackoverflow.com/questions/3391076/repeat-string-to-certain-length, http://stackoverflow.com/questions/14050824/add-sum-of-values-of-two-lists-into-new-list

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"

def rep(s, m):
    a, b = divmod(m, len(s))
    return s * a + s[:b]

def encrypt():
    list1 = []
    list2 = []
    for x in message:
        letter = associations.find(x)
        list1.append(letter)
    for y in key:
        keyletter = associations.find(y)
        list2.append(keyletter)
        list21 = rep(list2, len(list1))
    list3 = [x + y for x, y in zip(list1, list21)]
    list4 = []
    for n in list3:
        if n%86 == 0:
            l = "!"
        else:
            l = associations[n%86]
        list4.append(l)
    for x in list4:
        print(x, end='')

def decrypt():
    dlist1 = []
    dlist2 = []
    for x in dmessage:
        dletter = associations.find(x)
        dlist1.append(dletter)
    for y in dkey:
        dkeyletter = associations.find(y)
        dlist2.append(dkeyletter)
        list22 = rep(dlist2, len(dlist1))
    dlist3 = [x - y for x, y in zip(dlist1, list22)]
    dlist4 = []
    for m in dlist3:
        dl = associations[m]
        dlist4.append(dl)
    for x in dlist4:
        print(x, end='')

running = 0
while running == 0:
    command = str(input("Enter e to encrypt, d to decrypt, or q to quit: "))
    if command == "e":
        message = str(input("Message: "))
        key = str(input("Key: "))
        encrypt()
        print('\n')
    elif command == "d":
        dmessage = str(input("Message: "))
        dkey = str(input("Key: "))
        decrypt()
        print('\n')
    elif command == "q":
        print("Goodbye!")
        running += 1
    else:
        print("Did not understand command, try again.")
        print('\n')

"""
if command == "e":
    message = str(input("Message: "))
    key = str(input("Key: "))
    encrypt()
elif command == "d":
    dmessage = str(input("Message: "))
    dkey = str(input("Key: "))
    decrypt()
elif command == "q":
    print("Goodbye!")
else:
    print("Did not understand command, try again.")
"""<|endoftext|>


<commit_before>
from django.contrib import admin
from django import forms
from django.contrib.contenttypes import generic
from django.utils.safestring import mark_safe

from versioning.models import Revision

class DeltaWidget(forms.Widget):
    """
    Render a delta in a form.
    TODO: this needs some more work.
    """
    def render(self, name, data, attrs):
        return mark_safe(data.replace("\n", "
"))

class RevisionInline(generic.GenericTabularInline):
    model = Revision
    ct_field = "content_type"
    ct_fk_field = "object_pk"
    extra = 0
    fields = ("delta",)

    def formfield_for_dbfield(self, db_field, **kwargs):
        if db_field.name == "delta":
            kwargs["widget"] = DeltaWidget
            return db_field.formfield(**kwargs)
        super(RevisionInline, self).formfield_for_dbfield(db_field, **kwargs)

class RevisionAdmin(admin.ModelAdmin):
    list_display = ("sha1", "content_type", "object_pk", "created_at")
    list_filter = ("created_at", "content_type",)

    def formfield_for_dbfield(self, db_field, **kwargs):
        if db_field.name == "delta":
            kwargs["widget"] = DeltaWidget
            return db_field.formfield(**kwargs)
        super(RevisionAdmin, self).formfield_for_dbfield(db_field, **kwargs)

admin.site.register(Revision, RevisionAdmin)
<commit_msg>Added return statements to formfield_for_dbfield methods.<commit_after>
from django.contrib import admin
from django import forms
from django.contrib.contenttypes import generic
from django.utils.safestring import mark_safe

from versioning.models import Revision

class DeltaWidget(forms.Widget):
    """
    Render a delta in a form.
    TODO: this needs some more work.
    """
    def render(self, name, data, attrs):
        return mark_safe(data.replace("\n", "
"))

class RevisionInline(generic.GenericTabularInline):
    model = Revision
    ct_field = "content_type"
    ct_fk_field = "object_pk"
    extra = 0
    fields = ("delta",)

    def formfield_for_dbfield(self, db_field, **kwargs):
        if db_field.name == "delta":
            kwargs["widget"] = DeltaWidget
            return db_field.formfield(**kwargs)
        return super(RevisionInline, self).formfield_for_dbfield(db_field, **kwargs)

class RevisionAdmin(admin.ModelAdmin):
    list_display = ("sha1", "content_type", "object_pk", "created_at")
    list_filter = ("created_at", "content_type",)

    def formfield_for_dbfield(self, db_field, **kwargs):
        if db_field.name == "delta":
            kwargs["widget"] = DeltaWidget
            return db_field.formfield(**kwargs)
        return super(RevisionAdmin, self).formfield_for_dbfield(db_field, **kwargs)

admin.site.register(Revision, RevisionAdmin)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
#  boost_tst.py
#
#  test of generation of RGB images in C++
#
#  Copyright (C) 2014 Diamond Light Source
#
#  Author: Luis Fuentes-Montero (Luiso)
#
#  This code is distributed under the BSD license, a copy of which is
#  included in the root directory of this package.

from __future__ import absolute_import, division
from dials.array_family import flex

import boost.python
#from dials_viewer_ext import gen_font_img, rgb_img
from dials_viewer_ext import rgb_img, gen_str_tst
if __name__ == "__main__":
  tmp_not_needed = '''
  size_xyz = 7

  arr_2d = flex.double(flex.grid(size_xyz, size_xyz), 00)

  tot = 0.0
  for row in range(size_xyz):
    for col in range(size_xyz):
      arr_2d[row, col] += (row * 2 + col * 2)
      tot += arr_2d[row, col]

  print "arr_2d.as_numpy_array() = \n", arr_2d.as_numpy_array()
  #print "gen_font_img(arr_2d).as_numpy_array() = \n", gen_font_img(arr_2d).as_numpy_array()

  wx_bmp_arr = rgb_img()
  print wx_bmp_arr.gen_bmp(arr_2d).as_numpy_array()
  '''

  print "testing new Cpp code"
  size_lng = 555
  arr_1d = flex.double(flex.grid(size_lng, 1), 00)
  for i in xrange(size_lng):
    var = i * .5
    arr_1d[i, 0] = -.2 * var * var * var + 0.00000001
  gen_str_tst(arr_1d)
  #print "gen_str_tst(arr_1d).as_numpy_array() = \n", gen_str_tst(arr_1d).as_numpy_array()
<commit_msg>@luisodls says remove. Closes #524<commit_after><|endoftext|>


<commit_before>from re import match
from random import sample
from string import capwords, strip
import pandas as pd
import numpy as np
from pandas import Series, DataFrame
import fiona
from shapely.geometry import Point, asShape
from shapely.prepared import prep
from shapely import speedups
from rules import name_rules, geofeat_rules

DATA_ROOT = "" # TODO: wire this up

# Warn if a value is being assigned to a copy
pd.set_option('mode.chained_assignment', 'warn')

# Imports of needed data
try:
    gb = pd.read_pickle("../../data/pickles/gb.pk1")
except IOError:
    # TODO: In this case should do a bunch of stuff to get gb into namespace
    pass

try:
    namefam = pd.read_table("../../data/pickles")

### Setup Data Functions###
def set_gb(file_path):
    """Read the GB.txt file from geonames and return an appropriately
    filtered DataFrame"""
    gb = pd.read_csv(file_path)
    # Trim away an initial index column
    gb = gb.ix[:, 1:20]
    column_names = ['geoid', 'name', 'asciiname', 'altname', 'lat', 'long',
        'feature_class', 'feature_code', 'country_code', 'cc2', 'adm1', 'adm2',
        'adm3', 'adm4', 'pop', 'elev', 'delev', 'timezone', 'moddate']
    gb.columns = column_names
    # Removing rows that correspond to Cyprus, N. Ireland, cruft etc.
    remove_these_adm1 = ['05', '00', '01', 'NIR', '03']
    for i in remove_these_adm1:
        gb = gb[gb.adm1 != i]
    # Get rid of adm1 with null values
    gb = gb[~gb.adm1.isnull()]
    # Setting a list of alternate names from altname string field
    gb['ls_altname'] = gb['altname'].dropna().apply(lambda x: x.split(','))
    # Pick only feature codes that correspond to towns and cities, see
    # http://www.geonames.org/export/codes.html
    gb = gb[gb.feature_code.isin(geofeat_rules)]
    # Clean up index
    gb.index = range(len(gb))
    return gb

def set_fam(dfin):
    """In-place setup name families for df dataframe.

    Note: this only acts on the 'name' field. Use another function to setup
    altname families

    Column changes: will add a ls_namefam column

    TODO: This is a one run deal; not idempotent. Fix.
    """
    df = dfin.copy()
    df["ls_namefam"] = np.nan
    # Iterate over rows in dataframe
    for index, row in df.iterrows():
        # For each row, check each name_rule
        for namekey, ls_regex in name_rules.iteritems():
            result = patinstr(row.loc['name'], ls_regex)
            if result:
                cur = df.loc[index, 'ls_namefam']
                if not isinstance(cur, list):
                    df.loc[index, 'ls_namefam'] = list([namekey])
                else:
                    df.loc[index, 'ls_namefam'].append(namekey)
    return df

def set_alt(df, column_names=None):
    """DataFrame should only be the dataframe that
    comes after setgb if columns is left undefined; column_names is a list of
    column names that the resulting dataframe should contain. This list should
    include 'parent.'"""

    df_alt = df.copy()
    df_alt.drop(['altname'], inplace=True, axis=1)
    df_alt.drop(['ls_altname'], inplace=True, axis=1)
    df_alt['parent'] = np.nan

    if column_names is None:
        column_names = ['geoid', 'name', 'parent', 'asciiname', 'lat', 'long',
                     'feature_class', 'feature_code', 'country_code', 'cc2',
                     'adm1', 'adm2', 'adm3', 'adm4', 'pop', 'elev', 'delev',
                     'timezone', 'moddate']
    df_alt = df_alt[column_names]

    i = len(df)
    for index, row in df.iterrows():
        parent_name = row['name']
        try:
            a_names = iter(row['ls_altname'])
        except TypeError:
            pass
        else:
            for a_name in a_names:
                row['name'] = a_name
                row['parent'] = parent_name
                df_alt.ix[i] = row[column_names]
                i += 1

    df_alt[~df_alt['name'].str.contains('/')]
    return df_alt

def set_namefam_table(file_path):
    """Read the namfam csv file and load to a table"""
    return pd.read_csv(file_path)

def append_nuts3_region(dfin, shapefile_path):
    """ Take a pandas dataframe with lat and long columns and a shapefile.
    Convert coordinates to Shapely points to do a point-in-polygon check
    for each point. Append name of nuts3 region corresponding to point-in-polygon
    to the dataframe.
    This is extremely slow. Needs to be optimized with Shapely boundary box.
    """
    df = dfin.copy()
    df['nuts3'] = np.nan
    fc = fiona.open(shapefile_path)
    speedups.enable()
    for feature in fc:
        prepared_shape = prep(asShape(feature['geometry']))
        for index, row in df.iterrows():
            point = Point(row.loc['long'], row.loc['lat'])
            if prepared_shape.contains(point):
                df.loc[index, 'nuts3name'] = feature['properties']['NUTS315NM']
                df.loc[index, 'nuts3id'] = feature['properties']['NUTS315CD']
    return df

def append_2013_gva(dfin, csv_file_path):
    df = dfin.copy()
    gva = pd.read_csv(csv_file_path)
    gvasub = DataFrame(columns=['nuts3id', 'gva2013'])
    gvasub['nuts3id'], gvasub['gva2013'] = gva['nutsid'], gva['2013']
    df_gva = pd.merge(
        left=df,
        right=gvasub.dropna(),
        how='left',
        left_on='nuts3id',
        right_on='nuts3id')
    return df_gva

### Helper Query Functions ###
def patinls(slist, patlist):
    """Search each string in slist for a substring instance defined by patlist;
    return re.match if found, None if not found"""
    found = None
    try:
        strings = iter(slist)
    except TypeError:
        return
    for string in strings:
        if not found:
            for pat in patlist:
                found = match(pat, string)
                if found:
                    break
        else:
            break
    return found

def patinstr(string, patlist):
    """Search string for a substring instance defined by patlist; return
    re.match if found, None if not found"""
    found = None
    if not isinstance(string, str):
        return None
    for pat in patlist:
        found = match(pat, string)
        if found:
            break
    return found

### DataFrame Query Functions ###
def get_fam(df, namekey):
    """Get dataframe subset from df for rows belonging to the family
    which is a sub"""
    def _droidfind(listin):
        result = None
        try:
            result = namekey in listin
        except TypeError:
            pass
        return result if result is True else None
    mask = df['ls_namefam'].map(lambda x: _droidfind(x))
    return df[~mask.isnull()]

def query_random(df):
    """Return a sub-dataframe corresponding to a particular namefamily.
    Also return a sample placename from that namefamily"""
    namekey = sample(name_rules.keys())
    subdf = get_fam(df, namekey)
    placename = sample(subdf['name'])
    return subdf, placename

def query_name(df, placestring):
    """Return a sub-DataFrame containing boolean matches to place name.

    This is intended as a Helper function. Will attempt a more literal
    match, and if this fails, will try a loose match.
    """
    # First try to query assuming that the user formatted the placename
    # correctly, then try a looser search
    place = capwords(placestring).strip()
    query = df['name'].str.contains('.*' + place + '.*')
    if not query.any():
        place = placestring.lower().strip()
        query = df['name'].str.contains('.*' + place + '.*')
    return query

def query_placename(df, placestring):
    """Attempt to match placestring to a city with a family pattern;
    return the matching sub-DataFrame, namekey, and full placename if a match
    is made, else None

    If there are multiple associated name families, then one will be
    picked at random"""
    query = query_name(df, placestring)
    # Return a dataframe row with the placename contained;
    # assure that this row actually has a namefam associated
    try:
        namefam_row = df[query & (~df['ls_namefam'].isnull())].sample()
    except ValueError:
        # In this case, there is no namefam_row match
        return None

    namekey = namefam_row['ls_namefam'].map(lambda x: sample(x, 1))
    # An ugly way to get the string out
    namekey = namekey.values[0][0]
    placename = namefam_row['name']
    return get_fam(df, namekey), namekey, placename.values[0]

def query_name_or_fam(df, placestring):
    """Attempt to match placestring to a city with a family pattern; else
    attempt to find a place of the same name. Return None otherwise.

    If namefamily is found: return the matching sub-DataFrame, namekey,
    and full placenameif a match

    If namefamily is not found, but placename is: return the singular placename
    row as a DataFrame, None for namekey, and full placename

    Otherwise return None."""

    result = query_placename(df, placestring)
    if result is None:
        # Case of place with no namefam
        query = query_name(df, placestring)
        try:
            row = df[query].sample()
            result = row, None, row['name'].values[0]
        except ValueError:
            result = None
    return result

def query_pop_slice(df, popthresh):
    """Return a sub-DataFrame from DataFrame df that excludes all population
    values below the threshold"""

    return df[df['pop'] >= popthresh]

def query_namefam_table(namekey):
    """Return a dictionary containing strings of all of the elements of
    a namefam table in human readable string format"""
    row =

if __name__ == "__main__":
    gbf = 'data/pristine/GB.txt'
    gb = set_gb(gbf)
    gb2 = set_fam(gb)

    print gb2['ls_namefam'].dropna()
<commit_msg>added a query_namefam_table() to human format data from the namefam table<commit_after>from re import match
from random import sample
from string import capwords, strip
import pandas as pd
import numpy as np
from pandas import Series, DataFrame
import fiona
from shapely.geometry import Point, asShape
from shapely.prepared import prep
from shapely import speedups
from rules import name_rules, geofeat_rules, wiki_codes

DATA_ROOT = "" # TODO: wire this up

# Warn if a value is being assigned to a copy
pd.set_option('mode.chained_assignment', 'warn')

# Imports of needed data
try:
    GB = pd.read_pickle("../../data/pickles/gb.pk1")
except IOError:
    # TODO: In this case should do a bunch of stuff to get gb into namespace
    pass

try:
    NAMEFAM = pd.read_table("../../data/pickles")
except IOError:
    # TODO: In this case should do a bunch of stuff to get gb into namespace
    pass

### Setup Data Functions###
def set_gb(file_path):
    """Read the GB.txt file from geonames and return an appropriately
    filtered DataFrame"""
    gb = pd.read_csv(file_path)
    # Trim away an initial index column
    gb = gb.ix[:, 1:20]
    column_names = ['geoid', 'name', 'asciiname', 'altname', 'lat', 'long',
        'feature_class', 'feature_code', 'country_code', 'cc2', 'adm1', 'adm2',
        'adm3', 'adm4', 'pop', 'elev', 'delev', 'timezone', 'moddate']
    gb.columns = column_names
    # Removing rows that correspond to Cyprus, N. Ireland, cruft etc.
    remove_these_adm1 = ['05', '00', '01', 'NIR', '03']
    for i in remove_these_adm1:
        gb = gb[gb.adm1 != i]
    # Get rid of adm1 with null values
    gb = gb[~gb.adm1.isnull()]
    # Setting a list of alternate names from altname string field
    gb['ls_altname'] = gb['altname'].dropna().apply(lambda x: x.split(','))
    # Pick only feature codes that correspond to towns and cities, see
    # http://www.geonames.org/export/codes.html
    gb = gb[gb.feature_code.isin(geofeat_rules)]
    # Clean up index
    gb.index = range(len(gb))
    return gb

def set_fam(dfin):
    """In-place setup name families for df dataframe.

    Note: this only acts on the 'name' field. Use another function to setup
    altname families

    Column changes: will add a ls_namefam column

    TODO: This is a one run deal; not idempotent. Fix.
    """
    df = dfin.copy()
    df["ls_namefam"] = np.nan
    # Iterate over rows in dataframe
    for index, row in df.iterrows():
        # For each row, check each name_rule
        for namekey, ls_regex in name_rules.iteritems():
            result = patinstr(row.loc['name'], ls_regex)
            if result:
                cur = df.loc[index, 'ls_namefam']
                if not isinstance(cur, list):
                    df.loc[index, 'ls_namefam'] = list([namekey])
                else:
                    df.loc[index, 'ls_namefam'].append(namekey)
    return df

def set_alt(df, column_names=None):
    """DataFrame should only be the dataframe that
    comes after setgb if columns is left undefined; column_names is a list of
    column names that the resulting dataframe should contain. This list should
    include 'parent.'"""

    df_alt = df.copy()
    df_alt.drop(['altname'], inplace=True, axis=1)
    df_alt.drop(['ls_altname'], inplace=True, axis=1)
    df_alt['parent'] = np.nan

    if column_names is None:
        column_names = ['geoid', 'name', 'parent', 'asciiname', 'lat', 'long',
                     'feature_class', 'feature_code', 'country_code', 'cc2',
                     'adm1', 'adm2', 'adm3', 'adm4', 'pop', 'elev', 'delev',
                     'timezone', 'moddate']
    df_alt = df_alt[column_names]

    i = len(df)
    for index, row in df.iterrows():
        parent_name = row['name']
        try:
            a_names = iter(row['ls_altname'])
        except TypeError:
            pass
        else:
            for a_name in a_names:
                row['name'] = a_name
                row['parent'] = parent_name
                df_alt.ix[i] = row[column_names]
                i += 1

    df_alt[~df_alt['name'].str.contains('/')]
    return df_alt

def set_namefam_table(file_path):
    """Read the namfam csv file and load to a table"""
    return pd.read_csv(file_path)

def append_nuts3_region(dfin, shapefile_path):
    """ Take a pandas dataframe with lat and long columns and a shapefile.
    Convert coordinates to Shapely points to do a point-in-polygon check
    for each point. Append name of nuts3 region corresponding to point-in-polygon
    to the dataframe.
    This is extremely slow. Needs to be optimized with Shapely boundary box.
    """
    df = dfin.copy()
    df['nuts3'] = np.nan
    fc = fiona.open(shapefile_path)
    speedups.enable()
    for feature in fc:
        prepared_shape = prep(asShape(feature['geometry']))
        for index, row in df.iterrows():
            point = Point(row.loc['long'], row.loc['lat'])
            if prepared_shape.contains(point):
                df.loc[index, 'nuts3name'] = feature['properties']['NUTS315NM']
                df.loc[index, 'nuts3id'] = feature['properties']['NUTS315CD']
    return df

def append_2013_gva(dfin, csv_file_path):
    df = dfin.copy()
    gva = pd.read_csv(csv_file_path)
    gvasub = DataFrame(columns=['nuts3id', 'gva2013'])
    gvasub['nuts3id'], gvasub['gva2013'] = gva['nutsid'], gva['2013']
    df_gva = pd.merge(
        left=df,
        right=gvasub.dropna(),
        how='left',
        left_on='nuts3id',
        right_on='nuts3id')
    return df_gva

### Helper Query Functions ###
def patinls(slist, patlist):
    """Search each string in slist for a substring instance defined by patlist;
    return re.match if found, None if not found"""
    found = None
    try:
        strings = iter(slist)
    except TypeError:
        return
    for string in strings:
        if not found:
            for pat in patlist:
                found = match(pat, string)
                if found:
                    break
        else:
            break
    return found

def patinstr(string, patlist):
    """Search string for a substring instance defined by patlist; return
    re.match if found, None if not found"""
    found = None
    if not isinstance(string, str):
        return None
    for pat in patlist:
        found = match(pat, string)
        if found:
            break
    return found

### DataFrame Query Functions ###
def get_fam(df, namekey):
    """Get dataframe subset from df for rows belonging to the family
    which is a sub"""
    def _droidfind(listin):
        result = None
        try:
            result = namekey in listin
        except TypeError:
            pass
        return result if result is True else None
    mask = df['ls_namefam'].map(lambda x: _droidfind(x))
    return df[~mask.isnull()]

def query_random(df):
    """Return a sub-dataframe corresponding to a particular namefamily.
    Also return a sample placename from that namefamily"""
    namekey = sample(name_rules.keys())
    subdf = get_fam(df, namekey)
    placename = sample(subdf['name'])
    return subdf, placename

def query_name(df, placestring):
    """Return a sub-DataFrame containing boolean matches to place name.

    This is intended as a Helper function. Will attempt a more literal
    match, and if this fails, will try a loose match.
    """
    # First try to query assuming that the user formatted the placename
    # correctly, then try a looser search
    place = capwords(placestring).strip()
    query = df['name'].str.contains('.*' + place + '.*')
    if not query.any():
        place = placestring.lower().strip()
        query = df['name'].str.contains('.*' + place + '.*')
    return query

def query_placename(df, placestring):
    """Attempt to match placestring to a city with a family pattern;
    return the matching sub-DataFrame, namekey, and full placename if a match
    is made, else None

    If there are multiple associated name families, then one will be
    picked at random"""
    query = query_name(df, placestring)
    # Return a dataframe row with the placename contained;
    # assure that this row actually has a namefam associated
    try:
        namefam_row = df[query & (~df['ls_namefam'].isnull())].sample()
    except ValueError:
        # In this case, there is no namefam_row match
        return None

    namekey = namefam_row['ls_namefam'].map(lambda x: sample(x, 1))
    # An ugly way to get the string out
    namekey = namekey.values[0][0]
    placename = namefam_row['name']
    return get_fam(df, namekey), namekey, placename.values[0]

def query_name_or_fam(df, placestring):
    """Attempt to match placestring to a city with a family pattern; else
    attempt to find a place of the same name. Return None otherwise.

    If namefamily is found: return the matching sub-DataFrame, namekey,
    and full placenameif a match

    If namefamily is not found, but placename is: return the singular placename
    row as a DataFrame, None for namekey, and full placename

    Otherwise return None."""

    result = query_placename(df, placestring)
    if result is None:
        # Case of place with no namefam
        query = query_name(df, placestring)
        try:
            row = df[query].sample()
            result = row, None, row['name'].values[0]
        except ValueError:
            result = None
    return result

def query_pop_slice(df, popthresh):
    """Return a sub-DataFrame from DataFrame df that excludes all population
    values below the threshold"""

    return df[df['pop'] >= popthresh]

def query_namefam_table(namekey):
    """Return a dictionary containing strings of all of the elements of
    a namefam table in human readable string format"""
    row = NAMEFAM[NAMEFAM['namekey'] == namekey]
    if row.shape[0] == 0:
    # Case of asking for a namekey that doesn't exist
        return None
    toreturn = dict()

    # Turning wiki_code characters into a human readable string
    wiki_str = ""
    for code in row['wiki_codes'].values[0].split():
        frag = wiki_codes.get(code.rstrip(',').lstrip())
        frag = frag if frag is not None else ""
        wiki_str += "{}, ".format(frag)
    # Maybe refactor these iteratively later
    toreturn['wiki_codes'] = wiki_str.rstrip(", ")
    toreturn['namekey'] = row['namekey'].values[0]
    toreturn['humandef'] = row['humandef'].values[0]
    toreturn['lan_notes'] = row['lan_notes'].values[0]
    toreturn['human_namekey'] = row['human_namekey'].values[0]
    return toreturn

if __name__ == "__main__":
    gbf = 'data/pristine/GB.txt'
    gb = set_gb(gbf)
    gb2 = set_fam(gb)

    print gb2['ls_namefam'].dropna()
<|endoftext|>


<commit_before>import requests
from bs4 import BeautifulSoup

import numpy as np
import pandas as pd

commments = []

def crawl_comments(type='book', item_id='26708119', pn=2):
    url = 'https://' + type + '.douban.com/subject/' + str(item_id) + '/comments/hot?p=' + str(pn)
    print('crawling page %s ' % url)
    headers = {
        'Accept': 'application/json, text/javascript, */*; q=0.01',
        'Host': '%s.douban.com' % type,
        'Referer': 'https://' + type + '.douban.com/subject/' + str(item_id) + '/comments/hot?p=1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36'
    }
    response = requests.get(url, headers=headers)
    if response.status_code == 200:
        soup = BeautifulSoup(response.text, 'html5lib')
        for li in soup.find('div', id="comments").find_all('li'):
            cid = li['data-cid']
            comm = li.find('p', class_="comment-content").get_text()

            tmp = li.find('span', class_="comment-info").find_all('span')[0]
            rate = int(tmp['class'][1].replace('allstar', '')) / 10 if tmp.has_attr('class') else ''
            cdate = li.find('span', class_="comment-info").find_all('span')[-1].get_text()
            commments.append([cid, comm, rate, cdate])
    else:
        print(response.status_code)

if __name__ == '__main__':
    for i in range(2, 237):
        crawl_comments(pn=i, item_id='10750155')

    col = ['ID', 'Comment', 'Rate', 'Date']
    df = pd.DataFrame(commments, columns=col)
    df.to_excel(excel_writer='./数学之美.xlsx', sheet_name='DoubanBook', index=False)
<commit_msg>add sleep to douban comments crawler<commit_after>import time
import requests
from bs4 import BeautifulSoup

import numpy as np
import pandas as pd

commments = []

def crawl_comments(type='book', item_id='26708119', pn=2):
    url = 'https://' + type + '.douban.com/subject/' + str(item_id) + '/comments/hot?p=' + str(pn)
    print('crawling page %s ' % url)
    headers = {
        'Accept': 'application/json, text/javascript, */*; q=0.01',
        'Host': '%s.douban.com' % type,
        'Referer': 'https://' + type + '.douban.com/subject/' + str(item_id) + '/comments/hot?p=1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36'
    }
    response = requests.get(url, headers=headers)
    if response.status_code == 200:
        soup = BeautifulSoup(response.text, 'html5lib')
        for li in soup.find('div', id="comments").find_all('li'):
            cid = li['data-cid']
            comm = li.find('p', class_="comment-content").get_text()

            tmp = li.find('span', class_="comment-info").find_all('span')[0]
            rate = int(tmp['class'][1].replace('allstar', '')) / 10 if tmp.has_attr('class') else ''
            cdate = li.find('span', class_="comment-info").find_all('span')[-1].get_text()
            commments.append([cid, comm, rate, cdate])
    else:
        print(response.status_code)

if __name__ == '__main__':
    for i in range(2, 237):
        crawl_comments(pn=i, item_id='10750155')
        time.sleep(np.random.randint(2, 5))

    col = ['ID', 'Comment', 'Rate', 'Date']
    df = pd.DataFrame(commments, columns=col)
    df.to_excel(excel_writer='./数学之美.xlsx', sheet_name='DoubanBook', index=False)
<|endoftext|>


<commit_before>#!/usr/bin/env python
try :
    from collections import Counter
except ImportError :
    from counterbackport import Counter
from copy import deepcopy
from itertools import groupby

def group_rows(column_names, rows, grouped_columns_ids, aggregations):
    """
    Given a list of rows (header row first), group by given column ids
    grouped_column_ids should be zero indexed
    """
    #Define key fn that makes tuple of given indices from row
    keyfn = lambda x: tuple(x[i] for i in grouped_columns_ids)

    yield list(keyfn(column_names)) + map(
        lambda a: a.get_column_name(column_names), aggregations)

    #Group
    for group, group_rows in groupby(rows, key=keyfn):
        aggregations_copy = deepcopy(aggregations)
        for row in group_rows:
            for agg in aggregations_copy:
                agg.take_row(row)
        yield list(group) + map(lambda a: a.get_result(), aggregations_copy)

class Aggregator(object):
    cast_type = None
    name_format = '%s'

    def __init__(self, column_id):
        self.column_id = column_id
        self.result = None

    def get_value(self, row):
        if self.cast_type:
            return self.cast_type(row[self.column_id])
        else:
            return row[self.column_id]

    def take_row(self, row):
        raise NotImplementedError()

    def get_result(self):
        return self.result

    def get_column_name(self, column_names):
        return self.name_format % column_names[self.column_id]

class CommonAggregator(Aggregator):
    """Returns most common value from column"""

    name_format = '%s'

    def __init__(self, column_id):
        super(CommonAggregator, self).__init__(column_id)
        self.counter = Counter()

    def take_row(self, row):
        self.counter[self.get_value(row)] += 1

    def get_result(self):
        return self.counter.most_common(1)[0][0]

class FunAggregator(Aggregator):
    cast_type = int
    name_format = '(%s)'
    fun = None

    def take_row(self, row):
        value = self.get_value(row)
        if self.result is not None:
            self.result = self.fun(value, self.result)
        else:
            self.result = value

class MaxAggregator(FunAggregator):
    name_format = 'max(%s)'
    fun = staticmethod(max)

class MinAggregator(FunAggregator):
    name_format = 'min(%s)'
    fun = staticmethod(min)

class SumAggregator(FunAggregator):
    name_format = 'sum(%s)'
    fun = staticmethod(lambda a, b: a + b)

class ConditionCountAggregator(Aggregator):
    def condition(self, value):
        return True

    def take_row(self, row):
        value = self.get_value(row)
        if self.result is not None:
            if self.condition(value):
                self.result += 1
        else:
            if self.condition(value):
                self.result = 1
            else:
                self.result = 0

class CountAggregator(ConditionCountAggregator):
    name_format = 'count(%s)'
    pass

class CountAAggregator(ConditionCountAggregator):
    name_format = 'countA(%s)'
    cast_type = int

    def condition(self, value):
        return value > 0

aggregate_functions = {
    'min': MinAggregator,
    'max': MaxAggregator,
    'sum': SumAggregator,
    'count': CountAggregator,
    'countA': CountAAggregator,
    'common': CommonAggregator,

}
<commit_msg>sumf aggregator<commit_after>#!/usr/bin/env python
try :
    from collections import Counter
except ImportError :
    from counterbackport import Counter
from copy import deepcopy
from itertools import groupby

def group_rows(column_names, rows, grouped_columns_ids, aggregations):
    """
    Given a list of rows (header row first), group by given column ids
    grouped_column_ids should be zero indexed
    """
    #Define key fn that makes tuple of given indices from row
    keyfn = lambda x: tuple(x[i] for i in grouped_columns_ids)

    yield list(keyfn(column_names)) + map(
        lambda a: a.get_column_name(column_names), aggregations)

    #Group
    for group, group_rows in groupby(rows, key=keyfn):
        aggregations_copy = deepcopy(aggregations)
        for row in group_rows:
            for agg in aggregations_copy:
                agg.take_row(row)
        yield list(group) + map(lambda a: a.get_result(), aggregations_copy)

class Aggregator(object):
    cast_type = None
    name_format = '%s'

    def __init__(self, column_id):
        self.column_id = column_id
        self.result = None

    def get_value(self, row):
        if self.cast_type:
            return self.cast_type(row[self.column_id])
        else:
            return row[self.column_id]

    def take_row(self, row):
        raise NotImplementedError()

    def get_result(self):
        return self.result

    def get_column_name(self, column_names):
        return self.name_format % column_names[self.column_id]

class CommonAggregator(Aggregator):
    """Returns most common value from column"""

    name_format = '%s'

    def __init__(self, column_id):
        super(CommonAggregator, self).__init__(column_id)
        self.counter = Counter()

    def take_row(self, row):
        self.counter[self.get_value(row)] += 1

    def get_result(self):
        return self.counter.most_common(1)[0][0]

class FunAggregator(Aggregator):
    cast_type = int
    name_format = '(%s)'
    fun = None

    def take_row(self, row):
        value = self.get_value(row)
        if self.result is not None:
            self.result = self.fun(value, self.result)
        else:
            self.result = value

class MaxAggregator(FunAggregator):
    name_format = 'max(%s)'
    fun = staticmethod(max)

class MinAggregator(FunAggregator):
    name_format = 'min(%s)'
    fun = staticmethod(min)

class SumAggregator(FunAggregator):
    name_format = 'sum(%s)'
    fun = staticmethod(lambda a, b: a + b)

class SumfAggregator(FunAggregator):
    cast_type = float
    name_format = 'sumf(%s)'
    fun = staticmethod(lambda a, b: a + b)

class ConditionCountAggregator(Aggregator):
    def condition(self, value):
        return True

    def take_row(self, row):
        value = self.get_value(row)
        if self.result is not None:
            if self.condition(value):
                self.result += 1
        else:
            if self.condition(value):
                self.result = 1
            else:
                self.result = 0

class CountAggregator(ConditionCountAggregator):
    name_format = 'count(%s)'
    pass

class CountAAggregator(ConditionCountAggregator):
    name_format = 'countA(%s)'
    cast_type = int

    def condition(self, value):
        return value > 0

aggregate_functions = {
    'min': MinAggregator,
    'max': MaxAggregator,
    'sum': SumAggregator,
    'sumf': SumfAggregator,
    'count': CountAggregator,
    'countA': CountAAggregator,
    'common': CommonAggregator,

}
<|endoftext|>


<commit_before>import matplotlib.pyplot as plt
import pandas as pd
from pandas_flavor import register_dataframe_method, register_series_method

@register_series_method
@register_dataframe_method
def plot_cast(df, secondary_y=False, label=None, *args, **kwargs):
    """
    Plot a CTD variable with the index in the y-axis instead of x-axis.

    """

    ax = kwargs.pop("ax", None)
    fignums = plt.get_fignums()
    if ax is None and not fignums:
        ax = plt.axes()
        fig = ax.get_figure()
        fig.set_size_inches((5.25, 6.75))
    else:
        ax = plt.gca()
        fig = plt.gcf()

    figsize = kwargs.pop("figsize", fig.get_size_inches())
    fig.set_size_inches(figsize)

    y_inverted = False
    if not getattr(ax, "y_inverted", False):
        setattr(ax, "y_inverted", True)
        y_inverted = True

    if secondary_y:
        ax = ax.twiny()

    xlabel = getattr(df, "name", None)
    ylabel = getattr(df.index, "name", None)

    if isinstance(df, pd.DataFrame):
        labels = label if label else df.columns
        for k, (col, series) in enumerate(df.iteritems()):
            ax.plot(series, series.index, label=labels[k])
    elif isinstance(df, pd.Series):
        label = label if label else str(df.name)
        ax.plot(df.values, df.index, label=label, *args, **kwargs)

    ax.set_ylabel(ylabel)
    ax.set_xlabel(xlabel)
    if y_inverted and not secondary_y:
        ax.invert_yaxis()
    return ax
<commit_msg>matplotlib recommended way to wrap ax<commit_after>import matplotlib.pyplot as plt
import pandas as pd
from pandas_flavor import register_dataframe_method, register_series_method

@register_series_method
@register_dataframe_method
def plot_cast(df, secondary_y=False, label=None, ax=None, *args, **kwargs):
    """
    Plot a CTD variable with the index in the y-axis instead of x-axis.

    """

    fignums = plt.get_fignums()
    if ax is None and not fignums:
        ax = plt.axes()
        fig = ax.get_figure()
    else:
        ax = plt.gca()
        fig = plt.gcf()

    kwargs.setdefault("figsize", (5.25, 6.75))
    figsize = kwargs.pop("figsize")
    fig.set_size_inches(figsize)

    y_inverted = False
    if not getattr(ax, "y_inverted", False):
        setattr(ax, "y_inverted", True)
        y_inverted = True

    if secondary_y:
        ax = ax.twiny()

    xlabel = getattr(df, "name", None)
    ylabel = getattr(df.index, "name", None)

    if isinstance(df, pd.DataFrame):
        labels = label if label else df.columns
        for k, (col, series) in enumerate(df.iteritems()):
            ax.plot(series, series.index, label=labels[k])
    elif isinstance(df, pd.Series):
        label = label if label else str(df.name)
        ax.plot(df.values, df.index, label=label, *args, **kwargs)

    ax.set_ylabel(ylabel)
    ax.set_xlabel(xlabel)
    if y_inverted and not secondary_y:
        ax.invert_yaxis()
    return ax
<|endoftext|>