<commit_before>import logging
import numpy as np
from PyQt5.QtCore import QObject, pyqtSignal

from line import Line

class ToolBase(QObject):
  cleared = pyqtSignal()
  added = pyqtSignal(Line)

  def __init__(self):
    super().__init__()
    self.lines = []

  def clear(self):
    self.lines = []
    self.cleared.emit()

  def add(self, x, y, name):
    x, y = Line.cleanUp(x, y)
    l = Line(x, y, name)
    self.lines.append(l)
    self.added.emit(l)

  def getLines(self):
    raise NotImplementedError()

class NopTool(ToolBase):
  name = 'Nop'

  def getLines(self):
    return self.lines

class FitTool(NopTool):
  name = 'Fit'

class IADTool(ToolBase):
  name = 'iad'
  label = 'IAD'
  xoffUpdated = pyqtSignal(name='xoffUpdated')
  iadYUpdated = pyqtSignal(name='iadYUpdated')
  peaksUpdated = pyqtSignal(name='peaksUpdated')

  def __init__(self):
    super().__init__()
    self.mode = 'orig'
    self.base = -1
    self.bgsub = None
    self.interp = None
    self.interpdx = 0.1
    self.threshold = 1e-10
    self.lines = None

  def doInterp(self, line, *args):
    return self.interp.do(line, self.interpdx, *args)

  def calcXoff(self, lines, base):
    xoff = [0] * len(lines)

    for _p in range(100):
      X1_, X2_ = self.linesInnerRange(lines, xoff)
      if X1_ == X2_: break

      for i, line in enumerate(lines):
        if line == base:
          continue

        cnt = 0
        while True:
          X1, X2 = self.linesInnerRange(lines, xoff)
          wc1 = self.doInterp(base,               (X1, X2)).weightCenter()
          wc2 = self.doInterp(line.xoff(xoff[i]), (X1, X2)).weightCenter()

          dx = wc1 - wc2
          cnt += 1
          if abs(dx) < self.threshold or cnt > 10:
            break

          xoff[i] += dx

      if X1 == X1_ and X2 == X2_:
        break

    else:
      raise RuntimeError('Maximum loop count exceeded')

    return xoff, X1, X2

  def updatePeaks(self, lines):
    self.peaks = [l.peak() for l in lines]
    self.peaksUpdated.emit()
    return lines

  def linesInnerRange(self, lines, xoff):
    X1 = max([min(l.x+o) for l, o in zip(lines, xoff) if len(l.x) >= 1])
    X2 = min([max(l.x+o) for l, o in zip(lines, xoff) if len(l.x) >= 1])
    return X1, X2

  def getLines(self, mode=None):
    if not self.lines:
      return []

    if mode is None:
      mode = self.mode

    baseidx = self.base
    try:
      base = self.lines[baseidx]
    except IndexError:
      base = self.lines[-1]
      baseidx = -1

    lines = self.lines
    if self.bgsub:
      logging.info('Subtract bg: %s' % self.bgsub.label)
      lines = [l-Line(l.x, self.bgsub.calcY(l, l.x), 'bg') for l in lines]

    if mode == 'orig':
      return self.updatePeaks(lines)

    self.xoff, X1, X2 = self.calcXoff(lines, base)
    self.wc = [self.doInterp(line.xoff(xoff), (X1, X2)).weightCenter()
               for line, xoff in zip(lines, self.xoff)]
    self.xoffUpdated.emit()

    lines_off = [self.doInterp(l.xoff(xoff), (X1, X2)).normalize()
                 for l, xoff in zip(lines, self.xoff)]

    diff = []
    base = lines_off[baseidx]
    for l in lines_off:
      if len(l.x) == 0 or len(base.x) == 0:
        diff.append(Line([], [], l.name))
        continue
      diff.append(l - base)

    x = self.iadX
    y = [sum(np.abs(d.y)) for d in diff]
    self.iadY = y
    self.iadYUpdated.emit()
    IAD = Line(x, y, 'IAD')

    if mode == 'xoff':
      return self.updatePeaks(lines_off)

    if mode == 'diff':
      return diff

    if mode == 'iad':
      return [IAD]

    raise RuntimeError()
<commit_msg>[IAD] Add performance profiler<commit_after>import logging
import numpy as np
from PyQt5.QtCore import QObject, pyqtSignal

from line import Line

class ToolBase(QObject):
  cleared = pyqtSignal()
  added = pyqtSignal(Line)

  def __init__(self):
    super().__init__()
    self.lines = []

  def clear(self):
    self.lines = []
    self.cleared.emit()

  def add(self, x, y, name):
    x, y = Line.cleanUp(x, y)
    l = Line(x, y, name)
    self.lines.append(l)
    self.added.emit(l)

  def getLines(self):
    raise NotImplementedError()

class NopTool(ToolBase):
  name = 'Nop'

  def getLines(self):
    return self.lines

class FitTool(NopTool):
  name = 'Fit'

class IADTool(ToolBase):
  name = 'iad'
  label = 'IAD'
  xoffUpdated = pyqtSignal(name='xoffUpdated')
  iadYUpdated = pyqtSignal(name='iadYUpdated')
  peaksUpdated = pyqtSignal(name='peaksUpdated')

  def __init__(self):
    super().__init__()
    self.mode = 'orig'
    self.base = -1
    self.bgsub = None
    self.interp = None
    self.interpdx = 0.1
    self.threshold = 1e-10
    self.lines = None

  def doInterp(self, line, *args):
    return self.interp.do(line, self.interpdx, *args)

  def calcXoff(self, lines, base):
    import cProfile, pstats, io
    pr = cProfile.Profile()
    pr.enable()

    try:
      xoff = [0] * len(lines)

      for _p in range(len(lines)*2):
        X1_, X2_ = self.linesInnerRange(lines, xoff)
        if X1_ == X2_: break

        for i, line in enumerate(lines):
          if line == base:
            continue

          cnt = 0
          while True:
            X1, X2 = self.linesInnerRange(lines, xoff)
            wc1 = self.doInterp(base,               (X1, X2)).weightCenter()
            wc2 = self.doInterp(line.xoff(xoff[i]), (X1, X2)).weightCenter()

            dx = wc1 - wc2
            cnt += 1
            if abs(dx) < self.threshold or cnt > 10:
              break

            xoff[i] += dx

        if X1 == X1_ and X2 == X2_:
          break

      else:
        raise RuntimeError('Maximum loop count exceeded')

    finally:
      pr.disable()
      s = io.StringIO()
      sortby = 'cumulative'
      ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
      ps.print_stats()
      logging.info(s.getvalue())

    return xoff, X1, X2

  def updatePeaks(self, lines):
    self.peaks = [l.peak() for l in lines]
    self.peaksUpdated.emit()
    return lines

  def linesInnerRange(self, lines, xoff):
    X1 = max([min(l.x+o) for l, o in zip(lines, xoff) if len(l.x) >= 1])
    X2 = min([max(l.x+o) for l, o in zip(lines, xoff) if len(l.x) >= 1])
    return X1, X2

  def getLines(self, mode=None):
    if not self.lines:
      return []

    if mode is None:
      mode = self.mode

    baseidx = self.base
    try:
      base = self.lines[baseidx]
    except IndexError:
      base = self.lines[-1]
      baseidx = -1

    lines = self.lines
    if self.bgsub:
      logging.info('Subtract bg: %s' % self.bgsub.label)
      lines = [l-Line(l.x, self.bgsub.calcY(l, l.x), 'bg') for l in lines]

    if mode == 'orig':
      return self.updatePeaks(lines)

    self.xoff, X1, X2 = self.calcXoff(lines, base)
    self.wc = [self.doInterp(line.xoff(xoff), (X1, X2)).weightCenter()
               for line, xoff in zip(lines, self.xoff)]
    self.xoffUpdated.emit()

    lines_off = [self.doInterp(l.xoff(xoff), (X1, X2)).normalize()
                 for l, xoff in zip(lines, self.xoff)]

    diff = []
    base = lines_off[baseidx]
    for l in lines_off:
      if len(l.x) == 0 or len(base.x) == 0:
        diff.append(Line([], [], l.name))
        continue
      diff.append(l - base)

    x = self.iadX
    y = [sum(np.abs(d.y)) for d in diff]
    self.iadY = y
    self.iadYUpdated.emit()
    IAD = Line(x, y, 'IAD')

    if mode == 'xoff':
      return self.updatePeaks(lines_off)

    if mode == 'diff':
      return diff

    if mode == 'iad':
      return [IAD]

    raise RuntimeError()
<|endoftext|>


<commit_before>"""
This module contains all the classes that interface with pfdminer directly.
"""

import pdfminer.layout
import pdfminer.pdfdevice
import pdfminer.pdfdocument
import pdfminer.pdfinterp
import pdfminer.pdfpage
import pdfminer.pdfparser
import pdfminer.pdftypes
import pdfminer.utils

from .content import Page, Shape, Image, Lettering
from . import color

class ColoredState(pdfminer.pdfinterp.PDFGraphicState):

    """
    Expands the graphic state with fill and stroke color parameters.
    """

    def __init__(self):
        super(ColoredState, self).__init__()
        self.fill_color = None
        self.stroke_color = None

    def copy(self):
        obj = self.__class__()
        obj.linewidth = self.linewidth
        obj.linecap = self.linecap
        obj.linejoin = self.linejoin
        obj.miterlimit = self.miterlimit
        obj.dash = self.dash
        obj.intent = self.intent
        obj.flatness = self.flatness
        obj.fill_color = self.fill_color
        obj.stroke_color = self.stroke_color
        return obj

class StrokeState(object):

    """
    An object that encapsulates the stroking parameters.
    """

    def __init__(self):
        self.color = None
        self.linewidth = 2
        self.linecap = 0
        self.linejoin = 0
        self.miterlimit = 10
        self.dash = ([], 0)
        self.stroke_adjustment = False

    @classmethod
    def from_gs(cls, graphics):
        "Creates a new StrokeState from a ColoredState object."
        ret = cls()
        ret.color = graphics.stroke_color
        ret.linewidth = graphics.linewidth
        ret.linecap = graphics.linecap
        ret.linejoin = graphics.linejoin
        ret.miterlimit = graphics.miterlimit
        ret.dash = graphics.dash
        return ret

    def __repr__(self):
        return ("<%s: color=%r, line-width=%r, line-cap=%r "
                + "line-join=%r miter-limit=%r dash-pattern=%r>") %(
                    self.__class__.__name__, self.color, self.linewidth,
                    self.linecap, self.linejoin, self.miterlimit,
                    self.dash)

class FillState(object):

    """
    An object that encapsulates the fill parameters.
    """

    def __init__(self):
        self.color = None

    @classmethod
    def from_gs(cls, graphics):
        "Creates a new FillState from a ColoredState object."
        ret = cls()
        ret.color = graphics.fill_color
        return ret

    def __repr__(self):
        return ("<%s: color=%r>") % (self.__class__.__name__, self.color)

class ColoredInterpreter(pdfminer.pdfinterp.PDFPageInterpreter):

    """
    A PDF interpreter that can handle color commands.
    """

    # The ColoredInterpreter extends the PDFPageInterpreter found in pdfminer
    # by keeping track of the stroke and fill colors. It keeps a `ColorSpace`
    # for both stroke and fill operations, and uses it to create `Color`
    # objects for each of stroke and fill.
    #
    # The ColoredInterpreter manages the current color spaces the same way
    # that the PDFPageInterpreter does (pdfminer needs to care about the
    # number of components in the color space, so it has a thin version of
    # color spaces already). To plug into this machinery, it overrides the
    # setting of `self.csmap` in `self.init_resources()`. csmap is a
    # dictionary mapping names as found in the /Resources dict of a page to
    # instances of the colorspaces, initialized according to the parameters
    # found in /Resources.

    def __init__(self, *args, **kwargs):
        super(ColoredInterpreter, self).__init__(*args, **kwargs)
        # This is here to allow for independent testing of init_state and
        # init_resources, as well as to avoid pylint warnings ;)
        self.csmap = {}
        self.graphicstate = None

    def init_state(self, ctm):
        # Extends the parent method to install our custom graphic state
        super(ColoredInterpreter, self).init_state(ctm)
        self.graphicstate = ColoredState()

    def init_resources(self, resources):
        # Extends the parent method to install our custom color spaces
        if resources:
            resources = pdfminer.pdftypes.dict_value(resources)
            spaces = resources.pop('ColorSpace', {})
        else:
            spaces = {}
        super(ColoredInterpreter, self).init_resources(resources)
        self.csmap.clear()
        # Per the PDF spec, (p. 287), "The names DeviceGray, DeviceRGB,
        # DeviceCMYK, and Pattern always identify the corresponding color
        # spaces directly; they never refer to resources in the ColorSpace
        # subdictionary." We implement this behavior by overriding any
        # entries in the csmap with this name with the original color spaces.
        for csname, spec in pdfminer.pdftypes.dict_value(spaces).iteritems():
            if isinstance(spec, list):
                name = pdfminer.psparser.literal_name(spec[0])
                params = spec[1:]
            else:
                name = pdfminer.psparser.literal_name(spec)
                params = []
            self.csmap[csname] = color.FAMILIES[name].make_space(params)
        self.csmap.update(
            (name, color.FAMILIES[name].make_space())
            for name in ('DeviceGray', 'DeviceRGB', 'DeviceCMYK')
        )
        # The next loop ensures that device color spaces are overriden by
        # their defaults, if any
        for csname in ('DefaultGray', 'DefaultRGB', 'DefaultCMYK'):
            try:
                space = self.csmap[csname]
            except KeyError:
                pass
            else:
                self.csmap[csname.replace('Default', 'Device')] = space

    # setgray-stroking
    def do_G(self, gray):
        self.do_CS(pdfminer.pdfinterp.LITERAL_DEVICE_GRAY)
        self.graphicstate.stroke_color = self.scs.make_color((gray,))

    # setgray-non-stroking
    def do_g(self, gray):
        self.do_cs(pdfminer.pdfinterp.LITERAL_DEVICE_GRAY)
        self.graphicstate.fill_color = self.ncs.make_color((gray,))

    # setrgb-stroking
    def do_RG(self, r, g, b):
        self.do_CS(pdfminer.pdfinterp.LITERAL_DEVICE_RGB)
        self.graphicstate.stroke_color = self.scs.make_color((r, g, b))

    # setrgb-non-stroking
    def do_rg(self, r, g, b):
        self.do_cs(pdfminer.pdfinterp.LITERAL_DEVICE_RGB)
        self.graphicstate.fill_color = self.ncs.make_color((r, g, b))

    # setcmyk-stroking
    def do_K(self, c, m, y, k):
        self.do_CS(pdfminer.pdfinterp.LITERAL_DEVICE_CMYK)
        self.graphicstate.stroke_color = self.scs.make_color((c, m, y, k))

    # setcmyk-non-stroking
    def do_k(self, c, m, y, k):
        self.do_cs(pdfminer.pdfinterp.LITERAL_DEVICE_CMYK)
        self.graphicstate.fill_color = self.ncs.make_color((c, m, y, k))

    # setcolor-stroking
    def do_SCN(self):
        if self.scs:
            samples = self.scs.ncomponents
        else:
            raise pdfminer.pdfinterp.PDFInterpreterError(
                'No colorspace specified!')
        self.graphicstate.stroke_color = self.scs.make_color(
            self.pop(samples))

    # setcolor-non-stroking
    def do_scn(self):
        if self.ncs:
            samples = self.ncs.ncomponents
        else:
            raise pdfminer.pdfinterp.PDFInterpreterError(
                'No colorspace specified!')
        self.graphicstate.fill_color = self.ncs.make_color(self.pop(samples))

class DeviceLoader(pdfminer.pdfdevice.PDFTextDevice):

    """
    An interpreter that creates `Page` objects.
    """

    def __init__(self, rsrcmgr):
        super(DeviceLoader, self).__init__(rsrcmgr)
        self.page = None
        self.str_container = None
        self.unit = 1

    def __repr__(self):
        return object.__repr__(self)

    def begin_page(self, page, ctm):
        self.page = Page(page)
        self.unit = pdfminer.pdftypes.resolve1(page.attrs.get('UserUnit', 1))

    def set_ctm(self, ctm):
        # pdfminer adjusts the ctm for the page rotation and MediaBox,
        # so we just need to adjust for the UserUnit
        self.ctm = tuple(c * self.unit for c in ctm)

    def paint_path(self, graphicstate, stroked, filled, evenodd, path):
        # Converts path to device coordinates and adds the path to the page
        device_path = []
        for segment in path:
            coords = iter(segment[1:])
            for x in coords:  #pylint: disable=C0103
                y = next(coords)  #pylint: disable=C0103
                device_path.append(
                    (segment[0],)
                    + pdfminer.utils.apply_matrix_pt(self.ctm, (x, y)))
        stroke = StrokeState.from_gs(graphicstate) if stroked else None
        fill = FillState.from_gs(graphicstate) if filled else None
        self.page.add_shape(Shape(stroke, fill, evenodd, device_path))

    def render_image(self, name, stream):
        self.page.add_image(Image(self.ctm, stream))

    def render_string_horizontal(self, *args):
        return self.render_string_hv('horizontal', *args)

    def render_string_vertical(self, *args):
        return self.render_string_hv('vertical', *args)

    def render_char(self, matrix, font, fontsize, scaling, rise, cid):
        # Essentials copied from
        # pdfminer.converter.PDFLayoutAnalyzer.render_char
        text = font.to_unichr(cid)
        textwidth = font.char_width(cid)
        textdisp = font.char_disp(cid)
        item = pdfminer.layout.LTChar(matrix, font, fontsize, scaling, rise,
                                      text, textwidth, textdisp)
        self.str_container.add(item)
        return item.adv

    def render_string_hv(self, hv, seq, matrix, vec, font, fontsize,
                         scaling, charspace, wordspace, rise,
                         dxscale):
        """
        Calculate the bounding box in user coordinates for a string.

        `hv` -- one of 'horizontal' or 'vertical', for the type of string to
                render
        `seq` -- The array of strings/numbers to render
        `matrix` -- The matrix mapping text coordinates to user coordinates,
                    (T_m x CTM, part of T_rm)
        `vec` -- the user coordinates of the text origin
        `font` -- the font to use for rendering the string
        `fontsize` -- the fontsize in the text state
        `scaling` -- the horizontal scaling factor (Tz * .01)
        `charspace` -- additional space to insert b/ween characters, scaled
                       (Tc * Tz * .01) in text coordinates
        `wordspace` -- additional space to insert b/ween words, scaled
                       (Tw * Tz * .01) in text coordinates
        `rise` -- the text rise parameter, scaled (Ts * Tz * .01) in text
                  coordinates
        `dxscale` -- the size in user coordinates to skip when rendering `1`.

        """
        vec = list(vec)
        hv = ('horizontal', 'vertical').index(hv)
        needcharspace = False
        for obj in seq:
            if pdfminer.utils.isnumber(obj):
                vec[hv] -= obj * dxscale
                needcharspace = True
            else:
                string = []
                self.str_container = pdfminer.layout.LTExpandableContainer()
                for cid in font.decode(obj):
                    if needcharspace:
                        vec[hv] += charspace
                    vec[hv] += self.render_char(
                        pdfminer.utils.translate_matrix(matrix, vec),
                        font, fontsize, scaling, rise, cid)
                    if cid == 32 and wordspace:
                        vec[hv] += wordspace
                    needcharspace = True
                    string.append(font.to_unichr(cid))
                self.page.add_lettering(Lettering(
                    u''.join(string), self.str_container.bbox, hv == 0))
                self.str_container = None
        return tuple(vec)

class Document(object):

    """
    An in-memory PDF document.
    """

    def __init__(self, pdffile):
        res_mgr = pdfminer.pdfinterp.PDFResourceManager()
        self.device = DeviceLoader(res_mgr)
        self.interpreter = ColoredInterpreter(res_mgr, self.device)
        self.parser = pdfminer.pdfparser.PDFParser(pdffile)
        self.doc = pdfminer.pdfdocument.PDFDocument(self.parser, caching=True)

    def iter_pages(self):
        "Iterate through all the pages in a document."
        for page in pdfminer.pdfpage.PDFPage.create_pages(self.doc):
            self.interpreter.process_page(page)
            yield self.device.page

    def get_page(self, num):
        """
        Get a specific page in the document.

        The number refers to the 0-based index of the page in the document
        display order, not the numbering system used in the document.

        """
        for i, page in enumerate(
                pdfminer.pdfpage.PDFPage.create_pages(self.doc)):
            if i == num:
                self.interpreter.process_page(page)
                return self.device.page
<commit_msg>fixed application of CTM to path points<commit_after>"""
This module contains all the classes that interface with pfdminer directly.
"""

import pdfminer.layout
import pdfminer.pdfdevice
import pdfminer.pdfdocument
import pdfminer.pdfinterp
import pdfminer.pdfpage
import pdfminer.pdfparser
import pdfminer.pdftypes
import pdfminer.utils

from .content import Page, Shape, Image, Lettering
from . import color

class ColoredState(pdfminer.pdfinterp.PDFGraphicState):

    """
    Expands the graphic state with fill and stroke color parameters.
    """

    def __init__(self):
        super(ColoredState, self).__init__()
        self.fill_color = None
        self.stroke_color = None

    def copy(self):
        obj = self.__class__()
        obj.linewidth = self.linewidth
        obj.linecap = self.linecap
        obj.linejoin = self.linejoin
        obj.miterlimit = self.miterlimit
        obj.dash = self.dash
        obj.intent = self.intent
        obj.flatness = self.flatness
        obj.fill_color = self.fill_color
        obj.stroke_color = self.stroke_color
        return obj

class StrokeState(object):

    """
    An object that encapsulates the stroking parameters.
    """

    def __init__(self):
        self.color = None
        self.linewidth = 2
        self.linecap = 0
        self.linejoin = 0
        self.miterlimit = 10
        self.dash = ([], 0)
        self.stroke_adjustment = False

    @classmethod
    def from_gs(cls, graphics):
        "Creates a new StrokeState from a ColoredState object."
        ret = cls()
        ret.color = graphics.stroke_color
        ret.linewidth = graphics.linewidth
        ret.linecap = graphics.linecap
        ret.linejoin = graphics.linejoin
        ret.miterlimit = graphics.miterlimit
        ret.dash = graphics.dash
        return ret

    def __repr__(self):
        return ("<%s: color=%r, line-width=%r, line-cap=%r "
                + "line-join=%r miter-limit=%r dash-pattern=%r>") %(
                    self.__class__.__name__, self.color, self.linewidth,
                    self.linecap, self.linejoin, self.miterlimit,
                    self.dash)

class FillState(object):

    """
    An object that encapsulates the fill parameters.
    """

    def __init__(self):
        self.color = None

    @classmethod
    def from_gs(cls, graphics):
        "Creates a new FillState from a ColoredState object."
        ret = cls()
        ret.color = graphics.fill_color
        return ret

    def __repr__(self):
        return ("<%s: color=%r>") % (self.__class__.__name__, self.color)

class ColoredInterpreter(pdfminer.pdfinterp.PDFPageInterpreter):

    """
    A PDF interpreter that can handle color commands.
    """

    # The ColoredInterpreter extends the PDFPageInterpreter found in pdfminer
    # by keeping track of the stroke and fill colors. It keeps a `ColorSpace`
    # for both stroke and fill operations, and uses it to create `Color`
    # objects for each of stroke and fill.
    #
    # The ColoredInterpreter manages the current color spaces the same way
    # that the PDFPageInterpreter does (pdfminer needs to care about the
    # number of components in the color space, so it has a thin version of
    # color spaces already). To plug into this machinery, it overrides the
    # setting of `self.csmap` in `self.init_resources()`. csmap is a
    # dictionary mapping names as found in the /Resources dict of a page to
    # instances of the colorspaces, initialized according to the parameters
    # found in /Resources.

    def __init__(self, *args, **kwargs):
        super(ColoredInterpreter, self).__init__(*args, **kwargs)
        # This is here to allow for independent testing of init_state and
        # init_resources, as well as to avoid pylint warnings ;)
        self.csmap = {}
        self.graphicstate = None

    def init_state(self, ctm):
        # Extends the parent method to install our custom graphic state
        super(ColoredInterpreter, self).init_state(ctm)
        self.graphicstate = ColoredState()

    def init_resources(self, resources):
        # Extends the parent method to install our custom color spaces
        if resources:
            resources = pdfminer.pdftypes.dict_value(resources)
            spaces = resources.pop('ColorSpace', {})
        else:
            spaces = {}
        super(ColoredInterpreter, self).init_resources(resources)
        self.csmap.clear()
        # Per the PDF spec, (p. 287), "The names DeviceGray, DeviceRGB,
        # DeviceCMYK, and Pattern always identify the corresponding color
        # spaces directly; they never refer to resources in the ColorSpace
        # subdictionary." We implement this behavior by overriding any
        # entries in the csmap with this name with the original color spaces.
        for csname, spec in pdfminer.pdftypes.dict_value(spaces).iteritems():
            if isinstance(spec, list):
                name = pdfminer.psparser.literal_name(spec[0])
                params = spec[1:]
            else:
                name = pdfminer.psparser.literal_name(spec)
                params = []
            self.csmap[csname] = color.FAMILIES[name].make_space(params)
        self.csmap.update(
            (name, color.FAMILIES[name].make_space())
            for name in ('DeviceGray', 'DeviceRGB', 'DeviceCMYK')
        )
        # The next loop ensures that device color spaces are overriden by
        # their defaults, if any
        for csname in ('DefaultGray', 'DefaultRGB', 'DefaultCMYK'):
            try:
                space = self.csmap[csname]
            except KeyError:
                pass
            else:
                self.csmap[csname.replace('Default', 'Device')] = space

    # setgray-stroking
    def do_G(self, gray):
        self.do_CS(pdfminer.pdfinterp.LITERAL_DEVICE_GRAY)
        self.graphicstate.stroke_color = self.scs.make_color((gray,))

    # setgray-non-stroking
    def do_g(self, gray):
        self.do_cs(pdfminer.pdfinterp.LITERAL_DEVICE_GRAY)
        self.graphicstate.fill_color = self.ncs.make_color((gray,))

    # setrgb-stroking
    def do_RG(self, r, g, b):
        self.do_CS(pdfminer.pdfinterp.LITERAL_DEVICE_RGB)
        self.graphicstate.stroke_color = self.scs.make_color((r, g, b))

    # setrgb-non-stroking
    def do_rg(self, r, g, b):
        self.do_cs(pdfminer.pdfinterp.LITERAL_DEVICE_RGB)
        self.graphicstate.fill_color = self.ncs.make_color((r, g, b))

    # setcmyk-stroking
    def do_K(self, c, m, y, k):
        self.do_CS(pdfminer.pdfinterp.LITERAL_DEVICE_CMYK)
        self.graphicstate.stroke_color = self.scs.make_color((c, m, y, k))

    # setcmyk-non-stroking
    def do_k(self, c, m, y, k):
        self.do_cs(pdfminer.pdfinterp.LITERAL_DEVICE_CMYK)
        self.graphicstate.fill_color = self.ncs.make_color((c, m, y, k))

    # setcolor-stroking
    def do_SCN(self):
        if self.scs:
            samples = self.scs.ncomponents
        else:
            raise pdfminer.pdfinterp.PDFInterpreterError(
                'No colorspace specified!')
        self.graphicstate.stroke_color = self.scs.make_color(
            self.pop(samples))

    # setcolor-non-stroking
    def do_scn(self):
        if self.ncs:
            samples = self.ncs.ncomponents
        else:
            raise pdfminer.pdfinterp.PDFInterpreterError(
                'No colorspace specified!')
        self.graphicstate.fill_color = self.ncs.make_color(self.pop(samples))

class DeviceLoader(pdfminer.pdfdevice.PDFTextDevice):

    """
    An interpreter that creates `Page` objects.
    """

    def __init__(self, rsrcmgr):
        super(DeviceLoader, self).__init__(rsrcmgr)
        self.page = None
        self.str_container = None
        self.unit = 1

    def __repr__(self):
        return object.__repr__(self)

    def begin_page(self, page, ctm):
        self.page = Page(page)
        self.unit = pdfminer.pdftypes.resolve1(page.attrs.get('UserUnit', 1))

    def set_ctm(self, ctm):
        # pdfminer adjusts the ctm for the page rotation and MediaBox,
        # so we just need to adjust for the UserUnit
        self.ctm = tuple(c * self.unit for c in ctm)

    def paint_path(self, graphicstate, stroked, filled, evenodd, path):
        # Converts path to device coordinates and adds the path to the page
        device_path = []
        for segment in path:
            coords = iter(segment[1:])
            new_seg = [segment[0]]
            for x in coords:  #pylint: disable=C0103
                y = next(coords)  #pylint: disable=C0103
                new_seg.extend(pdfminer.utils.apply_matrix_pt(self.ctm,
                                                              (x, y)))
            device_path.append(tuple(new_seg))
        stroke = StrokeState.from_gs(graphicstate) if stroked else None
        fill = FillState.from_gs(graphicstate) if filled else None
        self.page.add_shape(Shape(stroke, fill, evenodd, device_path))

    def render_image(self, name, stream):
        self.page.add_image(Image(self.ctm, stream))

    def render_string_horizontal(self, *args):
        return self.render_string_hv('horizontal', *args)

    def render_string_vertical(self, *args):
        return self.render_string_hv('vertical', *args)

    def render_char(self, matrix, font, fontsize, scaling, rise, cid):
        # Essentials copied from
        # pdfminer.converter.PDFLayoutAnalyzer.render_char
        text = font.to_unichr(cid)
        textwidth = font.char_width(cid)
        textdisp = font.char_disp(cid)
        item = pdfminer.layout.LTChar(matrix, font, fontsize, scaling, rise,
                                      text, textwidth, textdisp)
        self.str_container.add(item)
        return item.adv

    def render_string_hv(self, hv, seq, matrix, vec, font, fontsize,
                         scaling, charspace, wordspace, rise,
                         dxscale):
        """
        Calculate the bounding box in user coordinates for a string.

        `hv` -- one of 'horizontal' or 'vertical', for the type of string to
                render
        `seq` -- The array of strings/numbers to render
        `matrix` -- The matrix mapping text coordinates to user coordinates,
                    (T_m x CTM, part of T_rm)
        `vec` -- the user coordinates of the text origin
        `font` -- the font to use for rendering the string
        `fontsize` -- the fontsize in the text state
        `scaling` -- the horizontal scaling factor (Tz * .01)
        `charspace` -- additional space to insert b/ween characters, scaled
                       (Tc * Tz * .01) in text coordinates
        `wordspace` -- additional space to insert b/ween words, scaled
                       (Tw * Tz * .01) in text coordinates
        `rise` -- the text rise parameter, scaled (Ts * Tz * .01) in text
                  coordinates
        `dxscale` -- the size in user coordinates to skip when rendering `1`.

        """
        vec = list(vec)
        hv = ('horizontal', 'vertical').index(hv)
        needcharspace = False
        for obj in seq:
            if pdfminer.utils.isnumber(obj):
                vec[hv] -= obj * dxscale
                needcharspace = True
            else:
                string = []
                self.str_container = pdfminer.layout.LTExpandableContainer()
                for cid in font.decode(obj):
                    if needcharspace:
                        vec[hv] += charspace
                    vec[hv] += self.render_char(
                        pdfminer.utils.translate_matrix(matrix, vec),
                        font, fontsize, scaling, rise, cid)
                    if cid == 32 and wordspace:
                        vec[hv] += wordspace
                    needcharspace = True
                    string.append(font.to_unichr(cid))
                self.page.add_lettering(Lettering(
                    u''.join(string), self.str_container.bbox, hv == 0))
                self.str_container = None
        return tuple(vec)

class Document(object):

    """
    An in-memory PDF document.
    """

    def __init__(self, pdffile):
        res_mgr = pdfminer.pdfinterp.PDFResourceManager()
        self.device = DeviceLoader(res_mgr)
        self.interpreter = ColoredInterpreter(res_mgr, self.device)
        self.parser = pdfminer.pdfparser.PDFParser(pdffile)
        self.doc = pdfminer.pdfdocument.PDFDocument(self.parser, caching=True)

    def iter_pages(self):
        "Iterate through all the pages in a document."
        for page in pdfminer.pdfpage.PDFPage.create_pages(self.doc):
            self.interpreter.process_page(page)
            yield self.device.page

    def get_page(self, num):
        """
        Get a specific page in the document.

        The number refers to the 0-based index of the page in the document
        display order, not the numbering system used in the document.

        """
        for i, page in enumerate(
                pdfminer.pdfpage.PDFPage.create_pages(self.doc)):
            if i == num:
                self.interpreter.process_page(page)
                return self.device.page
<|endoftext|>


<commit_before>"""Provides an interface for interacting with the model."""

import argparse
import msvcrt
import os
import shutil
import sys

import numpy as np
import tensorflow as tf

import architecture
import constants as c
from preprocessing import ImagePreprocessor

def train(steps, resuming):
    """
    Trains the model and saves the result.

    # Parameters
        steps (int):
            - Amount of images to train on.
        resuming (bool):
            - Whether to train from scratch or resume training from a saved model.
    """
    # TensorBoard refuses to simply overwrite old data, so this is necessary.
    if c.TENSORBOARD_DIR in os.listdir():
        shutil.rmtree(c.TENSORBOARD_DIR)
    with tf.Session() as sess:
        if resuming:
            loader = tf.train.import_meta_graph(c.SAVEMODEL_DIR + '.meta')
            loader.restore(sess, c.SAVEMODEL_DIR)
            graph = tf.get_default_graph()
            input_ = graph.get_tensor_by_name('input:0')
            label = graph.get_tensor_by_name('label:0')
            optimizer = graph.get_operation_by_name('optimizer')
            graph.get_operation_by_name('objective_summary')
        else:  # Else, we need to build the graph from scratch!
            input_ = tf.placeholder(tf.float32, shape=[1, c.ROWS, c.COLS, c.CHAN], name='input')
            model = architecture.model(input_, name='model')
            label = tf.placeholder(tf.float32, shape=c.LABEL_SHAPE, name='label')
            objective = tf.reduce_mean(tf.squared_difference(label, model), name='objective')
            optimizer = tf.train.MomentumOptimizer(c.LR, c.DC, use_nesterov=True,
                                                   name='optimizer').minimize(objective)
            tf.summary.scalar('objective_summary', objective)
            sess.run(tf.global_variables_initializer())
        summary = tf.summary.merge_all()
        writer = tf.summary.FileWriter(c.TENSORBOARD_DIR, graph=tf.get_default_graph())
        preprocessor = ImagePreprocessor([c.COLS, c.ROWS], c.COLOR_SPACE)
        for step, input_arg, label_arg in preprocessor.preprocess_classes(steps, c.TRAIN_DIR,
                                                                          c.ENCODING):
            print('Step: {}/{}'.format(step, steps))
            sess.run(optimizer, feed_dict={input_: input_arg, label: label_arg})
            step_summary = sess.run(summary, feed_dict={input_: input_arg, label: label_arg})
            writer.add_summary(step_summary, global_step=step)
        tf.train.Saver().save(sess, c.SAVEMODEL_DIR)
        # A sound to signal the end of training.
        print('\a')

def classify(path):
    """
    Does one of 3 things:
    1. Given a path to an image file on disk (WITH A FILE-EXTENSION), classifies it.
    2. Given a path to a directory on disk, classifies all images found in it (excluding
       subdirectories and files with unsupported formats).
    3. Given a URL to an image, classifies it.

    # Parameters
        path (str):
            - Can be a normal path to an image on disk.
            - Can also be a URL that returns an image.
    # Returns
        - If given path to an image file on disk, or a URL to an image, returns a string that is
          either 'cat' or 'dog'.
        - If given path to a directory, returns a dictionary {'filename': 'guessed animal'}
    """
    preprocessor = ImagePreprocessor([c.COLS, c.ROWS], c.COLOR_SPACE)
    with tf.Session() as sess:
        loader = tf.train.import_meta_graph(c.SAVEMODEL_DIR + '.meta')
        loader.restore(sess, c.SAVEMODEL_DIR)
        graph = tf.get_default_graph()
        input_ = graph.get_tensor_by_name('input:0')
        model_output = graph.get_tensor_by_name('model/output:0')
        if os.path.isdir(path):
            results = {}
            for image_name, preprocessed_image in preprocessor.preprocess_directory(path):
                input_arg = np.expand_dims(preprocessed_image, axis=0)
                result = sess.run(model_output, feed_dict={input_: input_arg})
                if np.argmax(result) == 0:
                    results[image_name] = 'cat'
                else:
                    results[image_name] = 'dog'
            return results
        # Else, `path` is either a file on disk or a URL.
        input_arg = np.expand_dims(preprocessor.preprocess_image(path), axis=0)
        result = sess.run(model_output, feed_dict={input_: input_arg})
        if np.argmax(result) == np.argmax(c.ENCODING['cats']):
            return 'cat'
        return 'dog'

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Need help? See: https://github.com/MarxSoul55/cats_vs_dogs#using-the-cli')
    parser.add_argument('--train', action='store_true')
    parser.add_argument('--resuming', action='store_true')
    parser.add_argument('--steps', type=int)
    parser.add_argument('--classify', action='store_true')
    parser.add_argument('--source')
    parser.set_defaults(resuming=False)
    args = parser.parse_args()
    if args.train:
        print('WARNING: Training will overwrite the saved model (if it exists). EXECUTE Y/N?')
        if msvcrt.getch().decode().lower() == 'y':
            train(args.steps, args.resuming)
        else:
            sys.exit('Program closed.')
    elif args.classify:
        print(classify(args.source))
<commit_msg>Gonna test out Adam<commit_after>"""Provides an interface for interacting with the model."""

import argparse
import msvcrt
import os
import shutil
import sys

import numpy as np
import tensorflow as tf

import architecture
import constants as c
from preprocessing import ImagePreprocessor

def train(steps, resuming):
    """
    Trains the model and saves the result.

    # Parameters
        steps (int):
            - Amount of images to train on.
        resuming (bool):
            - Whether to train from scratch or resume training from a saved model.
    """
    # TensorBoard refuses to simply overwrite old data, so this is necessary.
    if c.TENSORBOARD_DIR in os.listdir():
        shutil.rmtree(c.TENSORBOARD_DIR)
    with tf.Session() as sess:
        if resuming:
            loader = tf.train.import_meta_graph(c.SAVEMODEL_DIR + '.meta')
            loader.restore(sess, c.SAVEMODEL_DIR)
            graph = tf.get_default_graph()
            input_ = graph.get_tensor_by_name('input:0')
            label = graph.get_tensor_by_name('label:0')
            optimizer = graph.get_operation_by_name('optimizer')
            graph.get_operation_by_name('objective_summary')
        else:  # Else, we need to build the graph from scratch!
            input_ = tf.placeholder(tf.float32, shape=[1, c.ROWS, c.COLS, c.CHAN], name='input')
            model = architecture.model(input_, name='model')
            label = tf.placeholder(tf.float32, shape=c.LABEL_SHAPE, name='label')
            objective = tf.reduce_mean(tf.squared_difference(label, model), name='objective')
            optimizer = tf.train.AdamOptimizer(learning_rate=0.001, beta1=0.9, beta2=0.999,
                                               epsilon=1.0, name='optimizer').minimize(objective)
            tf.summary.scalar('objective_summary', objective)
            sess.run(tf.global_variables_initializer())
        summary = tf.summary.merge_all()
        writer = tf.summary.FileWriter(c.TENSORBOARD_DIR, graph=tf.get_default_graph())
        preprocessor = ImagePreprocessor([c.COLS, c.ROWS], c.COLOR_SPACE)
        for step, input_arg, label_arg in preprocessor.preprocess_classes(steps, c.TRAIN_DIR,
                                                                          c.ENCODING):
            print('Step: {}/{}'.format(step, steps))
            sess.run(optimizer, feed_dict={input_: input_arg, label: label_arg})
            step_summary = sess.run(summary, feed_dict={input_: input_arg, label: label_arg})
            writer.add_summary(step_summary, global_step=step)
        tf.train.Saver().save(sess, c.SAVEMODEL_DIR)
        # A sound to signal the end of training.
        print('\a')

def classify(path):
    """
    Does one of 3 things:
    1. Given a path to an image file on disk (WITH A FILE-EXTENSION), classifies it.
    2. Given a path to a directory on disk, classifies all images found in it (excluding
       subdirectories and files with unsupported formats).
    3. Given a URL to an image, classifies it.

    # Parameters
        path (str):
            - Can be a normal path to an image on disk.
            - Can also be a URL that returns an image.
    # Returns
        - If given path to an image file on disk, or a URL to an image, returns a string that is
          either 'cat' or 'dog'.
        - If given path to a directory, returns a dictionary {'filename': 'guessed animal'}
    """
    preprocessor = ImagePreprocessor([c.COLS, c.ROWS], c.COLOR_SPACE)
    with tf.Session() as sess:
        loader = tf.train.import_meta_graph(c.SAVEMODEL_DIR + '.meta')
        loader.restore(sess, c.SAVEMODEL_DIR)
        graph = tf.get_default_graph()
        input_ = graph.get_tensor_by_name('input:0')
        model_output = graph.get_tensor_by_name('model/output:0')
        if os.path.isdir(path):
            results = {}
            for image_name, preprocessed_image in preprocessor.preprocess_directory(path):
                input_arg = np.expand_dims(preprocessed_image, axis=0)
                result = sess.run(model_output, feed_dict={input_: input_arg})
                if np.argmax(result) == 0:
                    results[image_name] = 'cat'
                else:
                    results[image_name] = 'dog'
            return results
        # Else, `path` is either a file on disk or a URL.
        input_arg = np.expand_dims(preprocessor.preprocess_image(path), axis=0)
        result = sess.run(model_output, feed_dict={input_: input_arg})
        if np.argmax(result) == np.argmax(c.ENCODING['cats']):
            return 'cat'
        return 'dog'

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Need help? See: https://github.com/MarxSoul55/cats_vs_dogs#using-the-cli')
    parser.add_argument('--train', action='store_true')
    parser.add_argument('--resuming', action='store_true')
    parser.add_argument('--steps', type=int)
    parser.add_argument('--classify', action='store_true')
    parser.add_argument('--source')
    parser.set_defaults(resuming=False)
    args = parser.parse_args()
    if args.train:
        print('WARNING: Training will overwrite the saved model (if it exists). EXECUTE Y/N?')
        if msvcrt.getch().decode().lower() == 'y':
            train(args.steps, args.resuming)
        else:
            sys.exit('Program closed.')
    elif args.classify:
        print(classify(args.source))
<|endoftext|>


<commit_before>from stats import *
import glob
import numpy as np
from sklearn import linear_model
import matplotlib.pyplot as plt

#prepares data to be fit using only raw averages of all games (but the last) of each players
def raw_averages(season):
    averages = []
    next_match_points = []
    players = glob.glob('data' + os.sep + season + os.sep + 'player_stats' + os.sep + "*.pkl")
    for file in players:
        playerID = file[26:-4]
        averages.append(average(season, playerID)[0])
        next_match_points.append(compute_fantasy(season, playerID))
    X = np.array(averages)
    y = np.array(next_match_points)

    return X, y

#sliding average using raw averages for one player
def sliding_raw_average(season, playerID):
    averages = []
    next_match_points = []
    player = pickle.load(open('data' + os.sep + season + os.sep + 'player_stats' + os.sep + playerID + '.pkl', 'rb'))
    games_num = len(player['stats'])

    for i in range(1, games_num - 1):
        averages.append(average(season, playerID, i)[0])
        next_match_points.append(compute_fantasy(season, playerID, i + 1))

    X = np.array(averages)
    y = np.array(next_match_points)

    return X, y

#sliding averages for all players in one season
def sliding_raw_averages(season):
    Xs = []
    ys = []
    players = glob.glob('data' + os.sep + season + os.sep + 'player_stats' + os.sep + "*.pkl")
    for file in players:
        playerID = file[26:-4]
        X, y = sliding_raw_average(season, playerID)

        if X.shape != (0,):
            Xs.append(X)
            ys.append(y)

    return np.concatenate(Xs), np.concatenate(ys)

#sliding averages for all players in all seasons
# def sliding_raw_averages(seasons):
#     averages = []
#     next_match_points = []
#     for season in seasons:
#         players = glob.glob('data' + os.sep + season + os.sep + 'player_stats' + os.sep + "*.pkl")
#         for file in players:
#             playerID = file[26:-4]
#             player = pickle.load(open('data' + os.sep + season + os.sep + 'player_stats' + os.sep + playerID + '.pkl', 'rb'))
#             games_num = len(player['stats'])
#
#             for i in range(1, games_num - 1):
#                 averages.append(average(season, playerID, i)[0])
#                 next_match_points.append(compute_fantasy(season, playerID, i + 1))
#
#     X = np.array(averages)
#     y = np.array(next_match_points)
#
#     return X, y

def train_linear(X, y, normalize = False):
    lr = linear_model.LinearRegression(normalize=normalize)
    lr.fit(X,y)

    return lr

#X, y = sliding_raw_averages('2011-12')
#X, y = raw_averages('2006-07')
# X, y = sliding_raw_average('2011-12', '101107')
# print X.shape, y.shape
#model = train_linear(X, y)
#modeln = train_linear(X, y, True)

#computed error given model as input
def error(model, X, y):
    predictions = model.predict(X)
    avg_error = 0.
    max_error = 0.
    errors = []
    for i, prediction in enumerate(predictions):
        error = abs(prediction - y[i])
        avg_error += error
        max_error = error if error > max_error else max_error
        errors.append(error)

    #plt.plot(errors)
    #plt.show()

    return avg_error/predictions.shape[0], max_error

#print error(model, X, y)
#print error(modeln, X, y)

#all but one fold error over seasons using sliding raw averages
def ABOF_error(seasons):
    data = map(sliding_raw_averages, seasons)
    errors = []
    avg_error = 0.
    avg_max = 0.

    for i, season in enumerate(seasons):
        print "Testing on season %s (Training on the rest)" % season
        Xs = [dat[0] for dat in data]
        ys = [dat[1] for dat in data]
        testX, testy = Xs.pop(i), ys.pop(i)
        trainX, trainy = np.concatenate(Xs), np.concatenate(ys)
        model = train_linear(trainX, trainy)
        err = error(model, testX, testy)
        errors.append(err[0])
        avg_error += err[0]
        avg_max += err[1]

        print "error for this season is %s" % (err,)

    result = avg_error/len(season), avg_max/len(season)
    print "Average error and Averaged max error over all seasons is %s" % (result,)

    return result

#seasons = ['2006-07', '2007-08', '2008-09', '2009-10', '2010-11', '2011-12', '2012-13', '2013-14']
seasons = ['2006-07', '2007-08']

ABOF_error(seasons)

<commit_msg>now created a file containing X and y's to gain time<commit_after>from stats import *
import glob
import numpy as np
from sklearn import linear_model
import matplotlib.pyplot as plt
import shutil

#prepares data to be fit using only raw averages of all games (but the last) of each players
def raw_averages(season):
    averages = []
    next_match_points = []
    players = glob.glob('data' + os.sep + season + os.sep + 'player_stats' + os.sep + "*.pkl")
    for file in players:
        playerID = file[26:-4]
        averages.append(average(season, playerID)[0])
        next_match_points.append(compute_fantasy(season, playerID))
    X = np.array(averages)
    y = np.array(next_match_points)

    if os.path.exists('data' + os.sep + season + os.sep + 'averages' + os.sep + 'raw_X' + '.pkl'):
        os.remove('data' + os.sep + season + os.sep + 'averages' + os.sep + 'raw_X' + '.pkl')
        os.remove('data' + os.sep + season + os.sep + 'averages' + os.sep + 'raw_y' + '.pkl')

    pickle.dump(X, open('data' + os.sep + season + os.sep + 'averages' + os.sep + 'raw_X' + '.pkl', 'wb'))
    pickle.dump(y, open('data' + os.sep + season + os.sep + 'averages' + os.sep + 'raw_y' + '.pkl', 'wb'))

    return X, y

#sliding average using raw averages for one player
def sliding_average(season, playerID):
    averages = []
    next_match_points = []
    player = pickle.load(open('data' + os.sep + season + os.sep + 'player_stats' + os.sep + playerID + '.pkl', 'rb'))
    games_num = len(player['stats'])

    for i in range(1, games_num - 1):
        averages.append(average(season, playerID, i)[0])
        next_match_points.append(compute_fantasy(season, playerID, i + 1))

    X = np.array(averages)
    y = np.array(next_match_points)

    return X, y

#sliding averages for all players in one season
def sliding_averages(season):
    Xs = []
    ys = []
    players = glob.glob('data' + os.sep + season + os.sep + 'player_stats' + os.sep + "*.pkl")
    for file in players:
        playerID = file[26:-4]
        X, y = sliding_average(season, playerID)

        if X.shape != (0,):
            Xs.append(X)
            ys.append(y)

    Xf = np.concatenate(Xs)
    yf = np.concatenate(ys)

    if os.path.exists('data' + os.sep + season + os.sep + 'averages' + os.sep + 'sliding_X' + '.pkl'):
        os.remove('data' + os.sep + season + os.sep + 'averages' + os.sep + 'sliding_X' + '.pkl')
        os.remove('data' + os.sep + season + os.sep + 'averages' + os.sep + 'sliding_y' + '.pkl')

    pickle.dump(Xf, open('data' + os.sep + season + os.sep + 'averages' + os.sep + 'sliding_X' + '.pkl', 'wb'))
    pickle.dump(yf, open('data' + os.sep + season + os.sep + 'averages' + os.sep + 'sliding_y' + '.pkl', 'wb'))

    return Xf, yf

#sliding averages for all players in all seasons
# def sliding_averages(seasons):
#     averages = []
#     next_match_points = []
#     for season in seasons:
#         players = glob.glob('data' + os.sep + season + os.sep + 'player_stats' + os.sep + "*.pkl")
#         for file in players:
#             playerID = file[26:-4]
#             player = pickle.load(open('data' + os.sep + season + os.sep + 'player_stats' + os.sep + playerID + '.pkl', 'rb'))
#             games_num = len(player['stats'])
#
#             for i in range(1, games_num - 1):
#                 averages.append(average(season, playerID, i)[0])
#                 next_match_points.append(compute_fantasy(season, playerID, i + 1))
#
#     X = np.array(averages)
#     y = np.array(next_match_points)
#
#     return X, y

def train_linear(X, y, normalize = False):
    lr = linear_model.LinearRegression(normalize=normalize)
    lr.fit(X, y)

    return lr

#X, y = sliding_averages('2011-12')
#X, y = raw_averages('2006-07')
# X, y = sliding_average('2011-12', '101107')
# print X.shape, y.shape
#model = train_linear(X, y)
#modeln = train_linear(X, y, True)

#computed error given model as input
def error(model, X, y):
    predictions = model.predict(X)
    avg_error = 0.
    max_error = 0.
    errors = []
    for i, prediction in enumerate(predictions):
        error = abs(prediction - y[i])
        avg_error += error
        max_error = error if error > max_error else max_error
        errors.append(error)

    #plt.plot(errors)
    #plt.show()

    return avg_error/predictions.shape[0], max_error

#print error(model, X, y)
#print error(modeln, X, y)

#all but one fold error over seasons using sliding raw averages
def ABOF_error(seasons):
    data = map(sliding_averages, seasons)
    errors = []
    avg_error = 0.
    avg_max = 0.

    for i, season in enumerate(seasons):
        print "Testing on season %s (Training on the rest)" % season
        Xs = [dat[0] for dat in data]
        ys = [dat[1] for dat in data]
        testX, testy = Xs.pop(i), ys.pop(i)
        trainX, trainy = np.concatenate(Xs), np.concatenate(ys)
        model = train_linear(trainX, trainy)
        err = error(model, testX, testy)
        errors.append(err[0])
        avg_error += err[0]
        avg_max += err[1]

        print "error for this season is %s" % (err,)

    result = avg_error/len(seasons), avg_max/len(seasons)
    print "Average error and Averaged max error over all seasons is %s" % (result,)

    return result

seasons = ['2005-06', '2006-07', '2007-08', '2008-09', '2009-10', '2010-11', '2011-12', '2012-13', '2013-14']
#seasons = ['2006-07', '2007-08']
#ABOF_error(seasons)

<|endoftext|>


<commit_before># -*- coding: iso-8859-1 -*-
# -----------------------------------------------------------------------------
# utils.py - Miscellaneous system utilities
# -----------------------------------------------------------------------------
# $Id$
#
# -----------------------------------------------------------------------------
# Copyright (C) 2006 Dirk Meyer, Jason Tackaberry
#
# First Edition: Jason Tackaberry <tack@urandom.ca>
# Maintainer:    Jason Tackaberry <tack@urandom.ca>
#
# Please see the file AUTHORS for a complete list of authors.
#
# This library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version
# 2.1 as published by the Free Software Foundation.
#
# This library is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA
#
# -----------------------------------------------------------------------------

__all__ = [ ]

import sys
import os
import stat
import time
import imp
import logging

import kaa
import _utils

# get logging object
log = logging.getLogger('kaa')

def which(file, path = None):
    """
    Does what which(1) does: searches the PATH for a given file
    name and returns a list of matches.
    """
    if not path:
        path = os.getenv("PATH")

    for p in path.split(":"):
        fullpath = os.path.join(p, file)
        try:
            st = os.stat(fullpath)
        except OSError:
            continue

        if os.geteuid() == st[stat.ST_UID]:
            mask = stat.S_IXUSR
        elif st[stat.ST_GID] in os.getgroups():
            mask = stat.S_IXGRP
        else:
            mask = stat.S_IXOTH

        if stat.S_IMODE(st[stat.ST_MODE]) & mask:
            return fullpath

    return None

class Lock(object):
    def __init__(self):
        self._read, self._write = os.pipe()

    def release(self, exitcode):
        os.write(self._write, str(exitcode))
        os.close(self._read)
        os.close(self._write)

    def wait(self):
        exitcode = os.read(self._read, 1)
        os.close(self._read)
        os.close(self._write)
        return int(exitcode)

    def ignore(self):
        os.close(self._read)
        os.close(self._write)

def daemonize(stdin = '/dev/null', stdout = '/dev/null', stderr = None,
              pidfile=None, exit = True, wait = False):
    """
    Does a double-fork to daemonize the current process using the technique
    described at http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16 .

    If exit is True (default), parent exits immediately.  If false, caller will receive
    the pid of the forked child.
    """

    lock = 0
    if wait:
        lock = Lock()

    # First fork.
    try:
        pid = os.fork()
        if pid > 0:
            if wait:
                exitcode = lock.wait()
                if exitcode:
                    sys.exit(exitcode)
            if exit:
                # Exit from the first parent.
                sys.exit(0)

            # Wait for child to fork again (otherwise we have a zombie)
            os.waitpid(pid, 0)
            return pid
    except OSError, e:
        log.error("Initial daemonize fork failed: %d, %s\n", e.errno, e.strerror)
        sys.exit(1)

    os.chdir("/")
    os.setsid()

    # Second fork.
    try:
        pid = os.fork()
        if pid > 0:
            # Exit from the second parent.
            sys.exit(0)
    except OSError, e:
        log.error("Second daemonize fork failed: %d, %s\n", e.errno, e.strerror)
        sys.exit(1)

    # Create new standard file descriptors.
    if not stderr:
        stderr = stdout
    stdin = file(stdin, 'r')
    stdout = file(stdout, 'a+')
    stderr = file(stderr, 'a+', 0)
    if pidfile:
        file(pidfile, 'w+').write("%d\n" % os.getpid())

    # Remap standard fds.
    os.dup2(stdin.fileno(), sys.stdin.fileno())
    os.dup2(stdout.fileno(), sys.stdout.fileno())
    os.dup2(stderr.fileno(), sys.stderr.fileno())

    # Replace any existing thread notifier pipe, otherwise we'll be listening
    # to our parent's thread notifier.
    from kaa.notifier.thread import create_thread_notifier_pipe
    create_thread_notifier_pipe(new=False, purge=True)

    return lock

def is_running(name):
    """
    Check if the program with the given name is running. The program
    must have called set_running itself. Returns the pid or 0.
    """
    if not os.path.isfile(kaa.tempfile('run/' + name)):
        return 0
    run = open(kaa.tempfile('run/' + name))
    pid = run.readline().strip()
    cmdline = run.readline()
    run.close()
    if not os.path.exists('/proc/%s/cmdline' % pid):
        return 0
    current = open('/proc/%s/cmdline' % pid).readline()
    if current == cmdline or current.strip('\x00') == name:
        return int(pid)
    return 0

def set_running(name, modify = True):
    """
    Set this program as running with the given name.  If modify is True,
    the process name is updated as described in set_process_name().
    """
    cmdline = open('/proc/%s/cmdline' % os.getpid()).readline()
    run = open(kaa.tempfile('run/' + name), 'w')
    run.write(str(os.getpid()) + '\n')
    run.write(cmdline)
    run.close()
    if modify:
        _utils.set_process_name(name, len(cmdline))

def set_process_name(name):
    """
    On Linux systems later than 2.6.9, this function sets the process name as it
    appears in ps, and so that it can be found with killall.

    Note: name will be truncated to the cumulative length of the original
    process name and all its arguments; once updated, passed arguments will no
    longer be visible.
    """
    cmdline = open('/proc/%s/cmdline' % os.getpid()).readline()
    _utils.set_process_name(name, len(cmdline))

def get_plugins(path, include_files=True, include_directories=True):
    """
    Get a list of plugins in the given plugin directory. The 'path' argument
    can also be a full path of an __init__ file.
    """
    if os.path.isfile(path):
        path = os.path.dirname(path)
    result = []
    for plugin in os.listdir(path):
        for ext in ('.py', '.pyc', '.pyo'):
            if plugin.endswith(ext) and include_files:
                plugin = plugin[:-len(ext)]
                break
        else:
            if not include_directories or not os.path.isdir(os.path.join(path, plugin)):
                continue
        if not plugin in result and not plugin == '__init__' and \
               not plugin.startswith('.'):
            result.append(plugin)
    return result

class Singleton(object):
    """
    Create Singleton object from classref on demand.
    """

    class MemberFunction(object):
        def __init__(self, singleton, name):
            self._singleton = singleton
            self._name = name

        def __call__(self, *args, **kwargs):
            return getattr(self._singleton(), self._name)(*args, **kwargs)

    def __init__(self, classref):
        self._singleton = None
        self._class = classref

    def __call__(self):
        if self._singleton is None:
            self._singleton = self._class()
        return self._singleton

    def __getattr__(self, attr):
        if self._singleton is None:
            return Singleton.MemberFunction(self, attr)
        return getattr(self._singleton, attr)

class property(property):
    """
    Replaces built-in property function to extend it as per
    http://bugs.python.org/issue1416
    """
    def __init__(self, fget = None, fset = None, fdel = None, doc = None):
        super(property, self).__init__(fget, fset, fdel)
        self.__doc__ = doc or fget.__doc__

    def _add_doc(self, prop, doc = None):
        prop.__doc__ = doc or self.__doc__
        return prop

    def setter(self, fset):
        if isinstance(fset, property):
            # Wrapping another property, use deleter.
            self, fset = fset, fset.fdel
        return self._add_doc(property(self.fget, fset, self.fdel))

    def deleter(self, fdel):
        if isinstance(fdel, property):
            # Wrapping another property, use setter.
            self, fdel = fdel, fdel.fset
        return self._add_doc(property(self.fget, self.fset, fdel))

    def getter(self, fget):
        return self._add_doc(property(fget, self.fset, self.fdel), fget.__doc__ or self.fget.__doc__)

def sysimport(name):
    """
    Help to import modules with name conflict. E.g. thread.py in notifier
    uses sysimport('thread').
    """
    # Fast path: see if the module has already been imported.
    try:
        return sys.modules[name]
    except KeyError:
        pass

    # Remove the current directory and anything below it from the
    # search path.
    cwd = os.path.realpath(os.getcwd())
    path = [ x for x in sys.path if x and not os.path.realpath(x).startswith(cwd) ]
    fp, pathname, description = imp.find_module(name, path)
    try:
        return imp.load_module(name, fp, pathname, description)
    finally:
        # Since we may exit via an exception, close fp explicitly.
        if fp:
            fp.close()
<commit_msg>Utility function to get number of processors in a reasonably portable way.<commit_after># -*- coding: iso-8859-1 -*-
# -----------------------------------------------------------------------------
# utils.py - Miscellaneous system utilities
# -----------------------------------------------------------------------------
# $Id$
#
# -----------------------------------------------------------------------------
# Copyright (C) 2006 Dirk Meyer, Jason Tackaberry
#
# First Edition: Jason Tackaberry <tack@urandom.ca>
# Maintainer:    Jason Tackaberry <tack@urandom.ca>
#
# Please see the file AUTHORS for a complete list of authors.
#
# This library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version
# 2.1 as published by the Free Software Foundation.
#
# This library is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA
#
# -----------------------------------------------------------------------------

__all__ = [ ]

import sys
import os
import stat
import time
import imp
import logging

import kaa
import _utils

# get logging object
log = logging.getLogger('kaa')

def which(file, path = None):
    """
    Does what which(1) does: searches the PATH for a given file
    name and returns a list of matches.
    """
    if not path:
        path = os.getenv("PATH")

    for p in path.split(":"):
        fullpath = os.path.join(p, file)
        try:
            st = os.stat(fullpath)
        except OSError:
            continue

        if os.geteuid() == st[stat.ST_UID]:
            mask = stat.S_IXUSR
        elif st[stat.ST_GID] in os.getgroups():
            mask = stat.S_IXGRP
        else:
            mask = stat.S_IXOTH

        if stat.S_IMODE(st[stat.ST_MODE]) & mask:
            return fullpath

    return None

class Lock(object):
    def __init__(self):
        self._read, self._write = os.pipe()

    def release(self, exitcode):
        os.write(self._write, str(exitcode))
        os.close(self._read)
        os.close(self._write)

    def wait(self):
        exitcode = os.read(self._read, 1)
        os.close(self._read)
        os.close(self._write)
        return int(exitcode)

    def ignore(self):
        os.close(self._read)
        os.close(self._write)

def daemonize(stdin = '/dev/null', stdout = '/dev/null', stderr = None,
              pidfile=None, exit = True, wait = False):
    """
    Does a double-fork to daemonize the current process using the technique
    described at http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16 .

    If exit is True (default), parent exits immediately.  If false, caller will receive
    the pid of the forked child.
    """

    lock = 0
    if wait:
        lock = Lock()

    # First fork.
    try:
        pid = os.fork()
        if pid > 0:
            if wait:
                exitcode = lock.wait()
                if exitcode:
                    sys.exit(exitcode)
            if exit:
                # Exit from the first parent.
                sys.exit(0)

            # Wait for child to fork again (otherwise we have a zombie)
            os.waitpid(pid, 0)
            return pid
    except OSError, e:
        log.error("Initial daemonize fork failed: %d, %s\n", e.errno, e.strerror)
        sys.exit(1)

    os.chdir("/")
    os.setsid()

    # Second fork.
    try:
        pid = os.fork()
        if pid > 0:
            # Exit from the second parent.
            sys.exit(0)
    except OSError, e:
        log.error("Second daemonize fork failed: %d, %s\n", e.errno, e.strerror)
        sys.exit(1)

    # Create new standard file descriptors.
    if not stderr:
        stderr = stdout
    stdin = file(stdin, 'r')
    stdout = file(stdout, 'a+')
    stderr = file(stderr, 'a+', 0)
    if pidfile:
        file(pidfile, 'w+').write("%d\n" % os.getpid())

    # Remap standard fds.
    os.dup2(stdin.fileno(), sys.stdin.fileno())
    os.dup2(stdout.fileno(), sys.stdout.fileno())
    os.dup2(stderr.fileno(), sys.stderr.fileno())

    # Replace any existing thread notifier pipe, otherwise we'll be listening
    # to our parent's thread notifier.
    from kaa.notifier.thread import create_thread_notifier_pipe
    create_thread_notifier_pipe(new=False, purge=True)

    return lock

def is_running(name):
    """
    Check if the program with the given name is running. The program
    must have called set_running itself. Returns the pid or 0.
    """
    if not os.path.isfile(kaa.tempfile('run/' + name)):
        return 0
    run = open(kaa.tempfile('run/' + name))
    pid = run.readline().strip()
    cmdline = run.readline()
    run.close()
    if not os.path.exists('/proc/%s/cmdline' % pid):
        return 0
    current = open('/proc/%s/cmdline' % pid).readline()
    if current == cmdline or current.strip('\x00') == name:
        return int(pid)
    return 0

def set_running(name, modify = True):
    """
    Set this program as running with the given name.  If modify is True,
    the process name is updated as described in set_process_name().
    """
    cmdline = open('/proc/%s/cmdline' % os.getpid()).readline()
    run = open(kaa.tempfile('run/' + name), 'w')
    run.write(str(os.getpid()) + '\n')
    run.write(cmdline)
    run.close()
    if modify:
        _utils.set_process_name(name, len(cmdline))

def set_process_name(name):
    """
    On Linux systems later than 2.6.9, this function sets the process name as it
    appears in ps, and so that it can be found with killall.

    Note: name will be truncated to the cumulative length of the original
    process name and all its arguments; once updated, passed arguments will no
    longer be visible.
    """
    cmdline = open('/proc/%s/cmdline' % os.getpid()).readline()
    _utils.set_process_name(name, len(cmdline))

def get_num_processors():
    """
    Returns the number of processors on the system, or raises RuntimeError
    if that value cannot be determined.
    """
    try:
        if sys.platform == 'win32':
            return int(os.environ['NUMBER_OF_PROCESSORS'])
        elif sys.platform == 'darwin':
            return int(os.popen('sysctl -n hw.ncpu').read())
        else:
            return os.sysconf('SC_NPROCESSORS_ONLN')
    except (KeyError, ValueError, OSError, AttributeError):
        pass

    raise RuntimeError('Could not determine number of processors')

def get_plugins(path, include_files=True, include_directories=True):
    """
    Get a list of plugins in the given plugin directory. The 'path' argument
    can also be a full path of an __init__ file.
    """
    if os.path.isfile(path):
        path = os.path.dirname(path)
    result = []
    for plugin in os.listdir(path):
        for ext in ('.py', '.pyc', '.pyo'):
            if plugin.endswith(ext) and include_files:
                plugin = plugin[:-len(ext)]
                break
        else:
            if not include_directories or not os.path.isdir(os.path.join(path, plugin)):
                continue
        if not plugin in result and not plugin == '__init__' and \
               not plugin.startswith('.'):
            result.append(plugin)
    return result

class Singleton(object):
    """
    Create Singleton object from classref on demand.
    """

    class MemberFunction(object):
        def __init__(self, singleton, name):
            self._singleton = singleton
            self._name = name

        def __call__(self, *args, **kwargs):
            return getattr(self._singleton(), self._name)(*args, **kwargs)

    def __init__(self, classref):
        self._singleton = None
        self._class = classref

    def __call__(self):
        if self._singleton is None:
            self._singleton = self._class()
        return self._singleton

    def __getattr__(self, attr):
        if self._singleton is None:
            return Singleton.MemberFunction(self, attr)
        return getattr(self._singleton, attr)

class property(property):
    """
    Replaces built-in property function to extend it as per
    http://bugs.python.org/issue1416
    """
    def __init__(self, fget = None, fset = None, fdel = None, doc = None):
        super(property, self).__init__(fget, fset, fdel)
        self.__doc__ = doc or fget.__doc__

    def _add_doc(self, prop, doc = None):
        prop.__doc__ = doc or self.__doc__
        return prop

    def setter(self, fset):
        if isinstance(fset, property):
            # Wrapping another property, use deleter.
            self, fset = fset, fset.fdel
        return self._add_doc(property(self.fget, fset, self.fdel))

    def deleter(self, fdel):
        if isinstance(fdel, property):
            # Wrapping another property, use setter.
            self, fdel = fdel, fdel.fset
        return self._add_doc(property(self.fget, self.fset, fdel))

    def getter(self, fget):
        return self._add_doc(property(fget, self.fset, self.fdel), fget.__doc__ or self.fget.__doc__)

def sysimport(name):
    """
    Help to import modules with name conflict. E.g. thread.py in notifier
    uses sysimport('thread').
    """
    # Fast path: see if the module has already been imported.
    try:
        return sys.modules[name]
    except KeyError:
        pass

    # Remove the current directory and anything below it from the
    # search path.
    cwd = os.path.realpath(os.getcwd())
    path = [ x for x in sys.path if x and not os.path.realpath(x).startswith(cwd) ]
    fp, pathname, description = imp.find_module(name, path)
    try:
        return imp.load_module(name, fp, pathname, description)
    finally:
        # Since we may exit via an exception, close fp explicitly.
        if fp:
            fp.close()
<|endoftext|>


<commit_before>from __future__ import print_function
import random
from players import RandPlayer

def play_random_game(game):
    """Plays a game taking uniformly random moves"""
    print(play_match(game, [RandPlayer(), RandPlayer()]))

def play_match(game, players, verbose=True):
    """Plays a match between the given players"""
    game = game.copy()
    if verbose:
        print(game)
    while not game.is_over():
        cur_player = players[game.cur_player()]
        move = cur_player.chooseMove(game.copy())
        game.make_move(move)
        if verbose:
            print(game)
    return game.outcomes()

def play_series(game, players, n_matches=100):
    """Plays a series of 'n_matches' matches between the given players"""
    stats = {'p1_wins': 0, 'p2_wins': 0, 'draws': 0}
    for __ in range(n_matches):
        outcomes = play_match(game, players)
        if outcomes[0] == 'W':
            stats['p1_wins'] += 1
        elif outcomes[1] == 'W':
            stats['p2_wins'] += 1
        else:
            stats['draws'] += 1
        game.reset()
    print(stats)
<commit_msg>Update<commit_after>from __future__ import print_function
import random
from players import RandPlayer

def play_random_game(game):
    """Plays a game taking uniformly random moves"""
    print(play_match(game, [RandPlayer(), RandPlayer()]))

def play_match(game, players, verbose=True):
    """Plays a match between the given players"""
    game = game.copy()
    if verbose:
        print(game)
    while not game.is_over():
        cur_player = players[game.cur_player()]
        move = cur_player.chooseMove(game.copy())
        game.make_move(move)
        if verbose:
            print(game)
    return game.outcomes()

def play_series(game, players, n_matches=100):
    """Plays a series of 'n_matches' matches between the given players"""
    stats = {'p1_wins': 0, 'p2_wins': 0, 'draws': 0}
    for __ in range(n_matches):
        outcomes = play_match(game, players, verbose=False)
        if outcomes[0] == 'W':
            stats['p1_wins'] += 1
        elif outcomes[1] == 'W':
            stats['p2_wins'] += 1
        else:
            stats['draws'] += 1
        game.reset()
    print(stats)
<|endoftext|>