<commit_before>import cgen as c

from devito.dimension import IncrDimension
from devito.ir.iet import (Expression, Iteration, List, ntags, FindAdjacentIterations,
                           FindNodes, IsPerfectIteration, NestedTransformer, Transformer,
                           compose_nodes, retrieve_iteration_tree)
from devito.symbolics import as_symbol, xreplace_indices
from devito.tools import as_tuple

__all__ = ['fold_blockable_tree', 'unfold_blocked_tree']

def fold_blockable_tree(node, exclude_innermost=False):
    """
    Create :class:`IterationFold`s from sequences of nested :class:`Iteration`.
    """
    found = FindAdjacentIterations().visit(node)
    found.pop('seen_iteration')

    mapper = {}
    for k, v in found.items():
        for i in v:
            # Pre-condition: they all must be perfect iterations
            assert len(i) > 1
            if any(not IsPerfectIteration().visit(j) for j in i):
                continue
            # Only retain consecutive trees having same depth
            trees = [retrieve_iteration_tree(j)[0] for j in i]
            handle = []
            for j in trees:
                if len(j) != len(trees[0]):
                    break
                handle.append(j)
            trees = handle
            if not trees:
                continue
            # Check foldability
            pairwise_folds = list(zip(*reversed(trees)))
            if any(not is_foldable(j) for j in pairwise_folds):
                continue
            # Maybe heuristically exclude innermost Iteration
            if exclude_innermost is True:
                pairwise_folds = pairwise_folds[:-1]
            # Perhaps there's nothing to fold
            if len(pairwise_folds) == 1:
                continue
            # Perform folding
            for j in pairwise_folds:
                root, remainder = j[0], j[1:]
                folds = [(tuple(y-x for x, y in zip(i.offsets, root.offsets)), i.nodes)
                         for i in remainder]
                mapper[root] = IterationFold(folds=folds, **root.args)
                for k in remainder:
                    mapper[k] = None

    # Insert the IterationFolds in the Iteration/Expression tree
    processed = NestedTransformer(mapper).visit(node)

    return processed

def unfold_blocked_tree(node):
    """
    Unfold nested :class:`IterationFold`.

    :Example:

    Given a section of Iteration/Expression tree as below: ::

        for i = 1 to N-1  // folded
          for j = 1 to N-1  // folded
            foo1()

    Assuming a fold with offset 1 in both /i/ and /j/ and body ``foo2()``, create: ::

        for i = 1 to N-1
          for j = 1 to N-1
            foo1()
        for i = 2 to N-2
          for j = 2 to N-2
            foo2()
    """
    # Search the unfolding candidates
    candidates = []
    for tree in retrieve_iteration_tree(node):
        handle = tuple(i for i in tree if i.is_IterationFold)
        if handle:
            # Sanity check
            assert IsPerfectIteration().visit(handle[0])
            candidates.append(handle)

    # Perform unfolding
    tag = ntags()
    mapper = {}
    for tree in candidates:
        trees = list(zip(*[i.unfold() for i in tree]))
        # Update tag
        for i, _tree in enumerate(list(trees)):
            trees[i] = tuple(j.retag(tag + i) for j in _tree)
        trees = optimize_unfolded_tree(trees[:-1], trees[-1])
        mapper[tree[0]] = List(body=trees)

    # Insert the unfolded Iterations in the Iteration/Expression tree
    processed = Transformer(mapper).visit(node)

    return processed

def is_foldable(nodes):
    """
    Return True if the iterable ``nodes`` consists of foldable :class:`Iteration`s,
    False otherwise.
    """
    nodes = as_tuple(nodes)
    if len(nodes) <= 1 or any(not i.is_Iteration for i in nodes):
        return False
    main = nodes[0]
    return all(i.dim == main.dim and i.limits == main.limits and i.index == main.index
               and i.properties == main.properties for i in nodes)

def optimize_unfolded_tree(unfolded, root):
    """
    Transform folded trees to reduce the memory footprint.

    Examples
    ========
    Given:

        .. code-block::
            for i = 1 to N - 1  # Folded tree
              for j = 1 to N - 1
                tmp[i,j] = ...
            for i = 2 to N - 2  # Root
              for j = 2 to N - 2
                ... = ... tmp[i,j] ...

    The temporary ``tmp`` has shape ``(N-1, N-1)``. However, as soon as the
    iteration space is blocked, with blocks of shape ``(i_bs, j_bs)``, the
    ``tmp`` shape can be shrunk to ``(i_bs-1, j_bs-1)``. The resulting
    iteration tree becomes:

        .. code-block::
            for i = 1 to i_bs + 1  # Folded tree
              for j = 1 to j_bs + 1
                i' = i + i_block - 2
                j' = j + j_block - 2
                tmp[i,j] = ... # use i' and j'
            for i = i_block to i_block + i_bs  # Root
              for j = j_block to j_block + j_bs
                i' = i - x_block
                j' = j - j_block
                ... = ... tmp[i',j'] ...
    """
    processed = []
    for i, tree in enumerate(unfolded):
        assert len(tree) == len(root)

        # We can optimize the folded trees only if they compute temporary
        # arrays, but not if they compute input data
        exprs = FindNodes(Expression).visit(tree[-1])
        writes = [j.write for j in exprs if j.is_tensor]
        if not all(j.is_Array for j in writes):
            processed.append(compose_nodes(tree))
            root = compose_nodes(root)
            continue

        modified_tree = []
        modified_root = []
        mapper = {}

        # "Shrink" the iteration space
        for t1, t2 in zip(tree, root):
            t1_udim = IncrDimension(t1.dim, t1.limits[0], 1, "%ss%d" % (t1.index, i))
            limits = (0, t1.limits[1] - t1.limits[0], t1.symbolic_incr)
            modified_tree.append(t1._rebuild(limits=limits,
                                             uindices=t1.uindices + (t1_udim,)))

            t2_udim = IncrDimension(t1.dim, -t1.limits[0], 1, "%ss%d" % (t1.index, i))
            modified_root.append(t2._rebuild(uindices=t2.uindices + (t2_udim,)))

            mapper[t1.dim] = t1_udim

        # Temporary arrays can now be moved onto the stack
        if all(not j.is_Remainder for j in modified_tree):
            dimensions = tuple(j.limits[0] for j in modified_root)
            for j in writes:
                if j.is_Array:
                    j_dimensions = dimensions + j.dimensions[len(modified_root):]
                    j_shape = tuple(k.symbolic_size for k in j_dimensions)
                    j.update(shape=j_shape, dimensions=j_dimensions, onstack=True)

        # Substitute iteration variables within the folded trees
        modified_tree = compose_nodes(modified_tree)
        replaced = xreplace_indices([j.expr for j in exprs], mapper, only_rhs=True)
        subs = [j._rebuild(expr=k) for j, k in zip(exprs, replaced)]
        processed.append(Transformer(dict(zip(exprs, subs))).visit(modified_tree))

        # Introduce the new iteration variables within /root/
        modified_root = compose_nodes(modified_root)
        exprs = FindNodes(Expression).visit(modified_root)
        candidates = [as_symbol(j.output) for j in subs]
        replaced = xreplace_indices([j.expr for j in exprs], mapper, candidates)
        subs = [j._rebuild(expr=k) for j, k in zip(exprs, replaced)]
        root = Transformer(dict(zip(exprs, subs))).visit(modified_root)

    return processed + [root]

class IterationFold(Iteration):

    """
    An IterationFold is a special :class:`Iteration` object that represents
    a sequence of consecutive (in program order) Iterations. In an IterationFold,
    all Iterations of the sequence but the so called ``root`` are "hidden"; that is,
    they cannot be visited by an Iteration/Expression tree visitor.

    The Iterations in the sequence represented by the IterationFold all have same
    dimension and properties. However, their extent is relative to that of the ``root``.
    """

    is_IterationFold = True

    def __init__(self, *args, **kwargs):
        self.folds = kwargs.pop('folds', None)
        super(IterationFold, self).__init__(*args, **kwargs)

    def __repr__(self):
        properties = ""
        if self.properties:
            properties = [str(i) for i in self.properties]
            properties = "WithProperties[%s]::" % ",".join(properties)
        index = self.index
        if self.uindices:
            index += '[%s]' % ','.join(i.name for i in self.uindices)
        length = "Length %d" % len(self.folds)
        return "<%sIterationFold %s; %s; %s>" % (properties, index, self.limits, length)

    @property
    def ccode(self):
        comment = c.Comment('This IterationFold is "hiding" one or more Iterations')
        code = super(IterationFold, self).ccode
        return c.Module([comment, code])

    def unfold(self):
        """
        Return the corresponding :class:`Iteration` objects from each fold in ``self``.
        """
        args = self.args
        args.pop('folds')

        # Construct the root Iteration
        root = Iteration(**args)

        # Construct the folds
        args.pop('nodes')
        ofs = args.pop('offsets')
        try:
            start, end, incr = args.pop('limits')
        except TypeError:
            start, end, incr = self.limits
        folds = tuple(Iteration(nodes, limits=(start, end, incr),
                                offsets=tuple(i-j for i, j in zip(ofs, shift)), **args)
                      for shift, nodes in self.folds)

        return folds + as_tuple(root)
<commit_msg>dle: Do not fold iterations if not writing to temporaries<commit_after>import cgen as c

from devito.dimension import IncrDimension
from devito.ir.iet import (Expression, Iteration, List, ntags, FindAdjacentIterations,
                           FindNodes, IsPerfectIteration, NestedTransformer, Transformer,
                           compose_nodes, retrieve_iteration_tree)
from devito.symbolics import as_symbol, xreplace_indices
from devito.tools import as_tuple, flatten

__all__ = ['fold_blockable_tree', 'unfold_blocked_tree']

def fold_blockable_tree(node, exclude_innermost=False):
    """
    Create :class:`IterationFold`s from sequences of nested :class:`Iteration`.
    """
    found = FindAdjacentIterations().visit(node)
    found.pop('seen_iteration')

    mapper = {}
    for k, v in found.items():
        for i in v:
            # Pre-condition: they all must be perfect iterations
            assert len(i) > 1
            if any(not IsPerfectIteration().visit(j) for j in i):
                continue
            # Only retain consecutive trees having same depth
            trees = [retrieve_iteration_tree(j)[0] for j in i]
            handle = []
            for j in trees:
                if len(j) != len(trees[0]):
                    break
                handle.append(j)
            trees = handle
            if not trees:
                continue
            # Check foldability
            pairwise_folds = list(zip(*reversed(trees)))
            if any(not is_foldable(j) for j in pairwise_folds):
                continue
            # Maybe heuristically exclude innermost Iteration
            if exclude_innermost is True:
                pairwise_folds = pairwise_folds[:-1]
            # Perhaps there's nothing to fold
            if len(pairwise_folds) == 1:
                continue
            # TODO: we do not currently support blocking if any of the foldable
            # iterations writes to user data (need min/max loop bounds?)
            exprs = flatten(FindNodes(Expression).visit(j.root) for j in trees[:-1])
            if any(j.write.is_Input for j in exprs):
                continue
            # Perform folding
            for j in pairwise_folds:
                root, remainder = j[0], j[1:]
                folds = [(tuple(y-x for x, y in zip(i.offsets, root.offsets)), i.nodes)
                         for i in remainder]
                mapper[root] = IterationFold(folds=folds, **root.args)
                for k in remainder:
                    mapper[k] = None

    # Insert the IterationFolds in the Iteration/Expression tree
    processed = NestedTransformer(mapper).visit(node)

    return processed

def unfold_blocked_tree(node):
    """
    Unfold nested :class:`IterationFold`.

    :Example:

    Given a section of Iteration/Expression tree as below: ::

        for i = 1 to N-1  // folded
          for j = 1 to N-1  // folded
            foo1()

    Assuming a fold with offset 1 in both /i/ and /j/ and body ``foo2()``, create: ::

        for i = 1 to N-1
          for j = 1 to N-1
            foo1()
        for i = 2 to N-2
          for j = 2 to N-2
            foo2()
    """
    # Search the unfolding candidates
    candidates = []
    for tree in retrieve_iteration_tree(node):
        handle = tuple(i for i in tree if i.is_IterationFold)
        if handle:
            # Sanity check
            assert IsPerfectIteration().visit(handle[0])
            candidates.append(handle)

    # Perform unfolding
    tag = ntags()
    mapper = {}
    for tree in candidates:
        trees = list(zip(*[i.unfold() for i in tree]))
        # Update tag
        for i, _tree in enumerate(list(trees)):
            trees[i] = tuple(j.retag(tag + i) for j in _tree)
        trees = optimize_unfolded_tree(trees[:-1], trees[-1])
        mapper[tree[0]] = List(body=trees)

    # Insert the unfolded Iterations in the Iteration/Expression tree
    processed = Transformer(mapper).visit(node)

    return processed

def is_foldable(nodes):
    """
    Return True if the iterable ``nodes`` consists of foldable :class:`Iteration`s,
    False otherwise.
    """
    nodes = as_tuple(nodes)
    if len(nodes) <= 1 or any(not i.is_Iteration for i in nodes):
        return False
    main = nodes[0]
    return all(i.dim == main.dim and i.limits == main.limits and i.index == main.index
               and i.properties == main.properties for i in nodes)

def optimize_unfolded_tree(unfolded, root):
    """
    Transform folded trees to reduce the memory footprint.

    Examples
    ========
    Given:

        .. code-block::
            for i = 1 to N - 1  # Folded tree
              for j = 1 to N - 1
                tmp[i,j] = ...
            for i = 2 to N - 2  # Root
              for j = 2 to N - 2
                ... = ... tmp[i,j] ...

    The temporary ``tmp`` has shape ``(N-1, N-1)``. However, as soon as the
    iteration space is blocked, with blocks of shape ``(i_bs, j_bs)``, the
    ``tmp`` shape can be shrunk to ``(i_bs-1, j_bs-1)``. The resulting
    iteration tree becomes:

        .. code-block::
            for i = 1 to i_bs + 1  # Folded tree
              for j = 1 to j_bs + 1
                i' = i + i_block - 2
                j' = j + j_block - 2
                tmp[i,j] = ... # use i' and j'
            for i = i_block to i_block + i_bs  # Root
              for j = j_block to j_block + j_bs
                i' = i - x_block
                j' = j - j_block
                ... = ... tmp[i',j'] ...
    """
    processed = []
    for i, tree in enumerate(unfolded):
        assert len(tree) == len(root)

        # We can optimize the folded trees only if they compute temporary
        # arrays, but not if they compute input data
        exprs = FindNodes(Expression).visit(tree[-1])
        writes = [j.write for j in exprs if j.is_tensor]
        if not all(j.is_Array for j in writes):
            processed.append(compose_nodes(tree))
            root = compose_nodes(root)
            continue

        modified_tree = []
        modified_root = []
        mapper = {}

        # "Shrink" the iteration space
        for t1, t2 in zip(tree, root):
            t1_udim = IncrDimension(t1.dim, t1.limits[0], 1, "%ss%d" % (t1.index, i))
            limits = (0, t1.limits[1] - t1.limits[0], t1.symbolic_incr)
            modified_tree.append(t1._rebuild(limits=limits,
                                             uindices=t1.uindices + (t1_udim,)))

            t2_udim = IncrDimension(t1.dim, -t1.limits[0], 1, "%ss%d" % (t1.index, i))
            modified_root.append(t2._rebuild(uindices=t2.uindices + (t2_udim,)))

            mapper[t1.dim] = t1_udim

        # Temporary arrays can now be moved onto the stack
        if all(not j.is_Remainder for j in modified_tree):
            dimensions = tuple(j.limits[0] for j in modified_root)
            for j in writes:
                if j.is_Array:
                    j_dimensions = dimensions + j.dimensions[len(modified_root):]
                    j_shape = tuple(k.symbolic_size for k in j_dimensions)
                    j.update(shape=j_shape, dimensions=j_dimensions, onstack=True)

        # Substitute iteration variables within the folded trees
        modified_tree = compose_nodes(modified_tree)
        replaced = xreplace_indices([j.expr for j in exprs], mapper, only_rhs=True)
        subs = [j._rebuild(expr=k) for j, k in zip(exprs, replaced)]
        processed.append(Transformer(dict(zip(exprs, subs))).visit(modified_tree))

        # Introduce the new iteration variables within /root/
        modified_root = compose_nodes(modified_root)
        exprs = FindNodes(Expression).visit(modified_root)
        candidates = [as_symbol(j.output) for j in subs]
        replaced = xreplace_indices([j.expr for j in exprs], mapper, candidates)
        subs = [j._rebuild(expr=k) for j, k in zip(exprs, replaced)]
        root = Transformer(dict(zip(exprs, subs))).visit(modified_root)

    return processed + [root]

class IterationFold(Iteration):

    """
    An IterationFold is a special :class:`Iteration` object that represents
    a sequence of consecutive (in program order) Iterations. In an IterationFold,
    all Iterations of the sequence but the so called ``root`` are "hidden"; that is,
    they cannot be visited by an Iteration/Expression tree visitor.

    The Iterations in the sequence represented by the IterationFold all have same
    dimension and properties. However, their extent is relative to that of the ``root``.
    """

    is_IterationFold = True

    def __init__(self, *args, **kwargs):
        self.folds = kwargs.pop('folds', None)
        super(IterationFold, self).__init__(*args, **kwargs)

    def __repr__(self):
        properties = ""
        if self.properties:
            properties = [str(i) for i in self.properties]
            properties = "WithProperties[%s]::" % ",".join(properties)
        index = self.index
        if self.uindices:
            index += '[%s]' % ','.join(i.name for i in self.uindices)
        length = "Length %d" % len(self.folds)
        return "<%sIterationFold %s; %s; %s>" % (properties, index, self.limits, length)

    @property
    def ccode(self):
        comment = c.Comment('This IterationFold is "hiding" one or more Iterations')
        code = super(IterationFold, self).ccode
        return c.Module([comment, code])

    def unfold(self):
        """
        Return the corresponding :class:`Iteration` objects from each fold in ``self``.
        """
        args = self.args
        args.pop('folds')

        # Construct the root Iteration
        root = Iteration(**args)

        # Construct the folds
        args.pop('nodes')
        ofs = args.pop('offsets')
        try:
            start, end, incr = args.pop('limits')
        except TypeError:
            start, end, incr = self.limits
        folds = tuple(Iteration(nodes, limits=(start, end, incr),
                                offsets=tuple(i-j for i, j in zip(ofs, shift)), **args)
                      for shift, nodes in self.folds)

        return folds + as_tuple(root)
<|endoftext|>


<commit_before>#!/usr/bin/python

def he_control_loop(dummy,state):
  from time import sleep
  from datetime import datetime, timedelta
  import RPi.GPIO as GPIO
  import config as conf

  GPIO.setmode(GPIO.BCM)
  GPIO.setup(conf.he_pin, GPIO.OUT)
  GPIO.output(conf.he_pin,0)

  heating = False

  try:
    while True:
      if state['snoozeon'] == True :
        now = datetime.now()
        dt = datetime.strptime(state['snooze'],'%H:%M')
        if dt.hour == now.hour and dt.minute == now.minute :
          state['snoozeon'] = False

      avgpid = state['avgpid']

      if state['snoozeon']:
        state['heating'] = False
        GPIO.output(conf.he_pin,0)
        sleep(1)
      else:
        if avgpid >= 100 :
          state['heating'] = True
          GPIO.output(conf.he_pin,1)
          sleep(1)
        #elif avgpid > 0 and avgpid < 100 and state['tempf'] < state['settemp']+1 :
        elif avgpid > 0 and avgpid < 100:
          state['heating'] = True
          GPIO.output(conf.he_pin,1)
          sleep(avgpid/100.)
          GPIO.output(conf.he_pin,0)
          sleep(1-(avgpid/100.))
          state['heating'] = False
        else:
          GPIO.output(conf.he_pin,0)
          state['heating'] = False
          sleep(1)

  finally:
    GPIO.output(conf.he_pin,0)
    GPIO.cleanup()

def pid_loop(dummy,state):
  import sys
  from time import sleep, time
  from math import isnan
  import Adafruit_GPIO.SPI as SPI
  import Adafruit_MAX31855.MAX31855 as MAX31855
  import PID as PID
  import config as conf

  def c_to_f(c):
    return c * 9.0 / 5.0 + 32.0

  sensor = MAX31855.MAX31855(spi=SPI.SpiDev(conf.spi_port, conf.spi_dev))

  pid = PID.PID(conf.Pcold,conf.Icold,conf.Dcold)
  pid.SetPoint = state['settemp']
  pid.setSampleTime(conf.sample_time*5)

  nanct=0
  i=0
  j=0
  pidhist = [0.,0.,0.,0.,0.,0.,0.,0.,0.,0.]
  avgpid = 0.
  temphist = [0.,0.,0.,0.,0.]
  avgtemp = 0.
  lastsettemp = state['settemp']
  lasttime = time()
  sleeptime = 0
  lastcold = 0
  iscold = True

  try:
    while True : # Loops 10x/second
      tempc = sensor.readTempC()
      if isnan(tempc) :
        nanct += 1
        if nanct > 100000 :
          sys.exit
        continue
      else:
        nanct = 0

      tempf = c_to_f(tempc)
      temphist[i%5] = tempf
      avgtemp = sum(temphist)/len(temphist)

      if state['settemp'] != lastsettemp :
        pid.SetPoint = state['settemp']
        lastsettemp = state['settemp']

      if avgtemp < 140:
        lastcold = i

      #if i-(60/conf.sample_time*15) > lastcold:
      #  #machine is warm
      #  pid.setKp(conf.Pwarm)
      #  pid.setKi(conf.Iwarm)
      #  pid.setKd(conf.Dwarm)
      #  iscold = False
      #else:
      #  #machine is cold
      #  pid.setKp(conf.Pcold)
      #  pid.setKi(conf.Icold)
      #  pid.setKd(conf.Dcold)
      #  iscold = True

      if i%10 == 0 :
        pid.update(avgtemp)
        pidout = pid.output
        pidhist[i/10%10] = pidout
        avgpid = sum(pidhist)/len(pidhist)

      state['i'] = i
      state['tempf'] = round(tempf,2)
      state['avgtemp'] = round(avgtemp,2)
      state['pidval'] = round(pidout,2)
      state['avgpid'] = round(avgpid,2)
      state['pterm'] = round(pid.PTerm,2)
      if iscold:
        state['iterm'] = round(pid.ITerm * conf.Icold,2)
        state['dterm'] = round(pid.DTerm * conf.Dcold,2)
      else:
        state['iterm'] = round(pid.ITerm * conf.Iwarm,2)
        state['dterm'] = round(pid.DTerm * conf.Dwarm,2)

      print time(), state

      sleeptime = lasttime+conf.sample_time-time()
      if sleeptime < 0 :
        sleeptime = 0
      sleep(sleeptime)
      i += 1
      lasttime = time()

  finally:
    pid.clear

def rest_server(dummy,state):
  from bottle import route, run, get, post, request, static_file, abort
  from subprocess import call
  from datetime import datetime
  import config as conf
  import os

  basedir = os.path.dirname(__file__)
  wwwdir = basedir+'/www/'

  @route('/')
  def docroot():
    return static_file('index.html',wwwdir)

  @route('/<filepath:path>')
  def servfile(filepath):
    return static_file(filepath,wwwdir)

  @route('/curtemp')
  def curtemp():
    return str(state['avgtemp'])

  @get('/settemp')
  def settemp():
    return str(state['settemp'])

  @post('/settemp')
  def post_settemp():
    try:
      settemp = float(request.forms.get('settemp'))
      if settemp >= 200 and settemp <= 260 :
        state['settemp'] = settemp
        return str(settemp)
      else:
        abort(400,'Set temp out of range 200-260.')
    except:
      abort(400,'Invalid number for set temp.')

  @get('/snooze')
  def get_snooze():
    return str(state['snooze'])

  @post('/snooze')
  def post_snooze():
    snooze = request.forms.get('snooze')
    try:
      datetime.strptime(snooze,'%H:%M')
    except:
      abort(400,'Invalid time format.')
    state['snoozeon'] = True
    state['snooze'] = snooze
    return str(snooze)

  @post('/resetsnooze')
  def reset_snooze():
    state['snoozeon'] = False
    return True

  @get('/allstats')
  def allstats():
    return dict(state)

  @route('/restart')
  def restart():
    call(["reboot"])
    return '';

  @route('/shutdown')
  def shutdown():
    call(["shutdown","-h","now"])
    return '';

  @get('/healthcheck')
  def healthcheck():
    return 'OK'

  run(host='0.0.0.0',port=conf.port,server='cherrypy')

if __name__ == '__main__':
  from multiprocessing import Process, Manager
  from time import sleep
  from urllib2 import urlopen
  import config as conf

  manager = Manager()
  pidstate = manager.dict()
  pidstate['snooze'] = conf.snooze
  pidstate['snoozeon'] = False
  pidstate['i'] = 0
  pidstate['settemp'] = conf.set_temp
  pidstate['avgpid'] = 0.

  p = Process(target=pid_loop,args=(1,pidstate))
  p.daemon = True
  p.start()

  h = Process(target=he_control_loop,args=(1,pidstate))
  h.daemon = True
  h.start()

  r = Process(target=rest_server,args=(1,pidstate))
  r.daemon = True
  r.start()

  #Start Watchdog loop
  piderr = 0
  weberr = 0
  weberrflag = 0
  urlhc = 'http://localhost:'+str(conf.port)+'/healthcheck'

  lasti = pidstate['i']
  sleep(1)

  while True:
    curi = pidstate['i']
    if curi == lasti :
      piderr = piderr + 1
    else :
      piderr = 0

    lasti = curi

    if piderr > 9 :
      print 'ERROR IN PID THREAD, RESTARTING'
      p.terminate()
      sleep(60)
      p.run()
      sleep(2)

    try:
      hc = urlopen(urlhc,timeout=2)
    except:
      weberrflag = 1
    else:
      if hc.getcode() != 200 :
        weberrflag = 1

    if weberrflag != 0 :
      weberr = weberr + 1

    if weberr > 9 :
      print 'ERROR IN WEB SERVER THREAD, RESTARTING'
      r.terminate()
      sleep(2)
      r.run()
      sleep(2)

    weberrflag = 0

    sleep(1)
<commit_msg>removed hot/cold, modified watchdog thread<commit_after>#!/usr/bin/python

def he_control_loop(dummy,state):
  from time import sleep
  from datetime import datetime, timedelta
  import RPi.GPIO as GPIO
  import config as conf

  GPIO.setmode(GPIO.BCM)
  GPIO.setup(conf.he_pin, GPIO.OUT)
  GPIO.output(conf.he_pin,0)

  heating = False

  try:
    while True:
      if state['snoozeon'] == True :
        now = datetime.now()
        dt = datetime.strptime(state['snooze'],'%H:%M')
        if dt.hour == now.hour and dt.minute == now.minute :
          state['snoozeon'] = False

      avgpid = state['avgpid']

      if state['snoozeon']:
        state['heating'] = False
        GPIO.output(conf.he_pin,0)
        sleep(1)
      else:
        if avgpid >= 100 :
          state['heating'] = True
          GPIO.output(conf.he_pin,1)
          sleep(1)
        #elif avgpid > 0 and avgpid < 100 and state['tempf'] < state['settemp']+1 :
        elif avgpid > 0 and avgpid < 100:
          state['heating'] = True
          GPIO.output(conf.he_pin,1)
          sleep(avgpid/100.)
          GPIO.output(conf.he_pin,0)
          sleep(1-(avgpid/100.))
          state['heating'] = False
        else:
          GPIO.output(conf.he_pin,0)
          state['heating'] = False
          sleep(1)

  finally:
    GPIO.output(conf.he_pin,0)
    GPIO.cleanup()

def pid_loop(dummy,state):
  import sys
  from time import sleep, time
  from math import isnan
  import Adafruit_GPIO.SPI as SPI
  import Adafruit_MAX31855.MAX31855 as MAX31855
  import PID as PID
  import config as conf

  def c_to_f(c):
    return c * 9.0 / 5.0 + 32.0

  sensor = MAX31855.MAX31855(spi=SPI.SpiDev(conf.spi_port, conf.spi_dev))

  pid = PID.PID(conf.P,conf.I,conf.D)
  pid.SetPoint = state['settemp']
  pid.setSampleTime(conf.sample_time*5)

  nanct=0
  i=0
  j=0
  pidhist = [0.,0.,0.,0.,0.,0.,0.,0.,0.,0.]
  avgpid = 0.
  temphist = [0.,0.,0.,0.,0.]
  avgtemp = 0.
  lastsettemp = state['settemp']
  lasttime = time()
  sleeptime = 0

  try:
    while True : # Loops 10x/second
      tempc = sensor.readTempC()
      if isnan(tempc) :
        nanct += 1
        if nanct > 100000 :
          sys.exit
        continue
      else:
        nanct = 0

      tempf = c_to_f(tempc)
      temphist[i%5] = tempf
      avgtemp = sum(temphist)/len(temphist)

      if state['settemp'] != lastsettemp :
        pid.SetPoint = state['settemp']
        lastsettemp = state['settemp']

      if i%10 == 0 :
        pid.update(avgtemp)
        pidout = pid.output
        pidhist[i/10%10] = pidout
        avgpid = sum(pidhist)/len(pidhist)

      state['i'] = i
      state['tempf'] = round(tempf,2)
      state['avgtemp'] = round(avgtemp,2)
      state['pidval'] = round(pidout,2)
      state['avgpid'] = round(avgpid,2)
      state['pterm'] = round(pid.PTerm,2)
      state['iterm'] = round(pid.ITerm * conf.I,2)
      state['dterm'] = round(pid.DTerm * conf.D,2)

      print time(), state

      sleeptime = lasttime+conf.sample_time-time()
      if sleeptime < 0 :
        sleeptime = 0
      sleep(sleeptime)
      i += 1
      lasttime = time()

  finally:
    pid.clear

def rest_server(dummy,state):
  from bottle import route, run, get, post, request, static_file, abort
  from subprocess import call
  from datetime import datetime
  import config as conf
  import os

  basedir = os.path.dirname(__file__)
  wwwdir = basedir+'/www/'

  @route('/')
  def docroot():
    return static_file('index.html',wwwdir)

  @route('/<filepath:path>')
  def servfile(filepath):
    return static_file(filepath,wwwdir)

  @route('/curtemp')
  def curtemp():
    return str(state['avgtemp'])

  @get('/settemp')
  def settemp():
    return str(state['settemp'])

  @post('/settemp')
  def post_settemp():
    try:
      settemp = float(request.forms.get('settemp'))
      if settemp >= 200 and settemp <= 260 :
        state['settemp'] = settemp
        return str(settemp)
      else:
        abort(400,'Set temp out of range 200-260.')
    except:
      abort(400,'Invalid number for set temp.')

  @get('/snooze')
  def get_snooze():
    return str(state['snooze'])

  @post('/snooze')
  def post_snooze():
    snooze = request.forms.get('snooze')
    try:
      datetime.strptime(snooze,'%H:%M')
    except:
      abort(400,'Invalid time format.')
    state['snoozeon'] = True
    state['snooze'] = snooze
    return str(snooze)

  @post('/resetsnooze')
  def reset_snooze():
    state['snoozeon'] = False
    return True

  @get('/allstats')
  def allstats():
    return dict(state)

  @route('/restart')
  def restart():
    call(["reboot"])
    return '';

  @route('/shutdown')
  def shutdown():
    call(["shutdown","-h","now"])
    return '';

  @get('/healthcheck')
  def healthcheck():
    return 'OK'

  run(host='0.0.0.0',port=conf.port,server='cherrypy')

if __name__ == '__main__':
  from multiprocessing import Process, Manager
  from time import sleep
  from urllib2 import urlopen
  import config as conf

  manager = Manager()
  pidstate = manager.dict()
  pidstate['snooze'] = conf.snooze
  pidstate['snoozeon'] = False
  pidstate['i'] = 0
  pidstate['settemp'] = conf.set_temp
  pidstate['avgpid'] = 0.

  p = Process(target=pid_loop,args=(1,pidstate))
  p.daemon = True
  p.start()

  h = Process(target=he_control_loop,args=(1,pidstate))
  h.daemon = True
  h.start()

  r = Process(target=rest_server,args=(1,pidstate))
  r.daemon = True
  r.start()

  #Start Watchdog loop
  piderr = 0
  weberr = 0
  weberrflag = 0
  urlhc = 'http://localhost:'+str(conf.port)+'/healthcheck'

  lasti = pidstate['i']
  sleep(1)

  while p.is_alive() and h.is_alive() and r.is_alive():
    curi = pidstate['i']
    if curi == lasti :
      piderr = piderr + 1
    else :
      piderr = 0

    lasti = curi

    if piderr > 9 :
      print 'ERROR IN PID THREAD, RESTARTING'
      p.terminate()

    try:
      hc = urlopen(urlhc,timeout=2)
    except:
      weberrflag = 1
    else:
      if hc.getcode() != 200 :
        weberrflag = 1

    if weberrflag != 0 :
      weberr = weberr + 1

    if weberr > 9 :
      print 'ERROR IN WEB SERVER THREAD, RESTARTING'
      r.terminate()

    weberrflag = 0

    sleep(1)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import sys, tempfile, os, configparser, time
from datetime import timedelta, datetime
import subprocess as sp

sample_config = '''[jobe]
# Command to execute
# Can also refer to any files in this folder (i.e sh foo.sh)
command = uname -a

# Base name of your results branch
name = run_date

# Don't wait for the job to complete
detach = yes

# Time to execute the command (server timezone).
# To execute instantly, set this to a date in the past.
run_at = 2000-00-00T12:00:00.0

# Print all debug information
verbose = no
'''

class Worker:
    def __init__(self, job_id):
        self.job_id = job_id

    def spawn(self, detach):
        proc = sp.Popen(["python3", os.path.realpath(__file__), self.job_id], stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)

        if not detach:
            stdout, stderr = proc.communicate()

            p.debug(stdout)
            p.debug(stderr)

    def execute(self, config, repo):
        if config.wait > 0:
            time.sleep(config.wait)
        proc = sp.Popen(config.command, stdout=repo.open_file("stdout.log"),
                stderr=repo.open_file("stderr.log"), cwd=repo.work_dir, shell=True)
        proc.wait(timeout = config.timeout)
        with repo.open_file("exitcode.log") as exit_file:
            print(proc.returncode, file=exit_file)

    def run(self):
        with Repo(repo_dir) as repo:
            repo.clone()
            repo.checkout(self.job_id)
            config = Config(repo, p)
            self.execute(config, repo)
            repo.add_all()
            repo.commit("Job complete")
            repo.push()

class Printer:
    verbose = False

    info = lambda _, output: print('\033[94m>>> ' + output + '\033[0m')
    ok = lambda _, output: print('\033[92m>>> ' + output + '\033[0m')
    warn = lambda _, output: print('\033[93m>>> ' + output + '\033[0m')
    err = lambda _, output: print('\033[91m>>> ' + output + '\033[0m')
    banner = lambda self: self.info("JOBE - git\'in the job done")

    def debug(self, output):
        if self.verbose:
            self.warn(str(output))

class Repo:
    def __init__(self, location):
        self.location = location

    def __enter__(self):
        self.__tmp_dir = tempfile.TemporaryDirectory()
        self.work_dir = self.__tmp_dir.name
        return self

    def __exit__(self, type, value, traceback):
        self.__tmp_dir.cleanup()

    def reset(self):
        p.debug("Rolling back master")
        self.checkout("master")

        f = self.open_file("jobe.ini", 'a+')

        if f.read() != sample_config or len(os.listdir(self.work_dir)) > 2:
                f.close()
                self.git("rm *")

                with self.open_file("jobe.ini", 'w') as new_sample:
                    new_sample.write(sample_config)

                self.add("jobe.ini")
                self.commit("JOBE ready for submissions.")

    def open_file(self, file_name, mode = 'w+'):
        return open(os.path.join(self.work_dir, file_name), mode)

    def clone(self):
        self.git("clone " + self.location + " " + self.work_dir)

    def branch(self, branch_name):
        self.git("checkout -b " + branch_name)

    def checkout(self, branch_name):
        self.git("checkout " + branch_name)

    def add_all(self):
        self.git("add -A")

    def add(self, file_name):
        self.git("add " + file_name)

    def commit(self, message):
        self.git("commit -a -m '" + message + "'")

    def push(self, branch = "--all"):
        print(self.git("push " + branch))

    def short_hash(self):
        return self.git("rev-parse --short HEAD")

    def git(self, command):
        proc = sp.Popen("git --git-dir=" + self.work_dir + "/.git" + " " + command, shell=True, cwd = self.work_dir,
             stdout=sp.PIPE, stderr=sp.PIPE, universal_newlines=True)
        stdout, stderr = proc.communicate()

        p.debug(stdout)
        p.debug(stderr)

        return stdout

class Config:
    filename = 'jobe.ini'
    valid = False

    def __init__(self, repo, printer):
        self.config_file = configparser.ConfigParser()
        self.config_file.read(os.path.join(repo.work_dir, self.filename))

        if not 'jobe' in self.config_file:
            return

        self.command = self.config_file['jobe'].get('command')

        self.job_id = self.config_file['jobe'].get('name', 'job') + '-' + repo.short_hash()

        self.timeout = self.config_file['jobe'].get('timeout', 10)

        self.detach = self.config_file['jobe'].getboolean('detach', False)
        p.verbose = self.config_file['jobe'].getboolean('verbose', False)

        self.run_at = self.config_file['jobe'].get('run_at')
        self.run_at = datetime.strptime(self.run_at, "%Y-%m-%dT%H:%M:%S.%f")

        self.wait = (self.run_at - datetime.now()).total_seconds()
        self.valid = True

def branch_only_master():
    lines = sys.stdin.readlines()
    p.debug(lines)
    refs = lines[0].split(' ')
    return len(lines) == 1 and "refs/heads/master" in refs[2]

if __name__ == "__main__":
    p = Printer()

    repo_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir))

    if len(sys.argv) <= 1:
        # Standard push
        p.banner()
        if branch_only_master():
            with Repo(repo_dir) as repo:
                repo.clone()
                config = Config(repo, p)

                if not config.valid:
                    p.info("Invalid config, adding sample config file.")
                    p.info("Please pull and try again.")
                    repo.reset()
                    repo.push("origin master")
                    sys.exit(0)

                p.info("Received job")
                repo.branch(config.job_id)
                repo.reset()
                repo.push()

                w = Worker(config.job_id)
                w.spawn(config.detach)
                p.ok("Job submitted, id " + config.job_id)
                p.ok("To retrieve your job, do:")
                p.ok("git pull --all && git checkout " + config.job_id)
        else:
            p.warn("Push only to master to create jobs")

    else:
        w = Worker(sys.argv[1])
        w.run()

    sys.exit(0)
<commit_msg>Make sample date format valid<commit_after>#!/usr/bin/env python3

import sys, tempfile, os, configparser, time
from datetime import timedelta, datetime
import subprocess as sp

sample_config = '''[jobe]
# Command to execute
# Can also refer to any files in this folder (i.e sh foo.sh)
command = uname -a

# Base name of your results branch
name = run_date

# Don't wait for the job to complete
detach = yes

# Time to execute the command (server timezone).
# To execute instantly, set this to a date in the past.
run_at = 2000-01-01T12:00:00.0

# Print all debug information
verbose = no
'''

class Worker:
    def __init__(self, job_id):
        self.job_id = job_id

    def spawn(self, detach):
        proc = sp.Popen(["python3", os.path.realpath(__file__), self.job_id], stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)

        if not detach:
            stdout, stderr = proc.communicate()

            p.debug(stdout)
            p.debug(stderr)

    def execute(self, config, repo):
        if config.wait > 0:
            time.sleep(config.wait)
        proc = sp.Popen(config.command, stdout=repo.open_file("stdout.log"),
                stderr=repo.open_file("stderr.log"), cwd=repo.work_dir, shell=True)
        proc.wait(timeout = config.timeout)
        with repo.open_file("exitcode.log") as exit_file:
            print(proc.returncode, file=exit_file)

    def run(self):
        with Repo(repo_dir) as repo:
            repo.clone()
            repo.checkout(self.job_id)
            config = Config(repo, p)
            self.execute(config, repo)
            repo.add_all()
            repo.commit("Job complete")
            repo.push()

class Printer:
    verbose = False

    info = lambda _, output: print('\033[94m>>> ' + output + '\033[0m')
    ok = lambda _, output: print('\033[92m>>> ' + output + '\033[0m')
    warn = lambda _, output: print('\033[93m>>> ' + output + '\033[0m')
    err = lambda _, output: print('\033[91m>>> ' + output + '\033[0m')
    banner = lambda self: self.info("JOBE - git\'in the job done")

    def debug(self, output):
        if self.verbose:
            self.warn(str(output))

class Repo:
    def __init__(self, location):
        self.location = location

    def __enter__(self):
        self.__tmp_dir = tempfile.TemporaryDirectory()
        self.work_dir = self.__tmp_dir.name
        return self

    def __exit__(self, type, value, traceback):
        self.__tmp_dir.cleanup()

    def reset(self):
        p.debug("Rolling back master")
        self.checkout("master")

        f = self.open_file("jobe.ini", 'a+')

        if f.read() != sample_config or len(os.listdir(self.work_dir)) > 2:
                f.close()
                self.git("rm *")

                with self.open_file("jobe.ini", 'w') as new_sample:
                    new_sample.write(sample_config)

                self.add("jobe.ini")
                self.commit("JOBE ready for submissions.")

    def open_file(self, file_name, mode = 'w+'):
        return open(os.path.join(self.work_dir, file_name), mode)

    def clone(self):
        self.git("clone " + self.location + " " + self.work_dir)

    def branch(self, branch_name):
        self.git("checkout -b " + branch_name)

    def checkout(self, branch_name):
        self.git("checkout " + branch_name)

    def add_all(self):
        self.git("add -A")

    def add(self, file_name):
        self.git("add " + file_name)

    def commit(self, message):
        self.git("commit -a -m '" + message + "'")

    def push(self, branch = "--all"):
        print(self.git("push " + branch))

    def short_hash(self):
        return self.git("rev-parse --short HEAD")

    def git(self, command):
        proc = sp.Popen("git --git-dir=" + self.work_dir + "/.git" + " " + command, shell=True, cwd = self.work_dir,
             stdout=sp.PIPE, stderr=sp.PIPE, universal_newlines=True)
        stdout, stderr = proc.communicate()

        p.debug(stdout)
        p.debug(stderr)

        return stdout

class Config:
    filename = 'jobe.ini'
    valid = False

    def __init__(self, repo, printer):
        self.config_file = configparser.ConfigParser()
        self.config_file.read(os.path.join(repo.work_dir, self.filename))

        if not 'jobe' in self.config_file:
            return

        self.command = self.config_file['jobe'].get('command')

        self.job_id = self.config_file['jobe'].get('name', 'job') + '-' + repo.short_hash()

        self.timeout = self.config_file['jobe'].get('timeout', 10)

        self.detach = self.config_file['jobe'].getboolean('detach', False)
        p.verbose = self.config_file['jobe'].getboolean('verbose', False)

        self.run_at = self.config_file['jobe'].get('run_at')
        self.run_at = datetime.strptime(self.run_at, "%Y-%m-%dT%H:%M:%S.%f")

        self.wait = (self.run_at - datetime.now()).total_seconds()
        self.valid = True

def branch_only_master():
    lines = sys.stdin.readlines()
    p.debug(lines)
    refs = lines[0].split(' ')
    return len(lines) == 1 and "refs/heads/master" in refs[2]

if __name__ == "__main__":
    p = Printer()

    repo_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir))

    if len(sys.argv) <= 1:
        # Standard push
        p.banner()
        if branch_only_master():
            with Repo(repo_dir) as repo:
                repo.clone()
                config = Config(repo, p)

                if not config.valid:
                    p.info("Invalid config, adding sample config file.")
                    p.info("Please pull and try again.")
                    repo.reset()
                    repo.push("origin master")
                    sys.exit(0)

                p.info("Received job")
                repo.branch(config.job_id)
                repo.reset()
                repo.push()

                w = Worker(config.job_id)
                w.spawn(config.detach)
                p.ok("Job submitted, id " + config.job_id)
                p.ok("To retrieve your job, do:")
                p.ok("git pull --all && git checkout " + config.job_id)
        else:
            p.warn("Push only to master to create jobs")

    else:
        w = Worker(sys.argv[1])
        w.run()

    sys.exit(0)
<|endoftext|>


<commit_before>from simblin import create_app

application = create_app()
<commit_msg>added lines to wsgi file for virtualenv<commit_after>from simblin import create_app

activate_this = '/var/www/blog/env/bin/activate_this.py'
execfile(activate_this, dict(__file__=activate_this))

application = create_app()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

import mxnet as mx
import sys
import numpy as np
import time
from collections import namedtuple
import logging
from common import Data

def DNN_def(nLayers, nodesPerLayer):
    Param = namedtuple('Param', ["weight", "bias"]) # Currently, param(eters) are not used.
    param = []

    data = mx.sym.Variable('data')
    label = mx.sym.Variable('softmax_label')

    hidden = data
    for i in xrange(nLayers):
        #param.append(Param(weight = mx.sym.Variable("l%d_weight" % (i+1)),
        #                   bias   = mx.sym.Variable("l%d_bias" % (i+1))
        #                  ))
        #next_layer_z = mx.sym.FullyConnected(data=hidden, num_hidden=nodesPerLayer,
        #                                     weight=param[i].weight, bias=param[i].bias,
        #                                     name="Linear%d" % (i+1))
        next_layer_z = mx.sym.FullyConnected(data=hidden, num_hidden=nodesPerLayer,
                                             name="Linear%d" % (i+1))
        hidden = mx.sym.Activation(next_layer_z, act_type="sigmoid", name="Activation%d"%(i+1))

    sm = mx.sym.SoftmaxOutput(data=hidden, label=label, ignore_label=0, use_ignore=False,
                              name='softmax')
    return sm

#def get_initializer():
#    return mx.initializer.Xavier(magnitude=0.05)
#
#def do_traininit(module, data_train, data_val):
#
#    module.bind(data_shapes = data_train.provide_data,
#                label_shapes = data_train.provide_label,
#                for_training = True)
#    module.init_params(initializer=get_initializer())
#
#    module.init_optimizer(kvstore="local",
#                          optimizer="sgd",
#                          optimizer_params={'learning_rate'=0.01})
#
#    while True:
#        for data_batch in data_train:
#            module.forward_backward(data_batch)
#            module.update()

if __name__=="__main__":
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)

    # load data
    lld = Data('lld')
    lld.load_training_data()
    lld.load_test_data()

    #train_iter = mx.io.NDArrayIter(lld.feature_train, lld.label_train[:, 0], batch_size = batchsize)
    #test_iter  = mx.io.NDArrayIter(lld.feature_test, lld.label_test[:, 0], batch_size = batchsize)
    train_iter = mx.io.NDArrayIter(lld.feature_train, batch_size = batchsize)
    test_iter  = mx.io.NDArrayIter(lld.feature_test, batch_size = batchsize)
    train_iter.label = mx.io._init_data(lld.label_train[:, 0], allow_empty=True, default_name='softmax_label')
    test_iter.label  = mx.io._init_data(lld.label_test[:, 0], allow_empty=True, default_name='softmax_label')

    # Define Network Architectures
    nLayers = 3
    nodesPerLayer = 1024
    batchsize = 256
    sym = DNN_def(nLayers, nodesPerLayer)

    # Training the Model
    '''
    # To do
    #module = mx.mod.Module(sym)
    #do_training(module, data_train, data_val)
    '''
    model = mx.model.FeedForward(
        ctx = mx.cpu(0),
        symbol = sym,
        num_epoch = 10,
        learning_rate = 0.1,
        momentum = 0.9,
        wd = 0.00001,
    )
    model.fit(
        X = train_iter,
        eval_data = test_iter,
        batch_end_callback = mx.callback.Speedometer(batchsize, 200),
    )

    print model.score(test_iter)*100, "%"
<commit_msg>fix problem in simpleDNN<commit_after>#!/usr/bin/env python
# encoding: utf-8

import mxnet as mx
import sys
import numpy as np
import time
from collections import namedtuple
import logging
from common import Data

def DNN_def(nLayers, nodesPerLayer):
    Param = namedtuple('Param', ["weight", "bias"]) # Currently, param(eters) are not used.
    param = []

    data = mx.sym.Variable('data')
    label = mx.sym.Variable('softmax_label')

    hidden = data
    for i in xrange(nLayers):
        #param.append(Param(weight = mx.sym.Variable("l%d_weight" % (i+1)),
        #                   bias   = mx.sym.Variable("l%d_bias" % (i+1))
        #                  ))
        #next_layer_z = mx.sym.FullyConnected(data=hidden, num_hidden=nodesPerLayer,
        #                                     weight=param[i].weight, bias=param[i].bias,
        #                                     name="Linear%d" % (i+1))
        next_layer_z = mx.sym.FullyConnected(data=hidden, num_hidden=nodesPerLayer,
                                             name="Linear%d" % (i+1))
        hidden = mx.sym.Activation(next_layer_z, act_type="relu", name="Activation%d"%(i+1))
    hidden = mx.sym.FullyConnected(data = hidden ,num_hidden=5, name="output")
    sm = mx.sym.SoftmaxOutput(data=hidden, label=label, ignore_label=0, use_ignore=False,
                              name='softmax')
    return sm

#def get_initializer():
#    return mx.initializer.Xavier(magnitude=0.05)
#
#def do_traininit(module, data_train, data_val):
#
#    module.bind(data_shapes = data_train.provide_data,
#                label_shapes = data_train.provide_label,
#                for_training = True)
#    module.init_params(initializer=get_initializer())
#
#    module.init_optimizer(kvstore="local",
#                          optimizer="sgd",
#                          optimizer_params={'learning_rate'=0.01})
#
#    while True:
#        for data_batch in data_train:
#            module.forward_backward(data_batch)
#            module.update()

if __name__=="__main__":
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)

    # load data
    lld = Data('lld')
    lld.load_training_data()
    lld.load_test_data()

    # Define Network Architectures
    nLayers = 2
    nodesPerLayer = 1024
    batchsize = 256
    sym = DNN_def(nLayers, nodesPerLayer)

    #train_iter = mx.io.NDArrayIter(lld.feature_train, lld.label_train[:, 0], batch_size = batchsize)
    #test_iter  = mx.io.NDArrayIter(lld.feature_test, lld.label_test[:, 0], batch_size = batchsize)
    train_iter = mx.io.NDArrayIter(lld.feature_train, batch_size = batchsize)

    test_iter  = mx.io.NDArrayIter(lld.feature_test, batch_size = batchsize)
    train_iter.label = mx.io._init_data(lld.label_train[:, 0], allow_empty=True, default_name='softmax_label')
    test_iter.label  = mx.io._init_data(lld.label_test[:, 0], allow_empty=True, default_name='softmax_label')
    # train_iter.next()
    # print train_iter.getdata()[0].asnumpy()[:10,0]

    # Training the Model
    '''
    # To do
    #module = mx.mod.Module(sym)
    #do_training(module, data_train, data_val)
    '''
    model = mx.model.FeedForward(
        ctx = mx.cpu(0),
        symbol = sym,
        num_epoch = 10,
        learning_rate = 0.001,
        momentum = 0.9,
        wd = 0.00001,
    )
    model.fit(
        X = train_iter,
        eval_data = train_iter,
        eval_metric        = ['accuracy'],
        batch_end_callback = mx.callback.Speedometer(batchsize, 200),
    )

    # print model.predict(train_iter)
    print model.score(test_iter)*100, "%"
<|endoftext|>


<commit_before>from pygit2 import *
import collections
import enum
from hashwords import FriendlyHash
class NoRepoError(Exception) : pass

StatusEntry = collections.namedtuple("StatusEntry", "file statuses")

def statusforcode(code):
	statuses = []
	if code & GIT_STATUS_INDEX_NEW > 0 or code & GIT_STATUS_WT_NEW > 0:
		statuses += ["new"]
	if code & GIT_STATUS_INDEX_MODIFIED > 0 or code & GIT_STATUS_WT_MODIFIED > 0:
		statuses += ["modified"]
	if code & GIT_STATUS_INDEX_DELETED > 0 or code & GIT_STATUS_WT_DELETED > 0:
		statuses += ["deleted"]
	if code & GIT_STATUS_IGNORED > 0:
		statuses += ["ignored"]
	if len(statuses) == 0:
		statuses += ["current"]
	return statuses

class Repo:
	def __init__(self, directory, force=False):
		path = ""
		self.git = None
		try:
			path = discover_repository(directory)
			self.git = Repository(path)
			self.__master = self.git
		except KeyError as err:
			print(err)
			if force:
				self.git = init_repository(directory)
			else:
				raise NoRepoError("{0} does not contain a repository.".format(directory))

	def status(self):
		return [StatusEntry(x[0], statusforcode(x[1])) for x in self.git.status().items()]

	def log_entries(self):
		last = self.git[self.git.head.target]
		for commit in self.git.walk(last.id, GIT_SORT_TIME):
			yield (FriendlyHash(commit.id), commit)

<commit_msg>Updated simplegit to use the new module convention.<commit_after>from pygit2 import *
import collections
import enum
from hash.words import dehash
class NoRepoError(Exception) : pass

StatusEntry = collections.namedtuple("StatusEntry", "file statuses")

def statusforcode(code):
	statuses = []
	if code & GIT_STATUS_INDEX_NEW > 0 or code & GIT_STATUS_WT_NEW > 0:
		statuses += ["new"]
	if code & GIT_STATUS_INDEX_MODIFIED > 0 or code & GIT_STATUS_WT_MODIFIED > 0:
		statuses += ["modified"]
	if code & GIT_STATUS_INDEX_DELETED > 0 or code & GIT_STATUS_WT_DELETED > 0:
		statuses += ["deleted"]
	if code & GIT_STATUS_IGNORED > 0:
		statuses += ["ignored"]
	if len(statuses) == 0:
		statuses += ["current"]
	return statuses

class Repo:
	def __init__(self, directory, force=False):
		path = ""
		self.git = None
		try:
			path = discover_repository(directory)
			self.git = Repository(path)
			self.__master = self.git
		except KeyError as err:
			print(err)
			if force:
				self.git = init_repository(directory)
			else:
				raise NoRepoError("{0} does not contain a repository.".format(directory))

	def status(self):
		return [StatusEntry(x[0], statusforcode(x[1])) for x in self.git.status().items()]

	def log_entries(self):
		last = self.git[self.git.head.target]
		for commit in self.git.walk(last.id, GIT_SORT_TIME):
			yield (dehash(str(commit.id)), commit)

<|endoftext|>


<commit_before># Python script to pre-process MOFFITT data:
# o reorganizes the directory structure into LabCAS archive format
# o builds the file description by using the MOFIITT filename conventions
# o creates the dataset metadata file

import os
import sys
from glob import glob
from shutil import copyfile
import pydicom
from gov.nasa.jpl.edrn.labcas.server.dicom_met_extractor import get_top_dataset_name

# 1)
# COLLECTION_NAME = "Combined_Imaging_and_Blood_Biomarkers_for_Breast_Cancer_Diagnosis"
# INSTITUTION = "Moffitt"

# 2)
# COLLECTION_NAME = "Automated_System_For_Breast_Cancer_Biomarker_Analysis"
# INSTITUTION = "Moffitt"

# 3)
COLLECTION_NAME='Automated_Quantitative_Measures_of_Breast_Density_Data'
INSTITUTION = "Moffitt"

DATA_DIR=os.environ['LABCAS_ARCHIVE'] + "/" + COLLECTION_NAME
TEMPLATE_FILE = "%s/TEMPLATE_%s.cfg" % (DATA_DIR, INSTITUTION)

def read_metadata_from_template(datasetName):

    # read in template metadata file
    with open(TEMPLATE_FILE) as f:
        metadata = f.read()

    # replace metadata
    datasetDescription = datasetName + " mammography images"
    metadata = metadata.replace("DATASET_NAME", datasetName)
    metadata = metadata.replace("DATASET_DESCRIPTION", datasetDescription)

    return metadata

def write_metadata(dirPath, metadata):

    (parentDir, thisDir) = os.path.split(dirPath)
    datasetMetadataFile = dirPath + "/" + thisDir + ".cfg"
    print("\tWriting out metadata to file: %s" % datasetMetadataFile)

    with open(datasetMetadataFile, 'w') as f:
        f.write(metadata)

def get_sub_dataset_name(subDirPath):

    (parentDir, thisDir) = os.path.split(subDirPath)

    if thisDir.lower() == 'primary':
        subDatasetName = 'Primary Images'
    elif thisDir.lower() == 'mammograms':
        subDatasetName = 'Mammograms'
    elif thisDir.lower() == 'truth':
        subDatasetName = 'Truth files'
    elif thisDir.lower() == '2d':
        subDatasetName = 'Full-Field Digital Mammography (FFDM) images (2D)'
    elif thisDir.lower() == 'volume':
        subDatasetName = 'Digital Breast Tomosynthesis  (DBT) images (3D)'
    elif thisDir.lower() == 'proc':
        subDatasetName = 'Processed (for display)'
    elif thisDir.lower() == 'raw':
        subDatasetName = 'Raw (for processing)'
    elif thisDir.lower() == 'cview':
        subDatasetName = 'C-View'
    elif thisDir.lower() == 'mask':
        subDatasetName = 'Mask'
    else:
        subDatasetName = subDirPath

    print("\tSub-dataset name=%s" % subDatasetName)
    return subDatasetName

def main():

    # loop over 1st level directories
    subDirs = os.listdir(DATA_DIR)
    for subDir in subDirs:

        subDirPath = os.path.join(DATA_DIR, subDir)
        if os.path.isdir(subDirPath):
            print("Processing directory: %s" % subDir)

            # read and parse metadata from template file
            datasetName = get_top_dataset_name(subDir[0], subDir[1:5])
            metadata = read_metadata_from_template(datasetName)

            # write out the dataset metadata to file
            write_metadata(subDirPath, metadata)

        # traverse all sub-directories
        for dirName, subDirList, fileList in os.walk(subDirPath):

            # skip the first directory
            if dirName != subDirPath:
                print('\tProcessing sub-directory: %s' % dirName)
                subDatasetName = get_sub_dataset_name(dirName)

                # write out the sub-dataset metadata to file
                metadata = "[Dataset]"
                metadata += "\nDatasetName=%s" % subDatasetName
                metadata += "\n"
                write_metadata(dirName, metadata)

if __name__ == "__main__":
    main()
<commit_msg>Updating the preprocessing script for Duke's mammographies.<commit_after># Python script to pre-process MOFFITT data:
# o reorganizes the directory structure into LabCAS archive format
# o builds the file description by using the MOFIITT filename conventions
# o creates the dataset metadata file

import os
import sys
from glob import glob
from shutil import copyfile
import pydicom
from gov.nasa.jpl.edrn.labcas.server.dicom_met_extractor import get_top_dataset_name

# 1)
COLLECTION_NAME = "Combined_Imaging_and_Blood_Biomarkers_for_Breast_Cancer_Diagnosis"
# INSTITUTION = "Moffitt"
INSTITUTION = "Duke"

# 2)
# COLLECTION_NAME = "Automated_System_For_Breast_Cancer_Biomarker_Analysis"
# INSTITUTION = "Moffitt"

# 3)
#COLLECTION_NAME='Automated_Quantitative_Measures_of_Breast_Density_Data'
#INSTITUTION = "Moffitt"

DATA_DIR=os.environ['LABCAS_ARCHIVE'] + "/" + COLLECTION_NAME
TEMPLATE_FILE = "%s/TEMPLATE_%s.cfg" % (DATA_DIR, INSTITUTION)

def read_metadata_from_template(datasetName):

    # read in template metadata file
    with open(TEMPLATE_FILE) as f:
        metadata = f.read()

    # replace metadata
    datasetDescription = datasetName + " mammography images"
    metadata = metadata.replace("DATASET_NAME", datasetName)
    metadata = metadata.replace("DATASET_DESCRIPTION", datasetDescription)

    return metadata

def write_metadata(dirPath, metadata):

    (parentDir, thisDir) = os.path.split(dirPath)
    datasetMetadataFile = dirPath + "/" + thisDir + ".cfg"
    print("\tWriting out metadata to file: %s" % datasetMetadataFile)

    with open(datasetMetadataFile, 'w') as f:
        f.write(metadata)

def get_sub_dataset_name(subDirPath):

    (parentDir, thisDir) = os.path.split(subDirPath)

    if thisDir.lower() == 'primary':
        subDatasetName = 'Primary Images'
    elif thisDir.lower() == 'mammograms':
        subDatasetName = 'Mammograms'
    elif thisDir.lower() == 'truth':
        subDatasetName = 'Truth files'
    elif thisDir.lower() == '2d':
        subDatasetName = 'Full-Field Digital Mammography (FFDM) images (2D)'
    elif thisDir.lower() == 'volume':
        subDatasetName = 'Digital Breast Tomosynthesis  (DBT) images (3D)'
    elif thisDir.lower() == 'proc':
        subDatasetName = 'Processed (for display)'
    elif thisDir.lower() == 'raw':
        subDatasetName = 'Raw (for processing)'
    elif thisDir.lower() == 'cview':
        subDatasetName = 'C-View'
    elif thisDir.lower() == 'mask':
        subDatasetName = 'Mask'
    else:
        subDatasetName = subDirPath

    print("\tSub-dataset name=%s" % subDatasetName)
    return subDatasetName

def main():

    # loop over 1st level directories
    subDirs = os.listdir(DATA_DIR)
    for subDir in subDirs:

        subDirPath = os.path.join(DATA_DIR, subDir)
        if os.path.isdir(subDirPath):
            print("Processing directory: %s" % subDir)

            # read and parse metadata from template file
            datasetName = get_top_dataset_name(subDir[0], subDir[1:5])
            metadata = read_metadata_from_template(datasetName)

            # write out the dataset metadata to file
            write_metadata(subDirPath, metadata)

        # traverse all sub-directories
        for dirName, subDirList, fileList in os.walk(subDirPath):

            # skip the first directory
            if dirName != subDirPath:
                print('\tProcessing sub-directory: %s' % dirName)
                subDatasetName = get_sub_dataset_name(dirName)

                # write out the sub-dataset metadata to file
                metadata = "[Dataset]"
                metadata += "\nDatasetName=%s" % subDatasetName
                metadata += "\n"
                write_metadata(dirName, metadata)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import requests
import re
from BeautifulSoup import BeautifulSoup

class Show:
    def __init__(self, name, url):
        self.name = name
        self.url = url

    def name(self):
        return self.name

    def url(self):
        return self.name

class Shows:
    def __init__(self, name):
        self.name = name
        self.list = []

    def add(self, show):
        self.list.append(show)

    def all(self):
        return self.list

class Episode:
    def __init__(self, name, url):
        self.name = name
        self.url = url

    def name(self):
        return self.name

    def url(self):
        return self.name

class Episodes:
    def __init__(self, name, image):
        self.name = name
        self.list = []
        self.image = image

    def add(self, episode):
        self.list.append(episode)

    def all(self):
        return self.list

    def image(self):
        return self.image

class PlayerSite:
    def __init__(self, url):
        self.url = url
        self.name = re.sub(r".*?://([^/]*)/.*", r"\1", url)

    def url(self):
        return self.url

    def name(self):
        return self.name

class PlayerSites:
    def __init__(self, name, image):
        self.name = name
        self.list = []
        self.image = image

    def add(self, stream):
        self.list.append(stream)

    def all(self):
        return self.list

    def image(self):
        return self.image

def shows():
    page = requests.get('http://www.tvseriesonline.pl')
    soup = BeautifulSoup(page.content, convertEntities=BeautifulSoup.HTML_ENTITIES)
    links = soup.body.find('ul', {"id": "categories"}).findAll('a')

    show_list = Shows("All")
    for link in links:
        show_list.add(Show(link.text.encode('utf-8'), link.get("href").encode('utf-8')))

    return show_list

def episodes(show_url):
    page = requests.get(show_url)
    soup = BeautifulSoup(page.content, convertEntities=BeautifulSoup.HTML_ENTITIES)
    headers = soup.body.find('div', {"id": "contentwrap2"}).findAll('h3')
    title = soup.body.find('div', {"class": "catdesc"}).find('h2').text
    image = soup.body.find('div', {"class": "catdesc"}).find('div', {"class": "catImage"}).find('img').get('src')

    episode_list = Episodes(title.encode('utf-8'), image.encode('utf-8'))
    for header in headers:
        for link in header.findAll('a'):
            episode_list.add(Episode(link.text.encode('utf-8'), link.get("href").encode('utf-8')))

    return episode_list

def player_sites(episode_url):
    page = requests.get(episode_url)
    soup = BeautifulSoup(page.content, convertEntities=BeautifulSoup.HTML_ENTITIES)
    main_link = soup.body.find('div', {"class": "seriale"}).findNextSibling('a').get('href')
    other_links = soup.body.find('div', {"class": "seriale"}).findAll('li')
    image = soup.body.find('div', {"class": "catdesc"}).find('div', {"class": "catImage"}).find('img').get('src')

    sites = PlayerSites("All", image)

    page = requests.get(main_link)
    link = re.search('.*window.location.href=\'(.*)\'.*', page.content).group(1)
    sites.add(PlayerSite(link))

    for link in other_links:
        for a in link.findAll('a'):
            sites.add(PlayerSite(a.get('href').encode('utf-8')))

    return sites
<commit_msg>Fix the main link retrieval<commit_after>import requests
import re
from BeautifulSoup import BeautifulSoup

class Show:
    def __init__(self, name, url):
        self.name = name
        self.url = url

    def name(self):
        return self.name

    def url(self):
        return self.name

class Shows:
    def __init__(self, name):
        self.name = name
        self.list = []

    def add(self, show):
        self.list.append(show)

    def all(self):
        return self.list

class Episode:
    def __init__(self, name, url):
        self.name = name
        self.url = url

    def name(self):
        return self.name

    def url(self):
        return self.name

class Episodes:
    def __init__(self, name, image):
        self.name = name
        self.list = []
        self.image = image

    def add(self, episode):
        self.list.append(episode)

    def all(self):
        return self.list

    def image(self):
        return self.image

class PlayerSite:
    def __init__(self, url):
        self.url = url
        self.name = re.sub(r".*?://([^/]*)/.*", r"\1", url)

    def url(self):
        return self.url

    def name(self):
        return self.name

class PlayerSites:
    def __init__(self, name, image):
        self.name = name
        self.list = []
        self.image = image

    def add(self, stream):
        self.list.append(stream)

    def all(self):
        return self.list

    def image(self):
        return self.image

def shows():
    page = requests.get('http://www.tvseriesonline.pl')
    soup = BeautifulSoup(page.content, convertEntities=BeautifulSoup.HTML_ENTITIES)
    links = soup.body.find('ul', {"id": "categories"}).findAll('a')

    show_list = Shows("All")
    for link in links:
        show_list.add(Show(link.text.encode('utf-8'), link.get("href").encode('utf-8')))

    return show_list

def episodes(show_url):
    page = requests.get(show_url)
    soup = BeautifulSoup(page.content, convertEntities=BeautifulSoup.HTML_ENTITIES)
    headers = soup.body.find('div', {"id": "contentwrap2"}).findAll('h3')
    title = soup.body.find('div', {"class": "catdesc"}).find('h2').text
    image = soup.body.find('div', {"class": "catdesc"}).find('div', {"class": "catImage"}).find('img').get('src')

    episode_list = Episodes(title.encode('utf-8'), image.encode('utf-8'))
    for header in headers:
        for link in header.findAll('a'):
            episode_list.add(Episode(link.text.encode('utf-8'), link.get("href").encode('utf-8')))

    return episode_list

def player_sites(episode_url):
    page = requests.get(episode_url)
    soup = BeautifulSoup(page.content, convertEntities=BeautifulSoup.HTML_ENTITIES)
    main_link = soup.body.find('div', {"class": "seriale"}).findNextSibling('a').get('href')
    other_links = soup.body.find('div', {"class": "seriale"}).findAll('li')
    image = soup.body.find('div', {"class": "catdesc"}).find('div', {"class": "catImage"}).find('img').get('src')

    sites = PlayerSites("All", image)

    page = requests.get(main_link)
    link = re.search('.*window.location.href\s*=\s*\'(.*)\'.*', page.content).group(1)
    sites.add(PlayerSite(link))

    for link in other_links:
        for a in link.findAll('a'):
            sites.add(PlayerSite(a.get('href').encode('utf-8')))

    return sites
<|endoftext|>


<commit_before># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Simple image classification with Inception.

Run image classification with Inception trained on ImageNet 2012 Challenge data
set.

This program creates a graph from a saved GraphDef protocol buffer,
and runs inference on an input JPEG image. It outputs human readable
strings of the top 5 predictions along with their probabilities.

Change the --image_file argument to any jpg image to compute a
classification of that image.

Please see the tutorial and website for a detailed description of how
to use this script to perform image recognition.

https://tensorflow.org/tutorials/image_recognition/
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os.path
import re
import sys
import tarfile

import numpy as np
from six.moves import urllib
import tensorflow as tf

FLAGS = tf.app.flags.FLAGS

# classify_image_graph_def.pb:
#   Binary representation of the GraphDef protocol buffer.
# imagenet_synset_to_human_label_map.txt:
#   Map from synset ID to a human readable string.
# imagenet_2012_challenge_label_map_proto.pbtxt:
#   Text representation of a protocol buffer mapping a label to synset ID.
tf.app.flags.DEFINE_string(
    'model_dir', '/tmp/imagenet',
    """Path to classify_image_graph_def.pb, """
    """imagenet_synset_to_human_label_map.txt, and """
    """imagenet_2012_challenge_label_map_proto.pbtxt.""")
tf.app.flags.DEFINE_string('image_file', '',
                           """Absolute path to image file.""")
tf.app.flags.DEFINE_integer('num_top_predictions', 5,
                            """Display this many predictions.""")

# pylint: disable=line-too-long
DATA_URL = 'http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz'
# pylint: enable=line-too-long

class NodeLookup(object):
  """Converts integer node ID's to human readable labels."""

  def __init__(self,
               label_lookup_path=None,
               uid_lookup_path=None):
    if not label_lookup_path:
      label_lookup_path = os.path.join(
          FLAGS.model_dir, 'imagenet_2012_challenge_label_map_proto.pbtxt')
    if not uid_lookup_path:
      uid_lookup_path = os.path.join(
          FLAGS.model_dir, 'imagenet_synset_to_human_label_map.txt')
    self.node_lookup = self.load(label_lookup_path, uid_lookup_path)

  def load(self, label_lookup_path, uid_lookup_path):
    """Loads a human readable English name for each softmax node.

    Args:
      label_lookup_path: string UID to integer node ID.
      uid_lookup_path: string UID to human-readable string.

    Returns:
      dict from integer node ID to human-readable string.
    """
    if not tf.gfile.Exists(uid_lookup_path):
      tf.logging.fatal('File does not exist %s', uid_lookup_path)
    if not tf.gfile.Exists(label_lookup_path):
      tf.logging.fatal('File does not exist %s', label_lookup_path)

    # Loads mapping from string UID to human-readable string
    proto_as_ascii_lines = tf.gfile.GFile(uid_lookup_path).readlines()
    uid_to_human = {}
    p = re.compile(r'[n\d]*[ \S,]*')
    for line in proto_as_ascii_lines:
      parsed_items = p.findall(line)
      uid = parsed_items[0]
      human_string = parsed_items[2]
      uid_to_human[uid] = human_string

    # Loads mapping from string UID to integer node ID.
    node_id_to_uid = {}
    proto_as_ascii = tf.gfile.GFile(label_lookup_path).readlines()
    for line in proto_as_ascii:
      if line.startswith('  target_class:'):
        target_class = int(line.split(': ')[1])
      if line.startswith('  target_class_string:'):
        target_class_string = line.split(': ')[1]
        node_id_to_uid[target_class] = target_class_string[1:-2]

    # Loads the final mapping of integer node ID to human-readable string
    node_id_to_name = {}
    for key, val in node_id_to_uid.items():
      if val not in uid_to_human:
        tf.logging.fatal('Failed to locate: %s', val)
      name = uid_to_human[val]
      node_id_to_name[key] = name

    return node_id_to_name

  def id_to_string(self, node_id):
    if node_id not in self.node_lookup:
      return ''
    return self.node_lookup[node_id]

def create_graph():
  """Creates a graph from saved GraphDef file and returns a saver."""
  # Creates graph from saved graph_def.pb.
  with tf.gfile.FastGFile(os.path.join(
      FLAGS.model_dir, 'classify_image_graph_def.pb'), 'rb') as f:
    graph_def = tf.GraphDef()
    graph_def.ParseFromString(f.read())
    _ = tf.import_graph_def(graph_def, name='')

def run_inference_on_image(image):
  """Runs inference on an image.

  Args:
    image: Image file name.

  Returns:
    Nothing
  """
  if not tf.gfile.Exists(image):
    tf.logging.fatal('File does not exist %s', image)
  image_data = tf.gfile.FastGFile(image, 'rb').read()

  # Creates graph from saved GraphDef.
  create_graph()

  with tf.Session() as sess:
    # Some useful tensors:
    # 'softmax:0': A tensor containing the normalized prediction across
    #   1000 labels.
    # 'pool_3:0': A tensor containing the next-to-last layer containing 2048
    #   float description of the image.
    # 'DecodeJpeg/contents:0': A tensor containing a string providing JPEG
    #   encoding of the image.
    # Runs the softmax tensor by feeding the image_data as input to the graph.
    softmax_tensor = sess.graph.get_tensor_by_name('softmax:0')
    predictions = sess.run(softmax_tensor,
                           {'DecodeJpeg/contents:0': image_data})
    predictions = np.squeeze(predictions)

    # Creates node ID --> English string lookup.
    node_lookup = NodeLookup()

    top_k = predictions.argsort()[-FLAGS.num_top_predictions:][::-1]
    for node_id in top_k:
      human_string = node_lookup.id_to_string(node_id)
      score = predictions[node_id]
      print('%s (score = %.5f)' % (human_string, score))

def maybe_download_and_extract():
  """Download and extract model tar file."""
  dest_directory = FLAGS.model_dir
  if not os.path.exists(dest_directory):
    os.makedirs(dest_directory)
  filename = DATA_URL.split('/')[-1]
  filepath = os.path.join(dest_directory, filename)
  if not os.path.exists(filepath):
    def _progress(count, block_size, total_size):
      sys.stdout.write('\r>> Downloading %s %.1f%%' % (
          filename, float(count * block_size) / float(total_size) * 100.0))
      sys.stdout.flush()
    filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, _progress)
    print()
    statinfo = os.stat(filepath)
    print('Succesfully downloaded', filename, statinfo.st_size, 'bytes.')
  tarfile.open(filepath, 'r:gz').extractall(dest_directory)

def main(_):
  maybe_download_and_extract()
  image = (FLAGS.image_file if FLAGS.image_file else
           os.path.join(FLAGS.model_dir, 'cropped_panda.jpg'))
  run_inference_on_image(image)

if __name__ == '__main__':
  tf.app.run()
<commit_msg>fix for TIKA-2306 contributed by kranthigv<commit_after># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Simple image classification with Inception.

Run image classification with Inception trained on ImageNet 2012 Challenge data
set.

This program creates a graph from a saved GraphDef protocol buffer,
and runs inference on an input JPEG image. It outputs human readable
strings of the top 5 predictions along with their probabilities.

Change the --image_file argument to any jpg image to compute a
classification of that image.

Please see the tutorial and website for a detailed description of how
to use this script to perform image recognition.

https://tensorflow.org/tutorials/image_recognition/
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os.path
import re
import sys
import tarfile

import numpy as np
from six.moves import urllib
import tensorflow as tf

from datasets import imagenet, dataset_utils
from nets import inception
from preprocessing import inception_preprocessing

slim = tf.contrib.slim

FLAGS = tf.app.flags.FLAGS

# classify_image_graph_def.pb:
#   Binary representation of the GraphDef protocol buffer.
# imagenet_synset_to_human_label_map.txt:
#   Map from synset ID to a human readable string.
# imagenet_2012_challenge_label_map_proto.pbtxt:
#   Text representation of a protocol buffer mapping a label to synset ID.
tf.app.flags.DEFINE_string(
    'model_dir', '/tmp/imagenet',
    """Path to classify_image_graph_def.pb, """
    """imagenet_synset_to_human_label_map.txt, and """
    """imagenet_2012_challenge_label_map_proto.pbtxt.""")
tf.app.flags.DEFINE_string('image_file', '',
                           """Absolute path to image file.""")
tf.app.flags.DEFINE_integer('num_top_predictions', 5,
                            """Display this many predictions.""")

# pylint: disable=line-too-long
DATA_URL = 'http://download.tensorflow.org/models/inception_v4_2016_09_09.tar.gz'
# pylint: enable=line-too-long

def create_readable_names_for_imagenet_labels():
  """Create a dict mapping label id to human readable string.

  Returns:
      labels_to_names: dictionary where keys are integers from to 1000
      and values are human-readable names.

  We retrieve a synset file, which contains a list of valid synset labels used
  by ILSVRC competition. There is one synset one per line, eg.
          #   n01440764
          #   n01443537
  We also retrieve a synset_to_human_file, which contains a mapping from synsets
  to human-readable names for every synset in Imagenet. These are stored in a
  tsv format, as follows:
          #   n02119247    black fox
          #   n02119359    silver fox
  We assign each synset (in alphabetical order) an integer, starting from 1
  (since 0 is reserved for the background class).

  Code is based on
  https://github.com/tensorflow/models/blob/master/inception/inception/data/build_imagenet_data.py#L463
  """

  # pylint: disable=line-too-long

  dest_directory = FLAGS.model_dir

  synset_list = [s.strip() for s in open(os.path.join(dest_directory, 'imagenet_lsvrc_2015_synsets.txt')).readlines()]
  num_synsets_in_ilsvrc = len(synset_list)
  assert num_synsets_in_ilsvrc == 1000

  synset_to_human_list = open(os.path.join(dest_directory, 'imagenet_metadata.txt')).readlines()
  num_synsets_in_all_imagenet = len(synset_to_human_list)
  assert num_synsets_in_all_imagenet == 21842

  synset_to_human = {}
  for s in synset_to_human_list:
    parts = s.strip().split('\t')
    assert len(parts) == 2
    synset = parts[0]
    human = parts[1]
    synset_to_human[synset] = human

  label_index = 1
  labels_to_names = {0: 'background'}
  for synset in synset_list:
    name = synset_to_human[synset]
    labels_to_names[label_index] = name
    label_index += 1

  return labels_to_names

def run_inference_on_image(image):
  """Runs inference on an image.

  Args:
    image: Image file name.

  Returns:
    Nothing
  """
  dest_directory = FLAGS.model_dir

  image_size = inception.inception_v4.default_image_size

  if not tf.gfile.Exists(image):
    tf.logging.fatal('File does not exist %s', image)
  image_string = tf.gfile.FastGFile(image, 'rb').read()

  with tf.Graph().as_default():
    image = tf.image.decode_jpeg(image_string, channels=3)
    processed_image = inception_preprocessing.preprocess_image(image, image_size, image_size, is_training=False)
    processed_images  = tf.expand_dims(processed_image, 0)

    # Create the model, use the default arg scope to configure the batch norm parameters.
    with slim.arg_scope(inception.inception_v4_arg_scope()):
        logits, _ = inception.inception_v4(processed_images, num_classes=1001, is_training=False)
    probabilities = tf.nn.softmax(logits)

    init_fn = slim.assign_from_checkpoint_fn(
        os.path.join(dest_directory, 'inception_v4.ckpt'),
        slim.get_model_variables('InceptionV4'))

    with tf.Session() as sess:
        init_fn(sess)
        np_image, probabilities = sess.run([image, probabilities])
        probabilities = probabilities[0, 0:]
        sorted_inds = [i[0] for i in sorted(enumerate(-probabilities), key=lambda x:x[1])]

    names = create_readable_names_for_imagenet_labels()
    top_k = FLAGS.num_top_predictions
    for i in range(top_k):
        index = sorted_inds[i]
        print('%s (score = %.5f)' % (names[index], probabilities[index]))

def util_download(url, dest_directory):
  filename = url.split('/')[-1]
  filepath = os.path.join(dest_directory, filename)

  def _progress(count, block_size, total_size):
    sys.stdout.write('\r>> Downloading %s %.1f%%' % (
        filename, float(count * block_size) / float(total_size) * 100.0))
    sys.stdout.flush()
  filepath, _ = urllib.request.urlretrieve(url, filepath, _progress)
  print()
  statinfo = os.stat(filepath)
  print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')

def util_download_tar(url, dest_directory):
  filename = url.split('/')[-1]
  filepath = os.path.join(dest_directory, filename)

  def _progress(count, block_size, total_size):
    sys.stdout.write('\r>> Downloading %s %.1f%%' % (
        filename, float(count * block_size) / float(total_size) * 100.0))
    sys.stdout.flush()
  filepath, _ = urllib.request.urlretrieve(url, filepath, _progress)
  print()
  statinfo = os.stat(filepath)
  print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')
  tarfile.open(filepath, 'r:gz').extractall(dest_directory)

def maybe_download_and_extract():
  """Download and extract model tar file."""
  dest_directory = FLAGS.model_dir
  if not tf.gfile.Exists(dest_directory):
    tf.gfile.MakeDirs(dest_directory)
  if not tf.gfile.Exists(os.path.join(dest_directory, 'inception_v4.ckpt')):
    util_download_tar(DATA_URL, dest_directory)
  # pylint: disable=line-too-long
  if not tf.gfile.Exists(os.path.join(dest_directory, 'imagenet_lsvrc_2015_synsets.txt')):
    util_download('https://raw.githubusercontent.com/tensorflow/models/master/inception/inception/data/imagenet_lsvrc_2015_synsets.txt', dest_directory)
  if not tf.gfile.Exists(os.path.join(dest_directory, 'imagenet_metadata.txt')):
    util_download('https://raw.githubusercontent.com/tensorflow/models/master/inception/inception/data/imagenet_metadata.txt', dest_directory)
  # pylint: enable=line-too-long

def main(_):
  maybe_download_and_extract()
  image = (FLAGS.image_file if FLAGS.image_file else
           os.path.join(FLAGS.model_dir, 'lion.jpg'))
  #print(image)
  run_inference_on_image(image)

if __name__ == '__main__':
  tf.app.run()
<|endoftext|>


<commit_before>import sys
import pytest
import nbrmd
from testfixtures import compare

ACTIVE_ALL = {'.py': """# + {"active": "ipynb,py,R,Rmd"}
# This cell is active in all extensions
""",
              '.Rmd': """```{python active="ipynb,py,R,Rmd"}
# This cell is active in all extensions
```
""",
              '.R': """#+ active="ipynb,py,R,Rmd"
# This cell is active in all extensions
""",
              '.ipynb': {'cell_type': 'code',
                         'source': '# This cell is active in all extensions',
                         'metadata': {'active': 'ipynb,py,R,Rmd'},
                         'execution_count': None,
                         'outputs': []}}

@pytest.mark.parametrize('ext', ['.Rmd', '.py', '.R'])
def test_active_all(ext):
    nb = nbrmd.reads(ACTIVE_ALL[ext], ext=ext)
    assert len(nb.cells) == 1
    compare(nb.cells[0], ACTIVE_ALL['.ipynb'])
    compare(ACTIVE_ALL[ext], nbrmd.writes(nb, ext=ext))

ACTIVE_IPYNB = {'.py': """# + {"active": "ipynb", "endofcell": "-"}
# # This cell is active only in ipynb
# %matplotlib inline
# -
""",
                '.Rmd': """```{python active="ipynb", eval=FALSE}
# This cell is active only in ipynb
%matplotlib inline
```
""",
                '.R': """#+ language="python", active="ipynb", eval=FALSE
#' # This cell is active only in ipynb
#' %matplotlib inline
""",
                '.ipynb': {'cell_type': 'code',
                           'source': '# This cell is active only in ipynb\n'
                                     '%matplotlib inline',
                           'metadata': {'active': 'ipynb'},
                           'execution_count': None,
                           'outputs': []}}

@pytest.mark.skipif(sys.version_info < (3, 6),
                    reason="unordered dict result in changes in chunk options")
@pytest.mark.parametrize('ext', ['.Rmd', '.py', '.R'])
def test_active_ipynb(ext):
    nb = nbrmd.reads(ACTIVE_IPYNB[ext], ext=ext)
    assert len(nb.cells) == 1
    compare(nb.cells[0], ACTIVE_IPYNB['.ipynb'])
    if ext != '.R':
        compare(ACTIVE_IPYNB[ext], nbrmd.writes(nb, ext=ext))

ACTIVE_PY_IPYNB = {'.py': """# + {"active": "ipynb,py"}
# This cell is active in py and ipynb extensions
""",
                   '.Rmd': """```{python active="ipynb,py", eval=FALSE}
# This cell is active in py and ipynb extensions
```
""",
                   '.R': """#+ language="python", active="ipynb,py", eval=FALSE
#' # This cell is active in py and ipynb extensions
""",
                   '.ipynb': {'cell_type': 'code',
                              'source': '# This cell is active in py and '
                                        'ipynb extensions',
                              'metadata': {'active': 'ipynb,py'},
                              'execution_count': None,
                              'outputs': []}}

@pytest.mark.skipif(sys.version_info < (3, 6),
                    reason="unordered dict result in changes in chunk options")
@pytest.mark.parametrize('ext', ['.Rmd', '.py', '.R'])
def test_active_py_ipynb(ext):
    nb = nbrmd.reads(ACTIVE_PY_IPYNB[ext], ext=ext)
    assert len(nb.cells) == 1
    compare(nb.cells[0], ACTIVE_PY_IPYNB['.ipynb'])
    if ext != '.R':
        compare(ACTIVE_PY_IPYNB[ext], nbrmd.writes(nb, ext=ext))

ACTIVE_RMD = {'.py': """# + {"active": "Rmd", "endofcell": "-"}
# # This cell is active in Rmd only
# -
""",
              '.Rmd': """```{python active="Rmd"}
# This cell is active in Rmd only
```
""",
              '.R': """#+ language="python", active="Rmd", eval=FALSE
#' # This cell is active in Rmd only
""",
              '.ipynb': {'cell_type': 'raw',
                         'source': '# This cell is active in Rmd only',
                         'metadata': {'active': 'Rmd'}}}

@pytest.mark.skipif(sys.version_info < (3, 6),
                    reason="unordered dict result in changes in chunk options")
@pytest.mark.parametrize('ext', ['.Rmd', '.py', '.R'])
def test_active_rmd(ext):
    nb = nbrmd.reads(ACTIVE_RMD[ext], ext=ext)
    assert len(nb.cells) == 1
    compare(nb.cells[0], ACTIVE_RMD['.ipynb'])
    if ext != '.R':
        compare(ACTIVE_RMD[ext], nbrmd.writes(nb, ext=ext))

ACTIVE_NOT_INCLUDE_RMD = {'.py': """# + {"hide_output": true, "active": "Rmd", "endofcell": "-"}
# # This cell is active in Rmd only
# -
""",
                          '.Rmd': """```{python include=FALSE, active="Rmd"}
# This cell is active in Rmd only
```
""",
                          '.R': """#+ include=FALSE, active="Rmd", eval=FALSE
# # This cell is active in Rmd only
""",
                          '.ipynb':
                              {'cell_type': 'raw',
                               'source': '# This cell is active in Rmd only',
                               'metadata': {'active': 'Rmd',
                                            'hide_output': True}}}

@pytest.mark.parametrize('ext', ['.Rmd', '.py', '.R'])
def test_active_not_include_rmd(ext):
    nb = nbrmd.reads(ACTIVE_NOT_INCLUDE_RMD[ext], ext=ext)
    assert len(nb.cells) == 1
    compare(nb.cells[0], ACTIVE_NOT_INCLUDE_RMD['.ipynb'])
    compare(ACTIVE_NOT_INCLUDE_RMD[ext], nbrmd.writes(nb, ext=ext))
<commit_msg>Skip test when dict order not garanteed<commit_after>import sys
import pytest
import nbrmd
from testfixtures import compare

ACTIVE_ALL = {'.py': """# + {"active": "ipynb,py,R,Rmd"}
# This cell is active in all extensions
""",
              '.Rmd': """```{python active="ipynb,py,R,Rmd"}
# This cell is active in all extensions
```
""",
              '.R': """#+ active="ipynb,py,R,Rmd"
# This cell is active in all extensions
""",
              '.ipynb': {'cell_type': 'code',
                         'source': '# This cell is active in all extensions',
                         'metadata': {'active': 'ipynb,py,R,Rmd'},
                         'execution_count': None,
                         'outputs': []}}

@pytest.mark.parametrize('ext', ['.Rmd', '.py', '.R'])
def test_active_all(ext):
    nb = nbrmd.reads(ACTIVE_ALL[ext], ext=ext)
    assert len(nb.cells) == 1
    compare(nb.cells[0], ACTIVE_ALL['.ipynb'])
    compare(ACTIVE_ALL[ext], nbrmd.writes(nb, ext=ext))

ACTIVE_IPYNB = {'.py': """# + {"active": "ipynb", "endofcell": "-"}
# # This cell is active only in ipynb
# %matplotlib inline
# -
""",
                '.Rmd': """```{python active="ipynb", eval=FALSE}
# This cell is active only in ipynb
%matplotlib inline
```
""",
                '.R': """#+ language="python", active="ipynb", eval=FALSE
#' # This cell is active only in ipynb
#' %matplotlib inline
""",
                '.ipynb': {'cell_type': 'code',
                           'source': '# This cell is active only in ipynb\n'
                                     '%matplotlib inline',
                           'metadata': {'active': 'ipynb'},
                           'execution_count': None,
                           'outputs': []}}

@pytest.mark.skipif(sys.version_info < (3, 6),
                    reason="unordered dict result in changes in chunk options")
@pytest.mark.parametrize('ext', ['.Rmd', '.py', '.R'])
def test_active_ipynb(ext):
    nb = nbrmd.reads(ACTIVE_IPYNB[ext], ext=ext)
    assert len(nb.cells) == 1
    compare(nb.cells[0], ACTIVE_IPYNB['.ipynb'])
    if ext != '.R':
        compare(ACTIVE_IPYNB[ext], nbrmd.writes(nb, ext=ext))

ACTIVE_PY_IPYNB = {'.py': """# + {"active": "ipynb,py"}
# This cell is active in py and ipynb extensions
""",
                   '.Rmd': """```{python active="ipynb,py", eval=FALSE}
# This cell is active in py and ipynb extensions
```
""",
                   '.R': """#+ language="python", active="ipynb,py", eval=FALSE
#' # This cell is active in py and ipynb extensions
""",
                   '.ipynb': {'cell_type': 'code',
                              'source': '# This cell is active in py and '
                                        'ipynb extensions',
                              'metadata': {'active': 'ipynb,py'},
                              'execution_count': None,
                              'outputs': []}}

@pytest.mark.skipif(sys.version_info < (3, 6),
                    reason="unordered dict result in changes in chunk options")
@pytest.mark.parametrize('ext', ['.Rmd', '.py', '.R'])
def test_active_py_ipynb(ext):
    nb = nbrmd.reads(ACTIVE_PY_IPYNB[ext], ext=ext)
    assert len(nb.cells) == 1
    compare(nb.cells[0], ACTIVE_PY_IPYNB['.ipynb'])
    if ext != '.R':
        compare(ACTIVE_PY_IPYNB[ext], nbrmd.writes(nb, ext=ext))

ACTIVE_RMD = {'.py': """# + {"active": "Rmd", "endofcell": "-"}
# # This cell is active in Rmd only
# -
""",
              '.Rmd': """```{python active="Rmd"}
# This cell is active in Rmd only
```
""",
              '.R': """#+ language="python", active="Rmd", eval=FALSE
#' # This cell is active in Rmd only
""",
              '.ipynb': {'cell_type': 'raw',
                         'source': '# This cell is active in Rmd only',
                         'metadata': {'active': 'Rmd'}}}

@pytest.mark.skipif(sys.version_info < (3, 6),
                    reason="unordered dict result in changes in chunk options")
@pytest.mark.parametrize('ext', ['.Rmd', '.py', '.R'])
def test_active_rmd(ext):
    nb = nbrmd.reads(ACTIVE_RMD[ext], ext=ext)
    assert len(nb.cells) == 1
    compare(nb.cells[0], ACTIVE_RMD['.ipynb'])
    if ext != '.R':
        compare(ACTIVE_RMD[ext], nbrmd.writes(nb, ext=ext))

ACTIVE_NOT_INCLUDE_RMD = {'.py': """# + {"hide_output": true, "active": "Rmd", "endofcell": "-"}
# # This cell is active in Rmd only
# -
""",
                          '.Rmd': """```{python include=FALSE, active="Rmd"}
# This cell is active in Rmd only
```
""",
                          '.R': """#+ include=FALSE, active="Rmd", eval=FALSE
# # This cell is active in Rmd only
""",
                          '.ipynb':
                              {'cell_type': 'raw',
                               'source': '# This cell is active in Rmd only',
                               'metadata': {'active': 'Rmd',
                                            'hide_output': True}}}

@pytest.mark.skipif(sys.version_info < (3, 6),
                    reason="unordered dict result in changes in chunk options")
@pytest.mark.parametrize('ext', ['.Rmd', '.py', '.R'])
def test_active_not_include_rmd(ext):
    nb = nbrmd.reads(ACTIVE_NOT_INCLUDE_RMD[ext], ext=ext)
    assert len(nb.cells) == 1
    compare(nb.cells[0], ACTIVE_NOT_INCLUDE_RMD['.ipynb'])
    compare(ACTIVE_NOT_INCLUDE_RMD[ext], nbrmd.writes(nb, ext=ext))
<|endoftext|>