<commit_before># -*- coding: utf-8 -*-
"""
Generators for geometric graphs.
"""
#    Copyright (C) 2004-2011 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.

__author__ = "\n".join(['Aric Hagberg (hagberg@lanl.gov)',
                        'Dan Schult (dschult@colgate.edu)',
                        'Ben Edwards (BJEdwards@gmail.com)'])

__all__ = ['random_geometric_graph',
           'waxman_graph',
           'geographical_threshold_graph',
           'navigable_small_world_graph']

from bisect import bisect_left
from functools import reduce
from itertools import product
import math, random, sys
import networkx as nx

#---------------------------------------------------------------------------
#  Random Geometric Graphs
#---------------------------------------------------------------------------

def random_geometric_graph(n, radius, dim=2, pos=None):
    r"""Return the random geometric graph in the unit cube.

    The random geometric graph model places n nodes uniformly at random
    in the unit cube  Two nodes `u,v` are connected with an edge if
    `d(u,v)<=r` where `d` is the Euclidean distance and `r` is a radius
    threshold.

    Parameters
    ----------
    n : int
        Number of nodes
    radius: float
        Distance threshold value
    dim : int, optional
        Dimension of graph
    pos : dict, optional
        A dictionary keyed by node with node positions as values.

    Returns
    -------
    Graph

    Examples
    --------
    >>> G = nx.random_geometric_graph(20,0.1)

    Notes
    -----
    This uses an `n^2` algorithm to build the graph.  A faster algorithm
    is possible using k-d trees.

    The pos keyword can be used to specify node positions so you can create
    an arbitrary distribution and domain for positions.  If you need a distance
    function other than Euclidean you'll have to hack the algorithm.

    E.g to use a 2d Gaussian distribution of node positions with mean (0,0)
    and std. dev. 2

    >>> import random
    >>> n=20
    >>> p=dict((i,(random.gauss(0,2),random.gauss(0,2))) for i in range(n))
    >>> G = nx.random_geometric_graph(n,0.2,pos=p)

    References
    ----------
    .. [1] Penrose, Mathew, Random Geometric Graphs,
       Oxford Studies in Probability, 5, 2003.
    """
    G=nx.Graph()
    G.name="Random Geometric Graph"
    G.add_nodes_from(range(n))
    if pos is None:
        # random positions
        for n in G:
            G.node[n]['pos']=[random.random() for i in range(0,dim)]
    else:
        nx.set_node_attributes(G,'pos',pos)
    # connect nodes within "radius" of each other
    # n^2 algorithm, could use a k-d tree implementation
    nodes = G.nodes(data=True)
    while nodes:
        u,du = nodes.pop()
        pu = du['pos']
        for v,dv in nodes:
            pv = dv['pos']
            d = sum(((a-b)**2 for a,b in zip(pu,pv)))
            if d <= radius**2:
                G.add_edge(u,v)
    return G

def geographical_threshold_graph(n, theta, alpha=2, dim=2,
                                 pos=None, weight=None):
    r"""Return a geographical threshold graph.

    The geographical threshold graph model places n nodes uniformly at random
    in a rectangular domain.  Each node `u` is assigned a weight `w_u`.
    Two nodes `u,v` are connected with an edge if

    .. math::

       w_u + w_v \ge \theta r^{\alpha}

    where `r` is the Euclidean distance between `u` and `v`,
    and `\theta`, `\alpha` are parameters.

    Parameters
    ----------
    n : int
        Number of nodes
    theta: float
        Threshold value
    alpha: float, optional
        Exponent of distance function
    dim : int, optional
        Dimension of graph
    pos : dict
        Node positions as a dictionary of tuples keyed by node.
    weight : dict
        Node weights as a dictionary of numbers keyed by node.

    Returns
    -------
    Graph

    Examples
    --------
    >>> G = nx.geographical_threshold_graph(20,50)

    Notes
    -----
    If weights are not specified they are assigned to nodes by drawing randomly
    from an the exponential distribution with rate parameter `\lambda=1`.
    To specify a weights from a different distribution assign them to a
    dictionary and pass it as the weight= keyword

    >>> import random
    >>> n = 20
    >>> w=dict((i,random.expovariate(5.0)) for i in range(n))
    >>> G = nx.geographical_threshold_graph(20,50,weight=w)

    If node positions are not specified they are randomly assigned from the
    uniform distribution.

    References
    ----------
    .. [1] Masuda, N., Miwa, H., Konno, N.:
       Geographical threshold graphs with small-world and scale-free properties.
       Physical Review E 71, 036108 (2005)
    .. [2]  Milan Bradonjić, Aric Hagberg and Allon G. Percus,
       Giant component and connectivity in geographical threshold graphs,
       in Algorithms and Models for the Web-Graph (WAW 2007),
       Antony Bonato and Fan Chung (Eds), pp. 209--216, 2007
    """
    G=nx.Graph()
    # add n nodes
    G.add_nodes_from([v for v in range(n)])
    if weight is None:
        # choose weights from exponential distribution
        for n in G:
            G.node[n]['weight'] = random.expovariate(1.0)
    else:
        nx.set_node_attributes(G,'weight',weight)
    if pos is None:
        # random positions
        for n in G:
            G.node[n]['pos']=[random.random() for i in range(0,dim)]
    else:
        nx.set_node_attributes(G,'pos',pos)
    G.add_edges_from(geographical_threshold_edges(G, theta, alpha))
    return G

def geographical_threshold_edges(G, theta, alpha=2):
    # generate edges for a geographical threshold graph given a graph
    # with positions and weights assigned as node attributes 'pos' and 'weight'.
    nodes = G.nodes(data=True)
    while nodes:
        u,du = nodes.pop()
        wu = du['weight']
        pu = du['pos']
        for v,dv in nodes:
            wv = dv['weight']
            pv = dv['pos']
            r = math.sqrt(sum(((a-b)**2 for a,b in zip(pu,pv))))
            if wu+wv >= theta*r**alpha:
                yield(u,v)

def waxman_graph(n, alpha=0.4, beta=0.1, L=None, domain=(0,0,1,1)):
    r"""Return a Waxman random graph.

    The Waxman random graph models place n nodes uniformly at random
    in a rectangular domain. Two nodes u,v are connected with an edge
    with probability

    .. math::
            p = \alpha*exp(-d/(\beta*L)).

    This function implements both Waxman models.

    Waxman-1:  `L` not specified
       The distance `d` is the Euclidean distance between the nodes u and v.
       `L` is the maximum distance between all nodes in the graph.

    Waxman-2: `L` specified
       The distance `d` is chosen randomly in `[0,L]`.

    Parameters
    ----------
    n : int
        Number of nodes
    alpha: float
        Model parameter
    beta: float
        Model parameter
    L : float, optional
        Maximum distance between nodes.  If not specified the actual distance
        is calculated.
    domain : tuple of numbers, optional
         Domain size (xmin, ymin, xmax, ymax)

    Returns
    -------
    G: Graph

    References
    ----------
    .. [1]  B. M. Waxman, Routing of multipoint connections.
       IEEE J. Select. Areas Commun. 6(9),(1988) 1617-1622.
    """
    # build graph of n nodes with random positions in the unit square
    G = nx.Graph()
    G.add_nodes_from(range(n))
    (xmin,ymin,xmax,ymax)=domain
    for n in G:
        G.node[n]['pos']=((xmin + (xmax-xmin))*random.random(),
                          (ymin + (ymax-ymin))*random.random())
    if L is None:
        # find maximum distance L between two nodes
        l = 0
        pos = list(nx.get_node_attributes(G,'pos').values())
        while pos:
            x1,y1 = pos.pop()
            for x2,y2 in pos:
                r2 = (x1-x2)**2 + (y1-y2)**2
                if r2 > l:
                    l = r2
        l=math.sqrt(l)
    else:
        # user specified maximum distance
        l = L

    nodes=G.nodes()
    if L is None:
        # Waxman-1 model
        # try all pairs, connect randomly based on euclidean distance
        while nodes:
            u = nodes.pop()
            x1,y1 = G.node[u]['pos']
            for v in nodes:
                x2,y2 = G.node[v]['pos']
                r = math.sqrt((x1-x2)**2 + (y1-y2)**2)
                if random.random() < alpha*math.exp(-r/(beta*l)):
                    G.add_edge(u,v)
    else:
        # Waxman-2 model
        # try all pairs, connect randomly based on randomly chosen l
        while nodes:
            u = nodes.pop()
            for v in nodes:
                r = random.random()*l
                if random.random() < alpha*math.exp(-r/(beta*l)):
                    G.add_edge(u,v)
    return G

def navigable_small_world_graph(n, p=1, q=1, r=2, dim=2, seed=None):
    r"""Return a navigable small-world graph.

    A navigable small-world graph is a directed grid with additional
    long-range connections that are chosen randomly.  From [1]_:

    Begin with a set of nodes that are identified with the set of lattice
    points in an `n \times n` square, `{(i,j): i\in {1,2,\ldots,n}, j\in {1,2,\ldots,n}}`
    and define the lattice distance between two nodes `(i,j)` and `(k,l)`
    to be the number of "lattice steps" separating them: `d((i,j),(k,l)) = |k-i|+|l-j|`.

    For a universal constant `p`, the node `u` has a directed edge to every other
    node within lattice distance `p` (local contacts) .

    For universal constants `q\ge 0` and `r\ge 0` construct directed edges from `u` to `q`
    other nodes (long-range contacts) using independent random trials;  the i'th
    directed edge from `u` has endpoint `v` with probability proportional to `d(u,v)^{-r}`.

    Parameters
    ----------
    n : int
        The number of nodes.
    p : int
        The diameter of short range connections. Each node is connected
        to every other node within lattice distance p.
    q : int
        The number of long-range connections for each node.
    r : float
        Exponent for decaying probability of connections.  The probability of
        connecting to a node at lattice distance d is 1/d^r.
    dim : int
        Dimension of grid
    seed : int, optional
        Seed for random number generator (default=None).

    References
    ----------
    .. [1] J. Kleinberg. The small-world phenomenon: An algorithmic
       perspective. Proc. 32nd ACM Symposium on Theory of Computing, 2000.
    """
    if (p < 1):
        raise nx.NetworkXException("p must be >= 1")
    if (q < 0):
        raise nx.NetworkXException("q must be >= 0")
    if (r < 0):
        raise nx.NetworkXException("r must be >= 1")
    if not seed is None:
        random.seed(seed)
    G = nx.DiGraph()
    nodes = list(product(range(n),repeat=dim))
    for p1 in nodes:
        probs = [0]
        for p2 in nodes:
            if p1==p2:
                continue
            d = sum((abs(b-a) for a,b in zip(p1,p2)))
            if d <= p:
                G.add_edge(p1,p2)
            probs.append(d**-r)
        cdf = list(nx.utils.cumulative_sum(probs))
        for _ in range(q):
            target = nodes[bisect_left(cdf,random.uniform(0, cdf[-1]))]
            G.add_edge(p1,target)
    return G
<commit_msg>fix waxman_graph domain<commit_after># -*- coding: utf-8 -*-
"""
Generators for geometric graphs.
"""
#    Copyright (C) 2004-2011 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.

__author__ = "\n".join(['Aric Hagberg (hagberg@lanl.gov)',
                        'Dan Schult (dschult@colgate.edu)',
                        'Ben Edwards (BJEdwards@gmail.com)'])

__all__ = ['random_geometric_graph',
           'waxman_graph',
           'geographical_threshold_graph',
           'navigable_small_world_graph']

from bisect import bisect_left
from functools import reduce
from itertools import product
import math, random, sys
import networkx as nx

#---------------------------------------------------------------------------
#  Random Geometric Graphs
#---------------------------------------------------------------------------

def random_geometric_graph(n, radius, dim=2, pos=None):
    r"""Return the random geometric graph in the unit cube.

    The random geometric graph model places n nodes uniformly at random
    in the unit cube  Two nodes `u,v` are connected with an edge if
    `d(u,v)<=r` where `d` is the Euclidean distance and `r` is a radius
    threshold.

    Parameters
    ----------
    n : int
        Number of nodes
    radius: float
        Distance threshold value
    dim : int, optional
        Dimension of graph
    pos : dict, optional
        A dictionary keyed by node with node positions as values.

    Returns
    -------
    Graph

    Examples
    --------
    >>> G = nx.random_geometric_graph(20,0.1)

    Notes
    -----
    This uses an `n^2` algorithm to build the graph.  A faster algorithm
    is possible using k-d trees.

    The pos keyword can be used to specify node positions so you can create
    an arbitrary distribution and domain for positions.  If you need a distance
    function other than Euclidean you'll have to hack the algorithm.

    E.g to use a 2d Gaussian distribution of node positions with mean (0,0)
    and std. dev. 2

    >>> import random
    >>> n=20
    >>> p=dict((i,(random.gauss(0,2),random.gauss(0,2))) for i in range(n))
    >>> G = nx.random_geometric_graph(n,0.2,pos=p)

    References
    ----------
    .. [1] Penrose, Mathew, Random Geometric Graphs,
       Oxford Studies in Probability, 5, 2003.
    """
    G=nx.Graph()
    G.name="Random Geometric Graph"
    G.add_nodes_from(range(n))
    if pos is None:
        # random positions
        for n in G:
            G.node[n]['pos']=[random.random() for i in range(0,dim)]
    else:
        nx.set_node_attributes(G,'pos',pos)
    # connect nodes within "radius" of each other
    # n^2 algorithm, could use a k-d tree implementation
    nodes = G.nodes(data=True)
    while nodes:
        u,du = nodes.pop()
        pu = du['pos']
        for v,dv in nodes:
            pv = dv['pos']
            d = sum(((a-b)**2 for a,b in zip(pu,pv)))
            if d <= radius**2:
                G.add_edge(u,v)
    return G

def geographical_threshold_graph(n, theta, alpha=2, dim=2,
                                 pos=None, weight=None):
    r"""Return a geographical threshold graph.

    The geographical threshold graph model places n nodes uniformly at random
    in a rectangular domain.  Each node `u` is assigned a weight `w_u`.
    Two nodes `u,v` are connected with an edge if

    .. math::

       w_u + w_v \ge \theta r^{\alpha}

    where `r` is the Euclidean distance between `u` and `v`,
    and `\theta`, `\alpha` are parameters.

    Parameters
    ----------
    n : int
        Number of nodes
    theta: float
        Threshold value
    alpha: float, optional
        Exponent of distance function
    dim : int, optional
        Dimension of graph
    pos : dict
        Node positions as a dictionary of tuples keyed by node.
    weight : dict
        Node weights as a dictionary of numbers keyed by node.

    Returns
    -------
    Graph

    Examples
    --------
    >>> G = nx.geographical_threshold_graph(20,50)

    Notes
    -----
    If weights are not specified they are assigned to nodes by drawing randomly
    from an the exponential distribution with rate parameter `\lambda=1`.
    To specify a weights from a different distribution assign them to a
    dictionary and pass it as the weight= keyword

    >>> import random
    >>> n = 20
    >>> w=dict((i,random.expovariate(5.0)) for i in range(n))
    >>> G = nx.geographical_threshold_graph(20,50,weight=w)

    If node positions are not specified they are randomly assigned from the
    uniform distribution.

    References
    ----------
    .. [1] Masuda, N., Miwa, H., Konno, N.:
       Geographical threshold graphs with small-world and scale-free properties.
       Physical Review E 71, 036108 (2005)
    .. [2]  Milan Bradonjić, Aric Hagberg and Allon G. Percus,
       Giant component and connectivity in geographical threshold graphs,
       in Algorithms and Models for the Web-Graph (WAW 2007),
       Antony Bonato and Fan Chung (Eds), pp. 209--216, 2007
    """
    G=nx.Graph()
    # add n nodes
    G.add_nodes_from([v for v in range(n)])
    if weight is None:
        # choose weights from exponential distribution
        for n in G:
            G.node[n]['weight'] = random.expovariate(1.0)
    else:
        nx.set_node_attributes(G,'weight',weight)
    if pos is None:
        # random positions
        for n in G:
            G.node[n]['pos']=[random.random() for i in range(0,dim)]
    else:
        nx.set_node_attributes(G,'pos',pos)
    G.add_edges_from(geographical_threshold_edges(G, theta, alpha))
    return G

def geographical_threshold_edges(G, theta, alpha=2):
    # generate edges for a geographical threshold graph given a graph
    # with positions and weights assigned as node attributes 'pos' and 'weight'.
    nodes = G.nodes(data=True)
    while nodes:
        u,du = nodes.pop()
        wu = du['weight']
        pu = du['pos']
        for v,dv in nodes:
            wv = dv['weight']
            pv = dv['pos']
            r = math.sqrt(sum(((a-b)**2 for a,b in zip(pu,pv))))
            if wu+wv >= theta*r**alpha:
                yield(u,v)

def waxman_graph(n, alpha=0.4, beta=0.1, L=None, domain=(0,0,1,1)):
    r"""Return a Waxman random graph.

    The Waxman random graph models place n nodes uniformly at random
    in a rectangular domain. Two nodes u,v are connected with an edge
    with probability

    .. math::
            p = \alpha*exp(-d/(\beta*L)).

    This function implements both Waxman models.

    Waxman-1:  `L` not specified
       The distance `d` is the Euclidean distance between the nodes u and v.
       `L` is the maximum distance between all nodes in the graph.

    Waxman-2: `L` specified
       The distance `d` is chosen randomly in `[0,L]`.

    Parameters
    ----------
    n : int
        Number of nodes
    alpha: float
        Model parameter
    beta: float
        Model parameter
    L : float, optional
        Maximum distance between nodes.  If not specified the actual distance
        is calculated.
    domain : tuple of numbers, optional
         Domain size (xmin, ymin, xmax, ymax)

    Returns
    -------
    G: Graph

    References
    ----------
    .. [1]  B. M. Waxman, Routing of multipoint connections.
       IEEE J. Select. Areas Commun. 6(9),(1988) 1617-1622.
    """
    # build graph of n nodes with random positions in the unit square
    G = nx.Graph()
    G.add_nodes_from(range(n))
    (xmin,ymin,xmax,ymax)=domain
    for n in G:
        G.node[n]['pos']=(xmin + ((xmax-xmin)*random.random()),
                          ymin + ((ymax-ymin)*random.random()))
    if L is None:
        # find maximum distance L between two nodes
        l = 0
        pos = list(nx.get_node_attributes(G,'pos').values())
        while pos:
            x1,y1 = pos.pop()
            for x2,y2 in pos:
                r2 = (x1-x2)**2 + (y1-y2)**2
                if r2 > l:
                    l = r2
        l=math.sqrt(l)
    else:
        # user specified maximum distance
        l = L

    nodes=G.nodes()
    if L is None:
        # Waxman-1 model
        # try all pairs, connect randomly based on euclidean distance
        while nodes:
            u = nodes.pop()
            x1,y1 = G.node[u]['pos']
            for v in nodes:
                x2,y2 = G.node[v]['pos']
                r = math.sqrt((x1-x2)**2 + (y1-y2)**2)
                if random.random() < alpha*math.exp(-r/(beta*l)):
                    G.add_edge(u,v)
    else:
        # Waxman-2 model
        # try all pairs, connect randomly based on randomly chosen l
        while nodes:
            u = nodes.pop()
            for v in nodes:
                r = random.random()*l
                if random.random() < alpha*math.exp(-r/(beta*l)):
                    G.add_edge(u,v)
    return G

def navigable_small_world_graph(n, p=1, q=1, r=2, dim=2, seed=None):
    r"""Return a navigable small-world graph.

    A navigable small-world graph is a directed grid with additional
    long-range connections that are chosen randomly.  From [1]_:

    Begin with a set of nodes that are identified with the set of lattice
    points in an `n \times n` square, `{(i,j): i\in {1,2,\ldots,n}, j\in {1,2,\ldots,n}}`
    and define the lattice distance between two nodes `(i,j)` and `(k,l)`
    to be the number of "lattice steps" separating them: `d((i,j),(k,l)) = |k-i|+|l-j|`.

    For a universal constant `p`, the node `u` has a directed edge to every other
    node within lattice distance `p` (local contacts) .

    For universal constants `q\ge 0` and `r\ge 0` construct directed edges from `u` to `q`
    other nodes (long-range contacts) using independent random trials;  the i'th
    directed edge from `u` has endpoint `v` with probability proportional to `d(u,v)^{-r}`.

    Parameters
    ----------
    n : int
        The number of nodes.
    p : int
        The diameter of short range connections. Each node is connected
        to every other node within lattice distance p.
    q : int
        The number of long-range connections for each node.
    r : float
        Exponent for decaying probability of connections.  The probability of
        connecting to a node at lattice distance d is 1/d^r.
    dim : int
        Dimension of grid
    seed : int, optional
        Seed for random number generator (default=None).

    References
    ----------
    .. [1] J. Kleinberg. The small-world phenomenon: An algorithmic
       perspective. Proc. 32nd ACM Symposium on Theory of Computing, 2000.
    """
    if (p < 1):
        raise nx.NetworkXException("p must be >= 1")
    if (q < 0):
        raise nx.NetworkXException("q must be >= 0")
    if (r < 0):
        raise nx.NetworkXException("r must be >= 1")
    if not seed is None:
        random.seed(seed)
    G = nx.DiGraph()
    nodes = list(product(range(n),repeat=dim))
    for p1 in nodes:
        probs = [0]
        for p2 in nodes:
            if p1==p2:
                continue
            d = sum((abs(b-a) for a,b in zip(p1,p2)))
            if d <= p:
                G.add_edge(p1,p2)
            probs.append(d**-r)
        cdf = list(nx.utils.cumulative_sum(probs))
        for _ in range(q):
            target = nodes[bisect_left(cdf,random.uniform(0, cdf[-1]))]
            G.add_edge(p1,target)
    return G
<|endoftext|>


<commit_before>""" Common HTTP code. """

import asyncio
import os
import pickle

import aiohttp
import appdirs

from sacad import rate_watcher
from sacad import redo

IS_TRAVIS = os.getenv("CI") and os.getenv("TRAVIS")
HTTP_NORMAL_TIMEOUT_S = 30.1 if IS_TRAVIS else 9.1
HTTP_SHORT_TIMEOUT_S = 18.1 if IS_TRAVIS else 3.1
HTTP_MAX_ATTEMPTS = 20 if IS_TRAVIS else 3
DEFAULT_USER_AGENT = "Mozilla/5.0"

class Http:

  def __init__(self, *, allow_session_cookies, min_delay_between_accesses, logger):
    if not allow_session_cookies:
      cookie_jar = aiohttp.cookiejar.DummyCookieJar()
    else:
      cookie_jar = None
    self.session = aiohttp.ClientSession(cookie_jar=cookie_jar,
                                         loop=asyncio.get_event_loop())
    self.watcher_db_filepath = os.path.join(appdirs.user_cache_dir(appname="sacad",
                                                                   appauthor=False),
                                            "rate_watcher.sqlite")
    self.min_delay_between_accesses = min_delay_between_accesses
    self.logger = logger

  def __del__(self):
    self.session.close()  # silences a warning on shutdown

  @asyncio.coroutine
  def query(self, url, *, post_data=None, headers=None, verify=True, cache=None, pre_cache_callback=None):
    """ Send a GET/POST request or get data from cache, retry if it fails, and return a tuple of cache status, response content. """
    if cache is not None:
      # try from cache first
      if post_data is not None:
        if (url, post_data) in cache:
          self.logger.debug("Got data for URL '%s' %s from cache" % (url, dict(post_data)))
          return cache[(url, post_data)]
      elif url in cache:
        self.logger.debug("Got data for URL '%s' from cache" % (url))
        return cache[url]

    domain_rate_watcher = rate_watcher.AccessRateWatcher(self.watcher_db_filepath,
                                                         url,
                                                         self.min_delay_between_accesses,
                                                         logger=self.logger)

    for attempt, time_to_sleep in enumerate(redo.retrier(max_attempts=HTTP_MAX_ATTEMPTS,
                                                         sleeptime=1,
                                                         max_sleeptime=5,
                                                         sleepscale=1.5),
                                            1):
      yield from domain_rate_watcher.waitAccessAsync()

      try:
        if post_data is not None:
          response = yield from self.session.post(url,
                                                  data=post_data,
                                                  headers=self._buildHeaders(headers),
                                                  timeout=HTTP_NORMAL_TIMEOUT_S,
                                                  verify_ssl=verify)
        else:
          response = yield from self.session.get(url,
                                                 headers=self._buildHeaders(headers),
                                                 timeout=HTTP_NORMAL_TIMEOUT_S,
                                                 verify_ssl=verify)
        content = yield from response.read()

        if cache is not None:
          if pre_cache_callback is not None:
            # process
            try:
              data = yield from pre_cache_callback(content)
            except Exception:
              data = content
          else:
            data = content

          # add to cache
          if post_data is not None:
            cache[(url, post_data)] = data
          else:
            cache[url] = data

      except (asyncio.TimeoutError, aiohttp.ClientError) as e:
        self.logger.warning("Querying '%s' failed (attempt %u/%u): %s %s" % (url,
                                                                             attempt,
                                                                             HTTP_MAX_ATTEMPTS,
                                                                             e.__class__.__qualname__,
                                                                             e))
        if attempt == HTTP_MAX_ATTEMPTS:
          raise
        else:
          self.logger.debug("Retrying in %.3fs" % (time_to_sleep))
          yield from asyncio.sleep(time_to_sleep)

      else:
        break  # http retry loop

    # TODO don't store in cache if this raises
    response.raise_for_status()

    return content

  @asyncio.coroutine
  def isReachable(self, url, *, headers=None, verify=True, response_headers=None, cache=None):
    """ Send a HEAD request with short timeout or get data from cache, return True if ressource has 2xx status code, False instead. """
    if (cache is not None) and (url in cache):
      # try from cache first
      self.logger.debug("Got headers for URL '%s' from cache" % (url))
      resp_ok, response_headers = pickle.loads(cache[url])
      return resp_ok

    domain_rate_watcher = rate_watcher.AccessRateWatcher(self.watcher_db_filepath,
                                                         url,
                                                         self.min_delay_between_accesses,
                                                         logger=self.logger)
    resp_ok = True
    try:
      for attempt, time_to_sleep in enumerate(redo.retrier(max_attempts=HTTP_MAX_ATTEMPTS,
                                                           sleeptime=0.5,
                                                           max_sleeptime=2,
                                                           sleepscale=1.5),
                                              1):
        yield from domain_rate_watcher.waitAccessAsync()

        try:
          response = yield from self.session.head(url,
                                                  headers=self._buildHeaders(headers),
                                                  timeout=HTTP_SHORT_TIMEOUT_S,
                                                  verify_ssl=verify)

        except (asyncio.TimeoutError, aiohttp.ClientError) as e:
          self.logger.warning("Probing '%s' failed (attempt %u/%u): %s %s" % (url,
                                                                              attempt,
                                                                              HTTP_MAX_ATTEMPTS,
                                                                              e.__class__.__qualname__,
                                                                              e))
          if attempt == HTTP_MAX_ATTEMPTS:
            resp_ok = False
          else:
            self.logger.debug("Retrying in %.3fs" % (time_to_sleep))
            yield from asyncio.sleep(time_to_sleep)

        else:
          response.raise_for_status()

          if response_headers is not None:
            response_headers.update(response.headers)

          break  # http retry loop

    except aiohttp.ClientResponseError as e:
      self.logger.debug("Probing '%s' failed: %s %s" % (url, e.__class__.__qualname__, e))
      resp_ok = False

    if cache is not None:
      # store in cache
      cache[url] = pickle.dumps((resp_ok, response_headers))

    return resp_ok

  @asyncio.coroutine
  def fastStreamedQuery(self, url, *, headers=None, verify=True):
    """ Send a GET request with short timeout, do not retry, and return streamed response. """
    response = yield from self.session.get(url,
                                           headers=self._buildHeaders(headers),
                                           timeout=HTTP_SHORT_TIMEOUT_S,
                                           verify_ssl=verify)

    response.raise_for_status()

    return response

  def _buildHeaders(self, headers):
    """ Build HTTP headers dictionary. """
    if headers is None:
      headers = {}
    if "User-Agent" not in headers:
      headers["User-Agent"] = DEFAULT_USER_AGENT
    return headers
<commit_msg>Reduce HTTP retry count on Travis<commit_after>""" Common HTTP code. """

import asyncio
import os
import pickle

import aiohttp
import appdirs

from sacad import rate_watcher
from sacad import redo

IS_TRAVIS = os.getenv("CI") and os.getenv("TRAVIS")
HTTP_NORMAL_TIMEOUT_S = 30.1 if IS_TRAVIS else 9.1
HTTP_SHORT_TIMEOUT_S = 18.1 if IS_TRAVIS else 3.1
HTTP_MAX_ATTEMPTS = 10 if IS_TRAVIS else 3
DEFAULT_USER_AGENT = "Mozilla/5.0"

class Http:

  def __init__(self, *, allow_session_cookies, min_delay_between_accesses, logger):
    if not allow_session_cookies:
      cookie_jar = aiohttp.cookiejar.DummyCookieJar()
    else:
      cookie_jar = None
    self.session = aiohttp.ClientSession(cookie_jar=cookie_jar,
                                         loop=asyncio.get_event_loop())
    self.watcher_db_filepath = os.path.join(appdirs.user_cache_dir(appname="sacad",
                                                                   appauthor=False),
                                            "rate_watcher.sqlite")
    self.min_delay_between_accesses = min_delay_between_accesses
    self.logger = logger

  def __del__(self):
    self.session.close()  # silences a warning on shutdown

  @asyncio.coroutine
  def query(self, url, *, post_data=None, headers=None, verify=True, cache=None, pre_cache_callback=None):
    """ Send a GET/POST request or get data from cache, retry if it fails, and return a tuple of cache status, response content. """
    if cache is not None:
      # try from cache first
      if post_data is not None:
        if (url, post_data) in cache:
          self.logger.debug("Got data for URL '%s' %s from cache" % (url, dict(post_data)))
          return cache[(url, post_data)]
      elif url in cache:
        self.logger.debug("Got data for URL '%s' from cache" % (url))
        return cache[url]

    domain_rate_watcher = rate_watcher.AccessRateWatcher(self.watcher_db_filepath,
                                                         url,
                                                         self.min_delay_between_accesses,
                                                         logger=self.logger)

    for attempt, time_to_sleep in enumerate(redo.retrier(max_attempts=HTTP_MAX_ATTEMPTS,
                                                         sleeptime=1,
                                                         max_sleeptime=5,
                                                         sleepscale=1.5),
                                            1):
      yield from domain_rate_watcher.waitAccessAsync()

      try:
        if post_data is not None:
          response = yield from self.session.post(url,
                                                  data=post_data,
                                                  headers=self._buildHeaders(headers),
                                                  timeout=HTTP_NORMAL_TIMEOUT_S,
                                                  verify_ssl=verify)
        else:
          response = yield from self.session.get(url,
                                                 headers=self._buildHeaders(headers),
                                                 timeout=HTTP_NORMAL_TIMEOUT_S,
                                                 verify_ssl=verify)
        content = yield from response.read()

        if cache is not None:
          if pre_cache_callback is not None:
            # process
            try:
              data = yield from pre_cache_callback(content)
            except Exception:
              data = content
          else:
            data = content

          # add to cache
          if post_data is not None:
            cache[(url, post_data)] = data
          else:
            cache[url] = data

      except (asyncio.TimeoutError, aiohttp.ClientError) as e:
        self.logger.warning("Querying '%s' failed (attempt %u/%u): %s %s" % (url,
                                                                             attempt,
                                                                             HTTP_MAX_ATTEMPTS,
                                                                             e.__class__.__qualname__,
                                                                             e))
        if attempt == HTTP_MAX_ATTEMPTS:
          raise
        else:
          self.logger.debug("Retrying in %.3fs" % (time_to_sleep))
          yield from asyncio.sleep(time_to_sleep)

      else:
        break  # http retry loop

    # TODO don't store in cache if this raises
    response.raise_for_status()

    return content

  @asyncio.coroutine
  def isReachable(self, url, *, headers=None, verify=True, response_headers=None, cache=None):
    """ Send a HEAD request with short timeout or get data from cache, return True if ressource has 2xx status code, False instead. """
    if (cache is not None) and (url in cache):
      # try from cache first
      self.logger.debug("Got headers for URL '%s' from cache" % (url))
      resp_ok, response_headers = pickle.loads(cache[url])
      return resp_ok

    domain_rate_watcher = rate_watcher.AccessRateWatcher(self.watcher_db_filepath,
                                                         url,
                                                         self.min_delay_between_accesses,
                                                         logger=self.logger)
    resp_ok = True
    try:
      for attempt, time_to_sleep in enumerate(redo.retrier(max_attempts=HTTP_MAX_ATTEMPTS,
                                                           sleeptime=0.5,
                                                           max_sleeptime=2,
                                                           sleepscale=1.5),
                                              1):
        yield from domain_rate_watcher.waitAccessAsync()

        try:
          response = yield from self.session.head(url,
                                                  headers=self._buildHeaders(headers),
                                                  timeout=HTTP_SHORT_TIMEOUT_S,
                                                  verify_ssl=verify)

        except (asyncio.TimeoutError, aiohttp.ClientError) as e:
          self.logger.warning("Probing '%s' failed (attempt %u/%u): %s %s" % (url,
                                                                              attempt,
                                                                              HTTP_MAX_ATTEMPTS,
                                                                              e.__class__.__qualname__,
                                                                              e))
          if attempt == HTTP_MAX_ATTEMPTS:
            resp_ok = False
          else:
            self.logger.debug("Retrying in %.3fs" % (time_to_sleep))
            yield from asyncio.sleep(time_to_sleep)

        else:
          response.raise_for_status()

          if response_headers is not None:
            response_headers.update(response.headers)

          break  # http retry loop

    except aiohttp.ClientResponseError as e:
      self.logger.debug("Probing '%s' failed: %s %s" % (url, e.__class__.__qualname__, e))
      resp_ok = False

    if cache is not None:
      # store in cache
      cache[url] = pickle.dumps((resp_ok, response_headers))

    return resp_ok

  @asyncio.coroutine
  def fastStreamedQuery(self, url, *, headers=None, verify=True):
    """ Send a GET request with short timeout, do not retry, and return streamed response. """
    response = yield from self.session.get(url,
                                           headers=self._buildHeaders(headers),
                                           timeout=HTTP_SHORT_TIMEOUT_S,
                                           verify_ssl=verify)

    response.raise_for_status()

    return response

  def _buildHeaders(self, headers):
    """ Build HTTP headers dictionary. """
    if headers is None:
      headers = {}
    if "User-Agent" not in headers:
      headers["User-Agent"] = DEFAULT_USER_AGENT
    return headers
<|endoftext|>


<commit_before>#!/usr/bin/python

import os, sys, subprocess
import argparse
import mtoutils
import subprocess
import trajstorage, storage
import moving
from numpy import loadtxt
from numpy.linalg import inv
import matplotlib.pyplot as plt
import storage
import threading
import cvgenetic
import cfg_combination as cfgcomb
from configobj import ConfigObj
import timeit
from multiprocessing import Queue

class GeneticCompare(object):
    def __init__(self, motalist, IDlist):
        self.motalist = motalist
        self.IDlist = IDlist
        # lock = threading.Lock()

    def computeMOT(self, i):
        obj = trajstorage.CVsqlite(sqlite_files+str(i)+".sqlite")
        obj.loadObjects()

        motp, mota, mt, mme, fpt, gt = moving.computeClearMOT(cdb.annotations, obj.objects, args.matchDistance, firstFrame, lastFrame)
        # self.lock.acquire()
        self.IDlist.put(i)
        self.motalist.put(mota)
        obj.close()
        # self.lock.release()

        if args.PrintMOTA:
            print "MOTA: ", mota
            # print "MOTP: ", motp
            # print 'MOTP: {}'.format(motp)
            # print 'MOTA: {}'.format(mota)
            # print 'Number of missed objects.frames: {}'.format(mt)
            # print 'Number of mismatches: {}'.format(mme)
            # print 'Number of false alarms.frames: {}'.format(fpt)bestI
        return mota

if __name__ == '__main__' :
    parser = argparse.ArgumentParser(description="compare all sqlites that are created by cfg_combination.py to the Annotated version to find the ID of the best configuration")
    parser.add_argument('-d', '--database-file', dest ='databaseFile', help ="Name of the databaseFile.", required = True)
    parser.add_argument('-o', '--homography-file', dest ='homography', help = "Name of the homography file.", required = True)
    parser.add_argument('-f', '--First-ID', dest ='firstID', help = "the first ID of the range of ID", required = True, type = int)
    parser.add_argument('-l', '--Last-ID', dest ='lastID', help = "the last ID of the range of ID", required = True, type = int)
    parser.add_argument('-m', '--matching-distance', dest='matchDistance', help = "matchDistance", default = 10, type = float)
    parser.add_argument('-a', '--accuracy', dest ='accuracy', help = "accuracy parameter for genetic algorithm", type = int)
    parser.add_argument('-mota', '--print-MOTA', dest='PrintMOTA', action = 'store_true', help = "Print MOTA for each ID.")
    args = parser.parse_args()

    start = timeit.default_timer()

    dbfile = args.databaseFile;
    homography = loadtxt(args.homography)
    sqlite_files = "sql_files/Sqlite_ID_"

    cdb = trajstorage.CVsqlite(dbfile)
    cdb.open()
    cdb.getLatestAnnotation()
    cdb.createBoundingBoxTable(cdb.latestannotations, inv(homography))
    cdb.loadAnnotaion()
    for a in cdb.annotations:
        a.computeCentroidTrajectory(homography)
    print "Latest Annotaions in "+dbfile+": ", cdb.latestannotations
    # for row in cdb.boundingbox:
    #     print row
    cdb.frameNumbers = cdb.getFrameList()
    firstFrame = cdb.frameNumbers[0]
    lastFrame = cdb.frameNumbers[-1]

    # matplot
    foundmota = Queue()
    IDs = Queue()

    Comp = GeneticCompare(foundmota, IDs)
    config = ConfigObj('range.cfg')
    cfg_list = cfgcomb.CVConfigList()
    cfgcomb.config_to_list(cfg_list, config)
    if args.accuracy != None:
        GeneticCal = cvgenetic.CVGenetic(10, cfg_list, Comp.computeMOT, args.accuracy)
    else:
        GeneticCal = cvgenetic.CVGenetic(10, cfg_list, Comp.computeMOT)
    GeneticCal.run_thread()

    # tranform queues to lists
    foundmota = cvgenetic.Queue_to_list(foundmota)
    IDs = cvgenetic.Queue_to_list(IDs)

    Best_mota = max(foundmota)
    Best_ID = IDs[foundmota.index(Best_mota)]
    print "Best multiple object tracking accuracy (MOTA)", Best_mota
    print "ID:", Best_ID

    stop = timeit.default_timer()
    print str(stop-start) + "s"

    # matplot
    plt.plot(foundmota ,IDs ,'bo')
    plt.plot(Best_mota, Best_ID, 'ro')
    plt.axis([-1, 1, -1, args.lastID+1])
    plt.xlabel('mota')
    plt.ylabel('ID')

    plt.title(b'Best MOTA: '+str(Best_mota) +'\nwith ID: '+str(Best_ID))
    plt.show()

    cdb.close()
<commit_msg>add num_of_parents to argparser<commit_after>#!/usr/bin/python

import os, sys, subprocess
import argparse
import mtoutils
import subprocess
import trajstorage, storage
import moving
from numpy import loadtxt
from numpy.linalg import inv
import matplotlib.pyplot as plt
import storage
import threading
import cvgenetic
import cfg_combination as cfgcomb
from configobj import ConfigObj
import timeit
from multiprocessing import Queue

class GeneticCompare(object):
    def __init__(self, motalist, IDlist):
        self.motalist = motalist
        self.IDlist = IDlist
        # lock = threading.Lock()

    def computeMOT(self, i):
        obj = trajstorage.CVsqlite(sqlite_files+str(i)+".sqlite")
        obj.loadObjects()

        motp, mota, mt, mme, fpt, gt = moving.computeClearMOT(cdb.annotations, obj.objects, args.matchDistance, firstFrame, lastFrame)
        # self.lock.acquire()
        self.IDlist.put(i)
        self.motalist.put(mota)
        obj.close()
        # self.lock.release()

        if args.PrintMOTA:
            print "MOTA: ", mota
            # print "MOTP: ", motp
            # print 'MOTP: {}'.format(motp)
            # print 'MOTA: {}'.format(mota)
            # print 'Number of missed objects.frames: {}'.format(mt)
            # print 'Number of mismatches: {}'.format(mme)
            # print 'Number of false alarms.frames: {}'.format(fpt)bestI
        return mota

if __name__ == '__main__' :
    parser = argparse.ArgumentParser(description="compare all sqlites that are created by cfg_combination.py to the Annotated version to find the ID of the best configuration")
    parser.add_argument('-d', '--database-file', dest ='databaseFile', help ="Name of the databaseFile.", required = True)
    parser.add_argument('-o', '--homography-file', dest ='homography', help = "Name of the homography file.", required = True)
    parser.add_argument('-f', '--First-ID', dest ='firstID', help = "the first ID of the range of ID", required = True, type = int)
    parser.add_argument('-l', '--Last-ID', dest ='lastID', help = "the last ID of the range of ID", required = True, type = int)
    parser.add_argument('-m', '--matching-distance', dest='matchDistance', help = "matchDistance", default = 10, type = float)
    parser.add_argument('-a', '--accuracy', dest ='accuracy', help = "accuracy parameter for genetic algorithm", type = int)
    parser.add_argument('-p', '--num-of-parents', dest = 'num_of_parents', help = "Number of parents are selected each generation", type = int)
    parser.add_argument('-mota', '--print-MOTA', dest='PrintMOTA', action = 'store_true', help = "Print MOTA for each ID.")
    args = parser.parse_args()

    start = timeit.default_timer()

    dbfile = args.databaseFile;
    homography = loadtxt(args.homography)
    sqlite_files = "sql_files/Sqlite_ID_"

    cdb = trajstorage.CVsqlite(dbfile)
    cdb.open()
    cdb.getLatestAnnotation()
    cdb.createBoundingBoxTable(cdb.latestannotations, inv(homography))
    cdb.loadAnnotaion()
    for a in cdb.annotations:
        a.computeCentroidTrajectory(homography)
    print "Latest Annotaions in "+dbfile+": ", cdb.latestannotations
    # for row in cdb.boundingbox:
    #     print row
    cdb.frameNumbers = cdb.getFrameList()
    firstFrame = cdb.frameNumbers[0]
    lastFrame = cdb.frameNumbers[-1]

    # matplot
    foundmota = Queue()
    IDs = Queue()

    Comp = GeneticCompare(foundmota, IDs)
    config = ConfigObj('range.cfg')
    cfg_list = cfgcomb.CVConfigList()
    cfgcomb.config_to_list(cfg_list, config)
    if args.accuracy != None:
        GeneticCal = cvgenetic.CVGenetic(10, cfg_list, Comp.computeMOT, args.accuracy)
    else:
        GeneticCal = cvgenetic.CVGenetic(10, cfg_list, Comp.computeMOT)
    if args.num_of_parents != None:
        GeneticCal.run_thread(args.num_of_parents)
    else:
        GeneticCal.run_thread()

    # tranform queues to lists
    foundmota = cvgenetic.Queue_to_list(foundmota)
    IDs = cvgenetic.Queue_to_list(IDs)

    Best_mota = max(foundmota)
    Best_ID = IDs[foundmota.index(Best_mota)]
    print "Best multiple object tracking accuracy (MOTA)", Best_mota
    print "ID:", Best_ID

    stop = timeit.default_timer()
    print str(stop-start) + "s"

    # matplot
    plt.plot(foundmota ,IDs ,'bo')
    plt.plot(Best_mota, Best_ID, 'ro')
    plt.axis([-1, 1, -1, args.lastID+1])
    plt.xlabel('mota')
    plt.ylabel('ID')

    plt.title(b'Best MOTA: '+str(Best_mota) +'\nwith ID: '+str(Best_ID))
    plt.show()

    cdb.close()
<|endoftext|>


<commit_before>import mimetypes
import os
import re
from gzip import GzipFile
from StringIO import StringIO

import cloudfiles
from cloudfiles.errors import NoSuchObject, ResponseError

from django.core.files.base import File, ContentFile
from django.core.files.storage import Storage

from .settings import CUMULUS

HEADER_PATTERNS = tuple((re.compile(p), h) for p, h in CUMULUS.get('HEADERS', {}))

def sync_headers(cloud_obj, headers={}, header_patterns=HEADER_PATTERNS):
    """
    Overwrite the given cloud_obj's headers with the ones given as ``headers`
    and add additional headers as defined in the HEADERS setting depending on
    the cloud_obj's file name.
    """
    # don't set headers on directories
    content_type = getattr(cloud_obj, 'content_type', None)
    if content_type == 'application/directory':
        return
    matched_headers = {}
    for pattern, pattern_headers in header_patterns:
        if pattern.match(cloud_obj.name):
            matched_headers.update(pattern_headers.copy())
    matched_headers.update(cloud_obj.headers)  # preserve headers already set
    matched_headers.update(headers)  # explicitly set headers overwrite matches and already set headers
    if matched_headers != cloud_obj.headers:
        cloud_obj.headers = matched_headers
        cloud_obj.sync_metadata()

def get_gzipped_contents(input_file):
    """
    Return a gzipped version of a previously opened file's buffer.
    """
    zbuf = StringIO()
    zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
    zfile.write(input_file.read())
    zfile.close()
    return ContentFile(zbuf.getvalue())

class CloudFilesStorage(Storage):
    """
    Custom storage for Rackspace Cloud Files.
    """
    default_quick_listdir = True

    def __init__(self, username=None, api_key=None, container=None, timeout=None,
                 connection_kwargs=None):
        """
        Initialize the settings for the connection and container.
        """
        self.api_key = api_key or CUMULUS['API_KEY']
        self.auth_url = CUMULUS['AUTH_URL']
        self.connection_kwargs = connection_kwargs or {}
        self.container_name = container or CUMULUS['CONTAINER']
        self.timeout = timeout or CUMULUS['TIMEOUT']
        self.use_servicenet = CUMULUS['SERVICENET']
        self.username = username or CUMULUS['USERNAME']
        self.ttl = CUMULUS['TTL']
        self.use_ssl = CUMULUS['USE_SSL']

    def __getstate__(self):
        """
        Return a picklable representation of the storage.
        """
        return dict(username=self.username,
                    api_key=self.api_key,
                    container_name=self.container_name,
                    timeout=self.timeout,
                    use_servicenet=self.use_servicenet,
                    connection_kwargs=self.connection_kwargs)

    def _get_connection(self):
        if not hasattr(self, '_connection'):
            self._connection = cloudfiles.get_connection(
                                  username=self.username,
                                  api_key=self.api_key,
                                  authurl = self.auth_url,
                                  timeout=self.timeout,
                                  servicenet=self.use_servicenet,
                                  **self.connection_kwargs)
        return self._connection

    def _set_connection(self, value):
        self._connection = value

    connection = property(_get_connection, _set_connection)

    def _get_container(self):
        if not hasattr(self, '_container'):
            self.container = self.connection.get_container(
                                                        self.container_name)
        return self._container

    def _set_container(self, container):
        """
        Set the container (and, if needed, the configured TTL on it), making
        the container publicly available.
        """
        if container.cdn_ttl != self.ttl or not container.is_public():
            container.make_public(ttl=self.ttl)
        if hasattr(self, '_container_public_uri'):
            delattr(self, '_container_public_uri')
        self._container = container

    container = property(_get_container, _set_container)

    def _get_container_url(self):
        if not hasattr(self, '_container_public_uri'):
            if self.use_ssl:
                self._container_public_uri = self.container.public_ssl_uri()
            else:
                self._container_public_uri = self.container.public_uri()
        if CUMULUS['CNAMES'] and self._container_public_uri in CUMULUS['CNAMES']:
            self._container_public_uri = CUMULUS['CNAMES'][self._container_public_uri]
        return self._container_public_uri

    container_url = property(_get_container_url)

    def _get_cloud_obj(self, name):
        """
        Helper function to get retrieve the requested Cloud Files Object.
        """
        return self.container.get_object(name)

    def _open(self, name, mode='rb'):
        """
        Return the CloudFilesStorageFile.
        """
        return CloudFilesStorageFile(storage=self, name=name)

    def _save(self, name, content):
        """
        Use the Cloud Files service to write ``content`` to a remote file
        (called ``name``).
        """
        (path, last) = os.path.split(name)
        if path:
            try:
                self.container.get_object(path)
            except NoSuchObject:
                self._save(path, CloudStorageDirectory(path))

        content.open()
        cloud_obj = self.container.create_object(name)
        # If the content type is available, pass it in directly rather than
        # getting the cloud object to try to guess.
        if hasattr(content.file, 'content_type'):
            cloud_obj.content_type = content.file.content_type
        elif hasattr(content, 'content_type'):
            cloud_obj.content_type = content.content_type
        else:
            mime_type, encoding = mimetypes.guess_type(name)
            cloud_obj.content_type = mime_type
        # gzip the file if its of the right content type
        if cloud_obj.content_type in CUMULUS.get('GZIP_CONTENT_TYPES', []):
            content = get_gzipped_contents(content)
            cloud_obj.headers['Content-Encoding'] = 'gzip'
        # set file size
        if hasattr(content.file, 'size'):
            cloud_obj.size = content.file.size
        else:
            cloud_obj.size = content.size
        cloud_obj.send(content)
        content.close()
        sync_headers(cloud_obj)
        return name

    def delete(self, name):
        """
        Deletes the specified file from the storage system.
        """
        try:
            self.container.delete_object(name)
        except ResponseError, exc:
            if exc.status == 404:
                pass
            else:
                raise

    def exists(self, name):
        """
        Returns True if a file referenced by the given name already exists in
        the storage system, or False if the name is available for a new file.
        """
        try:
            self._get_cloud_obj(name)
            return True
        except NoSuchObject:
            return False

    def listdir(self, path):
        """
        Lists the contents of the specified path, returning a 2-tuple; the
        first being an empty list of directories (not available for quick-
        listing), the second being a list of filenames.

        If the list of directories is required, use the full_listdir method.
        """
        files = []
        if path and not path.endswith('/'):
            path = '%s/' % path
        path_len = len(path)
        for name in self.container.list_objects(path=path):
            files.append(name[path_len:])
        return ([], files)

    def full_listdir(self, path):
        """
        Lists the contents of the specified path, returning a 2-tuple of lists;
        the first item being directories, the second item being files.

        On large containers, this may be a slow operation for root containers
        because every single object must be returned (cloudfiles does not
        provide an explicit way of listing directories).
        """
        dirs = set()
        files = []
        if path and not path.endswith('/'):
            path = '%s/' % path
        path_len = len(path)
        for name in self.container.list_objects(prefix=path):
            name = name[path_len:]
            slash = name[1:-1].find('/') + 1
            if slash:
                dirs.add(name[:slash])
            elif name:
                files.append(name)
        dirs = list(dirs)
        dirs.sort()
        return (dirs, files)

    def size(self, name):
        """
        Returns the total size, in bytes, of the file specified by name.
        """
        return self._get_cloud_obj(name).size

    def url(self, name):
        """
        Returns an absolute URL where the file's contents can be accessed
        directly by a web browser.
        """
        return '%s/%s' % (self.container_url, name)

    def modified_time(self, name):
        # CloudFiles return modified date in different formats
        # depending on whether or not we pre-loaded objects.
        # When pre-loaded, timezone is not included but we
        # assume UTC. Since FileStorage returns localtime, and
        # collectstatic compares these dates, we need to depend
        # on dateutil to help us convert timezones.
        try:
           from dateutil import parser, tz
        except ImportError:
            raise NotImplementedError("This functionality requires dateutil to be installed")

        obj = self.container.get_object(name)

        # convert to string to date
        date = parser.parse(obj.last_modified)

        # if the date has no timzone, assume UTC
        if date.tzinfo == None:
            date = date.replace(tzinfo=tz.tzutc())

        # convert date to local time w/o timezone
        date = date.astimezone(tz.tzlocal()).replace(tzinfo=None)
        return date

class CloudStorageDirectory(File):
    """
    A File-like object that creates a directory at cloudfiles
    """

    def __init__(self, name):
        super(CloudStorageDirectory, self).__init__(StringIO(), name=name)
        self.file.content_type = 'application/directory'
        self.size = 0

    def __str__(self):
        return 'directory'

    def __nonzero__(self):
        return True

    def open(self, mode=None):
        self.seek(0)

    def close(self):
        pass

class CloudFilesStaticStorage(CloudFilesStorage):
    """
    Subclasses CloudFilesStorage to automatically set the container to the one
    specified in CUMULUS['STATIC_CONTAINER']. This provides the ability to
    specify a separate storage backend for Django's collectstatic command.

    To use, make sure CUMULUS['STATIC_CONTAINER'] is set to something other
    than CUMULUS['CONTAINER']. Then, tell Django's staticfiles app by setting
    STATICFILES_STORAGE = 'cumulus.storage.CloudFilesStaticStorage'.
    """
    def __init__(self, *args, **kwargs):
        if not 'container' in kwargs:
            kwargs['container'] = CUMULUS['STATIC_CONTAINER']
        super(CloudFilesStaticStorage, self).__init__(*args, **kwargs)

class CloudFilesStorageFile(File):
    closed = False

    def __init__(self, storage, name, *args, **kwargs):
        self._storage = storage
        self._pos = 0
        super(CloudFilesStorageFile, self).__init__(file=None, name=name,
                                                    *args, **kwargs)

    def _get_pos(self):
        return self._pos

    def _get_size(self):
        if not hasattr(self, '_size'):
            self._size = self._storage.size(self.name)
        return self._size

    def _set_size(self, size):
        self._size = size

    size = property(_get_size, _set_size)

    def _get_file(self):
        if not hasattr(self, '_file'):
            self._file = self._storage._get_cloud_obj(self.name)
            self._file.tell = self._get_pos
        return self._file

    def _set_file(self, value):
        if value is None:
            if hasattr(self, '_file'):
                del self._file
        else:
            self._file = value

    file = property(_get_file, _set_file)

    def read(self, num_bytes=0):
        if self._pos == self._get_size():
            return ""
        if num_bytes and self._pos + num_bytes > self._get_size():
            num_bytes = self._get_size() - self._pos
        data = self.file.read(size=num_bytes or -1, offset=self._pos)
        self._pos += len(data)
        return data

    def open(self, *args, **kwargs):
        """
        Open the cloud file object.
        """
        self._pos = 0

    def close(self, *args, **kwargs):
        self._pos = 0

    @property
    def closed(self):
        return not hasattr(self, '_file')

    def seek(self, pos):
        self._pos = pos

class ThreadSafeCloudFilesStorage(CloudFilesStorage):
    """
    Extends CloudFilesStorage to make it mostly thread safe.

    As long as you don't pass container or cloud objects
    between threads, you'll be thread safe.

    Uses one cloudfiles connection per thread.
    """

    def __init__(self, *args, **kwargs):
        super(ThreadSafeCloudFilesStorage, self).__init__(*args, **kwargs)

        import threading
        self.local_cache = threading.local()

    def _get_connection(self):
        if not hasattr(self.local_cache, 'connection'):
            connection = cloudfiles.get_connection(self.username,
                                    self.api_key, **self.connection_kwargs)
            self.local_cache.connection = connection

        return self.local_cache.connection

    connection = property(_get_connection, CloudFilesStorage._set_connection)

    def _get_container(self):
        if not hasattr(self.local_cache, 'container'):
            container = self.connection.get_container(self.container_name)
            self.local_cache.container = container

        return self.local_cache.container

    container = property(_get_container, CloudFilesStorage._set_container)
<commit_msg>added the CONTAINER_URI settings: if specified, we don't need to query cloudfiles to resolve every file URL.<commit_after>import mimetypes
import os
import re
from gzip import GzipFile
from StringIO import StringIO

import cloudfiles
from cloudfiles.errors import NoSuchObject, ResponseError

from django.core.files.base import File, ContentFile
from django.core.files.storage import Storage

from .settings import CUMULUS

HEADER_PATTERNS = tuple((re.compile(p), h) for p, h in CUMULUS.get('HEADERS', {}))

def sync_headers(cloud_obj, headers={}, header_patterns=HEADER_PATTERNS):
    """
    Overwrite the given cloud_obj's headers with the ones given as ``headers`
    and add additional headers as defined in the HEADERS setting depending on
    the cloud_obj's file name.
    """
    # don't set headers on directories
    content_type = getattr(cloud_obj, 'content_type', None)
    if content_type == 'application/directory':
        return
    matched_headers = {}
    for pattern, pattern_headers in header_patterns:
        if pattern.match(cloud_obj.name):
            matched_headers.update(pattern_headers.copy())
    matched_headers.update(cloud_obj.headers)  # preserve headers already set
    matched_headers.update(headers)  # explicitly set headers overwrite matches and already set headers
    if matched_headers != cloud_obj.headers:
        cloud_obj.headers = matched_headers
        cloud_obj.sync_metadata()

def get_gzipped_contents(input_file):
    """
    Return a gzipped version of a previously opened file's buffer.
    """
    zbuf = StringIO()
    zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
    zfile.write(input_file.read())
    zfile.close()
    return ContentFile(zbuf.getvalue())

class CloudFilesStorage(Storage):
    """
    Custom storage for Rackspace Cloud Files.
    """
    default_quick_listdir = True

    def __init__(self, username=None, api_key=None, container=None, timeout=None,
                 connection_kwargs=None):
        """
        Initialize the settings for the connection and container.
        """
        self.api_key = api_key or CUMULUS['API_KEY']
        self.auth_url = CUMULUS['AUTH_URL']
        self.connection_kwargs = connection_kwargs or {}
        self.container_name = container or CUMULUS['CONTAINER']
        self.timeout = timeout or CUMULUS['TIMEOUT']
        self.use_servicenet = CUMULUS['SERVICENET']
        self.username = username or CUMULUS['USERNAME']
        self.ttl = CUMULUS['TTL']
        self.use_ssl = CUMULUS['USE_SSL']

        if 'CONTAINER_URI' in CUMULUS:
            self._container_public_uri = CUMULUS['CONTAINER_URI']

    def __getstate__(self):
        """
        Return a picklable representation of the storage.
        """
        return dict(username=self.username,
                    api_key=self.api_key,
                    container_name=self.container_name,
                    timeout=self.timeout,
                    use_servicenet=self.use_servicenet,
                    connection_kwargs=self.connection_kwargs)

    def _get_connection(self):
        if not hasattr(self, '_connection'):
            self._connection = cloudfiles.get_connection(
                                  username=self.username,
                                  api_key=self.api_key,
                                  authurl = self.auth_url,
                                  timeout=self.timeout,
                                  servicenet=self.use_servicenet,
                                  **self.connection_kwargs)
        return self._connection

    def _set_connection(self, value):
        self._connection = value

    connection = property(_get_connection, _set_connection)

    def _get_container(self):
        if not hasattr(self, '_container'):
            self.container = self.connection.get_container(
                                                        self.container_name)
        return self._container

    def _set_container(self, container):
        """
        Set the container (and, if needed, the configured TTL on it), making
        the container publicly available.
        """
        if container.cdn_ttl != self.ttl or not container.is_public():
            container.make_public(ttl=self.ttl)
        if hasattr(self, '_container_public_uri'):
            delattr(self, '_container_public_uri')
        self._container = container

    container = property(_get_container, _set_container)

    def _get_container_url(self):
        if not hasattr(self, '_container_public_uri'):
            if self.use_ssl:
                self._container_public_uri = self.container.public_ssl_uri()
            else:
                self._container_public_uri = self.container.public_uri()
        if CUMULUS['CNAMES'] and self._container_public_uri in CUMULUS['CNAMES']:
            self._container_public_uri = CUMULUS['CNAMES'][self._container_public_uri]
        return self._container_public_uri

    container_url = property(_get_container_url)

    def _get_cloud_obj(self, name):
        """
        Helper function to get retrieve the requested Cloud Files Object.
        """
        return self.container.get_object(name)

    def _open(self, name, mode='rb'):
        """
        Return the CloudFilesStorageFile.
        """
        return CloudFilesStorageFile(storage=self, name=name)

    def _save(self, name, content):
        """
        Use the Cloud Files service to write ``content`` to a remote file
        (called ``name``).
        """
        (path, last) = os.path.split(name)
        if path:
            try:
                self.container.get_object(path)
            except NoSuchObject:
                self._save(path, CloudStorageDirectory(path))

        content.open()
        cloud_obj = self.container.create_object(name)
        # If the content type is available, pass it in directly rather than
        # getting the cloud object to try to guess.
        if hasattr(content.file, 'content_type'):
            cloud_obj.content_type = content.file.content_type
        elif hasattr(content, 'content_type'):
            cloud_obj.content_type = content.content_type
        else:
            mime_type, encoding = mimetypes.guess_type(name)
            cloud_obj.content_type = mime_type
        # gzip the file if its of the right content type
        if cloud_obj.content_type in CUMULUS.get('GZIP_CONTENT_TYPES', []):
            content = get_gzipped_contents(content)
            cloud_obj.headers['Content-Encoding'] = 'gzip'
        # set file size
        if hasattr(content.file, 'size'):
            cloud_obj.size = content.file.size
        else:
            cloud_obj.size = content.size
        cloud_obj.send(content)
        content.close()
        sync_headers(cloud_obj)
        return name

    def delete(self, name):
        """
        Deletes the specified file from the storage system.
        """
        try:
            self.container.delete_object(name)
        except ResponseError, exc:
            if exc.status == 404:
                pass
            else:
                raise

    def exists(self, name):
        """
        Returns True if a file referenced by the given name already exists in
        the storage system, or False if the name is available for a new file.
        """
        try:
            self._get_cloud_obj(name)
            return True
        except NoSuchObject:
            return False

    def listdir(self, path):
        """
        Lists the contents of the specified path, returning a 2-tuple; the
        first being an empty list of directories (not available for quick-
        listing), the second being a list of filenames.

        If the list of directories is required, use the full_listdir method.
        """
        files = []
        if path and not path.endswith('/'):
            path = '%s/' % path
        path_len = len(path)
        for name in self.container.list_objects(path=path):
            files.append(name[path_len:])
        return ([], files)

    def full_listdir(self, path):
        """
        Lists the contents of the specified path, returning a 2-tuple of lists;
        the first item being directories, the second item being files.

        On large containers, this may be a slow operation for root containers
        because every single object must be returned (cloudfiles does not
        provide an explicit way of listing directories).
        """
        dirs = set()
        files = []
        if path and not path.endswith('/'):
            path = '%s/' % path
        path_len = len(path)
        for name in self.container.list_objects(prefix=path):
            name = name[path_len:]
            slash = name[1:-1].find('/') + 1
            if slash:
                dirs.add(name[:slash])
            elif name:
                files.append(name)
        dirs = list(dirs)
        dirs.sort()
        return (dirs, files)

    def size(self, name):
        """
        Returns the total size, in bytes, of the file specified by name.
        """
        return self._get_cloud_obj(name).size

    def url(self, name):
        """
        Returns an absolute URL where the file's contents can be accessed
        directly by a web browser.
        """
        return '%s/%s' % (self.container_url, name)

    def modified_time(self, name):
        # CloudFiles return modified date in different formats
        # depending on whether or not we pre-loaded objects.
        # When pre-loaded, timezone is not included but we
        # assume UTC. Since FileStorage returns localtime, and
        # collectstatic compares these dates, we need to depend
        # on dateutil to help us convert timezones.
        try:
           from dateutil import parser, tz
        except ImportError:
            raise NotImplementedError("This functionality requires dateutil to be installed")

        obj = self.container.get_object(name)

        # convert to string to date
        date = parser.parse(obj.last_modified)

        # if the date has no timzone, assume UTC
        if date.tzinfo == None:
            date = date.replace(tzinfo=tz.tzutc())

        # convert date to local time w/o timezone
        date = date.astimezone(tz.tzlocal()).replace(tzinfo=None)
        return date

class CloudStorageDirectory(File):
    """
    A File-like object that creates a directory at cloudfiles
    """

    def __init__(self, name):
        super(CloudStorageDirectory, self).__init__(StringIO(), name=name)
        self.file.content_type = 'application/directory'
        self.size = 0

    def __str__(self):
        return 'directory'

    def __nonzero__(self):
        return True

    def open(self, mode=None):
        self.seek(0)

    def close(self):
        pass

class CloudFilesStaticStorage(CloudFilesStorage):
    """
    Subclasses CloudFilesStorage to automatically set the container to the one
    specified in CUMULUS['STATIC_CONTAINER']. This provides the ability to
    specify a separate storage backend for Django's collectstatic command.

    To use, make sure CUMULUS['STATIC_CONTAINER'] is set to something other
    than CUMULUS['CONTAINER']. Then, tell Django's staticfiles app by setting
    STATICFILES_STORAGE = 'cumulus.storage.CloudFilesStaticStorage'.
    """
    def __init__(self, *args, **kwargs):
        if not 'container' in kwargs:
            kwargs['container'] = CUMULUS['STATIC_CONTAINER']
        super(CloudFilesStaticStorage, self).__init__(*args, **kwargs)

class CloudFilesStorageFile(File):
    closed = False

    def __init__(self, storage, name, *args, **kwargs):
        self._storage = storage
        self._pos = 0
        super(CloudFilesStorageFile, self).__init__(file=None, name=name,
                                                    *args, **kwargs)

    def _get_pos(self):
        return self._pos

    def _get_size(self):
        if not hasattr(self, '_size'):
            self._size = self._storage.size(self.name)
        return self._size

    def _set_size(self, size):
        self._size = size

    size = property(_get_size, _set_size)

    def _get_file(self):
        if not hasattr(self, '_file'):
            self._file = self._storage._get_cloud_obj(self.name)
            self._file.tell = self._get_pos
        return self._file

    def _set_file(self, value):
        if value is None:
            if hasattr(self, '_file'):
                del self._file
        else:
            self._file = value

    file = property(_get_file, _set_file)

    def read(self, num_bytes=0):
        if self._pos == self._get_size():
            return ""
        if num_bytes and self._pos + num_bytes > self._get_size():
            num_bytes = self._get_size() - self._pos
        data = self.file.read(size=num_bytes or -1, offset=self._pos)
        self._pos += len(data)
        return data

    def open(self, *args, **kwargs):
        """
        Open the cloud file object.
        """
        self._pos = 0

    def close(self, *args, **kwargs):
        self._pos = 0

    @property
    def closed(self):
        return not hasattr(self, '_file')

    def seek(self, pos):
        self._pos = pos

class ThreadSafeCloudFilesStorage(CloudFilesStorage):
    """
    Extends CloudFilesStorage to make it mostly thread safe.

    As long as you don't pass container or cloud objects
    between threads, you'll be thread safe.

    Uses one cloudfiles connection per thread.
    """

    def __init__(self, *args, **kwargs):
        super(ThreadSafeCloudFilesStorage, self).__init__(*args, **kwargs)

        import threading
        self.local_cache = threading.local()

    def _get_connection(self):
        if not hasattr(self.local_cache, 'connection'):
            connection = cloudfiles.get_connection(self.username,
                                    self.api_key, **self.connection_kwargs)
            self.local_cache.connection = connection

        return self.local_cache.connection

    connection = property(_get_connection, CloudFilesStorage._set_connection)

    def _get_container(self):
        if not hasattr(self.local_cache, 'container'):
            container = self.connection.get_container(self.container_name)
            self.local_cache.container = container

        return self.local_cache.container

    container = property(_get_container, CloudFilesStorage._set_container)
<|endoftext|>


<commit_before>import imp

from keras.models import Sequential
from keras.layers.core import Flatten, Dense, Dropout, Activation
from keras.layers.convolutional import Convolution2D, MaxPooling2D, ZeroPadding2D
from keras.optimizers import SGD
from keras.utils import np_utils, generic_utils

from keras.preprocessing.image import ImageDataGenerator

pyvec_api = imp.load_source('api', './pyvec/pyvec/core/api.py')

def augment_data(train_data):
    augmented_data_generator = ImageDataGenerator(
        rotation_range=20,
        horizontal_flip=True
    )

    augmented_data_generator.fit(train_data)
    return augmented_data_generator

def create_model(input_size, number_of_classes):
    model = Sequential()

    model.add(ZeroPadding2D((1,1), input_shape=(input_size[0], input_size[1])))
    model.add(Convolution2D(64, 3, 3))
    model.add(Activation("relu"))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(64, 3, 3))
    model.add(Activation("relu"))

    model.add(MaxPooling2D((2,2), strides=(2,2)))

    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(128, 3, 3))
    model.add(Activation("relu"))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(128, 3, 3))
    model.add(Activation("relu"))

    model.add(MaxPooling2D((2,2), strides=(2,2)))

    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(256, 3, 3))
    model.add(Activation("relu"))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(256, 3, 3))
    model.add(Activation("relu"))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(256, 3, 3))
    model.add(Activation("relu"))

    model.add(MaxPooling2D((2,2), strides=(2,2)))

    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3))
    model.add(Activation("relu"))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3))
    model.add(Activation("relu"))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3))
    model.add(Activation("relu"))

    model.add(MaxPooling2D((2,2), strides=(2,2)))

    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3))
    model.add(Activation("relu"))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3))
    model.add(Activation("relu"))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3))
    model.add(Activation("relu"))

    model.add(MaxPooling2D((2,2), strides=(2,2)))

    model.add(Flatten())
    model.add(Dense(4096,))
    model.add(Activation("relu"))
    model.add(Dropout(0.5))

    model.add(Dense(4096))
    model.add(Activation("relu"))
    model.add(Dropout(0.5))

    model.add(Dense(number_of_classes))
    model.add(Activation("softmax"))

    sgd = SGD()
    model.compile(loss="multiple_crossentropy", optimizer=sgd)
    return model

def train_model_and_test(number_of_epochs, train_data, train_label, augmented_data_generator):
    for e in range(number_of_epochs):
        progress_bar = generic_utils.Progbar(train_data.shape[0])
        for train_d_batch, train_l_batch in augmented_data_generator(train_data):
            score = model.train_on_batch(train_d_batch, train_l_batch)
            progress_bar.add(train_d_batch[0], values=[('train loss', score[0])])

if __name__ == "__main__":
    pictures_directory = "/home/keo7/Pictures/plant_images" # The directory containing ALL of the plant images.
    labels = "../file_preperation/labels/file.tsv" # The tsv file of which contains the image name and its correlated label. (correlate_photos_to_phenotype.py)

    input_size = (224, 224)
    split = 0.9 # 90% of the dataset used for training, leaving 10% out for testing
    number_of_epochs = 20

    (train_data, train_label), (test_data, test_label) = pyvec_api.load_images_with_tsv(pictures_directory, labels, input_size[0], input_size[1], split)
    number_of_classes = len(set(train_label))

    augmented_data_generator = augment_data(train_data)

    model = create_model(input_size, number_of_classes)

    train_model_and_test(number_of_epochs, train_data, train_label, augmented_data_generator)<commit_msg>Whoops<commit_after>import imp

from keras.models import Sequential
from keras.layers.core import Flatten, Dense, Dropout, Activation
from keras.layers.convolutional import Convolution2D, MaxPooling2D, ZeroPadding2D
from keras.optimizers import SGD
from keras.utils import np_utils, generic_utils

from keras.preprocessing.image import ImageDataGenerator

pyvec_api = imp.load_source('api', './pyvec/pyvec/core/api.py')

def augment_data(train_data):
    augmented_data_generator = ImageDataGenerator(
        rotation_range=20,
        horizontal_flip=True
    )

    augmented_data_generator.fit(train_data)
    return augmented_data_generator

def create_model(input_size, number_of_classes):
    model = Sequential()

    model.add(ZeroPadding2D((1,1), input_shape=(input_size[0], input_size[1])))
    model.add(Convolution2D(64, 3, 3))
    model.add(Activation("relu"))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(64, 3, 3))
    model.add(Activation("relu"))

    model.add(MaxPooling2D((2,2), strides=(2,2)))

    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(128, 3, 3))
    model.add(Activation("relu"))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(128, 3, 3))
    model.add(Activation("relu"))

    model.add(MaxPooling2D((2,2), strides=(2,2)))

    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(256, 3, 3))
    model.add(Activation("relu"))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(256, 3, 3))
    model.add(Activation("relu"))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(256, 3, 3))
    model.add(Activation("relu"))

    model.add(MaxPooling2D((2,2), strides=(2,2)))

    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3))
    model.add(Activation("relu"))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3))
    model.add(Activation("relu"))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3))
    model.add(Activation("relu"))

    model.add(MaxPooling2D((2,2), strides=(2,2)))

    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3))
    model.add(Activation("relu"))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3))
    model.add(Activation("relu"))
    model.add(ZeroPadding2D((1,1)))
    model.add(Convolution2D(512, 3, 3))
    model.add(Activation("relu"))

    model.add(MaxPooling2D((2,2), strides=(2,2)))

    model.add(Flatten())
    model.add(Dense(4096,))
    model.add(Activation("relu"))
    model.add(Dropout(0.5))

    model.add(Dense(4096))
    model.add(Activation("relu"))
    model.add(Dropout(0.5))

    model.add(Dense(number_of_classes))
    model.add(Activation("softmax"))

    sgd = SGD()
    model.compile(loss="multiple_crossentropy", optimizer=sgd)
    return model

def train_model_and_test(number_of_epochs, number_of_classes, train_data, train_label, augmented_data_generator):
    train_label = np_utils.to_categorical(train_label, number_of_classes)

    for e in range(number_of_epochs):
        progress_bar = generic_utils.Progbar(train_data.shape[0])
        for train_d_batch, train_l_batch in augmented_data_generator(train_data, train_label):
            score = model.train_on_batch(train_d_batch, train_l_batch)
            progress_bar.add(train_d_batch[0], values=[('train loss', score[0])])

if __name__ == "__main__":
    pictures_directory = "/home/keo7/Pictures/plant_images" # The directory containing ALL of the plant images.
    labels = "../file_preperation/labels/file.tsv" # The tsv file of which contains the image name and its correlated label. (correlate_photos_to_phenotype.py)

    input_size = (224, 224)
    split = 0.9 # 90% of the dataset used for training, leaving 10% out for testing
    number_of_epochs = 20

    (train_data, train_label), (test_data, test_label) = pyvec_api.load_images_with_tsv(pictures_directory, labels, input_size[0], input_size[1], split)
    number_of_classes = len(set(train_label))

    augmented_data_generator = augment_data(train_data)

    model = create_model(input_size, number_of_classes)

    train_model_and_test(number_of_epochs, number_of_classes, train_data, train_label, augmented_data_generator)<|endoftext|>


<commit_before>""" Code to help using the logging module with tqdm progress bars. """

import contextlib
import logging

class TqdmLoggingHandler(logging.Handler):

  """ Logging handler sending messages to the tqdm write method (avoids overlap). """

  def __init__(self, tqdm, *args, **kwargs):
    self.tqdm = tqdm
    super().__init__(*args, **kwargs)

  def emit(self, record):
    msg = self.format(record)
    self.tqdm.write(msg)

@contextlib.contextmanager
def redirect_logging(tqdm_obj, logger=logging.getLogger()):
  """ Context manager to redirect logging to a TqdmLoggingHandler object and then restore the original. """
  # remove current handler
  assert(len(logger.handlers) == 1)
  prev_handler = logger.handlers[0]
  logger.removeHandler(prev_handler)

  # add tqdm handler
  tqdm_handler = TqdmLoggingHandler(tqdm_obj)
  if prev_handler.formatter is not None:
    tqdm_handler.setFormatter(prev_handler.formatter)
  logger.addHandler(tqdm_handler)

  try:
    yield
  finally:
    # restore handler
    logger.removeHandler(tqdm_handler)
    logger.addHandler(prev_handler)
<commit_msg>Fix tqdm_logging thread safety<commit_after>""" Code to help using the logging module with tqdm progress bars. """

import contextlib
import logging

import threading

logging_handlers_lock = threading.Lock()

class TqdmLoggingHandler(logging.Handler):

  """ Logging handler sending messages to the tqdm write method (avoids overlap). """

  def __init__(self, tqdm, *args, **kwargs):
    self.tqdm = tqdm
    super().__init__(*args, **kwargs)

  def emit(self, record):
    msg = self.format(record)
    self.tqdm.write(msg)

@contextlib.contextmanager
def redirect_logging(tqdm_obj, logger=logging.getLogger()):
  """ Context manager to redirect logging to a TqdmLoggingHandler object and then restore the original logging behavior. """
  with logging_handlers_lock:
    # remove current handlers
    prev_handlers = []
    for handler in logger.handlers.copy():
      prev_handlers.append(handler)
      logger.removeHandler(handler)

    # add tqdm handler
    tqdm_handler = TqdmLoggingHandler(tqdm_obj)
    if prev_handlers[-1].formatter is not None:
      tqdm_handler.setFormatter(prev_handlers[-1].formatter)
    logger.addHandler(tqdm_handler)

  try:
    yield
  finally:
    # restore handlers
    with logging_handlers_lock:
      logger.removeHandler(tqdm_handler)
      for handler in prev_handlers:
        logger.addHandler(handler)
<|endoftext|>