<commit_before>#!/usr/bin/env python
#
# tournament.py -- implementation of a Swiss-system tournament
#

import psycopg2
import psycopg2.extras

def connect():
    """Connect to the PostgreSQL database.

    Returns:
      A database connection."""
    return psycopg2.connect("dbname=tournament")

def crud_operation(is_proc, operation, query, params, expected_rows,
                    has_return_id):
    """Perform CRUD operations on database

   Args:
      is_proc: True if we call a stored procedure
      operation: create, read, update, delete  according to database operation
      query: SQL query statement
      params: Array with params for query either stored procedure or query
      expected_rows: 'one' or 'many' if we expect a single row or multiple
      has_return_id: True if we make an INSERT and the new id must return
    Returns:
      None: If the operation does not generate any result
      row: If a single row is returned, it can be the generated key
           from last INSERT
      rows: If multiple rows are returned
    """
    rows = None
    db = connect()
    c = db.cursor(cursor_factory=psycopg2.extras.DictCursor)
    if is_proc:
        c.callproc(query, params)
    else:
        c.execute(query, params)
    if operation == "read":
        rows = c.fetchone() if expected_rows == "one" else c.fetchall()
    else:
        if operation == "create" and has_return_id:
            rows = c.fetchone()
    db.commit()
    db.close()
    return rows;

def delete_event(event_id):
    """Remove an event and all its related data from the database, without
    erasing registered players.

    Args:
      event_id: the id's event.
    """
    query = "DELETE FROM events WHERE id=%s"
    crud_operation(False, "delete", query, [event_id], None, None)

def delete_all_events():
    """Remove all events and all their related data from the database,
    without erasing registered players."""
    query = "DELETE FROM matches"
    crud_operation(False, "delete", query, [], None, None)
    query = "DELETE FROM playersInEvent"
    crud_operation(False, "delete", query, [], None, None)
    query = "DELETE FROM events"
    crud_operation(False, "delete", query, [], None, None)

def delete_all_matches():
    """Remove all the match records from the database."""
    query = "DELETE FROM matches"
    crud_operation(False, "delete", query, [], None, None)

def delete_matches_from_event(event_id):
    """Remove all the match records from an event.\

    Args:
      event_id: the id's event.
    """
    query = "DELETE FROM matches WHERE event=%s"
    crud_operation(False, "delete", query, [event_id], None, None)

def delete_players():
    """Remove all the player records from the database."""
    query = "DELETE FROM players"
    crud_operation(False, "delete", query, [], None, None)

def register_event(name, event_date):
    """Adds a new event to the tournament database.

    The database assigns a unique serial id number for the event.

    Args:
      name: the event's full name (need not be unique).
      event_date: this date could be in a future time.
    Returns:
      id: Key created from last INSERT
    """
    query = "INSERT INTO events (name, event_date) \
             VALUES (%s, %s) RETURNING id"
    row = crud_operation(False, "create", query, [name, event_date],
                         None, True)
    return row["id"]

def count_events():
    """Returns the number of events currently registered.

    Returns:
      num: Total number of events in the database
    """
    query = "SELECT count(*) as num FROM events"
    row =  crud_operation(False, "read", query, [], "one", None)
    return row["num"]

def count_players():
    """Returns the number of players currently registered.

    Returns:
      num: Total number of players registered in the database
    """
    query = "SELECT count(*) as num FROM players"
    row =  crud_operation(False, "read", query, [], "one", None)
    return row["num"]

def register_player(firstname, lastname):
    """Adds a player to the tournament database.

    The database assigns a unique serial id number for the player.  (This
    should be handled by your SQL database schema, not in your Python code.)

    Args:
      firstname: the player's firstname (need not be unique).
      lastname: the player's lastname (need not be unique).
    Returns:
      id: Key created from last INSERT
    """
    query = "INSERT INTO players (firstname, lastname) \
             VALUES (%s, %s) RETURNING id"
    row = crud_operation(False, "create", query, [firstname, lastname],
                         None, True)
    return row["id"]

def add_player_to_event(event_id, player_id):
    """Adds a player into an existing event.

    Args:
      event_id: the id's event.
      player_id: the id's player.
    """
    query = "INSERT INTO playersInEvent (event, player) VALUES (%s, %s)"
    crud_operation(False, "create", query, [event_id, player_id], None, False)

def remove_player_from_event(event_id, player_id):
    """Removes a single player from an existing event.

    Args:
      event_id: the id's event.
      player_id: the id's player.
    """
    query = "DELETE FROM playersInEvent WHERE event=%s AND player=%s"
    crud_operation(False, "delete", query, [event_id, player_id], None, False)

def count_players_in_event(event_id):
    """Returns the number of players in an specified event.

    Args:
      event_id: the id's event.
    Returns:
      num: Number of players in the event
    """
    query = "SELECT count(*) as num FROM playersInEvent WHERE event=%s"
    row =  crud_operation(False, "read", query, [event_id], "one", None)
    return row["num"]

def player_standings(event_id):
    """Returns a list of the players and their win records, sorted by
    ontained points from an event.

    The first entry in the list should be the player in first place,
    or a player tied for first place if there is currently a tie.

   Args:
      event_id: the id's event.
    Returns:
      A list of tuples, each of which contains (id, name, points, matches):
        id: the player's unique id (assigned by the database)
        name: the player's full name (as registered)
        points: the number of matches the player has won
        matches: the number of matches the player has played
    """
    procedure = "standings"
    rows =  crud_operation(True, "read", procedure, [event_id], "all", None)
    return rows

def report_match(event_id, round_number, player_one_id, player_one_points,
                 player_two_id, player_two_points):
    """Records the outcome of a single match between two players.
    If a player won obtains one point, if is a tie, half point to each one

    Args:
      event_id: the id's event
      round_number: The round that has played
      player_one_id:  the id number of the first player
      player_one_points: Number of points obtained in this match
      player_one_id:  the id number of the second player
      player_two_points: Number of points obtained in this match
    """
    query = "INSERT INTO matches (player_one, player_two, player_one_score, \
             player_two_score, event, round_number) \
             VALUES (%s, %s, %s, %s, %s, %s)"
    crud_operation(False, "create", query, [player_one_id, player_two_id,
        player_one_points, player_two_points, event_id, round_number],
        None, False)

def swiss_pairings(event_id):
    """Returns a list of pairs of players for the next round of a match.

    Assuming that there are an even number of players registered, each player
    appears exactly once in the pairings.  Each player is paired with another
    player with an equal or nearly-equal win record, that is, a player adjacent
    to him or her in the standings.
    Args:
      event_id: the id's event.
    Returns:
      A list of tuples, each of which contains (id1, name1, id2, name2)
        id1: the first player's unique id
        name1: the first player's name
        id2: the second player's unique id
        name2: the second player's name
    """
    standings = player_standings(event_id)
    pairings = []
    for i in range (0, len(standings), 2):
        (id1, name1, points1, matches1) = standings[i]
        (id2, name2, points2, matches2) = standings[i+1]
        pairings.append((id1, name1, id2, name2))
    return pairings

<commit_msg>feat: Prevent rematches between players<commit_after>#!/usr/bin/env python
#
# tournament.py -- implementation of a Swiss-system tournament
#

import psycopg2
import psycopg2.extras

def connect():
    """Connect to the PostgreSQL database.

    Returns:
      A database connection."""
    return psycopg2.connect("dbname=tournament")

def crud_operation(is_proc, operation, query, params, expected_rows,
                    has_return_id):
    """Perform CRUD operations on database

   Args:
      is_proc: True if we call a stored procedure
      operation: create, read, update, delete  according to database operation
      query: SQL query statement
      params: Array with params for query either stored procedure or query
      expected_rows: 'one' or 'many' if we expect a single row or multiple
      has_return_id: True if we make an INSERT and the new id must return
    Returns:
      None: If the operation does not generate any result
      row: If a single row is returned, it can be the generated key
           from last INSERT
      rows: If multiple rows are returned
    """
    rows = None
    db = connect()
    c = db.cursor(cursor_factory=psycopg2.extras.DictCursor)
    if is_proc:
        c.callproc(query, params)
    else:
        c.execute(query, params)
    if operation == "read":
        rows = c.fetchone() if expected_rows == "one" else c.fetchall()
    else:
        if operation == "create" and has_return_id:
            rows = c.fetchone()
    db.commit()
    db.close()
    return rows;

def delete_event(event_id):
    """Remove an event and all its related data from the database, without
    erasing registered players.

    Args:
      event_id: the id's event.
    """
    query = "DELETE FROM events WHERE id=%s"
    crud_operation(False, "delete", query, [event_id], None, None)

def delete_all_events():
    """Remove all events and all their related data from the database,
    without erasing registered players."""
    query = "DELETE FROM matches"
    crud_operation(False, "delete", query, [], None, None)
    query = "DELETE FROM playersInEvent"
    crud_operation(False, "delete", query, [], None, None)
    query = "DELETE FROM events"
    crud_operation(False, "delete", query, [], None, None)

def delete_all_matches():
    """Remove all the match records from the database."""
    query = "DELETE FROM matches"
    crud_operation(False, "delete", query, [], None, None)

def delete_matches_from_event(event_id):
    """Remove all the match records from an event.\

    Args:
      event_id: the id's event.
    """
    query = "DELETE FROM matches WHERE event=%s"
    crud_operation(False, "delete", query, [event_id], None, None)

def delete_players():
    """Remove all the player records from the database."""
    query = "DELETE FROM players"
    crud_operation(False, "delete", query, [], None, None)

def register_event(name, event_date):
    """Adds a new event to the tournament database.

    The database assigns a unique serial id number for the event.

    Args:
      name: the event's full name (need not be unique).
      event_date: this date could be in a future time.
    Returns:
      id: Key created from last INSERT
    """
    query = "INSERT INTO events (name, event_date) \
             VALUES (%s, %s) RETURNING id"
    row = crud_operation(False, "create", query, [name, event_date],
                         None, True)
    return row["id"]

def count_events():
    """Returns the number of events currently registered.

    Returns:
      num: Total number of events in the database
    """
    query = "SELECT count(*) as num FROM events"
    row =  crud_operation(False, "read", query, [], "one", None)
    return row["num"]

def count_players():
    """Returns the number of players currently registered.

    Returns:
      num: Total number of players registered in the database
    """
    query = "SELECT count(*) as num FROM players"
    row =  crud_operation(False, "read", query, [], "one", None)
    return row["num"]

def register_player(firstname, lastname):
    """Adds a player to the tournament database.

    The database assigns a unique serial id number for the player.  (This
    should be handled by your SQL database schema, not in your Python code.)

    Args:
      firstname: the player's firstname (need not be unique).
      lastname: the player's lastname (need not be unique).
    Returns:
      id: Key created from last INSERT
    """
    query = "INSERT INTO players (firstname, lastname) \
             VALUES (%s, %s) RETURNING id"
    row = crud_operation(False, "create", query, [firstname, lastname],
                         None, True)
    return row["id"]

def add_player_to_event(event_id, player_id):
    """Adds a player into an existing event.

    Args:
      event_id: the id's event.
      player_id: the id's player.
    """
    query = "INSERT INTO playersInEvent (event, player) VALUES (%s, %s)"
    crud_operation(False, "create", query, [event_id, player_id], None, False)

def remove_player_from_event(event_id, player_id):
    """Removes a single player from an existing event.

    Args:
      event_id: the id's event.
      player_id: the id's player.
    """
    query = "DELETE FROM playersInEvent WHERE event=%s AND player=%s"
    crud_operation(False, "delete", query, [event_id, player_id], None, False)

def count_players_in_event(event_id):
    """Returns the number of players in an specified event.

    Args:
      event_id: the id's event.
    Returns:
      num: Number of players in the event
    """
    query = "SELECT count(*) as num FROM playersInEvent WHERE event=%s"
    row =  crud_operation(False, "read", query, [event_id], "one", None)
    return row["num"]

def player_standings(event_id):
    """Returns a list of the players and their win records, sorted by
    ontained points from an event.

    The first entry in the list should be the player in first place,
    or a player tied for first place if there is currently a tie.

   Args:
      event_id: the id's event.
    Returns:
      A list of tuples, each of which contains (id, name, points, matches):
        id: the player's unique id (assigned by the database)
        name: the player's full name (as registered)
        points: the number of matches the player has won
        matches: the number of matches the player has played
    """
    procedure = "standings"
    rows =  crud_operation(True, "read", procedure, [event_id], "all", None)
    return rows

def report_match(event_id, round_number, player_one_id, player_one_points,
                 player_two_id, player_two_points):
    """Records the outcome of a single match between two players.
    If a player won obtains one point, if is a tie, half point to each one

    Args:
      event_id: the id's event
      round_number: The round that has played
      player_one_id:  the id number of the first player
      player_one_points: Number of points obtained in this match
      player_two_id:  the id number of the second player
      player_two_points: Number of points obtained in this match
    """
    query = "INSERT INTO matches (player_one, player_two, player_one_score, \
             player_two_score, event, round_number) \
             VALUES (%s, %s, %s, %s, %s, %s)"
    crud_operation(False, "create", query, [player_one_id, player_two_id,
        player_one_points, player_two_points, event_id, round_number],
        None, False)

def have_played(player_one, player_two):
    """Returns True if there is an existing match among players

    Args:
      player_one:  the id number of the first player
      player_two:  the id number of the second player
    """
    query = "SELECT * FROM matches WHERE (player_one = %s \
        AND player_two = %s) or (player_one = %s \
        AND player_two = %s)"
    rows =  crud_operation(False, "read", query,
            [player_one, player_two, player_two, player_one],
             "many", None)
    if len(rows) > 0:
        return True
    return False

def swiss_pairings(event_id):
    """Returns a list of pairs of players for the next round of a match.

    Assuming that there are an even number of players registered, each player
    appears exactly once in the pairings.  Each player is paired with another
    player with an equal or nearly-equal win record, that is, a player adjacent
    to him or her in the standings.
    Args:
      event_id: the id's event.
    Returns:
      A list of tuples, each of which contains (id1, name1, id2, name2)
        id1: the first player's unique id
        name1: the first player's name
        id2: the second player's unique id
        name2: the second player's name
    """
    standings = player_standings(event_id)
    pairings = []
    selected = set([])
    for i in range (0, len(standings)):
        (id1, name1, points1, matches1) = standings[i]
        if id1 not in selected:
            # This player hasn't been selected
            # Finding the next opponent
            for j in range(i+1,len(standings)):
                (id2, name2, points2, matches2) = standings[j]
                if id2 not in selected and not have_played(id1, id2):
                    # If they have not been selected neither
                    # played, we can pair them
                    pairings.append((id1, name1, id2, name2))
                    selected.add(id1)
                    selected.add(id2)
                    break

    return pairings

<|endoftext|>


<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2018, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

from __future__ import print_function
import os
import sys
import traceback
import numpy as np
import time

import torch
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms

from htmresearch.frameworks.pytorch.benchmark_utils import (
  register_nonzero_counter, unregister_counter_nonzero)
from htmresearch.support.expsuite import PyExperimentSuite

from htmresearch.frameworks.pytorch.image_transforms import RandomNoise
from htmresearch.frameworks.pytorch.sparse_net import SparseNet
from htmresearch.frameworks.pytorch.duty_cycle_metrics import plotDutyCycles
from htmresearch.frameworks.pytorch.dataset_utils import createValidationDataSampler

class MNISTSparseExperiment(PyExperimentSuite):
  """
  Allows running multiple sparse MNIST experiments in parallel
  """

  def parse_cfg(self):
    super(MNISTSparseExperiment, self).parse_cfg()
    # Change the current working directory to be relative to 'experiments.cfg'
    projectDir = os.path.dirname(self.options.config)
    projectDir = os.path.abspath(projectDir)
    os.chdir(projectDir)

  def reset(self, params, repetition):
    """
    Called once at the beginning of each experiment.
    """
    self.startTime = time.time()
    print(params)
    torch.manual_seed(params["seed"] + repetition)
    np.random.seed(params["seed"] + repetition)

    # Get our directories correct
    self.dataDir = params["datadir"]
    self.resultsDir = os.path.join(params["path"], params["name"], "plots")

    if not os.path.exists(self.resultsDir):
      os.makedirs(self.resultsDir)

    self.use_cuda = not params["no_cuda"] and torch.cuda.is_available()
    self.device = torch.device("cuda" if self.use_cuda else "cpu")

    transform = transforms.Compose([transforms.ToTensor(),
                                    transforms.Normalize((0.1307,), (0.3081,))])

    train_dataset = datasets.MNIST(self.dataDir, train=True, download=True,
                                   transform=transform)

    # Create training and validation sampler from MNIST dataset by training on
    # random X% of the training set and validating on the remaining (1-X)%,
    # where X can be tuned via the "validation" parameter
    validation = params.get("validation", 50000.0 / 60000.0)
    if validation < 1.0:
      self.train_sampler, self.validation_sampler = createValidationDataSampler(
        train_dataset, validation)

      self.train_loader = torch.utils.data.DataLoader(train_dataset,
                                                      batch_size=params["batch_size"],
                                                      sampler=self.train_sampler)

      self.validation_loader = torch.utils.data.DataLoader(train_dataset,
                                                           batch_size=params["batch_size"],
                                                           sampler=self.validation_sampler)
    else:
      # No validation. Normal training dataset
      self.validation_loader = None
      self.validation_sampler = None
      self.train_sampler = None
      self.train_loader = torch.utils.data.DataLoader(train_dataset,
                                                      batch_size=params["batch_size"],
                                                      shuffle=True)

    self.test_loader = torch.utils.data.DataLoader(
      datasets.MNIST(self.dataDir, train=False, transform=transform),
      batch_size=params["test_batch_size"], shuffle=True)

    # Parse 'n' and 'k' parameters
    n = params["n"]
    k = params["k"]
    if isinstance(n, basestring):
      n = map(int, n.split("_"))
    if isinstance(k, basestring):
      k = map(int, k.split("_"))

    if params["use_cnn"]:
      c1_out_channels = params["c1_out_channels"]
      c1_k = params["c1_k"]
      if isinstance(c1_out_channels, basestring):
        c1_out_channels = map(int, c1_out_channels.split("_"))
      if isinstance(c1_k, basestring):
        c1_k = map(int, c1_k.split("_"))

      sp_model = SparseNet(
        inputSize=(1, 28, 28),
        outChannels=c1_out_channels,
        c_k=c1_k,
        dropout=params["dropout"],
        n=n,
        k=k,
        boostStrength=params["boost_strength"],
        weightSparsity=params["weight_sparsity"],
        boostStrengthFactor=params["boost_strength_factor"],
        kInferenceFactor=params["k_inference_factor"],
        useBatchNorm=params["use_batch_norm"],
      )
      print("c1OutputLength=", sp_model.cnnSdr[0].outputLength)
    else:
      sp_model = SparseNet(
        n=n,
        k=k,
        boostStrength=params["boost_strength"],
        weightSparsity=params["weight_sparsity"],
        boostStrengthFactor=params["boost_strength_factor"],
        kInferenceFactor=params["k_inference_factor"],
        dropout=params["dropout"],
        useBatchNorm=params["use_batch_norm"],
      )
    if torch.cuda.device_count() > 1:
      print("Using", torch.cuda.device_count(), "GPUs")
      sp_model = torch.nn.DataParallel(sp_model)

    self.model = sp_model.to(self.device)
    self.learningRate = params["learning_rate"]
    self.optimizer = self.createOptimizer(params, self.model)
    self.lr_scheduler = self.createLearningRateScheduler(params, self.optimizer)

  def iterate(self, params, repetition, iteration):
    """
    Called once for each training iteration (== epoch here).
    """
    try:
      print("\nStarting iteration",iteration)
      t1 = time.time()
      ret = {}

      # Update learning rate using learning rate scheduler if configured
      if self.lr_scheduler is not None:
        # ReduceLROnPlateau lr_scheduler step should be called after validation,
        # all other lr_schedulers should be called before training
        if params["lr_scheduler"] != "ReduceLROnPlateau":
          self.lr_scheduler.step()

      self.train(params, epoch=iteration)

      # Run validation test
      if self.validation_loader is not None:
        validation = self.test(params, self.validation_loader)

        # ReduceLROnPlateau step should be called after validation
        if params["lr_scheduler"] == "ReduceLROnPlateau":
          self.lr_scheduler.step(validation["test_loss"])

        ret["validation"] = validation
        print("Validation: Test error=", validation["testerror"],
              "entropy=", validation["entropy"])

      # Run noise test
      if (params["test_noise_every_epoch"] or
          iteration == params["iterations"] - 1):
        ret.update(self.runNoiseTests(params))
        print("Noise test results: totalCorrect=", ret["totalCorrect"],
              "Test error=", ret["testerror"], ", entropy=", ret["entropy"])
        if ret["totalCorrect"] > 100000 and ret["testerror"] > 98.3:
          print("*******")
          print(params)

      ret.update({"elapsedTime": time.time() - self.startTime})
      ret.update({"learningRate": self.learningRate if self.lr_scheduler is None
                                                    else self.lr_scheduler.get_lr()})

      print("Iteration time= {0:.3f} secs, "
            "total elapsed time= {1:.3f} mins".format(
              time.time() - t1,ret["elapsedTime"]/60.0))

    except Exception as e:
      # Tracebacks are not printed if using multiprocessing so we do it here
      tb = sys.exc_info()[2]
      traceback.print_tb(tb)
      raise RuntimeError("Something went wrong in iterate")

    return ret

  def finalize(self, params, rep):
    """
    Called once we are done.
    """
    if params.get("saveNet", True):
      self.pruneWeights(params)
      self.pruneDutyCycles(params)

      # Save the full model once we are done.
      saveDir = os.path.join(params["path"], params["name"], "model.pt")
      torch.save(self.model, saveDir)

  def createLearningRateScheduler(self, params, optimizer):
    """
    Creates the learning rate scheduler and attach the optimizer
    """
    lr_scheduler = params.get("lr_scheduler", None)
    if lr_scheduler is None:
      return None

    if lr_scheduler == "StepLR":
      lr_scheduler_params = "{'step_size': 1, 'gamma':" + str(params["learning_rate_factor"]) + "}"

    else:
      lr_scheduler_params = params.get("lr_scheduler_params", None)
      if lr_scheduler_params is None:
        raise ValueError("Missing 'lr_scheduler_params' for {}".format(lr_scheduler))

    # Get lr_scheduler class by name
    clazz = eval("torch.optim.lr_scheduler.{}".format(lr_scheduler))

    # Parse scheduler parameters from config
    lr_scheduler_params = eval(lr_scheduler_params)

    return clazz(optimizer, **lr_scheduler_params)

  def createOptimizer(self, params, model):
    """
    Create a new instance of the optimizer
    """
    lr = params["learning_rate"]
    print("Creating optimizer with learning rate=", lr)
    if params["optimizer"] == "SGD":
      optimizer = optim.SGD(model.parameters(), lr=lr,
                            momentum=params["momentum"])
    elif params["optimizer"] == "Adam":
      optimizer = optim.Adam(model.parameters(), lr=lr)
    else:
      raise LookupError("Incorrect optimizer value")

    return optimizer

  def train(self, params, epoch):
    """
    Train one epoch of this model by iterating through mini batches. An epoch
    ends after one pass through the training set, or if the number of mini
    batches exceeds the parameter "batches_in_epoch".
    """
    self.model.train()
    for batch_idx, (data, target) in enumerate(self.train_loader):

      data, target = data.to(self.device), target.to(self.device)
      self.optimizer.zero_grad()
      output = self.model(data)
      loss = F.nll_loss(output, target)
      loss.backward()
      self.optimizer.step()

      # Log info every log_interval mini batches
      if batch_idx % params["log_interval"] == 0:
        entropy = self.model.entropy()
        print("logging: ",self.model.getLearningIterations(),
              " learning iterations, elapsedTime", time.time() - self.startTime,
              " entropy:", float(entropy)," / ", self.model.maxEntropy())
        if params["create_plots"]:
          plotDutyCycles(self.model.dutyCycle,
                         self.resultsDir + "/figure_"+str(epoch)+"_"+str(
                           self.model.getLearningIterations()))

      if batch_idx >= params["batches_in_epoch"]:
        break

    self.model.postEpoch()

  def test(self, params, test_loader):
    """
    Test the model using the given loader and return test metrics
    """
    self.model.eval()
    test_loss = 0
    correct = 0

    nonzeros = None
    count_nonzeros = params.get("count_nonzeros", False)
    if count_nonzeros:
      nonzeros = {}
      register_nonzero_counter(self.model, nonzeros)

    with torch.no_grad():
      for data, target in test_loader:
        data, target = data.to(self.device), target.to(self.device)
        output = self.model(data)
        test_loss += F.nll_loss(output, target, reduction='sum').item()
        pred = output.max(1, keepdim=True)[1]
        correct += pred.eq(target.view_as(pred)).sum().item()

        # count nonzeros only once
        if count_nonzeros:
          count_nonzeros = False
          unregister_counter_nonzero(self.model)

    test_loss /= len(test_loader.sampler)
    test_error = 100. * correct / len(test_loader.sampler)

    entropy = self.model.entropy()
    ret = {"num_correct": correct,
           "test_loss": test_loss,
           "testerror": test_error,
           "entropy": float(entropy)}

    if nonzeros is not None:
      ret["nonzeros"] = nonzeros

    return ret

  def runNoiseTests(self, params):
    """
    Test the model with different noise values and return test metrics.
    """
    ret = {}

    # Noise on validation data
    validation = {} if self.validation_sampler is not None else None

    # Test with noise
    total_correct = 0
    validation_total_correct = 0
    for noise in [0.0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5]:
      transform = transforms.Compose([
        transforms.ToTensor(),
        RandomNoise(noise, whiteValue=0.1307 + 2*0.3081),
        transforms.Normalize((0.1307,), (0.3081,))
      ])
      test_loader = torch.utils.data.DataLoader(
        datasets.MNIST(self.dataDir, train=False, transform=transform),
        batch_size=params["test_batch_size"], shuffle=True)

      testResult = self.test(params, test_loader)
      total_correct += testResult["num_correct"]
      ret[noise]= testResult

      if validation is not None:
        validation_loader = torch.utils.data.DataLoader(
          datasets.MNIST(self.dataDir, train=True, transform=transform),
          sampler=self.validation_sampler,
          batch_size=params["test_batch_size"])

        validationResult = self.test(params, validation_loader)
        validation_total_correct += validationResult["num_correct"]
        validation[noise] = validationResult

    ret["totalCorrect"] = total_correct
    ret["testerror"] = ret[0.0]["testerror"]
    ret["entropy"] = ret[0.0]["entropy"]

    if "nonzeros" in ret[0.0]:
      ret["nonzeros"] = ret[0.0]["nonzeros"]

    if validation is not None:
      validation["totalCorrect"] = validation_total_correct
      validation["testerror"] = validation[0.0]["testerror"]
      validation["entropy"] = validation[0.0]["entropy"]
      ret["validation"] = validation

    return ret

  def pruneWeights(self, params):
    """
    Prune the weights whose absolute magnitude is < params["min_weight"]
    """
    self.model.pruneWeights(params.get("min_weight", 0.0))

  def pruneDutyCycles(self, params):
    """
    Prune the DutyCycles whose absolute magnitude is < params["min_dutycycle"]
    """
    self.model.pruneDutyCycles(params.get("min_dutycycle", 0.0))

if __name__ == '__main__':
  suite = MNISTSparseExperiment()
  suite.start()
<commit_msg>Moved prune weight to a different file<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2018, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

from __future__ import print_function
import os
import sys
import traceback
import numpy as np
import time

import torch
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms

from htmresearch.frameworks.pytorch.benchmark_utils import (
  register_nonzero_counter, unregister_counter_nonzero)
from htmresearch.support.expsuite import PyExperimentSuite

from htmresearch.frameworks.pytorch.image_transforms import RandomNoise
from htmresearch.frameworks.pytorch.sparse_net import SparseNet
from htmresearch.frameworks.pytorch.duty_cycle_metrics import plotDutyCycles
from htmresearch.frameworks.pytorch.dataset_utils import createValidationDataSampler

class MNISTSparseExperiment(PyExperimentSuite):
  """
  Allows running multiple sparse MNIST experiments in parallel
  """

  def parse_cfg(self):
    super(MNISTSparseExperiment, self).parse_cfg()
    # Change the current working directory to be relative to 'experiments.cfg'
    projectDir = os.path.dirname(self.options.config)
    projectDir = os.path.abspath(projectDir)
    os.chdir(projectDir)

  def reset(self, params, repetition):
    """
    Called once at the beginning of each experiment.
    """
    self.startTime = time.time()
    print(params)
    torch.manual_seed(params["seed"] + repetition)
    np.random.seed(params["seed"] + repetition)

    # Get our directories correct
    self.dataDir = params["datadir"]
    self.resultsDir = os.path.join(params["path"], params["name"], "plots")

    if not os.path.exists(self.resultsDir):
      os.makedirs(self.resultsDir)

    self.use_cuda = not params["no_cuda"] and torch.cuda.is_available()
    self.device = torch.device("cuda" if self.use_cuda else "cpu")

    transform = transforms.Compose([transforms.ToTensor(),
                                    transforms.Normalize((0.1307,), (0.3081,))])

    train_dataset = datasets.MNIST(self.dataDir, train=True, download=True,
                                   transform=transform)

    # Create training and validation sampler from MNIST dataset by training on
    # random X% of the training set and validating on the remaining (1-X)%,
    # where X can be tuned via the "validation" parameter
    validation = params.get("validation", 50000.0 / 60000.0)
    if validation < 1.0:
      self.train_sampler, self.validation_sampler = createValidationDataSampler(
        train_dataset, validation)

      self.train_loader = torch.utils.data.DataLoader(train_dataset,
                                                      batch_size=params["batch_size"],
                                                      sampler=self.train_sampler)

      self.validation_loader = torch.utils.data.DataLoader(train_dataset,
                                                           batch_size=params["batch_size"],
                                                           sampler=self.validation_sampler)
    else:
      # No validation. Normal training dataset
      self.validation_loader = None
      self.validation_sampler = None
      self.train_sampler = None
      self.train_loader = torch.utils.data.DataLoader(train_dataset,
                                                      batch_size=params["batch_size"],
                                                      shuffle=True)

    self.test_loader = torch.utils.data.DataLoader(
      datasets.MNIST(self.dataDir, train=False, transform=transform),
      batch_size=params["test_batch_size"], shuffle=True)

    # Parse 'n' and 'k' parameters
    n = params["n"]
    k = params["k"]
    if isinstance(n, basestring):
      n = map(int, n.split("_"))
    if isinstance(k, basestring):
      k = map(int, k.split("_"))

    if params["use_cnn"]:
      c1_out_channels = params["c1_out_channels"]
      c1_k = params["c1_k"]
      if isinstance(c1_out_channels, basestring):
        c1_out_channels = map(int, c1_out_channels.split("_"))
      if isinstance(c1_k, basestring):
        c1_k = map(int, c1_k.split("_"))

      sp_model = SparseNet(
        inputSize=(1, 28, 28),
        outChannels=c1_out_channels,
        c_k=c1_k,
        dropout=params["dropout"],
        n=n,
        k=k,
        boostStrength=params["boost_strength"],
        weightSparsity=params["weight_sparsity"],
        boostStrengthFactor=params["boost_strength_factor"],
        kInferenceFactor=params["k_inference_factor"],
        useBatchNorm=params["use_batch_norm"],
      )
      print("c1OutputLength=", sp_model.cnnSdr[0].outputLength)
    else:
      sp_model = SparseNet(
        n=n,
        k=k,
        boostStrength=params["boost_strength"],
        weightSparsity=params["weight_sparsity"],
        boostStrengthFactor=params["boost_strength_factor"],
        kInferenceFactor=params["k_inference_factor"],
        dropout=params["dropout"],
        useBatchNorm=params["use_batch_norm"],
      )
    if torch.cuda.device_count() > 1:
      print("Using", torch.cuda.device_count(), "GPUs")
      sp_model = torch.nn.DataParallel(sp_model)

    self.model = sp_model.to(self.device)
    self.learningRate = params["learning_rate"]
    self.optimizer = self.createOptimizer(params, self.model)
    self.lr_scheduler = self.createLearningRateScheduler(params, self.optimizer)

  def iterate(self, params, repetition, iteration):
    """
    Called once for each training iteration (== epoch here).
    """
    try:
      print("\nStarting iteration",iteration)
      t1 = time.time()
      ret = {}

      # Update learning rate using learning rate scheduler if configured
      if self.lr_scheduler is not None:
        # ReduceLROnPlateau lr_scheduler step should be called after validation,
        # all other lr_schedulers should be called before training
        if params["lr_scheduler"] != "ReduceLROnPlateau":
          self.lr_scheduler.step()

      self.train(params, epoch=iteration)

      # Run validation test
      if self.validation_loader is not None:
        validation = self.test(params, self.validation_loader)

        # ReduceLROnPlateau step should be called after validation
        if params["lr_scheduler"] == "ReduceLROnPlateau":
          self.lr_scheduler.step(validation["test_loss"])

        ret["validation"] = validation
        print("Validation: Test error=", validation["testerror"],
              "entropy=", validation["entropy"])

      # Run noise test
      if (params["test_noise_every_epoch"] or
          iteration == params["iterations"] - 1):
        ret.update(self.runNoiseTests(params))
        print("Noise test results: totalCorrect=", ret["totalCorrect"],
              "Test error=", ret["testerror"], ", entropy=", ret["entropy"])
        if ret["totalCorrect"] > 100000 and ret["testerror"] > 98.3:
          print("*******")
          print(params)

      ret.update({"elapsedTime": time.time() - self.startTime})
      ret.update({"learningRate": self.learningRate if self.lr_scheduler is None
                                                    else self.lr_scheduler.get_lr()})

      print("Iteration time= {0:.3f} secs, "
            "total elapsed time= {1:.3f} mins".format(
              time.time() - t1,ret["elapsedTime"]/60.0))

    except Exception as e:
      # Tracebacks are not printed if using multiprocessing so we do it here
      tb = sys.exc_info()[2]
      traceback.print_tb(tb)
      raise RuntimeError("Something went wrong in iterate")

    return ret

  def finalize(self, params, rep):
    """
    Called once we are done.
    """
    if params.get("saveNet", True):
      # Save the full model once we are done.
      saveDir = os.path.join(params["path"], params["name"], "model.pt")
      torch.save(self.model, saveDir)

  def createLearningRateScheduler(self, params, optimizer):
    """
    Creates the learning rate scheduler and attach the optimizer
    """
    lr_scheduler = params.get("lr_scheduler", None)
    if lr_scheduler is None:
      return None

    if lr_scheduler == "StepLR":
      lr_scheduler_params = "{'step_size': 1, 'gamma':" + str(params["learning_rate_factor"]) + "}"

    else:
      lr_scheduler_params = params.get("lr_scheduler_params", None)
      if lr_scheduler_params is None:
        raise ValueError("Missing 'lr_scheduler_params' for {}".format(lr_scheduler))

    # Get lr_scheduler class by name
    clazz = eval("torch.optim.lr_scheduler.{}".format(lr_scheduler))

    # Parse scheduler parameters from config
    lr_scheduler_params = eval(lr_scheduler_params)

    return clazz(optimizer, **lr_scheduler_params)

  def createOptimizer(self, params, model):
    """
    Create a new instance of the optimizer
    """
    lr = params["learning_rate"]
    print("Creating optimizer with learning rate=", lr)
    if params["optimizer"] == "SGD":
      optimizer = optim.SGD(model.parameters(), lr=lr,
                            momentum=params["momentum"])
    elif params["optimizer"] == "Adam":
      optimizer = optim.Adam(model.parameters(), lr=lr)
    else:
      raise LookupError("Incorrect optimizer value")

    return optimizer

  def train(self, params, epoch):
    """
    Train one epoch of this model by iterating through mini batches. An epoch
    ends after one pass through the training set, or if the number of mini
    batches exceeds the parameter "batches_in_epoch".
    """
    self.model.train()
    for batch_idx, (data, target) in enumerate(self.train_loader):

      data, target = data.to(self.device), target.to(self.device)
      self.optimizer.zero_grad()
      output = self.model(data)
      loss = F.nll_loss(output, target)
      loss.backward()
      self.optimizer.step()

      # Log info every log_interval mini batches
      if batch_idx % params["log_interval"] == 0:
        entropy = self.model.entropy()
        print("logging: ",self.model.getLearningIterations(),
              " learning iterations, elapsedTime", time.time() - self.startTime,
              " entropy:", float(entropy)," / ", self.model.maxEntropy())
        if params["create_plots"]:
          plotDutyCycles(self.model.dutyCycle,
                         self.resultsDir + "/figure_"+str(epoch)+"_"+str(
                           self.model.getLearningIterations()))

      if batch_idx >= params["batches_in_epoch"]:
        break

    self.model.postEpoch()

  def test(self, params, test_loader):
    """
    Test the model using the given loader and return test metrics
    """
    self.model.eval()
    test_loss = 0
    correct = 0

    nonzeros = None
    count_nonzeros = params.get("count_nonzeros", False)
    if count_nonzeros:
      nonzeros = {}
      register_nonzero_counter(self.model, nonzeros)

    with torch.no_grad():
      for data, target in test_loader:
        data, target = data.to(self.device), target.to(self.device)
        output = self.model(data)
        test_loss += F.nll_loss(output, target, reduction='sum').item()
        pred = output.max(1, keepdim=True)[1]
        correct += pred.eq(target.view_as(pred)).sum().item()

        # count nonzeros only once
        if count_nonzeros:
          count_nonzeros = False
          unregister_counter_nonzero(self.model)

    test_loss /= len(test_loader.sampler)
    test_error = 100. * correct / len(test_loader.sampler)

    entropy = self.model.entropy()
    ret = {"num_correct": correct,
           "test_loss": test_loss,
           "testerror": test_error,
           "entropy": float(entropy)}

    if nonzeros is not None:
      ret["nonzeros"] = nonzeros

    return ret

  def runNoiseTests(self, params):
    """
    Test the model with different noise values and return test metrics.
    """
    ret = {}

    # Noise on validation data
    validation = {} if self.validation_sampler is not None else None

    # Test with noise
    total_correct = 0
    validation_total_correct = 0
    for noise in [0.0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5]:
      transform = transforms.Compose([
        transforms.ToTensor(),
        RandomNoise(noise, whiteValue=0.1307 + 2*0.3081),
        transforms.Normalize((0.1307,), (0.3081,))
      ])
      test_loader = torch.utils.data.DataLoader(
        datasets.MNIST(self.dataDir, train=False, transform=transform),
        batch_size=params["test_batch_size"], shuffle=True)

      testResult = self.test(params, test_loader)
      total_correct += testResult["num_correct"]
      ret[noise]= testResult

      if validation is not None:
        validation_loader = torch.utils.data.DataLoader(
          datasets.MNIST(self.dataDir, train=True, transform=transform),
          sampler=self.validation_sampler,
          batch_size=params["test_batch_size"])

        validationResult = self.test(params, validation_loader)
        validation_total_correct += validationResult["num_correct"]
        validation[noise] = validationResult

    ret["totalCorrect"] = total_correct
    ret["testerror"] = ret[0.0]["testerror"]
    ret["entropy"] = ret[0.0]["entropy"]

    if "nonzeros" in ret[0.0]:
      ret["nonzeros"] = ret[0.0]["nonzeros"]

    if validation is not None:
      validation["totalCorrect"] = validation_total_correct
      validation["testerror"] = validation[0.0]["testerror"]
      validation["entropy"] = validation[0.0]["entropy"]
      ret["validation"] = validation

    return ret

if __name__ == '__main__':
  suite = MNISTSparseExperiment()
  suite.start()
<|endoftext|>


<commit_before>from django.conf.urls import include, url
from django.contrib import admin
from django.core.urlresolvers import reverse_lazy
from django.views.generic.base import RedirectView
from django.conf import settings
from django.conf.urls.static import static

from pos.views.shift import ShiftViewSet
from pos.views.stock import CategoryViewSet, IngredientViewSet, ItemViewSet, OrderLineViewSet, OrderViewSet, PurchaseViewSet
from pos.views.user import UserViewSet

from rest_framework import routers

# Routers provide an easy way of automatically determining the URL conf.
router = routers.SimpleRouter()
router.register(r'users', UserViewSet)
router.register(r'categories', CategoryViewSet)
router.register(r'orderlines', OrderLineViewSet)
router.register(r'ingredients', IngredientViewSet)
router.register(r'items', ItemViewSet)
router.register(r'orders', OrderViewSet)
router.register(r'shifts', ShiftViewSet)
router.register(r'purchases', PurchaseViewSet, 'purchase')
urlpatterns = [
    url(r'^$', RedirectView.as_view(url=reverse_lazy('admin:index'))),
    url(r'^admin/', include(admin.site.urls)),
    url(r'^', include(router.urls)),
] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
<commit_msg>Code cleanup<commit_after>from django.conf import settings
from django.conf.urls import include, url
from django.conf.urls.static import static
from django.contrib import admin
from django.core.urlresolvers import reverse_lazy
from django.views.generic.base import RedirectView

from pos.views.shift import ShiftViewSet
from pos.views.stock import CategoryViewSet, IngredientViewSet, ItemViewSet, OrderLineViewSet, OrderViewSet, PurchaseViewSet
from pos.views.user import UserViewSet

from rest_framework import routers

# Routers provide an easy way of automatically determining the URL conf.
router = routers.SimpleRouter()
router.register(r'users', UserViewSet)
router.register(r'categories', CategoryViewSet)
router.register(r'orderlines', OrderLineViewSet)
router.register(r'ingredients', IngredientViewSet)
router.register(r'items', ItemViewSet)
router.register(r'orders', OrderViewSet)
router.register(r'shifts', ShiftViewSet)
router.register(r'purchases', PurchaseViewSet, 'purchase')
urlpatterns = [
    url(r'^$', RedirectView.as_view(url=reverse_lazy('admin:index'))),
    url(r'^admin/', include(admin.site.urls)),
    url(r'^', include(router.urls)),
] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
<|endoftext|>


<commit_before># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Runs plain netperf in a few modes.

docs:
http://www.netperf.org/svn/netperf2/tags/netperf-2.4.5/doc/netperf.html#TCP_005fRR
manpage: http://manpages.ubuntu.com/manpages/maverick/man1/netperf.1.html

Runs TCP_RR, TCP_CRR, and TCP_STREAM benchmarks from netperf across two
machines.
"""

import csv
import io
import logging

from perfkitbenchmarker import flags
from perfkitbenchmarker import sample
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.packages import netperf

flags.DEFINE_integer('netperf_max_iter', None,
                     'Maximum number of iterations to run during '
                     'confidence interval estimation. If unset, '
                     'a single iteration will be run.',
                     lower_bound=3, upper_bound=30)

flags.DEFINE_integer('netperf_test_length', 60,
                     'netperf test length, in seconds',
                     lower_bound=1)

FLAGS = flags.FLAGS

BENCHMARK_INFO = {'name': 'netperf',
                  'description': 'Run TCP_RR, TCP_CRR, UDP_RR and TCP_STREAM '
                  'netperf benchmarks',
                  'scratch_disk': False,
                  'num_machines': 2}

MBPS = 'Mbits/sec'
TRANSACTIONS_PER_SECOND = 'transactions_per_second'

NETPERF_BENCHMARKS = ['TCP_RR', 'TCP_CRR', 'TCP_STREAM', 'UDP_RR']
COMMAND_PORT = 20000
DATA_PORT = 20001

def GetInfo():
  return BENCHMARK_INFO

def PrepareNetperf(vm):
  """Installs netperf on a single vm."""
  vm.Install('netperf')

def Prepare(benchmark_spec):
  """Install netperf on the target vm.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.
  """
  vms = benchmark_spec.vms
  vms = vms[:2]
  vm_util.RunThreaded(PrepareNetperf, vms)

  if vm_util.ShouldRunOnExternalIpAddress():
    fw = benchmark_spec.firewall
    fw.AllowPort(vms[1], COMMAND_PORT)
    fw.AllowPort(vms[1], DATA_PORT)

  vms[1].RemoteCommand('%s -p %s' %
                       (netperf.NETSERVER_PATH, COMMAND_PORT))

def RunNetperf(vm, benchmark_name, server_ip):
  """Spawns netperf on a remove VM, parses results.

  Args:
    vm: The VM that the netperf TCP_RR benchmark will be run upon.
    benchmark_name: The netperf benchmark to run, see the documentation.
    server_ip: A machine that is running netserver.

  Returns:
    A sample.Sample object with the result.
  """
  # Flags:
  # -o specifies keys to include in CSV output.
  # -j keeps additional latency numbers
  # -I specifies the confidence % and width - here 99% confidence that the true
  #    value is within +/- 2.5% of the reported value
  # -i specifies the maximum and minimum number of iterations.
  confidence = ('-I 99,5 -i {0},3'.format(FLAGS.netperf_max_iter)
                if FLAGS.netperf_max_iter else '')
  netperf_cmd = ('{netperf_path} -p {command_port} -j '
                 '-t {benchmark_name} -H {server_ip} -l {length} {confidence} '
                 ' -- '
                 '-P {data_port} '
                 '-o THROUGHPUT,THROUGHPUT_UNITS,P50_LATENCY,P90_LATENCY,'
                 'P99_LATENCY,STDDEV_LATENCY,'
                 'CONFIDENCE_ITERATION,THROUGHPUT_CONFID').format(
                     netperf_path=netperf.NETPERF_PATH,
                     benchmark_name=benchmark_name,
                     server_ip=server_ip, command_port=COMMAND_PORT,
                     data_port=DATA_PORT,
                     length=FLAGS.netperf_test_length,
                     confidence=confidence)
  stdout, _ = vm.RemoteCommand(netperf_cmd, should_log=True)

  fp = io.StringIO(stdout)
  # "-o" flag above specifies CSV output, but there is one extra header line:
  banner = next(fp)
  assert banner.startswith('MIGRATED'), stdout
  r = csv.DictReader(fp)
  row = next(r)
  logging.info('Netperf Results: %s', row)
  assert 'Throughput' in row, row

  value = float(row['Throughput'])
  unit = {'Trans/s': TRANSACTIONS_PER_SECOND,
          '10^6bits/s': MBPS}[row['Throughput Units']]
  if unit == MBPS:
    metric = '%s_Throughput' % benchmark_name
  else:
    metric = '%s_Transaction_Rate' % benchmark_name

  meta_keys = [('Confidence Iterations Run', 'confidence_iter'),
               ('Throughput Confidence Width (%)', 'confidence_width_percent')]
  metadata = {meta_key: row[np_key] for np_key, meta_key in meta_keys}
  metadata.update(netperf_test_length=FLAGS.netperf_test_length,
                  max_iter=FLAGS.netperf_max_iter or 1)

  samples = [sample.Sample(metric, value, unit, metadata)]

  # No tail latency for throughput.
  if unit == MBPS:
    return samples

  for metric_key, metric_name in [
      ('50th Percentile Latency Microseconds', 'p50'),
      ('90th Percentile Latency Microseconds', 'p90'),
      ('99th Percentile Latency Microseconds', 'p99'),
      ('Stddev Latency Microseconds', 'stddev')]:
    samples.append(
        sample.Sample('%s_Latency_%s' % (benchmark_name, metric_name),
                      float(row[metric_key]), 'us', metadata))
  return samples

def Run(benchmark_spec):
  """Run netperf TCP_RR on the target vm.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.

  Returns:
    A list of sample.Sample objects.
  """
  vms = benchmark_spec.vms
  vm = vms[0]
  server_vm = vms[1]
  logging.info('netperf running on %s', vm)
  results = []
  metadata = {
      'ip_type': 'external',
      'server_machine_type': server_vm.machine_type,
      'server_zone': server_vm.zone,
      'receiving_zone': server_vm.zone,
      'client_machine_type': vm.machine_type,
      'client_zone': vm.zone,
      'sending_zone': vm.zone
  }
  for netperf_benchmark in NETPERF_BENCHMARKS:

    if vm_util.ShouldRunOnExternalIpAddress():
      external_ip_results = RunNetperf(vm, netperf_benchmark,
                                       server_vm.ip_address)
      for external_ip_result in external_ip_results:
        external_ip_result.metadata.update(metadata)
      results.extend(external_ip_results)

    if vm_util.ShouldRunOnInternalIpAddress(vm, server_vm):
      internal_ip_results = RunNetperf(vm, netperf_benchmark,
                                       server_vm.internal_ip)
      for internal_ip_result in internal_ip_results:
        internal_ip_result.metadata.update(metadata)
        internal_ip_result.metadata['ip_type'] = 'internal'
      results.extend(internal_ip_results)

  return results

def Cleanup(benchmark_spec):
  """Cleanup netperf on the target vm (by uninstalling).

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.
  """
  vms = benchmark_spec.vms
  vms[1].RemoteCommand('sudo pkill netserver')
<commit_msg>Add a timeout to netperf tests<commit_after># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Runs plain netperf in a few modes.

docs:
http://www.netperf.org/svn/netperf2/tags/netperf-2.4.5/doc/netperf.html#TCP_005fRR
manpage: http://manpages.ubuntu.com/manpages/maverick/man1/netperf.1.html

Runs TCP_RR, TCP_CRR, and TCP_STREAM benchmarks from netperf across two
machines.
"""

import csv
import io
import logging

from perfkitbenchmarker import flags
from perfkitbenchmarker import sample
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.packages import netperf

flags.DEFINE_integer('netperf_max_iter', None,
                     'Maximum number of iterations to run during '
                     'confidence interval estimation. If unset, '
                     'a single iteration will be run.',
                     lower_bound=3, upper_bound=30)

flags.DEFINE_integer('netperf_test_length', 60,
                     'netperf test length, in seconds',
                     lower_bound=1)

FLAGS = flags.FLAGS

BENCHMARK_INFO = {'name': 'netperf',
                  'description': 'Run TCP_RR, TCP_CRR, UDP_RR and TCP_STREAM '
                  'netperf benchmarks',
                  'scratch_disk': False,
                  'num_machines': 2}

MBPS = 'Mbits/sec'
TRANSACTIONS_PER_SECOND = 'transactions_per_second'

NETPERF_BENCHMARKS = ['TCP_RR', 'TCP_CRR', 'TCP_STREAM', 'UDP_RR']
COMMAND_PORT = 20000
DATA_PORT = 20001

def GetInfo():
  return BENCHMARK_INFO

def PrepareNetperf(vm):
  """Installs netperf on a single vm."""
  vm.Install('netperf')

def Prepare(benchmark_spec):
  """Install netperf on the target vm.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.
  """
  vms = benchmark_spec.vms
  vms = vms[:2]
  vm_util.RunThreaded(PrepareNetperf, vms)

  if vm_util.ShouldRunOnExternalIpAddress():
    fw = benchmark_spec.firewall
    fw.AllowPort(vms[1], COMMAND_PORT)
    fw.AllowPort(vms[1], DATA_PORT)

  vms[1].RemoteCommand('%s -p %s' %
                       (netperf.NETSERVER_PATH, COMMAND_PORT))

def RunNetperf(vm, benchmark_name, server_ip):
  """Spawns netperf on a remove VM, parses results.

  Args:
    vm: The VM that the netperf TCP_RR benchmark will be run upon.
    benchmark_name: The netperf benchmark to run, see the documentation.
    server_ip: A machine that is running netserver.

  Returns:
    A sample.Sample object with the result.
  """
  # Flags:
  # -o specifies keys to include in CSV output.
  # -j keeps additional latency numbers
  # -I specifies the confidence % and width - here 99% confidence that the true
  #    value is within +/- 2.5% of the reported value
  # -i specifies the maximum and minimum number of iterations.
  confidence = ('-I 99,5 -i {0},3'.format(FLAGS.netperf_max_iter)
                if FLAGS.netperf_max_iter else '')
  netperf_cmd = ('{netperf_path} -p {command_port} -j '
                 '-t {benchmark_name} -H {server_ip} -l {length} {confidence} '
                 ' -- '
                 '-P {data_port} '
                 '-o THROUGHPUT,THROUGHPUT_UNITS,P50_LATENCY,P90_LATENCY,'
                 'P99_LATENCY,STDDEV_LATENCY,'
                 'CONFIDENCE_ITERATION,THROUGHPUT_CONFID').format(
                     netperf_path=netperf.NETPERF_PATH,
                     benchmark_name=benchmark_name,
                     server_ip=server_ip, command_port=COMMAND_PORT,
                     data_port=DATA_PORT,
                     length=FLAGS.netperf_test_length,
                     confidence=confidence)
  stdout, _ = vm.RemoteCommand(netperf_cmd, should_log=True,
                               timeout=2 * FLAGS.netperf_test_length)

  fp = io.StringIO(stdout)
  # "-o" flag above specifies CSV output, but there is one extra header line:
  banner = next(fp)
  assert banner.startswith('MIGRATED'), stdout
  r = csv.DictReader(fp)
  row = next(r)
  logging.info('Netperf Results: %s', row)
  assert 'Throughput' in row, row

  value = float(row['Throughput'])
  unit = {'Trans/s': TRANSACTIONS_PER_SECOND,
          '10^6bits/s': MBPS}[row['Throughput Units']]
  if unit == MBPS:
    metric = '%s_Throughput' % benchmark_name
  else:
    metric = '%s_Transaction_Rate' % benchmark_name

  meta_keys = [('Confidence Iterations Run', 'confidence_iter'),
               ('Throughput Confidence Width (%)', 'confidence_width_percent')]
  metadata = {meta_key: row[np_key] for np_key, meta_key in meta_keys}
  metadata.update(netperf_test_length=FLAGS.netperf_test_length,
                  max_iter=FLAGS.netperf_max_iter or 1)

  samples = [sample.Sample(metric, value, unit, metadata)]

  # No tail latency for throughput.
  if unit == MBPS:
    return samples

  for metric_key, metric_name in [
      ('50th Percentile Latency Microseconds', 'p50'),
      ('90th Percentile Latency Microseconds', 'p90'),
      ('99th Percentile Latency Microseconds', 'p99'),
      ('Stddev Latency Microseconds', 'stddev')]:
    samples.append(
        sample.Sample('%s_Latency_%s' % (benchmark_name, metric_name),
                      float(row[metric_key]), 'us', metadata))
  return samples

def Run(benchmark_spec):
  """Run netperf TCP_RR on the target vm.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.

  Returns:
    A list of sample.Sample objects.
  """
  vms = benchmark_spec.vms
  vm = vms[0]
  server_vm = vms[1]
  logging.info('netperf running on %s', vm)
  results = []
  metadata = {
      'ip_type': 'external',
      'server_machine_type': server_vm.machine_type,
      'server_zone': server_vm.zone,
      'receiving_zone': server_vm.zone,
      'client_machine_type': vm.machine_type,
      'client_zone': vm.zone,
      'sending_zone': vm.zone
  }
  for netperf_benchmark in NETPERF_BENCHMARKS:

    if vm_util.ShouldRunOnExternalIpAddress():
      external_ip_results = RunNetperf(vm, netperf_benchmark,
                                       server_vm.ip_address)
      for external_ip_result in external_ip_results:
        external_ip_result.metadata.update(metadata)
      results.extend(external_ip_results)

    if vm_util.ShouldRunOnInternalIpAddress(vm, server_vm):
      internal_ip_results = RunNetperf(vm, netperf_benchmark,
                                       server_vm.internal_ip)
      for internal_ip_result in internal_ip_results:
        internal_ip_result.metadata.update(metadata)
        internal_ip_result.metadata['ip_type'] = 'internal'
      results.extend(internal_ip_results)

  return results

def Cleanup(benchmark_spec):
  """Cleanup netperf on the target vm (by uninstalling).

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.
  """
  vms = benchmark_spec.vms
  vms[1].RemoteCommand('sudo pkill netserver')
<|endoftext|>


<commit_before>from django import template

from waitinglist.forms import WaitingListEntryForm

register = template.Library()

@register.assignment_tag
def waitinglist_entry_form():
    """
    Get a (new) form object to post a new comment.

    Syntax::

        {% waitinglist_entry_form as [varname] %}

    """
    return WaitingListEntryForm()
<commit_msg>corrected import path in templatetags<commit_after>from django import template

from ..forms import WaitingListEntryForm

register = template.Library()

@register.assignment_tag
def waitinglist_entry_form():
    """
    Get a (new) form object to post a new comment.

    Syntax::

        {% waitinglist_entry_form as [varname] %}

    """
    return WaitingListEntryForm()
<|endoftext|>


<commit_before># Code to handle displaying and logging of results.
# Anything in binwalk that prints results to screen should use this class.

import sys
import csv as pycsv
import datetime
import binwalk.core.common
from binwalk.core.compat import *

CHK_MD5 = False

class Display(object):
    '''
    Class to handle display of output and writing to log files.
    This class is instantiated for all modules implicitly and should not need to be invoked directly by most modules.
    '''
    SCREEN_WIDTH = 0
    HEADER_WIDTH = 80
    DEFAULT_FORMAT = "%s\n"

    def __init__(self, quiet=False, verbose=False, log=None, csv=False, fit_to_screen=False):
        self.quiet = quiet
        self.verbose = verbose
        self.fit_to_screen = fit_to_screen
        self.fp = None
        self.csv = None
        self.num_columns = 0
        self.custom_verbose_format = ""
        self.custom_verbose_args = []

        self._configure_formatting()

        if log:
            self.fp = open(log, "a")
            if csv:
                self.csv = pycsv.writer(self.fp)

    def format_strings(self, header, result):
        self.result_format = result
        self.header_format = header

        if self.num_columns == 0:
            self.num_columns = len(header.split())

    def log(self, fmt, columns):
        if self.fp:
            if self.csv:
                self.csv.writerow(columns)
            else:
                self.fp.write(fmt % tuple(columns))

            self.fp.flush()

    def add_custom_header(self, fmt, args):
        self.custom_verbose_format = fmt
        self.custom_verbose_args = args

    def header(self, *args, **kwargs):
        global CHK_MD5
        file_name = None
        self.num_columns = len(args)

        if has_key(kwargs, 'file_name'):
            file_name = kwargs['file_name']

        if self.verbose and file_name:
            md5sum = binwalk.core.common.file_md5(file_name)
            timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")

            if self.csv:
                self.log("", ["FILE", "MD5SUM", "TIMESTAMP"])
                self.log("", [file_name, md5sum, timestamp])

            self._fprint("%s", "\n", csv=False)
            #self._fprint("Scan Time:     %s\n", [timestamp], csv=False, filter=False)
            #self._fprint("Target File:   %s\n", [file_name], csv=False, filter=False)
            if CHK_MD5:
                self._fprint("MD5 Checksum:  %s\n", [md5sum], csv=False, filter=False)
            else:
                CHK_MD5 = True
            if self.custom_verbose_format and self.custom_verbose_args:
                self._fprint(self.custom_verbose_format, self.custom_verbose_args, csv=False, filter=False)

        self._fprint("%s", "\n", csv=False, filter=False)
        self._fprint(self.header_format, args, filter=False)
        self._fprint("%s", ["-" * self.HEADER_WIDTH + "\n"], csv=False, filter=False)

    def result(self, *args):
        # Convert to list for item assignment
        args = list(args)

        # Replace multiple spaces with single spaces. This is to prevent accidentally putting
        # four spaces in the description string, which would break auto-formatting.
        for i in range(len(args)):
            if isinstance(args[i], str):
                while "    " in args[i]:
                    args[i] = args[i].replace("  " , " ")

        self._fprint(self.result_format, tuple(args))

    def footer(self):
        self._fprint("%s", "\n", csv=False, filter=False)

    def _fprint(self, fmt, columns, csv=True, stdout=True, filter=True):
        line = fmt % tuple(columns)

        if not self.quiet and stdout:
            try:
                w_line = self._format_line(line.strip())
                if w_line != '':
                    sys.stdout.write(w_line + "\n")
                sys.stdout.flush()
            except IOError as e:
                pass

        if self.fp and not (self.csv and not csv):
            self.log(fmt, columns)

    def _append_to_data_parts(self, data, start, end):
        '''
        Intelligently appends data to self.string_parts.
        For use by self._format.
        '''
        try:
            while data[start] == ' ':
                start += 1

            if start == end:
                end = len(data[start:])

            self.string_parts.append(data[start:end])
        except KeyboardInterrupt as e:
            raise e
        except Exception:
            try:
                self.string_parts.append(data[start:])
            except KeyboardInterrupt as e:
                raise e
            except Exception:
                pass

        return start

    def _format_line(self, line):
        '''
        Formats a line of text to fit in the terminal window.
        For Tim.
        '''
        delim = '\n'
        offset = 0
        self.string_parts = []

        # Split the line into an array of columns, e.g., ['0', '0x00000000', 'Some description here']
        line_columns = line.split(None, self.num_columns-1)
        if line_columns:
            # Find where the start of the last column (description) starts in the line of text.
            # All line wraps need to be aligned to this offset.
            offset = line.rfind(line_columns[-1])
            # The delimiter will be a newline followed by spaces padding out the line wrap to the alignment offset.
            delim += ' ' * offset

        if line_columns and self.fit_to_screen and len(line) > self.SCREEN_WIDTH:
            # Calculate the maximum length that each wrapped line can be
            max_line_wrap_length = self.SCREEN_WIDTH - offset
            # Append all but the last column to formatted_line
            formatted_line = line[:offset]

            # Loop to split up line into multiple max_line_wrap_length pieces
            while len(line[offset:]) > max_line_wrap_length:
                # Find the nearest space to wrap the line at (so we don't split a word across two lines)
                split_offset = line[offset:offset+max_line_wrap_length].rfind(' ')
                # If there were no good places to split the line, just truncate it at max_line_wrap_length
                if split_offset < 1:
                    split_offset = max_line_wrap_length

                self._append_to_data_parts(line, offset, offset+split_offset)
                offset += split_offset

            # Add any remaining data (guarunteed to be max_line_wrap_length long or shorter) to self.string_parts
            self._append_to_data_parts(line, offset, offset+len(line[offset:]))

            # Append self.string_parts to formatted_line; each part seperated by delim
            formatted_line += delim.join(self.string_parts)
        else:
            formatted_line = line
        if 'JFFS' in formatted_line:
            return ''
        return formatted_line

    def _configure_formatting(self):
        '''
        Configures output formatting, and fitting output to the current terminal width.

        Returns None.
        '''
        self.format_strings(self.DEFAULT_FORMAT, self.DEFAULT_FORMAT)

        if self.fit_to_screen:
            try:
                import fcntl
                import struct
                import termios

                # Get the terminal window width
                hw = struct.unpack('hh', fcntl.ioctl(1, termios.TIOCGWINSZ, '1234'))
                self.SCREEN_WIDTH = self.HEADER_WIDTH = hw[1]
            except KeyboardInterrupt as e:
                raise e
            except Exception:
                pass

<commit_msg>Updated to remove jffs fs from log entries<commit_after># Code to handle displaying and logging of results.
# Anything in binwalk that prints results to screen should use this class.

import sys
import csv as pycsv
import datetime
import binwalk.core.common
from binwalk.core.compat import *

CHK_MD5 = False

class Display(object):
    '''
    Class to handle display of output and writing to log files.
    This class is instantiated for all modules implicitly and should not need to be invoked directly by most modules.
    '''
    SCREEN_WIDTH = 0
    HEADER_WIDTH = 80
    DEFAULT_FORMAT = "%s\n"

    def __init__(self, quiet=False, verbose=False, log=None, csv=False, fit_to_screen=False):
        self.quiet = quiet
        self.verbose = verbose
        self.fit_to_screen = fit_to_screen
        self.fp = None
        self.csv = None
        self.num_columns = 0
        self.custom_verbose_format = ""
        self.custom_verbose_args = []

        self._configure_formatting()

        if log:
            self.fp = open(log, "a")
            if csv:
                self.csv = pycsv.writer(self.fp)

    def format_strings(self, header, result):
        self.result_format = result
        self.header_format = header

        if self.num_columns == 0:
            self.num_columns = len(header.split())

    def log(self, fmt, columns):
        if self.fp:
            if self.csv:
                self.csv.writerow(columns)
            else:
                self.fp.write(fmt % tuple(columns))

            self.fp.flush()

    def add_custom_header(self, fmt, args):
        self.custom_verbose_format = fmt
        self.custom_verbose_args = args

    def header(self, *args, **kwargs):
        global CHK_MD5
        file_name = None
        self.num_columns = len(args)

        if has_key(kwargs, 'file_name'):
            file_name = kwargs['file_name']

        if self.verbose and file_name:
            md5sum = binwalk.core.common.file_md5(file_name)
            timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")

            if self.csv:
                self.log("", ["FILE", "MD5SUM", "TIMESTAMP"])
                self.log("", [file_name, md5sum, timestamp])

            self._fprint("%s", "\n", csv=False)
            #self._fprint("Scan Time:     %s\n", [timestamp], csv=False, filter=False)
            #self._fprint("Target File:   %s\n", [file_name], csv=False, filter=False)
            if CHK_MD5:
                self._fprint("MD5 Checksum:  %s\n", [md5sum], csv=False, filter=False)
            else:
                CHK_MD5 = True
            if self.custom_verbose_format and self.custom_verbose_args:
                self._fprint(self.custom_verbose_format, self.custom_verbose_args, csv=False, filter=False)

        self._fprint("%s", "\n", csv=False, filter=False)
        self._fprint(self.header_format, args, filter=False)
        self._fprint("%s", ["-" * self.HEADER_WIDTH + "\n"], csv=False, filter=False)

    def result(self, *args):
        # Convert to list for item assignment
        args = list(args)

        # Replace multiple spaces with single spaces. This is to prevent accidentally putting
        # four spaces in the description string, which would break auto-formatting.
        for i in range(len(args)):
            if isinstance(args[i], str):
                while "    " in args[i]:
                    args[i] = args[i].replace("  " , " ")

        self._fprint(self.result_format, tuple(args))

    def footer(self):
        self._fprint("%s", "\n", csv=False, filter=False)

    def _fprint(self, fmt, columns, csv=True, stdout=True, filter=True):
        line = fmt % tuple(columns)
        w_line = self._format_line(line.strip())

        if not self.quiet and stdout:
            try:
                if w_line != '':
                    sys.stdout.write(w_line + "\n")
                sys.stdout.flush()
            except IOError as e:
                pass

        if self.fp and not (self.csv and not csv):
            if w_line != '':
                self.log(fmt, columns)

    def _append_to_data_parts(self, data, start, end):
        '''
        Intelligently appends data to self.string_parts.
        For use by self._format.
        '''
        try:
            while data[start] == ' ':
                start += 1

            if start == end:
                end = len(data[start:])

            self.string_parts.append(data[start:end])
        except KeyboardInterrupt as e:
            raise e
        except Exception:
            try:
                self.string_parts.append(data[start:])
            except KeyboardInterrupt as e:
                raise e
            except Exception:
                pass

        return start

    def _format_line(self, line):
        '''
        Formats a line of text to fit in the terminal window.
        For Tim.
        '''
        delim = '\n'
        offset = 0
        self.string_parts = []

        # Split the line into an array of columns, e.g., ['0', '0x00000000', 'Some description here']
        line_columns = line.split(None, self.num_columns-1)
        if line_columns:
            # Find where the start of the last column (description) starts in the line of text.
            # All line wraps need to be aligned to this offset.
            offset = line.rfind(line_columns[-1])
            # The delimiter will be a newline followed by spaces padding out the line wrap to the alignment offset.
            delim += ' ' * offset

        if line_columns and self.fit_to_screen and len(line) > self.SCREEN_WIDTH:
            # Calculate the maximum length that each wrapped line can be
            max_line_wrap_length = self.SCREEN_WIDTH - offset
            # Append all but the last column to formatted_line
            formatted_line = line[:offset]

            # Loop to split up line into multiple max_line_wrap_length pieces
            while len(line[offset:]) > max_line_wrap_length:
                # Find the nearest space to wrap the line at (so we don't split a word across two lines)
                split_offset = line[offset:offset+max_line_wrap_length].rfind(' ')
                # If there were no good places to split the line, just truncate it at max_line_wrap_length
                if split_offset < 1:
                    split_offset = max_line_wrap_length

                self._append_to_data_parts(line, offset, offset+split_offset)
                offset += split_offset

            # Add any remaining data (guarunteed to be max_line_wrap_length long or shorter) to self.string_parts
            self._append_to_data_parts(line, offset, offset+len(line[offset:]))

            # Append self.string_parts to formatted_line; each part seperated by delim
            formatted_line += delim.join(self.string_parts)
        else:
            formatted_line = line
        if 'JFFS' in formatted_line:
            return ''
        return formatted_line

    def _configure_formatting(self):
        '''
        Configures output formatting, and fitting output to the current terminal width.

        Returns None.
        '''
        self.format_strings(self.DEFAULT_FORMAT, self.DEFAULT_FORMAT)

        if self.fit_to_screen:
            try:
                import fcntl
                import struct
                import termios

                # Get the terminal window width
                hw = struct.unpack('hh', fcntl.ioctl(1, termios.TIOCGWINSZ, '1234'))
                self.SCREEN_WIDTH = self.HEADER_WIDTH = hw[1]
            except KeyboardInterrupt as e:
                raise e
            except Exception:
                pass

<|endoftext|>


<commit_before>import re

class Translator(object):
    """
    Simple regular expression to convert Trac wiki to Github markdown.
    """
    def __init__(self, repo, ticketsToIssuesMap, trac_url=None, attachmentsPrefix=None):
        self.repo_url = r'https://github.com/{login}/{name}'.format(login=repo.owner.login, name=repo.name)
        self.trac_url = trac_url
        self.ticketsToIssuesMap = ticketsToIssuesMap
        self.subs = self.compile_subs()
        self.attachmentsPrefix = attachmentsPrefix

    def compile_subs(self):
        subs = [
            # This first handles things like #!xml
            [r"\{\{\{\s*?[\n\r]{1,2}#!([a-z\/]+)(.*?)\}\}\}", r"```\1\2```"],
            # Single line block quotes
            [r"\{\{\{([^\n]*?)\}\}\}",  r"`\1`"],
            # Multi line block quotes
            [r"\{\{\{(.*?)\}\}\}",  r"```\1```"],
            # These next are for headings
            [r"====\s([^\n]+?)\s====(\s*[\n\r]+)", r'#### \1\2'],
            [r"===\s([^\n]+?)\s===(\s*[\n\r]+)", r'### \1\2'],
            [r"==\s([^\n]+?)\s==(\s*[\n\r]+)", r'## \1\2'],
            [r"=\s([^\n]+?)\s=(\s*[\n\r]+)", r'# \1\2'],
            [r"\!(([A-Z][a-z0-9]+){2,})", r'\1'],
            # These next are italics, bold
            [r"'''(.+)'''", r'*\1*'],
            [r"''(.+)''", r'_\1_'],
            # These next 2 are various bulleted or numbered lists
            # This next should probably be beginning-of-line, not beginning-of-field
            # So need to give the MULTILINE, M flag re.M. Or change ^ to \n
            [r"^\s\*", r'*'],
            # Hmm. This next rule turns any #d list at start of field into a header. That makes no sense
            # Probably '\s\d.\s' changed to '\d.\s' is better
#            [r"^\s\d\.", r'#'],
            # This next looks like it tries to strip leading !, getting rid of wikilink escapes
            [r"!(\w)", r"\1"],
            #            [r"(^|\n)[ ]{6,}", r"\1"], # AH: This stripped leading whitespace from lines, which messes up block quotes
            [r"\[([^\s\n\,\]\.\(\)]{6,}?)\s{1,}([^\n]+?)\]", r"[\2](\1)"],
            # This next line turns hashes into links. Require at least 15 chars to avoid mistaken links
            [r"(\s+|\()(changeset:|commit:|:)?([0-9a-f]{15,})([^0-9a-f\-])", r"\1[\3]({repo_url}/commit/\3)\4".format(repo_url=self.repo_url)],
            [r"source:branches/([\w\-]*)", r"[\1](../tree/\1)"],
            [r"source:fipy/([\w/\.]*)@([0-9a-f]{5,40})", r"[\1@\2](../tree/\2/\1)"],
            [r"source:([\w/\.\-\_\d\~]+)", r"[\1](../tree/master/\1)"],
            [r"blog:(\w*)", r"[blog:\1]({trac_url}/blog/\1)".format(trac_url=self.trac_url)],
            [r"(\b)([0-9a-f]{5,40})\.", r"\1\2"],
            [r" (\w*?)::", r"#### \1"],
            [r"\[([0-9]{1,4})\/(.+?)\]", r"[\1/\2]({trac_url}/changeset/\1/historical/\2)".format(trac_url=self.trac_url)],
            [r'\[changeset:"(\S*?)\/fipy"\]', r"\1"],
            [r'\^([0-9]{1,5})\^', r"\1"],
            [r'diff:@([0-9]{1,5}):([0-9]{1,5})', r'[diff:@\1:\2]({trac_url}/changeset?new=\2&old=\1)'.format(trac_url=self.trac_url)],
            #[r"\[([^\[\]\s\n\r\E\"\{][^\[\]\s\n\r\"\{]+)\s+([^\]]+)]", r"[\2](\1)"]
            # Try to exclude things that are used in apache error logs
            [r"\[(?!Thu|Mon|Tue|Wed|Fri|Sat|Sun|client|Last|Native)([^\]\[\{E\s\n\r\t\"\'][^\]\[\{\s\n\r\"\'\t]+)\s+([^\]]+)]", r"[\2](\1)"]
            ]

        # AH: Last one converts wiki links [URL text] to markdown links [text](URL)
        # piece at beginning excludes things that start with certain characters. Then it says rest of target may not
        # have certain characters
        regex = r"ticket:([0-9]{1,3})"
        sub = lambda m: r"issue #{0}".format(self.ticketsToIssuesMap[int(m.group(1))])
        subs.append([regex, sub])

        return [[re.compile(r, re.DOTALL), s] for r, s in subs]

    def no_compile_subs(self, ticketId):
        #print 'ticketId',ticketId
        subs = [[r"\[\[Image\((\S*?)\,\s{0,}\S*?\)\]\]", r"![\1]({attachmentsPrefix}/{ticketId}/\1)".format(attachmentsPrefix=self.attachmentsPrefix, ticketId=ticketId)],
                [r"\[\[Image\((\S*?)\)\]\]", r"![\1]({attachmentsPrefix}/{ticketId}/\1)".format(attachmentsPrefix=self.attachmentsPrefix, ticketId=ticketId)],
                [r"attachment:(\S*?)", r"{attachmentsPrefix}/{ticketId}/\1".format(attachmentsPrefix=self.attachmentsPrefix, ticketId=ticketId)]]

        return subs

    def translate(self, text, ticketId=''):
        subs = self.no_compile_subs(ticketId)
        for r, s in subs:
            p = re.compile(r, re.DOTALL)
            #ot = text
            text = p.sub(s, text)
            #if ot != text:
            #    print "regex '%s' changed \n'%s' to \n'%s'" % (r, ot, text)
        for p, s in self.subs:
            #ot = text
            text = p.sub(s, text)
            #if ot != text:
            #    print "regex '%s' changed \n'%s' to \n'%s'" % (p.pattern, ot, text)
        return text

class NullTranslator(Translator):
    def translate(self, text):
        return text
<commit_msg>avoid turning double colon like from python class to member printouts into a header or partial header<commit_after>import re

class Translator(object):
    """
    Simple regular expression to convert Trac wiki to Github markdown.
    """
    def __init__(self, repo, ticketsToIssuesMap, trac_url=None, attachmentsPrefix=None):
        self.repo_url = r'https://github.com/{login}/{name}'.format(login=repo.owner.login, name=repo.name)
        self.trac_url = trac_url
        self.ticketsToIssuesMap = ticketsToIssuesMap
        self.subs = self.compile_subs()
        self.attachmentsPrefix = attachmentsPrefix

    def compile_subs(self):
        subs = [
            # This first handles things like #!xml
            [r"\{\{\{\s*?[\n\r]{1,2}#!([a-z\/]+)(.*?)\}\}\}", r"```\1\2```"],
            # Single line block quotes
            [r"\{\{\{([^\n]*?)\}\}\}",  r"`\1`"],
            # Multi line block quotes
            [r"\{\{\{(.*?)\}\}\}",  r"```\1```"],
            # These next are for headings
            [r"====\s([^\n]+?)\s====(\s*[\n\r]+)", r'#### \1\2'],
            [r"===\s([^\n]+?)\s===(\s*[\n\r]+)", r'### \1\2'],
            [r"==\s([^\n]+?)\s==(\s*[\n\r]+)", r'## \1\2'],
            [r"=\s([^\n]+?)\s=(\s*[\n\r]+)", r'# \1\2'],
            [r"\!(([A-Z][a-z0-9]+){2,})", r'\1'],
            # These next are italics, bold
            [r"'''(.+)'''", r'*\1*'],
            [r"''(.+)''", r'_\1_'],
            # These next 2 are various bulleted or numbered lists
            # This next should probably be beginning-of-line, not beginning-of-field
            # So need to give the MULTILINE, M flag re.M. Or change ^ to \n
            [r"^\s\*", r'*'],
            # Hmm. This next rule turns any #d list at start of field into a header. That makes no sense
            # Probably '\s\d.\s' changed to '\d.\s' is better
#            [r"^\s\d\.", r'#'],
            # This next looks like it tries to strip leading !, getting rid of wikilink escapes
            [r"!(\w)", r"\1"],
            #            [r"(^|\n)[ ]{6,}", r"\1"], # AH: This stripped leading whitespace from lines, which messes up block quotes
            [r"\[([^\s\n\,\]\.\(\)]{6,}?)\s{1,}([^\n]+?)\]", r"[\2](\1)"],
            # This next line turns hashes into links. Require at least 15 chars to avoid mistaken links
            [r"(\s+|\()(changeset:|commit:|:)?([0-9a-f]{15,})([^0-9a-f\-])", r"\1[\3]({repo_url}/commit/\3)\4".format(repo_url=self.repo_url)],
            [r"source:branches/([\w\-]*)", r"[\1](../tree/\1)"],
            [r"source:fipy/([\w/\.]*)@([0-9a-f]{5,40})", r"[\1@\2](../tree/\2/\1)"],
            [r"source:([\w/\.\-\_\d\~]+)", r"[\1](../tree/master/\1)"],
            [r"blog:(\w*)", r"[blog:\1]({trac_url}/blog/\1)".format(trac_url=self.trac_url)],
            [r"(\b)([0-9a-f]{5,40})\.", r"\1\2"],
            [r" (\w*?)::[\s\n\r]+", r"####\n \1"], # Is this meant for a definition list?
            [r"\[([0-9]{1,4})\/(.+?)\]", r"[\1/\2]({trac_url}/changeset/\1/historical/\2)".format(trac_url=self.trac_url)],
            [r'\[changeset:"(\S*?)\/fipy"\]', r"\1"],
            [r'\^([0-9]{1,5})\^', r"\1"],
            [r'diff:@([0-9]{1,5}):([0-9]{1,5})', r'[diff:@\1:\2]({trac_url}/changeset?new=\2&old=\1)'.format(trac_url=self.trac_url)],
            #[r"\[([^\[\]\s\n\r\E\"\{][^\[\]\s\n\r\"\{]+)\s+([^\]]+)]", r"[\2](\1)"]
            # Try to exclude things that are used in apache error logs
            [r"\[(?!Thu|Mon|Tue|Wed|Fri|Sat|Sun|client|Last|Native)([^\]\[\{E\s\n\r\t\"\'][^\]\[\{\s\n\r\"\'\t]+)\s+([^\]]+)]", r"[\2](\1)"]
            ]

        # AH: Last one converts wiki links [URL text] to markdown links [text](URL)
        # piece at beginning excludes things that start with certain characters. Then it says rest of target may not
        # have certain characters
        regex = r"ticket:([0-9]{1,3})"
        sub = lambda m: r"issue #{0}".format(self.ticketsToIssuesMap[int(m.group(1))])
        subs.append([regex, sub])

        return [[re.compile(r, re.DOTALL), s] for r, s in subs]

    def no_compile_subs(self, ticketId):
        #print 'ticketId',ticketId
        subs = [[r"\[\[Image\((\S*?)\,\s{0,}\S*?\)\]\]", r"![\1]({attachmentsPrefix}/{ticketId}/\1)".format(attachmentsPrefix=self.attachmentsPrefix, ticketId=ticketId)],
                [r"\[\[Image\((\S*?)\)\]\]", r"![\1]({attachmentsPrefix}/{ticketId}/\1)".format(attachmentsPrefix=self.attachmentsPrefix, ticketId=ticketId)],
                [r"attachment:(\S*?)", r"{attachmentsPrefix}/{ticketId}/\1".format(attachmentsPrefix=self.attachmentsPrefix, ticketId=ticketId)]]

        return subs

    def translate(self, text, ticketId=''):
        subs = self.no_compile_subs(ticketId)
        for r, s in subs:
            p = re.compile(r, re.DOTALL)
            #ot = text
            text = p.sub(s, text)
            #if ot != text:
            #    print "regex '%s' changed \n'%s' to \n'%s'" % (r, ot, text)
        for p, s in self.subs:
            #ot = text
            text = p.sub(s, text)
            #if ot != text:
            #    print "regex '%s' changed \n'%s' to \n'%s'" % (p.pattern, ot, text)
        return text

class NullTranslator(Translator):
    def translate(self, text):
        return text
<|endoftext|>