<commit_before>#!/usr/bin/env python

import sys
import collections

from google.protobuf.compiler import plugin_pb2 as plugin
import itertools
import json
from google.protobuf.descriptor_pb2 import DescriptorProto, EnumDescriptorProto, EnumValueDescriptorProto, FieldDescriptorProto

def convert_protodef_to_editable(proto):
    class Editable(object):
        def __init__(self, prot):
            self.kind = type(prot)
            self.name = prot.name
            self.comment = ""
            if isinstance(prot, EnumDescriptorProto):
                self.value = [convert_protodef_to_editable(x) for x in prot.value]
            elif isinstance(prot, DescriptorProto):
                self.field = [convert_protodef_to_editable(x) for x in prot.field]
            elif isinstance(prot, EnumValueDescriptorProto):
                self.number = prot.number
            elif isinstance(prot, FieldDescriptorProto):
                self.type = prot.type
            else:
                raise Exception, type(prot)

    return Editable(proto)

def traverse(proto_file):

    def _collapse_comments(comments):
        return (comments["leading_comments"] + comments["trailing_comments"]).strip()

    def _traverse(package, items, tree):
        for item_index, item in enumerate(items):
            item = convert_protodef_to_editable(item)
            if item_index in tree:
                comments = tree[item_index]
                if "leading_comments" in comments or "trailing_comments" in comments:
                    item.comment = _collapse_comments(comments)
                    #raise Exception, item.__dict__
                    del comments["leading_comments"]
                    del comments["trailing_comments"]
                if item.kind is EnumDescriptorProto:
                    if 2 in comments: # value in EnumDescriptorProto
                        for k in comments[2]:
                            value_comment = comments[2][k]
                            if value_comment != {}:
                                item.value[k].comment = _collapse_comments(value_comment)
                elif item.kind is DescriptorProto:
                    if 2 in comments: # field in DescriptorProto
                        for k in comments[2]:
                            field_comment = comments[2][k]
                            if field_comment != {}:
                                item.field[k].comment = _collapse_comments(field_comment)
                else:
                    raise Exception, item.kind

            yield item, package

            if isinstance(item, DescriptorProto):
                for enum in item.enum_type:
                    yield enum, package

                for nested in item.nested_type:
                    nested_package = package + item.name

                    for nested_item in _traverse(nested, nested_package):
                        yield nested_item, nested_package

    import pprint
    open("dump", "w").write(pprint.pformat(proto_file.source_code_info))

    tree = collections.defaultdict(collections.defaultdict)
    for loc in proto_file.source_code_info.location:
        if loc.leading_comments or loc.trailing_comments:
            place = tree
            for p in loc.path:
                if not place.has_key(p):
                    place[p] = collections.defaultdict(collections.defaultdict)
                place = place[p]
            place["leading_comments"] = loc.leading_comments
            place["trailing_comments"] = loc.trailing_comments

    if set(tree.keys()).difference(set([4,5])) != set():
        raise Exception, sorted(tree.keys())

    return {"types":
        itertools.chain(
            _traverse(proto_file.package, proto_file.enum_type, tree[5]), # 5 is enum_type in FileDescriptorProto
            _traverse(proto_file.package, proto_file.message_type, tree[4]), # 4 is enum_type in FileDescriptorProto
        ),
        "file": ["".join(x.leading_detached_comments) for x in proto_file.source_code_info.location if len(x.leading_detached_comments) > 0]
    }

def generate_code(request, response):
    for proto_file in request.proto_file:
        types = []

        results = traverse(proto_file)
        for item, package in results["types"]:
            data = {
                'name': item.name,
                'doc': item.comment
            }

            if item.kind == DescriptorProto:
                data.update({
                    'type': 'message',
                    'fields': []
                })
                for f in item.field:
                    if f.type in [1]:
                        kind = "double"
                    elif f.type in [3]:
                        kind = "long"
                    elif f.type in [5]:
                        kind = "integer"
                    elif f.type in [8]:
                        kind = "boolean"
                    elif f.type in [9]:
                        kind = "string"
                    elif f.type in [11]:
                        kind = "message"
                    elif f.type in [12]:
                        kind = "bytes"
                    elif f.type in [14]:
                        kind = "enum"
                    else:
                        raise Exception, f.type
                    data["fields"].append({
                        'name': f.name,
                        'type': kind,
                        'doc': f.comment
                        })

            elif item.kind == EnumDescriptorProto:
                comments = ["\n* `%s`: %s"%(v.name, v.comment) for v in item.value]
                data.update({
                    'type': 'enum',
                    'symbols': [v.name for v in item.value]
                })
                data["doc"] += " ".join(comments)

            types.append(data)

        comments = "".join(results["file"]).strip()
        output = {
            "types": types,
            "protocol": proto_file.name.split("/")[-1].split(".")[0],
            'doc': comments,
            "namespace": proto_file.package,
        }

        # Fill response
        f = response.file.add()
        f.name = proto_file.name + '.json'
        f.content = json.dumps(output, indent=2)

if __name__ == '__main__':
    # Read request message from stdin
    data = sys.stdin.read()

    # Parse request
    request = plugin.CodeGeneratorRequest()
    request.ParseFromString(data)

    # Create response
    response = plugin.CodeGeneratorResponse()

    # Generate code
    generate_code(request, response)

    # Serialise response message
    output = response.SerializeToString()

    # Write to stdout
    sys.stdout.write(output)
<commit_msg>Add service support<commit_after>#!/usr/bin/env python

import sys
import collections

from google.protobuf.compiler import plugin_pb2 as plugin
import itertools
import json
from google.protobuf.descriptor_pb2 import DescriptorProto, EnumDescriptorProto, EnumValueDescriptorProto, FieldDescriptorProto, ServiceDescriptorProto, MethodDescriptorProto

def convert_protodef_to_editable(proto):
    class Editable(object):
        def __init__(self, prot):
            self.kind = type(prot)
            self.name = prot.name
            self.comment = ""
            if isinstance(prot, EnumDescriptorProto):
                self.value = [convert_protodef_to_editable(x) for x in prot.value]
            elif isinstance(prot, DescriptorProto):
                self.field = [convert_protodef_to_editable(x) for x in prot.field]
            elif isinstance(prot, EnumValueDescriptorProto):
                self.number = prot.number
            elif isinstance(prot, FieldDescriptorProto):
                self.type = prot.type
            elif isinstance(prot, ServiceDescriptorProto):
                self.method = [convert_protodef_to_editable(x) for x in prot.method]
            elif isinstance(prot, MethodDescriptorProto):
                self.input_type = prot.input_type
                self.output_type = prot.output_type
            else:
                raise Exception, type(prot)

    return Editable(proto)

def traverse(proto_file):

    def _collapse_comments(comments):
        return (comments["leading_comments"] + comments["trailing_comments"]).strip()

    def _traverse(package, items, tree):
        for item_index, item in enumerate(items):
            item = convert_protodef_to_editable(item)
            if item_index in tree:
                comments = tree[item_index]
                if "leading_comments" in comments or "trailing_comments" in comments:
                    item.comment = _collapse_comments(comments)
                    #raise Exception, item.__dict__
                    del comments["leading_comments"]
                    del comments["trailing_comments"]
                if item.kind is EnumDescriptorProto:
                    if 2 in comments: # value in EnumDescriptorProto
                        for k in comments[2]:
                            value_comment = comments[2][k]
                            if value_comment != {}:
                                item.value[k].comment = _collapse_comments(value_comment)
                elif item.kind is DescriptorProto:
                    if 2 in comments: # field in DescriptorProto
                        for k in comments[2]:
                            field_comment = comments[2][k]
                            if field_comment != {}:
                                item.field[k].comment = _collapse_comments(field_comment)
                elif item.kind is ServiceDescriptorProto:
                    if 2 in comments: # method in ServiceDescriptorProto
                        for k in comments[2]:
                            method_comment = comments[2][k]
                            if method_comment != {}:
                                item.method[k].comment = _collapse_comments(method_comment)
                else:
                    raise Exception, item.kind

            yield item, package

            if isinstance(item, DescriptorProto):
                for enum in item.enum_type:
                    yield enum, package

                for nested in item.nested_type:
                    nested_package = package + item.name

                    for nested_item in _traverse(nested, nested_package):
                        yield nested_item, nested_package

    import pprint
    open("dump", "w").write(pprint.pformat(proto_file.source_code_info))

    tree = collections.defaultdict(collections.defaultdict)
    for loc in proto_file.source_code_info.location:
        if loc.leading_comments or loc.trailing_comments:
            place = tree
            for p in loc.path:
                if not place.has_key(p):
                    place[p] = collections.defaultdict(collections.defaultdict)
                place = place[p]
            place["leading_comments"] = loc.leading_comments
            place["trailing_comments"] = loc.trailing_comments

    if set(tree.keys()).difference(set([4,5,6])) != set():
        raise Exception, sorted(tree.keys())

    return {"types":
        itertools.chain(
            _traverse(proto_file.package, proto_file.service, tree[6]), # 5 is enum_type in FileDescriptorProto
            _traverse(proto_file.package, proto_file.enum_type, tree[5]), # 5 is enum_type in FileDescriptorProto
            _traverse(proto_file.package, proto_file.message_type, tree[4]), # 4 is message_type in FileDescriptorProto
        ),
        "file": ["".join(x.leading_detached_comments) for x in proto_file.source_code_info.location if len(x.leading_detached_comments) > 0]
    }

def generate_code(request, response):
    for proto_file in request.proto_file:
        types = []

        results = traverse(proto_file)
        for item, package in results["types"]:
            data = {
                'name': item.name,
                'doc': item.comment
            }

            if item.kind == DescriptorProto:
                data.update({
                    'type': 'message',
                    'fields': []
                })
                for f in item.field: # types from FieldDescriptorProto
                    if f.type in [1]:
                        kind = "double"
                    elif f.type in [2]:
                        kind = "float"
                    elif f.type in [3]:
                        kind = "long"
                    elif f.type in [5]:
                        kind = "integer"
                    elif f.type in [8]:
                        kind = "boolean"
                    elif f.type in [9]:
                        kind = "string"
                    elif f.type in [11]:
                        kind = "message"
                    elif f.type in [12]:
                        kind = "bytes"
                    elif f.type in [14]:
                        kind = "enum"
                    else:
                        raise Exception, f.type
                    data["fields"].append({
                        'name': f.name,
                        'type': kind,
                        'doc': f.comment
                        })

            elif item.kind == EnumDescriptorProto:
                comments = ["\n* `%s`: %s"%(v.name, v.comment) for v in item.value]
                data.update({
                    'type': 'enum',
                    'symbols': [v.name for v in item.value]
                })
                data["doc"] += " ".join(comments)
            elif item.kind == ServiceDescriptorProto:
                data.update({
                    'type': 'service',
                    'methods': [{"name": m.name, "input": m.input_type[1:], "output": m.output_type[1:]} for m in item.method]
                })
            else:
                raise Exception, item.kind

            types.append(data)

        comments = "".join(results["file"]).strip()
        output = {
            "types": types,
            "protocol": proto_file.name.split("/")[-1].split(".")[0],
            'doc': comments,
            "namespace": proto_file.package,
        }

        # Fill response
        f = response.file.add()
        f.name = proto_file.name + '.json'
        f.content = json.dumps(output, indent=2)

if __name__ == '__main__':
    # Read request message from stdin
    data = sys.stdin.read()

    # Parse request
    request = plugin.CodeGeneratorRequest()
    request.ParseFromString(data)

    # Create response
    response = plugin.CodeGeneratorResponse()

    # Generate code
    generate_code(request, response)

    # Serialise response message
    output = response.SerializeToString()

    # Write to stdout
    sys.stdout.write(output)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# This script is used for downloading the datasets used by the examples.
# The datasets used in examples are: iris, glass, and segment.
# The following table describes some informations related to the three datasets.

# === data ==== N ==== K ==== D ==
# |  iris   |  150  |  3  |   4  |
# ================================
# |  glass  |  214  |  6  |   9  |
# ================================
# | segment |  2310 |  7  |  19  |
# ================================

# Note: N is the number of examples, including both training and testing set;
#       K is the number of classes;
#       D is the input feature dimension, i.e. the length of feature vector.

import os
import urllib.request
import random

IRIS_URL = 'http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/iris.scale'
IRIS_SIZE = 150

GLASS_URL = 'http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/glass.scale'
GLASS_SIZE = 214

SEGMENT_URL = 'http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/segment.scale'
SEGMENT_SIZE = 2310

TARGET_PATH = os.path.dirname(os.path.realpath(__file__))

def main():
    print('downloading iris ...')
    rows = list(urllib.request.urlopen(IRIS_URL))
    selected = random.sample(rows, IRIS_SIZE)
    with open(TARGET_PATH + '/iris.txt', 'wb') as f:
        for row in selected:
            f.write(row)

    print('downloading glass ...')
    rows = list(urllib.request.urlopen(GLASS_URL))
    selected = random.sample(rows, GLASS_SIZE)
    with open(TARGET_PATH + '/glass.txt', 'wb') as f:
        for row in selected:
            f.write(row)

    print('downloading segment ...')
    rows = list(urllib.request.urlopen(SEGMENT_URL))
    selected = random.sample(rows, SEGMENT_SIZE)
    with open(TARGET_PATH + '/segment.txt', 'wb') as f:
        for row in selected:
            f.write(row)

if __name__ == '__main__':
    main()
<commit_msg>change to binary datasets, add detailed comments<commit_after># !/usr/bin/env python3
#
# The script is used for downloading the three datasets used by the examples.
#
# All three datasets come from LIBSVM website, and are stored in LIBSVM format.
# For more details, please refer to the following link:
# https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary.html
#
# Besides, all three datasets are for binary classification, since most of the
# state-of-the-arts active learning algorithms (query strategies) are only
# suitable for binary classification.
#
# The following table describes some informations about
# the three datasets: australian, diabetes, and heart.
#
# +------------+-----+----+
# |   dataset  |  N  |  D |
# +============+=====+====+
# | australian | 690 | 14 |
# +------------+-----+----+
# |  diabetes  | 768 |  8 |
# +------------+-----+----+
# |    heart   | 270 | 13 |
# +------------+-----+----+
#
# N is the number of samples, and D is the dimension of the input feature.
# labels y \in {-1, +1}

import os
import urllib.request
import random

AUS_URL = 'https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/australian_scale'
AUS_SIZE = 690

DB_URL = 'https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/diabetes_scale'
DB_SIZE = 768

HT_URL = 'https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/heart_scale'
HT_SIZE = 270

TARGET_PATH = os.path.dirname(os.path.realpath(__file__))

def main():
    print('downloading australian ...')
    rows = list(urllib.request.urlopen(AUS_URL))
    selected = random.sample(rows, AUS_SIZE)
    with open(TARGET_PATH + '/australian.txt', 'wb') as f:
        for row in selected:
            f.write(row)
    print('australian downloaded successfully !\n')

    print('downloading diabetes ...')
    rows = list(urllib.request.urlopen(DB_URL))
    selected = random.sample(rows, DB_SIZE)
    with open(TARGET_PATH + '/diabetes.txt', 'wb') as f:
        for row in selected:
            f.write(row)
    print('diabetes downloaded successfully !\n')

    print('downloading heart ...')
    rows = list(urllib.request.urlopen(HT_URL))
    selected = random.sample(rows, HT_SIZE)
    with open(TARGET_PATH + '/heart.txt', 'wb') as f:
        for row in selected:
            f.write(row)
    print('heart downloaded successfully !')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (c) 2006-2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# drmemory_analyze.py

''' Given a ThreadSanitizer output file, parses errors and uniques them.'''

import logging
import optparse
import os
import re
import subprocess
import sys
import time

class _StackTraceLine(object):
  def __init__(self, line, address, binary):
    self.raw_line_ = line
    self.address = address
    self.binary = binary
  def __str__(self):
    return self.raw_line_

class DrMemoryAnalyze:
  ''' Given a set of Dr.Memory output files, parse all the errors out of
  them, unique them and output the results.'''

  def __init__(self, source_dir, files):
    '''Reads in a set of files.

    Args:
      source_dir: Path to top of source tree for this build
      files: A list of filenames.
    '''

    self.races = []
    self.used_suppressions = {}
    for file in files:
      self.ParseReportFile(file)

  def ReadLine(self):
    self.line_ = self.cur_fd_.readline()
    self.stack_trace_line_ = None

  def ReadSection(self):
    FILE_PREFIXES_TO_CUT = [
        "build\\src\\",
        "crt_bld\\self_x86\\",
    ]
    CUT_STACK_BELOW = ".*testing.*Test.*Run.*"

    result = [self.line_]
    self.ReadLine()
    cnt = 1
    while len(self.line_.strip()) > 0:
      tmp_line = self.line_
      match1 = re.search("system call (.*)\n", tmp_line)
      if match1:
        result.append(" #%2i <sys.call> %s\n" % (cnt, match1.groups()[0]))
        cnt = cnt + 1
        self.ReadLine() # skip "<system call> line
        self.ReadLine()
        continue
      match1 = re.search("(0x[0-9a-fA-F]+) <.*> (.*)!([^+]*)"
                         "(?:\+0x[0-9a-fA-F]+)?\n", tmp_line)
      self.ReadLine()
      match2 = re.search("\s*(.*):([0-9]+)(?:\+0x[0-9a-fA-F]+)?", self.line_)
      if match2:
        self.ReadLine()
        if match1:
          pc, binary, fname = match1.groups()
          if re.search(CUT_STACK_BELOW, fname):
            break
          src, lineno = match2.groups()
          binary_fname = "%s!%s" % (binary, fname)
          report_line = (" #%2i %s %s" % (cnt, pc, binary_fname))
          if src != "??":
            for pat in FILE_PREFIXES_TO_CUT:
              idx = src.rfind(pat)
              if idx != -1:
                src = src[idx+len(pat):]
            if (len(binary_fname) < 40):
              report_line += " " * (40 - len(binary_fname))
            report_line += " " + src
            if int(lineno) != 0:
              report_line += ":%i" % int(lineno)
          result.append(report_line + "\n")
          cnt = cnt + 1
    return result

  def ParseReportFile(self, filename):
    self.cur_fd_ = open(filename, 'r')

    while True:
      self.ReadLine()
      if (self.line_ == ''):
        break

      if re.search("Grouping errors that", self.line_):
        # DrMemory has finished working.
        break

      tmp = []
      match = re.search("Error #[0-9]+: (.*)", self.line_)
      if match:
        self.line_ = match.groups()[0].strip() + "\n"
        tmp.extend(self.ReadSection())
        self.races.append(tmp)

    self.cur_fd_.close()

  def Report(self, check_sanity):
    sys.stdout.flush()
    #TODO(timurrrr): support positive tests / check_sanity==True

    if len(self.races) > 0:
      logging.error("Found %i error reports" % len(self.races))
      for report_list in self.races:
        report = ''
        for line in report_list:
          report += str(line)
        logging.error('\n' + report)
      return -1
    logging.info("PASS: No error reports found")
    return 0

if __name__ == '__main__':
  '''For testing only. The DrMemoryAnalyze class should be imported instead.'''
  retcode = 0
  parser = optparse.OptionParser("usage: %prog [options] <files to analyze>")
  parser.add_option("", "--source_dir",
                    help="path to top of source tree for this build"
                    "(used to normalize source paths in baseline)")

  (options, args) = parser.parse_args()
  if not len(args) >= 1:
    parser.error("no filename specified")
  filenames = args

  analyzer = DrMemoryAnalyze(options.source_dir, filenames)
  retcode = analyzer.Report(False)

  sys.exit(retcode)
<commit_msg>Fix a typo in Dr.Memory analyze script; add ASSERT lines to the list of the reports TBR=glider Review URL: http://codereview.chromium.org/2869010<commit_after>#!/usr/bin/python
# Copyright (c) 2006-2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# drmemory_analyze.py

''' Given a ThreadSanitizer output file, parses errors and uniques them.'''

import logging
import optparse
import os
import re
import subprocess
import sys
import time

class _StackTraceLine(object):
  def __init__(self, line, address, binary):
    self.raw_line_ = line
    self.address = address
    self.binary = binary
  def __str__(self):
    return self.raw_line_

class DrMemoryAnalyze:
  ''' Given a set of Dr.Memory output files, parse all the errors out of
  them, unique them and output the results.'''

  def __init__(self, source_dir, files):
    '''Reads in a set of files.

    Args:
      source_dir: Path to top of source tree for this build
      files: A list of filenames.
    '''

    self.reports = []
    self.used_suppressions = {}
    for file in files:
      self.ParseReportFile(file)

  def ReadLine(self):
    self.line_ = self.cur_fd_.readline()
    self.stack_trace_line_ = None

  def ReadSection(self):
    FILE_PREFIXES_TO_CUT = [
        "build\\src\\",
        "crt_bld\\self_x86\\",
    ]
    CUT_STACK_BELOW = ".*testing.*Test.*Run.*"

    result = [self.line_]
    self.ReadLine()
    cnt = 1
    while len(self.line_.strip()) > 0:
      tmp_line = self.line_
      match1 = re.search("system call (.*)\n", tmp_line)
      if match1:
        result.append(" #%2i <sys.call> %s\n" % (cnt, match1.groups()[0]))
        cnt = cnt + 1
        self.ReadLine() # skip "<system call> line
        self.ReadLine()
        continue
      match1 = re.search("(0x[0-9a-fA-F]+) <.*> (.*)!([^+]*)"
                         "(?:\+0x[0-9a-fA-F]+)?\n", tmp_line)
      self.ReadLine()
      match2 = re.search("\s*(.*):([0-9]+)(?:\+0x[0-9a-fA-F]+)?", self.line_)
      if match2:
        self.ReadLine()
        if match1:
          pc, binary, fname = match1.groups()
          if re.search(CUT_STACK_BELOW, fname):
            break
          src, lineno = match2.groups()
          binary_fname = "%s!%s" % (binary, fname)
          report_line = (" #%2i %s %s" % (cnt, pc, binary_fname))
          if src != "??":
            for pat in FILE_PREFIXES_TO_CUT:
              idx = src.rfind(pat)
              if idx != -1:
                src = src[idx+len(pat):]
            if (len(binary_fname) < 40):
              report_line += " " * (40 - len(binary_fname))
            report_line += " " + src
            if int(lineno) != 0:
              report_line += ":%i" % int(lineno)
          result.append(report_line + "\n")
          cnt = cnt + 1
    return result

  def ParseReportFile(self, filename):
    self.cur_fd_ = open(filename, 'r')

    while True:
      self.ReadLine()
      if (self.line_ == ''):
        break

      if re.search("Grouping errors that", self.line_):
        # DrMemory has finished working.
        break

      tmp = []
      match = re.search("Error #[0-9]+: (.*)", self.line_)
      if match:
        self.line_ = match.groups()[0].strip() + "\n"
        tmp.extend(self.ReadSection())
        self.reports.append(tmp)
      elif self.line_.startswith("ASSERT FAILURE"):
        self.reports.append(self.line_.strip())

    self.cur_fd_.close()

  def Report(self, check_sanity):
    sys.stdout.flush()
    #TODO(timurrrr): support positive tests / check_sanity==True

    if len(self.reports) > 0:
      logging.error("Found %i error reports" % len(self.reports))
      for report_list in self.reports:
        report = ''
        for line in report_list:
          report += str(line)
        logging.error('\n' + report)
      return -1
    logging.info("PASS: No error reports found")
    return 0

if __name__ == '__main__':
  '''For testing only. The DrMemoryAnalyze class should be imported instead.'''
  retcode = 0
  parser = optparse.OptionParser("usage: %prog [options] <files to analyze>")
  parser.add_option("", "--source_dir",
                    help="path to top of source tree for this build"
                    "(used to normalize source paths in baseline)")

  (options, args) = parser.parse_args()
  if not len(args) >= 1:
    parser.error("no filename specified")
  filenames = args

  analyzer = DrMemoryAnalyze(options.source_dir, filenames)
  retcode = analyzer.Report(False)

  sys.exit(retcode)
<|endoftext|>


<commit_before># -*- coding: utf-8  -*-

from re import sub, UNICODE

from markupsafe import escape

def highlight_delta(context, chain, delta):
    degree = chain.degree - 1
    highlights = [False] * degree
    block = [chain.START] * degree
    for word in chain.text.split() + ([chain.END] * degree):
        word = _strip_word(chain, word)
        tblock = tuple(block)
        if tblock in delta.chain and word in delta.chain[tblock]:
            highlights[-1 * degree:] = [True] * degree
            highlights.append(True)
        else:
            highlights.append(False)
        block.pop(0)
        block.append(word)

    i = degree
    numwords = len(chain.text.split())
    processed = []
    for paragraph in chain.text.split("\n"):
        words = []
        for i, word in enumerate(paragraph.split(), i):
            if highlights[i]:
                before = highlights[i - 1]
                after = highlights[i + 1]
                first = i == degree
                last = i - degree + 1 == numwords
                words.append(_highlight_word(word, before, after, first, last))
            else:
                words.append(unicode(escape(word)))
        processed.append(u" ".join(words))
        i += 1

    return u"
" + u"
\n
".join(processed) + u"
"

def _highlight_word(word, before, after, first, last):
    if before and after:
        # Word is in the middle of a highlighted block:
        res = unicode(escape(word))
        if first:
            res = u'' + res
        if last:
            res += u''
    elif after:
        # Word is the first in a highlighted block:
        res = u'' + _fade_word(word, u"in")
        if last:
            res += u""
    elif before:
        # Word is the last in a highlighted block:
        res = _fade_word(word, u"out") + u""
        if first:
            res = u'' + res
    else:
        res = unicode(escape(word))
    return res

def _fade_word(word, dir):
    if len(word) <= 4:
        word = unicode(escape(word))
        return u'{1}'.format(dir, word)
    if dir == u"out":
        before, after = unicode(escape(word[:-4])), unicode(escape(word[-4:]))
        base = u'{0}{1}'
        return base.format(before, after)
    else:
        before, after = unicode(escape(word[:4])), unicode(escape(word[4:]))
        base = u'{0}{1}'
        return base.format(before, after)

def _strip_word(chain, word):
    if word == chain.START or word == chain.END:
        return word
    return sub("[^\w\s-]", "", word.lower(), flags=UNICODE)
<commit_msg>Join adjacent lines with only one word.<commit_after># -*- coding: utf-8  -*-

from re import sub, UNICODE

from markupsafe import escape

def highlight_delta(context, chain, delta):
    degree = chain.degree - 1
    highlights = [False] * degree
    block = [chain.START] * degree
    for word in chain.text.split() + ([chain.END] * degree):
        word = _strip_word(chain, word)
        tblock = tuple(block)
        if tblock in delta.chain and word in delta.chain[tblock]:
            highlights[-1 * degree:] = [True] * degree
            highlights.append(True)
        else:
            highlights.append(False)
        block.pop(0)
        block.append(word)

    i = degree
    numwords = len(chain.text.split())
    processed = []
    paragraphs = chain.text.split("\n")
    while paragraphs:
        words = []
        for i, word in enumerate(_get_next(paragraphs), i):
            if highlights[i]:
                before = highlights[i - 1]
                after = highlights[i + 1]
                first = i == degree
                last = i - degree + 1 == numwords
                words.append(_highlight_word(word, before, after, first, last))
            else:
                words.append(unicode(escape(word)))
        processed.append(u" ".join(words))
        i += 1

    return u"
" + u"
\n
".join(processed) + u"
"

def _get_next(paragraphs):
    paragraph = paragraphs.pop(0)
    body = paragraph.split()
    if len(body) == 1:
        while paragraphs:
            next = paragraphs[0].split()
            if len(next) == 1:
                body += next
                paragraphs.pop(0)
            else:
                break
    return body

def _highlight_word(word, before, after, first, last):
    if before and after:
        # Word is in the middle of a highlighted block:
        res = unicode(escape(word))
        if first:
            res = u'' + res
        if last:
            res += u''
    elif after:
        # Word is the first in a highlighted block:
        res = u'' + _fade_word(word, u"in")
        if last:
            res += u""
    elif before:
        # Word is the last in a highlighted block:
        res = _fade_word(word, u"out") + u""
        if first:
            res = u'' + res
    else:
        res = unicode(escape(word))
    return res

def _fade_word(word, dir):
    if len(word) <= 4:
        word = unicode(escape(word))
        return u'{1}'.format(dir, word)
    if dir == u"out":
        before, after = unicode(escape(word[:-4])), unicode(escape(word[-4:]))
        base = u'{0}{1}'
        return base.format(before, after)
    else:
        before, after = unicode(escape(word[:4])), unicode(escape(word[4:]))
        base = u'{0}{1}'
        return base.format(before, after)

def _strip_word(chain, word):
    if word == chain.START or word == chain.END:
        return word
    return sub("[^\w\s-]", "", word.lower(), flags=UNICODE)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Akvo RSR is covered by the GNU Affero General Public License.
# See more details in the license.txt file located at the root folder of the Akvo RSR module.
# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.

from ....rsr.models.project import Project
from ....rsr.models.related_project import RelatedProject

from .. import ImportMapper

from django.core.exceptions import ObjectDoesNotExist

class RelatedProjects(ImportMapper):

    def __init__(self, iati_import_job, parent_elem, project, globals,
                 related_obj=None):
        super(RelatedProjects, self).__init__(iati_import_job, parent_elem,
                                              project, globals)
        self.model = RelatedProject

    def do_import(self):
        """
        Retrieve and store the related projects.
        The related projects will be extracted from the 'related-activity' elements.

        :return: List; contains fields that have changed
        """
        imported_related_projects = []
        changes = []

        for related_activity in self.parent_elem.findall('related-activity'):

            related_iati_id = self.get_attrib(related_activity, 'ref', 'related_iati_id')
            relation = self.get_attrib(related_activity, 'type', 'relation')

            # TODO: I don't fully understand the original logic, need Kasper's explanation
            try:
                related_project = Project.objects.get(iati_activity_id=related_iati_id)
            except ObjectDoesNotExist:
                related_project = None

            rp, created = RelatedProject.objects.get_or_create(
                project=self.project,
                related_project=related_project,
                related_iati_id=related_iati_id if not related_activity else '',
                relation=relation
            )
            if created:
                changes.append(u'added related project (id: {}): {}'.format(rp.pk, rp))
            imported_related_projects.append(rp)

        changes += self.delete_objects(
                self.project.related_projects, imported_related_projects, 'related project')
        return changes
<commit_msg>[#1828] Fix data assignment to RelatedProject<commit_after># -*- coding: utf-8 -*-

# Akvo RSR is covered by the GNU Affero General Public License.
# See more details in the license.txt file located at the root folder of the Akvo RSR module.
# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.

from ....rsr.models.project import Project
from ....rsr.models.related_project import RelatedProject

from .. import ImportMapper

from django.core.exceptions import ObjectDoesNotExist

class RelatedProjects(ImportMapper):

    def __init__(self, iati_import_job, parent_elem, project, globals,
                 related_obj=None):
        super(RelatedProjects, self).__init__(iati_import_job, parent_elem,
                                              project, globals)
        self.model = RelatedProject

    def do_import(self):
        """
        Retrieve and store the related projects.
        The related projects will be extracted from the 'related-activity' elements.

        :return: List; contains fields that have changed
        """
        imported_related_projects = []
        changes = []

        for related_activity in self.parent_elem.findall('related-activity'):

            related_iati_id = self.get_attrib(related_activity, 'ref', 'related_iati_id')
            relation = self.get_attrib(related_activity, 'type', 'relation')

            try:
                related_project = Project.objects.get(iati_activity_id=related_iati_id)
            except ObjectDoesNotExist:
                related_project = None

            rp, created = RelatedProject.objects.get_or_create(
                project=self.project,
                related_project=related_project,
                related_iati_id=related_iati_id if not related_project else '',
                relation=relation
            )
            if created:
                changes.append(u'added related project (id: {}): {}'.format(rp.pk, rp))
            imported_related_projects.append(rp)

        changes += self.delete_objects(
                self.project.related_projects, imported_related_projects, 'related project')
        return changes
<|endoftext|>


<commit_before>from django.utils.translation import ugettext_lazy as _
from django.core.urlresolvers import reverse
from django.core.validators import MinValueValidator
from django.db import models

from wafer.snippets.markdown_field import MarkdownTextField

class File(models.Model):
    """A file for use in sponsor and sponshorship package descriptions."""
    name = models.CharField(max_length=255)
    description = models.TextField()
    item = models.FileField(upload_to='sponsors_files')

class SponsorshipPackage(models.Model):
    """A description of a sponsorship package."""
    order = models.IntegerField()
    name = models.CharField(max_length=255)
    number_available = models.IntegerField(
        null=True, validators=[MinValueValidator(0)])
    currency = models.CharField(
        max_length=16,
        help_text=_("Currency symbol for the sponsorship amount."))
    price = models.DecimalField(
        max_digits=12, decimal_places=2,
        help_text=_("Amount to be sponsored."))
    short_description = models.TextField(
        help_text=_("One sentence overview of the package."))
    description = MarkdownTextField(
        help_text=_("Describe what the package gives the sponsor."))
    files = models.ManyToManyField(
        File, related_name="packages", null=True, blank=True,
        help_text=_("Images and other files for use in"
                    " the description markdown field."))

    class Meta:
        ordering = ['order', 'price', 'name']

    def __unicode__(self):
        return u'%s (amount: %.0f)' % (self.name, self.price)

class Sponsor(models.Model):
    """A conference sponsor."""
    name = models.CharField(max_length=255)
    packages = models.ManyToManyField(SponsorshipPackage,
                                      related_name="sponsors")
    description = MarkdownTextField(
        help_text=_("Write some nice things about the sponsor."))
    files = models.ManyToManyField(
        File, related_name="sponsors", null=True, blank=True,
        help_text=_("Images and other files for use in"
                    " the description markdown field."))

    def __unicode__(self):
        return u'%s' % (self.name,)

    def get_absolute_url(self):
        return reverse('wafer_sponsor', args=(self.pk,))
<commit_msg>Add defaults to new fields.<commit_after>from django.utils.translation import ugettext_lazy as _
from django.core.urlresolvers import reverse
from django.core.validators import MinValueValidator
from django.db import models

from wafer.snippets.markdown_field import MarkdownTextField

class File(models.Model):
    """A file for use in sponsor and sponshorship package descriptions."""
    name = models.CharField(max_length=255)
    description = models.TextField()
    item = models.FileField(upload_to='sponsors_files')

class SponsorshipPackage(models.Model):
    """A description of a sponsorship package."""
    order = models.IntegerField(default=1)
    name = models.CharField(max_length=255)
    number_available = models.IntegerField(
        null=True, validators=[MinValueValidator(0)])
    currency = models.CharField(
        max_length=16, default='$',
        help_text=_("Currency symbol for the sponsorship amount."))
    price = models.DecimalField(
        max_digits=12, decimal_places=2,
        help_text=_("Amount to be sponsored."))
    short_description = models.TextField(
        help_text=_("One sentence overview of the package."))
    description = MarkdownTextField(
        help_text=_("Describe what the package gives the sponsor."))
    files = models.ManyToManyField(
        File, related_name="packages", null=True, blank=True,
        help_text=_("Images and other files for use in"
                    " the description markdown field."))

    class Meta:
        ordering = ['order', 'price', 'name']

    def __unicode__(self):
        return u'%s (amount: %.0f)' % (self.name, self.price)

class Sponsor(models.Model):
    """A conference sponsor."""
    name = models.CharField(max_length=255)
    packages = models.ManyToManyField(SponsorshipPackage,
                                      related_name="sponsors")
    description = MarkdownTextField(
        help_text=_("Write some nice things about the sponsor."))
    files = models.ManyToManyField(
        File, related_name="sponsors", null=True, blank=True,
        help_text=_("Images and other files for use in"
                    " the description markdown field."))

    def __unicode__(self):
        return u'%s' % (self.name,)

    def get_absolute_url(self):
        return reverse('wafer_sponsor', args=(self.pk,))
<|endoftext|>


<commit_before>import json
import os, sys
from shutil import copyfile

from datetime import datetime
import inspect
import tensorflow as tf

from agents import test_traffic_deep

if 'SUMO_HOME' in os.environ:
    tools = os.path.join(os.environ['SUMO_HOME'], 'tools')
    sys.path.append(tools)
else:
    sys.exit("please declare environment variable 'SUMO_HOME'")

import gym

from baselines import deepq
import Traci_1_cross_env.Traci_1_cross_env
import Traci_2_cross_env.Traci_2_cross_env
import Traci_3_cross_env.Traci_3_cross_env
from baselines import logger, logger_utils
from BaseTraciEnv import BaseTraciEnv
from pathlib import Path

os.environ["CUDA_VISIBLE_DEVICES"] = "1"

def train_and_log(environment_name="Traci_3_cross_env-v0",
                  num_car_chances=1000,
                  action_function=BaseTraciEnv.set_light_phase_4_cross_green_dir,
                  reward_function=BaseTraciEnv.reward_total_waiting_vehicles,
                  lr=(1e-3 / 4.0),  # lr / 4.0 as described in (Tom Schaul 2016) when using prio. exp. repl.
                  max_timesteps=int(1e6),
                  buffer_size=200000,
                  exploration_initial_p=0.2,
                  exploration_fraction=0.5,
                  explore_final_eps=0.02,
                  train_freq=10,
                  batch_size=32,
                  checkpoint_freq=10000,
                  learning_starts=10000,
                  gamma=0.9,
                  target_network_update_freq=500,
                  start_car_probabilities=[1.0, 0.1],
                  # [0.1,0.1,0.1,0.1,0.1,0.1,0.1], #For traci_3_cross: Bigroad_spawn_prob,Smallroad_spawn_prob
                  end_car_probabilities=None,  # When set to None do not anneal
                  num_steps_from_start_car_probs_to_end_car_probs=1e5,
                  prioritized_replay=True,
                  prioritized_replay_alpha=0.6,
                  prioritized_replay_beta0=0.4,
                  prioritized_replay_beta_iters=None,
                  prioritized_replay_eps=1e-6,
                  num_cpu=8,
                  param_noise=False,
                  state_use_queue_length=True,
                  normalize_queue_lengths=True,
                  state_use_tl_state=True,
                  state_use_time_since_tl_change=True,
                  state_use_avg_speed=False,
                  hidden_layers=[64],
                  num_actions_pr_trafficlight=2,
                  num_history_states=2,
                  experiment_name="",
                  layer_norm=False):
    print("RUNNING train_and_log")

    # Print call values
    frame = inspect.currentframe()
    args, _, _, values = inspect.getargvalues(frame)

    call_params_string_array = ['function name "%s"' % inspect.getframeinfo(frame)[2]]
    for i in args:
        call_params_string_array.append("    %s = %s" % (i, values[i]))

    # Setup path of logging, name of environment and save the current arguments (this script)
    log_dir = [os.path.join(str(Path.home()), "Desktop"), environment_name + "deep_q_" + experiment_name]
    logger_path = logger_utils.path_with_date(log_dir[0], log_dir[1])

    # Create environment and initialize
    env = gym.make(environment_name)
    env.configure_traci(num_car_chances=num_car_chances,
                        start_car_probabilities=start_car_probabilities,
                        end_car_probabilities=end_car_probabilities,
                        num_steps_from_start_car_probs_to_end_car_probs=num_steps_from_start_car_probs_to_end_car_probs,
                        reward_func=reward_function,
                        action_func=action_function,
                        state_contain_num_cars_in_queue_history=state_use_queue_length,
                        state_contain_avg_speed_between_detectors_history=state_use_avg_speed,
                        state_contain_time_since_tl_change=state_use_time_since_tl_change,
                        state_contain_tl_state_history=state_use_tl_state,
                        num_actions_pr_trafficlight=num_actions_pr_trafficlight,
                        num_history_states=num_history_states,
                        normalize_queue_lengths=normalize_queue_lengths)
    env.render()

    # Initialize logger
    logger.reset()
    logger.configure(logger_path, ["tensorboard", "stdout"])
    logger.logtxt(call_params_string_array)

    with open(logger_path + "/params.txt", 'w') as file:
        file.write(json.dumps(call_params_string_array, indent=4))
    copyfile(__file__, logger_path + "/script.txt")

    # Create the training model
    model = deepq.models.mlp(hidden_layers, layer_norm=layer_norm)
    act = deepq.learn(
        env=env,
        q_func=model,
        lr=lr,
        max_timesteps=max_timesteps,
        buffer_size=buffer_size,
        exploration_initial_p=exploration_initial_p,
        exploration_fraction=exploration_fraction,
        exploration_final_eps=explore_final_eps,
        train_freq=train_freq,
        batch_size=batch_size,
        print_freq=1,
        checkpoint_freq=checkpoint_freq,
        learning_starts=learning_starts,
        gamma=gamma,
        target_network_update_freq=target_network_update_freq,
        prioritized_replay=prioritized_replay,
        prioritized_replay_alpha=prioritized_replay_alpha,
        prioritized_replay_beta0=prioritized_replay_beta0,
        prioritized_replay_beta_iters=prioritized_replay_beta_iters,
        prioritized_replay_eps=prioritized_replay_eps,
        num_cpu=num_cpu,
        param_noise=param_noise,
        callback=None,
        model_path=logger_path
    )
    save_path = logger_path + "/model-" + datetime.now().strftime('%Y-%m-%d_%H-%M-%S') + ".pkl"
    print("Saving last model to {}".format(save_path))
    act.save(save_path)

    # Run test
    test_environment = gym.make(environment_name)
    test_environment.configure_traci(num_car_chances=num_car_chances,
                                     start_car_probabilities=start_car_probabilities,
                                     enjoy_car_probs=False,
                                     reward_func=reward_function,
                                     action_func=action_function,
                                     state_contain_num_cars_in_queue_history=state_use_queue_length,
                                     state_contain_time_since_tl_change=state_use_time_since_tl_change,
                                     state_contain_tl_state_history=state_use_tl_state,
                                     state_contain_avg_speed_between_detectors_history=state_use_avg_speed,
                                     num_actions_pr_trafficlight=num_actions_pr_trafficlight,
                                     num_history_states=num_history_states)
    test_traffic_deep.test(environment_name=environment_name,
                           path_to_model=save_path,
                           configured_environment=test_environment,
                           act=act,
                           log_dir=logger_path)

def main():
    probabilities = [
        [0.25, 0.05],
        [1.0, 0.10]
    ]

    # Set this by hand!
    experiment_name = "normalize"

    exp_start = 0.2
    exp_end = 0.02

    for pr in probabilities:
        print("Now props:", pr)
        g = tf.Graph()
        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        sess = tf.InteractiveSession(graph=g, config=config)
        with g.as_default():
            train_and_log(start_car_probabilities=pr,
                          experiment_name=experiment_name,
                          exploration_initial_p=exp_start,
                          explore_final_eps=exp_end)

if __name__ == '__main__':
    main()
<commit_msg>do not render<commit_after>import json
import os, sys
from shutil import copyfile

from datetime import datetime
import inspect
import tensorflow as tf

from agents import test_traffic_deep

if 'SUMO_HOME' in os.environ:
    tools = os.path.join(os.environ['SUMO_HOME'], 'tools')
    sys.path.append(tools)
else:
    sys.exit("please declare environment variable 'SUMO_HOME'")

import gym

from baselines import deepq
import Traci_1_cross_env.Traci_1_cross_env
import Traci_2_cross_env.Traci_2_cross_env
import Traci_3_cross_env.Traci_3_cross_env
from baselines import logger, logger_utils
from BaseTraciEnv import BaseTraciEnv
from pathlib import Path

os.environ["CUDA_VISIBLE_DEVICES"] = "1"

def train_and_log(environment_name="Traci_3_cross_env-v0",
                  num_car_chances=1000,
                  action_function=BaseTraciEnv.set_light_phase_4_cross_green_dir,
                  reward_function=BaseTraciEnv.reward_total_waiting_vehicles,
                  lr=(1e-3 / 4.0),  # lr / 4.0 as described in (Tom Schaul 2016) when using prio. exp. repl.
                  max_timesteps=int(1e6),
                  buffer_size=200000,
                  exploration_initial_p=0.2,
                  exploration_fraction=0.5,
                  explore_final_eps=0.02,
                  train_freq=10,
                  batch_size=32,
                  checkpoint_freq=10000,
                  learning_starts=10000,
                  gamma=0.9,
                  target_network_update_freq=500,
                  start_car_probabilities=[1.0, 0.1],
                  # [0.1,0.1,0.1,0.1,0.1,0.1,0.1], #For traci_3_cross: Bigroad_spawn_prob,Smallroad_spawn_prob
                  end_car_probabilities=None,  # When set to None do not anneal
                  num_steps_from_start_car_probs_to_end_car_probs=1e5,
                  prioritized_replay=True,
                  prioritized_replay_alpha=0.6,
                  prioritized_replay_beta0=0.4,
                  prioritized_replay_beta_iters=None,
                  prioritized_replay_eps=1e-6,
                  num_cpu=8,
                  param_noise=False,
                  state_use_queue_length=True,
                  normalize_queue_lengths=True,
                  state_use_tl_state=True,
                  state_use_time_since_tl_change=True,
                  state_use_avg_speed=False,
                  hidden_layers=[64],
                  num_actions_pr_trafficlight=2,
                  num_history_states=2,
                  experiment_name="",
                  layer_norm=False):
    print("RUNNING train_and_log")

    # Print call values
    frame = inspect.currentframe()
    args, _, _, values = inspect.getargvalues(frame)

    call_params_string_array = ['function name "%s"' % inspect.getframeinfo(frame)[2]]
    for i in args:
        call_params_string_array.append("    %s = %s" % (i, values[i]))

    # Setup path of logging, name of environment and save the current arguments (this script)
    log_dir = [os.path.join(str(Path.home()), "Desktop"), environment_name + "deep_q_" + experiment_name]
    logger_path = logger_utils.path_with_date(log_dir[0], log_dir[1])

    # Create environment and initialize
    env = gym.make(environment_name)
    env.configure_traci(num_car_chances=num_car_chances,
                        start_car_probabilities=start_car_probabilities,
                        end_car_probabilities=end_car_probabilities,
                        num_steps_from_start_car_probs_to_end_car_probs=num_steps_from_start_car_probs_to_end_car_probs,
                        reward_func=reward_function,
                        action_func=action_function,
                        state_contain_num_cars_in_queue_history=state_use_queue_length,
                        state_contain_avg_speed_between_detectors_history=state_use_avg_speed,
                        state_contain_time_since_tl_change=state_use_time_since_tl_change,
                        state_contain_tl_state_history=state_use_tl_state,
                        num_actions_pr_trafficlight=num_actions_pr_trafficlight,
                        num_history_states=num_history_states,
                        normalize_queue_lengths=normalize_queue_lengths)
    # env.render()

    # Initialize logger
    logger.reset()
    logger.configure(logger_path, ["tensorboard", "stdout"])
    logger.logtxt(call_params_string_array)

    with open(logger_path + "/params.txt", 'w') as file:
        file.write(json.dumps(call_params_string_array, indent=4))
    copyfile(__file__, logger_path + "/script.txt")

    # Create the training model
    model = deepq.models.mlp(hidden_layers, layer_norm=layer_norm)
    act = deepq.learn(
        env=env,
        q_func=model,
        lr=lr,
        max_timesteps=max_timesteps,
        buffer_size=buffer_size,
        exploration_initial_p=exploration_initial_p,
        exploration_fraction=exploration_fraction,
        exploration_final_eps=explore_final_eps,
        train_freq=train_freq,
        batch_size=batch_size,
        print_freq=1,
        checkpoint_freq=checkpoint_freq,
        learning_starts=learning_starts,
        gamma=gamma,
        target_network_update_freq=target_network_update_freq,
        prioritized_replay=prioritized_replay,
        prioritized_replay_alpha=prioritized_replay_alpha,
        prioritized_replay_beta0=prioritized_replay_beta0,
        prioritized_replay_beta_iters=prioritized_replay_beta_iters,
        prioritized_replay_eps=prioritized_replay_eps,
        num_cpu=num_cpu,
        param_noise=param_noise,
        callback=None,
        model_path=logger_path
    )
    save_path = logger_path + "/model-" + datetime.now().strftime('%Y-%m-%d_%H-%M-%S') + ".pkl"
    print("Saving last model to {}".format(save_path))
    act.save(save_path)

    # Run test
    test_environment = gym.make(environment_name)
    test_environment.configure_traci(num_car_chances=num_car_chances,
                                     start_car_probabilities=start_car_probabilities,
                                     enjoy_car_probs=False,
                                     reward_func=reward_function,
                                     action_func=action_function,
                                     state_contain_num_cars_in_queue_history=state_use_queue_length,
                                     state_contain_time_since_tl_change=state_use_time_since_tl_change,
                                     state_contain_tl_state_history=state_use_tl_state,
                                     state_contain_avg_speed_between_detectors_history=state_use_avg_speed,
                                     num_actions_pr_trafficlight=num_actions_pr_trafficlight,
                                     num_history_states=num_history_states)
    test_traffic_deep.test(environment_name=environment_name,
                           path_to_model=save_path,
                           configured_environment=test_environment,
                           act=act,
                           log_dir=logger_path)

def main():
    probabilities = [
        [0.25, 0.05],
        [1.0, 0.10]
    ]

    # Set this by hand!
    experiment_name = "normalize"

    exp_start = 0.2
    exp_end = 0.02

    for pr in probabilities:
        print("Now props:", pr)
        g = tf.Graph()
        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        sess = tf.InteractiveSession(graph=g, config=config)
        with g.as_default():
            train_and_log(start_car_probabilities=pr,
                          experiment_name=experiment_name,
                          exploration_initial_p=exp_start,
                          explore_final_eps=exp_end)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from os.path import join, dirname
from setuptools import setup, find_packages

this_dir = dirname(__file__)

setup(name = 'devilry_subjectadmin',
      description = 'Subject (course) administrator interface for Devilry.',
      version = '1.0',
      license='BSD',
      url = 'https://github.com/devilry/devilry_subjectadmin',
      author = 'Espen Angell Kristiansen',
      packages=find_packages(exclude=['ez_setup']),
      install_requires = ['setuptools', 'Django', 'devilry'],
      include_package_data=True,
      long_description = open(join(this_dir, 'README.rst')).read().strip(),
      zip_safe=False,
      classifiers=[
                   'Development Status :: 5 - Production/Stable',
                   'Environment :: Web Environment',
                   'Framework :: Django',
                   'Intended Audience :: Developers',
                   'License :: OSI Approved',
                   'Operating System :: OS Independent',
                   'Programming Language :: Python'
                  ]
)
<commit_msg>devilry_subjectadmin.setup: Depend on django_seleniumhelpers.<commit_after>from os.path import join, dirname
from setuptools import setup, find_packages

this_dir = dirname(__file__)

setup(name = 'devilry_subjectadmin',
      description = 'Subject (course) administrator interface for Devilry.',
      version = '1.0',
      license='BSD',
      url = 'https://github.com/devilry/devilry_subjectadmin',
      author = 'Espen Angell Kristiansen',
      packages=find_packages(exclude=['ez_setup']),
      install_requires = ['setuptools', 'Django', 'devilry',
                          'django_seleniumhelpers'],
      include_package_data=True,
      long_description = open(join(this_dir, 'README.rst')).read().strip(),
      zip_safe=False,
      classifiers=[
                   'Development Status :: 5 - Production/Stable',
                   'Environment :: Web Environment',
                   'Framework :: Django',
                   'Intended Audience :: Developers',
                   'License :: OSI Approved',
                   'Operating System :: OS Independent',
                   'Programming Language :: Python'
                  ]
)
<|endoftext|>


<commit_before>import numpy as np
from copy import copy
from copy import deepcopy
from itertools import zip_longest
import traceback
from pycqed.utilities.general import temporary_value
from pycqed.measurement.calibration.two_qubit_gates import CalibBuilder
from pycqed.measurement.waveform_control.block import Block, ParametricValue
from pycqed.measurement.waveform_control.segment import UnresolvedPulse
from pycqed.measurement.sweep_points import SweepPoints
from pycqed.analysis import fitting_models as fit_mods
from pycqed.measurement.calibration.calibration_points import CalibrationPoints
import pycqed.measurement.awg_sweep_functions as awg_swf
import pycqed.analysis_v2.timedomain_analysis as tda
from pycqed.instrument_drivers.meta_instrument.qubit_objects.QuDev_transmon \
    import QuDev_transmon
from pycqed.measurement.multi_qubit_module import \
    get_multiplexed_readout_detector_functions
import logging
log = logging.getLogger(__name__)

class T1FrequencySweep(CalibBuilder):
    def __init__(self, dev, task_list=None, sweep_points=None,
                 qubits=None, **kw):
        """
        Flux pulse amplitude measurement used to determine the qubits energy in
        dependence of flux pulse amplitude.

        Timings of sequence

       |          ---|X180|  ------------------------------|RO|
       |          --------| --------- fluxpulse ---------- |

        :param dev: instance of Device class; see CalibBuilder docstring
        :param task_list: list of dicts; see CalibBuilder docstring
        :param sweep_points: SweepPoints class instance with first sweep
            dimension describing the flux pulse lengths and second dimension
            either the flux pulse amplitudes, qubit frequencies, or both.
            !!! If both amplitudes and frequencies are provided, they must be
            be specified in the order amplitudes, frequencies as shown:
            [{'pulse_length': (lengths, 's', 'Flux pulse length')},
             {'flux_pulse_amp': (amps, 'V', 'Flux pulse amplitude'),
              'qubit_freqs': (freqs, 'Hz', 'Qubit frequency')}]
            If this parameter is provided it will be used for all qubits.
        :param qubits: list of QuDev_transmon class instances
        :param kw: keyword arguments
            for_ef (bool, default: False): passed to get_cal_points; see
                docstring there.
            spectator_op_codes (list, default: []): see t1_flux_pulse_block
            all_fits (bool, default: True) passed to run_analysis; see
                docstring there

        Assumptions:
         - assumes there is one task for each qubit. If task_list is None, it
          will internally create it.
         - the entry "qubits_to_measure" in each task should contain one qubit
         name.

        """
        try:
            if task_list is None:
                if sweep_points is None or qubits is None:
                    raise ValueError('Please provide either "sweep_points" '
                                     'and "qubits," or "task_list" containing '
                                     'this information.')
                task_list = [{'qubits_to_measure': qb.name,
                              'sweep_points': sweep_points} for qb in qubits]
            if qubits is None:
                qubits = self.find_qubits_in_tasks(dev.qubits(), task_list)
            super().__init__(dev, qubits=qubits, **kw)

            self.analysis = None
            task_list = self.add_amplitude_sweep_points(task_list)
            self.task_list = task_list
            self.ro_qubits = self.get_ro_qubits()
            self.guess_label()
            self.data_to_fit = {qb.name: 'pe' for qb in self.ro_qubits}
            for_ef = kw.get('for_ef', False)
            kw['for_ef'] = for_ef
            self.create_cal_points(**kw)
            sweep_points = SweepPoints(from_dict_list=[{}, {}])
            self.sequences, sp = \
                self.parallel_sweep(sweep_points, self.task_list,
                                    self.t1_flux_pulse_block, **kw)
            self.hard_sweep_points, self.soft_sweep_points = sp
            self.exp_metadata.update({
                'data_to_fit': self.data_to_fit,
                'global_PCA': len(self.cal_points.states) == 0
            })

            if self.measure:
                # compress the 2nd sweep dimension completely onto the first
                self.run_measurement(
                    compression_seg_lim=np.product([len(s) for s in sp]) +
                                        len(self.cal_points.states), **kw)
            if self.analyze:
                self.run_analysis(**kw)
        except Exception as x:
            self.exception = x
            traceback.print_exc()

    def add_amplitude_sweep_points(self, task_list):
        """
        If flux pulse amplitudes are not in the sweep_points in each task, but
        qubit frequencies are, then amplitudes will be calculated based on
        the frequencies and the fit_ge_freq_from_flux_pulse_amp qubit parameter.
        sweep_points entry in each task_list will be updated.
        :param task_list: list of dictionaries describing the the measurement
            for each qubit.
        :return: updated task list
        """
        for task in task_list:
            this_qb = self.get_qubits(task['qubits_to_measure'])[0][0]
            sweep_points = task['sweep_points']
            if len(sweep_points) == 1:
                raise NotImplementedError('Sweep points must be two-dimensional'
                                          ' with dim 1 over flux pulse lengths,'
                                          ' and dim 2 over qubit frequencies'
                                          ' or flux pulse amplitudes.')
                # add the commented out code below when we know a good default
                # for the sweep points
                # # it must be the 1st sweep dimension, over flux pulse lengths
                # sweep_points = [sweep_points[0], {}]
                # sweep_points = SweepPoints(from_dict_list=sweep_points)

            if sweep_points.get_sweep_params_property('unit', 2).lower() == 'hz':
                fit_paras = deepcopy(this_qb.fit_ge_freq_from_flux_pulse_amp())
                if len(fit_paras) == 0:
                    raise ValueError(f'fit_ge_freq_from_flux_pulse_amp is empty'
                                     f' for {this_qb.name}. Cannot calculate '
                                     f'amplitudes from qubit frequencies.')
                amplitudes = np.array(fit_mods.Qubit_freq_to_dac(
                    sweep_points.get_sweep_params_property('values', 2),
                    **fit_paras))
                if np.any((amplitudes > abs(fit_paras['V_per_phi0']) / 2)):
                    amplitudes -= fit_paras['V_per_phi0']
                elif np.any((amplitudes < -abs(fit_paras['V_per_phi0']) / 2)):
                    amplitudes += fit_paras['V_per_phi0']
                if np.any(np.isnan(amplitudes)):
                    raise ValueError('Specified frequencies resulted in nan '
                                     'amplitude. Check frequency range!')
                amp_sweep_points = SweepPoints('amplitude', amplitudes,
                                               'V', 'Flux pulse amplitude')
                sweep_points.update([{}] + amp_sweep_points)
            task['sweep_points'] = sweep_points
        return task_list

    def t1_flux_pulse_block(self, qubits_to_measure, sweep_points,
                            prepend_pulse_dicts=None, **kw):
        """
        Function that constructs the experiment block for one qubit
        :param qubits_to_measure: name or list with the name of the qubit
            to measure. This function expect only one qubit to measure!
        :param sweep_points: SweepPoints class instance
        :param prepend_pulse_dicts: dictionary of pulses to prepend
        :param kw: keyword arguments
            spectator_op_codes: list of op_codes for spectator qubits
        :return: precompiled block
        """

        qubit_name = qubits_to_measure
        if isinstance(qubit_name, list):
            qubit_name = qubit_name[0]
        hard_sweep_dict, soft_sweep_dict = sweep_points
        pb = self.prepend_pulses_block(prepend_pulse_dicts)

        pulse_modifs = {'all': {'element_name': 'pi_pulse'}}
        pp = self.block_from_ops('pipulse',
                                 [f'X180 {qubit_name}'] +
                                 kw.get('spectator_op_codes', []),
                                 pulse_modifs=pulse_modifs)

        pulse_modifs = {'all': {'element_name': 'flux_pulse', 'pulse_delay': 0}}
        fp = self.block_from_ops('flux', [f'FP {qubit_name}'],
                                 pulse_modifs=pulse_modifs)
        for k in ['channel', 'channel2']:
            if k in fp.pulses[0]:
                if fp.pulses[0][k] not in self.channels_to_upload:
                    self.channels_to_upload.append(fp.pulses[0][k])
        for k in hard_sweep_dict:
            for p in fp.pulses:
                if k in p:
                    p[k] = ParametricValue(k)
        for k in soft_sweep_dict:
            for p in fp.pulses:
                if k in p:
                    p[k] = ParametricValue(k)

        return self.sequential_blocks(f't1 flux pulse {qubit_name}',
                                      [pb, pp, fp])

    def guess_label(self):
        """
        Default measurement label.
        """
        if self.label is None:
            self.label = f'T1_frequency_sweep' \
                         f'{self.dev.get_msmt_suffix(self.ro_qubits)}'

    def run_analysis(self, **kw):
        """
        Runs analysis and stores analysis instance in self.analysis.
        :param kw:
            all_fits (bool, default: True): whether to do all fits
        """

        self.all_fits = kw.get('all_fits', True)
        self.analysis = tda.T1FrequencySweepAnalysis(
            qb_names=[qb.name for qb in self.ro_qubits],
            options_dict=dict(TwoD=False, all_fits=self.all_fits))<commit_msg>T1FrequencySweep: optionally work without qubit objects; fixed combining of sweep_points and sweep points in task_list<commit_after>import numpy as np
from copy import copy
from copy import deepcopy
from itertools import zip_longest
import traceback
from pycqed.utilities.general import temporary_value
from pycqed.measurement.calibration.two_qubit_gates import CalibBuilder
from pycqed.measurement.waveform_control.block import Block, ParametricValue
from pycqed.measurement.waveform_control.segment import UnresolvedPulse
from pycqed.measurement.sweep_points import SweepPoints
from pycqed.analysis import fitting_models as fit_mods
from pycqed.measurement.calibration.calibration_points import CalibrationPoints
import pycqed.measurement.awg_sweep_functions as awg_swf
import pycqed.analysis_v2.timedomain_analysis as tda
from pycqed.instrument_drivers.meta_instrument.qubit_objects.QuDev_transmon \
    import QuDev_transmon
from pycqed.measurement.multi_qubit_module import \
    get_multiplexed_readout_detector_functions
import logging
log = logging.getLogger(__name__)

class T1FrequencySweep(CalibBuilder):
    def __init__(self, task_list=None, sweep_points=None, qubits=None, **kw):
        """
        Flux pulse amplitude measurement used to determine the qubits energy in
        dependence of flux pulse amplitude.

        Timings of sequence

       |          ---|X180|  ------------------------------|RO|
       |          --------| --------- fluxpulse ---------- |

        :param task_list: list of dicts; see CalibBuilder docstring
        :param sweep_points: SweepPoints class instance with first sweep
            dimension describing the flux pulse lengths and second dimension
            either the flux pulse amplitudes, qubit frequencies, or both.
            !!! If both amplitudes and frequencies are provided, they must be
            be specified in the order amplitudes, frequencies as shown:
            [{'pulse_length': (lengths, 's', 'Flux pulse length')},
             {'flux_pulse_amp': (amps, 'V', 'Flux pulse amplitude'),
              'qubit_freqs': (freqs, 'Hz', 'Qubit frequency')}]
            If this parameter is provided it will be used for all qubits.
        :param qubits: list of QuDev_transmon class instances
        :param kw: keyword arguments
            for_ef (bool, default: False): passed to get_cal_points; see
                docstring there.
            spectator_op_codes (list, default: []): see t1_flux_pulse_block
            all_fits (bool, default: True) passed to run_analysis; see
                docstring there

        Assumptions:
         - assumes there is one task for each qubit. If task_list is None, it
          will internally create it.
         - the entry "qubits_to_measure" in each task should contain one qubit
         name.

        """
        try:
            if task_list is None:
                if sweep_points is None or qubits is None:
                    raise ValueError('Please provide either "sweep_points" '
                                     'and "qubits," or "task_list" containing '
                                     'this information.')
                task_list = [{'qubits_to_measure': qb.name,
                              'sweep_points': sweep_points} for qb in qubits]
                # remove sweep_points since they are in the tasks now
                sweep_points = None

            super().__init__(task_list, qubits=qubits, **kw)

            self.analysis = None
            self.data_to_fit = {qb: 'pe' for qb in self.ro_qb_names}
            self.sweep_points = SweepPoints(
                from_dict_list=[{}, {}] if sweep_points is None
                else sweep_points)
            self.add_amplitude_sweep_points()

            self.sequences, self.mc_points = \
                self.parallel_sweep(self.sweep_points, self.task_list,
                                    self.t1_flux_pulse_block, **kw)
            self.exp_metadata.update({
                'global_PCA': len(self.cal_points.states) == 0
            })

            if kw.get('compression_seg_lim', None) is None:
                # compress the 2nd sweep dimension completely onto the first
                kw['compression_seg_lim'] = \
                    np.product([len(s) for s in self.mc_points]) \
                    + len(self.cal_points.states)
            if self.measure:
                self.run_measurement(**kw)
            if self.analyze:
                self.run_analysis(**kw)
        except Exception as x:
            self.exception = x
            traceback.print_exc()

    def add_amplitude_sweep_points(self, task_list=None):
        """
        If flux pulse amplitudes are not in the sweep_points in each task, but
        qubit frequencies are, then amplitudes will be calculated based on
        the frequencies and the fit_ge_freq_from_flux_pulse_amp qubit parameter.
        sweep_points entry in each task_list will be updated.
        :param task_list: list of dictionaries describing the the measurement
            for each qubit.
        :return: updated task list
        """
        if task_list is None:
            task_list = self.task_list
        # TODO: check combination of sweep points in task and in sweep_points
        for task in task_list:
            sweep_points = task.get('sweep_points', [{},{}])
            sweep_points = SweepPoints(from_dict_list=sweep_points)
            if len(sweep_points) == 1:
                sweep_points.add_sweep_dimension()
            if 'qubit_freqs' in sweep_points[1]:
                qubit_freqs = sweep_points[1]['qubit_freqs'][0]
            elif len(self.sweep_points) >= 2 and \
                    'qubit_freqs' in self.sweep_points[1]:
                qubit_freqs = self.sweep_points[1]['qubit_freqs'][0]
            else:
                qubit_freqs = None
            if qubit_freqs is not None:
                qubits, _ = self.get_qubits(task['qubits_to_measure'])
                if qubits is None:
                    raise KeyError('qubit_freqs specified in sweep_points, '
                                   'but no qubit objects available, so that '
                                   'the corresponding amplitudes cannot be '
                                   'computed.')
                this_qb = qubits[0]
                fit_paras = deepcopy(this_qb.fit_ge_freq_from_flux_pulse_amp())
                if len(fit_paras) == 0:
                    raise ValueError(f'fit_ge_freq_from_flux_pulse_amp is empty'
                                     f' for {this_qb.name}. Cannot calculate '
                                     f'amplitudes from qubit frequencies.')
                amplitudes = np.array(fit_mods.Qubit_freq_to_dac(
                    qubit_freqs, **fit_paras))
                if np.any((amplitudes > abs(fit_paras['V_per_phi0']) / 2)):
                    amplitudes -= fit_paras['V_per_phi0']
                elif np.any((amplitudes < -abs(fit_paras['V_per_phi0']) / 2)):
                    amplitudes += fit_paras['V_per_phi0']
                if np.any(np.isnan(amplitudes)):
                    raise ValueError('Specified frequencies resulted in nan '
                                     'amplitude. Check frequency range!')
                amp_sweep_points = SweepPoints('amplitude', amplitudes,
                                               'V', 'Flux pulse amplitude')
                sweep_points.update([{}] + amp_sweep_points)
            task['sweep_points'] = sweep_points
        return task_list

    def t1_flux_pulse_block(self, qubits_to_measure, sweep_points,
                            prepend_pulse_dicts=None, **kw):
        """
        Function that constructs the experiment block for one qubit
        :param qubits_to_measure: name or list with the name of the qubit
            to measure. This function expect only one qubit to measure!
        :param sweep_points: SweepPoints class instance
        :param prepend_pulse_dicts: dictionary of pulses to prepend
        :param kw: keyword arguments
            spectator_op_codes: list of op_codes for spectator qubits
        :return: precompiled block
        """

        qubit_name = qubits_to_measure
        if isinstance(qubit_name, list):
            qubit_name = qubit_name[0]
        hard_sweep_dict, soft_sweep_dict = sweep_points
        pb = self.prepend_pulses_block(prepend_pulse_dicts)

        pulse_modifs = {'all': {'element_name': 'pi_pulse'}}
        pp = self.block_from_ops('pipulse',
                                 [f'X180 {qubit_name}'] +
                                 kw.get('spectator_op_codes', []),
                                 pulse_modifs=pulse_modifs)

        pulse_modifs = {'all': {'element_name': 'flux_pulse', 'pulse_delay': 0}}
        fp = self.block_from_ops('flux', [f'FP {qubit_name}'],
                                 pulse_modifs=pulse_modifs)
        for k in ['channel', 'channel2']:
            if k in fp.pulses[0]:
                if fp.pulses[0][k] not in self.channels_to_upload:
                    self.channels_to_upload.append(fp.pulses[0][k])
        for k in hard_sweep_dict:
            for p in fp.pulses:
                if k in p:
                    p[k] = ParametricValue(k)
        for k in soft_sweep_dict:
            for p in fp.pulses:
                if k in p:
                    p[k] = ParametricValue(k)

        return self.sequential_blocks(f't1 flux pulse {qubit_name}',
                                      [pb, pp, fp])

    def guess_label(self, **kw):
        """
        Default measurement label.
        """
        if kw.get('experiment_name', None) is None:
            kw['experiment_name'] = 'T1_frequency_sweep'
        return super().guess_label(**kw)

    def run_analysis(self, **kw):
        """
        Runs analysis and stores analysis instance in self.analysis.
        :param kw:
            all_fits (bool, default: True): whether to do all fits
        """

        self.all_fits = kw.get('all_fits', True)
        self.analysis = tda.T1FrequencySweepAnalysis(
            qb_names=self.ro_qb_names,
            options_dict=dict(TwoD=False, all_fits=self.all_fits))<|endoftext|>


<commit_before>import time
import socket
import bernhard
from functools import wraps

def riemann_wrapper(client=bernhard.Client(),
                    prefix="python",
                    sep=".",
                    host=None,
                    exception_state='warning',
                    send_exceptions=True,
                    global_tags=['python']):
    """Yield a riemann wrapper with default values for
    the bernhard client, host and prefix"""

    global_client = client
    global_host = host

    def wrap_riemann(metric,
                     client=global_client,
                     host=global_host,
                     tags=[]):

        tags = global_tags + tags

        def send(event):
            if client:
                client.send(event)

        def riemann_decorator(f):
            @wraps(f)
            def decorated_function(*args, **kwargs):

                if host:
                    hostname = host
                else:
                    hostname = socket.gethostname()

                started = time.time()
                metric_name = prefix + sep + metric

                try:
                    response = f(*args, **kwargs)
                except Exception as e:

                    if _call_if_callable(send_exceptions):
                        send({'host': hostname,
                              'service': metric_name + "-exceptions",
                              'description': str(e),
                              'tags': tags + ['exception'],
                              'attributes': {'prefix': prefix},
                              'state': exception_state,
                              'metric': 1})
                    raise

                duration = (time.time() - started) * 1000
                send({'host': hostname,
                      'service': metric_name + "-time",
                      'attributes': {'prefix': prefix},
                      'tags': tags + ['duration'],
                      'metric': duration})
                return response
            return decorated_function
        return riemann_decorator
    return wrap_riemann

# default riemann wrapper
wrap_riemann = riemann_wrapper()

def _call_if_callable(x):
    return x() if callable(x) else x
<commit_msg>Swallow TransportError, log to an optional logger<commit_after>import time
import socket
import bernhard
from functools import wraps

def riemann_wrapper(client=bernhard.Client(),
                    prefix="python",
                    sep=".",
                    host=None,
                    exception_state='warning',
                    send_exceptions=True,
                    global_tags=['python'],
                    logger=None):
    """Yield a riemann wrapper with default values for
    the bernhard client, host and prefix"""

    global_client = client
    global_host = host
    global_logger = logger

    def wrap_riemann(metric,
                     client=global_client,
                     host=global_host,
                     tags=[],
                     logger=global_logger):

        tags = global_tags + tags

        def send(event):
            if client:
                try:
                    client.send(event)
                except bernhard.TransportError:
                    log = _call_if_callable(logger)
                    if log:
                        log.exception('Failed to send Riemann event.')

        def riemann_decorator(f):
            @wraps(f)
            def decorated_function(*args, **kwargs):

                if host:
                    hostname = host
                else:
                    hostname = socket.gethostname()

                started = time.time()
                metric_name = prefix + sep + metric

                try:
                    response = f(*args, **kwargs)
                except Exception as e:

                    if _call_if_callable(send_exceptions):
                        send({'host': hostname,
                              'service': metric_name + "-exceptions",
                              'description': str(e),
                              'tags': tags + ['exception'],
                              'attributes': {'prefix': prefix},
                              'state': exception_state,
                              'metric': 1})
                    raise

                duration = (time.time() - started) * 1000
                send({'host': hostname,
                      'service': metric_name + "-time",
                      'attributes': {'prefix': prefix},
                      'tags': tags + ['duration'],
                      'metric': duration})
                return response
            return decorated_function
        return riemann_decorator
    return wrap_riemann

# default riemann wrapper
wrap_riemann = riemann_wrapper()

def _call_if_callable(x):
    return x() if callable(x) else x
<|endoftext|>


<commit_before># SPDX-License-Identifier: Apache-2.0
# Copyright 2021 Eotvos Lorand University, Budapest, Hungary

compiler_common.current_compilation['is_multicompiled'] = True

part_count = compiler_common.current_compilation['multi']
multi_idx = compiler_common.current_compilation['multi_idx']

table_names = (table.short_name + ("/keyless" if table.key_bit_size == 0 else "") + ("/hidden" if table.is_hidden else "") for table in hlir.tables)
all_table_infos = sorted(zip(hlir.tables, table_names), key=lambda k: len(k[0].actions))
table_infos = list(ti for idx, ti in enumerate(all_table_infos) if idx % part_count == multi_idx)

all_ctl_stages = ((ctl, idx, comp) for ctl in hlir.controls for idx, comp in enumerate(ctl.body.components))
ctl_stages = list(cic for idx, cic in enumerate(all_ctl_stages) if (idx + len(all_table_infos)) % part_count == multi_idx)

if table_infos == [] and ctl_stages == []:
    compiler_common.current_compilation['skip_output'] = True
else:
    from utils.codegen import format_declaration, format_statement, format_expr, format_type, gen_format_type, get_method_call_env
    from compiler_log_warnings_errors import addError, addWarning
    from compiler_common import types, generate_var_name, get_hdrfld_name, unique_everseen

    #[ #include "dataplane_impl.h"
    #[ #include "gen_model.h"

    # TODO make this an import from hardware_indep
    #[ #include "dpdk_smem.h"
    #[

    for table, table_info in table_infos:
        tname = table.name
        #{     apply_result_t ${tname}_apply(STDPARAMS) {
        if 'key' not in table or table.key_bit_size == 0:
            #[         ENTRY(${tname})* entry = ${tname}_get_default_entry(STDPARAMS_IN);
            #[         bool hit = false;
            #[         ${tname}_apply_show_hit(entry->id, STDPARAMS_IN);
        else:
            #{         #ifdef T4P4S_DEBUG
            #[             char key_txt[4096];
            #[             int key_txt_idx = 0;
            #}         #endif

            #[         uint8_t key[${table.key_length_bytes}];
            #[         table_${tname}_key(pd, key  KEYTXTPARAMS_IN);

            #[         ENTRY($tname)* entry = (ENTRY($tname)*)${table.matchType.name}_lookup(tables[TABLE_$tname], key);
            #[         bool hit = entry != NULL && entry->id != INVALID_ACTION;
            #{         if (unlikely(!hit)) {
            #[             entry = ${tname}_get_default_entry(STDPARAMS_IN);
            #}         }

            #{         #ifdef T4P4S_DEBUG
            #[             if (likely(entry != NULL))    ${tname}_apply_show_hit_with_key(hit, entry  KEYTXTPARAM_IN, STDPARAMS_IN);
            #}         #endif

            #{         #ifdef T4P4S_STATS
            #[             t4p4s_stats_global.T4STAT(table,hit,$tname) = hit || t4p4s_stats_global.T4STAT(table,hit,$tname);
            #[             t4p4s_stats_global.T4STAT(table,miss,$tname) = !hit || t4p4s_stats_global.T4STAT(table,miss,$tname);
            #[             t4p4s_stats_per_packet.T4STAT(table,hit,$tname) = hit || t4p4s_stats_per_packet.T4STAT(table,hit,$tname);
            #[             t4p4s_stats_per_packet.T4STAT(table,miss,$tname) = !hit || t4p4s_stats_per_packet.T4STAT(table,miss,$tname);
            #}         #endif

        if len(table.direct_meters + table.direct_counters) > 0:
            #[         if (likely(hit))    ${tname}_apply_smems(STDPARAMS_IN);

        #[         if (unlikely(entry == NULL))    return (apply_result_t) { hit, INVALID_ACTION };

        #[         ${tname}_stats(entry->id, STDPARAMS_IN);

        # ACTIONS
        if len(table.actions) == 1:
            ao = table.actions[0].action_object
            if len(ao.body.components) != 0:
                #[         action_code_${ao.name}(entry->params.${ao.name}_params, SHORT_STDPARAMS_IN);
            #[         return (apply_result_t) { hit, action_${ao.name} };
        else:
            #{         switch (entry->id) {
            for action in table.actions.filter(lambda act: len(act.action_object.body.components) != 0):
                ao = action.action_object
                #{           case action_${ao.name}:
                #[              action_code_${ao.name}(entry->params.${ao.name}_params, SHORT_STDPARAMS_IN);
                #}              return (apply_result_t) { hit, entry->id };
            if len(acts := table.actions.filter(lambda act: len(act.action_object.body.components) == 0)) > 0:
                for act in acts:
                    #[           case action_${act.action_object.name}:
                #]              return (apply_result_t) { hit, entry->id };
            #[           default:
            #[               return (apply_result_t) {}; // unreachable
            #}         }

        #}     }
        #[

    ################################################################################

    for ctl, idx, comp in ctl_stages:
        #{ void control_stage_${ctl.name}_${idx}(control_locals_${ctl.name}_t* local_vars, STDPARAMS) {
        compiler_common.enclosing_control = ctl
        #= format_statement(comp, ctl)
        compiler_common.enclosing_control = None
        #} }
        #[
<commit_msg>Fix table miss (NoAction)<commit_after># SPDX-License-Identifier: Apache-2.0
# Copyright 2021 Eotvos Lorand University, Budapest, Hungary

compiler_common.current_compilation['is_multicompiled'] = True

part_count = compiler_common.current_compilation['multi']
multi_idx = compiler_common.current_compilation['multi_idx']

table_names = (table.short_name + ("/keyless" if table.key_bit_size == 0 else "") + ("/hidden" if table.is_hidden else "") for table in hlir.tables)
all_table_infos = sorted(zip(hlir.tables, table_names), key=lambda k: len(k[0].actions))
table_infos = list(ti for idx, ti in enumerate(all_table_infos) if idx % part_count == multi_idx)

all_ctl_stages = ((ctl, idx, comp) for ctl in hlir.controls for idx, comp in enumerate(ctl.body.components))
ctl_stages = list(cic for idx, cic in enumerate(all_ctl_stages) if (idx + len(all_table_infos)) % part_count == multi_idx)

if table_infos == [] and ctl_stages == []:
    compiler_common.current_compilation['skip_output'] = True
else:
    from utils.codegen import format_declaration, format_statement, format_expr, format_type, gen_format_type, get_method_call_env
    from compiler_log_warnings_errors import addError, addWarning
    from compiler_common import types, generate_var_name, get_hdrfld_name, unique_everseen

    #[ #include "dataplane_impl.h"
    #[ #include "gen_model.h"

    # TODO make this an import from hardware_indep
    #[ #include "dpdk_smem.h"
    #[

    for table, table_info in table_infos:
        tname = table.name
        #{     apply_result_t ${tname}_apply(STDPARAMS) {
        if 'key' not in table or table.key_bit_size == 0:
            #[         ENTRY(${tname})* entry = ${tname}_get_default_entry(STDPARAMS_IN);
            if 'key' not in table:
                #[         bool hit = true; // no key in table ${table.name}
            if table.key_bit_size == 0:
                #[         bool hit = true; // empty key in table ${table.name}
            #[         ${tname}_apply_show_hit(entry->id, STDPARAMS_IN);
        else:
            #{         #ifdef T4P4S_DEBUG
            #[             char key_txt[4096];
            #[             int key_txt_idx = 0;
            #}         #endif

            #[         uint8_t key[${table.key_length_bytes}];
            #[         table_${tname}_key(pd, key  KEYTXTPARAMS_IN);

            #[         ENTRY($tname)* entry = (ENTRY($tname)*)${table.matchType.name}_lookup(tables[TABLE_$tname], key);

            noaction_names = table.actions.map('expression.method.path').filter(lambda p: p.name.startswith('NoAction')).map('name')
            if len(noaction_names) == 0:
                #[         bool hit = true; // lookup on table ${table.name} cannot miss
            elif len(noaction_names) == 1:
                #[         bool hit = entry->id != action_${noaction_names[0]};
            else:
                addError('Finding NoAction', f'Too many NoActions ({len(noaction_names)}) found')

            #{         if (unlikely(!hit)) {
            #[             entry = ${tname}_get_default_entry(STDPARAMS_IN);
            #}         }

            #{         #ifdef T4P4S_DEBUG
            #[             if (likely(entry != NULL))    ${tname}_apply_show_hit_with_key(hit, entry  KEYTXTPARAM_IN, STDPARAMS_IN);
            #}         #endif

            #{         #ifdef T4P4S_STATS
            #[             t4p4s_stats_global.T4STAT(table,hit,$tname) = hit || t4p4s_stats_global.T4STAT(table,hit,$tname);
            #[             t4p4s_stats_global.T4STAT(table,miss,$tname) = !hit || t4p4s_stats_global.T4STAT(table,miss,$tname);
            #[             t4p4s_stats_per_packet.T4STAT(table,hit,$tname) = hit || t4p4s_stats_per_packet.T4STAT(table,hit,$tname);
            #[             t4p4s_stats_per_packet.T4STAT(table,miss,$tname) = !hit || t4p4s_stats_per_packet.T4STAT(table,miss,$tname);
            #}         #endif

        if len(table.direct_meters + table.direct_counters) > 0:
            #[         if (likely(hit))    ${tname}_apply_smems(STDPARAMS_IN);

        #[         if (unlikely(entry == NULL))    return (apply_result_t) { hit, INVALID_ACTION };

        #[         ${tname}_stats(entry->id, STDPARAMS_IN);

        # ACTIONS
        if len(table.actions) == 1:
            ao = table.actions[0].action_object
            if len(ao.body.components) != 0:
                #[         action_code_${ao.name}(entry->params.${ao.name}_params, SHORT_STDPARAMS_IN);
            #[         return (apply_result_t) { hit, action_${ao.name} };
        else:
            #{         switch (entry->id) {
            for action in table.actions.filter(lambda act: len(act.action_object.body.components) != 0):
                ao = action.action_object
                #{           case action_${ao.name}:
                #[              action_code_${ao.name}(entry->params.${ao.name}_params, SHORT_STDPARAMS_IN);
                #}              return (apply_result_t) { hit, entry->id };
            if len(acts := table.actions.filter(lambda act: len(act.action_object.body.components) == 0)) > 0:
                cases = ' '.join(f'case action_{act.action_object.name}:' for act in acts)
                #{           ${cases}
                #[              return (apply_result_t) { hit, entry->id };
                #}
            #[           default: return (apply_result_t) {}; // unreachable
            #}         }

        #}     }
        #[

    ################################################################################

    for ctl, idx, comp in ctl_stages:
        #{ void control_stage_${ctl.name}_${idx}(control_locals_${ctl.name}_t* local_vars, STDPARAMS) {
        compiler_common.enclosing_control = ctl
        #= format_statement(comp, ctl)
        compiler_common.enclosing_control = None
        #} }
        #[
<|endoftext|>