<commit_before>import unittest
from StringIO import StringIO

from robot.parsing.htmlreader import HtmlReader
from robot.utils.asserts import assert_equals

VALID_TABLES = [ "Variable", "Setting", "Test Case", "Test Suite", "Keyword" ]
ROW_TEMPLATE = '
%s	%s	%s
'

class PopulatorMock:

    def __init__(self):
        self.tables = {}
        self.current = None

    def start_table(self, header):
        name = header[0]
        if name in VALID_TABLES:
            self.tables[name] = []
            self.current = name
            return True
        else:
            self.current = None
            return False

    def add(self, cells):
        self.tables[self.current].append(cells)

    def eof(self):
        pass

class TestHtmlReader(unittest.TestCase):

    def setUp(self):
        self.reader = HtmlReader()

    def _read(self, *data):
        self.reader.read(StringIO('\n'.join(data)), PopulatorMock())

    def test_initial_state(self):
        self.reader.state = self.reader.IGNORE
        self._read('
')
        assert_equals(self.reader.state, self.reader.INITIAL)

    def test_empty_table(self):
        self._read('

')
        assert_equals(self.reader.state, self.reader.IGNORE)

    def test_start_valid_table(self):
        for name in VALID_TABLES:
            self._read('
', ROW_TEMPLATE % (name, 'Value 1', 'Value2'))
            assert_equals(self.reader.state, self.reader.PROCESS)
            assert_equals(self.reader.populator.current, name)

    def test_start_invalid_table(self):
        for name in ["Foo", "VariableTable"]:
            self._read('
', ROW_TEMPLATE % (name, 'Value 1', 'Value2'),
                       ROW_TEMPLATE % ('This', 'row', 'is ignored'))
            assert_equals(self.reader.state, self.reader.IGNORE)
            assert_equals(self.reader.populator.current, None)
            assert_equals(self.reader.populator.tables, {})

    def test_br(self):
        inp = ['x
y', '1
2', '

']
        exp = ['x\ny', '1\n2', '\n\n']
        for name in VALID_TABLES:
            self._read('
', ROW_TEMPLATE % (name, 'Value 1', 'Value2'),
                       ROW_TEMPLATE % tuple(inp), '
')
            assert_equals(self.reader.populator.tables[name], [exp])

    def test_processing(self):
        self._row_processing(ROW_TEMPLATE)

    def test_missing_end_td(self):
        self._row_processing('
%s%s	%s
')
        self._row_processing('
%s%s%s
')

    def test_missing_end_tr(self):
        self._row_processing('
%s%s	%s	')

    def test_extra_end_tr(self):
        self._row_processing('
%s%s	%s

')

    def test_missing_start_tr(self):
        self._row_processing('%s%s	%s

')

    def _row_processing(self, row_template):
        row_data = [['Just', 'some', 'data'],
                    ['here', '', 'for'],
                    ['', 'these', 'rows']]
        for name in VALID_TABLES:
            rows = ['
', row_template % (name, 'Value 1', 'Value2')] + \
                [row_template % tuple(row) for row in row_data] + ['
']
            self._read(*rows)
            assert_equals(self.reader.populator.tables[name], row_data)
            assert_equals(self.reader.state, self.reader.IGNORE)

class TestEntityAndCharRefs(unittest.TestCase):

    def test_handle_entityrefs(self):
        for inp, exp in [ ('nbsp', ' '),
                          ('apos', "'"),
                          ('tilde', '~'),
                          ('lt', '<'),
                          ('gt', '>'),
                          ('amp', '&'),
                          ('quot', '"'),
                          ('auml', u'\u00E4'),
                          ('ouml', u'\u00F6'),
                          ('uuml', u'\u00FC'),
                          ('aring', u'\u00E5'),
                          ('ntilde', u'\u00F1'),
                          ('Auml', u'\u00C4'),
                          ('Ouml', u'\u00D6'),
                          ('Uuml', u'\u00DC'),
                          ('Aring', u'\u00C5'),
                          ('Ntilde', u'\u00D1'),
                          ('nabla', u'\u2207'),
                          ('ldquo', u'\u201c'),
                          ('invalid', '&invalid;') ]:
            self._test('&%s;' % inp, exp)

    def test_handle_charrefs(self):
        for inp, exp in [ ('82', 'R'),
                          ('228', u'\u00E4'),
                          ('invalid', '&#invalid;') ]:
            self._test('&#%s;' % inp, exp)

    def _test(self, input, expected):
        result = []
        def collect_data(data, decode):
            result.append(data)
        reader = HtmlReader()
        reader.data = collect_data
        reader.read(StringIO(input), PopulatorMock())
        assert_equals(''.join(result), expected)

class TestEncoding(unittest.TestCase):

    def test_default_encoding(self):
        assert_equals(HtmlReader()._encoding, 'ISO-8859-1')

    def test_encoding_is_read_from_meta_tag(self):
        self._test_encoding('<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />', 'utf-8')
        self._test_encoding('<META HTTP-EQUIV="CONTENT-TYPE" CONTENT="TEXT/HTML; CHARSET=UTF-8">', 'UTF-8')

    def _test_encoding(self, data, expected):
        reader = HtmlReader()
        reader.read(StringIO(data), PopulatorMock())
        assert_equals(reader._encoding, expected)
        return reader

    def test_valid_http_equiv_is_required_in_meta(self):
        self._test_encoding('<meta content="text/html; charset=utf-8" />', 'ISO-8859-1')
        self._test_encoding('<meta http-equiv="Invalid" content="text/html; charset=utf-8" />', 'ISO-8859-1')

    def test_encoding_is_read_from_pi(self):
        self._test_encoding('<?xml version="1.0" encoding="UTF-8"?>', 'UTF-8')
        self._test_encoding('<?xml encoding=US-ASCII version="1.0"?>', 'US-ASCII')

    def test_encoding_and_entityrefs(self):
        reader = self._test_encoding('''<meta content="text/html; charset=utf-8"
                                        http-equiv="Content-Type"/>

Setting

äiti
''', 'utf-8')
        assert_equals(reader.populator.tables['Setting'][0], [u'\xe4iti'])

if __name__ == '__main__':
    unittest.main()
<commit_msg>Unit tests for parsing HTML w/ lxml.<commit_after>import unittest
from StringIO import StringIO

from robot.parsing.htmlreader import HtmlReader
from robot.parsing.stdhtmlparser import RobotHtmlParser as StdHtmlParser
try:
    from robot.parsing.lxmlhtmlparser import RobotHtmlParser as LxmlHtmlParser
except ImportError:
    def LxmlHtmlParser(*args):
        raise RuntimeError('This test requires lxml module to be installed')

from robot.utils.asserts import assert_equals

VALID_TABLES = [ "Variable", "Setting", "Test Case", "Test Suite", "Keyword" ]
ROW_TEMPLATE = '
%s	%s	%s
'

class PopulatorMock:

    def __init__(self):
        self.tables = {}
        self.current = None

    def start_table(self, header):
        name = header[0]
        if name in VALID_TABLES:
            self.tables[name] = []
            self.current = name
            return True
        else:
            self.current = None
            return False

    def add(self, cells):
        self.tables[self.current].append(cells)

    def eof(self):
        pass

class TestHtmlReaderWithStdHtmlParser(unittest.TestCase):
    parser = StdHtmlParser

    def setUp(self):
        self.reader = HtmlReader(parser=self.parser)

    def _read(self, *data):
        self.reader.read(StringIO('\n'.join(data)), PopulatorMock())

    def test_empty_table(self):
        self._read('

')
        assert_equals(self.reader.state, self.reader.IGNORE)

    def test_start_valid_table(self):
        for name in VALID_TABLES:
            self._read('
', ROW_TEMPLATE % (name, 'Value 1', 'Value2'))
            assert_equals(self.reader.populator.current, name)

    def test_start_invalid_table(self):
        for name in ["Foo", "VariableTable"]:
            self._read('
', ROW_TEMPLATE % (name, 'Value 1', 'Value2'),
                       ROW_TEMPLATE % ('This', 'row', 'is ignored'))
            assert_equals(self.reader.state, self.reader.IGNORE)
            assert_equals(self.reader.populator.current, None)
            assert_equals(self.reader.populator.tables, {})

    def test_br(self):
        inp = ['x
y', '1
2', '

']
        exp = ['x\ny', '1\n2', '\n\n']
        for name in VALID_TABLES:
            self._read('
', ROW_TEMPLATE % (name, 'Value 1', 'Value2'),
                       ROW_TEMPLATE % tuple(inp), '
')
            assert_equals(self.reader.populator.tables[name], [exp])

    def test_comment(self):
        self._read('
', ROW_TEMPLATE % ('Setting', 'Value 1', 'Value2'),
                   '', ROW_TEMPLATE % tuple('ABC'),
                   ROW_TEMPLATE % tuple('123'), '
')
        assert_equals(self.reader.populator.tables['Setting'],
                      [['A', 'B', 'C'], ['1', '2', '3']])

    def test_processing(self):
        self._row_processing(ROW_TEMPLATE)

    def test_missing_end_td(self):
        self._row_processing('
%s%s	%s
')
        self._row_processing('
%s%s%s
')

    def test_missing_end_tr(self):
        self._row_processing('
%s%s	%s	')

    def test_extra_end_tr(self):
        self._row_processing('
%s%s	%s

')

    def test_missing_start_tr(self):
        self._row_processing('%s%s	%s
')

    def _row_processing(self, row_template):
        row_data = [['Just', 'some', 'data'],
                    ['here', '', 'for'],
                    ['', 'these', 'rows']]
        for name in VALID_TABLES:
            rows = ['
', row_template % (name, 'Value 1', 'Value2')] + \
                [row_template % tuple(row) for row in row_data] + ['
']
            self._read(*rows)
            assert_equals(self.reader.populator.tables[name], row_data)
            assert_equals(self.reader.state, self.reader.IGNORE)

class TestHtmlReaderWithLxmlParser(TestHtmlReaderWithStdHtmlParser):
    parser=LxmlHtmlParser

    def test_missing_start_tr(self):
        # lxml doens't handle this: it ignores also
 if there's no

        pass

class TestEntityAndCharRefsWithStdHtmlParser(unittest.TestCase):
    parser = StdHtmlParser

    def test_entityrefs(self):
        for inp, exp in [('nbsp', ' '),
                         ('apos', "'"),
                         ('tilde', '~'),
                         ('lt', '<'),
                         ('gt', '>'),
                         ('amp', '&'),
                         ('quot', '"'),
                         ('auml', u'\u00E4'),
                         ('ouml', u'\u00F6'),
                         ('uuml', u'\u00FC'),
                         ('aring', u'\u00E5'),
                         ('ntilde', u'\u00F1'),
                         ('Auml', u'\u00C4'),
                         ('Ouml', u'\u00D6'),
                         ('Uuml', u'\u00DC'),
                         ('Aring', u'\u00C5'),
                         ('Ntilde', u'\u00D1'),
                         ('nabla', u'\u2207'),
                         ('ldquo', u'\u201c'),
                         ('invalid', '&invalid;')]:
            self._test('&%s;' % inp, exp)

    def test_charrefs(self):
        for inp, exp in [('82', 'R'), ('228', u'\u00E4')]:
            self._test('&#%s;' % inp, exp)

    def test_invalid_charref(self):
        self._test('&#invalid;', '&#invalid;')

    def _test(self, input, expected):
        result = []
        def collect_data(data):
            result.append(data)
        reader = HtmlReader(parser=self.parser)
        reader.data = collect_data
        reader.read(StringIO(input), PopulatorMock())
        msg = "'%s': %r != %r" % (input, ''.join(result), expected)
        assert_equals(''.join(result), expected, msg, values=False)

class TestEntityAndCharRefsWithLxmlParser(TestEntityAndCharRefsWithStdHtmlParser):
    parser = LxmlHtmlParser

    def test_invalid_charref(self):
        self._test('&#invalid;', 'invalid;')

class TestEncodingWithStdHtmlParser(unittest.TestCase):

    def test_default_encoding(self):
        assert_equals(StdHtmlParser(reader=None)._encoding, 'ISO-8859-1')

    def test_encoding_is_read_from_meta_tag(self):
        self._test_encoding('<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />', 'utf-8')
        self._test_encoding('<META HTTP-EQUIV="CONTENT-TYPE" CONTENT="TEXT/HTML; CHARSET=UTF-8">', 'UTF-8')

    def test_valid_http_equiv_is_required_in_meta(self):
        self._test_encoding('<meta content="text/html; charset=utf-8" />', 'ISO-8859-1')
        self._test_encoding('<meta http-equiv="Invalid" content="text/html; charset=utf-8" />', 'ISO-8859-1')

    def test_encoding_is_read_from_pi(self):
        self._test_encoding('<?xml version="1.0" encoding="UTF-8"?>', 'UTF-8')
        self._test_encoding('<?xml encoding=US-ASCII version="1.0"?>', 'US-ASCII')

    def _test_encoding(self, data, expected):
        parser = StdHtmlParser(reader=HtmlReader())
        parser.parse(StringIO(data))
        assert_equals(parser._encoding, expected)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright lowRISC contributors.
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
# SPDX-License-Identifier: Apache-2.0
r"""This script generates sparse FSM encodings that fulfill a minimum
Hamming distance requirement. It uses a heuristic that incrementally
draws random state encodings until a solution has been found.

Depending on the parameterization, the script may not find a solution right
away. In such cases, the script should be rerun after tweaking the d/m/n
parameters. E.g. in order to increase the chances for success, the state
space can be made more sparse by increasing n, or the Hamming distance
threshold d can be lowered.

Note however that the Hamming distance d should be set to 3 at minimum.
It is recommended to set this value to 4-5 for security critical FSMs.

The custom seed s can be used to make subsequent runs of the script
deterministic. If not specified, the script randomly picks a seed.

"""
import argparse
import logging
import math
import random
import sys

from lib.common import get_hd, hd_histogram, wrapped_docstring

MAX_DRAWS = 10000
MAX_RESTARTS = 10000

SV_INSTRUCTIONS = """
------------------------------------------------------
| COPY PASTE THE CODE TEMPLATE BELOW INTO YOUR RTL   |
| IMPLEMENTATION, INLUDING THE COMMENT AND PRIM_FLOP |
| IN ORDER TO EASE AUDITABILITY AND REPRODUCIBILITY. |
------------------------------------------------------
"""

C_INSTRUCTIONS = """
------------------------------------------------
| COPY PASTE THE CODE TEMPLATE BELOW INTO YOUR |
| C HEADER, INLUDING THE COMMENT IN ORDER TO   |
| EASE AUDITABILITY AND REPRODUCIBILITY.       |
------------------------------------------------
"""

RUST_INSTRUCTIONS = """
------------------------------------------------
| COPY PASTE THE CODE TEMPLATE BELOW INTO YOUR |
| RUST FILE, INLUDING THE COMMENT IN ORDER TO  |
| EASE AUDITABILITY AND REPRODUCIBILITY.       |
------------------------------------------------
"""

def main():
    log.basicConfig(level=log.INFO,
                    format="%(levelname)s: %(message)s")

    parser = argparse.ArgumentParser(
        prog="sparse-fsm-encode",
        description=wrapped_docstring(),
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument(
        '-d',
        type=int,
        default=5,
        metavar='<minimum HD>',
        help='Minimum Hamming distance between encoded states.')
    parser.add_argument('-m',
                        type=int,
                        default=7,
                        metavar='<#states>',
                        help='Number of states to encode.')
    parser.add_argument('-n',
                        type=int,
                        default=10,
                        metavar='<#nbits>',
                        help='Encoding length [bit].')
    parser.add_argument('-s',
                        type=int,
                        metavar='<seed>',
                        help='Custom seed for RNG.')
    parser.add_argument('--language',
                        choices=['sv', 'c', 'rust'],
                        default='sv',
                        help='Choose the language of the generated enum.')

    args = parser.parse_args()

    if args.language in ['c', 'rust']:
        if args.n not in [8, 16, 32]:
            logging.error(
                "When using C or Rust, widths must be a power-of-two "
                "at least a byte (8 bits) wide. You chose %d." % (args.n, ))
            sys.exit(1)

    if args.m > 2**args.n:
        logging.error(
            'Statespace 2^%d not large enough to accommodate %d states.' %
            (args.n, args.m))
        sys.exit(1)

    if args.d >= args.n:
        logging.error(
            'State is only %d bits wide, which is not enough to fulfill a '
            'minimum Hamming distance constraint of %d. ' % (args.n, args.d))
        sys.exit(1)

    if args.d <= 0:
        logging.error('Hamming distance must be > 0.')
        sys.exit(1)

    if args.d < 3:
        logging.warning(
            'A value of 4-5 is recommended for the minimum Hamming distance '
            'constraint. At a minimum, this should be set to 3.')

    # If no seed has been provided, we choose a seed and print it
    # into the generated output later on such that this run can be
    # reproduced.
    if args.s is None:
        random.seed()
        args.s = random.getrandbits(32)

    random.seed(args.s)

    # This is a heuristic that opportunistically draws random
    # state encodings and check whether they fulfill the minimum
    # Hamming distance constraint.
    # Other solutions that use a brute-force approach would be
    # possible as well (see e.g. https://math.stackexchange.com/
    # questions/891528/generating-a-binary-code-with-maximized-hamming-distance).
    # However, due to the sparse nature of the state space, this
    # probabilistic heuristic works pretty well for most practical
    # cases, and it scales favorably to large N.
    num_draws = 0
    num_restarts = 0
    rnd = random.getrandbits(args.n)
    encodings = [format(rnd, '0' + str(args.n) + 'b')]
    while len(encodings) < args.m:
        # if we iterate for too long, start over.
        if num_draws >= MAX_DRAWS:
            num_draws = 0
            num_restarts += 1
            rnd = random.getrandbits(args.n)
            encodings = [format(rnd, '0' + str(args.n) + 'b')]
        # if we restarted for too many times, abort.
        if num_restarts >= MAX_RESTARTS:
            logging.error(
                'Did not find a solution after restarting {} times. This is '
                'an indicator that not many (or even no) solutions exist for '
                'the current parameterization. Rerun the script and/or adjust '
                'the d/m/n parameters. E.g. make the state space more sparse by '
                'increasing n, or lower the minimum Hamming distance threshold d.'
                .format(num_restarts))
            sys.exit(1)
        num_draws += 1
        # draw a candidate and check whether it fulfills the minimum
        # distance requirement with respect to other encodings.
        rnd = random.getrandbits(args.n)
        cand = format(rnd, '0' + str(args.n) + 'b')
        # disallow all-zero and all-one states
        pop_cnt = cand.count('1')
        if pop_cnt < args.n and pop_cnt > 0:
            for k in encodings:
                # disallow candidates that are the complement of other states
                if int(cand, 2) == ~int(k, 2):
                    break
                # disallow candidates that are too close to other states
                if get_hd(cand, k) < args.d:
                    break
            else:
                encodings.append(cand)

    # Get Hamming distance statistics.
    stats = hd_histogram(encodings)

    if args.language == "sv":
        print(SV_INSTRUCTIONS)
        print("// Encoding generated with:\n"
              "// $ ./util/design/sparse-fsm-encode.py -d {} -m {} -n {} \\\n"
              "//      -s {} --language=sv\n"
              "//\n"
              "// Hamming distance histogram:\n"
              "//".format(args.d, args.m, args.n, args.s))
        for bar in stats['bars']:
            print('// ' + bar)
        print("//\n"
              "// Minimum Hamming distance: {}\n"
              "// Maximum Hamming distance: {}\n"
              "// Minimum Hamming weight: {}\n"
              "// Maximum Hamming weight: {}\n"
              "//\n"
              "localparam int StateWidth = {};\n"
              "typedef enum logic [StateWidth-1:0] {{".format(
                  stats['min_hd'], stats['max_hd'], stats['min_hw'],
                  stats['max_hw'], args.n))
        fmt_str = "  State{} {}= {}'b{}"
        state_str = ""
        for j, k in enumerate(encodings):
            pad = ""
            for i in range(len(str(args.m)) - len(str(j))):
                pad += " "
            comma = "," if j < len(encodings) - 1 else ""
            print(fmt_str.format(j, pad, args.n, k) + comma)
            state_str += "    State{}: ;\n".format(j)

        # print FSM template
        print('''}} state_e;

state_e state_d, state_q;

always_comb begin : p_fsm
  // Default assignments
  state_d = state_q;

  unique case (state_q)
{}    default: ; // Consider triggering an error or alert in this case.
  endcase
end

// This primitive is used to place a size-only constraint on the
// flops in order to prevent FSM state encoding optimizations.
logic [StateWidth-1:0] state_raw_q;
assign state_q = state_e'(state_raw_q);
prim_flop #(
  .Width(StateWidth),
  .ResetValue(StateWidth'(State0))
) u_state_regs (
  .clk_i,
  .rst_ni,
  .d_i ( state_d     ),
  .q_o ( state_raw_q )
);
'''.format(state_str))

    elif args.language == "c":
        print(C_INSTRUCTIONS)
        print("/*\n"
              " * Encoding generated with\n"
              " * $ ./util/design/sparse-fsm-encode.py -d {} -m {} -n {} \\\n"
              " *     -s {} --language=c\n"
              " *\n"
              " * Hamming distance histogram:\n"
              " *".format(args.d, args.m, args.n, args.s))
        for hist_bar in stats['bars']:
            print(" * " + hist_bar)
        print(" *\n"
              " * Minimum Hamming distance: {}\n"
              " * Maximum Hamming distance: {}\n"
              " * Minimum Hamming weight: {}\n"
              " * Maximum Hamming weight: {}\n"
              " */\n"
              "typedef enum my_state {{".format(stats['min_hd'],
                                                stats['max_hd'],
                                                stats['min_hw'],
                                                stats['max_hw']))
        fmt_str = "  kMyState{0:} {1:}= 0x{3:0" + str(math.ceil(
            args.n / 4)) + "x}"
        for j, k in enumerate(encodings):
            pad = ""
            for i in range(len(str(args.m)) - len(str(j))):
                pad += " "
            print(fmt_str.format(j, pad, args.n, int(k, 2)) + ",")

        # print FSM template
        print("} my_state_t;")
    elif args.language == 'rust':
        print(RUST_INSTRUCTIONS)
        print("///```text\n"
              "/// Encoding generated with\n"
              "/// $ ./util/design/sparse-fsm-encode.py -d {} -m {} -n {} \\\n"
              "///     -s {} --language=rust\n"
              "///\n"
              "/// Hamming distance histogram:\n"
              "///".format(args.d, args.m, args.n, args.s))
        for hist_bar in stats['bars']:
            print("/// " + hist_bar)
        print("///\n"
              "/// Minimum Hamming distance: {}\n"
              "/// Maximum Hamming distance: {}\n"
              "/// Minimum Hamming weight: {}\n"
              "/// Maximum Hamming weight: {}\n"
              "///```\n"
              "#[derive(Clone,Copy,Eq,PartialEq,Ord,ParitalOrd,Hash,Debug)]\n"
              "#[repr(transparent)]\n"
              "struct MyState(u{});\n"
              "\n"
              "impl MyState {{".format(stats['min_hd'], stats['max_hd'],
                                       stats['min_hw'], stats['max_hw'],
                                       args.n))
        fmt_str = "  const MY_STATE{0:}: MyState {1:}= MyState(0x{3:0" + str(
            math.ceil(args.n / 4)) + "x})"
        for j, k in enumerate(encodings):
            pad = ""
            for i in range(len(str(args.m)) - len(str(j))):
                pad += " "
            print(fmt_str.format(j, pad, args.n, int(k, 2)) + ";")
        print("}")

if __name__ == "__main__":
    main()
<commit_msg>[util/design] Minor fix in sparse FSM encoding script<commit_after>#!/usr/bin/env python3
# Copyright lowRISC contributors.
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
# SPDX-License-Identifier: Apache-2.0
r"""This script generates sparse FSM encodings that fulfill a minimum
Hamming distance requirement. It uses a heuristic that incrementally
draws random state encodings until a solution has been found.

Depending on the parameterization, the script may not find a solution right
away. In such cases, the script should be rerun after tweaking the d/m/n
parameters. E.g. in order to increase the chances for success, the state
space can be made more sparse by increasing n, or the Hamming distance
threshold d can be lowered.

Note however that the Hamming distance d should be set to 3 at minimum.
It is recommended to set this value to 4-5 for security critical FSMs.

The custom seed s can be used to make subsequent runs of the script
deterministic. If not specified, the script randomly picks a seed.

"""
import argparse
import logging as log
import math
import random
import sys

from lib.common import get_hd, hd_histogram, wrapped_docstring

MAX_DRAWS = 10000
MAX_RESTARTS = 10000

SV_INSTRUCTIONS = """
------------------------------------------------------
| COPY PASTE THE CODE TEMPLATE BELOW INTO YOUR RTL   |
| IMPLEMENTATION, INLUDING THE COMMENT AND PRIM_FLOP |
| IN ORDER TO EASE AUDITABILITY AND REPRODUCIBILITY. |
------------------------------------------------------
"""

C_INSTRUCTIONS = """
------------------------------------------------
| COPY PASTE THE CODE TEMPLATE BELOW INTO YOUR |
| C HEADER, INLUDING THE COMMENT IN ORDER TO   |
| EASE AUDITABILITY AND REPRODUCIBILITY.       |
------------------------------------------------
"""

RUST_INSTRUCTIONS = """
------------------------------------------------
| COPY PASTE THE CODE TEMPLATE BELOW INTO YOUR |
| RUST FILE, INLUDING THE COMMENT IN ORDER TO  |
| EASE AUDITABILITY AND REPRODUCIBILITY.       |
------------------------------------------------
"""

def main():
    log.basicConfig(level=log.INFO,
                    format="%(levelname)s: %(message)s")

    parser = argparse.ArgumentParser(
        prog="sparse-fsm-encode",
        description=wrapped_docstring(),
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument(
        '-d',
        type=int,
        default=5,
        metavar='<minimum HD>',
        help='Minimum Hamming distance between encoded states.')
    parser.add_argument('-m',
                        type=int,
                        default=7,
                        metavar='<#states>',
                        help='Number of states to encode.')
    parser.add_argument('-n',
                        type=int,
                        default=10,
                        metavar='<#nbits>',
                        help='Encoding length [bit].')
    parser.add_argument('-s',
                        type=int,
                        metavar='<seed>',
                        help='Custom seed for RNG.')
    parser.add_argument('--language',
                        choices=['sv', 'c', 'rust'],
                        default='sv',
                        help='Choose the language of the generated enum.')

    args = parser.parse_args()

    if args.language in ['c', 'rust']:
        if args.n not in [8, 16, 32]:
            log.error(
                "When using C or Rust, widths must be a power-of-two "
                "at least a byte (8 bits) wide. You chose %d." % (args.n, ))
            sys.exit(1)

    if args.m > 2**args.n:
        log.error(
            'Statespace 2^%d not large enough to accommodate %d states.' %
            (args.n, args.m))
        sys.exit(1)

    if args.d >= args.n:
        log.error(
            'State is only %d bits wide, which is not enough to fulfill a '
            'minimum Hamming distance constraint of %d. ' % (args.n, args.d))
        sys.exit(1)

    if args.d <= 0:
        log.error('Hamming distance must be > 0.')
        sys.exit(1)

    if args.d < 3:
        log.warning(
            'A value of 4-5 is recommended for the minimum Hamming distance '
            'constraint. At a minimum, this should be set to 3.')

    # If no seed has been provided, we choose a seed and print it
    # into the generated output later on such that this run can be
    # reproduced.
    if args.s is None:
        random.seed()
        args.s = random.getrandbits(32)

    random.seed(args.s)

    # This is a heuristic that opportunistically draws random
    # state encodings and check whether they fulfill the minimum
    # Hamming distance constraint.
    # Other solutions that use a brute-force approach would be
    # possible as well (see e.g. https://math.stackexchange.com/
    # questions/891528/generating-a-binary-code-with-maximized-hamming-distance).
    # However, due to the sparse nature of the state space, this
    # probabilistic heuristic works pretty well for most practical
    # cases, and it scales favorably to large N.
    num_draws = 0
    num_restarts = 0
    rnd = random.getrandbits(args.n)
    encodings = [format(rnd, '0' + str(args.n) + 'b')]
    while len(encodings) < args.m:
        # if we iterate for too long, start over.
        if num_draws >= MAX_DRAWS:
            num_draws = 0
            num_restarts += 1
            rnd = random.getrandbits(args.n)
            encodings = [format(rnd, '0' + str(args.n) + 'b')]
        # if we restarted for too many times, abort.
        if num_restarts >= MAX_RESTARTS:
            log.error(
                'Did not find a solution after restarting {} times. This is '
                'an indicator that not many (or even no) solutions exist for '
                'the current parameterization. Rerun the script and/or adjust '
                'the d/m/n parameters. E.g. make the state space more sparse by '
                'increasing n, or lower the minimum Hamming distance threshold d.'
                .format(num_restarts))
            sys.exit(1)
        num_draws += 1
        # draw a candidate and check whether it fulfills the minimum
        # distance requirement with respect to other encodings.
        rnd = random.getrandbits(args.n)
        cand = format(rnd, '0' + str(args.n) + 'b')
        # disallow all-zero and all-one states
        pop_cnt = cand.count('1')
        if pop_cnt < args.n and pop_cnt > 0:
            for k in encodings:
                # disallow candidates that are the complement of other states
                if int(cand, 2) == ~int(k, 2):
                    break
                # disallow candidates that are too close to other states
                if get_hd(cand, k) < args.d:
                    break
            else:
                encodings.append(cand)

    # Get Hamming distance statistics.
    stats = hd_histogram(encodings)

    if args.language == "sv":
        print(SV_INSTRUCTIONS)
        print("// Encoding generated with:\n"
              "// $ ./util/design/sparse-fsm-encode.py -d {} -m {} -n {} \\\n"
              "//      -s {} --language=sv\n"
              "//\n"
              "// Hamming distance histogram:\n"
              "//".format(args.d, args.m, args.n, args.s))
        for bar in stats['bars']:
            print('// ' + bar)
        print("//\n"
              "// Minimum Hamming distance: {}\n"
              "// Maximum Hamming distance: {}\n"
              "// Minimum Hamming weight: {}\n"
              "// Maximum Hamming weight: {}\n"
              "//\n"
              "localparam int StateWidth = {};\n"
              "typedef enum logic [StateWidth-1:0] {{".format(
                  stats['min_hd'], stats['max_hd'], stats['min_hw'],
                  stats['max_hw'], args.n))
        fmt_str = "  State{} {}= {}'b{}"
        state_str = ""
        for j, k in enumerate(encodings):
            pad = ""
            for i in range(len(str(args.m)) - len(str(j))):
                pad += " "
            comma = "," if j < len(encodings) - 1 else ""
            print(fmt_str.format(j, pad, args.n, k) + comma)
            state_str += "    State{}: ;\n".format(j)

        # print FSM template
        print('''}} state_e;

state_e state_d, state_q;

always_comb begin : p_fsm
  // Default assignments
  state_d = state_q;

  unique case (state_q)
{}    default: ; // Consider triggering an error or alert in this case.
  endcase
end

// This primitive is used to place a size-only constraint on the
// flops in order to prevent FSM state encoding optimizations.
logic [StateWidth-1:0] state_raw_q;
assign state_q = state_e'(state_raw_q);
prim_flop #(
  .Width(StateWidth),
  .ResetValue(StateWidth'(State0))
) u_state_regs (
  .clk_i,
  .rst_ni,
  .d_i ( state_d     ),
  .q_o ( state_raw_q )
);
'''.format(state_str))

    elif args.language == "c":
        print(C_INSTRUCTIONS)
        print("/*\n"
              " * Encoding generated with\n"
              " * $ ./util/design/sparse-fsm-encode.py -d {} -m {} -n {} \\\n"
              " *     -s {} --language=c\n"
              " *\n"
              " * Hamming distance histogram:\n"
              " *".format(args.d, args.m, args.n, args.s))
        for hist_bar in stats['bars']:
            print(" * " + hist_bar)
        print(" *\n"
              " * Minimum Hamming distance: {}\n"
              " * Maximum Hamming distance: {}\n"
              " * Minimum Hamming weight: {}\n"
              " * Maximum Hamming weight: {}\n"
              " */\n"
              "typedef enum my_state {{".format(stats['min_hd'],
                                                stats['max_hd'],
                                                stats['min_hw'],
                                                stats['max_hw']))
        fmt_str = "  kMyState{0:} {1:}= 0x{3:0" + str(math.ceil(
            args.n / 4)) + "x}"
        for j, k in enumerate(encodings):
            pad = ""
            for i in range(len(str(args.m)) - len(str(j))):
                pad += " "
            print(fmt_str.format(j, pad, args.n, int(k, 2)) + ",")

        # print FSM template
        print("} my_state_t;")
    elif args.language == 'rust':
        print(RUST_INSTRUCTIONS)
        print("///```text\n"
              "/// Encoding generated with\n"
              "/// $ ./util/design/sparse-fsm-encode.py -d {} -m {} -n {} \\\n"
              "///     -s {} --language=rust\n"
              "///\n"
              "/// Hamming distance histogram:\n"
              "///".format(args.d, args.m, args.n, args.s))
        for hist_bar in stats['bars']:
            print("/// " + hist_bar)
        print("///\n"
              "/// Minimum Hamming distance: {}\n"
              "/// Maximum Hamming distance: {}\n"
              "/// Minimum Hamming weight: {}\n"
              "/// Maximum Hamming weight: {}\n"
              "///```\n"
              "#[derive(Clone,Copy,Eq,PartialEq,Ord,ParitalOrd,Hash,Debug)]\n"
              "#[repr(transparent)]\n"
              "struct MyState(u{});\n"
              "\n"
              "impl MyState {{".format(stats['min_hd'], stats['max_hd'],
                                       stats['min_hw'], stats['max_hw'],
                                       args.n))
        fmt_str = "  const MY_STATE{0:}: MyState {1:}= MyState(0x{3:0" + str(
            math.ceil(args.n / 4)) + "x})"
        for j, k in enumerate(encodings):
            pad = ""
            for i in range(len(str(args.m)) - len(str(j))):
                pad += " "
            print(fmt_str.format(j, pad, args.n, int(k, 2)) + ";")
        print("}")

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from flask import current_app as app
from flask.ext.wtf import AnyOf, Required
from app.hermes.models import EventType, Commodity, CommodityType, Exchange
from app.hermes.models import DataSource
from app.cronus.models import Transaction

# For use with Connection
def portify(site):
	site = site.split('/')

	if site[2] == 'localhost':
		site[2] = 'localhost:%s' % app.config['PORT']

	return '/'.join(site)

# For flask-script
def get_init_values():
	values = [
		[
			('NYSE', 'New York Stock Exchange'), ('NASDAQ', 'NASDAQ'),
			('OTC', 'Over the counter'), ('N/A', 'Currency')],  # exchange
		[[('Yahoo')], [('Google')], [('XE')]],  # data_source
		[[('Security')], [('Currency')], [('Other')]],  # commodity_group
		[
			('Stock', 1), ('Bond', 1), ('Mutual Fund', 1), ('ETF', 1),
			('Currency', 2), ('Descriptor', 3)],  # commodity_type
		[
			('USD', 'US Dollar', 5, 3, 4),
			('EUR', 'Euro', 5, 3, 4),
			('GBP', 'Pound Sterling', 5, 3, 4),
			('TZS', 'Tanzanian Shilling', 5, 3, 4),
			('Multiple', 'Multiple', 6, 3, 4),
			('APL', 'Apple', 1, 1, 1),
			('Text', 'Text', 6, 3, 4)],  # commodity
		[
			[('Dividend')], [('Special Dividend')], [('Stock Split')],
			[('Name Change')], [('Ticker Change')]],  # event_type
		[],  # event
		[
			(2, 1, 1.2, '1/1/12'),
			(3, 1, 1.8, '1/1/12'),
			(4, 1, 1.0 / 1580.0, '1/1/12'),
			(6, 1, 300, '1/1/12')],  # price
		[(1, 'Reuben', 'Cummings', 'reubano@gmail.com')],  # person
		[
			('Scottrade', 'https://trading.scottrade.com/'),
			('Vanguard', 'http://vanguard.com/')],  # company
		[[('Brokerage')], [('Roth IRA')]],  # account_type
		[(1, 1, 1, 1, 'Scottrade'), (2, 2, 1, 1, 'Vanguard IRA')],  # account
		[(6, 1)],  # holding
		[[('Buy')], [('Sell')]],  # trxn_type
		[(1, 1, 8, 303, '1/1/12', True)]]  # transaction

	return values

def get_pop_values():
	values = [
		[],  # exchange
		[],  # data_source
		[],  # commodity_group
		[],  # commodity_type
		[
			('IBM', 'International Business Machines', 1, 1, 1),
			('WMT', 'Wal-Mart', 1, 1, 1),
			('CAT', 'Caterpillar', 1, 1, 1)],  # commodity
		[],  # event_type
		[(1, 6, 1, 10, '2/1/12'), (1, 8, 1, 12, '2/1/12')],  # event
		[
			(8, 1, 150, '1/1/12'),
			(9, 1, 90, '1/1/12'),
			(10, 1, 120, '1/1/12')],  # price
		[],  # person
		[],  # company
		[],  # account_type
		[],  # account
		[(8, 1), (9, 1), (10, 1)],  # holding
		[],  # trxn_type
		[
			(2, 1, 10, 148, '1/1/12', True),
			(3, 1, 12, 85, '1/1/12', True),
			(1, 1, 8, 320, '2/1/12', True),
			(4, 1, 14, 125, '2/1/12', True)]]  # transaction

	return values

# For views
def get_plural(word):
	if word[-1] == 'y':
		return word[:-1] + 'ies'
	else:
		return word + 's'

def get_kwargs(table, module, conn, form=None, post_table=True):
	plural_table = get_plural(table).replace('_', ' ')
	table_title = table.title().replace('_', ' ')
	plural_table_title = plural_table.title()
	form_fields, table_headers, results, keys = getattr(conn, table)
	rows = conn.values(results, keys)

	post_table = table if post_table else None
	form_caption = '%s Entry Form' % table_title
	heading = 'The %s database' % plural_table
	subheading = (
		'Add %s to the database and see them instantly updated in the lists '
		'below.' % plural_table)

	return {
		'id': table, 'title': plural_table_title, 'heading': heading,
		'subheading': subheading, 'rows': rows, 'form': form,
		'form_caption': form_caption, 'table_caption': '%s List' % table_title,
		'table_headers': table_headers, 'form_fields': form_fields,
		'post_location': '%s.add' % module, 'post_table': post_table}

def init_form(form):
	try:
		form = form.new()
	except AttributeError:
		pass

	return form

# For forms
def get_choices(a_class, value_field, *args, **kwargs):
	order = '%s.%s' % (a_class.__table__, args[0])

	try:
		filter = '%s.%s' % (a_class.__name__, kwargs['column'])
		value = kwargs['value']
		result = a_class.query.filter(eval(filter).in_(value)).order_by(order).all()
	except KeyError:
		result = a_class.query.order_by(order).all()

	values = [getattr(x, value_field) for x in result]
	combo = []

	for arg in args:
		try:
			new = [getattr(getattr(x, arg[0]), arg[1]) for x in result]
		except Exception:
			new = [getattr(x, arg) for x in result]

		combo.append(new)

	try:
		attr = [', '.join(x) for x in zip(combo[0], combo[1])]
	except IndexError:
		attr = combo[0]

	return zip(values, attr)

def get_validators(a_class, value_field):
	result = a_class.query.all()
	values = [getattr(x, value_field) for x in result]
	values = sorted(values)
	return [
		Required(), AnyOf(
			values, message=u'Invalid value, must be one of:'
			'%(values)s')]
<commit_msg>Change variable names<commit_after>from flask import current_app as app
from flask.ext.wtf import AnyOf, Required
from app.hermes.models import EventType, Commodity, CommodityType, Exchange
from app.hermes.models import DataSource
from app.cronus.models import Transaction

# For use with Connection
def portify(site):
	site = site.split('/')

	if site[2] == 'localhost':
		site[2] = 'localhost:%s' % app.config['PORT']

	return '/'.join(site)

# For flask-script
def get_init_values():
	values = [
		[
			('NYSE', 'New York Stock Exchange'), ('NASDAQ', 'NASDAQ'),
			('OTC', 'Over the counter'), ('N/A', 'Currency')],  # exchange
		[[('Yahoo')], [('Google')], [('XE')]],  # data_source
		[[('Security')], [('Currency')], [('Other')]],  # commodity_group
		[
			('Stock', 1), ('Bond', 1), ('Mutual Fund', 1), ('ETF', 1),
			('Currency', 2), ('Descriptor', 3)],  # commodity_type
		[
			('USD', 'US Dollar', 5, 3, 4),
			('EUR', 'Euro', 5, 3, 4),
			('GBP', 'Pound Sterling', 5, 3, 4),
			('TZS', 'Tanzanian Shilling', 5, 3, 4),
			('Multiple', 'Multiple', 6, 3, 4),
			('APL', 'Apple', 1, 1, 1),
			('Text', 'Text', 6, 3, 4)],  # commodity
		[
			[('Dividend')], [('Special Dividend')], [('Stock Split')],
			[('Name Change')], [('Ticker Change')]],  # event_type
		[],  # event
		[
			(2, 1, 1.2, '1/1/12'),
			(3, 1, 1.8, '1/1/12'),
			(4, 1, 1.0 / 1580.0, '1/1/12'),
			(6, 1, 300, '1/1/12')],  # price
		[(1, 'Reuben', 'Cummings', 'reubano@gmail.com')],  # person
		[
			('Scottrade', 'https://trading.scottrade.com/'),
			('Vanguard', 'http://vanguard.com/')],  # company
		[[('Brokerage')], [('Roth IRA')]],  # account_type
		[(1, 1, 1, 1, 'Scottrade'), (2, 2, 1, 1, 'Vanguard IRA')],  # account
		[(6, 1)],  # holding
		[[('Buy')], [('Sell')]],  # trxn_type
		[(1, 1, 8, 303, '1/1/12', True)]]  # transaction

	return values

def get_pop_values():
	values = [
		[],  # exchange
		[],  # data_source
		[],  # commodity_group
		[],  # commodity_type
		[
			('IBM', 'International Business Machines', 1, 1, 1),
			('WMT', 'Wal-Mart', 1, 1, 1),
			('CAT', 'Caterpillar', 1, 1, 1)],  # commodity
		[],  # event_type
		[(1, 6, 1, 10, '2/1/12'), (1, 8, 1, 12, '2/1/12')],  # event
		[
			(8, 1, 150, '1/1/12'),
			(9, 1, 90, '1/1/12'),
			(10, 1, 120, '1/1/12')],  # price
		[],  # person
		[],  # company
		[],  # account_type
		[],  # account
		[(8, 1), (9, 1), (10, 1)],  # holding
		[],  # trxn_type
		[
			(2, 1, 10, 148, '1/1/12', True),
			(3, 1, 12, 85, '1/1/12', True),
			(1, 1, 8, 320, '2/1/12', True),
			(4, 1, 14, 125, '2/1/12', True)]]  # transaction

	return values

# For views
def get_plural(word):
	if word[-1] == 'y':
		return word[:-1] + 'ies'
	else:
		return word + 's'

def get_kwargs(table, module, conn, form=None, post_table=True):
	plural_table = get_plural(table).replace('_', ' ')
	table_title = table.title().replace('_', ' ')
	plural_table_title = plural_table.title()
	form_fields, table_headers, results, keys = getattr(conn, table)
	rows = conn.values(results, keys)

	post_table = table if post_table else None
	form_caption = '%s Entry Form' % table_title
	heading = 'The %s database' % plural_table
	subheading = (
		'Add %s to the database and see them instantly updated in the lists '
		'below.' % plural_table)

	return {
		'id': table, 'title': plural_table_title, 'heading': heading,
		'subheading': subheading, 'rows': rows, 'form': form,
		'form_caption': form_caption, 'table_caption': '%s List' % table_title,
		'table_headers': table_headers, 'form_fields': form_fields,
		'post_location': '%s.add' % module, 'post_table': post_table}

def init_form(form):
	try:
		form = form.new()
	except AttributeError:
		pass

	return form

# For forms
def get_choices(a_class, value_field, *args, **kwargs):
	order = '%s.%s' % (a_class.__table__, args[0])

	try:
		filter = '%s.%s' % (a_class.__name__, kwargs['column'])
		value = kwargs['value']
		result = a_class.query.filter(eval(filter).in_(value)).order_by(order).all()
	except KeyError:
		result = a_class.query.order_by(order).all()

	values = [getattr(x, value_field) for x in result]
	combo = []

	for arg in args:
		try:
			new = [getattr(getattr(x, arg[0]), arg[1]) for x in result]
		except Exception:
			new = [getattr(x, arg) for x in result]

		combo.append(new)

	try:
		selection = [', '.join(x) for x in zip(combo[0], combo[1])]
	except IndexError:
		selection = combo[0]

	return zip(values, selection)

def get_validators(a_class, value_field):
	result = a_class.query.all()
	values = [getattr(x, value_field) for x in result]
	values = sorted(values)
	return [
		Required(), AnyOf(
			values, message=u'Invalid value, must be one of:'
			'%(values)s')]
<|endoftext|>


<commit_before># set user selected language (default spanish)

if request.vars.lang: session.lang=request.vars.lang
T.force(session.lang or "es")

# Return service unavailable
# for maintenance

SUSPEND_SERVICE = False
ALLOW_VOTE = False

######################################
### PARAMETERS
######################################
import datetime
try:
    from gluon.contrib.gql import *
except ImportError:
    is_gae=False
else:
    is_gae=True

VERSION=0.5
T.current_languages=['es','es-ar','es-es']

# If Developer Test, turn off email verificaiton and recaptcha checks,
#  db-pooling, use GCO sandbox, etc.:
# DEV_TEST=True   # settings suitable for development work
DEV_TEST=True# Deployed settings

if DEV_TEST:
    DBURI='sqlite://development.db'
    DBPOOLS=0
    # to test translations; example:  http://..../function?force_language=es
    if request.vars.force_language: session.language=request.vars.force_language
    if session.language: T.force(session.language)
else:
    # DBURI set in 0_private.py
    DBURI='postgres://web2py:password@localhost/pycon2011'
    DBPOOLS=0

TWITTER_HASH = "pyconar"
response.title=T('web2conf')
response.subtitle=''
response.footer=T("""Conference descriptiondates city (organized by users group).

More info: blog  Contact: mail address""")
response.keywords='python, free software'
response.description=T('Powered by web2py')

# Enable or disable dynamic menu
NAVBAR = False

# GOOGLEMAP_KEY set in 0_private.py - here just to ensure definition
GOOGLEMAP_KEY=''

# The following GOOGLE items set in 0_private.py - here to ensure defaults:
GOOGLE_MERCHANT_ID=''
GOOGLE_MERCHANT_KEY=''
GOOGLE_SANDBOX=DEV_TEST

# Event link in social networks
LINKEDIN_EVENT = ""
FACEBOOK_EVENT = ""

FOOD_PREFERENCES=('normal','vegetarian','vegan','kosher','halal')
FOOD_PREFERENCES_LABELS=(T('normal'),T('vegetarian'),T('vegan'),T('kosher'),T('halal'))

T_SHIRT_SIZES=('', 'S','M','L','XL','XXL','XXXL',)
T_SHIRT_SIZES_LABELS=(T('no, thanks'),    T("small"),T("medium"),T("large"),T("xlarge"),T("xxlarge"), T("xxxlarge"),)

# TODAY_DATE is here so that comparizons w/ cutoff dates
#  will work properly anywhere in web2conf
# NOTE: we add 6 hours since our server is EST, and this will cover Hawaii
#  will want to have these times be session time local in next rev.
TODAY_DATE=datetime.datetime.today()
PROPOSALS_DEADLINE_DATE=datetime.datetime(2012,5,31,23,00,00)
REVIEW_DEADLINE_DATE=datetime.datetime(2012,6,30,23,00,00)
EARLYBIRD_DATE=datetime.datetime(2021,2,22,6,0,0)
PRECONF_DATE=datetime.datetime(2012,3,19,23,59,0)
FACUTOFF_DATE=datetime.datetime(2012,7,10,23,59,0)
REGCLOSE_DATE=datetime.datetime(2012,9,22,1,00,00)

SIMPLIFIED_REGISTRATION=False # don't ask password on registration

### fix this ...
if TODAY_DATE<EARLYBIRD_DATE:  ### early registration!
   ATTENDEE_TYPES=(
     ('gratis',T('Gratuito, $0')),
   )
elif TODAY_DATE<PRECONF_DATE:  ### pre-conference registration!:
   ATTENDEE_TYPES=(
     ('gratis',T('Gratuito, $0')),
   )
else:
   ATTENDEE_TYPES=(
     ('gratis',T('Gratuito, $0')),
   )
if session.manager:
   ATTENDEE_TYPES=(
     ('gratis',T('Gratuito, $0')),
   )
ATTENDEE_TYPE_COST=dict(
     gratis=0.0,
   )
ATTENDEE_TYPE_COST[None]=0.0

TUTORIALS_LIST=(
)
TUTORIALS=dict(TUTORIALS_LIST) ### do not remove

TUTORIALS_CAPS={
}

COST_FIRST_TUTORIAL=120.0
COST_SECOND_TUTORIAL=80.0

# default activities
ACTIVITY_TYPES= (T('keynote'),T('panel'),T('plenary'),T('tutorial'),
                 T('talk'),T('extreme talk'),T('poster'),T('project'),
                 T('startup'),T('stand'),T('summit'),T('open space'),
                 T('social'),T('break'),T('lightning talk'),
                 T('sprint'),T('workshop'))

ACTIVITY_CATEGORIES=sorted(('py3k','gui','web','cli','herramientas',
                             'lenguaje','fomento','core','educación',
                             'ciencia','académico','comunidad','moviles',
                             'caso de estudio','redes','juegos','seguridad',
                             'testing'))

ACTIVITY_LEVELS=("Beginner","Intermediate","Advanced", "Extreme")
ACTIVITY_DURATION={'talk': 30, 'extreme_talk': 30, 'tutorial': 120, 'workshop': 0, 'poster': 0, 'project': 0}
ACTIVITY_ROOMS={1: "Auditorio", 2: "Aula A", 3: "Aula B", 4: "Aula C", 5: "Aula D", 6: "Sala Internet"}

ACTIVITY_SHOW_DESCRIPTION = False # hide desc to public

SPONSOR_LEVELS=("Organizer", "Sponsor Oro", "Sponsor Plata", "Sponsor Bronce", "Mecenas", "Agradecimiento Especial", "Medios / Auspicios", "")

# verify by email, unless running a developer test:
EMAIL_VERIFICATION= True #not DEV_TEST
EMAIL_SERVER='localhost:25' #or Configure!
EMAIL_AUTH=None # or 'username:password'
EMAIL_SENDER='pyconar2012@gmail.com'

# for FA applications / communication
FA_EMAIL_UPDATES=True
FA_EMAIL_TO=EMAIL_SENDER

# for testing:
#  disable recaptcha by setting DEV_TEST at the top of this file:
DO_RECAPTCHA= False #not DEV_TEST
# RECAPTCHA public and private keys are set in 0_private.py
#  - here to ensure defaults:
RECAPTCHA_PUBLIC_KEY=''
RECAPTCHA_PRIVATE_KEY=''

JANRAIN = False

# modules
ENABLE_TALKS=True
ENABLE_EXPENSES = False
ENABLE_FINANCIAL_AID = True
ENABLE_PAYMENTS = False

if True and DEV_TEST:    # for local development
    HOST='localhost:8000'
    HOST_NEXT='localhost:8000'
else:
    HOST=''
    HOST_NEXT=''

HOTELS=('unknown','Hyatt Regency','Crowne Plaza','other','none')

# for badge generation:
TRUETYPE_PATH='/usr/share/fonts/truetype/freefont'
GSFONTS_PATH='/usr/share/fonts/type1/gsfonts/'

EMAIL_VERIFY_SUBJECT=T("%s Registration Confirmation") % response.title
EMAIL_VERIFY_BODY=T("""
Dear Attendee,

To proceed with your registration and verify your email, click on the following link:

%s

--
%s
""") % ("http://%s%s/%%(key)s" % (request.env.http_host, URL(r=request,f='verify')), response.title)

PASSWORD_RETRIEVE_SUBJECT=T("%s Registration Password") % response.title
PASSWORD_RETRIEVE_BODY=T("Your new password is %(password)s")
INVOICE_HEADER = "This is a Conference Invoice!!!"

CONFERENCE_URL=None
CONFERENCE_COORDS=-20.2597103,-61.4510078
#-31.2597103,-61.4510078

COUNTRIES=['United States', 'Afghanistan', 'Albania', 'Algeria', 'Andorra', 'Angola', 'Antigua and Barbuda', 'Argentina', 'Armenia', 'Australia', 'Austria', 'Azerbaijan', 'Bahamas', 'Bahrain', 'Bangladesh', 'Barbados', 'Belarus', 'Belgium', 'Belize', 'Benin', 'Bhutan', 'Bolivia', 'Bosnia and Herzegovina', 'Botswana', 'Brazil', 'Brunei', 'Bulgaria', 'Burkina Faso', 'Burundi', 'Cambodia', 'Cameroon', 'Canada', 'Cape Verde', 'Central African Republic', 'Chad', 'Chile', 'China', 'Colombia', 'Comoros', 'Congo', 'Costa Rica', "Côte d'Ivoire", 'Croatia', 'Cuba', 'Cyprus', 'Czech Republic', 'Denmark', 'Djibouti', 'Dominica', 'Dominican Republic', 'East Timor', 'Ecuador', 'Egypt', 'El Salvador', 'Equatorial Guinea', 'Eritrea', 'Estonia', 'Ethiopia', 'Fiji', 'Finland', 'France', 'Gabon', 'Gambia', 'Georgia', 'Germany', 'Ghana', 'Greece', 'Grenada', 'Guatemala', 'Guinea', 'Guinea-Bissau', 'Guyana', 'Haiti', 'Honduras', 'Hong Kong', 'Hungary', 'Iceland', 'India', 'Indonesia', 'Iran', 'Iraq', 'Ireland', 'Israel', 'Italy', 'Jamaica', 'Japan', 'Jordan', 'Kazakhstan', 'Kenya', 'Kiribati', 'North Korea','South Korea', 'Kuwait', 'Kyrgyzstan', 'Laos', 'Latvia', 'Lebanon', 'Lesotho', 'Liberia', 'Libya', 'Liechtenstein', 'Lithuania', 'Luxembourg', 'Macedonia', 'Madagascar', 'Malawi', 'Malaysia', 'Maldives', 'Mali', 'Malta', 'Marshall Islands', 'Mauritania', 'Mauritius', 'Mexico', 'Micronesia', 'Moldova', 'Monaco', 'Mongolia', 'Montenegro', 'Morocco', 'Mozambique', 'Myanmar', 'Namibia', 'Nauru', 'Nepal', 'Netherlands', 'New Zealand', 'Nicaragua', 'Niger', 'Nigeria', 'Norway', 'Oman', 'Pakistan', 'Palau', 'Palestine', 'Panama', 'Papua New Guinea', 'Paraguay', 'Peru', 'Philippines', 'Poland', 'Portugal', 'Puerto Rico', 'Qatar', 'Romania', 'Russia', 'Rwanda', 'Saint Kitts and Nevis', 'Saint Lucia', 'Saint Vincent and the Grenadines', 'Samoa', 'San Marino', 'Sao Tome and Principe', 'Saudi Arabia', 'Senegal', 'Serbia and Montenegro', 'Seychelles', 'Sierra Leone', 'Singapore', 'Slovakia', 'Slovenia', 'Solomon Islands', 'Somalia', 'South Africa', 'Spain', 'Sri Lanka', 'Sudan', 'Suriname', 'Swaziland', 'Sweden', 'Switzerland', 'Syria', 'Taiwan', 'Tajikistan', 'Tanzania', 'Thailand', 'Togo', 'Tonga', 'Trinidad and Tobago', 'Tunisia', 'Turkey', 'Turkmenistan', 'Tuvalu', 'Uganda', 'Ukraine', 'United Arab Emirates', 'United Kingdom', 'Uruguay', 'Uzbekistan', 'Vanuatu', 'Vatican City', 'Venezuela', 'Vietnam', 'Yemen', 'Zambia', 'Zimbabwe']
<commit_msg>updated pyconar2012 activity configutarion<commit_after># set user selected language (default spanish)

if request.vars.lang: session.lang=request.vars.lang
T.force(session.lang or "es")

# Return service unavailable
# for maintenance

SUSPEND_SERVICE = False
ALLOW_VOTE = False

######################################
### PARAMETERS
######################################
import datetime
try:
    from gluon.contrib.gql import *
except ImportError:
    is_gae=False
else:
    is_gae=True

VERSION=0.5
T.current_languages=['es','es-ar','es-es']

# If Developer Test, turn off email verificaiton and recaptcha checks,
#  db-pooling, use GCO sandbox, etc.:
# DEV_TEST=True   # settings suitable for development work
DEV_TEST=True# Deployed settings

if DEV_TEST:
    DBURI='sqlite://development.db'
    DBPOOLS=0
    # to test translations; example:  http://..../function?force_language=es
    if request.vars.force_language: session.language=request.vars.force_language
    if session.language: T.force(session.language)
else:
    # DBURI set in 0_private.py
    DBURI='postgres://web2py:password@localhost/pycon2011'
    DBPOOLS=0

TWITTER_HASH = "pyconar"
response.title=T('web2conf')
response.subtitle=''
response.footer=T("""Conference descriptiondates city (organized by users group).

More info: blog  Contact: mail address""")
response.keywords='python, free software'
response.description=T('Powered by web2py')

# Enable or disable dynamic menu
NAVBAR = False

# GOOGLEMAP_KEY set in 0_private.py - here just to ensure definition
GOOGLEMAP_KEY=''

# The following GOOGLE items set in 0_private.py - here to ensure defaults:
GOOGLE_MERCHANT_ID=''
GOOGLE_MERCHANT_KEY=''
GOOGLE_SANDBOX=DEV_TEST

# Event link in social networks
LINKEDIN_EVENT = ""
FACEBOOK_EVENT = ""

FOOD_PREFERENCES=('normal','vegetarian','vegan','kosher','halal')
FOOD_PREFERENCES_LABELS=(T('normal'),T('vegetarian'),T('vegan'),T('kosher'),T('halal'))

T_SHIRT_SIZES=('', 'S','M','L','XL','XXL','XXXL',)
T_SHIRT_SIZES_LABELS=(T('no, thanks'),    T("small"),T("medium"),T("large"),T("xlarge"),T("xxlarge"), T("xxxlarge"),)

# TODAY_DATE is here so that comparizons w/ cutoff dates
#  will work properly anywhere in web2conf
# NOTE: we add 6 hours since our server is EST, and this will cover Hawaii
#  will want to have these times be session time local in next rev.
TODAY_DATE=datetime.datetime.today()
PROPOSALS_DEADLINE_DATE=datetime.datetime(2012,5,31,23,00,00)
REVIEW_DEADLINE_DATE=datetime.datetime(2012,6,30,23,00,00)
EARLYBIRD_DATE=datetime.datetime(2021,2,22,6,0,0)
PRECONF_DATE=datetime.datetime(2012,3,19,23,59,0)
FACUTOFF_DATE=datetime.datetime(2012,7,10,23,59,0)
REGCLOSE_DATE=datetime.datetime(2012,9,22,1,00,00)

SIMPLIFIED_REGISTRATION=False # don't ask password on registration

### fix this ...
if TODAY_DATE<EARLYBIRD_DATE:  ### early registration!
   ATTENDEE_TYPES=(
     ('gratis',T('Gratuito, $0')),
   )
elif TODAY_DATE<PRECONF_DATE:  ### pre-conference registration!:
   ATTENDEE_TYPES=(
     ('gratis',T('Gratuito, $0')),
   )
else:
   ATTENDEE_TYPES=(
     ('gratis',T('Gratuito, $0')),
   )
if session.manager:
   ATTENDEE_TYPES=(
     ('gratis',T('Gratuito, $0')),
   )
ATTENDEE_TYPE_COST=dict(
     gratis=0.0,
   )
ATTENDEE_TYPE_COST[None]=0.0

TUTORIALS_LIST=(
)
TUTORIALS=dict(TUTORIALS_LIST) ### do not remove

TUTORIALS_CAPS={
}

COST_FIRST_TUTORIAL=120.0
COST_SECOND_TUTORIAL=80.0

# default activities
ACTIVITY_TYPES= (T('talk'),T('extreme talk'),T('poster'),T('project'),
                 T('keynote'),T('panel'),T('plenary'),T('tutorial'),
                 T('stand'),T('summit'),T('open space'),
                 T('social'),T('break'),T('lightning talk'),
                 T('sprint'),T('workshop'),T('paper'))

ACTIVITY_CATEGORIES=sorted(('py3k','gui','web','cli','herramientas',
                             'lenguaje','fomento','core','educación',
                             'ciencia','académico','comunidad','moviles',
                             'caso de estudio','redes','juegos','seguridad',
                             'testing'))

ACTIVITY_LEVELS=("Beginner","Intermediate","Advanced")
ACTIVITY_TRACKS=("General", "Science", "Student Works", "Extreme")
ACTIVITY_DURATION={'talk': 40, 'extreme talk': 30, 'tutorial': 120, 'workshop': 0, 'poster': 0, 'project': 0, 'panel': 45, 'plenary': 60, 'keynote': 60}
ACTIVITY_ROOMS={1: "Auditorio", 2: "Aula A", 3: "Aula B", 4: "Aula C", 5: "Aula D", 6: "Sala Internet"}

ACTIVITY_SHOW_DESCRIPTION = False # hide desc to public

SPONSOR_LEVELS=("Organizer", "Sponsor Oro", "Sponsor Plata", "Sponsor Bronce", "Mecenas", "Agradecimiento Especial", "Medios / Auspicios", "")

# verify by email, unless running a developer test:
EMAIL_VERIFICATION= True #not DEV_TEST
EMAIL_SERVER='localhost:25' #or Configure!
EMAIL_AUTH=None # or 'username:password'
EMAIL_SENDER='pyconar2012@gmail.com'

# for FA applications / communication
FA_EMAIL_UPDATES=True
FA_EMAIL_TO=EMAIL_SENDER

# for testing:
#  disable recaptcha by setting DEV_TEST at the top of this file:
DO_RECAPTCHA= False #not DEV_TEST
# RECAPTCHA public and private keys are set in 0_private.py
#  - here to ensure defaults:
RECAPTCHA_PUBLIC_KEY=''
RECAPTCHA_PRIVATE_KEY=''

JANRAIN = False

# modules
ENABLE_TALKS=True
ENABLE_EXPENSES = False
ENABLE_FINANCIAL_AID = True
ENABLE_PAYMENTS = False

if True and DEV_TEST:    # for local development
    HOST='localhost:8000'
    HOST_NEXT='localhost:8000'
else:
    HOST=''
    HOST_NEXT=''

HOTELS=('unknown','Hyatt Regency','Crowne Plaza','other','none')

# for badge generation:
TRUETYPE_PATH='/usr/share/fonts/truetype/freefont'
GSFONTS_PATH='/usr/share/fonts/type1/gsfonts/'

EMAIL_VERIFY_SUBJECT=T("%s Registration Confirmation") % response.title
EMAIL_VERIFY_BODY=T("""
Dear Attendee,

To proceed with your registration and verify your email, click on the following link:

%s

--
%s
""") % ("http://%s%s/%%(key)s" % (request.env.http_host, URL(r=request,f='verify')), response.title)

PASSWORD_RETRIEVE_SUBJECT=T("%s Registration Password") % response.title
PASSWORD_RETRIEVE_BODY=T("Your new password is %(password)s")
INVOICE_HEADER = "This is a Conference Invoice!!!"

CONFERENCE_URL=None
CONFERENCE_COORDS=-20.2597103,-61.4510078
#-31.2597103,-61.4510078

COUNTRIES=['United States', 'Afghanistan', 'Albania', 'Algeria', 'Andorra', 'Angola', 'Antigua and Barbuda', 'Argentina', 'Armenia', 'Australia', 'Austria', 'Azerbaijan', 'Bahamas', 'Bahrain', 'Bangladesh', 'Barbados', 'Belarus', 'Belgium', 'Belize', 'Benin', 'Bhutan', 'Bolivia', 'Bosnia and Herzegovina', 'Botswana', 'Brazil', 'Brunei', 'Bulgaria', 'Burkina Faso', 'Burundi', 'Cambodia', 'Cameroon', 'Canada', 'Cape Verde', 'Central African Republic', 'Chad', 'Chile', 'China', 'Colombia', 'Comoros', 'Congo', 'Costa Rica', "Côte d'Ivoire", 'Croatia', 'Cuba', 'Cyprus', 'Czech Republic', 'Denmark', 'Djibouti', 'Dominica', 'Dominican Republic', 'East Timor', 'Ecuador', 'Egypt', 'El Salvador', 'Equatorial Guinea', 'Eritrea', 'Estonia', 'Ethiopia', 'Fiji', 'Finland', 'France', 'Gabon', 'Gambia', 'Georgia', 'Germany', 'Ghana', 'Greece', 'Grenada', 'Guatemala', 'Guinea', 'Guinea-Bissau', 'Guyana', 'Haiti', 'Honduras', 'Hong Kong', 'Hungary', 'Iceland', 'India', 'Indonesia', 'Iran', 'Iraq', 'Ireland', 'Israel', 'Italy', 'Jamaica', 'Japan', 'Jordan', 'Kazakhstan', 'Kenya', 'Kiribati', 'North Korea','South Korea', 'Kuwait', 'Kyrgyzstan', 'Laos', 'Latvia', 'Lebanon', 'Lesotho', 'Liberia', 'Libya', 'Liechtenstein', 'Lithuania', 'Luxembourg', 'Macedonia', 'Madagascar', 'Malawi', 'Malaysia', 'Maldives', 'Mali', 'Malta', 'Marshall Islands', 'Mauritania', 'Mauritius', 'Mexico', 'Micronesia', 'Moldova', 'Monaco', 'Mongolia', 'Montenegro', 'Morocco', 'Mozambique', 'Myanmar', 'Namibia', 'Nauru', 'Nepal', 'Netherlands', 'New Zealand', 'Nicaragua', 'Niger', 'Nigeria', 'Norway', 'Oman', 'Pakistan', 'Palau', 'Palestine', 'Panama', 'Papua New Guinea', 'Paraguay', 'Peru', 'Philippines', 'Poland', 'Portugal', 'Puerto Rico', 'Qatar', 'Romania', 'Russia', 'Rwanda', 'Saint Kitts and Nevis', 'Saint Lucia', 'Saint Vincent and the Grenadines', 'Samoa', 'San Marino', 'Sao Tome and Principe', 'Saudi Arabia', 'Senegal', 'Serbia and Montenegro', 'Seychelles', 'Sierra Leone', 'Singapore', 'Slovakia', 'Slovenia', 'Solomon Islands', 'Somalia', 'South Africa', 'Spain', 'Sri Lanka', 'Sudan', 'Suriname', 'Swaziland', 'Sweden', 'Switzerland', 'Syria', 'Taiwan', 'Tajikistan', 'Tanzania', 'Thailand', 'Togo', 'Tonga', 'Trinidad and Tobago', 'Tunisia', 'Turkey', 'Turkmenistan', 'Tuvalu', 'Uganda', 'Ukraine', 'United Arab Emirates', 'United Kingdom', 'Uruguay', 'Uzbekistan', 'Vanuatu', 'Vatican City', 'Venezuela', 'Vietnam', 'Yemen', 'Zambia', 'Zimbabwe']
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
#
# Here is implemented the preApplication and postApplication method
# along the Asynchronous event schedule

import os

from twisted.scripts._twistd_unix import UnixApplicationRunner
from twisted.internet import reactor, defer
from twisted.python.util import untilConcludes

from globaleaks.db import init_db, clean_untracked_files, \
    refresh_memory_variables, update_version

from globaleaks.db.appdata import update_appdata

from globaleaks.jobs import session_management_sched, statistics_sched, \
    notification_sched, delivery_sched, cleaning_sched, \
    pgp_check_sched

from globaleaks.settings import GLSettings
from globaleaks.utils.utility import log, datetime_now, disable_swap

test_reactor = None

class GlobaLeaksRunner(UnixApplicationRunner):
    """
    This runner is specific to Unix systems.
    """
    _reactor = reactor

    def start_asynchronous_jobs(self):
        """
        Initialize the asynchronous operation, scheduled in the system
        """
        if test_reactor:
            self._reactor = test_reactor

        # Scheduling the Delivery schedule to be executed every 2 seconds
        delivery_sched.DeliverySchedule().schedule(2, 1)

        # Scheduling the Anomalies Check schedule to be executed every 30 seconds
        statistics_sched.AnomaliesSchedule().schedule(30, 2)

        # Scheduling the Notification schedule to be executed every 60 seconds
        notification_sched.NotificationSchedule().schedule(60, 3)

        # Scheduling the Session Management schedule to be executed every minute
        session_management_sched.SessionManagementSchedule().schedule(60, 5)

        # Scheduling the Tip Cleaning scheduler to be executed every day at 00:00
        current_time = datetime_now()
        delay = (3600 * (24 + 0)) - (current_time.hour * 3600) - (current_time.minute * 60) - current_time.second
        clean = cleaning_sched.CleaningSchedule().schedule(3600 * 24, delay)

        # Scheduling the PGP Check scheduler to be executed every day at 02:00
        delay = (3600 * (24 + 2)) - (current_time.hour * 3600) - (current_time.minute * 60) - current_time.second
        pgp_check = pgp_check_sched.PGPCheckSchedule().schedule(3600 * 24, delay)

        # Scheduling the Statistics schedule to be executed every hour on the hour
        delay = (3600) - (current_time.minute * 60) - current_time.second
        stats = statistics_sched.StatisticsSchedule().schedule(3600, delay)

    @defer.inlineCallbacks
    def start_globaleaks(self):
        try:
            GLSettings.fix_file_permissions()
            GLSettings.drop_privileges()
            GLSettings.check_directories()

            if GLSettings.initialize_db:
                yield init_db()
            else:
                yield update_version()

            yield clean_untracked_files()

            yield refresh_memory_variables()

            self.start_asynchronous_jobs()

        except Exception as excep:
            log.err("ERROR: Cannot start GlobaLeaks; please manual check the error.")
            log.err("EXCEPTION: %s" % excep)
            self._reactor.stop()

    def postApplication(self):
        """
        Run the application.
        """
        try:
            self.startApplication(self.application)
        except Exception as ex:
            statusPipe = self.config.get("statusPipe", None)
            if statusPipe is not None:
                # Limit the total length to the passed string to 100
                strippedError = str(ex)[:98]
                untilConcludes(os.write, statusPipe, "1 %s" % (strippedError,))
                untilConcludes(os.close, statusPipe)
            self.removePID(self.config['pidfile'])
            raise
        else:
            statusPipe = self.config.get("statusPipe", None)
            if statusPipe is not None:
                untilConcludes(os.write, statusPipe, "0")
                untilConcludes(os.close, statusPipe)

        self._reactor.callLater(0, self.start_globaleaks)

        self.startReactor(None, self.oldstdout, self.oldstderr)

        self.removePID(self.config['pidfile'])
<commit_msg>Remove ripped twisted function identical to the original<commit_after># -*- encoding: utf-8 -*-
#
# Here is implemented the preApplication and postApplication method
# along the Asynchronous event schedule

import os

from twisted.scripts._twistd_unix import UnixApplicationRunner
from twisted.internet import reactor, defer
from twisted.python.util import untilConcludes

from globaleaks.db import init_db, clean_untracked_files, \
    refresh_memory_variables, update_version

from globaleaks.db.appdata import update_appdata

from globaleaks.jobs import session_management_sched, statistics_sched, \
    notification_sched, delivery_sched, cleaning_sched, \
    pgp_check_sched

from globaleaks.settings import GLSettings
from globaleaks.utils.utility import log, datetime_now, disable_swap

test_reactor = None

class GlobaLeaksRunner(UnixApplicationRunner):
    """
    This runner is specific to Unix systems.
    """
    _reactor = reactor

    def start_asynchronous_jobs(self):
        """
        Initialize the asynchronous operation, scheduled in the system
        """
        if test_reactor:
            self._reactor = test_reactor

        # Scheduling the Delivery schedule to be executed every 2 seconds
        delivery_sched.DeliverySchedule().schedule(2, 1)

        # Scheduling the Anomalies Check schedule to be executed every 30 seconds
        statistics_sched.AnomaliesSchedule().schedule(30, 2)

        # Scheduling the Notification schedule to be executed every 60 seconds
        notification_sched.NotificationSchedule().schedule(60, 3)

        # Scheduling the Session Management schedule to be executed every minute
        session_management_sched.SessionManagementSchedule().schedule(60, 5)

        # Scheduling the Tip Cleaning scheduler to be executed every day at 00:00
        current_time = datetime_now()
        delay = (3600 * (24 + 0)) - (current_time.hour * 3600) - (current_time.minute * 60) - current_time.second
        clean = cleaning_sched.CleaningSchedule().schedule(3600 * 24, delay)

        # Scheduling the PGP Check scheduler to be executed every day at 02:00
        delay = (3600 * (24 + 2)) - (current_time.hour * 3600) - (current_time.minute * 60) - current_time.second
        pgp_check = pgp_check_sched.PGPCheckSchedule().schedule(3600 * 24, delay)

        # Scheduling the Statistics schedule to be executed every hour on the hour
        delay = (3600) - (current_time.minute * 60) - current_time.second
        stats = statistics_sched.StatisticsSchedule().schedule(3600, delay)

    @defer.inlineCallbacks
    def start_globaleaks(self):
        try:
            GLSettings.fix_file_permissions()
            GLSettings.drop_privileges()
            GLSettings.check_directories()

            if GLSettings.initialize_db:
                yield init_db()
            else:
                yield update_version()

            yield clean_untracked_files()

            yield refresh_memory_variables()

            self.start_asynchronous_jobs()

        except Exception as excep:
            log.err("ERROR: Cannot start GlobaLeaks; please manual check the error.")
            log.err("EXCEPTION: %s" % excep)
            self._reactor.stop()
<|endoftext|>


<commit_before>#!/usr/bin/python2.7
# encoding: utf-8

import sys
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.tri as Tri
import matplotlib.ticker as ticker
from matplotlib.path import Path

def closest_point( pt_lon, pt_lat, lon, lat, debug=False):
    '''
    Finds the closest exact lon, lat centre indexes of an FVCOM class
    to given lon, lat coordinates.

    Inputs:
      - pt_lon = list of longitudes in degrees to find
      - pt_lat = list of latitudes in degrees to find
      - lon = list of longitudes in degrees to search in
      - lat = list of latitudes in degrees to search in
    Outputs:
      - closest_point_indexes = numpy array of grid indexes
    '''
    if debug:
        print 'Computing closest_point_indexes...'

    points = np.array([pt_lon, pt_lat]).T
    point_list = np.array([lon, lat]).T

    #closest_dist = (np.square((point_list[:, 0] - points[:, 0, None])) +
    #                np.square((point_list[:, 1] - points[:, 1, None])))

    #closest_point_indexes = np.argmin(closest_dist, axis=1)

    #Wesley's optimized version of this bottleneck
    #point_list0 = point_list[:, 0]
    #points0 = points[:, 0, None]
    #point_list1 = point_list[:, 1]
    #points1 = points[:, 1, None]
    #
    #closest_dist = ((point_list0 - points0) *
    #                (point_list0 - points0) +
    #                (point_list1 - points1) *
    #                (point_list1 - points1)
    #                )
    #closest_point_indexes = np.argmin(closest_dist, axis=1)

    #Thomas' optimized version of this bottleneck
    closest_point_indexes = np.zeros(points.shape[0])
    for i in range(points.shape[0]):
        dist=((point_list-points[i,:])**2).sum(axis=1)
        ndx = d.argsort()
        closest_point_indexes[i] = ndx[0]
    if debug:
        print 'Closest dist: ', closest_dist

    if debug:
        print 'closest_point_indexes', closest_point_indexes
        print '...Passed'

    return closest_point_indexes

def interpN_at_pt(var, pt_x, pt_y, xc, yc, index, trinodes,
                  aw0, awx, awy, debug=False):
    """
    Interpol node variable any given variables at any give location.
    Inputs:
      - var = variable, numpy array, dim=(node) or (time, node) or (time, level, node)
      - pt_x = x coordinate in m to find
      - pt_y = y coordinate in m to find
      - xc = list of x coordinates of var, numpy array, dim= ele
      - yc = list of y coordinates of var, numpy array, dim= ele
      - trinodes = FVCOM trinodes, numpy array, dim=(3,nele)
      - index = index of the nearest element
      - aw0, awx, awy = grid parameters
    Outputs:
      - varInterp = var interpolate at (pt_lon, pt_lat)
    """
    if debug:
        print 'Interpolating at node...'

    n1 = trinodes[index,0]
    n2 = trinodes[index,1]
    n3 = trinodes[index,2]
    x0 = pt_x - xc[index]
    y0 = pt_y - yc[index]

    if len(var.shape)==1:
        var0 = (aw0[0,index] * var[n1]) \
             + (aw0[1,index] * var[n2]) \
             + (aw0[2,index] * var[n3])
        varX = (awx[0,index] * var[n1]) \
             + (awx[1,index] * var[n2]) \
             + (awx[2,index] * var[n3])
        varY = (awy[0,index] * var[n1]) \
             + (awy[1,index] * var[n2]) \
             + (awy[2,index] * var[n3])
    elif len(var.shape)==2:
        var0 = (aw0[0,index] * var[:,n1]) \
             + (aw0[1,index] * var[:,n2]) \
             + (aw0[2,index] * var[:,n3])
        varX = (awx[0,index] * var[:,n1]) \
             + (awx[1,index] * var[:,n2]) \
             + (awx[2,index] * var[:,n3])
        varY = (awy[0,index] * var[:,n1]) \
             + (awy[1,index] * var[:,n2]) \
             + (awy[2,index] * var[:,n3])
    else:
        var0 = (aw0[0,index] * var[:,:,n1]) \
             + (aw0[1,index] * var[:,:,n2]) \
             + (aw0[2,index] * var[:,:,n3])
        varX = (awx[0,index] * var[:,:,n1]) \
             + (awx[1,index] * var[:,:,n2]) \
             + (awx[2,index] * var[:,:,n3])
        varY = (awy[0,index] * var[:,:,n1]) \
             + (awy[1,index] * var[:,:,n2]) \
             + (awy[2,index] * var[:,:,n3])
    varPt = var0 + (varX * x0) + (varY * y0)

    if debug:
        if len(var.shape)==1:
            zi = varPt
            print 'Varpt: ', zi
        print '...Passed'

    return varPt

def interpE_at_pt(var, pt_x, pt_y, xc, yc, index, triele, trinodes,
                  a1u, a2u, debug=False):
    """
    Interpol node variable any given variables at any give location.
    Inputs:
      - var = variable, numpy array, dim=(nele) or (time, nele) or (time, level, nele)
      - pt_x = x coordinate in m to find
      - pt_y = y coordinate in m to find
      - xc = list of x coordinates of var, numpy array, dim= nele
      - yc = list of y coordinates of var, numpy array, dim= nele
      - triele = FVCOM triele, numpy array, dim=(3,nele)
      - trinodes = FVCOM trinodes, numpy array, dim=(3,nele)
      - index = index of the nearest element
      - a1u, a2u = grid parameters
    Outputs:
      - varInterp = var interpolate at (pt_lon, pt_lat)
    """
    if debug:
        print 'Interpolating at element...'

    n1 = triele[index,0]
    n2 = triele[index,1]
    n3 = triele[index,2]
    #TR comment: not quiet sure what this step does
    if n1==-1: n1 = trinodes.shape[1] + 1
    if n2==-1: n2 = trinodes.shape[1] + 1
    if n3==-1: n3 = trinodes.shape[1] + 1

    x0 = pt_x - xc[index]
    y0 = pt_y - yc[index]

    if len(var.shape)==1:
        dvardx = (a1u[0,index] * var[index]) \
               + (a1u[1,index] * var[n1]) \
               + (a1u[2,index] * var[n2]) \
               + (a1u[3,index] * var[n3])
        dvardy = (a2u[0,index] * var[index]) \
               + (a2u[1,index] * var[n1]) \
               + (a2u[2,index] * var[n2]) \
               + (a2u[3,index] * var[n3])
        varPt = var[index] + (dvardx * x0) + (dvardy * y0)
    elif len(var.shape)==2:
        dvardx = (a1u[0,index] * var[:,index]) \
               + (a1u[1,index] * var[:,n1]) \
               + (a1u[2,index] * var[:,n2]) \
               + (a1u[3,index] * var[:,n3])
        dvardy = (a2u[0,index] * var[:,index]) \
               + (a2u[1,index] * var[:,n1]) \
               + (a2u[2,index] * var[:,n2]) \
               + (a2u[3,index] * var[:,n3])
        varPt = var[:,index] + (dvardx * x0) + (dvardy * y0)
    else:
        dvardx = (a1u[0,index] * var[:,:,index]) \
               + (a1u[1,index] * var[:,:,n1]) \
               + (a1u[2,index] * var[:,:,n2]) \
               + (a1u[3,index] * var[:,:,n3])
        dvardy = (a2u[0,index] * var[:,:,index]) \
               + (a2u[1,index] * var[:,:,n1]) \
               + (a2u[2,index] * var[:,:,n2]) \
               + (a2u[3,index] * var[:,:,n3])
        varPt = var[:,:,index] + (dvardx * x0) + (dvardy * y0)

    if debug:
        if len(var.shape)==1:
            zi = varPt
            print 'Varpt: ', zi
        print '...Passed'

    return varPt

def interp_at_point(var, pt_lon, pt_lat, lon, lat,
                    index=[], trinodes=[], tri=[], debug=False):
    """
    Interpol any given variables at any give location.

    Inputs:
    ------
      - var = variable, numpy array, dim=(time, nele or node)
      - pt_lon = longitude in degrees to find
      - pt_lat = latitude in degrees to find
      - lon = list of longitudes of var, numpy array, dim=(nele or node)
      - lat = list of latitudes of var, numpy array, dim=(nele or node)
      - trinodes = FVCOM trinodes, numpy array, dim=(3,nele)
    Outputs:
      - varInterp = var interpolate at (pt_lon, pt_lat)
    """
    if debug:
        print 'Interpolating at point...'
    #Finding the right indexes

    #Triangulation
    #if debug:
    #    print triIndex, lon[triIndex], lat[triIndex]
    triIndex = trinodes[index]
    if not tri:
        tri = Tri.Triangulation(lon[triIndex], lat[triIndex], np.array([[0,1,2]]))

    trif = tri.get_trifinder()
    trif.__call__(pt_lon, pt_lat)
    #if debug:
        #if len(var.shape)==1:
            #averEl = var[triIndex]
            #print 'Var', averEl
            #inter = Tri.LinearTriInterpolator(tri, averEl)
            #zi = inter(pt_lon, pt_lat)
            #print zi
            #fig = plt.figure(figsize=(18,10))
            #ax = fig.add_subplot(111,
            #     aspect=(1.0/np.cos(np.mean(lat[triIndex])*np.pi/180.0)))
            #plt.tricontourf(newtri, averEl, cmap=plt.cm.jet)
            #cbar = plt.colorbar()
            #plt.triplot(newtri, 'ko-')
            #plt.plot(pt_lon, pt_lat, 'ko-')
            #plt.show()

    #Choose the right interpolation depending on the variable
    if len(var.shape)==1:
        triVar = np.zeros(triIndex.shape)
        triVar = var[triIndex]
        inter = Tri.LinearTriInterpolator(tri, triVar[:])
        varInterp = inter(pt_lon, pt_lat)
    elif len(var.shape)==2:
        triVar = np.zeros((var.shape[0], triIndex.shape[0]))
        triVar = var[:, triIndex]
        varInterp = np.ones(triVar.shape[0])
        for i in range(triVar.shape[0]):
            inter = Tri.LinearTriInterpolator(tri, triVar[i,:])
            varInterp[i] = inter(pt_lon, pt_lat)
    else:
        triVar = np.zeros((var.shape[0], var.shape[1], triIndex.shape[0]))
        triVar = var[:, :, triIndex]
        varInterp = np.ones(triVar.shape[:-1])
        for i in range(triVar.shape[0]):
           for j in range(triVar.shape[1]):
               inter = Tri.LinearTriInterpolator(tri, triVar[i,j,:])
               varInterp[i,j] = inter(pt_lon, pt_lat)

    if debug:
        print '...Passed'

    return varInterp

<commit_msg>Optimized bottleneck<commit_after>#!/usr/bin/python2.7
# encoding: utf-8

import sys
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.tri as Tri
import matplotlib.ticker as ticker
from matplotlib.path import Path

def closest_point( pt_lon, pt_lat, lon, lat, debug=False):
    '''
    Finds the closest exact lon, lat centre indexes of an FVCOM class
    to given lon, lat coordinates.

    Inputs:
      - pt_lon = list of longitudes in degrees to find
      - pt_lat = list of latitudes in degrees to find
      - lon = list of longitudes in degrees to search in
      - lat = list of latitudes in degrees to search in
    Outputs:
      - closest_point_indexes = numpy array of grid indexes
    '''
    if debug:
        print 'Computing closest_point_indexes...'

    points = np.array([pt_lon, pt_lat]).T
    point_list = np.array([lon, lat]).T

    #closest_dist = (np.square((point_list[:, 0] - points[:, 0, None])) +
    #                np.square((point_list[:, 1] - points[:, 1, None])))

    #closest_point_indexes = np.argmin(closest_dist, axis=1)

    #Wesley's optimized version of this bottleneck
    point_list0 = point_list[:, 0]
    points0 = points[:, 0, None]
    point_list1 = point_list[:, 1]
    points1 = points[:, 1, None]

    closest_dist = ((point_list0 - points0) *
                    (point_list0 - points0) +
                    (point_list1 - points1) *
                    (point_list1 - points1)
                    )
    closest_point_indexes = np.argmin(closest_dist, axis=1)

    #Thomas' optimized version of this bottleneck
    #closest_point_indexes = np.zeros(points.shape[0])
    #for i in range(points.shape[0]):
    #    dist=((point_list-points[i,:])**2).sum(axis=1)
    #    ndx = d.argsort()
    #    closest_point_indexes[i] = ndx[0]
    if debug:
        print 'Closest dist: ', closest_dist

    if debug:
        print 'closest_point_indexes', closest_point_indexes
        print '...Passed'

    return closest_point_indexes

def interpN_at_pt(var, pt_x, pt_y, xc, yc, index, trinodes,
                  aw0, awx, awy, debug=False):
    """
    Interpol node variable any given variables at any give location.
    Inputs:
      - var = variable, numpy array, dim=(node) or (time, node) or (time, level, node)
      - pt_x = x coordinate in m to find
      - pt_y = y coordinate in m to find
      - xc = list of x coordinates of var, numpy array, dim= ele
      - yc = list of y coordinates of var, numpy array, dim= ele
      - trinodes = FVCOM trinodes, numpy array, dim=(3,nele)
      - index = index of the nearest element
      - aw0, awx, awy = grid parameters
    Outputs:
      - varInterp = var interpolate at (pt_lon, pt_lat)
    """
    if debug:
        print 'Interpolating at node...'

    n1 = trinodes[index,0]
    n2 = trinodes[index,1]
    n3 = trinodes[index,2]
    x0 = pt_x - xc[index]
    y0 = pt_y - yc[index]

    if len(var.shape)==1:
        var0 = (aw0[0,index] * var[n1]) \
             + (aw0[1,index] * var[n2]) \
             + (aw0[2,index] * var[n3])
        varX = (awx[0,index] * var[n1]) \
             + (awx[1,index] * var[n2]) \
             + (awx[2,index] * var[n3])
        varY = (awy[0,index] * var[n1]) \
             + (awy[1,index] * var[n2]) \
             + (awy[2,index] * var[n3])
    elif len(var.shape)==2:
        var0 = (aw0[0,index] * var[:,n1]) \
             + (aw0[1,index] * var[:,n2]) \
             + (aw0[2,index] * var[:,n3])
        varX = (awx[0,index] * var[:,n1]) \
             + (awx[1,index] * var[:,n2]) \
             + (awx[2,index] * var[:,n3])
        varY = (awy[0,index] * var[:,n1]) \
             + (awy[1,index] * var[:,n2]) \
             + (awy[2,index] * var[:,n3])
    else:
        var0 = (aw0[0,index] * var[:,:,n1]) \
             + (aw0[1,index] * var[:,:,n2]) \
             + (aw0[2,index] * var[:,:,n3])
        varX = (awx[0,index] * var[:,:,n1]) \
             + (awx[1,index] * var[:,:,n2]) \
             + (awx[2,index] * var[:,:,n3])
        varY = (awy[0,index] * var[:,:,n1]) \
             + (awy[1,index] * var[:,:,n2]) \
             + (awy[2,index] * var[:,:,n3])
    varPt = var0 + (varX * x0) + (varY * y0)

    if debug:
        if len(var.shape)==1:
            zi = varPt
            print 'Varpt: ', zi
        print '...Passed'

    return varPt

def interpE_at_pt(var, pt_x, pt_y, xc, yc, index, triele, trinodes,
                  a1u, a2u, debug=False):
    """
    Interpol node variable any given variables at any give location.
    Inputs:
      - var = variable, numpy array, dim=(nele) or (time, nele) or (time, level, nele)
      - pt_x = x coordinate in m to find
      - pt_y = y coordinate in m to find
      - xc = list of x coordinates of var, numpy array, dim= nele
      - yc = list of y coordinates of var, numpy array, dim= nele
      - triele = FVCOM triele, numpy array, dim=(3,nele)
      - trinodes = FVCOM trinodes, numpy array, dim=(3,nele)
      - index = index of the nearest element
      - a1u, a2u = grid parameters
    Outputs:
      - varInterp = var interpolate at (pt_lon, pt_lat)
    """
    if debug:
        print 'Interpolating at element...'

    n1 = triele[index,0]
    n2 = triele[index,1]
    n3 = triele[index,2]
    #TR comment: not quiet sure what this step does
    if n1==-1: n1 = trinodes.shape[1] + 1
    if n2==-1: n2 = trinodes.shape[1] + 1
    if n3==-1: n3 = trinodes.shape[1] + 1

    x0 = pt_x - xc[index]
    y0 = pt_y - yc[index]

    if len(var.shape)==1:
        dvardx = (a1u[0,index] * var[index]) \
               + (a1u[1,index] * var[n1]) \
               + (a1u[2,index] * var[n2]) \
               + (a1u[3,index] * var[n3])
        dvardy = (a2u[0,index] * var[index]) \
               + (a2u[1,index] * var[n1]) \
               + (a2u[2,index] * var[n2]) \
               + (a2u[3,index] * var[n3])
        varPt = var[index] + (dvardx * x0) + (dvardy * y0)
    elif len(var.shape)==2:
        dvardx = (a1u[0,index] * var[:,index]) \
               + (a1u[1,index] * var[:,n1]) \
               + (a1u[2,index] * var[:,n2]) \
               + (a1u[3,index] * var[:,n3])
        dvardy = (a2u[0,index] * var[:,index]) \
               + (a2u[1,index] * var[:,n1]) \
               + (a2u[2,index] * var[:,n2]) \
               + (a2u[3,index] * var[:,n3])
        varPt = var[:,index] + (dvardx * x0) + (dvardy * y0)
    else:
        dvardx = (a1u[0,index] * var[:,:,index]) \
               + (a1u[1,index] * var[:,:,n1]) \
               + (a1u[2,index] * var[:,:,n2]) \
               + (a1u[3,index] * var[:,:,n3])
        dvardy = (a2u[0,index] * var[:,:,index]) \
               + (a2u[1,index] * var[:,:,n1]) \
               + (a2u[2,index] * var[:,:,n2]) \
               + (a2u[3,index] * var[:,:,n3])
        varPt = var[:,:,index] + (dvardx * x0) + (dvardy * y0)

    if debug:
        if len(var.shape)==1:
            zi = varPt
            print 'Varpt: ', zi
        print '...Passed'

    return varPt

def interp_at_point(var, pt_lon, pt_lat, lon, lat,
                    index=[], trinodes=[], tri=[], debug=False):
    """
    Interpol any given variables at any give location.

    Inputs:
    ------
      - var = variable, numpy array, dim=(time, nele or node)
      - pt_lon = longitude in degrees to find
      - pt_lat = latitude in degrees to find
      - lon = list of longitudes of var, numpy array, dim=(nele or node)
      - lat = list of latitudes of var, numpy array, dim=(nele or node)
      - trinodes = FVCOM trinodes, numpy array, dim=(3,nele)
    Outputs:
      - varInterp = var interpolate at (pt_lon, pt_lat)
    """
    if debug:
        print 'Interpolating at point...'
    #Finding the right indexes

    #Triangulation
    #if debug:
    #    print triIndex, lon[triIndex], lat[triIndex]
    triIndex = trinodes[index]
    if not tri:
        tri = Tri.Triangulation(lon[triIndex], lat[triIndex], np.array([[0,1,2]]))

    trif = tri.get_trifinder()
    trif.__call__(pt_lon, pt_lat)
    #if debug:
        #if len(var.shape)==1:
            #averEl = var[triIndex]
            #print 'Var', averEl
            #inter = Tri.LinearTriInterpolator(tri, averEl)
            #zi = inter(pt_lon, pt_lat)
            #print zi
            #fig = plt.figure(figsize=(18,10))
            #ax = fig.add_subplot(111,
            #     aspect=(1.0/np.cos(np.mean(lat[triIndex])*np.pi/180.0)))
            #plt.tricontourf(newtri, averEl, cmap=plt.cm.jet)
            #cbar = plt.colorbar()
            #plt.triplot(newtri, 'ko-')
            #plt.plot(pt_lon, pt_lat, 'ko-')
            #plt.show()

    #Choose the right interpolation depending on the variable
    if len(var.shape)==1:
        triVar = np.zeros(triIndex.shape)
        triVar = var[triIndex]
        inter = Tri.LinearTriInterpolator(tri, triVar[:])
        varInterp = inter(pt_lon, pt_lat)
    elif len(var.shape)==2:
        triVar = np.zeros((var.shape[0], triIndex.shape[0]))
        triVar = var[:, triIndex]
        varInterp = np.ones(triVar.shape[0])
        for i in range(triVar.shape[0]):
            inter = Tri.LinearTriInterpolator(tri, triVar[i,:])
            varInterp[i] = inter(pt_lon, pt_lat)
    else:
        triVar = np.zeros((var.shape[0], var.shape[1], triIndex.shape[0]))
        triVar = var[:, :, triIndex]
        varInterp = np.ones(triVar.shape[:-1])
        for i in range(triVar.shape[0]):
           for j in range(triVar.shape[1]):
               inter = Tri.LinearTriInterpolator(tri, triVar[i,j,:])
               varInterp[i,j] = inter(pt_lon, pt_lat)

    if debug:
        print '...Passed'

    return varInterp

<|endoftext|>