<commit_before>import os, sh
import uuid

from PblCommand import PblCommand

class PblProjectCreator(PblCommand):
    name = 'new-project'
    help = 'Create a new Pebble project'

    def configure_subparser(self, parser):
        parser.add_argument("name", help = "Name of the project you want to create")

    def run(self, args):
        print "Creating new project {}".format(args.name)

        # User can give a path to a new project dir
        project_path = args.name
        project_name = os.path.split(project_path)[1]
        project_root = os.path.join(os.getcwd(), project_path)

        project_resources_src = os.path.join(project_root, os.path.join("resources","src"))
        project_src = os.path.join(project_root, "src")

        # Create directories
        os.makedirs(project_root)
        os.makedirs(project_resources_src)
        os.makedirs(project_src)

        # Create main .c file
        self.generate_main_file(os.path.join(project_src, "%s.c" % (project_name)))

        # Add resource file
        open(os.path.join(project_resources_src, "resource_map.json"), "w").write(FILE_DUMMY_RESOURCE_MAP)

        # Add wscript file
        open(os.path.join(project_root, "wscript"), "w").write(FILE_WSCRIPT)

        # Add .gitignore file
        open(os.path.join(project_root, ".gitignore"), "w").write(FILE_GITIGNORE)

    def generate_uuid_as_array(self):
        """
        Returns a freshly generated UUID value in string form formatted as
        a C array for inclusion in a template's "#define MY_UUID {...}"
        macro.
        """
        return ", ".join(["0x%02X" % ord(b) for b in uuid.uuid4().bytes])

    def generate_main_file(self, destination_filepath):
        """
        Generates the main file *and* replaces a dummy UUID
        value in it with a freshly generated value.
        """

        # This is the dummy UUID value in the template file.
        UUID_VALUE_TO_REPLACE="/* GENERATE YOURSELF USING `uuidgen` */ 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF"

        open(destination_filepath, "w").write(FILE_DUMMY_MAIN.replace(UUID_VALUE_TO_REPLACE, self.generate_uuid_as_array(), 1))

FILE_GITIGNORE = """
# Ignore build generated files
build
"""

FILE_WSCRIPT = """
#
# This file is the default set of rules to compile a Pebble project.
#
# Feel free to customize this to your needs.
#

top = '.'
out = 'build'

def options(ctx):
  ctx.load('pebble_sdk')

def configure(ctx):
  ctx.load('pebble_sdk')

def build(ctx):
  ctx.load('pebble_sdk')
"""

# When an empty resource map is required this can be used but...
FILE_DEFAULT_RESOURCE_MAP = """
{"friendlyVersion": "VERSION", "versionDefName": "VERSION", "media": []}
"""

# ...for the moment we need to have one with a dummy entry due to
# a bug that causes a hang when there's an empty resource map.
FILE_DUMMY_RESOURCE_MAP = """
{"friendlyVersion": "VERSION",
 "versionDefName": "VERSION",
 "media": [
     {
      "type":"raw",
      "defName":"DUMMY",
      "file":"resource_map.json"
     }
    ]
}
"""

FILE_DUMMY_MAIN = """#include <pebble_os.h>
#include <pebble_app.h>
#include <pebble_fonts.h>

#define MY_UUID { /* GENERATE YOURSELF USING `uuidgen` */ 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF }
PBL_APP_INFO(MY_UUID,
             "Template App", "Your Company",
             1, 0, /* App version */
             DEFAULT_MENU_ICON,
             APP_INFO_STANDARD_APP);

static Window *window;
static TextLayer *text_layer;

void select_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Select");
}

void up_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Up");
}

void down_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Down");
}

void config_provider(ClickConfig **config, Window *window) {
  config[BUTTON_ID_SELECT]->click.handler = select_click_handler;
  config[BUTTON_ID_UP]->click.handler = up_click_handler;
  config[BUTTON_ID_DOWN]->click.handler = down_click_handler;
}

void handle_init(void) {
  window = window_create();
  window_set_click_config_provider(window, (ClickConfigProvider) config_provider);
  window_stack_push(window, true /* Animated */);

  text_layer = text_layer_create(GRect(/* x: */ 0, /* y: */ 74,
                                       /* width: */ 144, /* height: */ 20));
  layer_add_child(window_get_root_layer(window), text_layer_get_layer(text_layer));

  text_layer_set_text(text_layer, "Press a button");
}

void handle_deinit(void) {
  text_layer_destroy(text_layer);
  window_destroy(window);
}

int main(void) {
  handle_init();

  APP_LOG(APP_LOG_LEVEL_DEBUG, "Done initializing, pushed window: %p", window);

  app_event_loop();
  handle_deinit();
}
"""
<commit_msg>Add --javascript option to pb-sdk new-project<commit_after>import os, sh
import uuid

from PblCommand import PblCommand

class PblProjectCreator(PblCommand):
    name = 'new-project'
    help = 'Create a new Pebble project'

    def configure_subparser(self, parser):
        parser.add_argument("name", help = "Name of the project you want to create")
        parser.add_argument("--javascript", action="store_true", help = "Generate javascript related files")

    def run(self, args):
        print "Creating new project {}".format(args.name)

        # User can give a path to a new project dir
        project_path = args.name
        project_name = os.path.split(project_path)[1]
        project_root = os.path.join(os.getcwd(), project_path)

        project_resources_src = os.path.join(project_root, os.path.join("resources","src"))
        project_src = os.path.join(project_root, "src")

        # Create directories
        os.makedirs(project_root)
        os.makedirs(project_resources_src)
        os.makedirs(project_src)

        # Create main .c file
        self.generate_main_file(os.path.join(project_src, "%s.c" % (project_name)))

        # Add resource file
        with open(os.path.join(project_resources_src, "resource_map.json"), "w") as f:
            f.write(FILE_DUMMY_RESOURCE_MAP)

        # Add wscript file
        with open(os.path.join(project_root, "wscript"), "w") as f:
            f.write(FILE_WSCRIPT)

        # Add .gitignore file
        with open(os.path.join(project_root, ".gitignore"), "w") as f:
            f.write(FILE_GITIGNORE)

        if args.javascript:
            project_js_src = os.path.join(project_src, "js")
            os.makedirs(project_js_src)

            with open(os.path.join(project_js_src, "appinfo.json"), "w") as f:
                f.write(FILE_DUMMY_JAVASCRIPT_APPINFO)

            with open(os.path.join(project_js_src, "pebble-js-app.js"), "w") as f:
                f.write(FILE_DUMMY_JAVASCRIPT_SRC)

    def generate_uuid_as_array(self):
        """
        Returns a freshly generated UUID value in string form formatted as
        a C array for inclusion in a template's "#define MY_UUID {...}"
        macro.
        """
        return ", ".join(["0x%02X" % ord(b) for b in uuid.uuid4().bytes])

    def generate_main_file(self, destination_filepath):
        """
        Generates the main file *and* replaces a dummy UUID
        value in it with a freshly generated value.
        """

        # This is the dummy UUID value in the template file.
        UUID_VALUE_TO_REPLACE="/* GENERATE YOURSELF USING `uuidgen` */ 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF"

        open(destination_filepath, "w").write(FILE_DUMMY_MAIN.replace(UUID_VALUE_TO_REPLACE, self.generate_uuid_as_array(), 1))

FILE_GITIGNORE = """
# Ignore build generated files
build
"""

FILE_WSCRIPT = """
#
# This file is the default set of rules to compile a Pebble project.
#
# Feel free to customize this to your needs.
#

top = '.'
out = 'build'

def options(ctx):
  ctx.load('pebble_sdk')

def configure(ctx):
  ctx.load('pebble_sdk')

def build(ctx):
  ctx.load('pebble_sdk')
"""

# When an empty resource map is required this can be used but...
FILE_DEFAULT_RESOURCE_MAP = """
{"friendlyVersion": "VERSION", "versionDefName": "VERSION", "media": []}
"""

# ...for the moment we need to have one with a dummy entry due to
# a bug that causes a hang when there's an empty resource map.
FILE_DUMMY_RESOURCE_MAP = """
{"friendlyVersion": "VERSION",
 "versionDefName": "VERSION",
 "media": [
     {
      "type":"raw",
      "defName":"DUMMY",
      "file":"resource_map.json"
     }
    ]
}
"""

FILE_DUMMY_MAIN = """#include <pebble_os.h>
#include <pebble_app.h>
#include <pebble_fonts.h>

#define MY_UUID { /* GENERATE YOURSELF USING `uuidgen` */ 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF, 0xBE, 0xEF }
PBL_APP_INFO(MY_UUID,
             "Template App", "Your Company",
             1, 0, /* App version */
             DEFAULT_MENU_ICON,
             APP_INFO_STANDARD_APP);

static Window *window;
static TextLayer *text_layer;

void select_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Select");
}

void up_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Up");
}

void down_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Down");
}

void config_provider(ClickConfig **config, Window *window) {
  config[BUTTON_ID_SELECT]->click.handler = select_click_handler;
  config[BUTTON_ID_UP]->click.handler = up_click_handler;
  config[BUTTON_ID_DOWN]->click.handler = down_click_handler;
}

void handle_init(void) {
  window = window_create();
  window_set_click_config_provider(window, (ClickConfigProvider) config_provider);
  window_stack_push(window, true /* Animated */);

  text_layer = text_layer_create(GRect(/* x: */ 0, /* y: */ 74,
                                       /* width: */ 144, /* height: */ 20));
  layer_add_child(window_get_root_layer(window), text_layer_get_layer(text_layer));

  text_layer_set_text(text_layer, "Press a button");
}

void handle_deinit(void) {
  text_layer_destroy(text_layer);
  window_destroy(window);
}

int main(void) {
  handle_init();

  APP_LOG(APP_LOG_LEVEL_DEBUG, "Done initializing, pushed window: %p", window);

  app_event_loop();
  handle_deinit();
}
"""

FILE_DUMMY_JAVASCRIPT_APPINFO = """{
  "info": {
    "app_id": "com.getpebble.example",
    "app_uuid": "FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF",
    "app_name": "Template Javascript App",
    "support_url": "http://www.yourwebsite.com",
    "version_code": 1,
    "version_label": "1.0.0",
    "app_key_type": "manual"
  },
  "app_keys": {
    "dummy": 0
  }
}
"""

FILE_DUMMY_JAVASCRIPT_SRC = """\
Pebble.showSimpleNotificationOnPebble("Hello world!", "Sent from your javascript application.")
"""

<|endoftext|>


<commit_before>"""
Transform trac wiki markup file(s) to GitHub markdown markup and write it to
new file with .md extension

Usage:
  python tractomd.py <trac markup>.wiki ...

!!!
This is not a fully fledged converter! It only takes over some repetitive tasks.
I strongly recommend to also compare both documents and adjust manually. See
reasons below.
!!!

On the one hand implementing an isolated trac syntax parser is not something you
do in a couple of hours. The guys at trac seem to have been working on this
for quite some time: https://trac.edgewall.org/wiki/WikiEngine

Additionally, we want to seize the opportunity of this conversion, to
update the documentation, i.e. each document needs to be reviewed
manually anyways.

Nevertheless, contribution to this collection of regexes are most definitely
appreciated.

Following trac syntac elements are ignored:
 - discussion
 - citation
 - macros
 - macro lists
 - list escaping
 - headings without closing `=`
 - bold italic
 - color
 - underline (because there is no underline in md)
 - escaping with preceding !
 - monospace escaping: `{{{` or {{{`}}}
 - list continuation
 - table headers
 - table row continuation
 - complex tables
 - table cell alignment
 - embedded images
 - wiki links
 - special links
"""

import sys, os, re

def main(argv):
  for wiki_filename in argv:
    if (os.path.isfile(wiki_filename) and wiki_filename.endswith(".wiki")):
      md_filename = os.path.splitext(wiki_filename)[0] + ".md"

      text = ""
      with open(wiki_filename, "r") as wiki_file:
        for line in wiki_file:
          # HEADINGS
          line = re.sub(r'(?m)^======\s+(.*?)\s+======\s*$', r'###### \1\n', line)
          line = re.sub(r'(?m)^=====\s+(.*?)\s+=====\s*$', r'##### \1\n', line)
          line = re.sub(r'(?m)^====\s+(.*?)\s+====\s*$', r'#### \1\n', line)
          line = re.sub(r'(?m)^===\s+(.*?)\s+===\s*$', r'### \1\n', line)
          line = re.sub(r'(?m)^==\s+(.*?)\s+==\s*$', r'## \1\n', line)
          line = re.sub(r'(?m)^=\s+(.*?)\s+=\s*$', r'# \1\n', line)

          # LINKS
          line = re.sub(r'\[(https?://[^\s\[\]]+)\s([^\[\]]+)\]', r'[\2](\1)', line)
          # Strip trac internal link escapes
          line = re.sub(r'!([A-Z])', r'\1', line)

          # EMPHASIS
          line = re.sub(r' \'\'\'(.*?)\'\'\' ', r' **\1** ', line) # bold
          line = re.sub(r' //(.*?)// ', r' *\1* ', line) # italic
          line = re.sub(r'\^(.*?)\^', r'\1', line) # superscript
          line = re.sub(r',,(.*?),,', r'\1', line) # subscript

          # MONOSPACE
          line = re.sub(r'\{\{\{(.*?)\n?\}\}\}', r'```\1```', line) # any code

          # LISTS
          line = re.sub(r'^(\s*)a\. (\w.*?)$', r'\g<1>1. \2', line)
          # Strip definition lists
          line = re.sub(r'::', r'', line)

          # TABLES
          line = re.sub(r'\|\|', r'|', line)

          # Strip TOC
          line = re.sub(r'\[\[TOC\(inline\)\]\]', r'', line)

          # LINEBREAKS
          line = re.sub(r'\\', r'\n', line)
          line = re.sub(r'\[\[BR\]\]', r'\n', line)

          text += line

        # Multiline replacing
        # CODE
        # http://stackoverflow.com/questions/4823468/comments-in-markdown
        comments = re.compile(r'\{\{\{\s*#!comment(.*?)\}\}\}', re.DOTALL)
        text = re.sub(comments, r'', text)

        language = re.compile(r'\{\{\{\s*#!(\w*)(.*?)\}\}\}', re.DOTALL)
        text = re.sub(language, r'```\1\2```', text)

        anycode = re.compile(r'\{\{\{(.*?)\}\}\}', re.DOTALL)
        text = re.sub(anycode, r'```\1```', text)

      with open(md_filename, "w") as md_file:
        md_file.write(text)

if __name__ == '__main__':
  main(sys.argv[1:])

<commit_msg>fix space issue<commit_after>"""
Transform trac wiki markup file(s) to GitHub markdown markup and write it to
new file with .md extension

Usage:
  python tractomd.py <trac markup>.wiki ...

!!!
This is not a fully fledged converter! It only takes over some repetitive tasks.
I strongly recommend to also compare both documents and adjust manually. See
reasons below.
!!!

On the one hand implementing an isolated trac syntax parser is not something you
do in a couple of hours. The guys at trac seem to have been working on this
for quite some time: https://trac.edgewall.org/wiki/WikiEngine

Additionally, we want to seize the opportunity of this conversion, to
update the documentation, i.e. each document needs to be reviewed
manually anyways.

Nevertheless, contribution to this collection of regexes are most definitely
appreciated.

Following trac syntac elements are ignored:
 - discussion
 - citation
 - macros (it does strip TOC)
 - macro lists
 - list escaping
 - headings and font markup without closing tag
 - bold italic
 - color
 - underline (because there is no underline in md)
 - monospace escaping: `{{{` or {{{`}}}
 - list continuation
 - table headers
 - table row continuation
 - complex tables
 - table cell alignment
 - embedded images
 - wiki links
 - special links
"""

import sys, os, re

def main(argv):
  for wiki_filename in argv:
    if (os.path.isfile(wiki_filename) and wiki_filename.endswith(".wiki")):
      md_filename = os.path.splitext(wiki_filename)[0] + ".md"

      text = ""
      with open(wiki_filename, "r") as wiki_file:
        for line in wiki_file:
          # HEADINGS
          line = re.sub(r'(?m)^======\s+(.*?)\s+======\s*$', r'###### \1\n', line)
          line = re.sub(r'(?m)^=====\s+(.*?)\s+=====\s*$', r'##### \1\n', line)
          line = re.sub(r'(?m)^====\s+(.*?)\s+====\s*$', r'#### \1\n', line)
          line = re.sub(r'(?m)^===\s+(.*?)\s+===\s*$', r'### \1\n', line)
          line = re.sub(r'(?m)^==\s+(.*?)\s+==\s*$', r'## \1\n', line)
          line = re.sub(r'(?m)^=\s+(.*?)\s+=\s*$', r'# \1\n', line)

          # LINKS
          line = re.sub(r'\[(https?://[^\s\[\]]+)\s([^\[\]]+)\]', r'[\2](\1)', line)
          # Strip trac internal link escapes
          line = re.sub(r'!([A-Z])', r'\1', line)

          # EMPHASIS
          line = re.sub(r'\'\'\'(.*?)\'\'\'', r'**\1**', line) # bold
          # Messy Magic:
          # (?<!:) is a `negative lookbehind assertion` that ensures that the ...
          # between two urls on a line (http:// ... http://) is not italicezed
          line = re.sub(r'(?<!:)//(.*?)//', r'*\1*', line) # italic
          line = re.sub(r'\^(.*?)\^', r'\1', line) # superscript
          line = re.sub(r',,(.*?),,', r'\1', line) # subscript

          # MONOSPACE
          line = re.sub(r'\{\{\{(.*?)\n?\}\}\}', r'```\1```', line) # any code

          # LISTS
          line = re.sub(r'^(\s*)a\. (\w.*?)$', r'\g<1>1. \2', line)
          # Strip definition lists
          line = re.sub(r'::', r'', line)

          # TABLES
          line = re.sub(r'\|\|', r'|', line)

          # Strip TOC
          line = re.sub(r'\[\[TOC\(inline\)\]\]', r'', line)

          # LINEBREAKS
          line = re.sub(r'\\', r'\n', line)
          line = re.sub(r'\[\[BR\]\]', r'\n', line)

          text += line

        # Multiline replacing
        # CODE
        # http://stackoverflow.com/questions/4823468/comments-in-markdown
        comments = re.compile(r'\{\{\{\s*#!comment(.*?)\}\}\}', re.DOTALL)
        text = re.sub(comments, r'', text)

        language = re.compile(r'\{\{\{\s*#!(\w*)(.*?)\}\}\}', re.DOTALL)
        text = re.sub(language, r'```\1\2```', text)

        anycode = re.compile(r'\{\{\{(.*?)\}\}\}', re.DOTALL)
        text = re.sub(anycode, r'```\1```', text)

      with open(md_filename, "w") as md_file:
        md_file.write(text)

if __name__ == '__main__':
  main(sys.argv[1:])

<|endoftext|>


<commit_before>import itertools
import logging
import unittest
from pathlib import Path, PurePath
from tempfile import TemporaryDirectory
from test.testutils import GraphHelper
from typing import Tuple, cast

import pytest

from rdflib import RDF, Graph, Literal, Namespace, URIRef
from rdflib.graph import ConjunctiveGraph

@pytest.mark.parametrize(
    "format, tuple_index, is_keyword",
    [
        (format, tuple_index, keyword)
        for format, (tuple_index, keyword) in itertools.product(
            ["turtle", "n3", "trig"],
            [
                (0, False),
                (1, True),
                (2, False),
            ],
        )
    ]
    + [("trig", 3, False)],
)
def test_rdf_type(format: str, tuple_index: int, is_keyword: bool) -> None:
    NS = Namespace("example:")
    graph = ConjunctiveGraph()
    graph.bind("eg", NS)
    nodes = [NS.subj, NS.pred, NS.obj, NS.graph]
    nodes[tuple_index] = RDF.type
    quad = cast(Tuple[URIRef, URIRef, URIRef, URIRef], tuple(nodes))
    graph.add(quad)
    data = graph.serialize(format=format)
    logging.info("data = %s", data)
    assert NS in data
    if is_keyword:
        assert str(RDF) not in data
    else:
        assert str(RDF) in data
    parsed_graph = ConjunctiveGraph()
    parsed_graph.parse(data=data, format=format)
    GraphHelper.assert_triple_sets_equals(graph, parsed_graph)

class TestSerialize(unittest.TestCase):
    def setUp(self) -> None:

        graph = Graph()
        subject = URIRef("example:subject")
        predicate = URIRef("example:predicate")
        object = Literal("日本語の表記体系", lang="jpx")
        self.triple = (
            subject,
            predicate,
            object,
        )
        graph.add(self.triple)
        self.graph = graph
        return super().setUp()

    def test_serialize_to_purepath(self):
        with TemporaryDirectory() as td:
            tfpath = PurePath(td) / "out.nt"
            self.graph.serialize(destination=tfpath, format="nt", encoding="utf-8")
            graph_check = Graph()
            graph_check.parse(source=tfpath, format="nt")

        self.assertEqual(self.triple, next(iter(graph_check)))

    def test_serialize_to_path(self):
        with TemporaryDirectory() as td:
            tfpath = Path(td) / "out.nt"
            self.graph.serialize(destination=tfpath, format="nt", encoding="utf-8")
            graph_check = Graph()
            graph_check.parse(source=tfpath, format="nt")

        self.assertEqual(self.triple, next(iter(graph_check)))

    def test_serialize_to_neturl(self):
        with self.assertRaises(ValueError) as raised:
            self.graph.serialize(
                destination="http://example.com/", format="nt", encoding="utf-8"
            )
        self.assertIn("destination", f"{raised.exception}")

    def test_serialize_to_fileurl(self):
        with TemporaryDirectory() as td:
            tfpath = Path(td) / "out.nt"
            tfurl = tfpath.as_uri()
            self.assertRegex(tfurl, r"^file:")
            self.assertFalse(tfpath.exists())
            self.graph.serialize(destination=tfurl, format="nt", encoding="utf-8")
            self.assertTrue(tfpath.exists())
            graph_check = Graph()
            graph_check.parse(source=tfpath, format="nt")
        self.assertEqual(self.triple, next(iter(graph_check)))
<commit_msg>test: convert `test/test_serializers/test_serializer.py` to pytest<commit_after>import itertools
import logging
import re
from pathlib import Path
from test.testutils import GraphHelper
from typing import Tuple, cast

import pytest

from rdflib import RDF, Graph, Literal, Namespace, URIRef
from rdflib.graph import ConjunctiveGraph

@pytest.mark.parametrize(
    "format, tuple_index, is_keyword",
    [
        (format, tuple_index, keyword)
        for format, (tuple_index, keyword) in itertools.product(
            ["turtle", "n3", "trig"],
            [
                (0, False),
                (1, True),
                (2, False),
            ],
        )
    ]
    + [("trig", 3, False)],
)
def test_rdf_type(format: str, tuple_index: int, is_keyword: bool) -> None:
    NS = Namespace("example:")
    graph = ConjunctiveGraph()
    graph.bind("eg", NS)
    nodes = [NS.subj, NS.pred, NS.obj, NS.graph]
    nodes[tuple_index] = RDF.type
    quad = cast(Tuple[URIRef, URIRef, URIRef, URIRef], tuple(nodes))
    graph.add(quad)
    data = graph.serialize(format=format)
    logging.info("data = %s", data)
    assert NS in data
    if is_keyword:
        assert str(RDF) not in data
    else:
        assert str(RDF) in data
    parsed_graph = ConjunctiveGraph()
    parsed_graph.parse(data=data, format=format)
    GraphHelper.assert_triple_sets_equals(graph, parsed_graph)

EG = Namespace("example:")

@pytest.fixture
def simple_graph() -> Graph:
    graph = Graph()
    graph.add((EG.subject, EG.predicate, Literal("日本語の表記体系", lang="jpx")))
    return graph

def test_serialize_to_purepath(tmp_path: Path, simple_graph: Graph):
    tfpath = tmp_path / "out.nt"
    simple_graph.serialize(destination=tfpath, format="nt", encoding="utf-8")
    graph_check = Graph()
    graph_check.parse(source=tfpath, format="nt")

    GraphHelper.assert_triple_sets_equals(simple_graph, graph_check)

def test_serialize_to_path(tmp_path: Path, simple_graph: Graph):
    tfpath = tmp_path / "out.nt"
    simple_graph.serialize(destination=tfpath, format="nt", encoding="utf-8")
    graph_check = Graph()
    graph_check.parse(source=tfpath, format="nt")

    GraphHelper.assert_triple_sets_equals(simple_graph, graph_check)

def test_serialize_to_neturl(simple_graph: Graph):
    with pytest.raises(ValueError) as raised:
        simple_graph.serialize(
            destination="http://example.com/", format="nt", encoding="utf-8"
        )
    assert "destination" in f"{raised.value}"

def test_serialize_to_fileurl(tmp_path: Path, simple_graph: Graph):
    tfpath = tmp_path / "out.nt"
    tfurl = tfpath.as_uri()
    assert re.match(r"^file:", tfurl)
    assert not tfpath.exists()
    simple_graph.serialize(destination=tfurl, format="nt", encoding="utf-8")
    assert tfpath.exists()
    graph_check = Graph()
    graph_check.parse(source=tfpath, format="nt")
    GraphHelper.assert_triple_sets_equals(simple_graph, graph_check)
<|endoftext|>


<commit_before>def run_command(command, cmds, opts):
    '''Locate and call `command` handler in a map `cmds`. The handler is called
    with `opts` as its first argument.

    Will throw a SystemExit(1) if command is not found.

    :param command: command as string
    :param cmds: map of commands, command name being the key, a function being the value
    :type cmds: map
    :param opts: options passed to command handler

    '''
    handler = cmds.get(command, None)
    if not handler:
        logging.error('command \'%s\' not supported', command)
        raise SystemExit(1)
    else:
        handler(opts)
<commit_msg>common: add missing import<commit_after>import logging

def run_command(command, cmds, opts):
    '''Locate and call `command` handler in a map `cmds`. The handler is called
    with `opts` as its first argument.

    Will throw a SystemExit(1) if command is not found.

    :param command: command as string
    :param cmds: map of commands, command name being the key, a function being the value
    :type cmds: map
    :param opts: options passed to command handler

    '''
    handler = cmds.get(command, None)
    if not handler:
        logging.error('command \'%s\' not supported', command)
        raise SystemExit(1)
    else:
        handler(opts)
<|endoftext|>


<commit_before>#!/usr/bin/python3

import sys
import gzip
import csv
import json

from entitygenerator import EntityElement, DictElement

class TradeElement(DictElement):
    def __init__(self, dataFile=None,
                       countryFilter = None,
                       **kwargs ):

        DictElement.__init__(self, **kwargs)

        trades = [ ]
        fields = None

        f = gzip.open(dataFile, 'rt', encoding='utf-8')

        # dictreader defaults to using first row as header (good!!)
        reader = csv.DictReader(f, delimiter=',', quotechar='"')
        for d in reader:
            if countryFilter is not None:
                if d['country'] not in countryFilter: continue

            trades.append(d)

        f.close()

        self.fields = fields
        self.trades = trades

        return

    @staticmethod
    def digits_of(number):
        return [int(i) for i in str(number)]

    @staticmethod
    def luhn_checksum(card_number):
        digits = TradeElement.digits_of(card_number)
        odd_digits = digits[-1::-2]
        even_digits = digits[-2::-2]
        total = sum(odd_digits)
        for digit in even_digits:
            total += sum(TradeElement.digits_of(2 * digit))
        return total % 10

    @staticmethod
    def is_luhn_valid(card_number):
        return luhn_checksum(card_number) == 0

    def create(self, **kwargs):
        r = int(EntityElement.pool.next() * len(self.trades))
        d = self.trades[r].copy()

        print(str(d))

        iin_start = int(d['iin_start'])
        iin_end = d['iin_end']
        iin_end = int(iin_end) if iin_end is not '' else iin_start
        iin_range = iin_end - iin_start

        if iin_start == iin_end:
            r = 0
        else:
            rnd = EntityElement.pool.next()
            r = int(rnd * (iin_range + 1))

        iin = iin_start + r
        iin = str(iin)

        if d['number_length'] is not "":
             acct_len = int(d['number_length'])
        else:
            acct_len = 16
            if d['scheme'] == 'AMEX':
                acct_len = 15

        # lots of fields... don't need them all.
        for key in [ 'iin_start', 'iin_end', 'bank_logo', 'number_length',
                     'bank_url', 'bank_city' ]:
            del(d[key])

        # build the random account number
        n = acct_len - 6 - 1   # less BIN number and Luhn check digit
        acct = iin + str(int(EntityElement.pool.next() * (10 ** n))).zfill(n)
        acct += str(self.luhn_checksum( acct ))
        d['account_no'] = acct

        DictElement.addChildren(self, d, **kwargs)
        return d

class USCreditAccount(TradeElement):
    def __init__( self, dataFile = 'data/ranges.csv.gz',
                        countryFilter = [ 'US' ],
                        **kwargs ):

        TradeElement.__init__(self, dataFile = dataFile,
                                    countryFilter = countryFilter,
                                    **kwargs)
        return

def main(argv):
    trade = USCreditAccount()

    for i in range(10000):
        print(json.dumps(trade.create()))

    return 0

if __name__ == '__main__':
    sys.exit(main(sys.argv))
<commit_msg>remove debug output<commit_after>#!/usr/bin/python3

import sys
import gzip
import csv
import json

from entitygenerator import EntityElement, DictElement

class TradeElement(DictElement):
    def __init__(self, dataFile=None,
                       countryFilter = None,
                       **kwargs ):

        DictElement.__init__(self, **kwargs)

        trades = [ ]
        fields = None

        f = gzip.open(dataFile, 'rt', encoding='utf-8')

        # dictreader defaults to using first row as header (good!!)
        reader = csv.DictReader(f, delimiter=',', quotechar='"')
        for d in reader:
            if countryFilter is not None:
                if d['country'] not in countryFilter: continue

            trades.append(d)

        f.close()

        self.fields = fields
        self.trades = trades

        return

    @staticmethod
    def digits_of(number):
        return [int(i) for i in str(number)]

    @staticmethod
    def luhn_checksum(card_number):
        digits = TradeElement.digits_of(card_number)
        odd_digits = digits[-1::-2]
        even_digits = digits[-2::-2]
        total = sum(odd_digits)
        for digit in even_digits:
            total += sum(TradeElement.digits_of(2 * digit))
        return total % 10

    @staticmethod
    def is_luhn_valid(card_number):
        return luhn_checksum(card_number) == 0

    def create(self, **kwargs):
        r = int(EntityElement.pool.next() * len(self.trades))
        d = self.trades[r].copy()

        #print(str(d))

        iin_start = int(d['iin_start'])
        iin_end = d['iin_end']
        iin_end = int(iin_end) if iin_end is not '' else iin_start
        iin_range = iin_end - iin_start

        if iin_start == iin_end:
            r = 0
        else:
            rnd = EntityElement.pool.next()
            r = int(rnd * (iin_range + 1))

        iin = iin_start + r
        iin = str(iin)

        if d['number_length'] is not "":
             acct_len = int(d['number_length'])
        else:
            acct_len = 16
            if d['scheme'] == 'AMEX':
                acct_len = 15

        # lots of fields... don't need them all.
        for key in [ 'iin_start', 'iin_end', 'bank_logo', 'number_length',
                     'bank_url', 'bank_city' ]:
            del(d[key])

        # build the random account number
        n = acct_len - 6 - 1   # less BIN number and Luhn check digit
        acct = iin + str(int(EntityElement.pool.next() * (10 ** n))).zfill(n)
        acct += str(self.luhn_checksum( acct ))
        d['account_no'] = acct

        DictElement.addChildren(self, d, **kwargs)
        return d

class USCreditAccount(TradeElement):
    def __init__( self, dataFile = 'data/ranges.csv.gz',
                        countryFilter = [ 'US' ],
                        **kwargs ):

        TradeElement.__init__(self, dataFile = dataFile,
                                    countryFilter = countryFilter,
                                    **kwargs)
        return

def main(argv):
    trade = USCreditAccount()

    for i in range(10000):
        print(json.dumps(trade.create()))

    return 0

if __name__ == '__main__':
    sys.exit(main(sys.argv))
<|endoftext|>


<commit_before>import os

import geopandas
from flask_restplus import Namespace, Resource, fields, abort
import shutil
from staticmap import StaticMap, Polygon

import cea.config
import cea.inputlocator
from cea.utilities.standardize_coordinates import get_geographic_coordinate_system

api = Namespace('Project', description='Current project for CEA')

# PATH_REGEX = r'(^[a-zA-Z]:\\[\\\S|*\S]?.*$)|(^(/[^/ ]*)+/?$)'

PROJECT_PATH_MODEL = api.model('Project Path', {
    'path': fields.String(description='Path of Project'),
    'scenario': fields.String(description='Path of Project')
})

PROJECT_MODEL = api.inherit('Project', PROJECT_PATH_MODEL, {
    'name': fields.String(description='Name of Project'),
    'scenario': fields.String(description='Name of Current Scenario'),
    'scenarios': fields.List(fields.String, description='Name of Current Scenario')
})

@api.route('/')
class Project(Resource):
    @api.marshal_with(PROJECT_MODEL)
    def get(self):
        config = cea.config.Configuration()
        name = os.path.basename(config.project)
        return {'name': name, 'path': config.project, 'scenario': config.scenario_name,
                'scenarios': config.get_parameter('general:scenario-name')._choices}

    @api.doc(body=PROJECT_PATH_MODEL, responses={200: 'Success', 400: 'Invalid Path given'})
    def post(self):
        config = cea.config.Configuration()
        payload = api.payload

        if 'path' in payload:
            path = payload['path']
            if os.path.exists(path):
                config.project = path
                if 'scenario' not in payload:
                    config.scenario_name = ''
                    config.save()
                    return {'message': 'Project path changed'}
            else:
                abort(400, 'Path given does not exist')

        if 'scenario' in payload:
            scenario = payload['scenario']
            choices = config.get_parameter('general:scenario-name')._choices
            if scenario in choices:
                config.scenario_name = scenario
                config.save()
                return {'message': 'Scenario changed'}
            else:
                abort(400, 'Scenario does not exist', choices=choices)

@api.route('/<string:scenario>')
class Scenario(Resource):
    def get(self, scenario):
        config = cea.config.Configuration()
        choices = config.get_parameter('general:scenario-name')._choices
        if scenario in choices:
            return {'name': scenario}
        else:
            abort(400, 'Scenario does not exist', choices=choices)

    def delete(self, scenario):
        config = cea.config.Configuration()
        choices = config.get_parameter('general:scenario-name')._choices
        if scenario in choices:
            scenario_path = os.path.join(config.project, scenario)
            try:
                shutil.rmtree(scenario_path)
            except WindowsError:
                abort(400, 'Make sure that the scenario you are trying to delete is not open in any application.
'
                           'Try and refresh the page again.')
            return {'scenarios': config.get_parameter('general:scenario-name')._choices}
        else:
            abort(400, 'Scenario does not exist', choices=choices)

@api.route('/<string:scenario>/image')
class ScenarioImage(Resource):
    def get(self, scenario):
        config = cea.config.Configuration()
        choices = config.get_parameter('general:scenario-name')._choices
        if scenario in choices:
            locator = cea.inputlocator.InputLocator(os.path.join(config.project, scenario))
            zone_path = locator.get_zone_geometry()
            if os.path.isfile(zone_path):
                cache_path = os.path.join(config.project, '.cache')
                image_path = os.path.join(cache_path, scenario + '.png')

                zone_modified = os.path.getmtime(zone_path)
                if not os.path.isfile(image_path):
                    image_modified = 0
                else:
                    image_modified = os.path.getmtime(image_path)

                if zone_modified > image_modified:
                    # Make sure .cache folder exists
                    if not os.path.exists(cache_path):
                        os.makedirs(cache_path)

                    try:
                        zone_df = geopandas.read_file(zone_path)
                        zone_df = zone_df.to_crs(get_geographic_coordinate_system())
                        polygons = zone_df['geometry']

                        polygons = [list(polygons.geometry.exterior[row_id].coords) for row_id in range(polygons.shape[0])]

                        m = StaticMap(256, 160)
                        for polygon in polygons:
                            out = Polygon(polygon, 'blue', 'black', False)
                            m.add_polygon(out)

                        image = m.render()
                        image.save(image_path)
                    except Exception as e:
                        abort(400, str(e))

                import base64
                with open(image_path, 'rb') as imgFile:
                    image = base64.b64encode(imgFile.read())

                return {'image': image}
            abort(400, 'Zone file not found')
        else:
            abort(400, 'Scenario does not exist', choices=choices)

<commit_msg>Set scenario to '' if delete current scenario<commit_after>import os

import geopandas
from flask_restplus import Namespace, Resource, fields, abort
import shutil
from staticmap import StaticMap, Polygon

import cea.config
import cea.inputlocator
from cea.utilities.standardize_coordinates import get_geographic_coordinate_system

api = Namespace('Project', description='Current project for CEA')

# PATH_REGEX = r'(^[a-zA-Z]:\\[\\\S|*\S]?.*$)|(^(/[^/ ]*)+/?$)'

PROJECT_PATH_MODEL = api.model('Project Path', {
    'path': fields.String(description='Path of Project'),
    'scenario': fields.String(description='Path of Project')
})

PROJECT_MODEL = api.inherit('Project', PROJECT_PATH_MODEL, {
    'name': fields.String(description='Name of Project'),
    'scenario': fields.String(description='Name of Current Scenario'),
    'scenarios': fields.List(fields.String, description='Name of Current Scenario')
})

@api.route('/')
class Project(Resource):
    @api.marshal_with(PROJECT_MODEL)
    def get(self):
        config = cea.config.Configuration()
        name = os.path.basename(config.project)
        return {'name': name, 'path': config.project, 'scenario': config.scenario_name,
                'scenarios': config.get_parameter('general:scenario-name')._choices}

    @api.doc(body=PROJECT_PATH_MODEL, responses={200: 'Success', 400: 'Invalid Path given'})
    def post(self):
        config = cea.config.Configuration()
        payload = api.payload

        if 'path' in payload:
            path = payload['path']
            if os.path.exists(path):
                config.project = path
                if 'scenario' not in payload:
                    config.scenario_name = ''
                    config.save()
                    return {'message': 'Project path changed'}
            else:
                abort(400, 'Path given does not exist')

        if 'scenario' in payload:
            scenario = payload['scenario']
            choices = config.get_parameter('general:scenario-name')._choices
            if scenario in choices:
                config.scenario_name = scenario
                config.save()
                return {'message': 'Scenario changed'}
            else:
                abort(400, 'Scenario does not exist', choices=choices)

@api.route('/<string:scenario>')
class Scenario(Resource):
    def get(self, scenario):
        config = cea.config.Configuration()
        choices = config.get_parameter('general:scenario-name')._choices
        if scenario in choices:
            return {'name': scenario}
        else:
            abort(400, 'Scenario does not exist', choices=choices)

    def delete(self, scenario):
        config = cea.config.Configuration()
        choices = config.get_parameter('general:scenario-name')._choices
        if scenario in choices:
            scenario_path = os.path.join(config.project, scenario)
            try:
                if config.scenario_name == scenario:
                    config.scenario_name = ''
                    config.save()
                shutil.rmtree(scenario_path)

            except WindowsError:
                abort(400, 'Make sure that the scenario you are trying to delete is not open in any application.
'
                           'Try and refresh the page again.')
            return {'scenarios': config.get_parameter('general:scenario-name')._choices}
        else:
            abort(400, 'Scenario does not exist', choices=choices)

@api.route('/<string:scenario>/image')
class ScenarioImage(Resource):
    def get(self, scenario):
        config = cea.config.Configuration()
        choices = config.get_parameter('general:scenario-name')._choices
        if scenario in choices:
            locator = cea.inputlocator.InputLocator(os.path.join(config.project, scenario))
            zone_path = locator.get_zone_geometry()
            if os.path.isfile(zone_path):
                cache_path = os.path.join(config.project, '.cache')
                image_path = os.path.join(cache_path, scenario + '.png')

                zone_modified = os.path.getmtime(zone_path)
                if not os.path.isfile(image_path):
                    image_modified = 0
                else:
                    image_modified = os.path.getmtime(image_path)

                if zone_modified > image_modified:
                    # Make sure .cache folder exists
                    if not os.path.exists(cache_path):
                        os.makedirs(cache_path)

                    try:
                        zone_df = geopandas.read_file(zone_path)
                        zone_df = zone_df.to_crs(get_geographic_coordinate_system())
                        polygons = zone_df['geometry']

                        polygons = [list(polygons.geometry.exterior[row_id].coords) for row_id in range(polygons.shape[0])]

                        m = StaticMap(256, 160)
                        for polygon in polygons:
                            out = Polygon(polygon, 'blue', 'black', False)
                            m.add_polygon(out)

                        image = m.render()
                        image.save(image_path)
                    except Exception as e:
                        abort(400, str(e))

                import base64
                with open(image_path, 'rb') as imgFile:
                    image = base64.b64encode(imgFile.read())

                return {'image': image}
            abort(400, 'Zone file not found')
        else:
            abort(400, 'Scenario does not exist', choices=choices)

<|endoftext|>


<commit_before># dashboard/views.py
# Jake Malley
# 23/01/15

"""
Define the routes for the dashboard blueprint.
"""

from flask import flash, redirect, render_template, \
                request, url_for, Blueprint
from flask.ext.login import login_required, current_user
from traininglog.models import Exercise
from datetime import datetime

# Setup the dashboard blueprint.
dashboard_blueprint = Blueprint(
    'dashboard',__name__,
    template_folder='templates'
    )

# Define the routes
@dashboard_blueprint.route('/')
@login_required
def dashboard():

    # Get all the exercise data.
    exercise_data = Exercise.query.filter_by(member=current_user).order_by(Exercise.id.desc()).limit(8).all()

    # Data for the pie chart
    pie_chart_data = [{
                    'value': get_hours_running(),
                    'color':"#F7464A",
                    'highlight': "#FF5A5E",
                    'label': "Running"
                },
                {
                    'value': get_hours_cycling(),
                    'color': "#46BFBD",
                    'highlight': "#5AD3D1",
                    'label': "Swimming"
                },
                {
                    'value': get_hours_swimming(),
                    'color': "#FDB45C",
                    'highlight': "#FFC870",
                    'label': "Cycling"
                }]

    # Data for the line chart.
    line_chart_data = [data.get_calories_burned() for data in exercise_data][::-1]
    # Empty labels for the line chart.
    line_chart_labels = [data.get_datetime() for data in exercise_data][::-1]

    # Get todays date.
    now = datetime.utcnow()

    # Get total exercised for this year.
    exercise_total_year = get_exercise_total(datetime(now.year,1,1))
    exercise_total_today = get_exercise_total(now)

    progress_today_width = int(float(exercise_total_today/4)*100)
    progress_year_width = int(float(exercise_total_today/1460)*100)

    return render_template('dashboard.html',pie_chart_data=pie_chart_data, exercise_data=exercise_data,line_chart_data=line_chart_data,line_chart_labels=line_chart_labels,exercise_total_year=exercise_total_year,exercise_total_today=exercise_total_today,progress_today_width=progress_today_width,progress_year_width=progress_year_width)

def get_hours_running():
    """
    Returns the total hours of running.
    """
    exercise_data = Exercise.query.filter_by(member=current_user,exercise_type='running').all()
    total = 0
    for data in exercise_data:
        total += data.exercise_duration
    return total

def get_hours_cycling():
    """
    Returns the total hours of cycling.
    """
    exercise_data = Exercise.query.filter_by(member=current_user,exercise_type='cycling').all()
    total = 0
    for data in exercise_data:
        total += data.exercise_duration
    return total

def get_hours_swimming():
    """
    Returns the total hours of swimming.
    """
    exercise_data = Exercise.query.filter_by(member=current_user,exercise_type='swimming').all()
    total = 0
    for data in exercise_data:
        total += data.exercise_duration
    return total

def get_exercise_total(date):
    """
    Returns the number of hours exercised on the date given.
    """
    exercise_data = Exercise.query.filter(Exercise.date>date.date()).filter_by(member=current_user).all()
    total = 0
    for data in exercise_data:
        total += data.exercise_duration
    return total<commit_msg>Fixed pie chart.<commit_after># dashboard/views.py
# Jake Malley
# 23/01/15

"""
Define the routes for the dashboard blueprint.
"""

from flask import flash, redirect, render_template, \
                request, url_for, Blueprint
from flask.ext.login import login_required, current_user
from traininglog.models import Exercise
from datetime import datetime

# Setup the dashboard blueprint.
dashboard_blueprint = Blueprint(
    'dashboard',__name__,
    template_folder='templates'
    )

# Define the routes
@dashboard_blueprint.route('/')
@login_required
def dashboard():

    # Get all the exercise data.
    exercise_data = Exercise.query.filter_by(member=current_user).order_by(Exercise.id.desc()).limit(8).all()

    # Data for the pie chart
    pie_chart_data = [{
                    'value': get_hours_running(),
                    'color':"#F7464A",
                    'highlight': "#FF5A5E",
                    'label': "Hours Running"
                },
                {
                    'value': get_hours_swimming(),
                    'color': "#46BFBD",
                    'highlight': "#5AD3D1",
                    'label': "Hours Swimming"
                },
                {
                    'value': get_hours_cycling(),
                    'color': "#FDB45C",
                    'highlight': "#FFC870",
                    'label': "Hours Cycling"
                }]

    # Data for the line chart.
    line_chart_data = [data.get_calories_burned() for data in exercise_data][::-1]
    # Empty labels for the line chart.
    line_chart_labels = [data.get_datetime() for data in exercise_data][::-1]

    # Get todays date.
    now = datetime.utcnow()

    # Get total exercised for this year.
    exercise_total_year = get_exercise_total(datetime(now.year,1,1))
    exercise_total_today = get_exercise_total(now)

    progress_today_width = int(float(exercise_total_today/4)*100)
    progress_year_width = int(float(exercise_total_today/1460)*100)

    return render_template('dashboard.html',pie_chart_data=pie_chart_data, exercise_data=exercise_data,line_chart_data=line_chart_data,line_chart_labels=line_chart_labels,exercise_total_year=exercise_total_year,exercise_total_today=exercise_total_today,progress_today_width=progress_today_width,progress_year_width=progress_year_width)

def get_hours_running():
    """
    Returns the total hours of running.
    """
    exercise_data = Exercise.query.filter_by(member=current_user,exercise_type='running').all()
    total = 0
    for data in exercise_data:
        total += data.exercise_duration
    return total

def get_hours_cycling():
    """
    Returns the total hours of cycling.
    """
    exercise_data = Exercise.query.filter_by(member=current_user,exercise_type='cycling').all()
    total = 0
    for data in exercise_data:
        total += data.exercise_duration
    return total

def get_hours_swimming():
    """
    Returns the total hours of swimming.
    """
    exercise_data = Exercise.query.filter_by(member=current_user,exercise_type='swimming').all()
    total = 0
    for data in exercise_data:
        total += data.exercise_duration
    return total

def get_exercise_total(date):
    """
    Returns the number of hours exercised on the date given.
    """
    exercise_data = Exercise.query.filter(Exercise.date>date.date()).filter_by(member=current_user).all()
    total = 0
    for data in exercise_data:
        total += data.exercise_duration
    return total<|endoftext|>


<commit_before>from optparse import OptionParser

from fabric.api import run

from perfrunner.helpers import Helper
from perfrunner.helpers.rest import RestHelper
from perfrunner.helpers.remote import RemoteHelper, all_hosts

class ClusterManager(Helper):

    def __init__(self, cluster_spec, test_config):
        super(ClusterManager, self).__init__(cluster_spec, test_config)

        self.rest_helper = RestHelper(cluster_spec)

    @all_hosts
    def clean_data_path(self):
        for path in (self.data_path, self.index_path):
            run('rm -fr {0}/*'.format(path))

    def set_data_path(self):
        self.clean_data_path()
        for cluster in self.clusters:
            for host_port in cluster:
                self.rest_helper.set_data_path(host_port)

    def set_auth(self):
        for cluster in self.clusters:
            for host_port in cluster:
                self.rest_helper.set_auth(host_port)

    def set_mem_quota(self):
        for cluster in self.clusters:
            for host_port in cluster:
                self.rest_helper.set_mem_quota(host_port, self.mem_quota)

    def add_nodes(self):
        for cluster in self.clusters:
            master = cluster[0]
            for host_port in cluster[1:self.initial_nodes]:
                host = host_port.split(':')[0]
                self.rest_helper.add_node(master, host)

    def rebalance(self):
        for cluster in self.clusters:
            master = cluster[0]
            known_nodes = cluster[:self.initial_nodes]
            ejected_nodes = []
            self.rest_helper.rebalance(master, known_nodes, ejected_nodes)
            self.rest_helper.monitor_rebalance(master)

    def create_buckets(self):
        ram_quota = self.mem_quota / self.initial_nodes
        buckets = ('bucket-{0}'.format(i + 1) for i in xrange(self.buckets))
        for cluster in self.clusters:
            master = cluster[0]
            for name in buckets:
                self.rest_helper.create_bucket(master, name, ram_quota)

def get_options():
    usage = '%prog -c cluster -t test_config'

    parser = OptionParser(usage)

    parser.add_option('-c', dest='cluster',
                      help='path to cluster specification file',
                      metavar='cluster.spec')
    parser.add_option('-t', dest='test_config',
                      help='path to test configuration file',
                      metavar='my_test.test')

    options, _ = parser.parse_args()
    if not options.cluster or not options.test_config:
        parser.error('Missing mandatory parameter')

    return options

def main():
    options = get_options()

    cm = ClusterManager(options.cluster, options.test_config)
    cm.set_data_path()
    cm.set_auth()
    cm.set_mem_quota()
    cm.add_nodes()
    cm.rebalance()
    cm.create_buckets()

    rh = RemoteHelper(options.cluster)
    rh.reset_swap()

if __name__ == '__main__':
    main()
<commit_msg>fix bucket ram quota calculation<commit_after>from optparse import OptionParser

from fabric.api import run

from perfrunner.helpers import Helper
from perfrunner.helpers.rest import RestHelper
from perfrunner.helpers.remote import RemoteHelper, all_hosts

class ClusterManager(Helper):

    def __init__(self, cluster_spec, test_config):
        super(ClusterManager, self).__init__(cluster_spec, test_config)

        self.rest_helper = RestHelper(cluster_spec)

    @all_hosts
    def clean_data_path(self):
        for path in (self.data_path, self.index_path):
            run('rm -fr {0}/*'.format(path))

    def set_data_path(self):
        self.clean_data_path()
        for cluster in self.clusters:
            for host_port in cluster:
                self.rest_helper.set_data_path(host_port)

    def set_auth(self):
        for cluster in self.clusters:
            for host_port in cluster:
                self.rest_helper.set_auth(host_port)

    def set_mem_quota(self):
        for cluster in self.clusters:
            for host_port in cluster:
                self.rest_helper.set_mem_quota(host_port, self.mem_quota)

    def add_nodes(self):
        for cluster in self.clusters:
            master = cluster[0]
            for host_port in cluster[1:self.initial_nodes]:
                host = host_port.split(':')[0]
                self.rest_helper.add_node(master, host)

    def rebalance(self):
        for cluster in self.clusters:
            master = cluster[0]
            known_nodes = cluster[:self.initial_nodes]
            ejected_nodes = []
            self.rest_helper.rebalance(master, known_nodes, ejected_nodes)
            self.rest_helper.monitor_rebalance(master)

    def create_buckets(self):
        ram_quota = self.mem_quota / self.buckets
        buckets = ('bucket-{0}'.format(i + 1) for i in xrange(self.buckets))
        for cluster in self.clusters:
            master = cluster[0]
            for name in buckets:
                self.rest_helper.create_bucket(master, name, ram_quota)

def get_options():
    usage = '%prog -c cluster -t test_config'

    parser = OptionParser(usage)

    parser.add_option('-c', dest='cluster',
                      help='path to cluster specification file',
                      metavar='cluster.spec')
    parser.add_option('-t', dest='test_config',
                      help='path to test configuration file',
                      metavar='my_test.test')

    options, _ = parser.parse_args()
    if not options.cluster or not options.test_config:
        parser.error('Missing mandatory parameter')

    return options

def main():
    options = get_options()

    cm = ClusterManager(options.cluster, options.test_config)
    cm.set_data_path()
    cm.set_auth()
    cm.set_mem_quota()
    cm.add_nodes()
    cm.rebalance()
    cm.create_buckets()

    rh = RemoteHelper(options.cluster)
    rh.reset_swap()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from nose.tools import *
from substitute import *
from substitute.complaint import *
expect_that = assert_true

# arg = Argument
# kwarg = Keyword Argument

def test_expect_received_data():
    '''Substitute - Verify (args): Method was called with data'''
    component = Substitute()
    component.method('hello')
    expect_that(component.method.received('hello'))

def test_expect_received_type():
    '''Substitute - Verify (args): Method was called with type'''
    component = Substitute()
    component.method('hello')
    expect_that(component.method.received_any(str))

@raises(MissingCallComplaint)
def test_was_never_called_fails():
    '''Substitute - Verify (args): Method was never called (data)'''
    component = Substitute()
    expect_that(component.method.received('hello'))

@raises(MissingCallComplaint)
def text_expect_was_never_called_fails_2():
    '''Substitute - Verify (args): Method was never called (type)'''
    component = Substitute()
    expect_that(component.method.received_any(str))

@raises(UnexpectedKeywordComplaint)
def test_expect_received_unexpected_keyword_fails():
    '''Substitute - Verify (args): Method was called with unexpected kwarg'''
    component = Substitute()
    component.method(name='hello')
    expect_that(component.method.received('hello'))

@raises(ArgumentMissmatchComplaint)
def test_expect_received_wrong_data_fails():
    '''Substitute - Verify (args): Method was called with wrong data'''
    component = Substitute()
    component.method('hello, honey')
    expect_that(component.method.received('hello'))

#@raises(MissingArgumentComplaint)
def test_expect_received_no_type_fails():
    component = Substitute()
    component.method()
    expect_that(component.method.received_any(str))

@raises(WrongArgumentTypeComplaint)
def test_expect_received_wrong_type_fails():
    '''Substitute - Verify (args): Method was called with wrong type'''
    component = Substitute()
    component.method(5)
    expect_that(component.method.received_any(str))

@raises(WrongArgumentTypeComplaint)
def test_expect_received_wrong_signature_fails():
    '''Substitute - Verify (args): Method was called with wrong signature'''
    component = Substitute()
    component.method(5, 'hello', {})
    expect_that(component.method.received_any(str, dict))

@raises(MissingArgumentComplaint)
def test_expect_received_no_argument_fails():
    '''Substitute - Verify (args): Method was called without arguments'''
    component = Substitute()
    component.method()
    expect_that(component.method.received('hello'))

<commit_msg>Woops, forgot to put @raises over the method<commit_after>from nose.tools import *
from substitute import *
from substitute.complaint import *
expect_that = assert_true

# arg = Argument
# kwarg = Keyword Argument

def test_expect_received_data():
    '''Substitute - Verify (args): Method was called with data'''
    component = Substitute()
    component.method('hello')
    expect_that(component.method.received('hello'))

def test_expect_received_type():
    '''Substitute - Verify (args): Method was called with type'''
    component = Substitute()
    component.method('hello')
    expect_that(component.method.received_any(str))

@raises(MissingCallComplaint)
def test_was_never_called_fails():
    '''Substitute - Verify (args): Method was never called (data)'''
    component = Substitute()
    expect_that(component.method.received('hello'))

@raises(MissingCallComplaint)
def text_expect_was_never_called_fails_2():
    '''Substitute - Verify (args): Method was never called (type)'''
    component = Substitute()
    expect_that(component.method.received_any(str))

@raises(UnexpectedKeywordComplaint)
def test_expect_received_unexpected_keyword_fails():
    '''Substitute - Verify (args): Method was called with unexpected kwarg'''
    component = Substitute()
    component.method(name='hello')
    expect_that(component.method.received('hello'))

@raises(ArgumentMissmatchComplaint)
def test_expect_received_wrong_data_fails():
    '''Substitute - Verify (args): Method was called with wrong data'''
    component = Substitute()
    component.method('hello, honey')
    expect_that(component.method.received('hello'))

@raises(MissingArgumentComplaint)
def test_expect_received_no_type_fails():
    component = Substitute()
    component.method()
    expect_that(component.method.received_any(str))

@raises(WrongArgumentTypeComplaint)
def test_expect_received_wrong_type_fails():
    '''Substitute - Verify (args): Method was called with wrong type'''
    component = Substitute()
    component.method(5)
    expect_that(component.method.received_any(str))

@raises(WrongArgumentTypeComplaint)
def test_expect_received_wrong_signature_fails():
    '''Substitute - Verify (args): Method was called with wrong signature'''
    component = Substitute()
    component.method(5, 'hello', {})
    expect_that(component.method.received_any(str, dict))

@raises(MissingArgumentComplaint)
def test_expect_received_no_argument_fails():
    '''Substitute - Verify (args): Method was called without arguments'''
    component = Substitute()
    component.method()
    expect_that(component.method.received('hello'))

<|endoftext|>


<commit_before>#!/usr/bin/env python
import BaseHTTPServer
import SimpleHTTPServer
import SocketServer
from cStringIO import StringIO
import PIL.Image
import threading
import evdev

from uinput_map import js_map

SERVER_PORT = 8080

send_key_lock = threading.Lock()
def send_key(js_keycode, state):
	if js_keycode not in js_map:
		raise KeyError("Unkown javascript keycode: {}".format(js_keycode))
	send_key_lock.acquire()
	with evdev.UInput() as uinput:
		uinput.write(evdev.ecodes.EV_KEY, js_map[js_keycode], 1 if state else 0)
		uinput.syn()
	send_key_lock.release()

class ThreadedHTTPServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer):
	"""Handle requests in a separate thread."""
	daemon_threads = True

# https://github.com/ev3dev/ev3dev/wiki/Using-the-LCD
fbmap = {chr(i): ''.join(chr(int(x)) for x in '{:08b}'.format(i)[::-1]) for i in range(256)}
def jpg_from_fb():
	with open('/dev/fb0', 'r') as f:
		x = ''.join(fbmap[char] for char in f.read())
	sio = StringIO()
	im = PIL.Image.fromstring('P', (192,128), x)
	im.putpalette((124,145,111, 0,0,0))
	im.convert('RGB').crop((0,0,178,128)).save(sio, 'JPEG')
	jpg = sio.getvalue()
	sio.close()
	return jpg

class MyHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
	def do_GET(self):
		"""Serve a GET request."""
		if self.path.endswith('mjpg'):
			self.send_response(200)
			self.send_header('Content-Type', 'multipart/x-mixed-replace;boundary=boundarydonotcross')
			self.end_headers()
			i = 0
			while True:
				jpg = jpg_from_fb()
				self.send_header('Content-type', 'image/jpeg')
				self.send_header('Content-length', len(jpg))
				self.end_headers()
				self.wfile.write(jpg)
				self.wfile.write('--boundarydonotcross')
				print('written {}'.format(i))
				i+=1
		else:
			f = self.send_head()
			if f:
				try:
					self.copyfile(f, self.wfile)
				finally:
					f.close()
	def do_POST(self):
		if self.path.endswith('key'):
			self.send_response(200)
			content_len = int(self.headers.getheader('content-length', 0))
			content = self.rfile.read(content_len)
			jskc, state = content.split(',')
			send_key(int(jskc), int(state))

if __name__ == '__main__':
	print("Running on port {}".format(SERVER_PORT))
	httpd = ThreadedHTTPServer(('', SERVER_PORT), MyHandler)
	httpd.serve_forever()
<commit_msg>Made jpg_from_fb leverage PIL's built in least-sig-bit mode<commit_after>#!/usr/bin/env python
import BaseHTTPServer
import SimpleHTTPServer
import SocketServer
from cStringIO import StringIO
import PIL.Image
import threading
import evdev

from uinput_map import js_map

SERVER_PORT = 8080

send_key_lock = threading.Lock()
def send_key(js_keycode, state):
	if js_keycode not in js_map:
		raise KeyError("Unkown javascript keycode: {}".format(js_keycode))
	send_key_lock.acquire()
	with evdev.UInput() as uinput:
		uinput.write(evdev.ecodes.EV_KEY, js_map[js_keycode], 1 if state else 0)
		uinput.syn()
	send_key_lock.release()

class ThreadedHTTPServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer):
	"""Handle requests in a separate thread."""
	daemon_threads = True

# https://github.com/ev3dev/ev3dev/wiki/Using-the-LCD
fb_palette = (124,145,111)+(0,0,0)*255
def jpg_from_fb():
	sio = StringIO()
	with open('/dev/fb0', 'r') as f:
		x = f.read()
	im = PIL.Image.fromstring('1', (192,128), x, 'raw', '1;R')
	im = im.convert('P')
	im.putpalette(fb_palette)
	im.convert('RGB').crop((0,0,178,128)).save(sio, 'JPEG')
	jpg = sio.getvalue()
	sio.close()
	return jpg

class MyHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
	def do_GET(self):
		"""Serve a GET request."""
		if self.path.endswith('mjpg'):
			self.send_response(200)
			self.send_header('Content-Type', 'multipart/x-mixed-replace;boundary=boundarydonotcross')
			self.end_headers()
			i = 0
			while True:
				jpg = jpg_from_fb()
				self.send_header('Content-type', 'image/jpeg')
				self.send_header('Content-length', len(jpg))
				self.end_headers()
				self.wfile.write(jpg)
				self.wfile.write('--boundarydonotcross')
				print('written {}'.format(i))
				i+=1
		else:
			f = self.send_head()
			if f:
				try:
					self.copyfile(f, self.wfile)
				finally:
					f.close()
	def do_POST(self):
		if self.path.endswith('key'):
			self.send_response(200)
			content_len = int(self.headers.getheader('content-length', 0))
			content = self.rfile.read(content_len)
			jskc, state = content.split(',')
			send_key(int(jskc), int(state))

if __name__ == '__main__':
	print("Running on port {}".format(SERVER_PORT))
	httpd = ThreadedHTTPServer(('', SERVER_PORT), MyHandler)
	httpd.serve_forever()
<|endoftext|>


<commit_before>__author__ = 'vyakoviv'

# TODO: more =P

import os
import json
import requests

REQUIRED_RESULT_KEYS = ('component', 'suite', 'test_id')

class TRAggrAPIClient(object):

    def __init__(self, url):

        self.base_url = url

    def _compile_url(self, path):

        return '%s/%s' % (self.base_url.rstrip('/'), path.lstrip('/'))

    def ping(self):

        path = '/ping'
        url = self._compile_url(path)
        response = requests.get(url)

        if response.status_code != 200:
            raise Exception('API does not respond.')

    def post_results(self, project,
                           sprint,
                           results):

        headers = {'Content-Type': 'text/json'}

        # Validate results.
        for result in results:
            for required_key in REQUIRED_RESULT_KEYS:
                if required_key not in result:
                    raise Exception('Result record %s does not contain a required key "%s"' % (record, required_key))
        # TODO: More validation.

        path = '/results/%s/%s' % (project, sprint)
        url = self._compile_url(path)

        response = requests.post(url, data=json.dumps(results), headers=headers)

        if response.status_code != 200:
            raise Exception('Failed to post results. Code: %s. Text: %s' %
                            (response.status_code, response.text))

if __name__ == '__main__':

    url = 'http://localhost:5001'

    project = 'proj'
    sprint = '2014-04'

    client = TRAggrAPIClient(url=url)
    client.ping()

    results = [{'component': 'API',
                'suite': 'Functions',
                'test_id': 'A-1',
                'other_attributes': {'title': 'Test for login',
                                     'description': 'Some more description',
                                     'types': ['Functional']},
                'result_attributes': {'result': 'passed',
                                      'error': 'Exception'}}]

    client.post_results(project=project,
                        sprint=sprint,
                        results=results)

# EOF
<commit_msg>Added manaul tests methods<commit_after>__author__ = 'vyakoviv'

# TODO: more =P

import os
import json
import requests

REQUIRED_RESULT_KEYS = ('component', 'suite', 'test_id')

class TRAggrAPIClient(object):

    def __init__(self, url):
        self.base_url = url

    def _compile_url(self, path):
        return '%s/%s' % (self.base_url.rstrip('/'), path.lstrip('/'))

    def ping(self):
        path = '/ping'
        url = self._compile_url(path)
        response = requests.get(url)

        if response.status_code != 200:
            raise Exception('API does not respond.')

    def post_results(self, project, sprint, results):
        headers = {'Content-Type': 'text/json'}

        # Validate results.
        for result in results:
            for required_key in REQUIRED_RESULT_KEYS:
                if required_key not in result:
                    raise Exception('Result record %s does not contain a required key "%s"' % (result, required_key))
        # TODO: More validation.

        path = '/results/%s/%s' % (project, sprint)
        url = self._compile_url(path)

        response = requests.post(url, data=json.dumps(results), headers=headers)

        if response.status_code != 200:
            raise Exception('Failed to post results. Code: %s. Text: %s' %
                            (response.status_code, response.text))

    def post_manual_test(self, project, tests):
        headers = {'Content-Type': 'text/json'}

        # Validate results.
        for test in tests:
            for required_key in REQUIRED_RESULT_KEYS:
                if required_key not in test:
                    raise Exception('Result record %s does not contain a required key "%s"' % (test, required_key))
        # TODO: More validation.

        path = '/manual/%s' % project
        url = self._compile_url(path)

        response = requests.post(url, data=json.dumps(results), headers=headers)

        if response.status_code != 200:
            raise Exception('Failed to post results. Code: %s. Text: %s' %
                            (response.status_code, response.text))

    def post_manual_results(self, project, sprint, results):
        headers = {'Content-Type': 'text/json'}

        # Validate results.
        for result in results:
            for required_key in REQUIRED_RESULT_KEYS:
                if required_key not in result:
                    raise Exception('Result record %s does not contain a required key "%s"' % (result, required_key))
        # TODO: More validation.

        path = '/manual/results/%s/%s' % (project, sprint)
        url = self._compile_url(path)

        response = requests.post(url, data=json.dumps(results), headers=headers)

        if response.status_code != 200:
            raise Exception('Failed to post results. Code: %s. Text: %s' %
                            (response.status_code, response.text))

if __name__ == '__main__':

    url = 'http://localhost:5001'

    project = 'proj'
    sprint = '2014-04'

    client = TRAggrAPIClient(url=url)
    client.ping()

    results = [{'component': 'API',
                'suite': 'Functions',
                'test_id': 'A-1',
                'other_attributes': {'title': 'Test for login',
                                     'description': 'Some more description',
                                     'types': ['Functional']},
                'result_attributes': {'result': 'passed',
                                      'error': 'Exception'}}]

    client.post_results(project=project,
                        sprint=sprint,
                        results=results)

# EOF
<|endoftext|>


<commit_before>from collections import defaultdict
from django.http import QueryDict
from google.appengine.ext import db
from google.appengine.ext.db import djangoforms
from charter.models import Chart, ChartDataSet, DataRow

class ChartForm(djangoforms.ModelForm):
    class Meta:
        model = Chart
        exclude = ('data', 'user', 'hash',)

class DataSetForm(djangoforms.ModelForm):
    class Meta:
        model = ChartDataSet
        exclude = ('previous_version',)

    def __init__(self, *args, **kwargs):
        foo = kwargs['instance'].data_rows
        super(DataSetForm, self).__init__(*args, **kwargs)

class DataRowForm(djangoforms.ModelForm):
    class Meta:
        model = DataRow

class DataRowFormSet(object):
    base_form = DataRowForm

    def __init__(self, data=None, instances=None):
        """
        Testing Concerns:
         - deleting an item
         - adding an item
         - altering the key
         - altering the value
        """
        formset_dict = defaultdict(QueryDict)
        self.forms = []

        if data:
            if not hasattr(data, '__iter__'):
                raise KeyError("data attribute must be an iterable.")
            if len(instances) == len(data):
                raise KeyError("data and instances iterables must be the same length")

            data._mutable = True
            for key, val in dict(data).iteritems():
                data.pop(key)
                num, key = key.split('-', 1)
                if num in formset_dict:
                    formset_dict[num].update({key:val})
                else:
                    formset_dict[num] = QueryDict({key:val})

            # FIXME: Validate that data and instances will always return in the same order
            for i, obj in enumerate(instances):
                if isinstance(obj, db.Key):
                    # fetch keys for validation
                    # FIXME: Can probably do a bulk get
                    obj = db.get(obj)
                data = formset_dict[str(i)]

                self.forms.append(
                    self.base_form(
                        instance=obj,
                        data=data,
                        prefix="%i" % i,
                    )
                )
        else:
            for i, obj in enumerate(instances):
                if isinstance(obj, db.Key):
                    # fetch keys for validation
                    # FIXME: Can probably do a bulk get
                    obj = db.get(obj)
                try:
                    data = formset_dict[i]
                except IndexError:
                    data = None

                self.forms.append(
                    self.base_form(
                        instance=obj,
                        prefix="%i" % i,
                    )
                )

    def is_valid(self):
        for dr in self.forms:
            if not dr.is_valid():
                return False
        return True
<commit_msg>Django can handle its own querydict decompisiton if we provide the proper prefixes. *sigh*<commit_after>from collections import defaultdict
from django.http import QueryDict
from google.appengine.ext import db
from google.appengine.ext.db import djangoforms
from charter.models import Chart, ChartDataSet, DataRow

class ChartForm(djangoforms.ModelForm):
    class Meta:
        model = Chart
        exclude = ('data', 'user', 'hash',)

class DataSetForm(djangoforms.ModelForm):
    class Meta:
        model = ChartDataSet
        exclude = ('previous_version',)

    def __init__(self, *args, **kwargs):
        foo = kwargs['instance'].data_rows
        super(DataSetForm, self).__init__(*args, **kwargs)

class DataRowForm(djangoforms.ModelForm):
    class Meta:
        model = DataRow

class DataRowFormSet(object):
    base_form = DataRowForm

    def __init__(self, data=None, instances=None):
        """
        Testing Concerns:
         - deleting an item
         - adding an item
         - altering the key
         - altering the value
        """
        self.forms = []
        for i, obj in enumerate(instances):
            if isinstance(obj, db.Key):
                # fetch keys for validation
                # FIXME: Can probably do a bulk get
                obj = db.get(obj)
            if data:
                self.forms.append(
                    self.base_form(
                        instance=obj,
                        data=data,
                        prefix="%i" % i,
                    )
                )

            else:
                self.forms.append(
                    self.base_form(
                        instance=obj,
                        prefix="%i" % i,
                    )
                )

    def is_valid(self):
        for dr in self.forms:
            if not dr.is_valid():
                return False
        return True
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Routines common to all posix systems."""

import os
import errno
import sys
import time
import glob

from psutil._compat import PY3, unicode
from psutil._error import TimeoutExpired
from psutil._common import nt_diskinfo, usage_percent, memoize

def pid_exists(pid):
    """Check whether pid exists in the current process table."""
    if pid < 0:
        return False
    try:
        os.kill(pid, 0)
    except OSError:
        err = sys.exc_info()[1]
        if err.errno == errno.ESRCH:
            # ESRCH == No such process
            return False
        elif err.errno == errno.EPERM:
            # EPERM clearly means there's a process to deny access to
            return True
        else:
            # According to "man 2 kill" possible error values are
            # (EINVAL, EPERM, ESRCH) therefore we should bever get
            # here. If we do let's be explicit in considering this
            # an error.
            raise err
    else:
        return True

def wait_pid(pid, timeout=None):
    """Wait for process with pid 'pid' to terminate and return its
    exit status code as an integer.

    If pid is not a children of os.getpid() (current process) just
    waits until the process disappears and return None.

    If pid does not exist at all return None immediately.

    Raise TimeoutExpired on timeout expired.
    """
    def check_timeout(delay):
        if timeout is not None:
            if timer() >= stop_at:
                raise TimeoutExpired(pid)
        time.sleep(delay)
        return min(delay * 2, 0.04)

    timer = getattr(time, 'monotonic', time.time)
    if timeout is not None:
        waitcall = lambda: os.waitpid(pid, os.WNOHANG)
        stop_at = timer() + timeout
    else:
        waitcall = lambda: os.waitpid(pid, 0)

    delay = 0.0001
    while 1:
        try:
            retpid, status = waitcall()
        except OSError:
            err = sys.exc_info()[1]
            if err.errno == errno.EINTR:
                delay = check_timeout(delay)
                continue
            elif err.errno == errno.ECHILD:
                # This has two meanings:
                # - pid is not a child of os.getpid() in which case
                #   we keep polling until it's gone
                # - pid never existed in the first place
                # In both cases we'll eventually return None as we
                # can't determine its exit status code.
                while 1:
                    if pid_exists(pid):
                        delay = check_timeout(delay)
                    else:
                        return
            else:
                raise
        else:
            if retpid == 0:
                # WNOHANG was used, pid is still running
                delay = check_timeout(delay)
                continue
            # process exited due to a signal; return the integer of
            # that signal
            if os.WIFSIGNALED(status):
                return os.WTERMSIG(status)
            # process exited using exit(2) system call; return the
            # integer exit(2) system call has been called with
            elif os.WIFEXITED(status):
                return os.WEXITSTATUS(status)
            else:
                # should never happen
                raise RuntimeError("unknown process exit status")

def get_disk_usage(path):
    """Return disk usage associated with path."""
    try:
        st = os.statvfs(path)
    except UnicodeEncodeError:
        if not PY3 and isinstance(path, unicode):
            # this is a bug with os.statvfs() and unicode on
            # Python 2, see:
            # - https://code.google.com/p/psutil/issues/detail?id=416
            # - http://bugs.python.org/issue18695
            try:
                path = path.encode(sys.getfilesystemencoding())
            except UnicodeEncodeError:
                pass
            st = os.statvfs(path)
        else:
            raise
    free = (st.f_bavail * st.f_frsize)
    total = (st.f_blocks * st.f_frsize)
    used = (st.f_blocks - st.f_bfree) * st.f_frsize
    percent = usage_percent(used, total, _round=1)
    # NB: the percentage is -5% than what shown by df due to
    # reserved blocks that we are currently not considering:
    # http://goo.gl/sWGbH
    return nt_diskinfo(total, used, free, percent)

@memoize
def _get_terminal_map():
    ret = {}
    ls = glob.glob('/dev/tty*') + glob.glob('/dev/pts/*')
    for name in ls:
        assert name not in ret
        try:
            ret[os.stat(name).st_rdev] = name
        except OSError:
            err = sys.exc_info()[1]
            if err.errno != errno.ENOENT:
                raise
    return ret
<commit_msg>pep8ify<commit_after>#!/usr/bin/env python

# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Routines common to all posix systems."""

import errno
import glob
import os
import sys
import time

from psutil._common import nt_diskinfo, usage_percent, memoize
from psutil._compat import PY3, unicode
from psutil._error import TimeoutExpired

def pid_exists(pid):
    """Check whether pid exists in the current process table."""
    if pid < 0:
        return False
    try:
        os.kill(pid, 0)
    except OSError:
        err = sys.exc_info()[1]
        if err.errno == errno.ESRCH:
            # ESRCH == No such process
            return False
        elif err.errno == errno.EPERM:
            # EPERM clearly means there's a process to deny access to
            return True
        else:
            # According to "man 2 kill" possible error values are
            # (EINVAL, EPERM, ESRCH) therefore we should bever get
            # here. If we do let's be explicit in considering this
            # an error.
            raise err
    else:
        return True

def wait_pid(pid, timeout=None):
    """Wait for process with pid 'pid' to terminate and return its
    exit status code as an integer.

    If pid is not a children of os.getpid() (current process) just
    waits until the process disappears and return None.

    If pid does not exist at all return None immediately.

    Raise TimeoutExpired on timeout expired.
    """
    def check_timeout(delay):
        if timeout is not None:
            if timer() >= stop_at:
                raise TimeoutExpired(pid)
        time.sleep(delay)
        return min(delay * 2, 0.04)

    timer = getattr(time, 'monotonic', time.time)
    if timeout is not None:
        waitcall = lambda: os.waitpid(pid, os.WNOHANG)
        stop_at = timer() + timeout
    else:
        waitcall = lambda: os.waitpid(pid, 0)

    delay = 0.0001
    while 1:
        try:
            retpid, status = waitcall()
        except OSError:
            err = sys.exc_info()[1]
            if err.errno == errno.EINTR:
                delay = check_timeout(delay)
                continue
            elif err.errno == errno.ECHILD:
                # This has two meanings:
                # - pid is not a child of os.getpid() in which case
                #   we keep polling until it's gone
                # - pid never existed in the first place
                # In both cases we'll eventually return None as we
                # can't determine its exit status code.
                while 1:
                    if pid_exists(pid):
                        delay = check_timeout(delay)
                    else:
                        return
            else:
                raise
        else:
            if retpid == 0:
                # WNOHANG was used, pid is still running
                delay = check_timeout(delay)
                continue
            # process exited due to a signal; return the integer of
            # that signal
            if os.WIFSIGNALED(status):
                return os.WTERMSIG(status)
            # process exited using exit(2) system call; return the
            # integer exit(2) system call has been called with
            elif os.WIFEXITED(status):
                return os.WEXITSTATUS(status)
            else:
                # should never happen
                raise RuntimeError("unknown process exit status")

def get_disk_usage(path):
    """Return disk usage associated with path."""
    try:
        st = os.statvfs(path)
    except UnicodeEncodeError:
        if not PY3 and isinstance(path, unicode):
            # this is a bug with os.statvfs() and unicode on
            # Python 2, see:
            # - https://code.google.com/p/psutil/issues/detail?id=416
            # - http://bugs.python.org/issue18695
            try:
                path = path.encode(sys.getfilesystemencoding())
            except UnicodeEncodeError:
                pass
            st = os.statvfs(path)
        else:
            raise
    free = (st.f_bavail * st.f_frsize)
    total = (st.f_blocks * st.f_frsize)
    used = (st.f_blocks - st.f_bfree) * st.f_frsize
    percent = usage_percent(used, total, _round=1)
    # NB: the percentage is -5% than what shown by df due to
    # reserved blocks that we are currently not considering:
    # http://goo.gl/sWGbH
    return nt_diskinfo(total, used, free, percent)

@memoize
def _get_terminal_map():
    ret = {}
    ls = glob.glob('/dev/tty*') + glob.glob('/dev/pts/*')
    for name in ls:
        assert name not in ret
        try:
            ret[os.stat(name).st_rdev] = name
        except OSError:
            err = sys.exc_info()[1]
            if err.errno != errno.ENOENT:
                raise
    return ret
<|endoftext|>


<commit_before># external dependencies
from sklearn import tree
from timeit import default_timer as timer
from random import randint, sample, shuffle
from os.path import isfile
import os
import csv
import pylru
# local dependencies
import bots
import decks

# how many can we fit into memory?
ML_MODEL_CACHE_SIZE = 50

# cache of previously trained models in memory
models = pylru.lrucache(ML_MODEL_CACHE_SIZE)

#
# train a new model using the training data collected
#
def trainModel (botname, deckname):
    global models

    # start timing, so we know how long we're spending doing training
    start = timer()

    # get the path of the CSV with the training data
    trainingdata = bots.getFileName(botname, deckname)
    # no training data available
    if isfile(trainingdata) == False:
        return None

    # train a decision tree
    data = []
    target = []
    headers = None
    trainingrecords = 0
    with open(trainingdata, "rb") as trainingfile:
        reader = csv.reader(trainingfile, delimiter=",")
        headers = reader.next()
        for row in reader:
            data.append(row[:-1])
            target.append(row[-1])
            trainingrecords += 1

    dt = tree.DecisionTreeClassifier()
    dt.fit(data, target)

    # stop timing
    end = timer()

    # cache the decision tree so we can reuse it without training
    models[trainingdata] = dt

    return { "model" : dt, "training" : { "time" : (end - start), "hands" : len(data) } }

#
# gets the predictive model - fetching a previously trained model
#  if one is available in the cache, or creating a new one otherwise
def getModel (botname, deckname):
    global models
    trainingdata = bots.getFileName(botname, deckname)
    try:
        return models[trainingdata]
    except KeyError:
        newtraining = trainModel(botname, deckname)
        if newtraining is None:
            return None
        else:
            return newtraining["model"]

def compareNumbers (first, second):
    if first > second:
        return 1
    elif first < second:
        return -1
    else:
        return 0

def sortChoices (first, second):
    if first["outcome"] == 1 and second["outcome"] == 1:
        return compareNumbers(first["confidence"], second["confidence"])
    elif first["outcome"] == 1:
        return 1
    elif second["outcome"] == 1:
        return -1
    elif first["outcome"] == 0 and second["outcome"] == 0:
        return compareNumbers(second["confidence"], first["confidence"])
    elif first["outcome"] == 0:
        return 1
    elif second["outcome"] == 0:
        return -1
    else:
        return compareNumbers(second["confidence"], first["confidence"])

def randomChoice (rules):
    choice = sample(rules.keys(), 1)[0]
    print "Computer chose : %s" % (choice)
    return choice

def predict (botname, deckname, card):
    model = getModel(botname, deckname)

    deck = decks.get(deckname)
    rules = deck["rules"]

    if model is None:
        return { "confidence" : 0, "choice" : randomChoice(rules) }

    data = []

    optionidx = 0
    for option in rules.keys():
        optionData = []
        for attribute in rules.keys():
            optionData.append(card[attribute])
        optionData.append(optionidx)
        optionidx += 1

        data.append(optionData)

    predictions = model.predict_proba(data)

    choices = []

    optionidx = 0
    for option in rules.keys():
        print option
        loss_prob = predictions[optionidx][0]
        draw_prob = predictions[optionidx][1]
        win_prob = predictions[optionidx][2]
        print "probability of losing " + str(predictions[optionidx][0])
        print "probability of drawing " + str(predictions[optionidx][1])
        print "probability of winning " + str(predictions[optionidx][2])
        if win_prob >= draw_prob and win_prob > loss_prob:
            choices.append({
                "choice" : option,
                "outcome" : 1,
                "confidence" : win_prob * 100
            })
        elif (draw_prob > win_prob and draw_prob > loss_prob) or (win_prob == loss_prob and win_prob > draw_prob):
            choices.append({
                "choice" : option,
                "outcome" : 0,
                "confidence" : draw_prob * 100
            })
        else:
            choices.append({
                "choice" : option,
                "outcome" : -1,
                "confidence" : loss_prob * 100
            })

        optionidx += 1

    sortedChoices = sorted(choices, cmp=sortChoices, reverse=True)

    choice = sortedChoices[0]["choice"]
    print "Computer chose : %s" % (choice)

    outcome = ""
    if sortedChoices[0]["outcome"] == 1:
        outcome = "win"
    elif sortedChoices[0]["outcome"] == 0:
        outcome = "at least draw"
    else:
        outcome = "draw"

    message = "(expecting that it would %s with %d percent confidence)"
    print message % (outcome, sortedChoices[0]["confidence"])

    return sortedChoices[0]

<commit_msg>fix: Support predict calls before there is enough training<commit_after># external dependencies
from sklearn import tree
from timeit import default_timer as timer
from random import randint, sample, shuffle
from os.path import isfile
import os
import csv
import pylru
# local dependencies
import bots
import decks

# how many can we fit into memory?
ML_MODEL_CACHE_SIZE = 50

# cache of previously trained models in memory
models = pylru.lrucache(ML_MODEL_CACHE_SIZE)

#
# train a new model using the training data collected
#
def trainModel (botname, deckname):
    global models

    # start timing, so we know how long we're spending doing training
    start = timer()

    # get the path of the CSV with the training data
    trainingdata = bots.getFileName(botname, deckname)
    # no training data available
    if isfile(trainingdata) == False:
        return None

    # train a decision tree
    data = []
    target = []
    headers = None
    trainingrecords = 0
    with open(trainingdata, "rb") as trainingfile:
        reader = csv.reader(trainingfile, delimiter=",")
        headers = reader.next()
        for row in reader:
            data.append(row[:-1])
            target.append(row[-1])
            trainingrecords += 1

    dt = tree.DecisionTreeClassifier()
    dt.fit(data, target)

    # stop timing
    end = timer()

    # cache the decision tree so we can reuse it without training
    models[trainingdata] = dt

    return { "model" : dt, "training" : { "time" : (end - start), "hands" : len(data) } }

#
# gets the predictive model - fetching a previously trained model
#  if one is available in the cache, or creating a new one otherwise
def getModel (botname, deckname):
    global models
    trainingdata = bots.getFileName(botname, deckname)
    try:
        return models[trainingdata]
    except KeyError:
        newtraining = trainModel(botname, deckname)
        if newtraining is None:
            return None
        else:
            return newtraining["model"]

def compareNumbers (first, second):
    if first > second:
        return 1
    elif first < second:
        return -1
    else:
        return 0

def sortChoices (first, second):
    if first["outcome"] == 1 and second["outcome"] == 1:
        return compareNumbers(first["confidence"], second["confidence"])
    elif first["outcome"] == 1:
        return 1
    elif second["outcome"] == 1:
        return -1
    elif first["outcome"] == 0 and second["outcome"] == 0:
        return compareNumbers(second["confidence"], first["confidence"])
    elif first["outcome"] == 0:
        return 1
    elif second["outcome"] == 0:
        return -1
    else:
        return compareNumbers(second["confidence"], first["confidence"])

def randomChoice (rules):
    choice = sample(rules.keys(), 1)[0]
    print "Computer chose : %s" % (choice)
    return choice

def predict (botname, deckname, card):
    model = getModel(botname, deckname)

    deck = decks.get(deckname)
    rules = deck["rules"]

    if model is None:
        return { "confidence" : 0, "choice" : randomChoice(rules) }

    data = []

    optionidx = 0
    for option in rules.keys():
        optionData = []
        for attribute in rules.keys():
            optionData.append(card[attribute])
        optionData.append(optionidx)
        optionidx += 1

        data.append(optionData)

    predictions = model.predict_proba(data)

    print predictions

    if len(predictions) != len(rules.keys()):
        print "Insufficient training. Resorting to random."
        return { "confidence" : 0, "choice" : randomChoice(rules) }

    choices = []
    optionidx = 0
    for option in rules.keys():
        print option

        print len(predictions[optionidx])

        # we need predictions for win/loss/draw
        #  which only happen once all options have been
        #  represented in the training data
        if len(predictions[optionidx]) != 3:
            print "Insufficient training. Resorting to random."
            return { "confidence" : 0, "choice" : randomChoice(rules) }

        loss_prob = predictions[optionidx][0]
        draw_prob = predictions[optionidx][1]
        win_prob = predictions[optionidx][2]
        print "probability of losing " + str(predictions[optionidx][0])
        print "probability of drawing " + str(predictions[optionidx][1])
        print "probability of winning " + str(predictions[optionidx][2])
        if win_prob >= draw_prob and win_prob > loss_prob:
            choices.append({
                "choice" : option,
                "outcome" : 1,
                "confidence" : win_prob * 100
            })
        elif (draw_prob > win_prob and draw_prob > loss_prob) or (win_prob == loss_prob and win_prob > draw_prob):
            choices.append({
                "choice" : option,
                "outcome" : 0,
                "confidence" : draw_prob * 100
            })
        else:
            choices.append({
                "choice" : option,
                "outcome" : -1,
                "confidence" : loss_prob * 100
            })

        optionidx += 1

    sortedChoices = sorted(choices, cmp=sortChoices, reverse=True)

    choice = sortedChoices[0]["choice"]
    print "Computer chose : %s" % (choice)

    outcome = ""
    if sortedChoices[0]["outcome"] == 1:
        outcome = "win"
    elif sortedChoices[0]["outcome"] == 0:
        outcome = "at least draw"
    else:
        outcome = "draw"

    message = "(expecting that it would %s with %d percent confidence)"
    print message % (outcome, sortedChoices[0]["confidence"])

    return sortedChoices[0]

<|endoftext|>