<commit_before># coding=utf-8

from __future__ import unicode_literals

import io
import shutil
import os

import tempman
from nose.tools import istest, assert_equal

from .testing import test_path

import mammoth
from mammoth import results

@istest
def docx_containing_one_paragraph_is_converted_to_single_p_element():
    with open(test_path("single-paragraph.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal("
Walking on imported air
", result.value)
        assert_equal([], result.messages)

@istest
def can_read_xml_files_with_utf8_bom():
    with open(test_path("utf8-bom.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal("
This XML has a byte order mark.
", result.value)
        assert_equal([], result.messages)

@istest
def empty_paragraphs_are_ignored_by_default():
    with open(test_path("empty.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal("", result.value)
        assert_equal([], result.messages)

@istest
def empty_paragraphs_are_preserved_if_ignore_empty_paragraphs_is_false():
    with open(test_path("empty.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, ignore_empty_paragraphs=False)
        assert_equal("

", result.value)
        assert_equal([], result.messages)

@istest
def embedded_style_map_is_used_if_present():
    with _copy_of_test_data("single-paragraph.docx") as fileobj:
        mammoth.embed_style_map(fileobj, "p => h1")
        result = mammoth.convert_to_html(fileobj=fileobj, ignore_empty_paragraphs=False)
        assert_equal("
Walking on imported air
", result.value)
        assert_equal([], result.messages)

@istest
def embedded_style_map_can_be_retrieved():
    with _copy_of_test_data("single-paragraph.docx") as fileobj:
        mammoth.embed_style_map(fileobj, "p => h1")
        assert_equal("p => h1", mammoth.read_embedded_style_map(fileobj))

@istest
def warning_if_style_mapping_is_not_understood():
    style_map = """
!!!!
p => h1"""
    with open(test_path("single-paragraph.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, style_map=style_map)
        assert_equal("
Walking on imported air
", result.value)
        warning = "Did not understand this style mapping, so ignored it: !!!!"
        assert_equal([results.warning(warning)], result.messages)

@istest
def inline_images_are_included_in_output():
    with open(test_path("tiny-picture.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal("""

""", result.value)
        assert_equal([], result.messages)

@istest
def images_stored_outside_of_document_are_included_in_output():
    with open(test_path("external-picture.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal("""

""", result.value)
        assert_equal([], result.messages)

@istest
def warn_if_images_stored_outside_of_document_are_specified_when_passing_fileobj_without_name():
    fileobj = io.BytesIO()
    with open(test_path("external-picture.docx"), "rb") as source_fileobj:
        shutil.copyfileobj(source_fileobj, fileobj)

    result = mammoth.convert_to_html(fileobj=fileobj)
    assert_equal("", result.value)
    assert_equal([results.warning("could not find external image 'tiny-picture.png', fileobj has no name")], result.messages)

@istest
def warn_if_images_stored_outside_of_document_are_not_found():
    with tempman.create_temp_dir() as temp_dir:
        document_path = os.path.join(temp_dir.path, "document.docx")
        with open(document_path, "wb") as fileobj:
            with open(test_path("external-picture.docx"), "rb") as source_fileobj:
                shutil.copyfileobj(source_fileobj, fileobj)

        with open(document_path, "rb") as fileobj:
            result = mammoth.convert_to_html(fileobj=fileobj)
            assert_equal("", result.value)
            expected_warning = "could not open external image: 'tiny-picture.png'"
            assert_equal("warning", result.messages[0].type)
            assert result.messages[0].message.startswith(expected_warning), "message was: " + result.messages[0].message
            assert_equal(1, len(result.messages))

@istest
def simple_list_is_converted_to_list_elements():
    with open(test_path("simple-list.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal([], result.messages)
        assert_equal("

Apple

Banana

", result.value)

@istest
def word_tables_are_converted_to_html_tables():
    expected_html = ("
Above
" +
        "
" +
        "

Top left

Top right

" +
        "

Bottom left

Bottom right

" +
        "
" +
        "
Below
")

    with open(test_path("tables.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal([], result.messages)
        assert_equal(expected_html, result.value)

@istest
def footnotes_are_appended_to_text():
    # TODO: don't duplicate footnotes with multiple references
    expected_html = ('
Ouch' +
        '[1].' +
        '[2]
' +
        '

 A tachyon walks into a bar. ↑

' +
        '

 Fin. ↑

')

    with open(test_path("footnotes.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, id_prefix="doc-42")
        assert_equal([], result.messages)
        assert_equal(expected_html, result.value)

@istest
def endnotes_are_appended_to_text():
    expected_html = ('
Ouch' +
        '[1].' +
        '[2]
' +
        '

 A tachyon walks into a bar. ↑

' +
        '

 Fin. ↑

')

    with open(test_path("endnotes.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, id_prefix="doc-42")
        assert_equal([], result.messages)
        assert_equal(expected_html, result.value)

@istest
def relationships_are_handled_properly_in_footnotes():
    expected_html = (
        '
[1]
' +
        '

 Example ↑

')

    with open(test_path("footnote-hyperlink.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, id_prefix="doc-42")
        assert_equal([], result.messages)
        assert_equal(expected_html, result.value)

@istest
def text_boxes_are_read():
    with open(test_path("text-box.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal('
Datum plane
', result.value)

@istest
def underline_is_ignored_by_defualt():
    with open(test_path("underline.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal('
The Sunset Tree
', result.value)

@istest
def underline_can_be_configured_with_convert_underline_option():
    with open(test_path("underline.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, convert_underline=mammoth.underline.element("em"))
        assert_equal('
The Sunset Tree
', result.value)

@istest
def underline_can_be_configured_with_style_mapping():
    with open(test_path("underline.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, style_map="u => em")
        assert_equal('
The Sunset Tree
', result.value)

@istest
def strikethrough_is_converted_to_s_element_by_default():
    with open(test_path("strikethrough.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal("
<s>Today's Special: Salmon</s> Sold out
", result.value)

@istest
def strikethrough_conversion_can_be_configured_with_style_mapping():
    with open(test_path("strikethrough.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, style_map="strike => del")
        assert_equal("
Today's Special: Salmon Sold out
", result.value)

@istest
def transform_document_is_applied_to_document_before_conversion():
    def transform_document(document):
        document.children[0].style_id = "Heading1"
        return document

    with open(test_path("single-paragraph.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, transform_document=transform_document)
        assert_equal("
Walking on imported air
", result.value)
        assert_equal([], result.messages)

@istest
def docx_containing_one_paragraph_can_be_converted_to_markdown():
    with open(test_path("single-paragraph.docx"), "rb") as fileobj:
        result = mammoth.convert_to_markdown(fileobj=fileobj)
        assert_equal("Walking on imported air\n\n", result.value)
        assert_equal([], result.messages)

@istest
def can_extract_raw_text():
    with open(test_path("simple-list.docx"), "rb") as fileobj:
        result = mammoth.extract_raw_text(fileobj=fileobj)
        assert_equal([], result.messages)
        assert_equal("Apple\n\nBanana\n\n", result.value)

def _copy_of_test_data(path):
    destination = io.BytesIO()
    with open(test_path(path), "rb") as source:
        shutil.copyfileobj(source, destination)
    return destination
<commit_msg>Fix typo in test name<commit_after># coding=utf-8

from __future__ import unicode_literals

import io
import shutil
import os

import tempman
from nose.tools import istest, assert_equal

from .testing import test_path

import mammoth
from mammoth import results

@istest
def docx_containing_one_paragraph_is_converted_to_single_p_element():
    with open(test_path("single-paragraph.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal("
Walking on imported air
", result.value)
        assert_equal([], result.messages)

@istest
def can_read_xml_files_with_utf8_bom():
    with open(test_path("utf8-bom.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal("
This XML has a byte order mark.
", result.value)
        assert_equal([], result.messages)

@istest
def empty_paragraphs_are_ignored_by_default():
    with open(test_path("empty.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal("", result.value)
        assert_equal([], result.messages)

@istest
def empty_paragraphs_are_preserved_if_ignore_empty_paragraphs_is_false():
    with open(test_path("empty.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, ignore_empty_paragraphs=False)
        assert_equal("

", result.value)
        assert_equal([], result.messages)

@istest
def embedded_style_map_is_used_if_present():
    with _copy_of_test_data("single-paragraph.docx") as fileobj:
        mammoth.embed_style_map(fileobj, "p => h1")
        result = mammoth.convert_to_html(fileobj=fileobj, ignore_empty_paragraphs=False)
        assert_equal("
Walking on imported air
", result.value)
        assert_equal([], result.messages)

@istest
def embedded_style_map_can_be_retrieved():
    with _copy_of_test_data("single-paragraph.docx") as fileobj:
        mammoth.embed_style_map(fileobj, "p => h1")
        assert_equal("p => h1", mammoth.read_embedded_style_map(fileobj))

@istest
def warning_if_style_mapping_is_not_understood():
    style_map = """
!!!!
p => h1"""
    with open(test_path("single-paragraph.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, style_map=style_map)
        assert_equal("
Walking on imported air
", result.value)
        warning = "Did not understand this style mapping, so ignored it: !!!!"
        assert_equal([results.warning(warning)], result.messages)

@istest
def inline_images_are_included_in_output():
    with open(test_path("tiny-picture.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal("""

""", result.value)
        assert_equal([], result.messages)

@istest
def images_stored_outside_of_document_are_included_in_output():
    with open(test_path("external-picture.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal("""

""", result.value)
        assert_equal([], result.messages)

@istest
def warn_if_images_stored_outside_of_document_are_specified_when_passing_fileobj_without_name():
    fileobj = io.BytesIO()
    with open(test_path("external-picture.docx"), "rb") as source_fileobj:
        shutil.copyfileobj(source_fileobj, fileobj)

    result = mammoth.convert_to_html(fileobj=fileobj)
    assert_equal("", result.value)
    assert_equal([results.warning("could not find external image 'tiny-picture.png', fileobj has no name")], result.messages)

@istest
def warn_if_images_stored_outside_of_document_are_not_found():
    with tempman.create_temp_dir() as temp_dir:
        document_path = os.path.join(temp_dir.path, "document.docx")
        with open(document_path, "wb") as fileobj:
            with open(test_path("external-picture.docx"), "rb") as source_fileobj:
                shutil.copyfileobj(source_fileobj, fileobj)

        with open(document_path, "rb") as fileobj:
            result = mammoth.convert_to_html(fileobj=fileobj)
            assert_equal("", result.value)
            expected_warning = "could not open external image: 'tiny-picture.png'"
            assert_equal("warning", result.messages[0].type)
            assert result.messages[0].message.startswith(expected_warning), "message was: " + result.messages[0].message
            assert_equal(1, len(result.messages))

@istest
def simple_list_is_converted_to_list_elements():
    with open(test_path("simple-list.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal([], result.messages)
        assert_equal("

Apple

Banana

", result.value)

@istest
def word_tables_are_converted_to_html_tables():
    expected_html = ("
Above
" +
        "
" +
        "

Top left

Top right

" +
        "

Bottom left

Bottom right

" +
        "
" +
        "
Below
")

    with open(test_path("tables.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal([], result.messages)
        assert_equal(expected_html, result.value)

@istest
def footnotes_are_appended_to_text():
    # TODO: don't duplicate footnotes with multiple references
    expected_html = ('
Ouch' +
        '[1].' +
        '[2]
' +
        '

 A tachyon walks into a bar. ↑

' +
        '

 Fin. ↑

')

    with open(test_path("footnotes.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, id_prefix="doc-42")
        assert_equal([], result.messages)
        assert_equal(expected_html, result.value)

@istest
def endnotes_are_appended_to_text():
    expected_html = ('
Ouch' +
        '[1].' +
        '[2]
' +
        '

 A tachyon walks into a bar. ↑

' +
        '

 Fin. ↑

')

    with open(test_path("endnotes.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, id_prefix="doc-42")
        assert_equal([], result.messages)
        assert_equal(expected_html, result.value)

@istest
def relationships_are_handled_properly_in_footnotes():
    expected_html = (
        '
[1]
' +
        '

 Example ↑

')

    with open(test_path("footnote-hyperlink.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, id_prefix="doc-42")
        assert_equal([], result.messages)
        assert_equal(expected_html, result.value)

@istest
def text_boxes_are_read():
    with open(test_path("text-box.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal('
Datum plane
', result.value)

@istest
def underline_is_ignored_by_default():
    with open(test_path("underline.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal('
The Sunset Tree
', result.value)

@istest
def underline_can_be_configured_with_convert_underline_option():
    with open(test_path("underline.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, convert_underline=mammoth.underline.element("em"))
        assert_equal('
The Sunset Tree
', result.value)

@istest
def underline_can_be_configured_with_style_mapping():
    with open(test_path("underline.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, style_map="u => em")
        assert_equal('
The Sunset Tree
', result.value)

@istest
def strikethrough_is_converted_to_s_element_by_default():
    with open(test_path("strikethrough.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj)
        assert_equal("
<s>Today's Special: Salmon</s> Sold out
", result.value)

@istest
def strikethrough_conversion_can_be_configured_with_style_mapping():
    with open(test_path("strikethrough.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, style_map="strike => del")
        assert_equal("
Today's Special: Salmon Sold out
", result.value)

@istest
def transform_document_is_applied_to_document_before_conversion():
    def transform_document(document):
        document.children[0].style_id = "Heading1"
        return document

    with open(test_path("single-paragraph.docx"), "rb") as fileobj:
        result = mammoth.convert_to_html(fileobj=fileobj, transform_document=transform_document)
        assert_equal("
Walking on imported air
", result.value)
        assert_equal([], result.messages)

@istest
def docx_containing_one_paragraph_can_be_converted_to_markdown():
    with open(test_path("single-paragraph.docx"), "rb") as fileobj:
        result = mammoth.convert_to_markdown(fileobj=fileobj)
        assert_equal("Walking on imported air\n\n", result.value)
        assert_equal([], result.messages)

@istest
def can_extract_raw_text():
    with open(test_path("simple-list.docx"), "rb") as fileobj:
        result = mammoth.extract_raw_text(fileobj=fileobj)
        assert_equal([], result.messages)
        assert_equal("Apple\n\nBanana\n\n", result.value)

def _copy_of_test_data(path):
    destination = io.BytesIO()
    with open(test_path(path), "rb") as source:
        shutil.copyfileobj(source, destination)
    return destination
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""An XmlRpcServer serving as entry points to a remote machine."""

import logging
import optparse
import os
from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler
from SimpleXMLRPCServer import SimpleXMLRPCServer
import subprocess
import tempfile
import threading
import xmlrpclib

class _RemoteExecutor(object):
  """Implements all methods exposed by a remote machine."""

  def __init__(self):
    """Default initialization.

    Attributes:
      _sid: The ID of the server.
      _autoupdate: The Autoupdate state, True for need update, False otherwise.
    """

    # The value that the sid method should return.
    self._sid = 1
    # The value that the autoupdate method should return.
    self._autoupdate = False

  def exists(self, os_path):  # pylint: disable-msg=R0201
    """Returns True if os_path exists.

    Args:
    os_path: The path to test existance of.

    Returns:
    True if os_path exists. False otherwise.
    """
    return os.path.exists(os_path)

  def makedirs(self, os_path):  # pylint: disable-msg=R0201
    """Creates all the inexistent dirs in os_path.

    Args:
    os_path: The path of dirs to make.

    Returns:
    0. Because the XmlRpcServer must return something, not None...
    """
    os.makedirs(os_path)
    return 0

  def upload(self, os_path, content):  # pylint: disable-msg=R0201
    """Upload content to os_path.

    Args:
    os_path: The path where to upload content.
    content: The content to upload to os_path. Can be of type xmlrpclib.Binary
    or a string.

    Returns:
    The os_path where the content was uploaded.
    """
    new_file = open(os_path, 'wb')
    if isinstance(content, xmlrpclib.Binary):
      new_file.write(content.data)
    else:
      new_file.write(content.encode('utf8', 'replace'))
      new_file.close()
      return os_path

  # pylint: disable-msg=R0201
  def start(self, command, args,
            unused_wait_for_completion, capture):
    """Starts the command with the given args and potentially wait for results.

    Args:
    command: The command to execute.
    args: The list of arguments to pass to the command.
    unused_wait_for_completion: We don't use it, we only rely on capture.
    capture: Specfifies if we should wait to capture all results.

    Returns:
    The identifier of the started process if we don't capture the results.
    -1 in case of errors.
    If the results are captured, we return a tuple containing the exit code
    and stdout. (-1, <error text>) in case of errors.
    """
    logging.debug('starting process: %s', [command] + args)

    if capture:
      # Use a temporary file descriptor for the stdout pipe out of Popen so that
      # we can read that file with another file object and not interfere. Also
      # avoiding the known deadlock bugs with subprocess default PIPE.
      (stdout_file_descriptor, stdout_file_name) = tempfile.mkstemp(text=True)
      stdout_file = open(stdout_file_name)

      def CleanupTempFiles():
        os.close(stdout_file_descriptor)
        stdout_file.close()
        os.remove(stdout_file_name)

    else:
      stdout_file_descriptor = subprocess.PIPE

    try:
      proc = subprocess.Popen([command] + args, stdout=stdout_file_descriptor,
                              bufsize=1, stderr=subprocess.STDOUT,
                              stdin=subprocess.PIPE)
    except OSError, e:
      logging.exception('Execution of %s raised exception: %s.',
                        str([command] + args), e)
      if capture:
        CleanupTempFiles()
        return (-1, e)
      else:
        return -1

    logging.debug('process id: %s', proc.pid)
    if capture:
      logging.debug('waiting to capture')
      proc.wait()
      logging.debug('done!')
      stdout_text = stdout_file.read()
      CleanupTempFiles()
      return (proc.returncode, stdout_text)
    else:
      return proc.pid

  def sid(self):
    """Returns the server ID.

    Returns:
      The ID of the server.
    """
    return self._sid

  def autoupdate(self):
    """Returns the autoupdate state.

    Returns:
      True if we should autoupdate, False otherwise.
    """
    return self._autoupdate

# Restrict to a particular path.
class RequestHandler(SimpleXMLRPCRequestHandler):
  """A specialized request handler to specify RPC2."""

  rpc_paths = ('/RPC2',)

class XmlRpcServer(object):
  """The class running the server so we can import and start it manually."""

  def __init__(self):
    """Default initialization.

    Attributes:
      _server: A SimpleXMLRPCServer so make our objects and method available.
      _thread: A Thread in which to run the shutdown code to avoid deadlocks.
    """
    self._server = None
    self._thread = None

  def _ShutdownServer(self):
    """Shutting down the server from another thread."""
    assert self._thread.ident == threading.currentThread().ident

    if self._server:
      logging.debug('XmlRpcServer shutting down the server')
      self._server.shutdown()
      self._server.server_close()
      self._server = None
      logging.debug('XmlRpcServer _Shutdown done')

  def Shutdown(self):
    """Shuts down the server.

    Returns:
      True.
    """
    # We must do this from another thread since it blocks and would create
    # a deadlock while the server waits for this request to be handled.
    if self._server:
      self._thread = threading.Thread(target=self._ShutdownServer)
      logging.debug('shutdown pending')
      self._thread.start()
    return True

  def Start(self, verbose, started_event=None, done_event=None):
    """Start the server.

    Args:
      verbose: True for verbose logging, False for errors only.
      started_event: An event to signal when we are ready to start.
          Used mainly for testing purposes. Defaults to None.
      done_event: An event to signal when we are done shutting down.
          Used mainly for testing purposes. Defaults to None.
    """
    if verbose:
      logging.getLogger().setLevel(logging.DEBUG)
    else:
      logging.getLogger().setLevel(logging.ERROR)

    logging.debug('Creating server')
    #TODO(user): Dynamically choose a port.
    self._server = SimpleXMLRPCServer(('', 7399), logRequests=verbose,
                                      requestHandler=RequestHandler)

    logging.debug('registering RemoteExecutor')
    self._server.register_instance(_RemoteExecutor())

    logging.debug('registering introspection functions')
    self._server.register_introspection_functions()

    logging.debug('registering Shutdown function')
    self._server.register_function(self.Shutdown)

    if started_event:
      started_event.set()

    # Run the server's main loop
    logging.debug('Serving until Shutdown is called.')
    self._server.serve_forever()
    logging.debug('shutdown complete')

    if self._thread.isAlive():
      logging.debug('joining shutdown thread')
      self._thread.join()
      logging.debug('joined shutdown thread')
    self._thread = None

    if done_event:
      logging.debug('signaling done event')
      done_event.set()

if __name__ == '__main__':
  _DESCRIPTION = ('This script runs an xmlrpc server to be used by Swarm.')

  parser = optparse.OptionParser(usage='%prog [options] [filename]',
                                 description=_DESCRIPTION)
  parser.add_option('-v', '--verbose', action='store_true',
                    help='Set logging level to DEBUG. Optional. '
                    'Defaults to ERROR level.')
  (options, other_args) = parser.parse_args()

  XmlRpcServer().Start(options.verbose)
<commit_msg>Fix indentation in xmlrpc_server.py<commit_after>#!/usr/bin/python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""An XmlRpcServer serving as entry points to a remote machine."""

import logging
import optparse
import os
from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler
from SimpleXMLRPCServer import SimpleXMLRPCServer
import subprocess
import tempfile
import threading
import xmlrpclib

class _RemoteExecutor(object):
  """Implements all methods exposed by a remote machine."""

  def __init__(self):
    """Default initialization.

    Attributes:
      _sid: The ID of the server.
      _autoupdate: The Autoupdate state, True for need update, False otherwise.
    """

    # The value that the sid method should return.
    self._sid = 1
    # The value that the autoupdate method should return.
    self._autoupdate = False

  def exists(self, os_path):  # pylint: disable-msg=R0201
    """Returns True if os_path exists.

    Args:
      os_path: The path to test existance of.

    Returns:
      True if os_path exists. False otherwise.
    """
    return os.path.exists(os_path)

  def makedirs(self, os_path):  # pylint: disable-msg=R0201
    """Creates all the inexistent dirs in os_path.

    Args:
      os_path: The path of dirs to make.

    Returns:
      0. Because the XmlRpcServer must return something, not None...
    """
    os.makedirs(os_path)
    return 0

  def upload(self, os_path, content):  # pylint: disable-msg=R0201
    """Upload content to os_path.

    Args:
      os_path: The path where to upload content.
      content: The content to upload to os_path. Can be of type xmlrpclib.Binary
          or a string.

    Returns:
      The os_path where the content was uploaded.
    """
    new_file = open(os_path, 'wb')
    if isinstance(content, xmlrpclib.Binary):
      new_file.write(content.data)
    else:
      new_file.write(content.encode('utf8', 'replace'))
    new_file.close()
    return os_path

  # pylint: disable-msg=R0201
  def start(self, command, args,
            unused_wait_for_completion, capture):
    """Starts the command with the given args and potentially wait for results.

    Args:
      command: The command to execute.
      args: The list of arguments to pass to the command.
      unused_wait_for_completion: We don't use it, we only rely on capture.
      capture: Specfifies if we should wait to capture all results.

    Returns:
      The identifier of the started process if we don't capture the results.
      -1 in case of errors.
      If the results are captured, we return a tuple containing the exit code
      and stdout. (-1, <error text>) in case of errors.
    """
    logging.debug('starting process: %s', [command] + args)

    if capture:
      # Use a temporary file descriptor for the stdout pipe out of Popen so that
      # we can read that file with another file object and not interfere. Also
      # avoiding the known deadlock bugs with subprocess default PIPE.
      (stdout_file_descriptor, stdout_file_name) = tempfile.mkstemp(text=True)
      stdout_file = open(stdout_file_name)

      def CleanupTempFiles():
        os.close(stdout_file_descriptor)
        stdout_file.close()
        os.remove(stdout_file_name)

    else:
      stdout_file_descriptor = subprocess.PIPE

    try:
      proc = subprocess.Popen([command] + args, stdout=stdout_file_descriptor,
                              bufsize=1, stderr=subprocess.STDOUT,
                              stdin=subprocess.PIPE)
    except OSError, e:
      logging.exception('Execution of %s raised exception: %s.',
                        str([command] + args), e)
      if capture:
        CleanupTempFiles()
        return (-1, e)
      else:
        return -1

    logging.debug('process id: %s', proc.pid)
    if capture:
      logging.debug('waiting to capture')
      proc.wait()
      logging.debug('done!')
      stdout_text = stdout_file.read()
      CleanupTempFiles()
      return (proc.returncode, stdout_text)
    else:
      return proc.pid

  def sid(self):
    """Returns the server ID.

    Returns:
      The ID of the server.
    """
    return self._sid

  def autoupdate(self):
    """Returns the autoupdate state.

    Returns:
      True if we should autoupdate, False otherwise.
    """
    return self._autoupdate

# Restrict to a particular path.
class RequestHandler(SimpleXMLRPCRequestHandler):
  """A specialized request handler to specify RPC2."""

  rpc_paths = ('/RPC2',)

class XmlRpcServer(object):
  """The class running the server so we can import and start it manually."""

  def __init__(self):
    """Default initialization.

    Attributes:
      _server: A SimpleXMLRPCServer so make our objects and method available.
      _thread: A Thread in which to run the shutdown code to avoid deadlocks.
    """
    self._server = None
    self._thread = None

  def _ShutdownServer(self):
    """Shutting down the server from another thread."""
    assert self._thread.ident == threading.currentThread().ident

    if self._server:
      logging.debug('XmlRpcServer shutting down the server')
      self._server.shutdown()
      self._server.server_close()
      self._server = None
      logging.debug('XmlRpcServer _Shutdown done')

  def Shutdown(self):
    """Shuts down the server.

    Returns:
      True.
    """
    # We must do this from another thread since it blocks and would create
    # a deadlock while the server waits for this request to be handled.
    if self._server:
      self._thread = threading.Thread(target=self._ShutdownServer)
      logging.debug('shutdown pending')
      self._thread.start()
    return True

  def Start(self, verbose, started_event=None, done_event=None):
    """Start the server.

    Args:
      verbose: True for verbose logging, False for errors only.
      started_event: An event to signal when we are ready to start.
          Used mainly for testing purposes. Defaults to None.
      done_event: An event to signal when we are done shutting down.
          Used mainly for testing purposes. Defaults to None.
    """
    if verbose:
      logging.getLogger().setLevel(logging.DEBUG)
    else:
      logging.getLogger().setLevel(logging.ERROR)

    logging.debug('Creating server')
    #TODO(user): Dynamically choose a port.
    self._server = SimpleXMLRPCServer(('', 7399), logRequests=verbose,
                                      requestHandler=RequestHandler)

    logging.debug('registering RemoteExecutor')
    self._server.register_instance(_RemoteExecutor())

    logging.debug('registering introspection functions')
    self._server.register_introspection_functions()

    logging.debug('registering Shutdown function')
    self._server.register_function(self.Shutdown)

    if started_event:
      started_event.set()

    # Run the server's main loop
    logging.debug('Serving until Shutdown is called.')
    self._server.serve_forever()
    logging.debug('shutdown complete')

    if self._thread.isAlive():
      logging.debug('joining shutdown thread')
      self._thread.join()
      logging.debug('joined shutdown thread')
    self._thread = None

    if done_event:
      logging.debug('signaling done event')
      done_event.set()

if __name__ == '__main__':
  _DESCRIPTION = ('This script runs an xmlrpc server to be used by Swarm.')

  parser = optparse.OptionParser(usage='%prog [options] [filename]',
                                 description=_DESCRIPTION)
  parser.add_option('-v', '--verbose', action='store_true',
                    help='Set logging level to DEBUG. Optional. '
                    'Defaults to ERROR level.')
  (options, other_args) = parser.parse_args()

  XmlRpcServer().Start(options.verbose)
<|endoftext|>


<commit_before># Copyright (c) 2015-2016 Claudiu Popa <pcmanticore@gmail.com>

# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER

"""Astroid hooks for the PyQT library."""

from astroid import MANAGER, register_module_extender
from astroid.builder import AstroidBuilder
from astroid import nodes
from astroid import parse

def _looks_like_signal(node, signal_name='pyqtSignal'):
    if '__class__' in node.instance_attrs:
        cls = node.instance_attrs['__class__'][0]
        return cls.name == signal_name
    return False

def transform_pyqt_signal(node):
    module = parse('''
    class pyqtSignal(object):
        def connect(self, slot, type=None, no_receiver_check=False):
            pass
        def disconnect(self, slot):
            pass
        def emit(self, *args):
            pass
    ''')
    signal_cls = module['pyqtSignal']
    node.instance_attrs['emit'] = signal_cls['emit']
    node.instance_attrs['disconnect'] = signal_cls['disconnect']
    node.instance_attrs['connect'] = signal_cls['connect']

def pyqt4_qtcore_transform():
    return AstroidBuilder(MANAGER).string_build('''

def SIGNAL(signal_name): pass

class QObject(object):
    def emit(self, signal): pass
''')

register_module_extender(MANAGER, 'PyQt4.QtCore', pyqt4_qtcore_transform)
MANAGER.register_transform(nodes.FunctionDef, transform_pyqt_signal,
                           _looks_like_signal)<commit_msg>Handle AttributeErrors when _looks_like_signal receives something unexpected<commit_after># Copyright (c) 2015-2016 Claudiu Popa <pcmanticore@gmail.com>

# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER

"""Astroid hooks for the PyQT library."""

from astroid import MANAGER, register_module_extender
from astroid.builder import AstroidBuilder
from astroid import nodes
from astroid import parse

def _looks_like_signal(node, signal_name='pyqtSignal'):
    if '__class__' in node.instance_attrs:
        try:
            cls = node.instance_attrs['__class__'][0]
            return cls.name == signal_name
        except AttributeError:
            # return False if the cls does not have a name attribute
            pass
    return False

def transform_pyqt_signal(node):
    module = parse('''
    class pyqtSignal(object):
        def connect(self, slot, type=None, no_receiver_check=False):
            pass
        def disconnect(self, slot):
            pass
        def emit(self, *args):
            pass
    ''')
    signal_cls = module['pyqtSignal']
    node.instance_attrs['emit'] = signal_cls['emit']
    node.instance_attrs['disconnect'] = signal_cls['disconnect']
    node.instance_attrs['connect'] = signal_cls['connect']

def pyqt4_qtcore_transform():
    return AstroidBuilder(MANAGER).string_build('''

def SIGNAL(signal_name): pass

class QObject(object):
    def emit(self, signal): pass
''')

register_module_extender(MANAGER, 'PyQt4.QtCore', pyqt4_qtcore_transform)
MANAGER.register_transform(nodes.FunctionDef, transform_pyqt_signal,
                           _looks_like_signal)<|endoftext|>


<commit_before>import os
import sys
import time
from json import load
from urllib2 import urlopen, HTTPError
from django.db import transaction

from memopol2.utils import get_or_create

from reps.models import Email, WebSite
from mps.models import MP

if not os.path.exists("dumps"):
    os.mkdir("dumps")

def read_or_dl(url, name):
    if not os.path.exists("dumps/%s" % name):
        open("dumps/%s" % name, "w").write(urlopen(url).read())

    return open("dumps/%s" % name, "r")

def update_personal_informations(_mp, mp):
    _mp.full_name = mp["nom"]
    _mp.last_name = mp["nom_de_famille"]
    _mp.first_name = mp["prenom"]
    _mp.an_webpage = mp["url_an"]
    _mp.profession = mp["profession"]

def get_new_websites(mp, _mp):
    if mp["sites_web"]:
        for website in mp["sites_web"]:
            get_or_create(WebSite, url=website["site"], representative=_mp.representative_ptr)

def get_new_emails(mp, _mp):
    for email in mp["emails"]:
        get_or_create(Email, email=email["email"], representative=_mp.representative_ptr)

def set_mps_unactives():
    print "Setting all mps to unactive"
    a = 0
    total = MP.objects.count()
    for mp in MP.objects.all():
        a += 1
        mp.active = False
        mp.save()
        sys.stdout.write("%i/%i\r" % (a, total))
        sys.stdout.flush()
    sys.stdout.write("\n")

if __name__ == "__main__":
    mps = load(read_or_dl("http://www.nosdeputes.fr/deputes/json", "all_mps"))

    with transaction.commit_on_success():
        set_mps_unactives()

        a = 0
        for depute in mps["deputes"]:
            a += 1
            try:
                an_id = depute["depute"]["url_an"].split("/")[-1].split(".")[0]
                mp = load(read_or_dl(depute["depute"]["url_nosdeputes_api"], an_id))["depute"]
            except HTTPError:
                try:
                    print "Warning, failed to get a deputy, retrying in one seconde (url: %s)" % depute["depute"]["api_url"]
                    time.sleep(1)
                    mp = load(urlopen(depute["depute"]["url_nosdeputes_api"]))["depute"]
                except HTTPError:
                    print "Didn't managed to get this deputy, abort"
                    print "Go repport the bug on irc.freenode.net#regardscitoyens"
                    sys.exit(1)
            print a, "-", mp["nom"].encode("Utf-8")
            #mp = load(open("test"))["depute"]
            _mp = MP.objects.filter(an_id=mp["url_an"].split("/")[-1].split(".")[0])
            if _mp:
                _mp = _mp[0]
                if not depute["depute"].get("ancien_depute"):
                    _mp.active = True
                update_personal_informations(_mp, mp)
                get_new_emails(mp, _mp)
                get_new_websites(mp, _mp)
                print mp.get("groupe")
                #if mp["groupe_sigle"] and mp["groupe_sigle"] != "NI":
                    #Group.objects.get(abbreviation=mp["groupe_sigle"])
                _mp.save()
<commit_msg>[mod] remove useless code/comments<commit_after>import os
import sys
import time
from json import load
from urllib2 import urlopen, HTTPError
from django.db import transaction

from memopol2.utils import get_or_create

from reps.models import Email, WebSite
from mps.models import MP

if not os.path.exists("dumps"):
    os.mkdir("dumps")

def read_or_dl(url, name):
    if not os.path.exists("dumps/%s" % name):
        open("dumps/%s" % name, "w").write(urlopen(url).read())

    return open("dumps/%s" % name, "r")

def update_personal_informations(_mp, mp):
    _mp.full_name = mp["nom"]
    _mp.last_name = mp["nom_de_famille"]
    _mp.first_name = mp["prenom"]
    _mp.an_webpage = mp["url_an"]
    _mp.profession = mp["profession"]

def get_new_websites(mp, _mp):
    if mp["sites_web"]:
        for website in mp["sites_web"]:
            get_or_create(WebSite, url=website["site"], representative=_mp.representative_ptr)

def get_new_emails(mp, _mp):
    for email in mp["emails"]:
        get_or_create(Email, email=email["email"], representative=_mp.representative_ptr)

def set_mps_unactives():
    print "Setting all mps to unactive"
    a = 0
    total = MP.objects.count()
    for mp in MP.objects.all():
        a += 1
        mp.active = False
        mp.save()
        sys.stdout.write("%i/%i\r" % (a, total))
        sys.stdout.flush()
    sys.stdout.write("\n")

if __name__ == "__main__":
    mps = load(read_or_dl("http://www.nosdeputes.fr/deputes/json", "all_mps"))

    with transaction.commit_on_success():
        set_mps_unactives()

        a = 0
        for depute in mps["deputes"]:
            a += 1
            try:
                an_id = depute["depute"]["url_an"].split("/")[-1].split(".")[0]
                mp = load(read_or_dl(depute["depute"]["url_nosdeputes_api"], an_id))["depute"]
            except HTTPError:
                try:
                    print "Warning, failed to get a deputy, retrying in one seconde (url: %s)" % depute["depute"]["api_url"]
                    time.sleep(1)
                    mp = load(urlopen(depute["depute"]["url_nosdeputes_api"]))["depute"]
                except HTTPError:
                    print "Didn't managed to get this deputy, abort"
                    print "Go repport the bug on irc.freenode.net#regardscitoyens"
                    sys.exit(1)
            print a, "-", mp["nom"].encode("Utf-8")
            _mp = MP.objects.filter(an_id=mp["url_an"].split("/")[-1].split(".")[0])
            if _mp:
                _mp = _mp[0]
                if not depute["depute"].get("ancien_depute"):
                    _mp.active = True
                update_personal_informations(_mp, mp)
                get_new_emails(mp, _mp)
                get_new_websites(mp, _mp)
                _mp.save()
<|endoftext|>


<commit_before>from django.shortcuts import render, redirect
from django.http import HttpResponseRedirect, HttpResponse,  HttpResponseForbidden, HttpResponseNotAllowed
import sqlalchemy as sqla
from .forms import InputForm, UploadFileForm, UploadMapForm
from django.views.generic import View
from django.template import RequestContext
import csv
import os
import oeplatform.securitysettings as sec
from django.views.decorators.csrf import csrf_exempt
import urllib.request as request
from django.core.exceptions import PermissionDenied

import re
session = None

""" This is the initial view that initialises the database connection """

def listdbs(request):
    return render(request, 'dataedit/dataedit_dblist.html',{})

def listschemas(request, db):
    print(db)
    insp = connect(db)
    schemas = {schema for schema in  insp.get_schema_names()}
    return render(request, 'dataedit/dataedit_schemalist.html',{'db':db, 'schemas':schemas})

def listtables(request, db, schema):
    print(db,schema)
    insp = connect(db)

    tables =  {table for table in insp.get_table_names(schema=schema)}
    return render(request, 'dataedit/dataedit_tablelist.html',{'db':db, 'schema':schema, 'tables':tables})

class DataView(View):

    """ This method handles the GET requests for the main page of data edit.
        Initialises the session data (if necessary)
    """

    def get(self, request, db, schema, table):
        if any((x not in request.session for x in ["table", "schema", "fields", "headers", "floatingRows"])) or (
                request.session['table'] != table or request.session['schema'] != schema):
            error = loadSessionData(request, connect(db), db, schema, table)
            if error:
                return error
        return render(request, 'dataedit/dataedit_overview.html',
                      {'data': request.session['floatingRows'], 'headers': request.session['headers']})

"""
This View handles new data input and edit
"""

class DataInputView(View):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, *kwargs)
        self.id = None

    """
    Handles the GET-request for the data edit and entry page. Requests
    may contain an id, if pending data shall be edited.
    """

    def get(self, request):
        self.form = InputForm(fields=request.session['fields'])

        did = None
        if "id" in request.GET:
            did = int(request.GET["id"])
            for field, entry in zip(request.session['headers'], request.session[
                                    'floatingRows'][did]):
                self.form.fields[field].initial = entry

        return render(request, 'dataedit/dataedit.html',
                      {'form': self.form, 'internal_oe_id': did, 'pks':request.session['primaryKeys']})

    """ Handles the POST-request for the data edit and entry page. Requests
        may contain an id, if pending data shall be edited.
    """

    def post(self, request):
        request.session.set_test_cookie()
        print(request.POST)
        newRow = [request.POST[head] for head in request.session['headers']]
        pkvals = {pk:request.POST[pk] for pk in request.session["primaryKeys"]}

        if not validatePks(request.session["db"],
            request.session["schema"],
            request.session["table"],
            pkvals):
            form = InputForm(fields=request.session['fields'],values=request.POST)
            return render(request, 'dataedit/dataedit.html',
                      {'form': form,
                      'internal_oe_id':request.POST["internal_oe_id"],
                      'pks':request.session['primaryKeys'],
                      'errors':["Primary Key is already present"]})

        did = request.POST["internal_oe_id"]
        if did not in [None, 'None']:
            did = int(did)
            request.session['floatingRows'][did] = newRow
        else:
            request.session['floatingRows'].append(newRow)
        return _renderTable(request)

"""
This View yields an upload functionality for CSV-files
"""

class DataUploadView(View):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, *kwargs)

    def post(self, request):
        form = UploadFileForm(request.POST, request.FILES)
        if form.is_valid():
            return render(request, 'dataedit/dataedit_map.html',
                          {'form': form})
        else:
            return render(
                request, 'dataedit/dataedit_upload.html', {'form': form})

    def get(self, request):
        form = UploadFileForm()
        return render(request, 'dataedit/dataedit_upload.html', {'form': form})

"""
This View allows the user to specifies how each the table columns relate to the
csv columns
"""

class DataMapView(View):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, *kwargs)

    def post(self, request):
        def store(f):
            dname = 'tmp/' + f.name
            with open(dname, 'wb+') as destination:
                for chunk in f.chunks():
                    destination.write(chunk)
            return dname
        datafile = store(request.FILES['file'])
        with open(datafile, 'r') as outf:
            reader = csv.reader(outf, delimiter='\t', quotechar='"')
            headers = next(reader)
            request.session['data'] = (
                headers, [next(reader) for _ in range(10)])
            form = UploadMapForm(
                fields=request.session['fields'],
                headers=headers)
        os.remove(datafile)  # TODO: Unsicher!
        return render(request, 'dataedit/dataedit_map.html', {'form': form})

"""
Deletes the request.POST["id"]-th entry
"""

def delete(request):
    if request.POST["id"] is not None:
        did = int(request.POST["id"])
        request.session['floatingRows'] = request.session['floatingRows'][
            :did] + request.session['floatingRows'][did + 1:]
    return redirect('/dataedit/view/{db}/{schema}/{table}'.format(db=request.session['db'],
                schema=request.session['schema'],
                table=request.session['table']),
                    {'data': request.session['floatingRows'],
                        'headers': request.session['headers']})

def commit(request):
    db = request.session["db"]
    if db in ["test"]:
        engine = _get_engine(db)

    # load schema name and check for sanity
    schema = request.session.pop("schema", None)
    if not is_pg_qual(schema):
        raise p.toolkit.ValidationError("Invalid schema name")
    # Check whether schema exists

    # load table name and check for sanity
    table = request.session.pop("table", None)
    if not is_pg_qual(table):
        raise p.toolkit.ValidationError("Invalid table name")

    fields = request.session.pop("fields",[])
    fieldnames = [f[0] for f in fields]
    if not fields == ["*"] and not all(map(is_pg_qual,fieldnames)):
        raise p.toolkit.ValidationError("Invalid field name")

    data = request.session.pop("floatingRows",[])
    meta = sqla.MetaData()
    tab = sqla.Table(table,meta,schema=schema,autoload=True,autoload_with=engine)
    connection = engine.connect()

    connection.execute(
        tab.insert(),
        *[dict(zip(fieldnames,row)) for row in data]
    )

"""
    Constructs the new rows for @table using the mapping specified in dataedit_map.html
"""

def confirm(request):
    (csv_headers, rows) = request.session["data"]
    for row in rows:
        newRow = []
        for head in request.session['headers']:
            if request.POST[head] == "---":
                newRow.append(None)
            else:
                newRow.append(row[csv_headers.index(request.POST[head])])
        request.session['floatingRows'].append(newRow)
    return _renderTable(request)

""" Connects to the specified table using the security infomation stored in
securitysettings.py"""

def loadSessionData(request, insp, db, schema, table):
    print(db,schema,table)
    if not all(map(is_pg_qual,[db,schema,table])):
        return HttpResponseForbidden()
    request.session['db'] = db
    request.session['table'] = table
    request.session['schema'] = schema

    fields = [(c["name"], str(c["type"]))
                                 for c in insp.get_columns(table, schema=schema)]
    if not all(map(is_pg_qual,[f[0] for f in fields])):
        return HttpResponseForbidden()
    request.session['fields'] = fields
    request.session['headers'] = [x[0] for x in request.session['fields']]
    request.session['primaryKeys'] = insp.get_primary_keys(table, schema=schema)
    request.session['floatingRows'] = []

@csrf_exempt
def dropSessionData(request):
    print(request.method)
    if not (request.is_ajax() or request.method=='POST'):
        return HttpResponseNotAllowed(['POST'])
    del request.session['floatingRows']
    request.session['floatingRows'] = []
    return HttpResponse('ok')

def _renderTable(request):
    L = request.session['floatingRows']
    del request.session['floatingRows']
    request.session['floatingRows'] = L
    return redirect('/dataedit/view/{db}/{schema}/{table}'.format(
                db=request.session['schema'],
                schema=request.session['schema'],
                table=request.session['table']),
                    {})

# DB Utils

def validatePks(db, schema, table, pkValues):
    engine = _get_engine(db)
    connection = engine.connect()
    meta = sqla.MetaData()
    tab = sqla.Table(table,meta,schema=schema,autoload=True,autoload_with=engine)
    where = " and ".join(["{k}={v}".format(k=k,v=v) for (k,v) in pkValues.items()])
    print(pkValues)
    result = connection.execute(tab.select(whereclause=where))
    f = result.first()
    print(f)
    return f == None

def connect(db):
    engine = _get_engine(db)
    insp = sqla.inspect(engine)
    return insp

def _get_engine(db):
    if not all(map(is_pg_qual,[db])):
        raise PermissionDenied()
    engine = sqla.create_engine(
        'postgresql://{0}:{1}@{2}:{3}/{4}'.format(
            sec.dbuser,
            sec.dbpasswd,
            sec.dbhost,
            sec.dbport,
            db))
    return engine
def is_pg_qual(x):
    pgsql_qualifier = re.compile(r"^[\w\d_]+$")
    return pgsql_qualifier.search(x)

<commit_msg>Remove obsolete constructors<commit_after>from django.shortcuts import render, redirect
from django.http import HttpResponseRedirect, HttpResponse,  HttpResponseForbidden, HttpResponseNotAllowed
import sqlalchemy as sqla
from .forms import InputForm, UploadFileForm, UploadMapForm
from django.views.generic import View
from django.template import RequestContext
import csv
import os
import oeplatform.securitysettings as sec
from django.views.decorators.csrf import csrf_exempt
import urllib.request as request
from django.core.exceptions import PermissionDenied

import re
session = None

""" This is the initial view that initialises the database connection """

def listdbs(request):
    return render(request, 'dataedit/dataedit_dblist.html',{})

def listschemas(request, db):
    print(db)
    insp = connect(db)
    schemas = {schema for schema in  insp.get_schema_names()}
    return render(request, 'dataedit/dataedit_schemalist.html',{'db':db, 'schemas':schemas})

def listtables(request, db, schema):
    print(db,schema)
    insp = connect(db)

    tables =  {table for table in insp.get_table_names(schema=schema)}
    return render(request, 'dataedit/dataedit_tablelist.html',{'db':db, 'schema':schema, 'tables':tables})

class DataView(View):

    """ This method handles the GET requests for the main page of data edit.
        Initialises the session data (if necessary)
    """

    def get(self, request, db, schema, table):
        if any((x not in request.session for x in ["table", "schema", "fields", "headers", "floatingRows"])) or (
                request.session['table'] != table or request.session['schema'] != schema):
            error = loadSessionData(request, connect(db), db, schema, table)
            if error:
                return error
        return render(request, 'dataedit/dataedit_overview.html',
                      {'data': request.session['floatingRows'], 'headers': request.session['headers']})

"""
This View handles new data input and edit
"""

class DataInputView(View):

    """
    Handles the GET-request for the data edit and entry page. Requests
    may contain an id, if pending data shall be edited.
    """

    def get(self, request):
        self.form = InputForm(fields=request.session['fields'])

        did = None
        if "id" in request.GET:
            did = int(request.GET["id"])
            for field, entry in zip(request.session['headers'], request.session[
                                    'floatingRows'][did]):
                self.form.fields[field].initial = entry

        return render(request, 'dataedit/dataedit.html',
                      {'form': self.form, 'internal_oe_id': did, 'pks':request.session['primaryKeys']})

    """ Handles the POST-request for the data edit and entry page. Requests
        may contain an id, if pending data shall be edited.
    """

    def post(self, request):
        request.session.set_test_cookie()
        print(request.POST)
        newRow = [request.POST[head] for head in request.session['headers']]
        pkvals = {pk:request.POST[pk] for pk in request.session["primaryKeys"]}

        if not validatePks(request.session["db"],
            request.session["schema"],
            request.session["table"],
            pkvals):
            form = InputForm(fields=request.session['fields'],values=request.POST)
            return render(request, 'dataedit/dataedit.html',
                      {'form': form,
                      'internal_oe_id':request.POST["internal_oe_id"],
                      'pks':request.session['primaryKeys'],
                      'errors':["Primary Key is already present"]})

        did = request.POST["internal_oe_id"]
        if did not in [None, 'None']:
            did = int(did)
            request.session['floatingRows'][did] = newRow
        else:
            request.session['floatingRows'].append(newRow)
        return _renderTable(request)

"""
This View yields an upload functionality for CSV-files
"""

class DataUploadView(View):

    def post(self, request):
        form = UploadFileForm(request.POST, request.FILES)
        if form.is_valid():
            return render(request, 'dataedit/dataedit_map.html',
                          {'form': form})
        else:
            return render(
                request, 'dataedit/dataedit_upload.html', {'form': form})

    def get(self, request):
        form = UploadFileForm()
        return render(request, 'dataedit/dataedit_upload.html', {'form': form})

"""
This View allows the user to specifies how each the table columns relate to the
csv columns
"""

class DataMapView(View):

    def post(self, request):
        def store(f):
            dname = 'tmp/' + f.name
            with open(dname, 'wb+') as destination:
                for chunk in f.chunks():
                    destination.write(chunk)
            return dname
        datafile = store(request.FILES['file'])
        with open(datafile, 'r') as outf:
            reader = csv.reader(outf, delimiter='\t', quotechar='"')
            headers = next(reader)
            request.session['data'] = (
                headers, [next(reader) for _ in range(10)])
            form = UploadMapForm(
                fields=request.session['fields'],
                headers=headers)
        os.remove(datafile)  # TODO: Unsicher!
        return render(request, 'dataedit/dataedit_map.html', {'form': form})

"""
Deletes the request.POST["id"]-th entry
"""

def delete(request):
    if request.POST["id"] is not None:
        did = int(request.POST["id"])
        request.session['floatingRows'] = request.session['floatingRows'][
            :did] + request.session['floatingRows'][did + 1:]
    return redirect('/dataedit/view/{db}/{schema}/{table}'.format(db=request.session['db'],
                schema=request.session['schema'],
                table=request.session['table']),
                    {'data': request.session['floatingRows'],
                        'headers': request.session['headers']})

def commit(request):
    db = request.session["db"]
    if db in ["test"]:
        engine = _get_engine(db)

    # load schema name and check for sanity
    schema = request.session.pop("schema", None)
    if not is_pg_qual(schema):
        raise p.toolkit.ValidationError("Invalid schema name")
    # Check whether schema exists

    # load table name and check for sanity
    table = request.session.pop("table", None)
    if not is_pg_qual(table):
        raise p.toolkit.ValidationError("Invalid table name")

    fields = request.session.pop("fields",[])
    fieldnames = [f[0] for f in fields]
    if not fields == ["*"] and not all(map(is_pg_qual,fieldnames)):
        raise p.toolkit.ValidationError("Invalid field name")

    data = request.session.pop("floatingRows",[])
    meta = sqla.MetaData()
    tab = sqla.Table(table,meta,schema=schema,autoload=True,autoload_with=engine)
    connection = engine.connect()

    connection.execute(
        tab.insert(),
        *[dict(zip(fieldnames,row)) for row in data]
    )

"""
    Constructs the new rows for @table using the mapping specified in dataedit_map.html
"""

def confirm(request):
    (csv_headers, rows) = request.session["data"]
    for row in rows:
        newRow = []
        for head in request.session['headers']:
            if request.POST[head] == "---":
                newRow.append(None)
            else:
                newRow.append(row[csv_headers.index(request.POST[head])])
        request.session['floatingRows'].append(newRow)
    return _renderTable(request)

""" Connects to the specified table using the security infomation stored in
securitysettings.py"""

def loadSessionData(request, insp, db, schema, table):
    print(db,schema,table)
    if not all(map(is_pg_qual,[db,schema,table])):
        return HttpResponseForbidden()
    request.session['db'] = db
    request.session['table'] = table
    request.session['schema'] = schema

    fields = [(c["name"], str(c["type"]))
                                 for c in insp.get_columns(table, schema=schema)]
    if not all(map(is_pg_qual,[f[0] for f in fields])):
        return HttpResponseForbidden()
    request.session['fields'] = fields
    request.session['headers'] = [x[0] for x in request.session['fields']]
    request.session['primaryKeys'] = insp.get_primary_keys(table, schema=schema)
    request.session['floatingRows'] = []

@csrf_exempt
def dropSessionData(request):
    print(request.method)
    if not (request.is_ajax() or request.method=='POST'):
        return HttpResponseNotAllowed(['POST'])
    del request.session['floatingRows']
    request.session['floatingRows'] = []
    return HttpResponse('ok')

def _renderTable(request):
    L = request.session['floatingRows']
    del request.session['floatingRows']
    request.session['floatingRows'] = L
    return redirect('/dataedit/view/{db}/{schema}/{table}'.format(
                db=request.session['schema'],
                schema=request.session['schema'],
                table=request.session['table']),
                    {})

# DB Utils

def validatePks(db, schema, table, pkValues):
    engine = _get_engine(db)
    connection = engine.connect()
    meta = sqla.MetaData()
    tab = sqla.Table(table,meta,schema=schema,autoload=True,autoload_with=engine)
    where = " and ".join(["{k}={v}".format(k=k,v=v) for (k,v) in pkValues.items()])
    print(pkValues)
    result = connection.execute(tab.select(whereclause=where))
    f = result.first()
    print(f)
    return f == None

def connect(db):
    engine = _get_engine(db)
    insp = sqla.inspect(engine)
    return insp

def _get_engine(db):
    if not all(map(is_pg_qual,[db])):
        raise PermissionDenied()
    engine = sqla.create_engine(
        'postgresql://{0}:{1}@{2}:{3}/{4}'.format(
            sec.dbuser,
            sec.dbpasswd,
            sec.dbhost,
            sec.dbport,
            db))
    return engine
def is_pg_qual(x):
    pgsql_qualifier = re.compile(r"^[\w\d_]+$")
    return pgsql_qualifier.search(x)

<|endoftext|>


<commit_before>from django import template
from django.conf import settings
from django.core.urlresolvers import reverse
from django.utils.importlib import import_module
from django.utils.safestring import mark_safe

register = template.Library()

def load_callable(path_to_callable):
    try:
        mod_name, func_name = path_to_callable.rsplit(".", 1)
    except ValueError:
        raise Exception("Improperly configured.")
    try:
        mod = import_module(mod_name)
    except ImportError:
        raise Exception("Could not import %s" % mod_name)
    try:
        func = getattr(mod, func_name)
    except AttributeError:
        raise Exception("The module '%s' does not contain '%s'." % (mod_name, func_name))
    return func

class RemindersNode(template.Node):

    @classmethod
    def handle_token(cls, parser, token):
        bits = token.split_contents()
        if len(bits) != 3:
            raise template.TemplateSyntaxError
        return cls(as_var = bits[2])

    def __init__(self, as_var):
        self.as_var = as_var

    def render(self, context):
        request = context["request"]
        reminders = []
        for label in settings.REMINDERS:
            if label not in request.session:
                test = settings.REMINDERS[label]["test"]
                message = settings.REMINDERS[label]["message"]
                if not callable(test):
                    test = load_callable(test)
                url = None
                if settings.REMINDERS[label].get("dismissable", True):
                    url = reverse("reminders_dismiss", kwargs={"label": label})
                reminders.append({
                    "message": mark_safe(message % test(request.user)),
                    "dismiss_url": url
                })
        context[self.as_var] = reminders
        return ""

@register.tag
def reminders(parser, token):
    """
    Usage::
        {% reminders as var %}

    Returns a list of reminders
    """
    return RemindersNode.handle_token(parser, token)
<commit_msg>Only add message if a result is returned<commit_after>from django import template
from django.conf import settings
from django.core.urlresolvers import reverse
from django.utils.importlib import import_module
from django.utils.safestring import mark_safe

register = template.Library()

def load_callable(path_to_callable):
    try:
        mod_name, func_name = path_to_callable.rsplit(".", 1)
    except ValueError:
        raise Exception("Improperly configured.")
    try:
        mod = import_module(mod_name)
    except ImportError:
        raise Exception("Could not import %s" % mod_name)
    try:
        func = getattr(mod, func_name)
    except AttributeError:
        raise Exception("The module '%s' does not contain '%s'." % (mod_name, func_name))
    return func

class RemindersNode(template.Node):

    @classmethod
    def handle_token(cls, parser, token):
        bits = token.split_contents()
        if len(bits) != 3:
            raise template.TemplateSyntaxError
        return cls(as_var = bits[2])

    def __init__(self, as_var):
        self.as_var = as_var

    def render(self, context):
        request = context["request"]
        reminders = []
        for label in settings.REMINDERS:
            if label not in request.session:
                test = settings.REMINDERS[label]["test"]
                message = settings.REMINDERS[label]["message"]
                if not callable(test):
                    test = load_callable(test)
                url = None
                if settings.REMINDERS[label].get("dismissable", True):
                    url = reverse("reminders_dismiss", kwargs={"label": label})
                result = test(request.user)
                if result:
                    reminders.append({
                        "message": mark_safe(message % test(request.user)),
                        "dismiss_url": url
                    })
        context[self.as_var] = reminders
        return ""

@register.tag
def reminders(parser, token):
    """
    Usage::
        {% reminders as var %}

    Returns a list of reminders
    """
    return RemindersNode.handle_token(parser, token)
<|endoftext|>


<commit_before>'''
Created on 4 Jul 2016

@author: ikalvari

Description: Genome's Project - Configuration File
Purpose: Contains all constants used in the genome's project. Uniprot and ENA
URLS can be accessed though the corresponding REST API webpages.

'''

from config import rfam_local as cfl

# -----------------------------------CONSTANTS---------------------------------

# ENA file formats
FORMATS = {"xml": ".xml", "fasta": ".fa"}

# Taxonomy Domains
DOMAINS = ['eukaryota', 'archaea', 'viruses', 'bacteria']

# Rfam's LSF GPFS location
RFAM_GPFS_LOC = cfl.RFAM_GPFS_LOC
LOC_PATH = cfl.LOC_PATH

# LSF JOB constraints & params
MEM = 5000
TMP_MEM = 1000
USER_EMAIL = cfl.USER_EMAIL

# LSF GROUP constraints
LSF_GROUPS_CMD = cfl.LSF_GROUPS_CMD
LSF_GEN_GROUP = cfl.LSF_GEN_GROUP
JOB_LIMIT = 5

# ENA GCA report file label
GCA_REP_LBL = "Sequence_Report"  # GCA ASSEMPLY REPORT FILE XML LABEL
GCA_REG_LBL = "Regions"  # GCA ASSEMPLY REGIONS FILE XML LABEL

# -----------------------------------URLs--------------------------------------

# This returns the list of reference proteomes from Uniprot
REF_PROT_LIST_URL = "http://www.uniprot.org/proteomes/?query=*&fil=reference%3Ayes&format=list"

# Retrieve a list of ref. Proteoms including GCA accessions and lineage
REF_PROT_REST_URL = ("http://www.uniprot.org/proteomes/?fil=reference:yes&"
                     "force=no&format=tab&columns=id,assembly,lineage")

# Retrieve a sorted list of ref. Proteoms including GCA accessions,
# species name, id and lineage
'''
REF_PROT_REST_URL = ("http://www.uniprot.org/proteomes/?sort=&desc=&"
                     "compress=no&query=&fil=reference:yes&force=no&"
                     "format=tab&columns=id,name,organism-id,assembly,lineage")
'''

# non sorted list
'''
REF_PROT_REST_URL = ("http://www.uniprot.org/proteomes/?fil=reference:yes&"
                     "force=no&format=tab&columns=id,name,organism-id,"
                     "assembly,lineage")
'''

# Retrieve the proteome rdf file
PROTEOME_URL = "http://www.uniprot.org/proteomes/%s.rdf"

# Retrieve the genome's xml file
ENA_XML_URL = "http://www.ebi.ac.uk/ena/data/view/%s&display=xml"

# ENA url for file download
ENA_DATA_URL = "http://www.ebi.ac.uk/ena/data/view/%s&display=%s&download=gzip"

# ENA url for assembly data retrieval via taxon id
ENA_TAX_URL = "http://www.ebi.ac.uk/ena/data/warehouse/search?query=\"tax_eq(%s)\"&result=assembly&display=xml"

# path to genome downloader executable
GEN_DWLD_EXEC = cfl.GEN_DWLD_EXEC

# -----------------------------------MODELS------------------------------------

GENOME_MODEL = "RfamLive.Genome"  # table names RfamLive.Genome
GENSEQ_MODEL = "RfamLive.Genseq"  # table names RfamLive.Genseq
<commit_msg>Add missing configuration values<commit_after>'''
Created on 4 Jul 2016

@author: ikalvari

Description: Genome's Project - Configuration File
Purpose: Contains all constants used in the genome's project. Uniprot and ENA
URLS can be accessed though the corresponding REST API webpages.

'''

from config import rfam_local as cfl

# -----------------------------------CONSTANTS---------------------------------

# ENA file formats
FORMATS = {"xml": ".xml", "fasta": ".fa"}

# Taxonomy Domains
DOMAINS = ['eukaryota', 'archaea', 'viruses', 'bacteria']

# Rfam's LSF GPFS location
RFAM_GPFS_LOC = cfl.RFAM_GPFS_LOC
LOC_PATH = cfl.LOC_PATH

# LSF JOB constraints & params
MEM = 5000
TMP_MEM = 1000
USER_EMAIL = cfl.USER_EMAIL

# LSF GROUP constraints
LSF_GROUPS_CMD = cfl.LSF_GROUPS_CMD
LSF_GEN_GROUP = cfl.LSF_GEN_GROUP
JOB_LIMIT = 5

# ENA GCA report file label
GCA_REP_LBL = "Sequence_Report"  # GCA ASSEMPLY REPORT FILE XML LABEL
GCA_REG_LBL = "Regions"  # GCA ASSEMPLY REGIONS FILE XML LABEL

# -----------------------------------URLs--------------------------------------

# This returns the list of reference proteomes from Uniprot
REF_PROT_LIST_URL = "http://www.uniprot.org/proteomes/?query=*&fil=reference%3Ayes&format=list"

# Retrieve a list of ref. Proteoms including GCA accessions and lineage
REF_PROT_REST_URL = ("http://www.uniprot.org/proteomes/?fil=reference:yes&"
                     "force=no&format=tab&columns=id,assembly,lineage")

# Retrieve a sorted list of ref. Proteoms including GCA accessions,
# species name, id and lineage
'''
REF_PROT_REST_URL = ("http://www.uniprot.org/proteomes/?sort=&desc=&"
                     "compress=no&query=&fil=reference:yes&force=no&"
                     "format=tab&columns=id,name,organism-id,assembly,lineage")
'''

# non sorted list
'''
REF_PROT_REST_URL = ("http://www.uniprot.org/proteomes/?fil=reference:yes&"
                     "force=no&format=tab&columns=id,name,organism-id,"
                     "assembly,lineage")
'''

# Retrieve the proteome rdf file
PROTEOME_URL = "http://www.uniprot.org/proteomes/%s.rdf"

# Proteome xml file
PROTEOME_XML_URL = "http://www.uniprot.org/proteomes/%s.xml"

NCBI_SEQ_URL = ''

# Retrieve the genome's xml file
ENA_XML_URL = "http://www.ebi.ac.uk/ena/data/view/%s&display=xml"

# ENA url for file download
ENA_DATA_URL = "http://www.ebi.ac.uk/ena/data/view/%s&display=%s&download=gzip"

# ENA url for assembly data retrieval via taxon id
ENA_TAX_URL = "http://www.ebi.ac.uk/ena/data/warehouse/search?query=\"tax_eq(%s)\"&result=assembly&display=xml"

# path to genome downloader executable
GEN_DWLD_EXEC = cfl.GEN_DWLD_EXEC

# -----------------------------------MODELS------------------------------------

GENOME_MODEL = "RfamLive.Genome"  # table names RfamLive.Genome
GENSEQ_MODEL = "RfamLive.Genseq"  # table names RfamLive.Genseq
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Elijah: Cloudlet Infrastructure for Mobile Computing
# Copyright (C) 2011-2012 Carnegie Mellon University
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of version 2 of the GNU General Public License as published
# by the Free Software Foundation.  A copy of the GNU General Public License
# should have been distributed along with this program in the file
# LICENSE.GPL.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#
import os
import sys
import socket
from optparse import OptionParser
import time
import struct
import math

def recv_all(sock, size):
    data = ''
    while len(data) < size:
        data += sock.recv(size - len(data))
    return data

def process_command_line(argv):
    global operation_mode

    parser = OptionParser(usage="usage: %prog [option]", version="MOPED Desktop Client")
    parser.add_option(
            '-i', '--input', action='store', type='string', dest='input_file',
            help='Set Input image directory')
    parser.add_option(
            '-s', '--server', action='store', type='string', dest='server_address', default="localhost",
            help='Set Input image directory')
    parser.add_option(
            '-p', '--port', action='store', type='int', dest='server_port', default=9093,
            help='Set Input image directory')
    settings, args = parser.parse_args(argv)
    if not len(args) == 0:
        parser.error('program takes no command-line arguments; "%s" ignored.' % (args,))

    return settings, args

def send_request(address, port, input_data):
    # connection
    try:
        print "Connecting to (%s, %d).." % (address, port)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.setblocking(True)
        sock.connect((address, port))
    except socket.error, msg:
        sys.stderr.write("Error, %s\n" % msg[1])
        sys.exit(1)

    # send requests
    current_duration = -1
    print "index\tstart\tend\tduration\tjitter"
    start_time = time.time()

    if input_data:
        loop_length = len(input_data)
    else:
        loop_length = 1000

    for index in xrange(loop_length):
        start_time_request = time.time()

        # send acc data
        if input_data:
            x_acc = float(input_data[index].split("  ")[1])
            y_acc = float(input_data[index].split("  ")[2])
        else:
            x_acc = -9.0
            y_acc = -1.0

        sent_size = sock.send(struct.pack("!ff", x_acc, y_acc))
        #print "Sent acc (%f, %f)" % (x_acc, y_acc)
        if not sent_size == 8:
            sys.strerr.write("Error, send wrong size of acc data: %d" + sent_size)
            sys.exit(1)

        # recv
        data = sock.recv(4)
        ret_size = struct.unpack("!I", data)[0]
        #print "Recv size : %d" % (ret_size)

        if not ret_size == 0:
            ret_data = recv_all(sock, ret_size)
            if not ret_size == len(ret_data):
                sys.stderr.write("Error, returned value size : %d" % (len(ret_data)))
                sys.exit(1)
        else:
            sys.stderr.write("Error, return size must not be zero")
            sys.exit(1)

        # print result
        end_time_request = time.time()
        prev_duration = current_duration
        current_duration = end_time_request-start_time_request

        if prev_duration == -1: # fisrt response
            print "%d\t%05.3f\t%05.3f\t%05.3f\t0" % (index, start_time_request,\
                    end_time_request, \
                    end_time_request-start_time_request)
        else:
            print "%d\t%05.3f\t%05.3f\t%05.3f\t%05.3f" % (index, round(start_time_request, 3), \
                    end_time_request, \
                    current_duration, \
                    math.fabs(current_duration-prev_duration))

    print "Total time : %05.3f" % (time.time()-start_time)

def main(argv=None):
    global LOCAL_IPADDRESS
    settings, args = process_command_line(sys.argv[1:])
    input_accs = None
    if settings.input_file and os.path.exists(settings.input_file):
        input_accs = open(settings.input_file, "r").read().split("\n")

    send_request(settings.server_address, settings.server_port, input_accs)

    return 0

if __name__ == "__main__":
    status = main()
    sys.exit(status)
<commit_msg>Data convertor to CDF<commit_after>#!/usr/bin/env python
#
# Elijah: Cloudlet Infrastructure for Mobile Computing
# Copyright (C) 2011-2012 Carnegie Mellon University
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of version 2 of the GNU General Public License as published
# by the Free Software Foundation.  A copy of the GNU General Public License
# should have been distributed along with this program in the file
# LICENSE.GPL.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#
import os
import sys
import socket
from optparse import OptionParser
import time
import struct
import math

def recv_all(sock, size):
    data = ''
    while len(data) < size:
        data += sock.recv(size - len(data))
    return data

def process_command_line(argv):
    global operation_mode

    parser = OptionParser(usage="usage: %prog [option]", version="MOPED Desktop Client")
    parser.add_option(
            '-i', '--input', action='store', type='string', dest='input_file',
            help='Set Input image directory')
    parser.add_option(
            '-s', '--server', action='store', type='string', dest='server_address', default="localhost",
            help='Set Input image directory')
    parser.add_option(
            '-p', '--port', action='store', type='int', dest='server_port', default=9093,
            help='Set Input image directory')
    settings, args = parser.parse_args(argv)
    if not len(args) == 0:
        parser.error('program takes no command-line arguments; "%s" ignored.' % (args,))

    return settings, args

def send_request(address, port, input_data):
    # connection
    try:
        print "Connecting to (%s, %d).." % (address, port)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.setblocking(True)
        sock.connect((address, port))
    except socket.error, msg:
        sys.stderr.write("Error, %s\n" % msg[1])
        sys.exit(1)

    # send requests
    current_duration = -1
    print "index\tstart\tend\tduration\tjitter"
    start_time = time.time()

    if input_data:
        loop_length = len(input_data)
    else:
        loop_length = 1000

    for index in xrange(loop_length):
        start_time_request = time.time()

        # send acc data
        if input_data:
            x_acc = float(input_data[index].split("  ")[1])
            y_acc = float(input_data[index].split("  ")[2])
        else:
            x_acc = -9.0
            y_acc = -1.0

        sent_size = sock.send(struct.pack("!ff", x_acc, y_acc))
        #print "Sent acc (%f, %f)" % (x_acc, y_acc)
        if not sent_size == 8:
            sys.strerr.write("Error, send wrong size of acc data: %d" + sent_size)
            sys.exit(1)

        # recv
        data = sock.recv(4)
        ret_size = struct.unpack("!I", data)[0]
        #print "Recv size : %d" % (ret_size)

        if not ret_size == 0:
            ret_data = recv_all(sock, ret_size)
            if not ret_size == len(ret_data):
                sys.stderr.write("Error, returned value size : %d" % (len(ret_data)))
                sys.exit(1)
        else:
            sys.stderr.write("Error, return size must not be zero")
            sys.exit(1)

        # print result
        end_time_request = time.time()
        prev_duration = current_duration
        current_duration = end_time_request-start_time_request

        if prev_duration == -1: # fisrt response
            print "%d\t%05.5f\t%05.5f\t%05.5f\t0" % (index, start_time_request,\
                    end_time_request, \
                    end_time_request-start_time_request)
        else:
            print "%d\t%05.5f\t%05.5f\t%05.5f\t%05.5f" % (index, round(start_time_request, 3), \
                    end_time_request, \
                    current_duration, \
                    math.fabs(current_duration-prev_duration))

    print "Total time : %05.5f" % (time.time()-start_time)

def main(argv=None):
    global LOCAL_IPADDRESS
    settings, args = process_command_line(sys.argv[1:])
    input_accs = None
    if settings.input_file and os.path.exists(settings.input_file):
        input_accs = open(settings.input_file, "r").read().split("\n")

    send_request(settings.server_address, settings.server_port, input_accs)

    return 0

if __name__ == "__main__":
    status = main()
    sys.exit(status)
<|endoftext|>


<commit_before>from datapipe import *

def test_example():

    # Create test file
    open('input.txt', 'w').close()

    from datapipe import Task, Input, LocalFile, require

    # New tasks are defined by inheriting from an existing Task
    class AddLines(Task):

        # The inputs can be anything the task depends on:
        # Local and remote files, python objects, numpy arrays, ...
        infile = Input()
        count = Input(default=1)
        text = Input(default='This is some text')

        # The outputs are defined dynamically (with access to the inputs)
        def outputs(self):
            return LocalFile(self.infile.get().replace('.txt', '.AddLines.txt'))

        # The actual task is defined as a function with access to inputs and outputs
        def run(self):
            with open(self.infile.get()) as f:
                with open(self.outputs().get(), 'w') as g:
                    g.write(f.read())
                    for i in range(self.count):
                        g.write(self.text + '\n')

    # Create initial Targets
    infile = LocalFile('input.txt')

    # Define the pipeline
    task1 = AddLines(infile, count=2)
    task2 = AddLines(task1.outputs(), count=3, text='This is some more text')

    # Require a target to execute all tasks needed to produce it
    require(task2.outputs())

<commit_msg>Modify test case<commit_after>from datapipe import *

def test_example():

    # Create test file
    open('/tmp/input.txt', 'w').close()

    from datapipe import Task, Input, LocalFile, require

    # New tasks are defined by inheriting from an existing Task
    class AddLines(Task):

        # The inputs can be anything the task depends on:
        # Local and remote files, python objects, numpy arrays, ...
        infile = Input()
        count = Input(default=1)
        text = Input(default='This is some text')

        # The outputs are defined dynamically (with access to the inputs)
        def outputs(self):
            return LocalFile(self.infile.path().replace('.txt', '.AddLines.txt'))

        # The actual task is defined as a function with access to inputs and outputs
        def run(self):
            with self.infile.open() as f:
                with self.outputs().open('w') as g:
                    g.write(f.read())
                    for i in range(self.count):
                        g.write(self.text + '\n')

    # Create initial Targets
    infile = LocalFile('/tmp/input.txt')

    # Define the pipeline
    task1 = AddLines(infile, count=2)
    task2 = AddLines(task1.outputs(), count=3, text='This is some more text')

    # Require a target to execute all tasks needed to produce it
    require(task2.outputs())

<|endoftext|>


<commit_before>#!/usr/bin/python
#+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
#|R|a|s|p|b|e|r|r|y|P|i|.|c|o|m|.|t|w|
#+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
# Copyright (c) 2016, raspberrypi.com.tw
# All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# hc_sr04_measure_distance.py
# Measure the distance between HC-SR04 and nearest wall or solid object.
#
# Author : sosorry
# Date   : 09/13/2014

import RPi.GPIO as GPIO
import time

GPIO.setmode(GPIO.BOARD)
TRIGGER_PIN = 16
ECHO_PIN    = 18
GPIO.setup(TRIGGER_PIN,  GPIO.OUT)
GPIO.setup(ECHO_PIN,     GPIO.IN)
GPIO.output(TRIGGER_PIN, GPIO.LOW)
v = 343		# (331 + 0.6*20)

def measure() :
    GPIO.output(TRIGGER_PIN, GPIO.HIGH)
    time.sleep(0.00001)	# 10uS
    GPIO.output(TRIGGER_PIN, GPIO.LOW)
    pulse_start = time.time()

    while GPIO.input(ECHO_PIN) == GPIO.LOW:
        pulse_start = time.time()

    while GPIO.input(ECHO_PIN) == GPIO.HIGH:
        pulse_end = time.time()

    t = pulse_end - pulse_start

    d = t * v
    d = d/2

    return d*100

def measure_average() :
    d1 = measure()
    time.sleep(0.05)
    d2 = measure()
    time.sleep(0.05)
    d3 = measure()
    distance = (d1 + d2 + d3) / 3

    return distance

try :
    while True:
        distance = measure_average()
        print "Distance : %.1f" (cm)% distance
        time.sleep(1)

except KeyboardInterrupt:
    print "Exception: KeyboardInterrupt"

finally:
    GPIO.cleanup()
<commit_msg>fix typo<commit_after>#!/usr/bin/python
#+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
#|R|a|s|p|b|e|r|r|y|P|i|.|c|o|m|.|t|w|
#+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
# Copyright (c) 2016, raspberrypi.com.tw
# All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# hc_sr04_measure_distance.py
# Measure the distance between HC-SR04 and nearest wall or solid object.
#
# Author : sosorry
# Date   : 09/13/2014

import RPi.GPIO as GPIO
import time

GPIO.setmode(GPIO.BOARD)
TRIGGER_PIN = 16
ECHO_PIN    = 18
GPIO.setup(TRIGGER_PIN,  GPIO.OUT)
GPIO.setup(ECHO_PIN,     GPIO.IN)
GPIO.output(TRIGGER_PIN, GPIO.LOW)
v = 343		# (331 + 0.6*20)

def measure() :
    GPIO.output(TRIGGER_PIN, GPIO.HIGH)
    time.sleep(0.00001)	# 10uS
    GPIO.output(TRIGGER_PIN, GPIO.LOW)
    pulse_start = time.time()

    while GPIO.input(ECHO_PIN) == GPIO.LOW:
        pulse_start = time.time()

    while GPIO.input(ECHO_PIN) == GPIO.HIGH:
        pulse_end = time.time()

    t = pulse_end - pulse_start

    d = t * v
    d = d/2

    return d*100

def measure_average() :
    d1 = measure()
    time.sleep(0.05)
    d2 = measure()
    time.sleep(0.05)
    d3 = measure()
    distance = (d1 + d2 + d3) / 3

    return distance

try :
    while True:
        distance = measure_average()
        print "Distance : %.1f (cm)" % distance
        time.sleep(1)

except KeyboardInterrupt:
    print "Exception: KeyboardInterrupt"

finally:
    GPIO.cleanup()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Written by Lucas Sinclair.
MIT Licensed.
Contact at www.sinclair.bio
"""

# Built-in modules #

# First party modules #

# Third party modules #

################################################################################
# This provides extra information for certain packages #

extra_clues = {
    'apt': "It is likely the `apt` command is missing because we are not "
           "currently running on the Ubuntu or Debian operating system.",
    'gsed': "The gsed package can be installed with `brew install gnu-sed`"
            " on macOS."
}

################################################################################
def check_cmd(cmd_name, exception=True, extra_msg=None):
    """
    Checks if the given executable is found in the path.
    Optionally, raises an exception if the executable `cmd_name` is not found.
    Optionally, print an extra message as a string alongside the exception.
    """
    # Use our own which implementation #
    from plumbing.common import which
    # Try to find it in the $PATH #
    if which(cmd_name, safe=True) is not None: return True
    # Make a detailed message #
    msg = f"The executable '{cmd_name}' is required for this operation." \
          f" Unfortunately it cannot be found anywhere in your $PATH."   \
          f" Either you need to install '{cmd_name}'"                    \
          f" or you need to fix the $PATH environment variable."
    # Add any hints #
    if cmd_name in extra_clues: msg += '\n\n' + extra_clues.get(cmd_name)
    # Add a custom extra message #
    if extra_msg: msg += '\n\n' + extra_msg
    # Raise an exception #
    if exception: raise Exception(msg)
    # Return #
    return False<commit_msg>Move check cmd found message to module base<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Written by Lucas Sinclair.
MIT Licensed.
Contact at www.sinclair.bio
"""

# Built-in modules #

# First party modules #

# Third party modules #

################################################################################
# This provides extra information for certain packages #
extra_clues = {
    'apt': "It is likely the `apt` command is missing because we are not "
           "currently running on the Ubuntu or Debian operating system.",
    'gsed': "The gsed package can be installed with `brew install gnu-sed`"
            " on macOS."
}

# Make a detailed message #
msg_base = "The executable '%s' is required for this operation." \
           " Unfortunately it cannot be found anywhere in your $PATH." \
           " Either you need to install '%s'" \
           " or you need to fix the $PATH environment variable."

################################################################################
def check_cmd(cmd_name, exception=True, extra_msg=None):
    """
    Checks if the given executable is found in the path.
    Optionally, raises an exception if the executable `cmd_name` is not found.
    Optionally, print an extra message as a string alongside the exception.
    """
    # Use our own which implementation #
    from plumbing.common import which
    # Try to find it in the $PATH #
    if which(cmd_name, safe=True) is not None: return True
    # Prepare the message #
    msg = msg_base % (cmd_name, cmd_name)
    # Add any hints #
    if cmd_name in extra_clues: msg += '\n\n' + extra_clues.get(cmd_name)
    # Add a custom extra message #
    if extra_msg: msg += '\n\n' + extra_msg
    # Raise an exception #
    if exception: raise Exception(msg)
    # Return #
    return False<|endoftext|>