<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals
import datetime
from contextlib import contextmanager

from couchdbkit import ResourceNotFound
from ddtrace import tracer

from corehq.form_processor.interfaces.dbaccessors import FormAccessors
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from corehq.form_processor.models import Attachment
from corehq.form_processor.utils import convert_xform_to_json, adjust_datetimes
from corehq.util.soft_assert.api import soft_assert
from couchforms import XMLSyntaxError
from couchforms.exceptions import MissingXMLNSError
from dimagi.utils.couch import release_lock
import six

@contextmanager
def locked_form(xform, interface):
    """Context manager that locks a form and checks/prepares for duplicates

    The lock is acquired on context manager enter and released on exit.

    Historically this locked both the new (possibly duplicate) form ID
    as well as any new ID generated in the process of handling a
    duplicate form ID. The submitted form ID is still locked, but the
    newly generated ID is not since it should be globally unique.
    """
    context = [xform]
    lock = interface.acquire_lock_for_xform(xform.form_id)
    try:
        if interface.is_duplicate(xform.form_id):
            new_form, dup_form = _handle_id_conflict(xform, xform.domain)
            if dup_form:
                assert dup_form.form_id != new_form.form_id, (new_form, dup_form)
                context = [new_form, dup_form]
            else:
                assert new_form is xform, (new_form, xform)
        yield context
    finally:
        release_lock(lock, degrade_gracefully=True)

class FormProcessingResult(object):

    def __init__(self, submitted_form):
        self.submitted_form = submitted_form
        self.interface = FormProcessorInterface(self.submitted_form.domain)

    def get_locked_forms(self):
        """Get a context manager whose context is a list of forms

        The first form in the list is the newly submitted form, and will
        always be present. The second is a form with a duplicate form ID
        (if one exists); it will be omitted if there is no duplicate.
        """
        return locked_form(self.submitted_form, self.interface)

@tracer.wrap(name='submission.process_form_xml')
def process_xform_xml(domain, instance, attachments=None, auth_context=None):
    """
    Create a new xform to ready to be saved to a database in a thread-safe manner

    attachments is a dictionary of the request.FILES that are not the xform;
    key is parameter name, value is
    `django.core.files.uploadedfile.UploadedFile` object.

    :returns: FormProcessingResult containing the new XFormInstance(SQL)
    or raises an exception if anything goes wrong.
    """
    attachments = attachments or {}

    try:
        return _create_new_xform(domain, instance, attachments=attachments, auth_context=auth_context)
    except (MissingXMLNSError, XMLSyntaxError) as e:
        return _get_submission_error(domain, instance, e)

def _create_new_xform(domain, instance_xml, attachments=None, auth_context=None):
    """
    create but do not save an XFormInstance from an xform payload (xml_string)
    optionally set the doc _id to a predefined value (_id)
    return doc _id of the created doc

    `process` is transformation to apply to the form right before saving
    This is to avoid having to save multiple times

    If xml_string is bad xml
      - raise couchforms.XMLSyntaxError
      :param domain:

    :returns: FormProcessingResult or raises an error
    """
    from corehq.form_processor.interfaces.processor import FormProcessorInterface
    interface = FormProcessorInterface(domain)

    assert attachments is not None
    form_data = convert_xform_to_json(instance_xml)
    if not form_data.get('@xmlns'):
        raise MissingXMLNSError("Form is missing a required field: XMLNS")

    adjust_datetimes(form_data)

    xform = interface.new_xform(form_data)
    xform.domain = domain
    xform.auth_context = auth_context

    # Maps all attachments to uniform format and adds form.xml to list before storing
    attachments = [Attachment(name=a[0], raw_content=a[1], content_type=a[1].content_type) for a in attachments.items()]
    attachments.append(Attachment(name='form.xml', raw_content=instance_xml, content_type='text/xml'))
    interface.store_attachments(xform, attachments)

    assert xform.is_normal
    return FormProcessingResult(xform)

def _get_submission_error(domain, instance, error):
    """
    Handle's a hard failure from posting a form to couch.
    :returns: xform error instance with raw xml as attachment
    """
    try:
        message = six.text_type(error)
    except UnicodeDecodeError:
        message = six.text_type(str(error), encoding='utf-8')

    xform = FormProcessorInterface(domain).submission_error_form_instance(instance, message)
    return FormProcessingResult(xform)

def _handle_id_conflict(xform, domain):
    """
    For id conflicts, we check if the files contain exactly the same content,
    If they do, we just log this as a dupe. If they don't, we deprecate the
    previous form and overwrite it with the new form's contents.

    :returns: A two-tuple: `(<new form>, <duplicate form or None>)`
    The new form may have a different `form_id` than `xform.form_id`.
    """

    assert domain
    conflict_id = xform.form_id

    interface = FormProcessorInterface(domain)
    if interface.is_duplicate(conflict_id, domain):
        # It looks like a duplicate/edit in the same domain so pursue that workflow.
        return _handle_duplicate(xform)
    else:
        # the same form was submitted to two domains, or a form was submitted with
        # an ID that belonged to a different doc type. these are likely developers
        # manually testing or broken API users. just resubmit with a generated ID.
        return interface.assign_new_id(xform), None

def _handle_duplicate(new_doc):
    """
    Handle duplicate xforms and xform editing ('deprecation')

    existing doc *must* be validated as an XFormInstance in the right domain
    and *must* include inline attachments

    :returns: A two-tuple: `(<new form>, <duplicate form or None>)`
    The new form may have a different `form_id` than `new_doc.form_id`.
    """
    interface = FormProcessorInterface(new_doc.domain)
    conflict_id = new_doc.form_id
    try:
        existing_doc = FormAccessors(new_doc.domain).get_with_attachments(conflict_id)
    except ResourceNotFound:
        # Original form processing failed but left behind a form doc with no
        # attachments. It's safe to delete this now since we're going to re-process
        # the form anyway.
        from couchforms.models import XFormInstance
        XFormInstance.get_db().delete_doc(conflict_id)
        return new_doc, None

    existing_md5 = existing_doc.xml_md5()
    new_md5 = new_doc.xml_md5()

    if existing_md5 != new_md5:
        _soft_assert = soft_assert(to='{}@{}.com'.format('skelly', 'dimagi'), exponential_backoff=False)
        if new_doc.xmlns != existing_doc.xmlns:
            # if the XMLNS has changed this probably isn't a form edit
            # it could be a UUID clash (yes we've had that before)
            # Assign a new ID to the form and process as normal + notify_admins
            xform = interface.assign_new_id(new_doc)
            _soft_assert(
                False, "Potential UUID clash", {
                    'incoming_form_id': conflict_id,
                    'existing_form_id': existing_doc.form_id,
                    'new_form_id': xform.form_id,
                    'incoming_xmlns': new_doc.xmlns,
                    'existing_xmlns': existing_doc.xmlns,
                    'domain': new_doc.domain,
                }
            )
            return xform, None
        else:
            # if the form contents are not the same:
            #  - "Deprecate" the old form by making a new document with the same contents
            #    but a different ID and a doc_type of XFormDeprecated
            #  - Save the new instance to the previous document to preserve the ID
            existing_doc, new_doc = apply_deprecation(existing_doc, new_doc, interface)
            _soft_assert(
                False, "Form edit", {
                    'form_id': new_doc.form_id,
                    'deprecated_form': existing_doc.form_id,
                    'domain': new_doc.domain,
                }
            )
            return new_doc, existing_doc
    else:
        # follow standard dupe handling, which simply saves a copy of the form
        # but a new doc_id, and a doc_type of XFormDuplicate
        duplicate = interface.deduplicate_xform(new_doc)
        return duplicate, existing_doc

def apply_deprecation(existing_xform, new_xform, interface=None):
    # if the form contents are not the same:
    #  - "Deprecate" the old form by making a new document with the same contents
    #    but a different ID and a doc_type of XFormDeprecated
    #  - Save the new instance to the previous document to preserve the ID

    interface = interface or FormProcessorInterface(existing_xform.domain)
    interface.copy_attachments(existing_xform, new_xform)
    interface.copy_form_operations(existing_xform, new_xform)
    new_xform.form_id = existing_xform.form_id
    existing_xform = interface.assign_new_id(existing_xform)
    existing_xform.orig_id = new_xform.form_id

    # and give the new doc server data of the old one and some metadata
    new_xform.received_on = existing_xform.received_on
    new_xform.deprecated_form_id = existing_xform.form_id
    new_xform.edited_on = datetime.datetime.utcnow()
    existing_xform.edited_on = new_xform.edited_on

    return interface.apply_deprecation(existing_xform, new_xform)
<commit_msg>more details to soft assert<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals
import datetime
from contextlib import contextmanager

from couchdbkit import ResourceNotFound
from ddtrace import tracer

from corehq.apps.cloudcare.const import DEVICE_ID as FORMPLAYER_DEVICE_ID
from corehq.form_processor.interfaces.dbaccessors import FormAccessors
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from corehq.form_processor.models import Attachment
from corehq.form_processor.utils import convert_xform_to_json, adjust_datetimes
from corehq.util.soft_assert.api import soft_assert
from couchforms import XMLSyntaxError
from couchforms.exceptions import MissingXMLNSError
from dimagi.utils.couch import release_lock
import six

@contextmanager
def locked_form(xform, interface):
    """Context manager that locks a form and checks/prepares for duplicates

    The lock is acquired on context manager enter and released on exit.

    Historically this locked both the new (possibly duplicate) form ID
    as well as any new ID generated in the process of handling a
    duplicate form ID. The submitted form ID is still locked, but the
    newly generated ID is not since it should be globally unique.
    """
    context = [xform]
    lock = interface.acquire_lock_for_xform(xform.form_id)
    try:
        if interface.is_duplicate(xform.form_id):
            new_form, dup_form = _handle_id_conflict(xform, xform.domain)
            if dup_form:
                assert dup_form.form_id != new_form.form_id, (new_form, dup_form)
                context = [new_form, dup_form]
            else:
                assert new_form is xform, (new_form, xform)
        yield context
    finally:
        release_lock(lock, degrade_gracefully=True)

class FormProcessingResult(object):

    def __init__(self, submitted_form):
        self.submitted_form = submitted_form
        self.interface = FormProcessorInterface(self.submitted_form.domain)

    def get_locked_forms(self):
        """Get a context manager whose context is a list of forms

        The first form in the list is the newly submitted form, and will
        always be present. The second is a form with a duplicate form ID
        (if one exists); it will be omitted if there is no duplicate.
        """
        return locked_form(self.submitted_form, self.interface)

@tracer.wrap(name='submission.process_form_xml')
def process_xform_xml(domain, instance, attachments=None, auth_context=None):
    """
    Create a new xform to ready to be saved to a database in a thread-safe manner

    attachments is a dictionary of the request.FILES that are not the xform;
    key is parameter name, value is
    `django.core.files.uploadedfile.UploadedFile` object.

    :returns: FormProcessingResult containing the new XFormInstance(SQL)
    or raises an exception if anything goes wrong.
    """
    attachments = attachments or {}

    try:
        return _create_new_xform(domain, instance, attachments=attachments, auth_context=auth_context)
    except (MissingXMLNSError, XMLSyntaxError) as e:
        return _get_submission_error(domain, instance, e)

def _create_new_xform(domain, instance_xml, attachments=None, auth_context=None):
    """
    create but do not save an XFormInstance from an xform payload (xml_string)
    optionally set the doc _id to a predefined value (_id)
    return doc _id of the created doc

    `process` is transformation to apply to the form right before saving
    This is to avoid having to save multiple times

    If xml_string is bad xml
      - raise couchforms.XMLSyntaxError
      :param domain:

    :returns: FormProcessingResult or raises an error
    """
    from corehq.form_processor.interfaces.processor import FormProcessorInterface
    interface = FormProcessorInterface(domain)

    assert attachments is not None
    form_data = convert_xform_to_json(instance_xml)
    if not form_data.get('@xmlns'):
        raise MissingXMLNSError("Form is missing a required field: XMLNS")

    adjust_datetimes(form_data)

    xform = interface.new_xform(form_data)
    xform.domain = domain
    xform.auth_context = auth_context

    # Maps all attachments to uniform format and adds form.xml to list before storing
    attachments = [Attachment(name=a[0], raw_content=a[1], content_type=a[1].content_type) for a in attachments.items()]
    attachments.append(Attachment(name='form.xml', raw_content=instance_xml, content_type='text/xml'))
    interface.store_attachments(xform, attachments)

    assert xform.is_normal
    return FormProcessingResult(xform)

def _get_submission_error(domain, instance, error):
    """
    Handle's a hard failure from posting a form to couch.
    :returns: xform error instance with raw xml as attachment
    """
    try:
        message = six.text_type(error)
    except UnicodeDecodeError:
        message = six.text_type(str(error), encoding='utf-8')

    xform = FormProcessorInterface(domain).submission_error_form_instance(instance, message)
    return FormProcessingResult(xform)

def _handle_id_conflict(xform, domain):
    """
    For id conflicts, we check if the files contain exactly the same content,
    If they do, we just log this as a dupe. If they don't, we deprecate the
    previous form and overwrite it with the new form's contents.

    :returns: A two-tuple: `(<new form>, <duplicate form or None>)`
    The new form may have a different `form_id` than `xform.form_id`.
    """

    assert domain
    conflict_id = xform.form_id

    interface = FormProcessorInterface(domain)
    if interface.is_duplicate(conflict_id, domain):
        # It looks like a duplicate/edit in the same domain so pursue that workflow.
        return _handle_duplicate(xform)
    else:
        # the same form was submitted to two domains, or a form was submitted with
        # an ID that belonged to a different doc type. these are likely developers
        # manually testing or broken API users. just resubmit with a generated ID.
        return interface.assign_new_id(xform), None

def _handle_duplicate(new_doc):
    """
    Handle duplicate xforms and xform editing ('deprecation')

    existing doc *must* be validated as an XFormInstance in the right domain
    and *must* include inline attachments

    :returns: A two-tuple: `(<new form>, <duplicate form or None>)`
    The new form may have a different `form_id` than `new_doc.form_id`.
    """
    interface = FormProcessorInterface(new_doc.domain)
    conflict_id = new_doc.form_id
    try:
        existing_doc = FormAccessors(new_doc.domain).get_with_attachments(conflict_id)
    except ResourceNotFound:
        # Original form processing failed but left behind a form doc with no
        # attachments. It's safe to delete this now since we're going to re-process
        # the form anyway.
        from couchforms.models import XFormInstance
        XFormInstance.get_db().delete_doc(conflict_id)
        return new_doc, None

    existing_md5 = existing_doc.xml_md5()
    new_md5 = new_doc.xml_md5()

    if existing_md5 != new_md5:
        _soft_assert = soft_assert(to='{}@{}.com'.format('skelly', 'dimagi'), exponential_backoff=False)
        if new_doc.xmlns != existing_doc.xmlns:
            # if the XMLNS has changed this probably isn't a form edit
            # it could be a UUID clash (yes we've had that before)
            # Assign a new ID to the form and process as normal + notify_admins
            xform = interface.assign_new_id(new_doc)
            _soft_assert(
                False, "Potential UUID clash", {
                    'incoming_form_id': conflict_id,
                    'existing_form_id': existing_doc.form_id,
                    'new_form_id': xform.form_id,
                    'incoming_xmlns': new_doc.xmlns,
                    'existing_xmlns': existing_doc.xmlns,
                    'domain': new_doc.domain,
                }
            )
            return xform, None
        else:
            # if the form contents are not the same:
            #  - "Deprecate" the old form by making a new document with the same contents
            #    but a different ID and a doc_type of XFormDeprecated
            #  - Save the new instance to the previous document to preserve the ID
            existing_doc, new_doc = apply_deprecation(existing_doc, new_doc, interface)
            device_id = new_doc.metadata.deviceID
            # we expect edits from formplayer so exclude those
            _soft_assert(
                device_id == FORMPLAYER_DEVICE_ID, "Form edit", {
                    'form_id': new_doc.form_id,
                    'deprecated_form': existing_doc.form_id,
                    'domain': new_doc.domain,
                    'device_id': device_id,
                    'cc_version': new_doc.metadata.appVersion,
                }
            )
            return new_doc, existing_doc
    else:
        # follow standard dupe handling, which simply saves a copy of the form
        # but a new doc_id, and a doc_type of XFormDuplicate
        duplicate = interface.deduplicate_xform(new_doc)
        return duplicate, existing_doc

def apply_deprecation(existing_xform, new_xform, interface=None):
    # if the form contents are not the same:
    #  - "Deprecate" the old form by making a new document with the same contents
    #    but a different ID and a doc_type of XFormDeprecated
    #  - Save the new instance to the previous document to preserve the ID

    interface = interface or FormProcessorInterface(existing_xform.domain)
    interface.copy_attachments(existing_xform, new_xform)
    interface.copy_form_operations(existing_xform, new_xform)
    new_xform.form_id = existing_xform.form_id
    existing_xform = interface.assign_new_id(existing_xform)
    existing_xform.orig_id = new_xform.form_id

    # and give the new doc server data of the old one and some metadata
    new_xform.received_on = existing_xform.received_on
    new_xform.deprecated_form_id = existing_xform.form_id
    new_xform.edited_on = datetime.datetime.utcnow()
    existing_xform.edited_on = new_xform.edited_on

    return interface.apply_deprecation(existing_xform, new_xform)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Copyright 2015 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import argparse
import logging
import os
import sys
import time
from xml.sax.saxutils import escape
from xml.sax.saxutils import quoteattr

from pydocstyle import check
from pydocstyle import ConfigurationParser
from pydocstyle import Error
from pydocstyle import log
log.setLevel(logging.INFO)

def main(argv=sys.argv[1:]):
    parser = argparse.ArgumentParser(
        description='Check docstrings against the style conventions in PEP 257.',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        '--ignore',
        nargs='*',
        default=['D100', 'D101', 'D102', 'D103', 'D104', 'D105', 'D203'],
        help='The pep257 categories to ignore')
    parser.add_argument(
        'paths',
        nargs='*',
        default=[os.curdir],
        help="The files or directories to check. For directories files ending "
             "in '.py' will be considered.")
    parser.add_argument(
        '--exclude',
        metavar='filename',
        nargs='*',
        default=[],
        dest='excludes',
        help='The filenames to exclude.')
    # not using a file handle directly
    # in order to prevent leaving an empty file when something fails early
    parser.add_argument(
        '--xunit-file',
        help='Generate a xunit compliant XML file')
    args = parser.parse_args(argv)

    if args.xunit_file:
        start_time = time.time()

    excludes = [os.path.abspath(e) for e in args.excludes]
    report = generate_pep257_report(args.paths, excludes, args.ignore)
    error_count = sum([len(r[1]) for r in report])

    # print summary
    if not error_count:
        print('No errors')
        rc = 0
    else:
        print('%d errors' % error_count)
        rc = 1

    # generate xunit file
    if args.xunit_file:
        folder_name = os.path.basename(os.path.dirname(args.xunit_file))
        file_name = os.path.basename(args.xunit_file)
        suffix = '.xml'
        if file_name.endswith(suffix):
            file_name = file_name[0:-len(suffix)]
            suffix = '.xunit'
            if file_name.endswith(suffix):
                file_name = file_name[0:-len(suffix)]
        testname = '%s.%s' % (folder_name, file_name)

        xml = get_xunit_content(report, testname, time.time() - start_time)
        path = os.path.dirname(os.path.abspath(args.xunit_file))
        if not os.path.exists(path):
            os.makedirs(path)
        with open(args.xunit_file, 'w') as f:
            f.write(xml)

    return rc

def generate_pep257_report(paths, excludes, ignore):
    conf = ConfigurationParser()
    sys_argv = sys.argv
    sys.argv = [
        'main',
        '--ignore=' + ','.join(ignore),
        '--match', '.*\.py',
        '--match-dir', '[^\._].*',
    ]
    sys.argv += paths
    conf.parse()
    sys.argv = sys_argv
    files_to_check = conf.get_files_to_check()

    report = []

    files_dict = {}
    for filename, select in files_to_check:
        if os.path.abspath(filename) in excludes:
            continue
        files_dict[filename] = select

    for filename in sorted(files_dict.keys()):
        print('checking', filename)
        errors = []
        pep257_errors = check([filename], select=files_dict[filename])
        for pep257_error in pep257_errors:
            if isinstance(pep257_error, Error):
                errors.append({
                    'category': pep257_error.code,
                    'linenumber': pep257_error.line,
                    'message': pep257_error.message,
                })
                print(
                    '%s:%d %s: %s' %
                    (pep257_error.filename, pep257_error.line, pep257_error.definition,
                     pep257_error.message))
            elif isinstance(pep257_error, SyntaxError):
                errors.append({
                    'category': str(type(pep257_error)),
                    'linenumber': '-',
                    'message': 'invalid syntax in file',
                })
                print('%s: invalid syntax' % filename)
            else:
                errors.append({
                    'category': 'unknown',
                    'linenumber': '-',
                    'message': str(pep257_error),
                })
                print('%s: %s' % (filename, pep257_error))
        report.append((filename, errors))
    return report

def get_xunit_content(report, testname, elapsed):
    test_count = sum([max(len(r[1]), 1) for r in report])
    error_count = sum([len(r[1]) for r in report])
    data = {
        'testname': testname,
        'test_count': test_count,
        'error_count': error_count,
        'time': '%.3f' % round(elapsed, 3),
    }
    xml = '''<?xml version="1.0" encoding="UTF-8"?>
<testsuite
  name="%(testname)s"
  tests="%(test_count)d"
  failures="%(error_count)d"
  time="%(time)s"
>
''' % data

    for (filename, errors) in report:

        if errors:
            # report each error as a failing testcase
            for error in errors:
                data = {
                    'quoted_location': quoteattr(
                        '%s (%s:%d)' % (
                            error['category'], filename, error['linenumber'])),
                    'testname': testname,
                    'quoted_message': quoteattr(error['message']),
                }
                xml += '''  <testcase
    name=%(quoted_location)s
    classname="%(testname)s"
  >
      <failure message=%(quoted_message)s/>
  </testcase>
''' % data

        else:
            # if there are no lint_cmake errors report a single successful test
            data = {
                'quoted_location': quoteattr(filename),
                'testname': testname,
            }
            xml += '''  <testcase
    name=%(quoted_location)s
    classname="%(testname)s"
    status="No errors"/>
''' % data

    # output list of checked files
    data = {
        'escaped_files': escape(''.join(['\n* %s' % r[0] for r in report])),
    }
    xml += '''  <system-out>Checked files:%(escaped_files)s</system-out>
''' % data

    xml += '</testsuite>\n'
    return xml

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>(fix) correct pydocstyle import for version 1.1.0<commit_after>#!/usr/bin/env python3

# Copyright 2015 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import argparse
import logging
import os
import sys
import time
from xml.sax.saxutils import escape
from xml.sax.saxutils import quoteattr

from pydocstyle import check

try:  # as of version 1.1.0
    from pydocstyle.config import ConfigurationParser
    from pydocstyle.violations import Error
    from pydocstyle.utils import log
except ImportError:  # try version 1.0.0
    from pydocstyle import ConfigurationParser
    from pydocstyle import Error
    from pydocstyle import log

log.setLevel(logging.INFO)

def main(argv=sys.argv[1:]):
    parser = argparse.ArgumentParser(
        description='Check docstrings against the style conventions in PEP 257.',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        '--ignore',
        nargs='*',
        default=['D100', 'D101', 'D102', 'D103', 'D104', 'D105', 'D203'],
        help='The pep257 categories to ignore')
    parser.add_argument(
        'paths',
        nargs='*',
        default=[os.curdir],
        help="The files or directories to check. For directories files ending "
             "in '.py' will be considered.")
    parser.add_argument(
        '--exclude',
        metavar='filename',
        nargs='*',
        default=[],
        dest='excludes',
        help='The filenames to exclude.')
    # not using a file handle directly
    # in order to prevent leaving an empty file when something fails early
    parser.add_argument(
        '--xunit-file',
        help='Generate a xunit compliant XML file')
    args = parser.parse_args(argv)

    if args.xunit_file:
        start_time = time.time()

    excludes = [os.path.abspath(e) for e in args.excludes]
    report = generate_pep257_report(args.paths, excludes, args.ignore)
    error_count = sum([len(r[1]) for r in report])

    # print summary
    if not error_count:
        print('No errors')
        rc = 0
    else:
        print('%d errors' % error_count)
        rc = 1

    # generate xunit file
    if args.xunit_file:
        folder_name = os.path.basename(os.path.dirname(args.xunit_file))
        file_name = os.path.basename(args.xunit_file)
        suffix = '.xml'
        if file_name.endswith(suffix):
            file_name = file_name[0:-len(suffix)]
            suffix = '.xunit'
            if file_name.endswith(suffix):
                file_name = file_name[0:-len(suffix)]
        testname = '%s.%s' % (folder_name, file_name)

        xml = get_xunit_content(report, testname, time.time() - start_time)
        path = os.path.dirname(os.path.abspath(args.xunit_file))
        if not os.path.exists(path):
            os.makedirs(path)
        with open(args.xunit_file, 'w') as f:
            f.write(xml)

    return rc

def generate_pep257_report(paths, excludes, ignore):
    conf = ConfigurationParser()
    sys_argv = sys.argv
    sys.argv = [
        'main',
        '--ignore=' + ','.join(ignore),
        '--match', '.*\.py',
        '--match-dir', '[^\._].*',
    ]
    sys.argv += paths
    conf.parse()
    sys.argv = sys_argv
    files_to_check = conf.get_files_to_check()

    report = []

    files_dict = {}
    for filename, select in files_to_check:
        if os.path.abspath(filename) in excludes:
            continue
        files_dict[filename] = select

    for filename in sorted(files_dict.keys()):
        print('checking', filename)
        errors = []
        pep257_errors = check([filename], select=files_dict[filename])
        for pep257_error in pep257_errors:
            if isinstance(pep257_error, Error):
                errors.append({
                    'category': pep257_error.code,
                    'linenumber': pep257_error.line,
                    'message': pep257_error.message,
                })
                print(
                    '%s:%d %s: %s' %
                    (pep257_error.filename, pep257_error.line, pep257_error.definition,
                     pep257_error.message))
            elif isinstance(pep257_error, SyntaxError):
                errors.append({
                    'category': str(type(pep257_error)),
                    'linenumber': '-',
                    'message': 'invalid syntax in file',
                })
                print('%s: invalid syntax' % filename)
            else:
                errors.append({
                    'category': 'unknown',
                    'linenumber': '-',
                    'message': str(pep257_error),
                })
                print('%s: %s' % (filename, pep257_error))
        report.append((filename, errors))
    return report

def get_xunit_content(report, testname, elapsed):
    test_count = sum([max(len(r[1]), 1) for r in report])
    error_count = sum([len(r[1]) for r in report])
    data = {
        'testname': testname,
        'test_count': test_count,
        'error_count': error_count,
        'time': '%.3f' % round(elapsed, 3),
    }
    xml = '''<?xml version="1.0" encoding="UTF-8"?>
<testsuite
  name="%(testname)s"
  tests="%(test_count)d"
  failures="%(error_count)d"
  time="%(time)s"
>
''' % data

    for (filename, errors) in report:

        if errors:
            # report each error as a failing testcase
            for error in errors:
                data = {
                    'quoted_location': quoteattr(
                        '%s (%s:%d)' % (
                            error['category'], filename, error['linenumber'])),
                    'testname': testname,
                    'quoted_message': quoteattr(error['message']),
                }
                xml += '''  <testcase
    name=%(quoted_location)s
    classname="%(testname)s"
  >
      <failure message=%(quoted_message)s/>
  </testcase>
''' % data

        else:
            # if there are no lint_cmake errors report a single successful test
            data = {
                'quoted_location': quoteattr(filename),
                'testname': testname,
            }
            xml += '''  <testcase
    name=%(quoted_location)s
    classname="%(testname)s"
    status="No errors"/>
''' % data

    # output list of checked files
    data = {
        'escaped_files': escape(''.join(['\n* %s' % r[0] for r in report])),
    }
    xml += '''  <system-out>Checked files:%(escaped_files)s</system-out>
''' % data

    xml += '</testsuite>\n'
    return xml

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>""" Pull replay from rocketleaguereplays and post turbo-carnival """
import StringIO
import time
import requests
import boto3

# To setup your keys
# aws configure --profile turbo-carnival
SESSION = boto3.Session(profile_name='turbo-carnival')
DYNAMO = SESSION.resource("dynamodb")
TABLE = DYNAMO.Table('turbo-carnival')

def getReplay(replay_hash):
    """ Grabs an individual replay and posts to turbo-carnival"""
    url = "https://media.rocketleaguereplays.com/uploads/replay_files/" + replay_hash + ".replay"
    req = requests.get(url)
    if str(req.status_code) == '200':
        str_file = StringIO.StringIO(req.content)
        files = {"file": (replay_hash + ".replay", str_file)}
        post = requests.post("http://rocketleague.carlosrendon.me/upload", files=files)
        if str(post.status_code) != '200':
            print("upload failed: " + str(post.status_code))
    else:
        print("download failed: " + str(req.status_code))

def listRLReplays():
    """ Grabs a list of replay hashs from the RL replays API. TODO handle paging """
    req = requests.get("http://www.rocketleaguereplays.com/api/replays")
    if str(req.status_code) != '200':
        print("Error querying api: " + str(req.status_code))
        print(req.content)
        return []
    resp = req.json()

    ids = []
    for result in resp['results']:
        if result['replay_id']:
            ids.append(result['replay_id'])
    return ids

REPLAYS = listRLReplays()
for replay in REPLAYS:
    ret = TABLE.get_item(Key={"replay_key": replay})
    if not ret.has_key('Item'):
        print("Grabbing " + replay)
        getReplay(replay)
        time.sleep(10)
    else:
        print("Skipping " + replay)

<commit_msg>Handle paging<commit_after>""" Pull replay from rocketleaguereplays and post turbo-carnival """
import StringIO
import time
import requests
import boto3

# To setup your keys
# aws configure --profile turbo-carnival
SESSION = boto3.Session(profile_name='turbo-carnival')
DYNAMO = SESSION.resource("dynamodb")
TABLE = DYNAMO.Table('turbo-carnival')

def getReplay(replay_hash):
    """ Grabs an individual replay and posts to turbo-carnival"""
    url = "https://media.rocketleaguereplays.com/uploads/replay_files/" + replay_hash + ".replay"
    req = requests.get(url)
    if str(req.status_code) == '200':
        str_file = StringIO.StringIO(req.content)
        files = {"file": (replay_hash + ".replay", str_file)}
        post = requests.post("http://rocketleague.carlosrendon.me/upload", files=files)
        if str(post.status_code) != '200':
            print("upload failed: " + str(post.status_code))
    else:
        print("download failed: " + str(req.status_code))

def listRLReplays(page):
    """ Grabs a list of replay hashs from the RL replays API."""
    if page == 1:
        req = requests.get("http://www.rocketleaguereplays.com/api/replays")
    else:
        req = requests.get("http://www.rocketleaguereplays.com/api/replays?page=" + str(page))
    if str(req.status_code) != '200':
        print("Error querying api: " + str(req.status_code))
        print(req.content)
        return []
    resp = req.json()

    ids = []
    for result in resp['results']:
        if result['replay_id']:
            ids.append(result['replay_id'])
    return ids

def getAllReplays():
    """ Get's all of the replays. Handle's API paging."""
    page = 1
    while True:
        replays = listRLReplays(page)
        for replay in replays:
            try:
                ret = TABLE.get_item(Key={"replay_key": replay})
            except:
                print("Exception")
                exit(1)
            if not ret.has_key('Item'):
                print("Grabbing " + replay)
                getReplay(replay)
                time.sleep(10)
            else:
                print("Skipping " + replay)
        page = page + 1

getAllReplays()
<|endoftext|>


<commit_before>import subprocess
from bcbio.distributed.transaction import tx_tmpdir, file_transaction
from bcbio.utils import chdir, which, safe_makedir
from bcbio import bam
from bcbio.log import logger
from bcbio.provenance import do

def prepare_rsem_reference(gtf, multifasta, build):
    """
    gtf: path to GTF file (must have gene_id and transcript_id)
    multifasta: path to multifasta file
    build: name of organism build (e.g. hg19)
    """
    if not which("rsem-prepare-reference"):
        logger.info("Skipping prepping RSEM reference because rsem-prepare-reference could "
                    "not be found.")
        return None

    cmd = "rsem-prepare-reference --gtf {gtf} {multifasta} {build}"
    with tx_tmpdir(remove=False) as rsem_genome_dir:
        with chdir(rsem_genome_dir):
            message = "Preparing rsem reference from %s" % gtf
            do.run(cmd.format(**locals()), message)
    return rsem_genome_dir

def rsem_calculate_expression(bam_file, rsem_genome_dir, samplename, build,
                              out_dir, cores=1):
    """
    works only in unstranded mode for now (--forward-prob 0.5)
    """
    if not which("rsem-calculate-expression"):
        logger.info("Skipping RSEM because rsem-calculate-expression could "
                    "not be found.")
        return None

    sentinel_file = os.path.join(out_dir, samplename + "Test.genes.results")
    if file_exists(sentinel_file):
        return out_dir

    paired_flag = "--paired" if bam.is_paired(bam_file) else ""
    core_flag = "-p {cores}".format(cores=cores)
    cmd = ("rsem-calculate-expression --bam {core_flag} {paired_flag} --no-bam-output "
           "--forward-prob 0.5 --estimate-rspd {bam_file} {rsem_genome_dir}/{build} "
           "{samplename}")
    message = "Calculating transcript expression of {bam_file} using RSEM."
    with file_transaction(out_dir) as tx_out_dir:
        safe_makedir(tx_out_dir)
        with chdir(tx_out_dir):
            do.run(cmd.format(**locals()), message.format(**locals()))
    return out_dir
<commit_msg>Fix issue #888<commit_after>import os
import subprocess
from bcbio.distributed.transaction import tx_tmpdir, file_transaction
from bcbio.utils import chdir, which, safe_makedir, file_exists
from bcbio import bam
from bcbio.log import logger
from bcbio.provenance import do

def prepare_rsem_reference(gtf, multifasta, build):
    """
    gtf: path to GTF file (must have gene_id and transcript_id)
    multifasta: path to multifasta file
    build: name of organism build (e.g. hg19)
    """
    if not which("rsem-prepare-reference"):
        logger.info("Skipping prepping RSEM reference because rsem-prepare-reference could "
                    "not be found.")
        return None

    cmd = "rsem-prepare-reference --gtf {gtf} {multifasta} {build}"
    with tx_tmpdir(remove=False) as rsem_genome_dir:
        with chdir(rsem_genome_dir):
            message = "Preparing rsem reference from %s" % gtf
            do.run(cmd.format(**locals()), message)
    return rsem_genome_dir

def rsem_calculate_expression(bam_file, rsem_genome_dir, samplename, build,
                              out_dir, cores=1):
    """
    works only in unstranded mode for now (--forward-prob 0.5)
    """
    if not which("rsem-calculate-expression"):
        logger.info("Skipping RSEM because rsem-calculate-expression could "
                    "not be found.")
        return None

    sentinel_file = os.path.join(out_dir, samplename + "Test.genes.results")
    if file_exists(sentinel_file):
        return out_dir

    paired_flag = "--paired" if bam.is_paired(bam_file) else ""
    core_flag = "-p {cores}".format(cores=cores)
    cmd = ("rsem-calculate-expression --bam {core_flag} {paired_flag} --no-bam-output "
           "--forward-prob 0.5 --estimate-rspd {bam_file} {rsem_genome_dir}/{build} "
           "{samplename}")
    message = "Calculating transcript expression of {bam_file} using RSEM."
    with file_transaction(out_dir) as tx_out_dir:
        safe_makedir(tx_out_dir)
        with chdir(tx_out_dir):
            do.run(cmd.format(**locals()), message.format(**locals()))
    return out_dir
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Copyright (C) 2013-2017 Danilo Bargen

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""
from __future__ import print_function, division, absolute_import, unicode_literals

import sys

from RPLCD import i2c, gpio

try:
    range = xrange
except NameError:  # Python 3
    pass

try:
    safe_input = raw_input
except NameError:  # Python 3
    safe_input = input

def print_usage():
    print('Usage: %s i2c <expander> <addr> <rows> <cols>' % sys.argv[0])
    print('       %s gpio <rows> <cols>' % sys.argv[0])
    print('')
    print('<expander>  Supported I²C port expanders are PCF8574 and MCP23008')
    print('<addr>      The I²C address (in hex format) can be found with')
    print('            `i2cdetect 1` from the i2c-tools package.')
    print('<rows>      The number of rows on your LCD, e.g. 2')
    print('<cols>      The number of columns on your LCD, e.g. 16')
    print('')
    sys.exit(1)

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print_usage()
    if sys.argv[1] == 'i2c':
        if len(sys.argv) != 6:
            print_usage()
        rows, cols = int(sys.argv[4]), int(sys.argv[5])
        lcd = i2c.CharLCD(int(sys.argv[3], 16), cols=cols, rows=rows, i2c_expander=sys.argv[2])
    elif sys.argv[1] == 'gpio':
        if len(sys.argv) != 4:
            print_usage()
        rows, cols = int(sys.argv[2]), int(sys.argv[3])
        lcd = gpio.CharLCD(cols=cols, rows=rows)
    else:
        print_usage()

    print('This tool shows the character map of your LCD on the display.')
    print('Press ctrl+c at any time to abort.\n')

    page = 0
    chars = rows * cols
    text_tpl = 'Displaying page %d (characters %d-%d). Press <ENTER> to continue.'

    try:
        while True:
            offset = page * chars
            start = offset
            end = offset + chars
            lcd.clear()
            for i in range(offset, offset + chars):
                if i > 255:
                    if i > start:
                        safe_input(text_tpl % (page + 1, start, i - 1))
                    else:
                        pass
                    sys.exit(0)
                lcd.write(i)
            safe_input(text_tpl % (page + 1, start, end - 1))
            page += 1
    except KeyboardInterrupt:
        print('Aborting.')
    finally:
        lcd.clear()
        try:
            lcd.backlight_enabled = False
        except ValueError:
            pass
        lcd.close()
        print('Test done. If you have a backlight, it should now be off.')
<commit_msg>show_charmap: Fix error caused by removing default value for i2c_expander<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Copyright (C) 2013-2017 Danilo Bargen

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""
from __future__ import print_function, division, absolute_import, unicode_literals

import sys

from RPLCD import i2c, gpio

try:
    range = xrange
except NameError:  # Python 3
    pass

try:
    safe_input = raw_input
except NameError:  # Python 3
    safe_input = input

def print_usage():
    print('Usage: %s i2c <expander> <addr> <rows> <cols>' % sys.argv[0])
    print('       %s gpio <rows> <cols>' % sys.argv[0])
    print('')
    print('<expander>  Supported I²C port expanders are PCF8574 and MCP23008')
    print('<addr>      The I²C address (in hex format) can be found with')
    print('            `i2cdetect 1` from the i2c-tools package.')
    print('<rows>      The number of rows on your LCD, e.g. 2')
    print('<cols>      The number of columns on your LCD, e.g. 16')
    print('')
    sys.exit(1)

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print_usage()
    if sys.argv[1] == 'i2c':
        if len(sys.argv) != 6:
            print_usage()
        rows, cols = int(sys.argv[4]), int(sys.argv[5])
        lcd = i2c.CharLCD(sys.argv[2], int(sys.argv[3], 16), cols=cols, rows=rows)
    elif sys.argv[1] == 'gpio':
        if len(sys.argv) != 4:
            print_usage()
        rows, cols = int(sys.argv[2]), int(sys.argv[3])
        lcd = gpio.CharLCD(cols=cols, rows=rows)
    else:
        print_usage()

    print('This tool shows the character map of your LCD on the display.')
    print('Press ctrl+c at any time to abort.\n')

    page = 0
    chars = rows * cols
    text_tpl = 'Displaying page %d (characters %d-%d). Press <ENTER> to continue.'

    try:
        while True:
            offset = page * chars
            start = offset
            end = offset + chars
            lcd.clear()
            for i in range(offset, offset + chars):
                if i > 255:
                    if i > start:
                        safe_input(text_tpl % (page + 1, start, i - 1))
                    else:
                        pass
                    sys.exit(0)
                lcd.write(i)
            safe_input(text_tpl % (page + 1, start, end - 1))
            page += 1
    except KeyboardInterrupt:
        print('Aborting.')
    finally:
        lcd.clear()
        try:
            lcd.backlight_enabled = False
        except ValueError:
            pass
        lcd.close()
        print('Test done. If you have a backlight, it should now be off.')
<|endoftext|>


<commit_before>from PIL import Image
import numpy
import os
import glob

def import_tiff(tiff_filename):
    """
    Import a TIFF file into a numpy array.

    Arguments:
        tiff_filename:  A string filename of a TIFF datafile

    Returns:
        A numpy array with data from the TIFF file
    """

    # Expand filename to be absolute
    tiff_filename = os.path.expanduser(tiff_filename)

    try:
        img = Image.open(tiff_filename)
    except Exception as e:
        raise ValueError("Could not load file {0} for conversion.".format(tiff_filename))
        raise

    return numpy.array(img)

def export_tiff(tiff_filename, numpy_data):
    """
    Export a numpy array to a TIFF file.

    Arguments:
        tiff_filename:  A filename to which to save the TIFF data
        numpy_data:     The numpy array to save to TIFF

    Returns:
        String. The expanded filename that now holds the TIFF data
    """

    # Expand filename to be absolute
    tiff_filename = os.path.expanduser(tiff_filename)

    if type(numpy_data) is str:
        fp = open(png_filename, "wb")
        fp.write(numpy_data)
        fp.close()
        return png_filename

    if numpy_data.dtype.name is not 'uint8':
        raise ValueError("Datatype is not uint8, you may experience a known PIL bug.")

    try:
        img = Image.fromarray(numpy_data)
        img.save(tiff_filename)
    except Exception as e:
        raise ValueError("Could not save TIFF file {0}.".format(tiff_filename))

    return tiff_filename

def export_tiff_collection(tiff_filename_base, numpy_data, start_layers_at=1):
    """
    Export a numpy array to a set of TIFF files, with each Z-index 2D
    array as its own 2D file.

    Arguments:
        tiff_filename_base:     A filename template, such as "my-image-*.tiff"
                                which will lead to a collection of files named
                                "my-image-0.tiff", "my-image-1.tiff", etc.
        numpy_data:             The numpy array data to save to TIFF.

    Returns:
        Array. A list of expanded filenames that hold TIFF data.
    """

    file_ext = tiff_filename_base.split('.')[-1]
    if file_ext in ['tif', 'tiff']:
        # Filename is "name*.tif[f]", set file_base to "name*".
        file_base = '.'.join(tiff_filename_base.split('.')[:-1])
    else:
        # Filename is "name*", set file_base to "name*".
        # That is, extension wasn't included.
        file_base = tiff_filename_base
        file_ext = ".tiff"

    file_base_array = file_base.split('*')

    # The array of filenames to return
    output_files = []

    # Filename 0-padding
    i = start_layers_at
    for layer in numpy_data:
        layer_filename = (str(i).zfill(6)).join(file_base_array) + file_ext
        output_files.append(export_tiff(layer_filename, layer))
        i += 1

    return output_files

def import_tiff_collection(tiff_filename_base):
    """
    Import all files matching the filename base given via `tiff_filename_base`.
    Images are ordered by alphabetical order, which means that you *MUST* 0-pad
    your numbers if they span a power of ten (e.g. 0999-1000 or 09-10). This is
    handled automatically by the complement function, `export_tiff_collection`.
    Also, look at how nicely these documentation lines are all the same length!

    Arguments:
        tiff_filename_base:     An asterisk-wildcard string that should refer
                                to all TIFFs in the stack. All * are replaced
                                according to command-line expansion rules.
    Returns:
        A numpy array holding a 3D dataset
    """

    # We expect images to be indexed by their alphabetical order.
    files = glob.glob(tiff_filename_base)
    files.sort()

    numpy_data = []
    for f in files:
        numpy_data.append(import_tiff(f))

    return numpy_data
<commit_msg>get rid of bug warning<commit_after>from PIL import Image
import numpy
import os
import glob

def import_tiff(tiff_filename):
    """
    Import a TIFF file into a numpy array.

    Arguments:
        tiff_filename:  A string filename of a TIFF datafile

    Returns:
        A numpy array with data from the TIFF file
    """

    # Expand filename to be absolute
    tiff_filename = os.path.expanduser(tiff_filename)

    try:
        img = Image.open(tiff_filename)
    except Exception as e:
        raise ValueError("Could not load file {0} for conversion.".format(tiff_filename))
        raise

    return numpy.array(img)

def export_tiff(tiff_filename, numpy_data):
    """
    Export a numpy array to a TIFF file.

    Arguments:
        tiff_filename:  A filename to which to save the TIFF data
        numpy_data:     The numpy array to save to TIFF

    Returns:
        String. The expanded filename that now holds the TIFF data
    """

    # Expand filename to be absolute
    tiff_filename = os.path.expanduser(tiff_filename)

    if type(numpy_data) is str:
        fp = open(png_filename, "wb")
        fp.write(numpy_data)
        fp.close()
        return png_filename

    # if numpy_data.dtype.name is not 'uint8':
    #     raise ValueError("Datatype is not uint8, you may experience a known PIL bug.")

    try:
        img = Image.fromarray(numpy_data)
        img.save(tiff_filename)
    except Exception as e:
        raise ValueError("Could not save TIFF file {0}.".format(tiff_filename))

    return tiff_filename

def export_tiff_collection(tiff_filename_base, numpy_data, start_layers_at=1):
    """
    Export a numpy array to a set of TIFF files, with each Z-index 2D
    array as its own 2D file.

    Arguments:
        tiff_filename_base:     A filename template, such as "my-image-*.tiff"
                                which will lead to a collection of files named
                                "my-image-0.tiff", "my-image-1.tiff", etc.
        numpy_data:             The numpy array data to save to TIFF.

    Returns:
        Array. A list of expanded filenames that hold TIFF data.
    """

    file_ext = tiff_filename_base.split('.')[-1]
    if file_ext in ['tif', 'tiff']:
        # Filename is "name*.tif[f]", set file_base to "name*".
        file_base = '.'.join(tiff_filename_base.split('.')[:-1])
    else:
        # Filename is "name*", set file_base to "name*".
        # That is, extension wasn't included.
        file_base = tiff_filename_base
        file_ext = ".tiff"

    file_base_array = file_base.split('*')

    # The array of filenames to return
    output_files = []

    # Filename 0-padding
    i = start_layers_at
    for layer in numpy_data:
        layer_filename = (str(i).zfill(6)).join(file_base_array) + file_ext
        output_files.append(export_tiff(layer_filename, layer))
        i += 1

    return output_files

def import_tiff_collection(tiff_filename_base):
    """
    Import all files matching the filename base given via `tiff_filename_base`.
    Images are ordered by alphabetical order, which means that you *MUST* 0-pad
    your numbers if they span a power of ten (e.g. 0999-1000 or 09-10). This is
    handled automatically by the complement function, `export_tiff_collection`.
    Also, look at how nicely these documentation lines are all the same length!

    Arguments:
        tiff_filename_base:     An asterisk-wildcard string that should refer
                                to all TIFFs in the stack. All * are replaced
                                according to command-line expansion rules.
    Returns:
        A numpy array holding a 3D dataset
    """

    # We expect images to be indexed by their alphabetical order.
    files = glob.glob(tiff_filename_base)
    files.sort()

    numpy_data = []
    for f in files:
        numpy_data.append(import_tiff(f))

    return numpy_data
<|endoftext|>


<commit_before>from __future__ import print_function

import os
import sys
import shutil
import subprocess

logstr = ""

def log(*args):
    global logstr
    print(*args)
    logstr += " ".join(args) + "\n"

def pathjoin(*args):
    return os.sep.join(args)

def isfile(path):
    if os.path.isfile(path):
        log("#     Found %s" % path)
        return True
    else:
        log("#     Could not find %s" % path)
        return False

def replacedir(path):
    log("#     Replacing directory", path)
    if os.path.isdir(path):
        shutil.rmtree(path)
    os.makedirs(path)
    return path

def call(cmd):
    log("#     Calling '%s'" % cmd)
    log("##")
    log("### CALL BEGINS")
    retcode = subprocess.call(cmd, shell=True)
    log("### CALL ENDS")
    log("##")
    return retcode

def diff(filename, diff_dict):
    with open(filename, "r") as a:
        with open(filename+".new", "w") as b:
            for line_number, line in enumerate(a):
                if line[:-1] in diff_dict:
                    newline = diff_dict[line[:-1]]+"\n"
                    log("#\n#     Change in %s"
                        "on line %i" % (filename, line_number + 1))
                    log("#     --", line[:-1][:70])
                    log("#     ++", newline[:70])
                    b.write(newline)
                else:
                    b.write(line)
    shutil.move(filename+".new", filename)

class SolverBackend(object):
    installed = False

    def __init__(self):
        log("# Looking for", self.name)
        location = self.look()
        if location is not None:
            log("# Found %s %s" % (self.name, location))
            if not hasattr(self, 'build'):
                self.installed = True
            else:
                log("#\n# Building %s..." % self.name)
                self.installed = self.build()
                status = "Done" if self.installed else "Failed"
                log("# %s building %s" % (status, self.name))
        else:
            log("# Did not find", self.name)
        log

class Mosek_CLI(SolverBackend):
    name = "mosek_cli"

    def look(self):
        try:
            log("#   Trying to run mskexpopt...")
            if call("mskexpopt") in (1052, 28):  # 28 for MacOSX
                return "in system path"
        except Exception:
            return None

class CVXopt(SolverBackend):
    name = "cvxopt"

    def look(self):
        try:
            log("#   Trying to import cvxopt...")
            import cvxopt
            return "in Python path"
        except ImportError:
            return None

class Mosek(SolverBackend):
    name = "mosek"

    # Some of the expopt code leaks log(statements onto stdout,)
    # instead of handing them to the task's stream message system.
    patches = {
        'dgopt.c': {
            # line 683:
            '          printf("Number of Hessian non-zeros: %d\\n",nlh[0]->numhesnz);':
            '          MSK_echotask(task,MSK_STREAM_MSG,"Number of Hessian non-zeros: %d\\n",nlh[0]->numhesnz);',
        },
        'expopt.c': {
            # line 1115:
            '    printf ("solsta = %d, prosta = %d\\n", (int)*solsta,(int)*prosta);':
            '    MSK_echotask(expopttask,MSK_STREAM_MSG, "solsta = %d, prosta = %d\\n", (int)*solsta,(int)*prosta);',
            """      printf("Warning: The variable with index '%d' has only positive coefficients akj.\\n The problem is possibly ill-posed.\\n.\\n",i);      """:
            """      MSK_echotask(expopttask,MSK_STREAM_MSG, "Warning: The variable with index '%d' has only positive coefficients akj.\\n The problem is possibly ill-posed.\\n.\\n",i);""",
            """      printf("Warning: The variable with index '%d' has only negative coefficients akj.\\n The problem is possibly ill-posed.\\n",i);      """:
            """      MSK_echotask(expopttask,MSK_STREAM_MSG, "Warning: The variable with index '%d' has only negative coefficients akj.\\n The problem is possibly ill-posed.\\n",i);""",
        }
    }

    def look(self):
        try:
            import ctypesgencore
        except ImportError:
            log("## SKIPPING MOSEK INSTALL: CTYPESGENCORE WAS NOT FOUND")
            return None

        if sys.platform == "win32":
            self.flags = "-Wl,--export-all-symbols,-R"
            try:
                self.dir = "C:\\Program Files\\Mosek"
                self.platform = "win64x86"
                self.libname = "mosek64_7_0.dll"
            except WindowsError:
                try:
                    self.dir = "C:\\Program Files (x86)\\Mosek"
                    self.platform = "win32x86"
                    self.libname = "mosek7_0.dll"
                except WindowsError:
                    return None
        elif sys.platform == "darwin":
            self.flags = "-Wl,-rpath"
            try:
                self.dir = pathjoin(os.path.expanduser("~"), "mosek")
                self.platform = "osx64x86"
                self.libname = "libmosek64.7.0.dylib"
            except OSError:
                return None
        elif sys.platform == "linux2":
            self.flags = "-Wl,--export-dynamic,-R"
            try:
                self.dir = pathjoin(os.path.expanduser("~"), "mosek")
                self.platform = "linux64x86"
                self.libname = "libmosek64.so"
            except OSError:
                return None
        else:
            log("# Build script does not support"
                " your platform (%s)" % sys.platform)
            return None

        if not os.path.isdir(self.dir):
            return None

        possible_versions = [f for f in os.listdir(self.dir) if len(f) == 1]
        self.version = sorted(possible_versions)[-1]
        self.tools_dir = pathjoin(self.dir, self.version, "tools")
        self.lib_dir = pathjoin(self.tools_dir, "platform", self.platform)
        self.h_path = pathjoin(self.lib_dir, "h", "mosek.h")
        self.bin_dir = pathjoin(self.lib_dir, "bin")
        self.lib_path = pathjoin(self.lib_dir, "bin", self.libname)

        if not isfile(self.h_path):
            return None
        if not isfile(self.lib_path):
            return None

        self.expopt_dir = pathjoin(self.tools_dir, "examples", "c")
        expopt_filenames = ["scopt-ext.c", "expopt.c", "dgopt.c",
                            "scopt-ext.h", "expopt.h", "dgopt.h"]
        self.expopt_files = [pathjoin(self.expopt_dir, fname)
                             for fname in expopt_filenames]
        self.expopt_files += [self.h_path]
        for expopt_file in self.expopt_files:
            if not isfile(expopt_file):
                return None

        global settings
        settings["mosek_bin_dir"] = self.bin_dir
        os.environ['PATH'] = os.environ['PATH']+':%s' % self.bin_dir

        return "version %s, installed to %s" % (self.version, self.dir)

    def build(self):
        lib_dir = replacedir(pathjoin("gpkit", "_mosek", "lib"))
        solib_dir = replacedir(pathjoin(os.path.expanduser("~"), ".gpkit"))
        f = open(pathjoin(lib_dir, "__init__.py"), 'w')
        f.close()

        build_dir = replacedir(pathjoin("gpkit", "_mosek", "build"))
        log("#\n#   Copying expopt library files to", build_dir)
        expopt_build_files = []
        for old_location in self.expopt_files:
            new_location = pathjoin(build_dir, os.path.basename(old_location))
            log("#     Copying %s" % old_location)
            shutil.copyfile(old_location, new_location)
            if new_location[-2:] == ".c":
                expopt_build_files.append(new_location)

        log("#\n#   Applying expopt patches...")
        for filename, patch in self.patches.items():
            diff(pathjoin(build_dir, filename), patch)

        log("#\n#   Building expopt library...")
        built_expopt_lib = call("gcc -fpic -shared" +
                                " %s %s" % (self.flags, self.bin_dir) +
                                "    " + " ".join(expopt_build_files) +
                                '   "' + self.lib_path + '"' +
                                " -o " + pathjoin(solib_dir, "expopt.so"))
        if built_expopt_lib != 0:
            return False

        log("#\n#   Building Python bindings for expopt and Mosek...")
        # mosek_h_path = pathjoin(lib_dir, "mosek_h.py")
        built_expopt_h = call("python gpkit/ctypesgen.py -a" +
                              " -l "+pathjoin(solib_dir, "expopt.so").replace("\\", "/") +
                              ' -l "' + self.lib_path.replace("\\", "/") + '"' +
                              # ' -o "' + mosek_h_path.replace("\\", "/") + '"'+
                              " -o "+pathjoin(lib_dir, "expopt_h.py") +
                              "    "+pathjoin(build_dir, "expopt.h"))

        if built_expopt_h != 0:
            return False

        return True

if isfile("__init__.py"):
    call("ls")
    log("#     Don't want to be in a folder with __init__.py, going up!")
    os.chdir("..")

log("Started building gpkit...\n")
settings = {}

log("Attempting to find and build solvers:\n")
solvers = [CVXopt(), Mosek(), Mosek_CLI()]
installed_solvers = [solver.name
                     for solver in solvers
                     if solver.installed]
if not installed_solvers:
    log("Can't find any solvers!\n")
#    sys.stderr.write("Can't find any solvers!\n")
#    sys.exit(70)

log("...finished building gpkit.")

# Choose default solver
settings["installed_solvers"] = ", ".join(installed_solvers)
log("\nFound the following solvers: " + settings["installed_solvers"])

# Write settings
envpath = pathjoin("gpkit", "env")
replacedir(envpath)
log("Replaced the directory gpkit/env\n")
settingspath = envpath + os.sep + "settings"
with open(settingspath, "w") as f:
    for setting, value in settings.items():
        f.write("%s : %s\n" % (setting, value))
    f.write("\n")

with open("gpkit/build.log", "w") as file:
    file.write(logstr)

call("ls")
call("echo \\# gpkit")
call("ls gpkit")
call("echo \\# gpkit/env")
call("ls gpkit/env")
<commit_msg>lint build.py<commit_after>from __future__ import print_function

import os
import sys
import shutil
import subprocess

logstr = ""

def log(*args):
    global logstr
    print(*args)
    logstr += " ".join(args) + "\n"

def pathjoin(*args):
    return os.sep.join(args)

def isfile(path):
    if os.path.isfile(path):
        log("#     Found %s" % path)
        return True
    else:
        log("#     Could not find %s" % path)
        return False

def replacedir(path):
    log("#     Replacing directory", path)
    if os.path.isdir(path):
        shutil.rmtree(path)
    os.makedirs(path)
    return path

def call(cmd):
    log("#     Calling '%s'" % cmd)
    log("##")
    log("### CALL BEGINS")
    retcode = subprocess.call(cmd, shell=True)
    log("### CALL ENDS")
    log("##")
    return retcode

def diff(filename, diff_dict):
    with open(filename, "r") as a:
        with open(filename+".new", "w") as b:
            for line_number, line in enumerate(a):
                if line[:-1] in diff_dict:
                    newline = diff_dict[line[:-1]]+"\n"
                    log("#\n#     Change in %s"
                        "on line %i" % (filename, line_number + 1))
                    log("#     --", line[:-1][:70])
                    log("#     ++", newline[:70])
                    b.write(newline)
                else:
                    b.write(line)
    shutil.move(filename+".new", filename)

class SolverBackend(object):
    installed = False

    def __init__(self):
        log("# Looking for", self.name)
        location = self.look()
        if location is not None:
            log("# Found %s %s" % (self.name, location))
            if not hasattr(self, 'build'):
                self.installed = True
            else:
                log("#\n# Building %s..." % self.name)
                self.installed = self.build()
                status = "Done" if self.installed else "Failed"
                log("# %s building %s" % (status, self.name))
        else:
            log("# Did not find", self.name)
        log

class Mosek_CLI(SolverBackend):
    name = "mosek_cli"

    def look(self):
        try:
            log("#   Trying to run mskexpopt...")
            if call("mskexpopt") in (1052, 28):  # 28 for MacOSX
                return "in system path"
        except Exception:
            return

class CVXopt(SolverBackend):
    name = "cvxopt"

    def look(self):
        try:
            log("#   Trying to import cvxopt...")
            import cvxopt
            return "in Python path"
        except ImportError:
            return

class Mosek(SolverBackend):
    name = "mosek"

    # Some of the expopt code leaks log(statements onto stdout,)
    # instead of handing them to the task's stream message system.
    patches = {
        'dgopt.c': {
            # line 683:
            '          printf("Number of Hessian non-zeros: %d\\n",nlh[0]->numhesnz);':
            '          MSK_echotask(task,MSK_STREAM_MSG,"Number of Hessian non-zeros: %d\\n",nlh[0]->numhesnz);',
        },
        'expopt.c': {
            # line 1115:
            '    printf ("solsta = %d, prosta = %d\\n", (int)*solsta,(int)*prosta);':
            '    MSK_echotask(expopttask,MSK_STREAM_MSG, "solsta = %d, prosta = %d\\n", (int)*solsta,(int)*prosta);',
            """      printf("Warning: The variable with index '%d' has only positive coefficients akj.\\n The problem is possibly ill-posed.\\n.\\n",i);      """:
            """      MSK_echotask(expopttask,MSK_STREAM_MSG, "Warning: The variable with index '%d' has only positive coefficients akj.\\n The problem is possibly ill-posed.\\n.\\n",i);""",
            """      printf("Warning: The variable with index '%d' has only negative coefficients akj.\\n The problem is possibly ill-posed.\\n",i);      """:
            """      MSK_echotask(expopttask,MSK_STREAM_MSG, "Warning: The variable with index '%d' has only negative coefficients akj.\\n The problem is possibly ill-posed.\\n",i);""",
        }
    }

    def look(self):
        try:
            import ctypesgencore
        except ImportError:
            log("## SKIPPING MOSEK INSTALL: CTYPESGENCORE WAS NOT FOUND")
            return

        if sys.platform == "win32":
            self.flags = "-Wl,--export-all-symbols,-R"
            try:
                self.dir = "C:\\Program Files\\Mosek"
                self.platform = "win64x86"
                self.libname = "mosek64_7_0.dll"
            except WindowsError:
                try:
                    self.dir = "C:\\Program Files (x86)\\Mosek"
                    self.platform = "win32x86"
                    self.libname = "mosek7_0.dll"
                except WindowsError:
                    return
        elif sys.platform == "darwin":
            self.flags = "-Wl,-rpath"
            try:
                self.dir = pathjoin(os.path.expanduser("~"), "mosek")
                self.platform = "osx64x86"
                self.libname = "libmosek64.7.0.dylib"
            except OSError:
                return
        elif sys.platform == "linux2":
            self.flags = "-Wl,--export-dynamic,-R"
            try:
                self.dir = pathjoin(os.path.expanduser("~"), "mosek")
                self.platform = "linux64x86"
                self.libname = "libmosek64.so"
            except OSError:
                return
        else:
            log("# Build script does not support"
                " your platform (%s)" % sys.platform)
            return

        if not os.path.isdir(self.dir):
            return

        possible_versions = [f for f in os.listdir(self.dir) if len(f) == 1]
        self.version = sorted(possible_versions)[-1]
        self.tools_dir = pathjoin(self.dir, self.version, "tools")
        self.lib_dir = pathjoin(self.tools_dir, "platform", self.platform)
        self.h_path = pathjoin(self.lib_dir, "h", "mosek.h")
        self.bin_dir = pathjoin(self.lib_dir, "bin")
        self.lib_path = pathjoin(self.lib_dir, "bin", self.libname)

        if not isfile(self.h_path):
            return
        if not isfile(self.lib_path):
            return

        self.expopt_dir = pathjoin(self.tools_dir, "examples", "c")
        expopt_filenames = ["scopt-ext.c", "expopt.c", "dgopt.c",
                            "scopt-ext.h", "expopt.h", "dgopt.h"]
        self.expopt_files = [pathjoin(self.expopt_dir, fname)
                             for fname in expopt_filenames]
        self.expopt_files += [self.h_path]
        for expopt_file in self.expopt_files:
            if not isfile(expopt_file):
                return

        global settings
        settings["mosek_bin_dir"] = self.bin_dir
        os.environ['PATH'] = os.environ['PATH']+':%s' % self.bin_dir

        return "version %s, installed to %s" % (self.version, self.dir)

    def build(self):
        lib_dir = replacedir(pathjoin("gpkit", "_mosek", "lib"))
        solib_dir = replacedir(pathjoin(os.path.expanduser("~"), ".gpkit"))
        f = open(pathjoin(lib_dir, "__init__.py"), 'w')
        f.close()

        build_dir = replacedir(pathjoin("gpkit", "_mosek", "build"))
        log("#\n#   Copying expopt library files to", build_dir)
        expopt_build_files = []
        for old_location in self.expopt_files:
            new_location = pathjoin(build_dir, os.path.basename(old_location))
            log("#     Copying %s" % old_location)
            shutil.copyfile(old_location, new_location)
            if new_location[-2:] == ".c":
                expopt_build_files.append(new_location)

        log("#\n#   Applying expopt patches...")
        for filename, patch in self.patches.items():
            diff(pathjoin(build_dir, filename), patch)

        log("#\n#   Building expopt library...")
        built_expopt_lib = call("gcc -fpic -shared" +
                                " %s %s" % (self.flags, self.bin_dir) +
                                "    " + " ".join(expopt_build_files) +
                                '   "' + self.lib_path + '"' +
                                " -o " + pathjoin(solib_dir, "expopt.so"))
        if built_expopt_lib != 0:
            return False

        log("#\n#   Building Python bindings for expopt and Mosek...")
        # mosek_h_path = pathjoin(lib_dir, "mosek_h.py")
        built_expopt_h = call("python gpkit/ctypesgen.py -a" +
                              " -l "+pathjoin(solib_dir, "expopt.so").replace("\\", "/") +
                              ' -l "' + self.lib_path.replace("\\", "/") + '"' +
                              # ' -o "' + mosek_h_path.replace("\\", "/") + '"'+
                              " -o "+pathjoin(lib_dir, "expopt_h.py") +
                              "    "+pathjoin(build_dir, "expopt.h"))

        if built_expopt_h != 0:
            return False

        return True

if isfile("__init__.py"):
    #call("ls")
    log("#     Don't want to be in a folder with __init__.py, going up!")
    os.chdir("..")

log("Started building gpkit...\n")
settings = {}

log("Attempting to find and build solvers:\n")
solvers = [CVXopt(), Mosek(), Mosek_CLI()]
installed_solvers = [solver.name
                     for solver in solvers
                     if solver.installed]
if not installed_solvers:
    log("Can't find any solvers!\n")
#    sys.stderr.write("Can't find any solvers!\n")
#    sys.exit(70)

log("...finished building gpkit.")

# Choose default solver
settings["installed_solvers"] = ", ".join(installed_solvers)
log("\nFound the following solvers: " + settings["installed_solvers"])

# Write settings
envpath = pathjoin("gpkit", "env")
replacedir(envpath)
log("Replaced the directory gpkit/env\n")
settingspath = envpath + os.sep + "settings"
with open(settingspath, "w") as f:
    for setting, value in settings.items():
        f.write("%s : %s\n" % (setting, value))
    f.write("\n")

with open("gpkit/build.log", "w") as file:
    file.write(logstr)

#call("ls")
#call("echo \\# gpkit")
#call("ls gpkit")
#call("echo \\# gpkit/env")
#call("ls gpkit/env")
<|endoftext|>


<commit_before>import os
import yaml
import toml
import errno
import shutil
import json
import grp

from pwd import getpwnam
from collections import OrderedDict

from moulinette import m18n
from moulinette.core import MoulinetteError

# Files & directories --------------------------------------------------

def read_file(file_path):
    """
    Read a regular text file

    Keyword argument:
        file_path -- Path to the text file
    """
    assert isinstance(
        file_path, str
    ), "Error: file_path '%s' should be a string but is of type '%s' instead" % (
        file_path,
        type(file_path),
    )

    # Check file exists
    if not os.path.isfile(file_path):
        raise MoulinetteError("file_not_exist", path=file_path)

    # Open file and read content
    try:
        with open(file_path, "r") as f:
            file_content = f.read()
    except IOError as e:
        raise MoulinetteError("cannot_open_file", file=file_path, error=str(e))
    except Exception as e:
        raise MoulinetteError(
            "unknown_error_reading_file", file=file_path, error=str(e)
        )

    return file_content

def read_json(file_path):
    """
    Read a json file

    Keyword argument:
        file_path -- Path to the json file
    """

    # Read file
    file_content = read_file(file_path)

    # Try to load json to check if it's syntaxically correct
    try:
        loaded_json = json.loads(file_content)
    except ValueError as e:
        raise MoulinetteError("corrupted_json", ressource=file_path, error=str(e))

    return loaded_json

def read_yaml(file_path):
    """
    Safely read a yaml file

    Keyword argument:
        file_path -- Path to the yaml file
    """

    # Read file
    file_content = read_file(file_path)

    # Try to load yaml to check if it's syntaxically correct
    try:
        loaded_yaml = yaml.safe_load(file_content)
    except Exception as e:
        raise MoulinetteError("corrupted_yaml", ressource=file_path, error=str(e))

    return loaded_yaml

def read_toml(file_path):
    """
    Safely read a toml file

    Keyword argument:
        file_path -- Path to the toml file
    """

    # Read file
    file_content = read_file(file_path)

    # Try to load toml to check if it's syntactically correct
    try:
        loaded_toml = toml.loads(file_content, _dict=OrderedDict)
    except Exception as e:
        raise MoulinetteError("corrupted_toml", ressource=file_path, error=str(e))

    return loaded_toml

def write_to_file(file_path, data, file_mode="w"):
    """
    Write a single string or a list of string to a text file.
    The text file will be overwritten by default.

    Keyword argument:
        file_path -- Path to the output file
        data -- The data to write (must be a string or list of string)
        file_mode -- Mode used when writing the file. Option meant to be used
        by append_to_file to avoid duplicating the code of this function.
    """
    assert isinstance(data, str) or isinstance(
        data, list
    ), "Error: data '%s' should be either a string or a list but is of type '%s'" % (
        data,
        type(data),
    )
    assert not os.path.isdir(file_path), (
        "Error: file_path '%s' point to a dir, it should be a file" % file_path
    )
    assert os.path.isdir(
        os.path.dirname(file_path)
    ), "Error: the path ('%s') base dir ('%s') is not a dir" % (
        file_path,
        os.path.dirname(file_path),
    )

    # If data is a list, check elements are strings and build a single string
    if not isinstance(data, str):
        for element in data:
            assert isinstance(
                element, str
            ), "Error: element '%s' should be a string but is of type '%s' instead" % (
                element,
                type(element),
            )
        data = "\n".join(data)

    try:
        with open(file_path, file_mode) as f:
            f.write(data)
    except IOError as e:
        raise MoulinetteError("cannot_write_file", file=file_path, error=str(e))
    except Exception as e:
        raise MoulinetteError("error_writing_file", file=file_path, error=str(e))

def append_to_file(file_path, data):
    """
    Append a single string or a list of string to a text file.

    Keyword argument:
        file_path -- Path to the output file
        data -- The data to write (must be a string or list of string)
    """

    write_to_file(file_path, data, file_mode="a")

def write_to_json(file_path, data, sort_keys=False, indent=None):
    """
    Write a dictionnary or a list to a json file

    Keyword argument:
        file_path -- Path to the output json file
        data -- The data to write (must be a dict or a list)
    """

    # Assumptions
    assert isinstance(
        file_path, str
    ), "Error: file_path '%s' should be a string but is of type '%s' instead" % (
        file_path,
        type(file_path),
    )
    assert isinstance(data, dict) or isinstance(
        data, list
    ), "Error: data '%s' should be a dict or a list but is of type '%s' instead" % (
        data,
        type(data),
    )
    assert not os.path.isdir(file_path), (
        "Error: file_path '%s' point to a dir, it should be a file" % file_path
    )
    assert os.path.isdir(
        os.path.dirname(file_path)
    ), "Error: the path ('%s') base dir ('%s') is not a dir" % (
        file_path,
        os.path.dirname(file_path),
    )

    # Write dict to file
    try:
        with open(file_path, "w") as f:
            json.dump(data, f, sort_keys=sort_keys, indent=indent)
    except IOError as e:
        raise MoulinetteError("cannot_write_file", file=file_path, error=str(e))
    except Exception as e:
        raise MoulinetteError("error_writing_file", file=file_path, error=str(e))

def write_to_yaml(file_path, data):
    """
    Write a dictionnary or a list to a yaml file

    Keyword argument:
        file_path -- Path to the output yaml file
        data -- The data to write (must be a dict or a list)
    """
    # Assumptions
    assert isinstance(file_path, str)
    assert isinstance(data, dict) or isinstance(data, list)
    assert not os.path.isdir(file_path)
    assert os.path.isdir(os.path.dirname(file_path))

    # Write dict to file
    try:
        with open(file_path, "w") as f:
            yaml.safe_dump(data, f, default_flow_style=False)
    except IOError as e:
        raise MoulinetteError("cannot_write_file", file=file_path, error=str(e))
    except Exception as e:
        raise MoulinetteError("error_writing_file", file=file_path, error=str(e))

def mkdir(path, mode=0o0777, parents=False, uid=None, gid=None, force=False):
    """Create a directory with optional features

    Create a directory and optionaly set its permissions to mode and its
    owner and/or group. If path refers to an existing path, nothing is done
    unless force is True.

    Keyword arguments:
        - path -- The directory to create
        - mode -- Numeric path mode to set
        - parents -- Make parent directories as needed
        - uid -- Numeric uid or user name
        - gid -- Numeric gid or group name
        - force -- Force directory creation and owning even if the path exists

    """
    if os.path.exists(path) and not force:
        raise OSError(errno.EEXIST, m18n.g("folder_exists", path=path))

    if parents:
        # Create parents directories as needed
        head, tail = os.path.split(path)
        if not tail:
            head, tail = os.path.split(head)
        if head and tail and not os.path.exists(head):
            try:
                mkdir(head, mode, parents, uid, gid, force)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
            if tail == os.curdir:
                return

    # Create directory and set permissions
    try:
        oldmask = os.umask(000)
        os.mkdir(path, mode)
        os.umask(oldmask)
    except OSError:
        # mimic Python3.2+ os.makedirs exist_ok behaviour
        if not force or not os.path.isdir(path):
            raise

    if uid is not None or gid is not None:
        chown(path, uid, gid)

def chown(path, uid=None, gid=None, recursive=False):
    """Change the owner and/or group of a path

    Keyword arguments:
        - uid -- Numeric uid or user name
        - gid -- Numeric gid or group name
        - recursive -- Operate on path recursively

    """
    if uid is None and gid is None:
        raise ValueError("either uid or gid argument is required")

    # Retrieve uid/gid
    if isinstance(uid, str):
        try:
            uid = getpwnam(uid).pw_uid
        except KeyError:
            raise MoulinetteError("unknown_user", user=uid)
    elif uid is None:
        uid = -1
    if isinstance(gid, str):
        try:
            gid = grp.getgrnam(gid).gr_gid
        except KeyError:
            raise MoulinetteError("unknown_group", group=gid)
    elif gid is None:
        gid = -1

    try:
        os.chown(path, uid, gid)
        if recursive and os.path.isdir(path):
            for root, dirs, files in os.walk(path):
                for d in dirs:
                    os.chown(os.path.join(root, d), uid, gid)
                for f in files:
                    os.chown(os.path.join(root, f), uid, gid)
    except Exception as e:
        raise MoulinetteError(
            "error_changing_file_permissions", path=path, error=str(e)
        )

def chmod(path, mode, fmode=None, recursive=False):
    """Change the mode of a path

    Keyword arguments:
        - mode -- Numeric path mode to set
        - fmode -- Numeric file mode to set in case of a recursive directory
        - recursive -- Operate on path recursively

    """

    try:
        os.chmod(path, mode)
        if recursive and os.path.isdir(path):
            if fmode is None:
                fmode = mode
            for root, dirs, files in os.walk(path):
                for d in dirs:
                    os.chmod(os.path.join(root, d), mode)
                for f in files:
                    os.chmod(os.path.join(root, f), fmode)
    except Exception as e:
        raise MoulinetteError(
            "error_changing_file_permissions", path=path, error=str(e)
        )

def rm(path, recursive=False, force=False):
    """Remove a file or directory

    Keyword arguments:
        - path -- The path to remove
        - recursive -- Remove directories and their contents recursively
        - force -- Ignore nonexistent files

    """
    if recursive and os.path.isdir(path):
        shutil.rmtree(path, ignore_errors=force)
    else:
        try:
            os.remove(path)
        except OSError as e:
            if not force:
                raise MoulinetteError("error_removing", path=path, error=str(e))
<commit_msg>[enh] read_yaml support stream<commit_after>import os
import yaml
import toml
import errno
import shutil
import json
import grp

from pwd import getpwnam
from collections import OrderedDict

from moulinette import m18n
from moulinette.core import MoulinetteError

# Files & directories --------------------------------------------------

def read_file(file_path):
    """
    Read a regular text file

    Keyword argument:
        file_path -- Path to the text file
    """
    assert isinstance(
        file_path, str
    ), "Error: file_path '%s' should be a string but is of type '%s' instead" % (
        file_path,
        type(file_path),
    )

    # Check file exists
    if not os.path.isfile(file_path):
        raise MoulinetteError("file_not_exist", path=file_path)

    # Open file and read content
    try:
        with open(file_path, "r") as f:
            file_content = f.read()
    except IOError as e:
        raise MoulinetteError("cannot_open_file", file=file_path, error=str(e))
    except Exception as e:
        raise MoulinetteError(
            "unknown_error_reading_file", file=file_path, error=str(e)
        )

    return file_content

def read_json(file_path):
    """
    Read a json file

    Keyword argument:
        file_path -- Path to the json file
    """

    # Read file
    file_content = read_file(file_path)

    # Try to load json to check if it's syntaxically correct
    try:
        loaded_json = json.loads(file_content)
    except ValueError as e:
        raise MoulinetteError("corrupted_json", ressource=file_path, error=str(e))

    return loaded_json

def read_yaml(file_):
    """
    Safely read a yaml file

    Keyword argument:
        file -- Path or stream to the yaml file
    """

    # Read file
    file_path = file_ if isinstance(file_, str) else file_.name
    file_content = read_file(file_) if isinstance(file_, str) else file_

    # Try to load yaml to check if it's syntaxically correct
    try:
        loaded_yaml = yaml.safe_load(file_content)
    except Exception as e:
        raise MoulinetteError("corrupted_yaml", ressource=file_path, error=str(e))

    return loaded_yaml

def read_toml(file_path):
    """
    Safely read a toml file

    Keyword argument:
        file_path -- Path to the toml file
    """

    # Read file
    file_content = read_file(file_path)

    # Try to load toml to check if it's syntactically correct
    try:
        loaded_toml = toml.loads(file_content, _dict=OrderedDict)
    except Exception as e:
        raise MoulinetteError("corrupted_toml", ressource=file_path, error=str(e))

    return loaded_toml

def write_to_file(file_path, data, file_mode="w"):
    """
    Write a single string or a list of string to a text file.
    The text file will be overwritten by default.

    Keyword argument:
        file_path -- Path to the output file
        data -- The data to write (must be a string or list of string)
        file_mode -- Mode used when writing the file. Option meant to be used
        by append_to_file to avoid duplicating the code of this function.
    """
    assert isinstance(data, str) or isinstance(
        data, list
    ), "Error: data '%s' should be either a string or a list but is of type '%s'" % (
        data,
        type(data),
    )
    assert not os.path.isdir(file_path), (
        "Error: file_path '%s' point to a dir, it should be a file" % file_path
    )
    assert os.path.isdir(
        os.path.dirname(file_path)
    ), "Error: the path ('%s') base dir ('%s') is not a dir" % (
        file_path,
        os.path.dirname(file_path),
    )

    # If data is a list, check elements are strings and build a single string
    if not isinstance(data, str):
        for element in data:
            assert isinstance(
                element, str
            ), "Error: element '%s' should be a string but is of type '%s' instead" % (
                element,
                type(element),
            )
        data = "\n".join(data)

    try:
        with open(file_path, file_mode) as f:
            f.write(data)
    except IOError as e:
        raise MoulinetteError("cannot_write_file", file=file_path, error=str(e))
    except Exception as e:
        raise MoulinetteError("error_writing_file", file=file_path, error=str(e))

def append_to_file(file_path, data):
    """
    Append a single string or a list of string to a text file.

    Keyword argument:
        file_path -- Path to the output file
        data -- The data to write (must be a string or list of string)
    """

    write_to_file(file_path, data, file_mode="a")

def write_to_json(file_path, data, sort_keys=False, indent=None):
    """
    Write a dictionnary or a list to a json file

    Keyword argument:
        file_path -- Path to the output json file
        data -- The data to write (must be a dict or a list)
    """

    # Assumptions
    assert isinstance(
        file_path, str
    ), "Error: file_path '%s' should be a string but is of type '%s' instead" % (
        file_path,
        type(file_path),
    )
    assert isinstance(data, dict) or isinstance(
        data, list
    ), "Error: data '%s' should be a dict or a list but is of type '%s' instead" % (
        data,
        type(data),
    )
    assert not os.path.isdir(file_path), (
        "Error: file_path '%s' point to a dir, it should be a file" % file_path
    )
    assert os.path.isdir(
        os.path.dirname(file_path)
    ), "Error: the path ('%s') base dir ('%s') is not a dir" % (
        file_path,
        os.path.dirname(file_path),
    )

    # Write dict to file
    try:
        with open(file_path, "w") as f:
            json.dump(data, f, sort_keys=sort_keys, indent=indent)
    except IOError as e:
        raise MoulinetteError("cannot_write_file", file=file_path, error=str(e))
    except Exception as e:
        raise MoulinetteError("error_writing_file", file=file_path, error=str(e))

def write_to_yaml(file_path, data):
    """
    Write a dictionnary or a list to a yaml file

    Keyword argument:
        file_path -- Path to the output yaml file
        data -- The data to write (must be a dict or a list)
    """
    # Assumptions
    assert isinstance(file_path, str)
    assert isinstance(data, dict) or isinstance(data, list)
    assert not os.path.isdir(file_path)
    assert os.path.isdir(os.path.dirname(file_path))

    # Write dict to file
    try:
        with open(file_path, "w") as f:
            yaml.safe_dump(data, f, default_flow_style=False)
    except IOError as e:
        raise MoulinetteError("cannot_write_file", file=file_path, error=str(e))
    except Exception as e:
        raise MoulinetteError("error_writing_file", file=file_path, error=str(e))

def mkdir(path, mode=0o0777, parents=False, uid=None, gid=None, force=False):
    """Create a directory with optional features

    Create a directory and optionaly set its permissions to mode and its
    owner and/or group. If path refers to an existing path, nothing is done
    unless force is True.

    Keyword arguments:
        - path -- The directory to create
        - mode -- Numeric path mode to set
        - parents -- Make parent directories as needed
        - uid -- Numeric uid or user name
        - gid -- Numeric gid or group name
        - force -- Force directory creation and owning even if the path exists

    """
    if os.path.exists(path) and not force:
        raise OSError(errno.EEXIST, m18n.g("folder_exists", path=path))

    if parents:
        # Create parents directories as needed
        head, tail = os.path.split(path)
        if not tail:
            head, tail = os.path.split(head)
        if head and tail and not os.path.exists(head):
            try:
                mkdir(head, mode, parents, uid, gid, force)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
            if tail == os.curdir:
                return

    # Create directory and set permissions
    try:
        oldmask = os.umask(000)
        os.mkdir(path, mode)
        os.umask(oldmask)
    except OSError:
        # mimic Python3.2+ os.makedirs exist_ok behaviour
        if not force or not os.path.isdir(path):
            raise

    if uid is not None or gid is not None:
        chown(path, uid, gid)

def chown(path, uid=None, gid=None, recursive=False):
    """Change the owner and/or group of a path

    Keyword arguments:
        - uid -- Numeric uid or user name
        - gid -- Numeric gid or group name
        - recursive -- Operate on path recursively

    """
    if uid is None and gid is None:
        raise ValueError("either uid or gid argument is required")

    # Retrieve uid/gid
    if isinstance(uid, str):
        try:
            uid = getpwnam(uid).pw_uid
        except KeyError:
            raise MoulinetteError("unknown_user", user=uid)
    elif uid is None:
        uid = -1
    if isinstance(gid, str):
        try:
            gid = grp.getgrnam(gid).gr_gid
        except KeyError:
            raise MoulinetteError("unknown_group", group=gid)
    elif gid is None:
        gid = -1

    try:
        os.chown(path, uid, gid)
        if recursive and os.path.isdir(path):
            for root, dirs, files in os.walk(path):
                for d in dirs:
                    os.chown(os.path.join(root, d), uid, gid)
                for f in files:
                    os.chown(os.path.join(root, f), uid, gid)
    except Exception as e:
        raise MoulinetteError(
            "error_changing_file_permissions", path=path, error=str(e)
        )

def chmod(path, mode, fmode=None, recursive=False):
    """Change the mode of a path

    Keyword arguments:
        - mode -- Numeric path mode to set
        - fmode -- Numeric file mode to set in case of a recursive directory
        - recursive -- Operate on path recursively

    """

    try:
        os.chmod(path, mode)
        if recursive and os.path.isdir(path):
            if fmode is None:
                fmode = mode
            for root, dirs, files in os.walk(path):
                for d in dirs:
                    os.chmod(os.path.join(root, d), mode)
                for f in files:
                    os.chmod(os.path.join(root, f), fmode)
    except Exception as e:
        raise MoulinetteError(
            "error_changing_file_permissions", path=path, error=str(e)
        )

def rm(path, recursive=False, force=False):
    """Remove a file or directory

    Keyword arguments:
        - path -- The path to remove
        - recursive -- Remove directories and their contents recursively
        - force -- Ignore nonexistent files

    """
    if recursive and os.path.isdir(path):
        shutil.rmtree(path, ignore_errors=force)
    else:
        try:
            os.remove(path)
        except OSError as e:
            if not force:
                raise MoulinetteError("error_removing", path=path, error=str(e))
<|endoftext|>