<commit_before>import triton
from . import core as tl

# Notes
# 1. triton doesn't support uint32, so we use int32 instead and benefit from the fact that two's complement operations are equivalent to uint operations.
# 2. multiply_low_high is currently inefficient.
# 3. Even though technically philox sampling outputs int, in many places we pretends they were actualy uints e.g. uint_to_uniform_float

PHILOX_KEY_A: tl.constexpr = -1640531527 # 0x9E3779B9
PHILOX_KEY_B: tl.constexpr = -1150833019 # 0xBB67AE85
PHILOX_ROUND_A: tl.constexpr = -766435501 # 0xD2511F53
PHILOX_ROUND_B: tl.constexpr = -845247145 # 0xCD9E8D57
N_ROUNDS_DEFAULT = 10 # Default number of rounds for philox

# -------------------
# randint
# -------------------

@triton.jit
def hacky_to_uint64(x):
    return ((x >> 1).to(tl.int64) << 1) + (x & 1).to(tl.int64)

@triton.jit
def philox_f(c0, c1, c2, c3, k0, k1, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT):
    """
    Run `n_rounds` rounds of Philox for state (c0, c1, c2, c3) and key (k0, k1).
    """
    for _ in range(n_rounds):
        # update random state
        A = PHILOX_ROUND_A
        B = PHILOX_ROUND_B
        _c0, _c2 = c0, c2
        c0 = tl.umulhi(B, _c2) ^ c1 ^ k0
        c2 = tl.umulhi(A, _c0) ^ c3 ^ k1
        c1 = B * _c2
        c3 = A * _c0
        # raise key
        k0 = k0 + PHILOX_KEY_A
        k1 = k1 + PHILOX_KEY_B
    return c0, c1, c2, c3

@triton.jit
def randint(seed, offset, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT):
    """
    Given a :code:`seed` scalar and an :code:`offset` block, returns a single
    block of random :code:`int32`.

    If you need multiple streams of random numbers,
    using `randint4x` is likely to be faster than calling `randint` 4 times.

    :param seed: The seed for generating random numbers.
    :param offsets: The offsets to generate random numbers for.
    """
    ret, _, _, _ = randint4x(seed, offset, n_rounds)
    return ret

@triton.jit
def randint4x(seed, offset, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT):
    """
    Given a :code:`seed` scalar and an :code:`offset` block, returns four
    blocks of random :code:`int32`.

    This is the maximally efficient entry point
    to Triton's Philox pseudo-random number generator.

    :param seed: The seed for generating random numbers.
    :param offsets: The offsets to generate random numbers for.
    """
    z = offset*0 #FIXME: just 0 doesn't work. Likelye some error with broadcasting
    seed = seed + 0
    seed = hacky_to_uint64(seed) # uint will solve this
    seed_hi = ((seed >> 32) & 0xffffffff).to(tl.int32)
    seed_lo = (seed & 0xffffffff).to(tl.int32)
    return philox_f(offset, z, z, z, seed_lo, seed_hi, n_rounds)

# -------------------
# rand
# -------------------

@triton.jit
def uint32_to_uniform_float(x):
    """
    Numerically stable function to convert a random integer into a random float uniformly sampled in [0, 1).
    This is originally designed from uint32, but it works with int32 too as long as the int32 uniformly
    covers all the possible values it can take.
    """
    max = 4.656613e-10 # = 1/MAX_INT = 1/2147483647.
    x = tl.where(x < 0, -x - 1, x)
    return x * max

@triton.jit
def rand(seed, offset, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT):
    """
    Given a :code:`seed` scalar and an :code:`offset` block,
    returns a block of random :code:`float32` in :math:`U(0, 1)`

    :param seed: The seed for generating random numbers.
    :param offsets: The offsets to generate random numbers for.
    """
    source = randint(seed, offset, n_rounds)
    return uint32_to_uniform_float(source)

@triton.jit
def rand4x(seed, offsets, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT):
    """
    Given a :code:`seed` scalar and an :code:`offsets` block,
    returns a 4 blocks of random :code:`float32` in :math:`U(0, 1)`

    :param seed: The seed for generating random numbers.
    :param offsets: The offsets to generate random numbers for.
    """
    i1, i2, i3, i4 = randint4x(seed, offsets, n_rounds)
    u1 = uint32_to_uniform_float(i1)
    u2 = uint32_to_uniform_float(i2)
    u3 = uint32_to_uniform_float(i3)
    u4 = uint32_to_uniform_float(i4)
    return u1, u2, u3, u4

# -------------------
# randn
# -------------------

@triton.jit
def pair_uniform_to_normal(u1, u2):
    """Box-Muller transform"""
    u1 = tl.maximum(1.0e-7, u1)
    th = 6.283185307179586 * u2
    r = tl.sqrt(-2.0 * tl.log(u1))
    return r * tl.cos(th), r * tl.sin(th)

@triton.jit
def randn(seed, offset, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT):
    """
    Given a :code:`seed` scalar and an :code:`offset` block,
    returns a block of random :code:`float32` in :math:`\\mathcal{N}(0, 1)`

    :param seed: The seed for generating random numbers.
    :param offsets: The offsets to generate random numbers for.
    """
    i1, i2, _, _ = randint4x(seed, offset, n_rounds)
    u1 = uint32_to_uniform_float(i1)
    u2 = uint32_to_uniform_float(i2)
    n1, _ = pair_uniform_to_normal(u1, u2)
    return n1

@triton.jit
def randn4x(seed, offset, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT):
    """
    Given a :code:`seed` scalar and an :code:`offset` block,
    returns a 4 blocks of random :code:`float32` in :math:`\\mathcal{N}(0, 1)`

    :param seed: The seed for generating random numbers.
    :param offsets: The offsets to generate random numbers for.
    """
    u1, u2, u3, u4 = rand4x(seed, offset, n_rounds)
    n1, n2 = pair_uniform_to_normal(u1, u2)
    n3, n4 = pair_uniform_to_normal(u3, u4)
    return n1, n2, n3, n4
<commit_msg>[FRONTEND] use unsigned integers to simplify RNG (#417)<commit_after>import triton
from . import core as tl

PHILOX_KEY_A: tl.constexpr = -1640531527  # 0x9E3779B9
PHILOX_KEY_B: tl.constexpr = -1150833019  # 0xBB67AE85
PHILOX_ROUND_A: tl.constexpr = -766435501  # 0xD2511F53
PHILOX_ROUND_B: tl.constexpr = -845247145  # 0xCD9E8D57
N_ROUNDS_DEFAULT = 10  # Default number of rounds for philox

# -------------------
# randint
# -------------------

@triton.jit
def philox_f(c0, c1, c2, c3, k0, k1, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT):
    """
    Run `n_rounds` rounds of Philox for state (c0, c1, c2, c3) and key (k0, k1).
    """
    for _ in range(n_rounds):
        # update random state
        A = PHILOX_ROUND_A
        B = PHILOX_ROUND_B
        _c0, _c2 = c0, c2
        c0 = tl.umulhi(B, _c2) ^ c1 ^ k0
        c2 = tl.umulhi(A, _c0) ^ c3 ^ k1
        c1 = B * _c2
        c3 = A * _c0
        # raise key
        k0 = k0 + PHILOX_KEY_A
        k1 = k1 + PHILOX_KEY_B
    return c0, c1, c2, c3

@triton.jit
def randint(seed, offset, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT):
    """
    Given a :code:`seed` scalar and an :code:`offset` block, returns a single
    block of random :code:`int32`.

    If you need multiple streams of random numbers,
    using `randint4x` is likely to be faster than calling `randint` 4 times.

    :param seed: The seed for generating random numbers.
    :param offsets: The offsets to generate random numbers for.
    """
    ret, _, _, _ = randint4x(seed, offset, n_rounds)
    return ret

@triton.jit
def randint4x(seed, offset, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT):
    """
    Given a :code:`seed` scalar and an :code:`offset` block, returns four
    blocks of random :code:`int32`.

    This is the maximally efficient entry point
    to Triton's Philox pseudo-random number generator.

    :param seed: The seed for generating random numbers.
    :param offsets: The offsets to generate random numbers for.
    """
    z = offset * 0  # FIXME: just 0 doesn't work. Likely some error with broadcasting
    seed = seed.to(tl.uint64)
    seed_hi = ((seed >> 32) & 0xffffffff).to(tl.uint32)
    seed_lo = (seed & 0xffffffff).to(tl.uint32)
    return philox_f(offset, z, z, z, seed_lo, seed_hi, n_rounds)

# -------------------
# rand
# -------------------

@triton.jit
def uint32_to_uniform_float(x):
    """
    Numerically stable function to convert a random uint32 into a random float uniformly sampled in [0, 1).
    """
    two_to_the_minus_32 = 2.328306e-10
    return x * two_to_the_minus_32

@triton.jit
def rand(seed, offset, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT):
    """
    Given a :code:`seed` scalar and an :code:`offset` block,
    returns a block of random :code:`float32` in :math:`U(0, 1)`

    :param seed: The seed for generating random numbers.
    :param offsets: The offsets to generate random numbers for.
    """
    source = randint(seed, offset, n_rounds)
    return uint32_to_uniform_float(source)

@triton.jit
def rand4x(seed, offsets, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT):
    """
    Given a :code:`seed` scalar and an :code:`offsets` block,
    returns a 4 blocks of random :code:`float32` in :math:`U(0, 1)`

    :param seed: The seed for generating random numbers.
    :param offsets: The offsets to generate random numbers for.
    """
    i1, i2, i3, i4 = randint4x(seed, offsets, n_rounds)
    u1 = uint32_to_uniform_float(i1)
    u2 = uint32_to_uniform_float(i2)
    u3 = uint32_to_uniform_float(i3)
    u4 = uint32_to_uniform_float(i4)
    return u1, u2, u3, u4

# -------------------
# randn
# -------------------

@triton.jit
def pair_uniform_to_normal(u1, u2):
    """Box-Muller transform"""
    u1 = tl.maximum(1.0e-7, u1)
    th = 6.283185307179586 * u2
    r = tl.sqrt(-2.0 * tl.log(u1))
    return r * tl.cos(th), r * tl.sin(th)

@triton.jit
def randn(seed, offset, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT):
    """
    Given a :code:`seed` scalar and an :code:`offset` block,
    returns a block of random :code:`float32` in :math:`\\mathcal{N}(0, 1)`

    :param seed: The seed for generating random numbers.
    :param offsets: The offsets to generate random numbers for.
    """
    i1, i2, _, _ = randint4x(seed, offset, n_rounds)
    u1 = uint32_to_uniform_float(i1)
    u2 = uint32_to_uniform_float(i2)
    n1, _ = pair_uniform_to_normal(u1, u2)
    return n1

@triton.jit
def randn4x(seed, offset, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT):
    """
    Given a :code:`seed` scalar and an :code:`offset` block,
    returns a 4 blocks of random :code:`float32` in :math:`\\mathcal{N}(0, 1)`

    :param seed: The seed for generating random numbers.
    :param offsets: The offsets to generate random numbers for.
    """
    u1, u2, u3, u4 = rand4x(seed, offset, n_rounds)
    n1, n2 = pair_uniform_to_normal(u1, u2)
    n3, n4 = pair_uniform_to_normal(u3, u4)
    return n1, n2, n3, n4
<|endoftext|>


<commit_before>class WatsonAnswer(object):
    """An answer received from Watson"""

    def __init__(self, answer_mapping):
        """Create a Watson Answer from the given mapping.

        :param answer_mapping: the Mapping representing a response from Watson
        :type answer_mapping: Mapping
        :return: Answer
        """
        pass
<commit_msg>Add dummy instance variable<commit_after>class WatsonAnswer(object):
    """An answer received from Watson"""

    def __init__(self, answer_mapping):
        """Create a Watson Answer from the given mapping.

        :param answer_mapping: the Mapping representing a response from Watson
        :type answer_mapping: Mapping
        :return: Answer
        """
        self.answer_mapping = answer_mapping
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""Special bot library containing UploadRobot.

Do not import classes directly from here but from specialbots.
"""
#
# (C) Pywikibot team, 2003-2020
#
# Distributed under the terms of the MIT license.
#
import os
import tempfile

from contextlib import closing
from urllib.parse import urlparse
from urllib.request import URLopener

import pywikibot
import pywikibot.data.api

from pywikibot import config

from pywikibot.bot import BaseBot, QuitKeyboardInterrupt
from pywikibot.tools import deprecated_args
from pywikibot.tools.formatter import color_format

class UploadRobot(BaseBot):

    """Upload bot."""

    @deprecated_args(urlEncoding='url_encoding', useFilename='use_filename',
                     keepFilename='keep_filename',
                     verifyDescription='verify_description',
                     ignoreWarning='ignore_warning', targetSite='target_site')
    def __init__(self, url: list, url_encoding=None, description='',
                 use_filename=None, keep_filename=False,
                 verify_description=True, ignore_warning=False,
                 target_site=None, aborts=[], chunk_size=0, summary=None,
                 filename_prefix=None, **kwargs):
        """
        Initializer.

        @param url: path to url or local file (deprecated), or list of urls or
            paths to local files.
        @param description: Description of file for its page. If multiple files
            are uploading the same description is used for every file.
        @type description: str
        @param use_filename: Specify title of the file's page. If multiple
            files are uploading it asks to change the name for second, third,
            etc. files, otherwise the last file will overwrite the other.
        @type use_filename: str
        @param keep_filename: Set to True to keep original names of urls and
            files, otherwise it will ask to enter a name for each file.
        @type keep_filename: bool
        @param summary: Summary of the upload
        @type summary: str
        @param verify_description: Set to True to proofread the description.
        @type verify_description: bool
        @param ignore_warning: Set this to True to upload even if another file
            would be overwritten or another mistake would be risked. Set it to
            an array of warning codes to selectively ignore specific warnings.
        @type ignore_warning: bool or list
        @param target_site: Set the site to upload to. If target site is not
            given it's taken from user-config.py.
        @type target_site: object
        @param aborts: List of the warning types to abort upload on. Set to
            True to abort on any warning.
        @type aborts: bool or list
        @param chunk_size: Upload the file in chunks (more overhead, but
            restartable) specified in bytes. If no value is specified the file
            will be uploaded as whole.
        @type chunk_size: integer
        @param filename_prefix: Specify prefix for the title of every
            file's page.
        @type filename_prefix: str
        @param always: Disables any input, requires that either ignore_warning
            or aborts are set to True and that the description is also set. It
            overwrites verify_description to False and keep_filename to True.
        @type always: bool
        """
        super().__init__(**kwargs)
        always = self.opt.always
        if always and ignore_warning is not True and aborts is not True:
            raise ValueError('When always is set to True, either '
                             'ignore_warning or aborts must be set to True.')
        if always and not description:
            raise ValueError('When always is set to True, the description '
                             'must be set.')
        self.url = url
        if isinstance(self.url, str):
            pywikibot.warning('url as string is deprecated. '
                              'Use an iterable instead.')
        self.url_encoding = url_encoding
        self.description = description
        self.use_filename = use_filename
        self.keep_filename = keep_filename or always
        self.verify_description = verify_description and not always
        self.ignore_warning = ignore_warning
        self.aborts = aborts
        self.chunk_size = chunk_size
        self.summary = summary
        self.filename_prefix = filename_prefix
        if config.upload_to_commons:
            self.target_site = target_site or pywikibot.Site('commons',
                                                             'commons')
        else:
            self.target_site = target_site or pywikibot.Site()

    def read_file_content(self, file_url=None):
        """Return name of temp file in which remote file is saved."""
        if not file_url:
            file_url = self.url
            pywikibot.warning('file_url is not given. '
                              'Set to self.url by default.')
        pywikibot.output('Reading file {}'.format(file_url))
        resume = False
        rlen = 0
        _contents = None
        dt = 15
        uo = URLopener()
        retrieved = False

        while not retrieved:
            if resume:
                pywikibot.output('Resume download...')
                uo.addheader('Range', 'bytes={}-'.format(rlen))

            with closing(uo.open(file_url)) as infile:
                info = infile.info()

                info_get = info.get
                content_type = info_get('Content-Type')
                content_len = info_get('Content-Length')
                accept_ranges = info_get('Accept-Ranges')

                if 'text/html' in content_type:
                    pywikibot.output(
                        "Couldn't download the image: "
                        'the requested URL was not found on server.')
                    return

                valid_ranges = accept_ranges == 'bytes'

                if resume:
                    _contents += infile.read()
                else:
                    _contents = infile.read()

            retrieved = True
            if content_len:
                rlen = len(_contents)
                content_len = int(content_len)
                if rlen < content_len:
                    retrieved = False
                    pywikibot.output(
                        'Connection closed at byte {} ({} left)'
                        .format(rlen, content_len))
                    if valid_ranges and rlen > 0:
                        resume = True
                    pywikibot.output('Sleeping for {} seconds...'.format(dt))
                    pywikibot.sleep(dt)
                    if dt <= 60:
                        dt += 15
                    elif dt < 360:
                        dt += 60
            else:
                pywikibot.log(
                    'WARNING: length check of retrieved data not possible.')
        handle, tempname = tempfile.mkstemp()
        with os.fdopen(handle, 'wb') as t:
            t.write(_contents)
        return tempname

    def _handle_warning(self, warning):
        """
        Return whether the warning cause an abort or be ignored.

        @param warning: The warning name
        @type warning: str
        @return: False if this warning should cause an abort, True if it should
            be ignored or None if this warning has no default handler.
        @rtype: bool or None
        """
        if self.aborts is not True:
            if warning in self.aborts:
                return False
        if self.ignore_warning is True or (self.ignore_warning is not False
                                           and warning in self.ignore_warning):
            return True
        return None if self.aborts is not True else False

    def _handle_warnings(self, warnings):
        messages = '\n'.join('{0.code}: {0.info}'.format(warning)
                             for warning in sorted(warnings,
                                                   key=lambda w: w.code))
        if len(warnings) > 1:
            messages = '\n' + messages
        pywikibot.output('We got the following warning(s): ' + messages)
        answer = True
        for warning in warnings:
            this_answer = self._handle_warning(warning.code)
            if this_answer is False:
                answer = False
                break
            elif this_answer is None:
                answer = None
        if answer is None:
            answer = pywikibot.input_yn('Do you want to ignore?',
                                        default=False, automatic_quit=False)
        return answer

    def process_filename(self, file_url=None):
        """Return base filename portion of file_url."""
        if not file_url:
            file_url = self.url
            pywikibot.warning('file_url is not given. '
                              'Set to self.url by default.')

        always = self.opt.always
        # Isolate the pure name
        filename = file_url
        # Filename may be either a URL or a local file path
        if '://' in filename:
            # extract the path portion of the URL
            filename = urlparse(filename).path
        filename = os.path.basename(filename)
        if self.use_filename:
            filename = self.use_filename
        if self.filename_prefix:
            filename = self.filename_prefix + filename
        if not self.keep_filename:
            pywikibot.output(
                'The filename on the target wiki will default to: %s'
                % filename)
            assert not always
            newfn = pywikibot.input(
                'Enter a better name, or press enter to accept:')
            if newfn != '':
                filename = newfn
        # FIXME: these 2 belong somewhere else, presumably in family
        # forbidden characters are handled by pywikibot/page.py
        forbidden = ':*?/\\'  # to be extended
        try:
            allowed_formats = self.target_site.siteinfo.get(
                'fileextensions', get_default=False)
        except KeyError:
            allowed_formats = []
        else:
            allowed_formats = [item['ext'] for item in allowed_formats]

        # ask until it's valid
        first_check = True
        while True:
            if not first_check:
                if always:
                    filename = None
                else:
                    filename = pywikibot.input('Enter a better name, or press '
                                               'enter to skip the file:')
                if not filename:
                    return None
            first_check = False
            ext = os.path.splitext(filename)[1].lower().strip('.')
            # are any chars in forbidden also in filename?
            invalid = set(forbidden) & set(filename)
            if invalid:
                c = ''.join(invalid)
                pywikibot.output(
                    'Invalid character(s): %s. Please try again' % c)
                continue
            if allowed_formats and ext not in allowed_formats:
                if always:
                    pywikibot.output('File format is not one of '
                                     '[{0}]'.format(' '.join(allowed_formats)))
                    continue
                elif not pywikibot.input_yn(
                        'File format is not one of [%s], but %s. Continue?'
                        % (' '.join(allowed_formats), ext),
                        default=False, automatic_quit=False):
                    continue
            potential_file_page = pywikibot.FilePage(self.target_site,
                                                     filename)
            if potential_file_page.exists():
                overwrite = self._handle_warning('exists')
                if overwrite is False:
                    pywikibot.output(
                        'File exists and you asked to abort. Skipping.')
                    return None
                if potential_file_page.has_permission():
                    if overwrite is None:
                        overwrite = not pywikibot.input_yn(
                            'File with name %s already exists. '
                            'Would you like to change the name? '
                            '(Otherwise file will be overwritten.)'
                            % filename, default=True,
                            automatic_quit=False)
                    if not overwrite:
                        continue
                    else:
                        break
                else:
                    pywikibot.output('File with name %s already exists and '
                                     'cannot be overwritten.' % filename)
                    continue
            else:
                try:
                    if potential_file_page.file_is_shared():
                        pywikibot.output(
                            'File with name %s already exists in shared '
                            'repository and cannot be overwritten.' % filename)
                        continue
                    else:
                        break
                except pywikibot.NoPage:
                    break

        # A proper description for the submission.
        # Empty descriptions are not accepted.
        if self.description:
            pywikibot.output('The suggested description is:\n%s'
                             % self.description)

        while not self.description or self.verify_description:
            if not self.description:
                pywikibot.output(color_format(
                    '{lightred}It is not possible to upload a file '
                    'without a description.{default}'))
            assert not always
            # if no description, ask if user want to add one or quit,
            # and loop until one is filled.
            # if self.verify_description, ask if user want to change it
            # or continue.
            if self.description:
                question = 'Do you want to change this description?'
            else:
                question = 'No description was given. Add one?'
            if pywikibot.input_yn(question, default=not self.description,
                                  automatic_quit=self.description):
                from pywikibot import editor as editarticle
                editor = editarticle.TextEditor()
                try:
                    new_description = editor.edit(self.description)
                except ImportError:
                    raise
                except Exception as e:
                    pywikibot.error(e)
                    continue
                # if user saved / didn't press Cancel
                if new_description:
                    self.description = new_description
            elif not self.description:
                raise QuitKeyboardInterrupt
            self.verify_description = False

        return filename

    def abort_on_warn(self, warn_code):
        """Determine if the warning message should cause an abort."""
        if self.aborts is True:
            return True
        else:
            return warn_code in self.aborts

    def ignore_on_warn(self, warn_code):
        """
        Determine if the warning message should be ignored.

        @param warn_code: The warning message
        @type warn_code: str
        """
        if self.ignore_warning is True:
            return True
        else:
            return warn_code in self.ignore_warning

    def upload_file(self, file_url, _file_key=None, _offset=0):
        """
        Upload the image at file_url to the target wiki.

        @see: U{https://www.mediawiki.org/wiki/API:Upload}

        Return the filename that was used to upload the image.
        If the upload fails, ask the user whether to try again or not.
        If the user chooses not to retry, return None.
        """
        filename = self.process_filename(file_url)
        if not filename:
            return None

        site = self.target_site
        imagepage = pywikibot.FilePage(site, filename)  # normalizes filename
        imagepage.text = self.description

        pywikibot.output('Uploading file to {0}...'.format(site))

        ignore_warnings = self.ignore_warning is True or self._handle_warnings
        if '://' in file_url and not site.has_right('upload_by_url'):
            file_url = self.read_file_content(file_url)

        try:
            success = imagepage.upload(file_url,
                                       ignore_warnings=ignore_warnings,
                                       chunk_size=self.chunk_size,
                                       _file_key=_file_key, _offset=_offset,
                                       comment=self.summary)
        except pywikibot.data.api.APIError as error:
            if error.code == 'uploaddisabled':
                pywikibot.error(
                    'Upload error: Local file uploads are disabled on %s.'
                    % site)
            else:
                pywikibot.error('Upload error: ', exc_info=True)
        except Exception:
            pywikibot.error('Upload error: ', exc_info=True)
        else:
            if success:
                # No warning, upload complete.
                pywikibot.output('Upload of %s successful.' % filename)
                self._save_counter += 1
                return filename  # data['filename']
            else:
                pywikibot.output('Upload aborted.')
        return None

    def run(self):
        """Run bot."""
        # early check that upload is enabled
        if self.target_site.is_uploaddisabled():
            pywikibot.error(
                'Upload error: Local file uploads are disabled on %s.'
                % self.target_site)
            return

        # early check that user has proper rights to upload
        self.target_site.login()
        if not self.target_site.has_right('upload'):
            pywikibot.error(
                "User '%s' does not have upload rights on site %s."
                % (self.target_site.user(), self.target_site))
            return

        try:
            if isinstance(self.url, str):
                self._treat_counter = 1
                return self.upload_file(self.url)
            for file_url in self.url:
                self.upload_file(file_url)
                self._treat_counter += 1
        except QuitKeyboardInterrupt:
            pywikibot.output('\nUser quit %s bot run...' %
                             self.__class__.__name__)
        except KeyboardInterrupt:
            if config.verbose_output:
                raise
            else:
                pywikibot.output('\nKeyboardInterrupt during %s bot run...' %
                                 self.__class__.__name__)
        finally:
            self.exit()
<commit_msg>[IMPR] Replace deprecated urllib.request.URLopener<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""Special bot library containing UploadRobot.

Do not import classes directly from here but from specialbots.
"""
#
# (C) Pywikibot team, 2003-2020
#
# Distributed under the terms of the MIT license.
#
import os
import requests
import tempfile

from pathlib import Path
from urllib.parse import urlparse

import pywikibot
import pywikibot.comms.http as http
import pywikibot.data.api

from pywikibot import config
from pywikibot.bot import BaseBot, QuitKeyboardInterrupt
from pywikibot.exceptions import FatalServerError
from pywikibot.tools import deprecated_args
from pywikibot.tools.formatter import color_format

class UploadRobot(BaseBot):

    """Upload bot."""

    @deprecated_args(urlEncoding='url_encoding', useFilename='use_filename',
                     keepFilename='keep_filename',
                     verifyDescription='verify_description',
                     ignoreWarning='ignore_warning', targetSite='target_site')
    def __init__(self, url: list, url_encoding=None, description='',
                 use_filename=None, keep_filename=False,
                 verify_description=True, ignore_warning=False,
                 target_site=None, aborts=[], chunk_size=0, summary=None,
                 filename_prefix=None, **kwargs):
        """
        Initializer.

        @param url: path to url or local file (deprecated), or list of urls or
            paths to local files.
        @param description: Description of file for its page. If multiple files
            are uploading the same description is used for every file.
        @type description: str
        @param use_filename: Specify title of the file's page. If multiple
            files are uploading it asks to change the name for second, third,
            etc. files, otherwise the last file will overwrite the other.
        @type use_filename: str
        @param keep_filename: Set to True to keep original names of urls and
            files, otherwise it will ask to enter a name for each file.
        @type keep_filename: bool
        @param summary: Summary of the upload
        @type summary: str
        @param verify_description: Set to True to proofread the description.
        @type verify_description: bool
        @param ignore_warning: Set this to True to upload even if another file
            would be overwritten or another mistake would be risked. Set it to
            an array of warning codes to selectively ignore specific warnings.
        @type ignore_warning: bool or list
        @param target_site: Set the site to upload to. If target site is not
            given it's taken from user-config.py.
        @type target_site: object
        @param aborts: List of the warning types to abort upload on. Set to
            True to abort on any warning.
        @type aborts: bool or list
        @param chunk_size: Upload the file in chunks (more overhead, but
            restartable) specified in bytes. If no value is specified the file
            will be uploaded as whole.
        @type chunk_size: integer
        @param filename_prefix: Specify prefix for the title of every
            file's page.
        @type filename_prefix: str
        @param always: Disables any input, requires that either ignore_warning
            or aborts are set to True and that the description is also set. It
            overwrites verify_description to False and keep_filename to True.
        @type always: bool
        """
        super().__init__(**kwargs)
        always = self.opt.always
        if always and ignore_warning is not True and aborts is not True:
            raise ValueError('When always is set to True, either '
                             'ignore_warning or aborts must be set to True.')
        if always and not description:
            raise ValueError('When always is set to True, the description '
                             'must be set.')
        self.url = url
        if isinstance(self.url, str):
            pywikibot.warning('url as string is deprecated. '
                              'Use an iterable instead.')
        self.url_encoding = url_encoding
        self.description = description
        self.use_filename = use_filename
        self.keep_filename = keep_filename or always
        self.verify_description = verify_description and not always
        self.ignore_warning = ignore_warning
        self.aborts = aborts
        self.chunk_size = chunk_size
        self.summary = summary
        self.filename_prefix = filename_prefix
        if config.upload_to_commons:
            self.target_site = target_site or pywikibot.Site('commons',
                                                             'commons')
        else:
            self.target_site = target_site or pywikibot.Site()

    def read_file_content(self, file_url=None):
        """Return name of temp file in which remote file is saved."""
        if not file_url:
            file_url = self.url
            pywikibot.warning('file_url is not given. '
                              'Set to self.url by default.')
        pywikibot.output('Reading file {}'.format(file_url))

        handle, tempname = tempfile.mkstemp()
        path = Path(tempname)
        size = 0

        dt_gen = (el for el in (15, 30, 45, 60, 120, 180, 240, 300))
        while True:
            file_len = path.stat().st_size
            if file_len:
                pywikibot.output('Download resumed.')
                headers = {'Range': 'bytes={}-'.format(file_len)}
            else:
                headers = {}

            with open(path, 'ab') as fd:
                os.lseek(handle, file_len, 0)
                try:
                    r = http.fetch(file_url, stream=True, headers=headers)
                    response = r.data
                    response.raise_for_status()

                    # get download info, if available
                    # Note: this is not enough to exclude pages
                    #       e.g. 'application/json' is also not a media
                    if 'text/' in response.headers['Content-Type']:
                        raise FatalServerError('The requested URL was not '
                                               'found on server.')
                    size = max(size,
                               int(response.headers.get('Content-Length', 0)))

                    # stream content to temp file (in chunks of 1Mb)
                    for chunk in response.iter_content(chunk_size=1024 * 1024):
                        fd.write(chunk)

                # raised from connection lost during response.iter_content()
                except requests.ConnectionError:
                    fd.flush()
                    pywikibot.output(
                        'Connection closed at byte %s' % path.stat().st_size)
                # raised from response.raise_for_status()
                except requests.HTTPError as e:
                    # exit criteria if size is not available
                    # error on last iteration is OK, we're requesting
                    #    {'Range': 'bytes=file_len-'}
                    if response.status_code == 416 and path.stat().st_size:
                        break
                    else:
                        raise FatalServerError(str(e)) from e

            if size and size == path.stat().st_size:
                break
            try:
                dt = next(dt_gen)
                pywikibot.output('Sleeping for {} seconds ...'.format(dt))
                pywikibot.sleep(dt)
            except StopIteration:
                raise FatalServerError('Download failed, too many retries!')

        pywikibot.output('Downloaded {} bytes'.format(path.stat().st_size))
        return tempname

    def _handle_warning(self, warning):
        """
        Return whether the warning cause an abort or be ignored.

        @param warning: The warning name
        @type warning: str
        @return: False if this warning should cause an abort, True if it should
            be ignored or None if this warning has no default handler.
        @rtype: bool or None
        """
        if self.aborts is not True:
            if warning in self.aborts:
                return False
        if self.ignore_warning is True or (self.ignore_warning is not False
                                           and warning in self.ignore_warning):
            return True
        return None if self.aborts is not True else False

    def _handle_warnings(self, warnings):
        messages = '\n'.join('{0.code}: {0.info}'.format(warning)
                             for warning in sorted(warnings,
                                                   key=lambda w: w.code))
        if len(warnings) > 1:
            messages = '\n' + messages
        pywikibot.output('We got the following warning(s): ' + messages)
        answer = True
        for warning in warnings:
            this_answer = self._handle_warning(warning.code)
            if this_answer is False:
                answer = False
                break
            elif this_answer is None:
                answer = None
        if answer is None:
            answer = pywikibot.input_yn('Do you want to ignore?',
                                        default=False, automatic_quit=False)
        return answer

    def process_filename(self, file_url=None):
        """Return base filename portion of file_url."""
        if not file_url:
            file_url = self.url
            pywikibot.warning('file_url is not given. '
                              'Set to self.url by default.')

        always = self.opt.always
        # Isolate the pure name
        filename = file_url
        # Filename may be either a URL or a local file path
        if '://' in filename:
            # extract the path portion of the URL
            filename = urlparse(filename).path
        filename = os.path.basename(filename)
        if self.use_filename:
            filename = self.use_filename
        if self.filename_prefix:
            filename = self.filename_prefix + filename
        if not self.keep_filename:
            pywikibot.output(
                'The filename on the target wiki will default to: %s'
                % filename)
            assert not always
            newfn = pywikibot.input(
                'Enter a better name, or press enter to accept:')
            if newfn != '':
                filename = newfn
        # FIXME: these 2 belong somewhere else, presumably in family
        # forbidden characters are handled by pywikibot/page.py
        forbidden = ':*?/\\'  # to be extended
        try:
            allowed_formats = self.target_site.siteinfo.get(
                'fileextensions', get_default=False)
        except KeyError:
            allowed_formats = []
        else:
            allowed_formats = [item['ext'] for item in allowed_formats]

        # ask until it's valid
        first_check = True
        while True:
            if not first_check:
                if always:
                    filename = None
                else:
                    filename = pywikibot.input('Enter a better name, or press '
                                               'enter to skip the file:')
                if not filename:
                    return None
            first_check = False
            ext = os.path.splitext(filename)[1].lower().strip('.')
            # are any chars in forbidden also in filename?
            invalid = set(forbidden) & set(filename)
            if invalid:
                c = ''.join(invalid)
                pywikibot.output(
                    'Invalid character(s): %s. Please try again' % c)
                continue
            if allowed_formats and ext not in allowed_formats:
                if always:
                    pywikibot.output('File format is not one of '
                                     '[{0}]'.format(' '.join(allowed_formats)))
                    continue
                elif not pywikibot.input_yn(
                        'File format is not one of [%s], but %s. Continue?'
                        % (' '.join(allowed_formats), ext),
                        default=False, automatic_quit=False):
                    continue
            potential_file_page = pywikibot.FilePage(self.target_site,
                                                     filename)
            if potential_file_page.exists():
                overwrite = self._handle_warning('exists')
                if overwrite is False:
                    pywikibot.output(
                        'File exists and you asked to abort. Skipping.')
                    return None
                if potential_file_page.has_permission():
                    if overwrite is None:
                        overwrite = not pywikibot.input_yn(
                            'File with name %s already exists. '
                            'Would you like to change the name? '
                            '(Otherwise file will be overwritten.)'
                            % filename, default=True,
                            automatic_quit=False)
                    if not overwrite:
                        continue
                    else:
                        break
                else:
                    pywikibot.output('File with name %s already exists and '
                                     'cannot be overwritten.' % filename)
                    continue
            else:
                try:
                    if potential_file_page.file_is_shared():
                        pywikibot.output(
                            'File with name %s already exists in shared '
                            'repository and cannot be overwritten.' % filename)
                        continue
                    else:
                        break
                except pywikibot.NoPage:
                    break

        # A proper description for the submission.
        # Empty descriptions are not accepted.
        if self.description:
            pywikibot.output('The suggested description is:\n%s'
                             % self.description)

        while not self.description or self.verify_description:
            if not self.description:
                pywikibot.output(color_format(
                    '{lightred}It is not possible to upload a file '
                    'without a description.{default}'))
            assert not always
            # if no description, ask if user want to add one or quit,
            # and loop until one is filled.
            # if self.verify_description, ask if user want to change it
            # or continue.
            if self.description:
                question = 'Do you want to change this description?'
            else:
                question = 'No description was given. Add one?'
            if pywikibot.input_yn(question, default=not self.description,
                                  automatic_quit=self.description):
                from pywikibot import editor as editarticle
                editor = editarticle.TextEditor()
                try:
                    new_description = editor.edit(self.description)
                except ImportError:
                    raise
                except Exception as e:
                    pywikibot.error(e)
                    continue
                # if user saved / didn't press Cancel
                if new_description:
                    self.description = new_description
            elif not self.description:
                raise QuitKeyboardInterrupt
            self.verify_description = False

        return filename

    def abort_on_warn(self, warn_code):
        """Determine if the warning message should cause an abort."""
        if self.aborts is True:
            return True
        else:
            return warn_code in self.aborts

    def ignore_on_warn(self, warn_code):
        """
        Determine if the warning message should be ignored.

        @param warn_code: The warning message
        @type warn_code: str
        """
        if self.ignore_warning is True:
            return True
        else:
            return warn_code in self.ignore_warning

    def upload_file(self, file_url, _file_key=None, _offset=0):
        """
        Upload the image at file_url to the target wiki.

        @see: U{https://www.mediawiki.org/wiki/API:Upload}

        Return the filename that was used to upload the image.
        If the upload fails, ask the user whether to try again or not.
        If the user chooses not to retry, return None.
        """
        filename = self.process_filename(file_url)
        if not filename:
            return None

        site = self.target_site
        imagepage = pywikibot.FilePage(site, filename)  # normalizes filename
        imagepage.text = self.description

        pywikibot.output('Uploading file to {0}...'.format(site))

        ignore_warnings = self.ignore_warning is True or self._handle_warnings
        if '://' in file_url and not site.has_right('upload_by_url'):
            try:
                file_url = self.read_file_content(file_url)
            except FatalServerError:
                return None

        try:
            success = imagepage.upload(file_url,
                                       ignore_warnings=ignore_warnings,
                                       chunk_size=self.chunk_size,
                                       _file_key=_file_key, _offset=_offset,
                                       comment=self.summary)
        except pywikibot.data.api.APIError as error:
            if error.code == 'uploaddisabled':
                pywikibot.error(
                    'Upload error: Local file uploads are disabled on %s.'
                    % site)
            else:
                pywikibot.error('Upload error: ', exc_info=True)
        except Exception:
            pywikibot.error('Upload error: ', exc_info=True)
        else:
            if success:
                # No warning, upload complete.
                pywikibot.output('Upload of %s successful.' % filename)
                self._save_counter += 1
                return filename  # data['filename']
            else:
                pywikibot.output('Upload aborted.')
        return None

    def run(self):
        """Run bot."""
        # early check that upload is enabled
        if self.target_site.is_uploaddisabled():
            pywikibot.error(
                'Upload error: Local file uploads are disabled on %s.'
                % self.target_site)
            return

        # early check that user has proper rights to upload
        self.target_site.login()
        if not self.target_site.has_right('upload'):
            pywikibot.error(
                "User '%s' does not have upload rights on site %s."
                % (self.target_site.user(), self.target_site))
            return

        try:
            if isinstance(self.url, str):
                self._treat_counter = 1
                return self.upload_file(self.url)
            for file_url in self.url:
                self.upload_file(file_url)
                self._treat_counter += 1
        except QuitKeyboardInterrupt:
            pywikibot.output('\nUser quit %s bot run...' %
                             self.__class__.__name__)
        except KeyboardInterrupt:
            if config.verbose_output:
                raise
            else:
                pywikibot.output('\nKeyboardInterrupt during %s bot run...' %
                                 self.__class__.__name__)
        finally:
            self.exit()
<|endoftext|>


<commit_before># GUI Application automation and testing library
# Copyright (C) 2006 Mark Mc Mahon
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public License
# as published by the Free Software Foundation; either version 2.1
# of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
#    Free Software Foundation, Inc.,
#    59 Temple Place,
#    Suite 330,
#    Boston, MA 02111-1307 USA

"Collect the wrapping classes and respond to request to wrap handles"

import re

import win32_controls
import common_controls
from pywinauto import handleprops

__revision__ = "$Revision $"

#====================================================================
# get all the classes/functions from win32_contols
_all_classes = win32_controls.__dict__.values()

# and add all the classes/functinos from common_controls
_all_classes.extend(common_controls.__dict__.values())

_wrapper_info = {}
_wrapper_cache = {}

for item in _all_classes:
    try:
        for classname_ in item.windowclasses:
            # set the info
            _wrapper_info[classname_] = (
                re.compile(classname_),
                item)

            _wrapper_cache[classname_] = item

    except AttributeError, e:
        pass

def _find_wrapper(class_name):
    """return the wrapper that handles this classname

    If there is no match found then return None.
    """

    try:

        # Optimization - return the item from the cache if it exists
        # in the cache
        return _wrapper_cache[class_name]

        # Optimization - then check if the control name matches
        # exactly before trying a re.match (and add it to the cache)
        #if classname in _wrapper_info:
        #    _wrapper_cache[classname] = _wrapper_info[classname][1]
        #    return _wrapper_info[classname][1]
    except KeyError:

        wrapper_match = None

        for regex, wrapper in _wrapper_info.values():
            if regex.match(class_name):

                wrapper_match = wrapper

    # save this classname - wrapper match for later
    # There are not so many classnames registerd usually so this saves
    # time
    _wrapper_cache[class_name] = wrapper_match

    # return the wrapper that matched
    return wrapper_match

#====================================================================
def WrapHandle(hwnd):
    """Return the hwnd wrapped with  the correct wraper

    Wrapper is chosen on the Class of the control
    """
    from HwndWrapper import HwndWrapper

    class_name = handleprops.classname(hwnd)

    _needs_image = False

    wrapper = _find_wrapper(class_name)

    if wrapper is None:
        # yet another optimization to not call this function too often
        is_top_level_win = handleprops.is_toplevel_window(hwnd)

        if is_top_level_win:
            wrapper = win32_controls.DialogWrapper
        else:
            wrapper = HwndWrapper

        if not is_top_level_win:
            _needs_image = True

    wrapped_hwnd = wrapper(hwnd)
    if _needs_image:
        wrapped_hwnd._NeedsImageProp = True

    return wrapped_hwnd
<commit_msg>Catch a key error - so that Pudge can still generate the documentation<commit_after># GUI Application automation and testing library
# Copyright (C) 2006 Mark Mc Mahon
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public License
# as published by the Free Software Foundation; either version 2.1
# of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
#    Free Software Foundation, Inc.,
#    59 Temple Place,
#    Suite 330,
#    Boston, MA 02111-1307 USA

"Collect the wrapping classes and respond to request to wrap handles"

import re

import win32_controls
import common_controls
from pywinauto import handleprops

__revision__ = "$Revision $"

#====================================================================
# get all the classes/functions from win32_contols
_all_classes = win32_controls.__dict__.values()

# and add all the classes/functinos from common_controls
_all_classes.extend(common_controls.__dict__.values())

_wrapper_info = {}
_wrapper_cache = {}

for item in _all_classes:
    try:
        for classname_ in item.windowclasses:
            # set the info
            _wrapper_info[classname_] = (
                re.compile(classname_),
                item)

            _wrapper_cache[classname_] = item

    # needed to add KeyError so that Pudge could
    # generate the documentation
    except (KeyError, AttributeError), e:
        pass

def _find_wrapper(class_name):
    """return the wrapper that handles this classname

    If there is no match found then return None.
    """

    try:

        # Optimization - return the item from the cache if it exists
        # in the cache
        return _wrapper_cache[class_name]

        # Optimization - then check if the control name matches
        # exactly before trying a re.match (and add it to the cache)
        #if classname in _wrapper_info:
        #    _wrapper_cache[classname] = _wrapper_info[classname][1]
        #    return _wrapper_info[classname][1]
    except KeyError:

        wrapper_match = None

        for regex, wrapper in _wrapper_info.values():
            if regex.match(class_name):

                wrapper_match = wrapper

    # save this classname - wrapper match for later
    # There are not so many classnames registerd usually so this saves
    # time
    _wrapper_cache[class_name] = wrapper_match

    # return the wrapper that matched
    return wrapper_match

#====================================================================
def WrapHandle(hwnd):
    """Return the hwnd wrapped with  the correct wraper

    Wrapper is chosen on the Class of the control
    """
    from HwndWrapper import HwndWrapper

    class_name = handleprops.classname(hwnd)

    _needs_image = False

    wrapper = _find_wrapper(class_name)

    if wrapper is None:
        # yet another optimization to not call this function too often
        is_top_level_win = handleprops.is_toplevel_window(hwnd)

        if is_top_level_win:
            wrapper = win32_controls.DialogWrapper
        else:
            wrapper = HwndWrapper

        if not is_top_level_win:
            _needs_image = True

    wrapped_hwnd = wrapper(hwnd)
    if _needs_image:
        wrapped_hwnd._NeedsImageProp = True

    return wrapped_hwnd
<|endoftext|>


<commit_before>"""
odk_validate.py
A python wrapper around ODK Validate
"""
import os
import re
import sys
from subprocess import Popen, PIPE
import threading
import signal

CURRENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
ODK_VALIDATE_JAR = os.path.join(CURRENT_DIRECTORY, "ODK_Validate.jar")

#Adapted from:
#http://betabug.ch/blogs/ch-athens/1093
def run_popen_with_timeout(command, timeout):
    """
    Run a sub-program in subprocess.Popen, pass it the input_data,
    kill it if the specified timeout has passed.
    returns a tuple of resultcode, timeout, stdout, stderr
    """
    kill_check = threading.Event()

    def _kill_process_after_a_timeout(pid):
        os.kill(pid, signal.SIGTERM)
        kill_check.set()  # tell the main routine that we had to kill
        # use SIGKILL if hard to kill...
        return
    p = Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
    pid = p.pid
    watchdog = threading.Timer(
        timeout, _kill_process_after_a_timeout, args=(pid, ))
    watchdog.start()
    (stdout, stderr) = p.communicate()
    watchdog.cancel()  # if it's still waiting to run
    timeout = kill_check.isSet()
    kill_check.clear()
    return (p.returncode, timeout, stdout, stderr)

def _cleanup_errors(error_message):
    def get_last_item(xpathStr):
        l = xpathStr.split("/")
        return l[len(l) - 1]

    def replace_function(match):
        strmatch = match.group()
        # eliminate e.g /html/body/select1[@ref=/id_string/elId]/item/value
        # instance('q4')/root/item[...]
        if strmatch.startswith("/html/body") \
                or strmatch.startswith("/root/item") \
                or strmatch.endswith("/item/value"):
            return strmatch
        return "${%s}" % get_last_item(match.group())
    pattern = "(/[a-z0-9\-_]+(?:/[a-z0-9\-_]+)+)"
    error_message = re.sub(pattern, replace_function, error_message, flags=re.I)
    k = []
    for line in error_message.splitlines():
        # has a java filename
        has_java_filename = line.find('.java') is not -1
        # starts with '    at java class path or method path'
        is_a_java_method = line.find('\tat') is not -1
        if not has_java_filename and not is_a_java_method:
            # remove java.lang.RuntimeException
            if line.startswith('java.lang.RuntimeException: '):
                line = line.replace('java.lang.RuntimeException: ', '')
            # remove java.lang.NullPointerException
            if line.startswith('java.lang.NullPointerException'):
                continue
            k.append(line)
    errStr = u'\n'.join(k)
    return errStr

def check_xform(path_to_xform):
    """
    Returns an array of warnings if the form is valid.
    Throws an exception if it is not
    """
    #resultcode indicates validity of the form
    #timeout indicates whether validation ran out of time to complete
    #stdout is not used because it has some warnings that always
    #appear and can be ignored.
    #stderr is treated as a warning if the form is valid or an error
    #if it is invalid.
    returncode, timeout, stdout, stderr = run_popen_with_timeout(
        ["java", "-jar", ODK_VALIDATE_JAR, path_to_xform], 100)
    warnings = []

    if timeout:
        return ["XForm took to long to completely validate."]
    else:
        if returncode > 0:  # Error invalid
            raise Exception(
                'ODK Validate Errors:\n' + _cleanup_errors(stderr))
        elif returncode == 0:
            if stderr:
                warnings.append('ODK Validate Warnings:\n' + stderr)
            return warnings
        elif returncode < 0:
            return ["Bad return code from ODK Validate."]

if __name__ == '__main__':
    print __doc__
    check_xform(sys.argv[1])
<commit_msg>pep8 cleanup<commit_after>"""
odk_validate.py
A python wrapper around ODK Validate
"""
import os
import re
import sys
from subprocess import Popen, PIPE
import threading
import signal

CURRENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
ODK_VALIDATE_JAR = os.path.join(CURRENT_DIRECTORY, "ODK_Validate.jar")

#Adapted from:
#http://betabug.ch/blogs/ch-athens/1093
def run_popen_with_timeout(command, timeout):
    """
    Run a sub-program in subprocess.Popen, pass it the input_data,
    kill it if the specified timeout has passed.
    returns a tuple of resultcode, timeout, stdout, stderr
    """
    kill_check = threading.Event()

    def _kill_process_after_a_timeout(pid):
        os.kill(pid, signal.SIGTERM)
        kill_check.set()  # tell the main routine that we had to kill
        # use SIGKILL if hard to kill...
        return
    p = Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE)
    pid = p.pid
    watchdog = threading.Timer(
        timeout, _kill_process_after_a_timeout, args=(pid, ))
    watchdog.start()
    (stdout, stderr) = p.communicate()
    watchdog.cancel()  # if it's still waiting to run
    timeout = kill_check.isSet()
    kill_check.clear()
    return (p.returncode, timeout, stdout, stderr)

def _cleanup_errors(error_message):
    def get_last_item(xpathStr):
        l = xpathStr.split("/")
        return l[len(l) - 1]

    def replace_function(match):
        strmatch = match.group()
        # eliminate e.g /html/body/select1[@ref=/id_string/elId]/item/value
        # instance('q4')/root/item[...]
        if strmatch.startswith("/html/body") \
                or strmatch.startswith("/root/item") \
                or strmatch.endswith("/item/value"):
            return strmatch
        return "${%s}" % get_last_item(match.group())
    pattern = "(/[a-z0-9\-_]+(?:/[a-z0-9\-_]+)+)"
    error_message = re.sub(
        pattern, replace_function, error_message, flags=re.I)
    k = []
    for line in error_message.splitlines():
        # has a java filename
        has_java_filename = line.find('.java') is not -1
        # starts with '    at java class path or method path'
        is_a_java_method = line.find('\tat') is not -1
        if not has_java_filename and not is_a_java_method:
            # remove java.lang.RuntimeException
            if line.startswith('java.lang.RuntimeException: '):
                line = line.replace('java.lang.RuntimeException: ', '')
            # remove java.lang.NullPointerException
            if line.startswith('java.lang.NullPointerException'):
                continue
            k.append(line)
    errStr = u'\n'.join(k)
    return errStr

def check_xform(path_to_xform):
    """
    Returns an array of warnings if the form is valid.
    Throws an exception if it is not
    """
    #resultcode indicates validity of the form
    #timeout indicates whether validation ran out of time to complete
    #stdout is not used because it has some warnings that always
    #appear and can be ignored.
    #stderr is treated as a warning if the form is valid or an error
    #if it is invalid.
    returncode, timeout, stdout, stderr = run_popen_with_timeout(
        ["java", "-jar", ODK_VALIDATE_JAR, path_to_xform], 100)
    warnings = []

    if timeout:
        return ["XForm took to long to completely validate."]
    else:
        if returncode > 0:  # Error invalid
            raise Exception(
                'ODK Validate Errors:\n' + _cleanup_errors(stderr))
        elif returncode == 0:
            if stderr:
                warnings.append('ODK Validate Warnings:\n' + stderr)
            return warnings
        elif returncode < 0:
            return ["Bad return code from ODK Validate."]

if __name__ == '__main__':
    print __doc__
    check_xform(sys.argv[1])
<|endoftext|>


<commit_before>from collections import namedtuple
from .analyzer import Analyzer, HtcHeader
from .wmi_ctrl_analyzer import WmiCtrlAnalyzer
from .htt_analyzer import HttAnalyzer

class AllAnalyzer(Analyzer):

    def __init__(self, wmi_ctrl_eid=1, htt_eid=2, short_htc_hdr=False,
                 wmi_unified=True, timestamps=False):

        Analyzer.__init__(self,
                          short_htc_hdr=short_htc_hdr,
                          timestamps=timestamps)

        self.wmi_ctrl_analyzer = WmiCtrlAnalyzer(eid=wmi_ctrl_eid,
                                                 short_htc_hdr=short_htc_hdr,
                                                 wmi_unified=wmi_unified,
                                                 timestamps=timestamps)

        self.htt_analyzer = HttAnalyzer(eid=htt_eid,
                                        short_htc_hdr=short_htc_hdr,
                                        timestamps=timestamps)

        self.cur_analyzer = None

        self.wmi_ctrl_eid = wmi_ctrl_eid
        self.htt_eid = htt_eid

        self.wmi_hdr = None
        self.wmi_cmd_enum = None
        self.wmi_evt_enum = None
        self.h2t_enum = None
        self.t2h_enum = None
        self.htt_id = None

    def __set_cur_analyzer(self, hexdata):

        hexdata_a = hexdata.split(' ', 15)
        eid = int(hexdata_a[0], 16)
        if eid == self.wmi_ctrl_eid:
            self.cur_analyzer = self.wmi_ctrl_analyzer
        elif eid == self.htt_eid:
            self.cur_analyzer = self.htt_analyzer
        else:
            self.cur_analyzer = None

    def parse_hexdata(self, hexdata):

        (ts, hexdata_no_ts) = self.parse_timestamp(hexdata)

        hexdata_split1 = hexdata_no_ts.split(': ', 1)
        addr = int(hexdata_split1[0], 16)
        if addr == 0:
            # Address = 0 means a new msg and thus, a new HTC
            # header. We use the eid from the HTC header to
            # decide which of our analyzers we should use
            self.__set_cur_analyzer(hexdata_split1[1])

        if self.cur_analyzer:
            return self.cur_analyzer.parse_hexdata(hexdata)
        else:
            return False

    def get_id(self):

        if not self.cur_analyzer:
            return None

        return self.cur_analyzer.get_id()

    def get_enums(self):

        if not self.cur_analyzer:
            return None

        return self.cur_analyzer.get_enums()

    def get_id_str(self):

        if not self.cur_analyzer:
            return ''

        return self.cur_analyzer.get_id_str()

    def get_data_str(self):

        if not self.cur_analyzer:
            return None

        return self.cur_analyzer.get_data_str()
<commit_msg>all_analyzer.py: removed unused variables<commit_after>from collections import namedtuple
from .analyzer import Analyzer, HtcHeader
from .wmi_ctrl_analyzer import WmiCtrlAnalyzer
from .htt_analyzer import HttAnalyzer

class AllAnalyzer(Analyzer):

    def __init__(self, wmi_ctrl_eid=1, htt_eid=2, short_htc_hdr=False,
                 wmi_unified=True, timestamps=False):

        Analyzer.__init__(self,
                          short_htc_hdr=short_htc_hdr,
                          timestamps=timestamps)

        self.wmi_ctrl_analyzer = WmiCtrlAnalyzer(eid=wmi_ctrl_eid,
                                                 short_htc_hdr=short_htc_hdr,
                                                 wmi_unified=wmi_unified,
                                                 timestamps=timestamps)

        self.htt_analyzer = HttAnalyzer(eid=htt_eid,
                                        short_htc_hdr=short_htc_hdr,
                                        timestamps=timestamps)

        self.cur_analyzer = None

        self.wmi_ctrl_eid = wmi_ctrl_eid
        self.htt_eid = htt_eid

    def __set_cur_analyzer(self, hexdata):

        hexdata_a = hexdata.split(' ', 15)
        eid = int(hexdata_a[0], 16)
        if eid == self.wmi_ctrl_eid:
            self.cur_analyzer = self.wmi_ctrl_analyzer
        elif eid == self.htt_eid:
            self.cur_analyzer = self.htt_analyzer
        else:
            self.cur_analyzer = None

    def parse_hexdata(self, hexdata):

        (ts, hexdata_no_ts) = self.parse_timestamp(hexdata)

        hexdata_split1 = hexdata_no_ts.split(': ', 1)
        addr = int(hexdata_split1[0], 16)
        if addr == 0:
            # Address = 0 means a new msg and thus, a new HTC
            # header. We use the eid from the HTC header to
            # decide which of our analyzers we should use
            self.__set_cur_analyzer(hexdata_split1[1])

        if self.cur_analyzer:
            return self.cur_analyzer.parse_hexdata(hexdata)
        else:
            return False

    def get_id(self):

        if not self.cur_analyzer:
            return None

        return self.cur_analyzer.get_id()

    def get_enums(self):

        if not self.cur_analyzer:
            return None

        return self.cur_analyzer.get_enums()

    def get_id_str(self):

        if not self.cur_analyzer:
            return ''

        return self.cur_analyzer.get_id_str()

    def get_data_str(self):

        if not self.cur_analyzer:
            return None

        return self.cur_analyzer.get_data_str()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# PYTHON_ARGCOMPLETE_OK
""" Install packages and setup home configs on fresh system. """
from __future__ import print_function
from BuildSrc import get_code
import argparse
import functools
import glob
import os
import shlex
import shutil
import subprocess
try:
    import apt
except ImportError:
    print("Don't use `debian` option.")
try:
    from argcomplete import autocomplete
except ImportError:
    def autocomplete(dummy):
        """ Dummy func. """
        pass

# Packages to install follow, broken down into categories.
MINIMUM = """ \
    vim build-essential git-core mercurial ccache fontconfig gdb lynx \
    colormake colordiff colorgcc exuberant-ctags dkms python-dev python-pip \
    p7zip-full rar zip unzip gzip lzip xz-utils liblzma-dev liblzma5 \
    htop iotop itop mytop dos2unix tree dfc \
    """

PROGRAMS = """ \
    htop iotop itop mytop pkg-config aptitude synaptic dos2unix \
    oxygen-molecule kubuntu-restricted-extras kubuntu-restricted-addons \
    brother-cups-wrapper-laser brother-cups-wrapper-laser1 \
    brother-lpr-drivers-laser brother-lpr-drivers-laser1 \
    k3b kid3 krita kolourpaint4 kchmviewer yakuake plasma-widget-quickaccess \
    kdirstat keepassx screen tmux \
    chromium-browser firefox rekonq \
    abiword baobab dvdrip easytag chm2pdf dia catdoc \
    gimp gimp-plugin-registry \
    fuse gparted quicksynergy tree dfc sharutils sharutils-doc \
    libnotify-bin libnotify-dev \
    redshift geoclue geoclue-hostip numlockx \
    samba samba-doc samba-dev samba-dbg cifs-utils wireshark libwireshark-dev \
    wondershaper vlc ffmpeg ffmpeg-doc mplayer mencoder \
    p7zip-full rar zip unzip gzip lzip xz-utils liblzma-dev liblzma5 \
    virtualbox-qt wine vagrant \
    ttf-xfree86-nonfree \
    """

KEYRINGS = """ \
    debian-keyring debian-archive-keyring gnome-keyring \
    debian-ports-archive-keyring python-gnomekeyring python-keyring \
    ubuntu-keyring \
    """

PROGRAMMING = """ \
    build-essential debianutils ubuntu-dev-tools mesa-utils openssh-server \
    automake automake1.9-doc ant ant-doc checkinstall checkbox \
    dkms docbook make-doc lynx kdiff3 kdiff3-doc patch rpm2cpio rpm \
    codeblocks kdevelop qt-sdk graphviz \
    colormake colordiff colorgcc jq nmap \
    vim vim-doc vim-gtk vim-rails vim-syntax-go vim-syntax-gtk vim-doc \
    flex flex-doc bison bison-doc exuberant-ctags sphinx-common sphinx-doc \
    bash-doc bash-builtins bashdb \
    zsh zsh-dbg zsh-dev zsh-doc zsh-lovers zshdb \
    clisp clisp-doc clisp-dev clisp-module-gdbm \
    coffeescript coffeescript-doc \
    docutils-common docutils-doc asciidoc xmlto docbook2x \
    erlang erlang-eunit \
    gdc golang golang-doc golang-src golang-codesearch-dev \
    gcc gcc-doc gcc-4.7-source libcunit1 gdb gdb-doc cgdb xxgdb ccache \
    libboost-all-dev libglm-dev libglew-dev libglfw-dev ncurses-doc \
    libncurses5-dev libncursesw5-dev libpcre3-dev zlib1g-dev libbz2-dev \
    libgnome2-dev libgnomeui-dev libgtk2.0-dev libatk1.0-dev libbonoboui2-dev \
    libcairo2-dev libx11-dev libxpm-dev libxt-dev libgmp3-dev libmpc-dev \
    libmpfr-dev libcurl4-openssl-dev libevent-dev libarchive-dev \
    libcurl4-gnutls-dev openmpi-bin openmpi-checkpoint openmpi-common \
    gfortran \
    ghc ghc-doc ghc-haddock ghc-prof haskell-debian-utils \
    haskell-devscripts haskell-doc cabal-install \
    junit junit-doc maven openjdk-7-doc openjdk-7-jdk openjdk-7-dbg \
    openjdk-7-source openjdk-7-demo icedtea-7-plugin \
    lua5.2 lua5.2-doc luadoc liblua5.2-dev \
    monodevelop-debugger-gdb monodevelop-nunit \
    nodejs nodejs-dev nodejs-legacy npm \
    perl perl-doc libperl-dev perl-modules libpadwalker-perl \
    libfile-next-perl \
    php5 php5-mysql phpunit php5-dev \
    swi-prolog swi-prolog-doc \
    python python-dev python-doc python-pip \
    python3 python3-dev python3-doc python3-pip jython jython-doc \
    pychecker pylint pep8 python-autopep8 ruby-full shunit2 \
    tcl-dev tcl-doc \
    tex-common texlive-latex-base texlive-font-utils \
    bzr bzr-builddeb bzr-doc python-bzrlib bzrtools git git-gui git-doc \
    mercurial subversion cvs \
    """

BABUN = """ \
    clisp lua perl python python3 ruby make cmake colordiff colorgcc \
    p7zip unzip zip gzip ncurses pcre mercurial bzr subversion \
    libboost-devel libboost_python-devel cppunit
    """

CABAL = "buildwrapper scion-browser hoogle terminfo happy hlint"

PY_PACKS = "argcomplete cram Pygments pytest trash-cli"

class NotSudo(Exception):
    """ Throw this if we aren't sudo but need to be. """
    pass

def make_cmd(src, dst):
    """ Generate a function helper. """
    def cmd_when_dst_empty(files, command, opts=None):
        """ When dst doesn't have file do:
                command(src + file, dst + file, *opts)
        """
        if opts is None:
            opts = ()
        for fil in files:
            sfile, dfile = [x + fil for x in (src, dst)]
            if not os.path.exists(dfile):
                print("{0} >>>>> {1}".format(dfile, sfile))
                command(sfile, dfile, *opts)
    return cmd_when_dst_empty

def home_config():
    """ Setup the dev environment, stuff goes in the user's home folder. """
    script_path = os.path.relpath(__file__)
    script_dir = os.path.dirname(script_path)
    if script_dir == '':
        script_dir = '.'

    # Leaving trailing sep for ease later.
    src = script_dir + os.sep + 'dot_files' + os.sep
    home = os.path.expanduser('~') + os.sep

    # Get shell utilities
    shell_dir = home + '.shell' + os.sep
    get_code('https://bitbucket.org/sjl/hg-prompt/', shell_dir + '.hg-prompt')

    git_urls = [
        'https://github.com/magicmonty/bash-git-prompt.git',
        'https://github.com/starcraftman/zsh-git-prompt.git',
        'https://github.com/zsh-users/zsh-completions.git',
        'https://github.com/zsh-users/zsh-syntax-highlighting.git',
        'https://github.com/starcraftman/hhighlighter.git',
    ]

    for url in git_urls:
        target = '.' + url[url.rindex('/')+1:url.rindex('.git')]
        get_code(url, shell_dir + target)

    # Setup powerline fonts if not done.
    ddir = home + '.fonts'
    dpow = ddir + os.sep + 'powerline-fonts'
    if not os.path.exists(ddir) and subprocess.call(['which', 'fc-cache']) == 0:
        os.mkdir(ddir)
        get_code('https://github.com/Lokaltog/powerline-fonts', dpow)
        subprocess.call(['fc-cache', '-vf', ddir])

    # Create dir for ccache
    ddir = home + '.ccache'
    if not os.path.exists(ddir):
        os.mkdir(ddir)

    # Glob all files in dot_files and link them to home
    files = glob.glob(src + '.*')
    files = [os.path.basename(x) for x in files]
    helper = make_cmd(src, home)
    helper(files, os.symlink)

    print("NOTE: Remember to add user to smb.\nsudo smbpasswd -a username")

def home_restore():
    """ Undo changes by home_config & restore backup if exists. """
    arc_dir = os.path.expanduser('~/.home_bak/')
    home = os.path.expanduser('~/')
    dot_dir = home + '.my_scripts' + os.path.sep + 'dot_files' + os.path.sep

    for folder in ('.shell', '.fonts', '.ccache',):
        try:
            shutil.rmtree(home + folder)
        except OSError:
            pass

    # Clear existing configs if they are symlinks
    files = glob.glob(dot_dir + '.*')
    files = [os.path.basename(x) for x in files]
    for fil in [home + fil for fil in files]:
        if os.path.islink(fil):
            os.remove(fil)

    arc_files = glob.glob(arc_dir + '.*')
    arc_files = [os.path.basename(x) for x in arc_files]

    helper = make_cmd(arc_dir, home)
    helper(arc_files, os.rename)

    try:
        os.rmdir(arc_dir)
    except OSError:
        pass

def home_save():
    """ Save existing home configs to a backup dir. """
    arc_dir = os.path.expanduser('~/.home_bak/')
    home = os.path.expanduser('~/')
    dot_dir = home + '.my_scripts' + os.path.sep + 'dot_files' + os.path.sep

    if not os.path.exists(arc_dir):
        os.makedirs(arc_dir)

    files = glob.glob(dot_dir + '.*')
    files = [os.path.basename(x) for x in files]
    files = [x for x in files if os.path.exists(home + x)]

    helper = make_cmd(home, arc_dir)
    helper(files, os.rename)

def packs_babun():
    """ Setup a fresh babun install. """
    # Install packages
    cmd = 'pact install ' + BABUN
    subprocess.call(shlex.split(cmd))

    # Now prepare then invoke regular setup link common files
    home = os.path.expanduser('~') + os.sep

    # Prevent fonts from running with home_config
    ddir = home + '.fonts'
    if not os.path.exists(ddir):
        os.mkdir(ddir)

    # Backup defaults and allow for new ones
    for name in ['.gitconfig', '.vim']:
        dfile = home + name
        dfile_bak = dfile + '_bak'
        if os.path.exists(dfile) and not os.path.exists(dfile_bak):
            os.rename(dfile, dfile_bak)

    home_config()

def packs_cabal():
    """ Installs haskell packages for Eclipse Haskell plugin. """
    cmd = 'cabal update'
    subprocess.call(shlex.split(cmd))

    cmd = 'cabal install ' + CABAL
    subprocess.call(shlex.split(cmd))

def packs_debian(server=False):
    """ Install packages on the current system. """
    if os.getuid() != 0:
        raise NotSudo

    if server:
        packages = MINIMUM.split()
    else:
        packages = (PROGRAMS + PROGRAMMING + KEYRINGS).split()

    cache = apt.Cache()
    print("One moment while we update cache.")
    cache.update()
    cache.open(None)
    print("Update done.")

    cmd = 'sudo apt-get install'.split()
    for pack in packages:
        try:
            package = cache[pack]
            if not package.is_installed:
                cmd.append(pack)
        except KeyError:
            print("Package couldn't be selected: %s" % pack)

    print("Please wait, running: " + " ".join(cmd))
    subprocess.call(cmd)

def packs_py():
    """ Installs python packages using pip. """
    if os.getuid() != 0:
        raise NotSudo

    # Use python package manager.
    cmd = 'sudo pip install ' + PY_PACKS
    subprocess.call(shlex.split(cmd))

    # Install python completion to system bash_completion.d.
    cmd = 'activate-global-python-argcomplete --user'
    subprocess.call(shlex.split(cmd))

def install_jshint():
    """ Setup jshint for progrmaming javascript with vim. """
    cmd = 'npm install jshint -g'
    subprocess.call(shlex.split(cmd))

def install_pipelight():
    """ Silverlight plugin for firefox/chrome on linux.
    http://www.webupd8.org/2013/08/pipelight-use-silverlight-in-your-linux.html
    """
    if os.getuid() != 0:
        raise NotSudo

    cmds = [
        'sudo apt-get remove flashplugin-installer',
        'sudo apt-add-repository ppa:pipelight/stable',
        'sudo apt-get update',
        'sudo apt-get install pipelight-multi',
        'pipelight-plugin --accept --enable silverlight',
        'pipelight-plugin --accept --enable flash',
    ]
    cmds = [x.split() for x in cmds]
    for cmd in cmds:
        subprocess.call(cmd)
    print("Installation over, remember to use a useragent switcher.")

def main():
    """ Main function. """
    mesg = """This script sets up a dev environment.

    choice       effect
    ------------------------------------------------------
    home         Setup home config files.
    home_restore Restore home files and undo home_save.
    home_save    Save existing home files.
    babun        Install babun packages.
    cabal        Install haskell packages for eclipse.
    debian       Install debian packages.
    server       Install debian packages for server, minimal.
    jshint       Install jshint via npm for javascript vim.
    pip          Install python libraries via pip.
    pipelight    Install pipelight flash & silverlight.
    """
    # Use a dict of funcs to process args
    actions = {
        'home':         home_config,
        'home_restore': home_restore,
        'home_save':    home_save,
        'babun':        packs_babun,
        'cabal':        packs_cabal,
        'debian':       packs_debian,
        'server':       functools.partial(packs_debian, True),
        'jshint':       install_jshint,
        'pip':          packs_py,
        'pipelight':    install_pipelight,
    }
    parser = argparse.ArgumentParser(description=mesg,
                                     formatter_class=argparse.
                                     RawDescriptionHelpFormatter)
    parser.add_argument('stages', nargs='+', help='stages to execute',
                        choices=sorted(actions.keys()))

    autocomplete(parser)
    args = parser.parse_args()  # Default parses argv[1:]

    try:
        for stage in args.stages:
            actions[stage]()
    except NotSudo:
        print("Rerun this part of script with sudo.")

if __name__ == '__main__':
    main()
<commit_msg>Simplify.<commit_after>#!/usr/bin/env python
# PYTHON_ARGCOMPLETE_OK
""" Install packages and setup home configs on fresh system. """
from __future__ import print_function
from BuildSrc import get_code
import argparse
import functools
import glob
import os
import shlex
import shutil
import subprocess
try:
    import apt
except ImportError:
    print("Don't use `debian` option.")
try:
    from argcomplete import autocomplete
except ImportError:
    def autocomplete(dummy):
        """ Dummy func. """
        pass

# Packages to install follow, broken down into categories.
MINIMUM = """ \
    vim build-essential git-core mercurial ccache fontconfig gdb lynx \
    colormake colordiff colorgcc exuberant-ctags dkms python-dev python-pip \
    p7zip-full rar zip unzip gzip lzip xz-utils liblzma-dev liblzma5 \
    htop iotop itop mytop dos2unix tree dfc \
    """

PROGRAMS = """ \
    htop iotop itop mytop pkg-config aptitude synaptic dos2unix \
    oxygen-molecule kubuntu-restricted-extras kubuntu-restricted-addons \
    brother-cups-wrapper-laser brother-cups-wrapper-laser1 \
    brother-lpr-drivers-laser brother-lpr-drivers-laser1 \
    k3b kid3 krita kolourpaint4 kchmviewer yakuake plasma-widget-quickaccess \
    kdirstat keepassx screen tmux \
    chromium-browser firefox rekonq \
    abiword baobab dvdrip easytag chm2pdf dia catdoc \
    gimp gimp-plugin-registry \
    fuse gparted quicksynergy tree dfc sharutils sharutils-doc \
    libnotify-bin libnotify-dev \
    redshift geoclue geoclue-hostip numlockx \
    samba samba-doc samba-dev samba-dbg cifs-utils wireshark libwireshark-dev \
    wondershaper vlc ffmpeg ffmpeg-doc mplayer mencoder \
    p7zip-full rar zip unzip gzip lzip xz-utils liblzma-dev liblzma5 \
    virtualbox-qt wine vagrant \
    ttf-xfree86-nonfree \
    """

KEYRINGS = """ \
    debian-keyring debian-archive-keyring gnome-keyring \
    debian-ports-archive-keyring python-gnomekeyring python-keyring \
    ubuntu-keyring \
    """

PROGRAMMING = """ \
    build-essential debianutils ubuntu-dev-tools mesa-utils openssh-server \
    automake automake1.9-doc ant ant-doc checkinstall checkbox \
    dkms docbook make-doc lynx kdiff3 kdiff3-doc patch rpm2cpio rpm \
    codeblocks kdevelop qt-sdk graphviz \
    colormake colordiff colorgcc jq nmap \
    vim vim-doc vim-gtk vim-rails vim-syntax-go vim-syntax-gtk vim-doc \
    flex flex-doc bison bison-doc exuberant-ctags sphinx-common sphinx-doc \
    bash-doc bash-builtins bashdb \
    zsh zsh-dbg zsh-dev zsh-doc zsh-lovers zshdb \
    clisp clisp-doc clisp-dev clisp-module-gdbm \
    coffeescript coffeescript-doc \
    docutils-common docutils-doc asciidoc xmlto docbook2x \
    erlang erlang-eunit \
    gdc golang golang-doc golang-src golang-codesearch-dev \
    gcc gcc-doc gcc-4.7-source libcunit1 gdb gdb-doc cgdb xxgdb ccache \
    libboost-all-dev libglm-dev libglew-dev libglfw-dev ncurses-doc \
    libncurses5-dev libncursesw5-dev libpcre3-dev zlib1g-dev libbz2-dev \
    libgnome2-dev libgnomeui-dev libgtk2.0-dev libatk1.0-dev libbonoboui2-dev \
    libcairo2-dev libx11-dev libxpm-dev libxt-dev libgmp3-dev libmpc-dev \
    libmpfr-dev libcurl4-openssl-dev libevent-dev libarchive-dev \
    libcurl4-gnutls-dev openmpi-bin openmpi-checkpoint openmpi-common \
    gfortran \
    ghc ghc-doc ghc-haddock ghc-prof haskell-debian-utils \
    haskell-devscripts haskell-doc cabal-install \
    junit junit-doc maven openjdk-7-doc openjdk-7-jdk openjdk-7-dbg \
    openjdk-7-source openjdk-7-demo icedtea-7-plugin \
    lua5.2 lua5.2-doc luadoc liblua5.2-dev \
    monodevelop-debugger-gdb monodevelop-nunit \
    nodejs nodejs-dev nodejs-legacy npm \
    perl perl-doc libperl-dev perl-modules libpadwalker-perl \
    libfile-next-perl \
    php5 php5-mysql phpunit php5-dev \
    swi-prolog swi-prolog-doc \
    python python-dev python-doc python-pip \
    python3 python3-dev python3-doc python3-pip jython jython-doc \
    pychecker pylint pep8 python-autopep8 ruby-full shunit2 \
    tcl-dev tcl-doc \
    tex-common texlive-latex-base texlive-font-utils \
    bzr bzr-builddeb bzr-doc python-bzrlib bzrtools git git-gui git-doc \
    mercurial subversion cvs \
    """

BABUN = """ \
    clisp lua perl python python3 ruby make cmake colordiff colorgcc \
    p7zip unzip zip gzip ncurses pcre mercurial bzr subversion \
    libboost-devel libboost_python-devel cppunit
    """

CABAL = "buildwrapper scion-browser hoogle terminfo happy hlint"

PY_PACKS = "argcomplete cram Pygments pytest trash-cli"

class NotSudo(Exception):
    """ Throw this if we aren't sudo but need to be. """
    pass

def make_cmd(src, dst):
    """ Generate a function helper. """
    def cmd_when_dst_empty(files, command, opts=None):
        """ When dst doesn't have file do:
                command(src + file, dst + file, *opts)
        """
        if opts is None:
            opts = ()
        for fil in files:
            sfile, dfile = [x + fil for x in (src, dst)]
            if not os.path.exists(dfile):
                print("{0} >>>>> {1}".format(sfile, dfile))
                command(sfile, dfile, *opts)
    return cmd_when_dst_empty

def home_config():
    """ Setup the dev environment, stuff goes in the user's home folder. """
    script_path = os.path.relpath(__file__)
    script_dir = os.path.dirname(script_path)
    if script_dir == '':
        script_dir = '.'

    dot_dir = script_dir + os.sep + 'dot_files' + os.sep
    home = os.path.expanduser('~') + os.sep

    # Get shell utilities
    shell_dir = home + '.shell' + os.sep
    get_code('https://bitbucket.org/sjl/hg-prompt/', shell_dir + '.hg-prompt')

    git_urls = [
        'https://github.com/magicmonty/bash-git-prompt.git',
        'https://github.com/starcraftman/zsh-git-prompt.git',
        'https://github.com/zsh-users/zsh-completions.git',
        'https://github.com/zsh-users/zsh-syntax-highlighting.git',
        'https://github.com/starcraftman/hhighlighter.git',
    ]

    for url in git_urls:
        target = '.' + url[url.rindex('/')+1:url.rindex('.git')]
        get_code(url, shell_dir + target)

    # Setup powerline fonts if not done.
    ddir = home + '.fonts'
    dpow = ddir + os.sep + 'powerline-fonts'
    if not os.path.exists(ddir) and subprocess.call(['which', 'fc-cache']) == 0:
        os.mkdir(ddir)
        get_code('https://github.com/Lokaltog/powerline-fonts', dpow)
        subprocess.call(['fc-cache', '-vf', ddir])

    ddir = home + '.ccache'
    if not os.path.exists(ddir):
        os.mkdir(ddir)

    files = [os.path.basename(x) for x in glob.glob(dot_dir + '.*')]
    helper = make_cmd(dot_dir, home)
    helper(files, os.symlink)

    print("NOTE: Remember to add user to smb.\nsudo smbpasswd -a username")

def home_restore():
    """ Undo changes by home_config & restore backup if exists. """
    arc_dir = os.path.expanduser('~/.home_bak/')
    home = os.path.expanduser('~/')
    dot_dir = home + '.my_scripts' + os.path.sep + 'dot_files' + os.path.sep

    for folder in ('.shell', '.fonts', '.ccache',):
        try:
            shutil.rmtree(home + folder)
        except OSError:
            pass

    # Clear existing configs if they are symlinks
    files = [os.path.basename(x) for x in glob.glob(dot_dir + '.*')]
    for fil in [home + fil for fil in files]:
        if os.path.islink(fil):
            os.remove(fil)

    arc_files = [os.path.basename(x) for x in glob.glob(arc_dir + '.*')]
    helper = make_cmd(arc_dir, home)
    helper(arc_files, os.rename)

    try:
        os.rmdir(arc_dir)
    except OSError:
        pass

def home_save():
    """ Save existing home configs to a backup dir. """
    arc_dir = os.path.expanduser('~/.home_bak/')
    home = os.path.expanduser('~/')
    dot_dir = home + '.my_scripts' + os.path.sep + 'dot_files' + os.path.sep

    if not os.path.exists(arc_dir):
        os.makedirs(arc_dir)

    files = [os.path.basename(x) for x in glob.glob(dot_dir + '.*')]
    files = [x for x in files if os.path.exists(home + x)]

    helper = make_cmd(home, arc_dir)
    helper(files, os.rename)

def packs_babun():
    """ Setup a fresh babun install. """
    # Install packages
    cmd = 'pact install ' + BABUN
    subprocess.call(shlex.split(cmd))

    # Now prepare then invoke regular setup link common files
    home = os.path.expanduser('~') + os.sep

    # Prevent fonts from running with home_config
    ddir = home + '.fonts'
    if not os.path.exists(ddir):
        os.mkdir(ddir)

    # Backup defaults and allow for new ones
    for name in ['.gitconfig', '.vim']:
        dfile = home + name
        dfile_bak = dfile + '_bak'
        if os.path.exists(dfile) and not os.path.exists(dfile_bak):
            os.rename(dfile, dfile_bak)

    home_config()

def packs_cabal():
    """ Installs haskell packages for Eclipse Haskell plugin. """
    cmd = 'cabal update'
    subprocess.call(shlex.split(cmd))

    cmd = 'cabal install ' + CABAL
    subprocess.call(shlex.split(cmd))

def packs_debian(server=False):
    """ Install packages on the current system. """
    if os.getuid() != 0:
        raise NotSudo

    if server:
        packages = MINIMUM.split()
    else:
        packages = (PROGRAMS + PROGRAMMING + KEYRINGS).split()

    cache = apt.Cache()
    print("One moment while we update cache.")
    cache.update()
    cache.open(None)
    print("Update done.")

    cmd = 'sudo apt-get install'.split()
    for pack in packages:
        try:
            package = cache[pack]
            if not package.is_installed:
                cmd.append(pack)
        except KeyError:
            print("Package couldn't be selected: %s" % pack)

    print("Please wait, running: " + " ".join(cmd))
    subprocess.call(cmd)

def packs_py():
    """ Installs python packages using pip. """
    if os.getuid() != 0:
        raise NotSudo

    # Use python package manager.
    cmd = 'sudo pip install ' + PY_PACKS
    subprocess.call(shlex.split(cmd))

    # Install python completion to system bash_completion.d.
    cmd = 'activate-global-python-argcomplete --user'
    subprocess.call(shlex.split(cmd))

def install_jshint():
    """ Setup jshint for progrmaming javascript with vim. """
    cmd = 'npm install jshint -g'
    subprocess.call(shlex.split(cmd))

def install_pipelight():
    """ Silverlight plugin for firefox/chrome on linux.
    http://www.webupd8.org/2013/08/pipelight-use-silverlight-in-your-linux.html
    """
    if os.getuid() != 0:
        raise NotSudo

    cmds = [
        'sudo apt-get remove flashplugin-installer',
        'sudo apt-add-repository ppa:pipelight/stable',
        'sudo apt-get update',
        'sudo apt-get install pipelight-multi',
        'pipelight-plugin --accept --enable silverlight',
        'pipelight-plugin --accept --enable flash',
    ]
    cmds = [x.split() for x in cmds]
    for cmd in cmds:
        subprocess.call(cmd)
    print("Installation over, remember to use a useragent switcher.")

def main():
    """ Main function. """
    mesg = """This script sets up a dev environment.

    choice       effect
    ------------------------------------------------------
    home         Setup home config files.
    home_restore Restore home files and undo home_save.
    home_save    Save existing home files.
    babun        Install babun packages.
    cabal        Install haskell packages for eclipse.
    debian       Install debian packages.
    server       Install debian packages for server, minimal.
    jshint       Install jshint via npm for javascript vim.
    pip          Install python libraries via pip.
    pipelight    Install pipelight flash & silverlight.
    """
    # Use a dict of funcs to process args
    actions = {
        'home':         home_config,
        'home_restore': home_restore,
        'home_save':    home_save,
        'babun':        packs_babun,
        'cabal':        packs_cabal,
        'debian':       packs_debian,
        'server':       functools.partial(packs_debian, True),
        'jshint':       install_jshint,
        'pip':          packs_py,
        'pipelight':    install_pipelight,
    }
    parser = argparse.ArgumentParser(description=mesg,
                                     formatter_class=argparse.
                                     RawDescriptionHelpFormatter)
    parser.add_argument('stages', nargs='+', help='stages to execute',
                        choices=sorted(actions.keys()))

    autocomplete(parser)
    args = parser.parse_args()  # Default parses argv[1:]

    try:
        for stage in args.stages:
            actions[stage]()
    except NotSudo:
        print("Rerun this part of script with sudo.")

if __name__ == '__main__':
    main()
<|endoftext|>