<commit_before># -*- encoding: utf-8 -*-
import os
import random
import tempfile
from distutils.spawn import find_executable

from twisted.python import usage
from twisted.internet import reactor, error

import txtorcon

from ooni.utils import log, onion
from ooni import nettest

class TorIsNotInstalled(Exception):
    pass

class UsageOptions(usage.Options):
    optParameters = [
        ['timeout', 't', 120,
         'Specify the timeout after which to consider '
         'the Tor bootstrapping process to have failed'], ]

class BridgeReachability(nettest.NetTestCase):
    name = "Bridge Reachability"
    description = "A test for checking if bridges are reachable " \
                  "from a given location."
    author = "Arturo Filastò"
    version = "0.1.2"

    usageOptions = UsageOptions

    inputFile = ['file', 'f', None,
                 'File containing bridges to test reachability for. '
                 'They should be one per line IP:ORPort or '
                 'TransportType IP:ORPort (ex. obfs2 127.0.0.1:443)']

    requiredOptions = ['file']

    def requirements(self):
        if not onion.find_tor_binary():
            raise TorIsNotInstalled(
                "For instructions on installing Tor see: "
                "https://www.torproject.org/download/download")

    def setUp(self):
        self.tor_progress = 0
        self.timeout = int(self.localOptions['timeout'])

        fd, self.tor_logfile = tempfile.mkstemp()
        os.close(fd)
        fd, self.obfsproxy_logfile = tempfile.mkstemp()
        os.close(fd)

        self.report['error'] = None
        self.report['success'] = None
        self.report['timeout'] = self.timeout
        self.report['transport_name'] = 'vanilla'
        self.report['tor_version'] = str(onion.tor_details['version'])
        self.report['tor_progress'] = 0
        self.report['tor_progress_tag'] = None
        self.report['tor_progress_summary'] = None
        self.report['tor_log'] = None
        self.report['obfsproxy_version'] = str(onion.obfsproxy_details['version'])
        self.report['obfsproxy_log'] = None
        self.report['bridge_address'] = None

        self.bridge = self.input
        if self.input.startswith('Bridge'):
            self.bridge = self.input.replace('Bridge ', '')
        self.pyobfsproxy_bin = onion.obfsproxy_details['binary']
        self.fteproxy_bin = find_executable('fteproxy')

    def postProcessor(self, measurements):
        if 'successes' not in self.summary:
            self.summary['successes'] = []
        if 'failures' not in self.summary:
            self.summary['failures'] = []

        details = {
            'address': self.report['bridge_address'],
            'transport_name': self.report['transport_name'],
            'tor_progress': self.report['tor_progress']
        }
        if self.report['success']:
            self.summary['successes'].append(details)
        else:
            self.summary['failures'].append(details)
        return self.report

    def displaySummary(self, summary):
        successful_count = {}
        failure_count = {}

        def count(results, counter):
            for result in results:
                if result['transport_name'] not in counter:
                    counter[result['transport_name']] = 0
                counter[result['transport_name']] += 1
        count(summary['successes'], successful_count)
        count(summary['failures'], failure_count)

        working_bridges = ', '.join(
            ["%s %s" % (x['transport_name'], x['address'])
             for x in summary['successes']])
        failing_bridges = ', '.join(
            ["%s %s (at %s%%)"
             % (x['transport_name'], x['address'], x['tor_progress'])
             for x in summary['failures']])

        log.msg("Total successes: %d" % len(summary['successes']))
        log.msg("Total failures: %d" % len(summary['failures']))

        for transport, count in successful_count.items():
            log.msg("%s successes: %d" % (transport.title(), count))
        for transport, count in failure_count.items():
            log.msg("%s failures: %d" % (transport.title(), count))

        log.msg("Working bridges: %s" % working_bridges)
        log.msg("Failing bridges: %s" % failing_bridges)

    def test_full_tor_connection(self):
        config = txtorcon.TorConfig()
        config.ControlPort = random.randint(2**14, 2**16)
        config.SocksPort = random.randint(2**14, 2**16)
        log.msg(
            "Connecting to %s with tor %s" %
            (self.bridge, onion.tor_details['version']))

        transport_name = onion.transport_name(self.bridge)

        if transport_name:
            self.report['transport_name'] = transport_name

        if transport_name and transport_name == 'fte' and self.fteproxy_bin:
            config.ClientTransportPlugin = "%s exec %s --managed" % (
                transport_name, self.fteproxy_bin)
            log.debug("Using fte from %s" % self.fteproxy_bin)
            self.report['bridge_address'] = self.bridge.split(' ')[1]
        elif transport_name and transport_name == 'fte'\
                and not self.fteproxy_bin:
            log.err("Unable to test bridge because fteproxy is not installed")
            self.report['error'] = 'missing-fteproxy'
            return
        elif transport_name and self.pyobfsproxy_bin:
            config.ClientTransportPlugin = ("%s exec %s --log-min-severity info "
                "--log-file %s managed") % (transport_name,
                                            self.pyobfsproxy_bin,
                                            self.obfsproxy_logfile)
            if onion.OBFSProxyVersion('0.2') > onion.obfsproxy_details['version']:
                log.err(
                    "The obfsproxy version you are using appears to be outdated."
                )
                self.report['error'] = 'old-obfsproxy'
                return
            log.debug("Using pyobfsproxy from %s" % self.pyobfsproxy_bin)
            self.report['bridge_address'] = self.bridge.split(' ')[1]
        elif transport_name and not self.pyobfsproxy_bin:
            log.err(
                "Unable to test bridge because pyobfsproxy is not installed")
            self.report['error'] = 'missing-pyobfsproxy'
            return
        else:
            self.report['bridge_address'] = self.bridge.split(' ')[0]

        if transport_name and transport_name == 'scramblesuit' and \
                onion.TorVersion('0.2.5.1') > onion.tor_details['version']:
            self.report['error'] = 'unsupported-tor-version'
            log.err("Unsupported Tor version.")
            return
        elif transport_name and \
                onion.TorVersion('0.2.4.1') > onion.tor_details['version']:
            self.report['error'] = 'unsupported-tor-version'
            log.err("Unsupported Tor version.")
            return

        config.Bridge = self.bridge
        config.UseBridges = 1
        config.log = ['notice stdout', 'notice file %s' % self.tor_logfile]
        config.save()

        def updates(prog, tag, summary):
            log.msg("%s: %s%%" % (self.bridge, prog))
            self.report['tor_progress'] = int(prog)
            self.report['tor_progress_tag'] = tag
            self.report['tor_progress_summary'] = summary

        d = txtorcon.launch_tor(config, reactor, timeout=self.timeout,
                                progress_updates=updates)

        @d.addCallback
        def setup_complete(proto):
            try:
                proto.transport.signalProcess('TERM')
            except error.ProcessExitedAlready:
                proto.transport.loseConnection()
            log.msg("Successfully connected to %s" % self.bridge)
            self.report['success'] = True

        @d.addErrback
        def setup_failed(failure):
            log.msg("Failed to connect to %s" % self.bridge)
            self.report['success'] = False
            self.report['error'] = 'timeout-reached'
            return

        @d.addCallback
        def write_log(_):
            with open(self.tor_logfile) as f:
                self.report['tor_log'] = f.read()
            os.remove(self.tor_logfile)
            with open(self.obfsproxy_logfile) as f:
                self.report['obfsproxy_log'] = f.read()
            os.remove(self.obfsproxy_logfile)

        return d
<commit_msg>factor out a level of if/elses<commit_after># -*- encoding: utf-8 -*-
import os
import random
import tempfile
from distutils.spawn import find_executable

from twisted.python import usage
from twisted.internet import reactor, error

import txtorcon

from ooni.utils import log, onion
from ooni import nettest

class TorIsNotInstalled(Exception):
    pass

class UsageOptions(usage.Options):
    optParameters = [
        ['timeout', 't', 120,
         'Specify the timeout after which to consider '
         'the Tor bootstrapping process to have failed'], ]

class BridgeReachability(nettest.NetTestCase):
    name = "Bridge Reachability"
    description = "A test for checking if bridges are reachable " \
                  "from a given location."
    author = "Arturo Filastò"
    version = "0.1.2"

    usageOptions = UsageOptions

    inputFile = ['file', 'f', None,
                 'File containing bridges to test reachability for. '
                 'They should be one per line IP:ORPort or '
                 'TransportType IP:ORPort (ex. obfs2 127.0.0.1:443)']

    requiredOptions = ['file']

    def requirements(self):
        if not onion.find_tor_binary():
            raise TorIsNotInstalled(
                "For instructions on installing Tor see: "
                "https://www.torproject.org/download/download")

    def setUp(self):
        self.tor_progress = 0
        self.timeout = int(self.localOptions['timeout'])

        fd, self.tor_logfile = tempfile.mkstemp()
        os.close(fd)
        fd, self.obfsproxy_logfile = tempfile.mkstemp()
        os.close(fd)

        self.report['error'] = None
        self.report['success'] = None
        self.report['timeout'] = self.timeout
        self.report['transport_name'] = 'vanilla'
        self.report['tor_version'] = str(onion.tor_details['version'])
        self.report['tor_progress'] = 0
        self.report['tor_progress_tag'] = None
        self.report['tor_progress_summary'] = None
        self.report['tor_log'] = None
        self.report['obfsproxy_version'] = str(onion.obfsproxy_details['version'])
        self.report['obfsproxy_log'] = None
        self.report['bridge_address'] = None

        self.bridge = self.input
        if self.input.startswith('Bridge'):
            self.bridge = self.input.replace('Bridge ', '')
        self.pyobfsproxy_bin = onion.obfsproxy_details['binary']
        self.fteproxy_bin = find_executable('fteproxy')

    def postProcessor(self, measurements):
        if 'successes' not in self.summary:
            self.summary['successes'] = []
        if 'failures' not in self.summary:
            self.summary['failures'] = []

        details = {
            'address': self.report['bridge_address'],
            'transport_name': self.report['transport_name'],
            'tor_progress': self.report['tor_progress']
        }
        if self.report['success']:
            self.summary['successes'].append(details)
        else:
            self.summary['failures'].append(details)
        return self.report

    def displaySummary(self, summary):
        successful_count = {}
        failure_count = {}

        def count(results, counter):
            for result in results:
                if result['transport_name'] not in counter:
                    counter[result['transport_name']] = 0
                counter[result['transport_name']] += 1
        count(summary['successes'], successful_count)
        count(summary['failures'], failure_count)

        working_bridges = ', '.join(
            ["%s %s" % (x['transport_name'], x['address'])
             for x in summary['successes']])
        failing_bridges = ', '.join(
            ["%s %s (at %s%%)"
             % (x['transport_name'], x['address'], x['tor_progress'])
             for x in summary['failures']])

        log.msg("Total successes: %d" % len(summary['successes']))
        log.msg("Total failures: %d" % len(summary['failures']))

        for transport, count in successful_count.items():
            log.msg("%s successes: %d" % (transport.title(), count))
        for transport, count in failure_count.items():
            log.msg("%s failures: %d" % (transport.title(), count))

        log.msg("Working bridges: %s" % working_bridges)
        log.msg("Failing bridges: %s" % failing_bridges)

    def test_full_tor_connection(self):
        config = txtorcon.TorConfig()
        config.ControlPort = random.randint(2**14, 2**16)
        config.SocksPort = random.randint(2**14, 2**16)
        log.msg(
            "Connecting to %s with tor %s" %
            (self.bridge, onion.tor_details['version']))

        transport_name = onion.transport_name(self.bridge)

        if transport_name:
            self.report['transport_name'] = transport_name
            if transport_name == 'fte' and self.fteproxy_bin:
                config.ClientTransportPlugin = "%s exec %s --managed" % (
                    transport_name, self.fteproxy_bin)
                log.debug("Using fte from %s" % self.fteproxy_bin)
                self.report['bridge_address'] = self.bridge.split(' ')[1]
            elif transport_name == 'fte' and not self.fteproxy_bin:
                log.err("Unable to test bridge because fteproxy is not installed")
                self.report['error'] = 'missing-fteproxy'
                return
            elif self.pyobfsproxy_bin:
                config.ClientTransportPlugin = ("%s exec %s --log-min-severity info "
                    "--log-file %s managed") % (transport_name,
                                                self.pyobfsproxy_bin,
                                                self.obfsproxy_logfile)
                if onion.OBFSProxyVersion('0.2') > onion.obfsproxy_details['version']:
                    log.err(
                        "The obfsproxy version you are using appears to be outdated."
                    )
                    self.report['error'] = 'old-obfsproxy'
                    return
                log.debug("Using pyobfsproxy from %s" % self.pyobfsproxy_bin)
                self.report['bridge_address'] = self.bridge.split(' ')[1]
            else:
                log.err(
                    "Unable to test bridge because pyobfsproxy is not installed")
                self.report['error'] = 'missing-pyobfsproxy'
                return
        else:
            self.report['bridge_address'] = self.bridge.split(' ')[0]

        if transport_name and transport_name == 'scramblesuit' and \
                onion.TorVersion('0.2.5.1') > onion.tor_details['version']:
            self.report['error'] = 'unsupported-tor-version'
            log.err("Unsupported Tor version.")
            return
        elif transport_name and \
                onion.TorVersion('0.2.4.1') > onion.tor_details['version']:
            self.report['error'] = 'unsupported-tor-version'
            log.err("Unsupported Tor version.")
            return

        config.Bridge = self.bridge
        config.UseBridges = 1
        config.log = ['notice stdout', 'notice file %s' % self.tor_logfile]
        config.save()

        def updates(prog, tag, summary):
            log.msg("%s: %s%%" % (self.bridge, prog))
            self.report['tor_progress'] = int(prog)
            self.report['tor_progress_tag'] = tag
            self.report['tor_progress_summary'] = summary

        d = txtorcon.launch_tor(config, reactor, timeout=self.timeout,
                                progress_updates=updates)

        @d.addCallback
        def setup_complete(proto):
            try:
                proto.transport.signalProcess('TERM')
            except error.ProcessExitedAlready:
                proto.transport.loseConnection()
            log.msg("Successfully connected to %s" % self.bridge)
            self.report['success'] = True

        @d.addErrback
        def setup_failed(failure):
            log.msg("Failed to connect to %s" % self.bridge)
            self.report['success'] = False
            self.report['error'] = 'timeout-reached'
            return

        @d.addCallback
        def write_log(_):
            with open(self.tor_logfile) as f:
                self.report['tor_log'] = f.read()
            os.remove(self.tor_logfile)
            with open(self.obfsproxy_logfile) as f:
                self.report['obfsproxy_log'] = f.read()
            os.remove(self.obfsproxy_logfile)

        return d
<|endoftext|>


<commit_before>#!/usr/bin/env python2
#-*- coding: utf-8 -*-

from setuptools import setup, find_packages
import os, sys

# Error-handling here is to allow package to be built w/o README included
try:
	readme = open(os.path.join(
		os.path.dirname(__file__), 'README.rst' )).read()
except IOError: readme = ''

setup(

	name = 'pulsectl',
	version = '16.8.8',
	author = 'George Filipkin, Mike Kazantsev',
	author_email = 'mk.fraggod@gmail.com',
	license = 'MIT',
	keywords = [
		'pulseaudio', 'libpulse', 'pulse', 'pa', 'bindings',
		'sound', 'audio',
		'ctypes', 'control', 'mixer', 'volume', 'mute', 'source', 'sink' ],

	url = 'http://github.com/mk-fg/python-pulse-control',

	description = 'Python high-level interface'
		' and ctypes-based bindings for PulseAudio (libpulse)',
	long_description = readme,

	classifiers = [
		'Development Status :: 4 - Beta',
		'Intended Audience :: Developers',
		'License :: OSI Approved :: MIT License',
		'Operating System :: POSIX',
		'Operating System :: POSIX :: Linux',
		'Programming Language :: Python',
		'Programming Language :: Python :: 2',
		'Programming Language :: Python :: 2.7',
		'Programming Language :: Python :: 3',
		'Programming Language :: Python :: 3.5',
		'Topic :: Multimedia',
		'Topic :: Multimedia :: Sound/Audio' ],

	install_requires = ['setuptools'],

	packages = find_packages(),
	include_package_data = True )
<commit_msg>setup: version bump<commit_after>#!/usr/bin/env python2
#-*- coding: utf-8 -*-

from setuptools import setup, find_packages
import os, sys

# Error-handling here is to allow package to be built w/o README included
try:
	readme = open(os.path.join(
		os.path.dirname(__file__), 'README.rst' )).read()
except IOError: readme = ''

setup(

	name = 'pulsectl',
	version = '16.8.14',
	author = 'George Filipkin, Mike Kazantsev',
	author_email = 'mk.fraggod@gmail.com',
	license = 'MIT',
	keywords = [
		'pulseaudio', 'libpulse', 'pulse', 'pa', 'bindings',
		'sound', 'audio',
		'ctypes', 'control', 'mixer', 'volume', 'mute', 'source', 'sink' ],

	url = 'http://github.com/mk-fg/python-pulse-control',

	description = 'Python high-level interface'
		' and ctypes-based bindings for PulseAudio (libpulse)',
	long_description = readme,

	classifiers = [
		'Development Status :: 4 - Beta',
		'Intended Audience :: Developers',
		'License :: OSI Approved :: MIT License',
		'Operating System :: POSIX',
		'Operating System :: POSIX :: Linux',
		'Programming Language :: Python',
		'Programming Language :: Python :: 2',
		'Programming Language :: Python :: 2.7',
		'Programming Language :: Python :: 3',
		'Programming Language :: Python :: 3.5',
		'Topic :: Multimedia',
		'Topic :: Multimedia :: Sound/Audio' ],

	install_requires = ['setuptools'],

	packages = find_packages(),
	include_package_data = True )
<|endoftext|>


<commit_before># Copyright 2019 DeepMind Technologies Ltd. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Python spiel example to use value iteration to solve a game."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import six

from open_spiel.python.algorithms import value_iteration
import pyspiel

def play_tic_tac_toe():
  """Solves tic tac toe."""
  game = pyspiel.load_game("tic_tac_toe")

  print("Solving the game; depth_limit = {}".format(-1))
  values = value_iteration.value_iteration(game, -1, 0.01)

  for state, value in six.iteritems(values):
    print("")
    print(str(state))
    print("Value = {}".format(value))

  initial_state = "...\n...\n..."
  cross_win_state = "...\n...\n.ox"
  naught_win_state = "x..\noo.\nxx."

  assert values[initial_state] == 0, "State should be drawn: \n" + initial_state
  assert values[cross_win_state] == 1, ("State should be won by player 0: \n" +
                                        cross_win_state)
  assert values[naught_win_state] == -1, (
      "State should be won by player 1: \n" + cross_win_state)

def main():
  parser = argparse.ArgumentParser("value_iteration")
  parser.add_argument("game", default="tic_tac_toe")
  args = parser.parse_args()

  if args.game == "tic_tac_toe":
    play_tic_tac_toe()
  else:
    raise NotImplementedError("Only tic tac toe is implemented.")

if __name__ == "__main__":
  main()
<commit_msg>fixing default game arg<commit_after># Copyright 2019 DeepMind Technologies Ltd. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Python spiel example to use value iteration to solve a game."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import six

from open_spiel.python.algorithms import value_iteration
import pyspiel

def play_tic_tac_toe():
  """Solves tic tac toe."""
  game = pyspiel.load_game("tic_tac_toe")

  print("Solving the game; depth_limit = {}".format(-1))
  values = value_iteration.value_iteration(game, -1, 0.01)

  for state, value in six.iteritems(values):
    print("")
    print(str(state))
    print("Value = {}".format(value))

  initial_state = "...\n...\n..."
  cross_win_state = "...\n...\n.ox"
  naught_win_state = "x..\noo.\nxx."

  assert values[initial_state] == 0, "State should be drawn: \n" + initial_state
  assert values[cross_win_state] == 1, ("State should be won by player 0: \n" +
                                        cross_win_state)
  assert values[naught_win_state] == -1, (
      "State should be won by player 1: \n" + cross_win_state)

def main():
  parser = argparse.ArgumentParser("value_iteration")
  parser.add_argument("game", nargs='?', default="tic_tac_toe")
  args = parser.parse_args()

  if args.game == "tic_tac_toe":
    play_tic_tac_toe()
  else:
    raise NotImplementedError("Only tic tac toe is implemented.")

if __name__ == "__main__":
  main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
from setuptools import setup, find_packages
import pkg_resources

__updated__ = "2015-06-07"
__author__ = "Aurélien Moreau"
__copyright__ = "Copyright 2015, Angus.ai"
__credits__ = ["Aurélien Moreau", "Gwennaël Gâté"]
__license__ = "Apache v2.0"
__maintainer__ = "Aurélien Moreau"
__status__ = "Production"

setup(name='angus-sdk-python',
      version=pkg_resources.resource_string(__name__, "/VERSION"),
      description='Angus python SDK',
      author=__author__,
      author_email='aurelien.moreau@yienyien.net',
      url='http://www.angus.ai/',
      install_requires=[
          "requests==2.5.3",  # fixme: issue with the ssl and python version
          "argparse==1.2.1",
          "pytest==2.6.4",
          "six",
          "requests-futures",
      ],
      license=__license__,
      packages=find_packages(exclude=['tests']),
      scripts=['bin/angusme']
      )
<commit_msg>Fix bug with VERSION file in setup.py and pip install under python3.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
import codecs

from setuptools import setup, find_packages
import pkg_resources

__updated__ = "2015-06-08"
__author__ = "Aurélien Moreau"
__copyright__ = "Copyright 2015, Angus.ai"
__credits__ = ["Aurélien Moreau", "Gwennaël Gâté"]
__license__ = "Apache v2.0"
__maintainer__ = "Aurélien Moreau"
__status__ = "Production"

with codecs.open('VERSION', 'r', 'utf8') as lf:
    version = lf.read()

setup(name='angus-sdk-python',
      version=version,
      description='Angus python SDK',
      author=__author__,
      author_email='aurelien.moreau@yienyien.net',
      url='http://www.angus.ai/',
      install_requires=[
          "requests==2.5.3",  # fixme: issue with the ssl and python version
          "argparse==1.2.1",
          "pytest==2.6.4",
          "six",
          "requests-futures",
      ],
      license=__license__,
      packages=find_packages(exclude=['tests']),
      scripts=['bin/angusme']
      )
<|endoftext|>


<commit_before>from setuptools import setup, find_packages

version = "0.0.0"

long_description = ""
try:
    long_description = file('README.md').read()
except Exception:
    pass

license = ""
try:
    license = file('LICENSE').read()
except Exception:
    pass

setup(
    name='matrix-bot',
    version=version,
    description='A matrix.org bot',
    author='Pablo Saavedra',
    author_email='saavedra.pablo@gmail.com',
    url='http://github.com/psaavedra/matrix-bot',
    packages=find_packages(),
    package_data={
    },
    scripts=[
        "tools/matrix-bot",
    ],
    zip_safe=False,
    install_requires=[
        "getconf",
        "ldap",
        "matrix-client",
    ],
    data_files=[
        ('/usr/share/doc/matrix-bot/',
            ['cfg/matrix-bot.cfg.example']),
    ],

    download_url='https://github.com/psaavedra/matrix-bot/zipball/master',
    classifiers=[
        "Development Status :: 2 - Pre-Alpha",
        "License :: OSI Approved :: MIT License",
        "Programming Language :: Python",
        "Topic :: Communications :: Chat",
    ],
    long_description=long_description,
    license=license,
    keywords="python matrix bot matrix-python-sdk",
)
<commit_msg>Updated the require packages names<commit_after>from setuptools import setup, find_packages

version = "0.0.0"

long_description = ""
try:
    long_description = file('README.md').read()
except Exception:
    pass

license = ""
try:
    license = file('LICENSE').read()
except Exception:
    pass

setup(
    name='matrix-bot',
    version=version,
    description='A matrix.org bot',
    author='Pablo Saavedra',
    author_email='saavedra.pablo@gmail.com',
    url='http://github.com/psaavedra/matrix-bot',
    packages=find_packages(),
    package_data={
    },
    scripts=[
        "tools/matrix-bot",
    ],
    zip_safe=False,
    install_requires=[
        "getconf",
        "matrix-client",
        "python-ldap",
    ],
    data_files=[
        ('/usr/share/doc/matrix-bot/',
            ['cfg/matrix-bot.cfg.example']),
    ],

    download_url='https://github.com/psaavedra/matrix-bot/zipball/master',
    classifiers=[
        "Development Status :: 2 - Pre-Alpha",
        "License :: OSI Approved :: MIT License",
        "Programming Language :: Python",
        "Topic :: Communications :: Chat",
    ],
    long_description=long_description,
    license=license,
    keywords="python matrix bot matrix-python-sdk",
)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import sys
import shutil
import unittest
from setuptools import setup
from setuptools.command.test import test as TestCommand # Need for 'test' command to be recognised
from numpy.distutils.misc_util import get_numpy_include_dirs

packages = ['cxroots', 'cxroots.tests', 'cxroots.contours']

# get the version, this will assign __version__
exec(open('cxroots/version.py').read())

# read the README_pip.rst
try:
    with open('README.rst') as file:
        long_description = file.read()
except:
    long_description = None

setup(
    name = 'cxroots',
    version = __version__,
    description = 'Find all the roots (zeros) of a complex analytic function within a given contour in the complex plane.',
    long_description = long_description,
    author = 'Robert Parini',
    author_email = 'robert.parini@gmail.com',
    url = 'https://rparini.github.io/cxroots/',
    license = 'BSD',
    data_files = [("", ["LICENSE"])],
    packages = packages,
    zip_safe = False,   # prevent cxroots from installing as a .egg zip file
    platforms = ['all'],
    dependency_links=['git+git://github.com/pbrod/numdifftools'],
    install_requires = ['pytest-runner', 'numpy', 'scipy', 'docrep', 'mpmath', 'numdifftools'],
    tests_require=['pytest'],
    keywords='roots zeros complex analytic functions',
    classifiers=[
	    'Development Status :: 4 - Beta',
	    'Intended Audience :: Science/Research',
	    'Topic :: Scientific/Engineering :: Mathematics',
	    'License :: OSI Approved :: BSD License',
	    'Programming Language :: Python :: 2.7',
	    'Programming Language :: Python :: 3',
	],
)
<commit_msg>Remove numpy.distutils import<commit_after>#!/usr/bin/env python
import os
import sys
import shutil
import unittest
from setuptools import setup
from setuptools.command.test import test as TestCommand # Need for 'test' command to be recognised

packages = ['cxroots', 'cxroots.tests', 'cxroots.contours']

# get the version, this will assign __version__
exec(open('cxroots/version.py').read())

# read the README_pip.rst
try:
    with open('README.rst') as file:
        long_description = file.read()
except:
    long_description = None

setup(
    name = 'cxroots',
    version = __version__,
    description = 'Find all the roots (zeros) of a complex analytic function within a given contour in the complex plane.',
    long_description = long_description,
    author = 'Robert Parini',
    author_email = 'robert.parini@gmail.com',
    url = 'https://rparini.github.io/cxroots/',
    license = 'BSD',
    data_files = [("", ["LICENSE"])],
    packages = packages,
    zip_safe = False,   # prevent cxroots from installing as a .egg zip file
    platforms = ['all'],
    dependency_links=['git+git://github.com/pbrod/numdifftools'],
    install_requires = ['pytest-runner', 'numpy', 'scipy', 'docrep', 'mpmath', 'numdifftools'],
    tests_require=['pytest'],
    keywords='roots zeros complex analytic functions',
    classifiers=[
	    'Development Status :: 4 - Beta',
	    'Intended Audience :: Science/Research',
	    'Topic :: Scientific/Engineering :: Mathematics',
	    'License :: OSI Approved :: BSD License',
	    'Programming Language :: Python :: 2.7',
	    'Programming Language :: Python :: 3',
	],
)
<|endoftext|>


<commit_before># coding=utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

"""setuptools install script"""

from setuptools import find_packages
from setuptools import setup

EXTRAS = {
    "test": [
        "codecov==2.0.15",
        "coverage>=4.5.4,<4.6",
        "distro>=1.3.0",
        "flake8==3.7.8",
        "flake8-commas==2.0.0",
        "flake8-isort==2.7.0",
        "flake8-quotes==2.1.0",
        "isort==4.3.21",
        "pylint==2.3.1",
        "pytest>=5.0.1,<5.1",
        "pytest-cov>=2.7.1,<2.8",
        "pytest-flake8>=1.0.4,<1.1",
        "pytest-pylint>=0.14.0,<0.15",
    ]}

if __name__ == "__main__":
    setup(name="funfuzz",
          version="0.7.0a1",
          entry_points={
              "console_scripts": ["funfuzz = funfuzz.bot:main"],
          },
          package_data={"funfuzz": [
              "autobisectjs/*",
              "ccoverage/*",
              "js/*",
              "js/jsfunfuzz/*",
              "js/shared/*",
              "util/*",
          ]},
          package_dir={"": "src"},
          packages=find_packages(where="src"),
          install_requires=[
              "boto>=2.49.0",
              "fasteners>=0.14.1,<0.15",
              # https://www.mercurial-scm.org/wiki/SupportedPythonVersions#Python_3.x_support
              # "mercurial>=4.7.2",  # Mercurial does not support Python 3 yet
              "requests>=2.20.1",
          ],
          extras_require=EXTRAS,
          python_requires=">=3.6",
          zip_safe=False)
<commit_msg>Bump pytest to 5.1.x and up.<commit_after># coding=utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

"""setuptools install script"""

from setuptools import find_packages
from setuptools import setup

EXTRAS = {
    "test": [
        "codecov==2.0.15",
        "coverage>=4.5.4,<4.6",
        "distro>=1.3.0",
        "flake8==3.7.8",
        "flake8-commas==2.0.0",
        "flake8-isort==2.7.0",
        "flake8-quotes==2.1.0",
        "isort==4.3.21",
        "pylint==2.3.1",
        "pytest>=5.1.1,<5.2",
        "pytest-cov>=2.7.1,<2.8",
        "pytest-flake8>=1.0.4,<1.1",
        "pytest-pylint>=0.14.0,<0.15",
    ]}

if __name__ == "__main__":
    setup(name="funfuzz",
          version="0.7.0a1",
          entry_points={
              "console_scripts": ["funfuzz = funfuzz.bot:main"],
          },
          package_data={"funfuzz": [
              "autobisectjs/*",
              "ccoverage/*",
              "js/*",
              "js/jsfunfuzz/*",
              "js/shared/*",
              "util/*",
          ]},
          package_dir={"": "src"},
          packages=find_packages(where="src"),
          install_requires=[
              "boto>=2.49.0",
              "fasteners>=0.14.1,<0.15",
              # https://www.mercurial-scm.org/wiki/SupportedPythonVersions#Python_3.x_support
              # "mercurial>=4.7.2",  # Mercurial does not support Python 3 yet
              "requests>=2.20.1",
          ],
          extras_require=EXTRAS,
          python_requires=">=3.6",
          zip_safe=False)
<|endoftext|>


<commit_before>#!/usr/bin/python
from distutils.sysconfig import get_python_lib
from distutils.core import setup
import glob
import os
from os.path import isfile, join

if isfile("MANIFEST"):
    os.unlink("MANIFEST")

# Get PYTHONLIB with no prefix so --prefix installs work.
PYTHONLIB = join(get_python_lib(standard_lib=1, prefix=''), 'site-packages')
ZONEINFO = join("dateutil", "zoneinfo")

setup(name="python-dateutil",
      version = "1.1",
      description = "Extensions to the standard python 2.3+ datetime module",
      author = "Gustavo Niemeyer",
      author_email = "gustavo@niemeyer.net",
      url = "http://labix.org/python-dateutil",
      license = "PSF License",
      long_description =
"""\
The dateutil module provides powerful extensions to the standard
datetime module, available in Python 2.3+.
""",
      packages = ["dateutil", "dateutil.zoneinfo"],
      data_files = [(join(PYTHONLIB, ZONEINFO),
                     glob.glob(join(ZONEINFO, "zoneinfo*.tar.*")))],
      )
<commit_msg>Import reordering.<commit_after>#!/usr/bin/python
from distutils.sysconfig import get_python_lib
from distutils.core import setup
from os.path import isfile, join
import glob
import os

if isfile("MANIFEST"):
    os.unlink("MANIFEST")

# Get PYTHONLIB with no prefix so --prefix installs work.
PYTHONLIB = join(get_python_lib(standard_lib=1, prefix=''), 'site-packages')
ZONEINFO = join("dateutil", "zoneinfo")

setup(name="python-dateutil",
      version = "1.1",
      description = "Extensions to the standard python 2.3+ datetime module",
      author = "Gustavo Niemeyer",
      author_email = "gustavo@niemeyer.net",
      url = "http://labix.org/python-dateutil",
      license = "PSF License",
      long_description =
"""\
The dateutil module provides powerful extensions to the standard
datetime module, available in Python 2.3+.
""",
      packages = ["dateutil", "dateutil.zoneinfo"],
      data_files = [(join(PYTHONLIB, ZONEINFO),
                     glob.glob(join(ZONEINFO, "zoneinfo*.tar.*")))],
      )
<|endoftext|>


<commit_before>from setuptools import setup, find_packages

long_description = '''
Mythril is a security analysis tool for Ethereum smart contracts. It
uses concolic analysis to detect various types of issues. Use it to
analyze source code or as a nmap-style black-box blockchain scanner (an
"ethermap" if you will).

Installation and setup
----------------------

Install from Pypi:

.. code:: bash

    $ pip install mythril

Or, clone the GitHub repo to install the newest master branch:

.. code:: bash

    $ git clone https://github.com/b-mueller/mythril/
    $ cd mythril
    $ python setup.py install

Note that Mythril requires Python 3.5 to work.

Function signatures
~~~~~~~~~~~~~~~~~~~

Whenever you disassemble or analyze binary code, Mythril will try to
resolve function names using its local signature database. The database
must be provided at ``~/.mythril/signatures.json``. You can start out
with the `default file <signatures.json>`__ as follows:

::

    $ cd ~/.mythril
    $ wget https://raw.githubusercontent.com/b-mueller/mythril/master/signatures.json

When you analyze Solidity code, new function signatures are added to the
database automatically.

Security analysis
-----------------

Run ``myth -x`` with one of the input options described below to run the
analysis. This will run the Python modules in the
`/analysis/modules <https://github.com/b-mueller/mythril/tree/master/mythril/analysis/modules>`__
directory.

Mythril detects a range of `security issues <security_checks.md>`__,
including integer underflows, owner-overwrite-to-Ether-withdrawal, and
others. However, the analysis will not detect business logic issues and
is not equivalent to formal verification.

Analyzing Solidity code
~~~~~~~~~~~~~~~~~~~~~~~

In order to work with Solidity source code files, the `solc command line
compiler <http://solidity.readthedocs.io/en/develop/using-the-compiler.html>`__
needs to be installed and in path. You can then provide the source
file(s) as positional arguments, e.g.:

.. code:: bash

    $ myth -x myContract.sol

Alternatively, compile the code on `Remix <http://remix.ethereum.org>`__
and pass the runtime binary code to Mythril:

.. code:: bash

    $ myth -x -c "0x5060(...)"

If you have multiple interdependent contracts, pass them to Mythril as
separate input files. Mythril will map the first contract to address
"0x0000(..)", the second one to "0x1111(...)", and so forth (make sure
that contract addresses are set accordingly in the source). The contract
passed in the first argument will be executed as the "main" contract.

.. code:: bash

    $ myth -x myContract.sol myLibrary.sol

Working with on-chain contracts
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

To analyze contracts on the blockchain you need an Ethereum node. By
default, Mythril will query a local node via RPC. Alternatively, you can
use `INFURA <https://infura.io>`__:

::

    $ myth --infura-mainnet -x -a 0x5c436ff914c458983414019195e0f4ecbef9e6dd

If you are planning to do batch operations or use the contract search
features, running a
`go-ethereum <https://github.com/ethereum/go-ethereum>`__ node is
recommended. Start your local node as follows:

.. code:: bash

    $ geth --rpc --rpcapi eth,debug --syncmode fast

Specify the target contract with the ``-a`` option:

.. code:: bash

    $ myth -x -a 0x5c436ff914c458983414019195e0f4ecbef9e6dd -v1

Adding the ``-l`` flag will cause Mythril to automatically retrieve
dependencies, such as library contracts:

.. code:: bash

    $  myth -x -a 0xEbFD99838cb0c132016B9E117563CB41f2B02264 -l -v1

Control flow graph
------------------

The ``-g FILENAME`` option generates an `interactive jsViz
graph <http://htmlpreview.github.io/?https://github.com/b-mueller/mythril/blob/master/static/mythril.html>`__:

.. code:: bash

    $ myth -g ./graph.html -a 0xEbFD99838cb0c132016B9E117563CB41f2B02264 -l

.. figure:: https://raw.githubusercontent.com/b-mueller/mythril/master/static/callgraph7.png
   :alt: Call graph

   callgraph

[STRIKEOUT:The "bounce" effect, while awesome (and thus enabled by
default), sometimes messes up the graph layout.] Try adding the
``--enable-physics`` flag for a very entertaining "bounce" effect that
unfortunately completely destroys usability.

Blockchain exploration
----------------------

Mythril builds its own contract database to enable fast search
operations. This enables operations like those described in the
`legendary "Mitch Brenner" blog
post <https://medium.com/@rtaylor30/how-i-snatched-your-153-037-eth-after-a-bad-tinder-date-d1d84422a50b>`__
in [STRIKEOUT:seconds] minutes instead of days. Unfortunately, the
initial sync process is slow. You don't need to sync the whole
blockchain right away though: If you abort the syncing process with
``ctrl+c``, it will be auto-resumed the next time you run the
``--init-db`` command.

.. code:: bash

    $ myth --init-db
    Starting synchronization from latest block: 4323706
    Processing block 4323000, 3 individual contracts in database
    (...)

The default behavior is to only sync contracts with a non-zero balance.
You can disable this behavior with the ``--sync-all`` flag, but be aware
that this will result in a huge (as in: dozens of GB) database.

Searching from the command line
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The search feature allows you to find contract instances that contain
specific function calls and opcode sequences. It supports simple boolean
expressions, such as:

.. code:: bash

    $ myth --search "func#changeMultisig(address)#"
    $ myth --search "code#PUSH1 0x50,POP#"
    $ myth --search "func#changeMultisig(address)# and code#PUSH1 0x50#"

Reading contract storage
~~~~~~~~~~~~~~~~~~~~~~~~

You can read the contents of storage slots from a deployed contract as
follows.

.. code:: bash

    ./myth --storage 0 -a "0x76799f77587738bfeef09452df215b63d2cfb08a"
    0x0000000000000000000000000000000000000000000000000000000000000003

Utilities
---------

Disassembler
~~~~~~~~~~~~

Use the ``-d`` flag to disassemble code. The disassembler accepts a
bytecode string or a contract address as its input.

.. code:: bash

    $ myth -d -c "0x6060"
    0 PUSH1 0x60

Specifying an address via ``-a ADDRESS`` will download the contract code
from your node.

.. code:: bash

    $ myth -d -a "0x2a0c0dbecc7e4d658f48e01e3fa353f44050c208"
    0 PUSH1 0x60
    2 PUSH1 0x40
    4 MSTORE
    (...)
    1135 - FUNCTION safeAdd(uint256,uint256) -
    1136 CALLVALUE
    1137 ISZERO

Finding cross-references
~~~~~~~~~~~~~~~~~~~~~~~~

It is often useful to find other contracts referenced by a particular
contract. E.g.:

.. code:: bash

    $ myth --search "code#DELEGATECALL#"
    Matched contract with code hash 07459966443977122e639cbf7804c446
    Address: 0x76799f77587738bfeef09452df215b63d2cfb08a, balance: 1000000000000000
    $ myth --xrefs -a 0x76799f77587738bfeef09452df215b63d2cfb08a
    5b9e8728e316bbeb692d22daaab74f6cbf2c4691

Calculating function hashes
~~~~~~~~~~~~~~~~~~~~~~~~~~~

To print the Keccak hash for a given function signature:

.. code:: bash

    $ myth --hash "setOwner(address)"
    0x13af4035

Credit
------

-  JSON RPC library is adapted from
   `ethjsonrpc <https://github.com/ConsenSys/ethjsonrpc>`__ (it doesn't
   seem to be maintained anymore, and I needed to make some changes to
   it).

-  The signature data in ``signatures.json`` was initially obtained from
   the `Ethereum Function Signature
   Database <https://www.4byte.directory>`__.
'''

setup(
    name='mythril',

    version='0.10.0',

    description='Security analysis tool for Ethereum smart contracts',
    long_description=long_description,

    url='https://github.com/b-mueller/mythril',

    author='Bernhard Mueller',
    author_email='bernhard.mueller11@gmail.com',

    license='MIT',

    classifiers=[
        'Development Status :: 3 - Alpha',

        'Intended Audience :: Science/Research',
        'Topic :: Software Development :: Disassemblers',

        'License :: OSI Approved :: MIT License',

        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
    ],

    keywords='hacking disassembler security ethereum',

    packages=find_packages(exclude=['contrib', 'docs', 'tests']),

    install_requires=[
        'ethereum>=2.0.4',
        'web3',
        'ZODB>=5.3.0',
        'z3-solver>=4.5',
        'laser-ethereum==0.4.0',
        'requests',
        'BTrees'
    ],

    python_requires='>=3.5',

    extras_require={
    },

    include_package_data=True,

    scripts=['myth']
)
<commit_msg>Bump version number<commit_after>from setuptools import setup, find_packages

long_description = '''
Mythril is a security analysis tool for Ethereum smart contracts. It
uses concolic analysis to detect various types of issues. Use it to
analyze source code or as a nmap-style black-box blockchain scanner (an
"ethermap" if you will).

Installation and setup
----------------------

Install from Pypi:

.. code:: bash

    $ pip install mythril

Or, clone the GitHub repo to install the newest master branch:

.. code:: bash

    $ git clone https://github.com/b-mueller/mythril/
    $ cd mythril
    $ python setup.py install

Note that Mythril requires Python 3.5 to work.

Function signatures
~~~~~~~~~~~~~~~~~~~

Whenever you disassemble or analyze binary code, Mythril will try to
resolve function names using its local signature database. The database
must be provided at ``~/.mythril/signatures.json``. You can start out
with the `default file <signatures.json>`__ as follows:

::

    $ cd ~/.mythril
    $ wget https://raw.githubusercontent.com/b-mueller/mythril/master/signatures.json

When you analyze Solidity code, new function signatures are added to the
database automatically.

Security analysis
-----------------

Run ``myth -x`` with one of the input options described below to run the
analysis. This will run the Python modules in the
`/analysis/modules <https://github.com/b-mueller/mythril/tree/master/mythril/analysis/modules>`__
directory.

Mythril detects a range of `security issues <security_checks.md>`__,
including integer underflows, owner-overwrite-to-Ether-withdrawal, and
others. However, the analysis will not detect business logic issues and
is not equivalent to formal verification.

Analyzing Solidity code
~~~~~~~~~~~~~~~~~~~~~~~

In order to work with Solidity source code files, the `solc command line
compiler <http://solidity.readthedocs.io/en/develop/using-the-compiler.html>`__
needs to be installed and in path. You can then provide the source
file(s) as positional arguments, e.g.:

.. code:: bash

    $ myth -x myContract.sol

Alternatively, compile the code on `Remix <http://remix.ethereum.org>`__
and pass the runtime binary code to Mythril:

.. code:: bash

    $ myth -x -c "0x5060(...)"

If you have multiple interdependent contracts, pass them to Mythril as
separate input files. Mythril will map the first contract to address
"0x0000(..)", the second one to "0x1111(...)", and so forth (make sure
that contract addresses are set accordingly in the source). The contract
passed in the first argument will be executed as the "main" contract.

.. code:: bash

    $ myth -x myContract.sol myLibrary.sol

Working with on-chain contracts
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

To analyze contracts on the blockchain you need an Ethereum node. By
default, Mythril will query a local node via RPC. Alternatively, you can
use `INFURA <https://infura.io>`__:

::

    $ myth --infura-mainnet -x -a 0x5c436ff914c458983414019195e0f4ecbef9e6dd

If you are planning to do batch operations or use the contract search
features, running a
`go-ethereum <https://github.com/ethereum/go-ethereum>`__ node is
recommended. Start your local node as follows:

.. code:: bash

    $ geth --rpc --rpcapi eth,debug --syncmode fast

Specify the target contract with the ``-a`` option:

.. code:: bash

    $ myth -x -a 0x5c436ff914c458983414019195e0f4ecbef9e6dd -v1

Adding the ``-l`` flag will cause Mythril to automatically retrieve
dependencies, such as library contracts:

.. code:: bash

    $  myth -x -a 0xEbFD99838cb0c132016B9E117563CB41f2B02264 -l -v1

Control flow graph
------------------

The ``-g FILENAME`` option generates an `interactive jsViz
graph <http://htmlpreview.github.io/?https://github.com/b-mueller/mythril/blob/master/static/mythril.html>`__:

.. code:: bash

    $ myth -g ./graph.html -a 0xEbFD99838cb0c132016B9E117563CB41f2B02264 -l

.. figure:: https://raw.githubusercontent.com/b-mueller/mythril/master/static/callgraph7.png
   :alt: Call graph

   callgraph

[STRIKEOUT:The "bounce" effect, while awesome (and thus enabled by
default), sometimes messes up the graph layout.] Try adding the
``--enable-physics`` flag for a very entertaining "bounce" effect that
unfortunately completely destroys usability.

Blockchain exploration
----------------------

Mythril builds its own contract database to enable fast search
operations. This enables operations like those described in the
`legendary "Mitch Brenner" blog
post <https://medium.com/@rtaylor30/how-i-snatched-your-153-037-eth-after-a-bad-tinder-date-d1d84422a50b>`__
in [STRIKEOUT:seconds] minutes instead of days. Unfortunately, the
initial sync process is slow. You don't need to sync the whole
blockchain right away though: If you abort the syncing process with
``ctrl+c``, it will be auto-resumed the next time you run the
``--init-db`` command.

.. code:: bash

    $ myth --init-db
    Starting synchronization from latest block: 4323706
    Processing block 4323000, 3 individual contracts in database
    (...)

The default behavior is to only sync contracts with a non-zero balance.
You can disable this behavior with the ``--sync-all`` flag, but be aware
that this will result in a huge (as in: dozens of GB) database.

Searching from the command line
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The search feature allows you to find contract instances that contain
specific function calls and opcode sequences. It supports simple boolean
expressions, such as:

.. code:: bash

    $ myth --search "func#changeMultisig(address)#"
    $ myth --search "code#PUSH1 0x50,POP#"
    $ myth --search "func#changeMultisig(address)# and code#PUSH1 0x50#"

Reading contract storage
~~~~~~~~~~~~~~~~~~~~~~~~

You can read the contents of storage slots from a deployed contract as
follows.

.. code:: bash

    ./myth --storage 0 -a "0x76799f77587738bfeef09452df215b63d2cfb08a"
    0x0000000000000000000000000000000000000000000000000000000000000003

Utilities
---------

Disassembler
~~~~~~~~~~~~

Use the ``-d`` flag to disassemble code. The disassembler accepts a
bytecode string or a contract address as its input.

.. code:: bash

    $ myth -d -c "0x6060"
    0 PUSH1 0x60

Specifying an address via ``-a ADDRESS`` will download the contract code
from your node.

.. code:: bash

    $ myth -d -a "0x2a0c0dbecc7e4d658f48e01e3fa353f44050c208"
    0 PUSH1 0x60
    2 PUSH1 0x40
    4 MSTORE
    (...)
    1135 - FUNCTION safeAdd(uint256,uint256) -
    1136 CALLVALUE
    1137 ISZERO

Finding cross-references
~~~~~~~~~~~~~~~~~~~~~~~~

It is often useful to find other contracts referenced by a particular
contract. E.g.:

.. code:: bash

    $ myth --search "code#DELEGATECALL#"
    Matched contract with code hash 07459966443977122e639cbf7804c446
    Address: 0x76799f77587738bfeef09452df215b63d2cfb08a, balance: 1000000000000000
    $ myth --xrefs -a 0x76799f77587738bfeef09452df215b63d2cfb08a
    5b9e8728e316bbeb692d22daaab74f6cbf2c4691

Calculating function hashes
~~~~~~~~~~~~~~~~~~~~~~~~~~~

To print the Keccak hash for a given function signature:

.. code:: bash

    $ myth --hash "setOwner(address)"
    0x13af4035

Credit
------

-  JSON RPC library is adapted from
   `ethjsonrpc <https://github.com/ConsenSys/ethjsonrpc>`__ (it doesn't
   seem to be maintained anymore, and I needed to make some changes to
   it).

-  The signature data in ``signatures.json`` was initially obtained from
   the `Ethereum Function Signature
   Database <https://www.4byte.directory>`__.
'''

setup(
    name='mythril',

    version='0.10.1',

    description='Security analysis tool for Ethereum smart contracts',
    long_description=long_description,

    url='https://github.com/b-mueller/mythril',

    author='Bernhard Mueller',
    author_email='bernhard.mueller11@gmail.com',

    license='MIT',

    classifiers=[
        'Development Status :: 3 - Alpha',

        'Intended Audience :: Science/Research',
        'Topic :: Software Development :: Disassemblers',

        'License :: OSI Approved :: MIT License',

        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
    ],

    keywords='hacking disassembler security ethereum',

    packages=find_packages(exclude=['contrib', 'docs', 'tests']),

    install_requires=[
        'ethereum>=2.0.4',
        'web3',
        'ZODB>=5.3.0',
        'z3-solver>=4.5',
        'laser-ethereum==0.4.0',
        'requests',
        'BTrees'
    ],

    python_requires='>=3.5',

    extras_require={
    },

    include_package_data=True,

    scripts=['myth']
)
<|endoftext|>


<commit_before>from os.path import abspath, basename, join, exists
from os import listdir, sep
from re import findall
from re import compile
from io import open

from .action_mapper import FileActionMapper

from logging import getLogger
log = getLogger(__name__)

# All output files marked with from_work_dir attributes will copied or downloaded
# this pattern picks up attiditional files to copy back - such as those
# associated with multiple outputs and metadata configuration. Set to .* to just
# copy everything
COPY_FROM_WORKING_DIRECTORY_PATTERN = compile(r"primary_.*|galaxy.json|metadata_.*")

class JobInputs(object):
    """
    Abstractions over dynamic inputs created for a given job (namely the command to
    execute and created configfiles).

    **Parameters**

    command_line : str
        Local command to execute for this job. (To be rewritten.)
    config_files : str
        Config files created for this job. (To be rewritten.)

    >>> import tempfile
    >>> tf = tempfile.NamedTemporaryFile()
    >>> def setup_inputs(tf):
    ...     open(tf.name, "w").write(u"world /path/to/input the rest")
    ...     inputs = JobInputs(u"hello /path/to/input", [tf.name])
    ...     return inputs
    >>> inputs = setup_inputs(tf)
    >>> inputs.rewrite_paths(u"/path/to/input", u'C:\\input')
    >>> inputs.rewritten_command_line == u'hello C:\\\\input'
    True
    >>> inputs.rewritten_config_files[tf.name] == u'world C:\\\\input the rest'
    True
    >>> tf.close()
    >>> tf = tempfile.NamedTemporaryFile()
    >>> inputs = setup_inputs(tf)
    >>> inputs.find_referenced_subfiles('/path/to') == [u'/path/to/input']
    True
    >>> inputs.path_referenced('/path/to')
    True
    >>> inputs.path_referenced(u'/path/to')
    True
    >>> inputs.path_referenced('/path/to/input')
    True
    >>> inputs.path_referenced('/path/to/notinput')
    False
    >>> tf.close()
    """

    def __init__(self, command_line, config_files):
        self.rewritten_command_line = command_line
        self.rewritten_config_files = {}
        for config_file in config_files or []:
            config_contents = _read(config_file)
            self.rewritten_config_files[config_file] = config_contents

    def find_referenced_subfiles(self, directory):
        """
        Return list of files below specified `directory` in job inputs. Could
        use more sophisticated logic (match quotes to handle spaces, handle
        subdirectories, etc...).

        **Parameters**

        directory : str
            Full path to directory to search.

        """
        pattern = r"(%s%s\S+)" % (directory, sep)
        referenced_files = set()
        for input_contents in self.__items():
            referenced_files.update(findall(pattern, input_contents))
        return list(referenced_files)

    def path_referenced(self, path):
        pattern = r"%s" % path
        found = False
        for input_contents in self.__items():
            if findall(pattern, input_contents):
                found = True
                break
        return found

    def rewrite_paths(self, local_path, remote_path):
        """
        Rewrite references to `local_path` with  `remote_path` in job inputs.
        """
        self.__rewrite_command_line(local_path, remote_path)
        self.__rewrite_config_files(local_path, remote_path)

    def __rewrite_command_line(self, local_path, remote_path):
        self.rewritten_command_line = self.rewritten_command_line.replace(local_path, remote_path)

    def __rewrite_config_files(self, local_path, remote_path):
        for config_file, rewritten_contents in self.rewritten_config_files.items():
            self.rewritten_config_files[config_file] = rewritten_contents.replace(local_path, remote_path)

    def __items(self):
        items = [self.rewritten_command_line]
        items.extend(self.rewritten_config_files.values())
        return items

class TransferTracker(object):

    def __init__(self, client, job_inputs):
        self.client = client
        self.action_mapper = FileActionMapper(client)
        self.job_inputs = job_inputs
        self.file_renames = {}

    def handle_transfer(self, path, type, name=None, contents=None):
        if contents:
            # If contents loaded in memory, no need to write out file and copy,
            # just transfer.
            action = ('transfer', )
        else:
            if not exists(path):
                message = "handle_tranfer called on non-existent file - [%s]" % path
                log.warn(message)
                raise Exception(message)
            action = self.__action(path, type)

        if action[0] in ['transfer', 'copy']:
            response = self.client.put_file(path, type, name=name, contents=contents)
            self.register_rewrite(path, response['path'], type, force=True)
        elif action[0] == 'none':
            # No action for this file.
            pass
        else:
            raise Exception("Unknown action type (%s) encountered for path (%s)" % (action[0], path))

    def register_rewrite(self, local_path, remote_path, type, force=False):
        action = self.__action(local_path, type)
        if action[0] in ['transfer', 'copy'] or force:
            self.file_renames[local_path] = remote_path

    def rewrite_input_paths(self):
        """
        For each file that has been transferred and renamed, updated
        command_line and configfiles to reflect that rewrite.
        """
        for local_path, remote_path in self.file_renames.items():
            self.job_inputs.rewrite_paths(local_path, remote_path)

    def __action(self, path, type):
        return self.action_mapper.action(path, type)

class FileStager(object):
    """
    Objects of the FileStager class interact with an LWR client object to
    stage the files required to run jobs on a remote LWR server.

    **Parameters**

    client : JobClient
        LWR client object.
    client_job_description : client_job_description
        Description of client view of job to stage and execute remotely.
    """

    def __init__(self, client, client_job_description, job_config):
        """
        """
        self.client = client
        self.command_line = client_job_description.command_line
        self.config_files = client_job_description.config_files
        self.input_files = client_job_description.input_files
        self.output_files = client_job_description.output_files
        self.tool_id = client_job_description.tool.id
        self.tool_version = client_job_description.tool.version
        self.tool_dir = abspath(client_job_description.tool.tool_dir)
        self.working_directory = client_job_description.working_directory

        # Setup job inputs, these will need to be rewritten before
        # shipping off to remote LWR server.
        self.job_inputs = JobInputs(self.command_line, self.config_files)

        self.transfer_tracker = TransferTracker(client, self.job_inputs)

        self.__handle_setup(job_config)
        self.__initialize_referenced_tool_files()
        self.__upload_tool_files()
        self.__upload_input_files()
        self.__upload_working_directory_files()
        self.__initialize_output_file_renames()
        self.__initialize_task_output_file_renames()
        self.__initialize_config_file_renames()
        self.__handle_rewrites()
        self.__upload_rewritten_config_files()

    def __handle_setup(self, job_config):
        if not job_config:
            job_config = self.client.setup(self.tool_id, self.tool_version)

        self.new_working_directory = job_config['working_directory']
        self.new_outputs_directory = job_config['outputs_directory']
        # Default configs_directory to match remote working_directory to mimic
        # behavior of older LWR servers.
        self.new_configs_drectory = job_config.get('configs_directory', self.new_working_directory)
        self.remote_path_separator = job_config['path_separator']
        # If remote LWR server assigned job id, use that otherwise
        # just use local job_id assigned.
        galaxy_job_id = self.client.job_id
        self.job_id = job_config.get('job_id', galaxy_job_id)
        if self.job_id != galaxy_job_id:
            # Remote LWR server assigned an id different than the
            # Galaxy job id, update client to reflect this.
            self.client.job_id = self.job_id

    def __initialize_referenced_tool_files(self):
        self.referenced_tool_files = self.job_inputs.find_referenced_subfiles(self.tool_dir)

    def __upload_tool_files(self):
        for referenced_tool_file in self.referenced_tool_files:
            self.transfer_tracker.handle_transfer(referenced_tool_file, 'tool')

    def __upload_input_files(self):
        for input_file in self.input_files:
            self.__upload_input_file(input_file)
            self.__upload_input_extra_files(input_file)

    def __upload_input_file(self, input_file):
        if self.job_inputs.path_referenced(input_file):
            if exists(input_file):
                self.transfer_tracker.handle_transfer(input_file, 'input')
            else:
                message = "LWR: __upload_input_file called on empty or missing dataset." + \
                          " So such file: [%s]" % input_file
                log.debug(message)

    def __upload_input_extra_files(self, input_file):
        # TODO: Determine if this is object store safe and what needs to be
        # done if it is not.
        files_path = "%s_files" % input_file[0:-len(".dat")]
        if exists(files_path) and self.job_inputs.path_referenced(files_path):
            for extra_file in listdir(files_path):
                extra_file_path = join(files_path, extra_file)
                relative_path = basename(files_path)
                extra_file_relative_path = join(relative_path, extra_file)
                self.transfer_tracker.handle_transfer(extra_file_path, 'input_extra', name=extra_file_relative_path)

    def __upload_working_directory_files(self):
        # Task manager stages files into working directory, these need to be
        # uploaded if present.
        for working_directory_file in listdir(self.working_directory):
            path = join(self.working_directory, working_directory_file)
            self.transfer_tracker.handle_transfer(path, 'work_dir')

    def __initialize_output_file_renames(self):
        for output_file in self.output_files:
            remote_path = r'%s%s%s' % (self.new_outputs_directory, self.remote_path_separator, basename(output_file))
            self.transfer_tracker.register_rewrite(output_file, remote_path, 'output')

    def __initialize_task_output_file_renames(self):
        for output_file in self.output_files:
            name = basename(output_file)
            task_file = join(self.working_directory, name)
            remote_path = r'%s%s%s' % (self.new_working_directory, self.remote_path_separator, name)
            self.transfer_tracker.register_rewrite(task_file, remote_path, 'output_task')

    def __initialize_config_file_renames(self):
        for config_file in self.config_files:
            remote_path = r'%s%s%s' % (self.new_configs_drectory, self.remote_path_separator, basename(config_file))
            self.transfer_tracker.register_rewrite(config_file, remote_path, 'config')

    def __handle_rewrites(self):
        """
        For each file that has been transferred and renamed, updated
        command_line and configfiles to reflect that rewrite.
        """
        self.transfer_tracker.rewrite_input_paths()

    def __upload_rewritten_config_files(self):
        for config_file, new_config_contents in self.job_inputs.rewritten_config_files.items():
            self.client.put_file(config_file, input_type='config', contents=new_config_contents)

    def get_rewritten_command_line(self):
        """
        Returns the rewritten version of the command line to execute suitable
        for remote host.
        """
        return self.job_inputs.rewritten_command_line

def finish_job(client, cleanup_job, job_completed_normally, working_directory, work_dir_outputs, output_files, working_directory_contents=[]):
    """
    """
    action_mapper = FileActionMapper(client)
    download_failure_exceptions = []
    downloaded_working_directory_files = []
    if job_completed_normally:
        # Fetch explicit working directory outputs.
        for source_file, output_file in work_dir_outputs:
            name = basename(source_file)
            try:
                action = action_mapper.action(output_file, 'output')
                client.fetch_work_dir_output(name, working_directory, output_file, action[0])
                downloaded_working_directory_files.append(name)
            except Exception as e:
                download_failure_exceptions.append(e)
            # Remove from full output_files list so don't try to download directly.
            output_files.remove(output_file)

        # Fetch output files.
        for output_file in output_files:
            try:
                action = action_mapper.action(output_file, 'output')
                client.fetch_output(output_file, working_directory=working_directory, action=action[0])
            except Exception as e:
                download_failure_exceptions.append(e)

        # Fetch remaining working directory outputs of interest.
        for name in working_directory_contents:
            if name in downloaded_working_directory_files:
                continue
            if COPY_FROM_WORKING_DIRECTORY_PATTERN.match(name):
                output_file = join(working_directory, name)
                action = action_mapper.action(output_file, 'output')
                client.fetch_work_dir_output(name, working_directory, output_file, action=action[0])
                downloaded_working_directory_files.append(name)

    return __clean(download_failure_exceptions, cleanup_job, client)

def __clean(download_failure_exceptions, cleanup_job, client):
    failed = (len(download_failure_exceptions) > 0)
    if not failed or cleanup_job == "always":
        try:
            client.clean()
        except:
            log.warn("Failed to cleanup remote LWR job")
    return failed

def submit_job(client, client_job_description, job_config=None):
    """
    """
    file_stager = FileStager(client, client_job_description, job_config)
    rebuilt_command_line = file_stager.get_rewritten_command_line()
    job_id = file_stager.job_id
    client.launch(rebuilt_command_line)
    return job_id

def _read(path):
    """
    Utility method to quickly read small files (config files and tool
    wrappers) into memory as bytes.
    """
    input = open(path, "r", encoding="utf-8")
    try:
        return input.read()
    finally:
        input.close()

class ClientJobDescription(object):
    """ A description of how client views job - command_line, inputs, etc..

    **Parameters**

    command_line : str
        The local command line to execute, this will be rewritten for the remote server.
    config_files : list
        List of Galaxy 'configfile's produced for this job. These will be rewritten and sent to remote server.
    input_files :  list
        List of input files used by job. These will be transferred and references rewritten.
    output_files : list
        List of output_files produced by job.
    tool_dir : str
        Directory containing tool to execute (if a wrapper is used, it will be transferred to remote server).
    working_directory : str
        Local path created by Galaxy for running this job.
    """

    def __init__(self, tool, command_line, config_files, input_files, output_files, working_directory):
        self.tool = tool
        self.command_line = command_line
        self.config_files = config_files
        self.input_files = input_files
        self.output_files = output_files
        self.working_directory = working_directory

__all__ = [submit_job, ClientJobDescription, finish_job]
<commit_msg>Fix bug related to cleaning of successful jobs.<commit_after>from os.path import abspath, basename, join, exists
from os import listdir, sep
from re import findall
from re import compile
from io import open

from .action_mapper import FileActionMapper

from logging import getLogger
log = getLogger(__name__)

# All output files marked with from_work_dir attributes will copied or downloaded
# this pattern picks up attiditional files to copy back - such as those
# associated with multiple outputs and metadata configuration. Set to .* to just
# copy everything
COPY_FROM_WORKING_DIRECTORY_PATTERN = compile(r"primary_.*|galaxy.json|metadata_.*")

class JobInputs(object):
    """
    Abstractions over dynamic inputs created for a given job (namely the command to
    execute and created configfiles).

    **Parameters**

    command_line : str
        Local command to execute for this job. (To be rewritten.)
    config_files : str
        Config files created for this job. (To be rewritten.)

    >>> import tempfile
    >>> tf = tempfile.NamedTemporaryFile()
    >>> def setup_inputs(tf):
    ...     open(tf.name, "w").write(u"world /path/to/input the rest")
    ...     inputs = JobInputs(u"hello /path/to/input", [tf.name])
    ...     return inputs
    >>> inputs = setup_inputs(tf)
    >>> inputs.rewrite_paths(u"/path/to/input", u'C:\\input')
    >>> inputs.rewritten_command_line == u'hello C:\\\\input'
    True
    >>> inputs.rewritten_config_files[tf.name] == u'world C:\\\\input the rest'
    True
    >>> tf.close()
    >>> tf = tempfile.NamedTemporaryFile()
    >>> inputs = setup_inputs(tf)
    >>> inputs.find_referenced_subfiles('/path/to') == [u'/path/to/input']
    True
    >>> inputs.path_referenced('/path/to')
    True
    >>> inputs.path_referenced(u'/path/to')
    True
    >>> inputs.path_referenced('/path/to/input')
    True
    >>> inputs.path_referenced('/path/to/notinput')
    False
    >>> tf.close()
    """

    def __init__(self, command_line, config_files):
        self.rewritten_command_line = command_line
        self.rewritten_config_files = {}
        for config_file in config_files or []:
            config_contents = _read(config_file)
            self.rewritten_config_files[config_file] = config_contents

    def find_referenced_subfiles(self, directory):
        """
        Return list of files below specified `directory` in job inputs. Could
        use more sophisticated logic (match quotes to handle spaces, handle
        subdirectories, etc...).

        **Parameters**

        directory : str
            Full path to directory to search.

        """
        pattern = r"(%s%s\S+)" % (directory, sep)
        referenced_files = set()
        for input_contents in self.__items():
            referenced_files.update(findall(pattern, input_contents))
        return list(referenced_files)

    def path_referenced(self, path):
        pattern = r"%s" % path
        found = False
        for input_contents in self.__items():
            if findall(pattern, input_contents):
                found = True
                break
        return found

    def rewrite_paths(self, local_path, remote_path):
        """
        Rewrite references to `local_path` with  `remote_path` in job inputs.
        """
        self.__rewrite_command_line(local_path, remote_path)
        self.__rewrite_config_files(local_path, remote_path)

    def __rewrite_command_line(self, local_path, remote_path):
        self.rewritten_command_line = self.rewritten_command_line.replace(local_path, remote_path)

    def __rewrite_config_files(self, local_path, remote_path):
        for config_file, rewritten_contents in self.rewritten_config_files.items():
            self.rewritten_config_files[config_file] = rewritten_contents.replace(local_path, remote_path)

    def __items(self):
        items = [self.rewritten_command_line]
        items.extend(self.rewritten_config_files.values())
        return items

class TransferTracker(object):

    def __init__(self, client, job_inputs):
        self.client = client
        self.action_mapper = FileActionMapper(client)
        self.job_inputs = job_inputs
        self.file_renames = {}

    def handle_transfer(self, path, type, name=None, contents=None):
        if contents:
            # If contents loaded in memory, no need to write out file and copy,
            # just transfer.
            action = ('transfer', )
        else:
            if not exists(path):
                message = "handle_tranfer called on non-existent file - [%s]" % path
                log.warn(message)
                raise Exception(message)
            action = self.__action(path, type)

        if action[0] in ['transfer', 'copy']:
            response = self.client.put_file(path, type, name=name, contents=contents)
            self.register_rewrite(path, response['path'], type, force=True)
        elif action[0] == 'none':
            # No action for this file.
            pass
        else:
            raise Exception("Unknown action type (%s) encountered for path (%s)" % (action[0], path))

    def register_rewrite(self, local_path, remote_path, type, force=False):
        action = self.__action(local_path, type)
        if action[0] in ['transfer', 'copy'] or force:
            self.file_renames[local_path] = remote_path

    def rewrite_input_paths(self):
        """
        For each file that has been transferred and renamed, updated
        command_line and configfiles to reflect that rewrite.
        """
        for local_path, remote_path in self.file_renames.items():
            self.job_inputs.rewrite_paths(local_path, remote_path)

    def __action(self, path, type):
        return self.action_mapper.action(path, type)

class FileStager(object):
    """
    Objects of the FileStager class interact with an LWR client object to
    stage the files required to run jobs on a remote LWR server.

    **Parameters**

    client : JobClient
        LWR client object.
    client_job_description : client_job_description
        Description of client view of job to stage and execute remotely.
    """

    def __init__(self, client, client_job_description, job_config):
        """
        """
        self.client = client
        self.command_line = client_job_description.command_line
        self.config_files = client_job_description.config_files
        self.input_files = client_job_description.input_files
        self.output_files = client_job_description.output_files
        self.tool_id = client_job_description.tool.id
        self.tool_version = client_job_description.tool.version
        self.tool_dir = abspath(client_job_description.tool.tool_dir)
        self.working_directory = client_job_description.working_directory

        # Setup job inputs, these will need to be rewritten before
        # shipping off to remote LWR server.
        self.job_inputs = JobInputs(self.command_line, self.config_files)

        self.transfer_tracker = TransferTracker(client, self.job_inputs)

        self.__handle_setup(job_config)
        self.__initialize_referenced_tool_files()
        self.__upload_tool_files()
        self.__upload_input_files()
        self.__upload_working_directory_files()
        self.__initialize_output_file_renames()
        self.__initialize_task_output_file_renames()
        self.__initialize_config_file_renames()
        self.__handle_rewrites()
        self.__upload_rewritten_config_files()

    def __handle_setup(self, job_config):
        if not job_config:
            job_config = self.client.setup(self.tool_id, self.tool_version)

        self.new_working_directory = job_config['working_directory']
        self.new_outputs_directory = job_config['outputs_directory']
        # Default configs_directory to match remote working_directory to mimic
        # behavior of older LWR servers.
        self.new_configs_drectory = job_config.get('configs_directory', self.new_working_directory)
        self.remote_path_separator = job_config['path_separator']
        # If remote LWR server assigned job id, use that otherwise
        # just use local job_id assigned.
        galaxy_job_id = self.client.job_id
        self.job_id = job_config.get('job_id', galaxy_job_id)
        if self.job_id != galaxy_job_id:
            # Remote LWR server assigned an id different than the
            # Galaxy job id, update client to reflect this.
            self.client.job_id = self.job_id

    def __initialize_referenced_tool_files(self):
        self.referenced_tool_files = self.job_inputs.find_referenced_subfiles(self.tool_dir)

    def __upload_tool_files(self):
        for referenced_tool_file in self.referenced_tool_files:
            self.transfer_tracker.handle_transfer(referenced_tool_file, 'tool')

    def __upload_input_files(self):
        for input_file in self.input_files:
            self.__upload_input_file(input_file)
            self.__upload_input_extra_files(input_file)

    def __upload_input_file(self, input_file):
        if self.job_inputs.path_referenced(input_file):
            if exists(input_file):
                self.transfer_tracker.handle_transfer(input_file, 'input')
            else:
                message = "LWR: __upload_input_file called on empty or missing dataset." + \
                          " So such file: [%s]" % input_file
                log.debug(message)

    def __upload_input_extra_files(self, input_file):
        # TODO: Determine if this is object store safe and what needs to be
        # done if it is not.
        files_path = "%s_files" % input_file[0:-len(".dat")]
        if exists(files_path) and self.job_inputs.path_referenced(files_path):
            for extra_file in listdir(files_path):
                extra_file_path = join(files_path, extra_file)
                relative_path = basename(files_path)
                extra_file_relative_path = join(relative_path, extra_file)
                self.transfer_tracker.handle_transfer(extra_file_path, 'input_extra', name=extra_file_relative_path)

    def __upload_working_directory_files(self):
        # Task manager stages files into working directory, these need to be
        # uploaded if present.
        for working_directory_file in listdir(self.working_directory):
            path = join(self.working_directory, working_directory_file)
            self.transfer_tracker.handle_transfer(path, 'work_dir')

    def __initialize_output_file_renames(self):
        for output_file in self.output_files:
            remote_path = r'%s%s%s' % (self.new_outputs_directory, self.remote_path_separator, basename(output_file))
            self.transfer_tracker.register_rewrite(output_file, remote_path, 'output')

    def __initialize_task_output_file_renames(self):
        for output_file in self.output_files:
            name = basename(output_file)
            task_file = join(self.working_directory, name)
            remote_path = r'%s%s%s' % (self.new_working_directory, self.remote_path_separator, name)
            self.transfer_tracker.register_rewrite(task_file, remote_path, 'output_task')

    def __initialize_config_file_renames(self):
        for config_file in self.config_files:
            remote_path = r'%s%s%s' % (self.new_configs_drectory, self.remote_path_separator, basename(config_file))
            self.transfer_tracker.register_rewrite(config_file, remote_path, 'config')

    def __handle_rewrites(self):
        """
        For each file that has been transferred and renamed, updated
        command_line and configfiles to reflect that rewrite.
        """
        self.transfer_tracker.rewrite_input_paths()

    def __upload_rewritten_config_files(self):
        for config_file, new_config_contents in self.job_inputs.rewritten_config_files.items():
            self.client.put_file(config_file, input_type='config', contents=new_config_contents)

    def get_rewritten_command_line(self):
        """
        Returns the rewritten version of the command line to execute suitable
        for remote host.
        """
        return self.job_inputs.rewritten_command_line

def finish_job(client, cleanup_job, job_completed_normally, working_directory, work_dir_outputs, output_files, working_directory_contents=[]):
    """
    """
    action_mapper = FileActionMapper(client)
    download_failure_exceptions = []
    downloaded_working_directory_files = []
    if job_completed_normally:
        # Fetch explicit working directory outputs.
        for source_file, output_file in work_dir_outputs:
            name = basename(source_file)
            try:
                action = action_mapper.action(output_file, 'output')
                client.fetch_work_dir_output(name, working_directory, output_file, action[0])
                downloaded_working_directory_files.append(name)
            except Exception as e:
                download_failure_exceptions.append(e)
            # Remove from full output_files list so don't try to download directly.
            output_files.remove(output_file)

        # Fetch output files.
        for output_file in output_files:
            try:
                action = action_mapper.action(output_file, 'output')
                client.fetch_output(output_file, working_directory=working_directory, action=action[0])
            except Exception as e:
                download_failure_exceptions.append(e)

        # Fetch remaining working directory outputs of interest.
        for name in working_directory_contents:
            if name in downloaded_working_directory_files:
                continue
            if COPY_FROM_WORKING_DIRECTORY_PATTERN.match(name):
                output_file = join(working_directory, name)
                action = action_mapper.action(output_file, 'output')
                client.fetch_work_dir_output(name, working_directory, output_file, action=action[0])
                downloaded_working_directory_files.append(name)

    return __clean(download_failure_exceptions, cleanup_job, client)

def __clean(download_failure_exceptions, cleanup_job, client):
    failed = (len(download_failure_exceptions) > 0)
    if (not failed and cleanup_job != "never") or cleanup_job == "always":
        try:
            client.clean()
        except:
            log.warn("Failed to cleanup remote LWR job")
    return failed

def submit_job(client, client_job_description, job_config=None):
    """
    """
    file_stager = FileStager(client, client_job_description, job_config)
    rebuilt_command_line = file_stager.get_rewritten_command_line()
    job_id = file_stager.job_id
    client.launch(rebuilt_command_line)
    return job_id

def _read(path):
    """
    Utility method to quickly read small files (config files and tool
    wrappers) into memory as bytes.
    """
    input = open(path, "r", encoding="utf-8")
    try:
        return input.read()
    finally:
        input.close()

class ClientJobDescription(object):
    """ A description of how client views job - command_line, inputs, etc..

    **Parameters**

    command_line : str
        The local command line to execute, this will be rewritten for the remote server.
    config_files : list
        List of Galaxy 'configfile's produced for this job. These will be rewritten and sent to remote server.
    input_files :  list
        List of input files used by job. These will be transferred and references rewritten.
    output_files : list
        List of output_files produced by job.
    tool_dir : str
        Directory containing tool to execute (if a wrapper is used, it will be transferred to remote server).
    working_directory : str
        Local path created by Galaxy for running this job.
    """

    def __init__(self, tool, command_line, config_files, input_files, output_files, working_directory):
        self.tool = tool
        self.command_line = command_line
        self.config_files = config_files
        self.input_files = input_files
        self.output_files = output_files
        self.working_directory = working_directory

__all__ = [submit_job, ClientJobDescription, finish_job]
<|endoftext|>


<commit_before>from distutils.core import setup

setup(
        name='coreclient',
        packages=['core'],
        version='0.2.1',
        license='BSD',
        description='Client libraries for accessing a CORE deployment',
        long_description=open('README').read(),
        author='Evan Borgstrom',
        author_email='evan@fatbox.ca',
        url='https://github.com/fatbox/coreclient',
        classifiers=[
            'Development Status :: 3 - Alpha',
            'Intended Audience :: Developers',
            'Intended Audience :: System Administrators',
            'License :: OSI Approved :: BSD License',
            'Natural Language :: English',
            'Topic :: Software Development :: Libraries',
            'Topic :: Software Development :: Libraries :: Python Modules',
            ],
        )
<commit_msg>Bump version<commit_after>from distutils.core import setup

setup(
        name='coreclient',
        packages=['core'],
        version='0.2.2',
        license='BSD',
        description='Client libraries for accessing a CORE deployment',
        long_description=open('README').read(),
        author='Evan Borgstrom',
        author_email='evan@fatbox.ca',
        url='https://github.com/fatbox/coreclient',
        classifiers=[
            'Development Status :: 3 - Alpha',
            'Intended Audience :: Developers',
            'Intended Audience :: System Administrators',
            'License :: OSI Approved :: BSD License',
            'Natural Language :: English',
            'Topic :: Software Development :: Libraries',
            'Topic :: Software Development :: Libraries :: Python Modules',
            ],
        )
<|endoftext|>


<commit_before>from setuptools import setup

setup(
    name = "PySPED",
    version = "0.1.3",
    author = "Aristides Caldeira",
    author_email = 'aristides.caldeira@tauga.com.br',
    test_suite='tests',
    keywords = ['nfe', 'nfse', 'cte', 'sped', 'edf', 'ecd'],
    classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Plugins',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Topic :: Software Development :: Libraries :: Python Modules',
    ],
    packages = [
        'pysped',
        'pysped.nfe',
        'pysped.nfe.leiaute',
        'pysped.nfe.danfe',
        'pysped.nfe.manual_300',
        'pysped.nfe.manual_401',
        'pysped.cte',
        'pysped.cte.leiaute',
        'pysped.cte.dacte',
        'pysped.efd',
        #'pysped.nfse',
        'pysped.xml_sped',
        'pysped.ecd',
        'pysped.nf_paulista',
        'pysped.relato_sped',
        'pysped.exemplos',
    ],
    package_data = {
        'pysped.nfe.danfe': ['fonts/*'],
        'pysped.relato_sped': ['fonts/*'],
        'pysped.nfe.leiaute': ['schema/*/*'],
        'pysped.cte.leiaute': ['schema/*/*'],
        'pysped.xml_sped': ['cadeia-certificadora/*/*']
    },
    url = 'https://github.com/aricaldeira/PySPED',
    license = 'LGPL-v2.1+',
    description = 'PySPED is a library to implement all requirements of the public system of bookkeeping digital',
    long_description = open('README.rst').read(),
    requires=[
        'Geraldo(>=0.4.16)',
        'xmlsec(>=1.0.5)',
        'qrcode(>=5.3)',
        'py3o.template(>=0.9.11)',
        'sh(>=1.12.9)'
    ],
    tests_require=[
        'pyflakes(>=0.6.1)',
    ],
)
<commit_msg>[FEA] Version bump<commit_after>from setuptools import setup

setup(
    name = "PySPED",
    version = "1.0.1",
    author = "Aristides Caldeira",
    author_email = 'aristides.caldeira@tauga.com.br',
    test_suite='tests',
    keywords = ['nfe', 'nfse', 'cte', 'sped', 'edf', 'ecd'],
    classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Plugins',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Topic :: Software Development :: Libraries :: Python Modules',
    ],
    packages = [
        'pysped',
        'pysped.nfe',
        'pysped.nfe.leiaute',
        'pysped.nfe.danfe',
        'pysped.nfe.manual_300',
        'pysped.nfe.manual_401',
        'pysped.cte',
        'pysped.cte.leiaute',
        'pysped.cte.dacte',
        'pysped.efd',
        #'pysped.nfse',
        'pysped.xml_sped',
        'pysped.ecd',
        'pysped.nf_paulista',
        'pysped.relato_sped',
        'pysped.exemplos',
    ],
    package_data = {
        'pysped.nfe.danfe': ['fonts/*'],
        'pysped.relato_sped': ['fonts/*'],
        'pysped.nfe.leiaute': ['schema/*/*'],
        'pysped.cte.leiaute': ['schema/*/*'],
        'pysped.xml_sped': ['cadeia-certificadora/*/*']
    },
    url = 'https://github.com/aricaldeira/PySPED',
    license = 'LGPL-v2.1+',
    description = 'PySPED is a library to implement all requirements of the public system of bookkeeping digital',
    long_description = open('README.rst').read(),
    requires=[
        'lxml(>=3.7.3))',
        'xmlsec(>=1.0.7)',
        'Geraldo(>=0.4.16)',
        'qrcode(>=5.3)',
        'py3o.template(>=0.9.11)',
        'sh(>=1.12.9)'
    ],
    tests_require=[
        'pyflakes(>=0.6.1)',
    ],
)
<|endoftext|>


<commit_before>from setuptools import setup, find_packages

setup(
    name="spinoff",
    version="0.2.3",
    packages=find_packages(),

    install_requires=[
        'twisted==12.0',
        'zope.interface',
        'txzmq',
        ],

    author="Erik Allik",
    author_email="erik.allik@skype.net",
)
<commit_msg>Bumped version to 0.2.4<commit_after>from setuptools import setup, find_packages

setup(
    name="spinoff",
    version="0.2.4",
    packages=find_packages(),

    install_requires=[
        'twisted==12.0',
        'zope.interface',
        'txzmq',
        ],

    author="Erik Allik",
    author_email="erik.allik@skype.net",
)
<|endoftext|>


<commit_before>import os
from setuptools import setup

dirs = ['/etc/intelmq', '/var/log/intelmq']
for dir in dirs:
    if not os.path.exists(dir):
        os.makedirs(dir)

setup(
    name='intelmq',
    version='0.1.0',
    maintainer='Tomas Lima',
    maintainer_email='tomas.lima@cert.pt',
    packages=[
                  'intelmq',
                  'intelmq.lib',
                  'intelmq.bots',
                  'intelmq.bots.inputs',
                  'intelmq.bots.inputs.arbor',
                  'intelmq.bots.inputs.abusehelper',
    ],
    scripts=['bin/intelmqctl'],
    url='http://pypi.python.org/pypi/IntelMQ/',
    license='MPL v2.0',
    description="IntelMQ Tool",
    long_description='IntelMQ is a solution for CERTs to process data feeds, pastebins, tweets throught a message queue.',
    package_data={
                  '': ['LICENSE', 'README.md', 'CONTRIBUTORS.md']
    },
    data_files=[('/etc/intelmq/', ['bin/conf/pipeline.conf.default', 'bin/conf/bots.conf.default', 'bin/conf/system.conf'])],
    install_requires=[
        "pika==0.9.14",
        "python-dateutil==1.5",
        "geoip2==0.5.1",
        "dnspython==1.11.1",
        "redis==2.4.9",
        "pymongo==2.7.1",
        "xmpppy==0.5.0rc1",
    ],
)
<commit_msg>preparing<commit_after>import os
from setuptools import setup, find_packages

dirs = ['/etc/intelmq', '/var/log/intelmq']
for dir in dirs:
    if not os.path.exists(dir):
        os.makedirs(dir)

setup(
    name='intelmq',
    version='0.1.0',
    maintainer='Tomas Lima',
    maintainer_email='tomas.lima@cert.pt',
    packages=find_packages(),
    scripts=['bin/intelmqctl'],
    url='http://pypi.python.org/pypi/IntelMQ/',
    license='MPL v2.0',
    description="IntelMQ Tool",
    long_description='IntelMQ is a solution for CERTs to process data feeds, pastebins, tweets throught a message queue.',
    package_data={
                  '': ['LICENSE', 'README.md', 'CONTRIBUTORS.md']
    },
    data_files=[('/etc/intelmq/', ['bin/conf/pipeline.conf.default', 'bin/conf/bots.conf.default', 'bin/conf/system.conf'])],
    install_requires=[
        "pika==0.9.14",
        "python-dateutil==1.5",
        "geoip2==0.5.1",
        "dnspython==1.11.1",
        "redis==2.4.9",
        "pymongo==2.7.1",
        "xmpppy==0.5.0rc1",
    ],
)
<|endoftext|>


<commit_before>#!/usr/bin/env python
from setuptools import setup, find_packages
import os, re

PKG="twitter_users"
verstr="unknown"

setup(name=PKG,
      version=verstr,
      description="Django app for Twitter authentication",
      author="Matt Diephouse",
      author_email="matt@diephouse.com",
      url="https://github.com/mdiep/django-twitter-users",
      packages = find_packages(),
      install_requires = ['Django', 'oauth2'],
      license = "BSD3",
      keywords="oauth",
      zip_safe = True
      )
<commit_msg>Set the version in setup.py<commit_after>#!/usr/bin/env python
from setuptools import setup, find_packages
import os, re

PKG="twitter_users"
verstr="0.1"

setup(name=PKG,
      version=verstr,
      description="Django app for Twitter authentication",
      author="Matt Diephouse",
      author_email="matt@diephouse.com",
      url="https://github.com/mdiep/django-twitter-users",
      packages = find_packages(),
      install_requires = ['Django', 'oauth2'],
      license = "BSD3",
      keywords="oauth",
      zip_safe = True
      )
<|endoftext|>