<commit_before>import sys
import os
import getopt
import tempfile
import shutil
import ConfigParser

from os.path import abspath, join, expanduser, isfile

from python import Python
from setuptools import Setuptools
from scp import SCP
from scm import SCMContainer
from exit import msg_exit, err_exit

pypiurl = "http://pypi.python.org/pypi"
maxaliasdepth = 23

version = "jarn.mkrelease 2.0b1"
usage = "Try 'mkrelease --help' for more information."
help = """\
Usage: mkrelease [options] [scm-url|scm-sandbox]

Release sdist eggs

Options:
  -C, --skip-checkin  Do not checkin modified files from the sandbox.
  -T, --skip-tag      Do not tag the release in SCM.
  -S, --skip-upload   Do not upload the release to dist-location.
  -D, --dry-run       Dry-run; equivalent to -CTS.

  --svn, --hg, --git  Select the SCM type. Only required if the SCM type
                      cannot be guessed from the argument.

  -s, --sign          Sign the release with GnuPG.
  -i identity, --identity=identity
                      The GnuPG identity to sign with.

  -d dist-location, --dist-location=dist-location
                      An scp destination specification, or an index server
                      configured in ~/.pypirc, or an alias name for either.
                      This option may be specified more than once.

  -p, --push          Push changes upstream (hg and git only).
  -q, --quiet         Suppress output of setuptools commands.
  -k, --keep-temp     Keep the temporary build directory.
  -h, --help          Print this help message and exit.
  -v, --version       Print the version string and exit.

  scm-url             The URL of a remote SCM repository.
  scm-sandbox         A local SCM sandbox; defaults to the current working
                      directory.
"""

class Defaults(object):

    def __init__(self):
        """Read config files.
        """
        self.parser = ConfigParser.ConfigParser()
        self.parser.read((expanduser('~/.pypirc'), '/etc/mkrelease',
                          expanduser('~/.mkrelease')))

        def get(section, key, default=None):
            if self.parser.has_option(section, key):
                return self.parser.get(section, key)
            return default

        self.python = sys.executable
        self.distbase = get('defaults', 'distbase', '')
        self.distdefault = get('defaults', 'distdefault', '')

        self.aliases = {}
        if self.parser.has_section('aliases'):
            for key, value in self.parser.items('aliases'):
                self.aliases[key] = value.split()

        self.servers = {}
        for server in get('distutils', 'index-servers', '').split():
            self.servers[server] = True
            url = get(server, 'repository', pypiurl)
            self.servers[url] = True

class Locations(object):

    def __init__(self, defaults):
        self.distbase = defaults.distbase
        self.distdefault = defaults.distdefault
        self.aliases = defaults.aliases
        self.servers = defaults.servers
        self.locations = []

    def __len__(self):
        """Return number of locations.
        """
        return len(self.locations)

    def __iter__(self):
        """Iterate over locations.
        """
        return iter(self.locations)

    def extend(self, location):
        """Extend list of locations.
        """
        self.locations.extend(location)

    def is_server(self, location):
        """Return True if 'location' is an index server.
        """
        return location in self.servers

    def has_host(self, location):
        """Return True if 'location' contains a host part.
        """
        colon = location.find(':')
        slash = location.find('/')
        return colon > 0 and (slash < 0 or slash > colon)

    def get_location(self, location, depth=0):
        """Resolve aliases and apply distbase.
        """
        if not location:
            return []
        if location in self.aliases:
            res = []
            if depth > maxaliasdepth:
                err_exit('Maximum alias depth exceeded: %(location)s' % locals())
            for loc in self.aliases[location]:
                res.extend(self.get_location(loc, depth+1))
            return res
        if self.is_server(location):
            return [location]
        if not self.has_host(location) and self.distbase:
            sep = '/'
            if self.distbase[-1] in (':', '/'):
                sep = ''
            return [self.distbase + sep + location]
        return [location]

    def default_location(self):
        """Return the default location.
        """
        return self.get_location(self.distdefault)

    def check_valid_locations(self, locations=None):
        """Fail if 'locations' is empty or contains bad scp destinations.
        """
        if locations is None:
            locations = self.locations
        if not locations:
            err_exit('mkrelease: option -d is required\n%s' % usage)
        for location in locations:
            if not self.is_server(location) and not self.has_host(location):
                err_exit('Scp destination must contain host part: %(location)s' % locals())

class ReleaseMaker(object):

    def __init__(self, args):
        """Set defaults.
        """
        self.skipcheckin = False
        self.skiptag = False
        self.skipupload = False
        self.keeptemp = False
        self.push = False
        self.quiet = False
        self.infoflags = ['--no-svn-revision', '--no-date', '--tag-build=""']
        self.sdistflags = ['--formats="zip"']
        self.uploadflags = []
        self.directory = os.curdir
        self.defaults = Defaults()
        self.locations = Locations(self.defaults)
        self.python = Python(self.defaults)
        self.setuptools = Setuptools(self.defaults)
        self.scp = SCP()
        self.scmcontainer = SCMContainer()
        self.scm = None
        self.scmtype = ''
        self.args = args

    def get_options(self):
        """Parse command line.
        """
        try:
            options, args = getopt.getopt(self.args, 'CDSTd:hi:kpqsv',
                ('skip-checkin', 'skip-tag', 'skip-upload', 'dry-run', 'keep-temp',
                 'sign', 'identity=', 'dist-location=', 'version', 'help',
                 'push', 'quiet', 'svn', 'hg', 'git', 'develop'))
        except getopt.GetoptError, e:
            err_exit('mkrelease: %s\n%s' % (e.msg, usage))

        for name, value in options:
            if name in ('-C', '--skip-checkin'):
                self.skipcheckin = True
            elif name in ('-T', '--skip-tag'):
                self.skiptag = True
            elif name in ('-S', '--skip-upload'):
                self.skipupload = True
            elif name in ('-D', '--dry-run'):
                self.skipcheckin = self.skiptag = self.skipupload = True
            elif name in ('-k', '--keep-temp'):
                self.keeptemp = True
            elif name in ('-p', '--push'):
                self.push = True
            elif name in ('-q', '--quiet'):
                self.quiet = True
            elif name in ('-s', '--sign'):
                self.uploadflags.append('--sign')
            elif name in ('-i', '--identity'):
                self.uploadflags.append('--identity="%s"' % value)
            elif name in ('-d', '--dist-location'):
                self.locations.extend(self.locations.get_location(value))
            elif name in ('-v', '--version'):
                msg_exit(version)
            elif name in ('-h', '--help'):
                msg_exit(help)
            elif name in ('--svn', '--hg', '--git'):
                self.scmtype = name[2:]
            elif name in ('--develop',):
                self.infoflags = []

        if self.uploadflags and '--sign' not in self.uploadflags:
            self.uploadflags.append('--sign')

        if not self.locations:
            self.locations.extend(self.locations.default_location())

        if not self.skipupload:
            self.locations.check_valid_locations()

        if len(args) > 1:
            err_exit('mkrelease: too many arguments\n%s' % usage)

        if args:
            self.directory = args[0]

    def get_packageurl(self):
        """Get URL to release.
        """
        directory = self.directory
        self.scm = self.scmcontainer.guess_scm(self.scmtype, directory)

        if self.scm.is_valid_url(directory):
            self.remoteurl = directory
            self.isremote = True
            self.push = True

            if directory.startswith('file://') and not directory.startswith('file:///'):
                err_exit('File URLs must be absolute: %(directory)s' % locals())
        else:
            directory = abspath(directory)
            self.scm.check_valid_sandbox(directory)
            self.setuptools.check_valid_package(directory)

            self.remoteurl = self.scm.get_url_from_sandbox(directory)
            self.isremote = False

            name, version = self.setuptools.get_package_info(directory)
            print 'Releasing', name, version

            if not self.skipcheckin:
                if self.scm.is_dirty_sandbox(directory):
                    self.scm.checkin_sandbox(
                        directory, name, version, self.push)

    def make_release(self):
        """Build and distribute the egg.
        """
        tempdir = abspath(tempfile.mkdtemp(prefix='mkrelease-'))
        directory = join(tempdir, 'checkout')
        scmtype = self.scm.name
        infoflags = self.infoflags
        sdistflags = self.sdistflags
        uploadflags = self.uploadflags

        try:
            if self.isremote:
                self.scm.checkout_url(self.remoteurl, directory)
            else:
                directory = abspath(self.directory)

            self.scm.check_valid_sandbox(directory)
            self.setuptools.check_valid_package(directory)
            self.scm.check_dirty_sandbox(directory)
            self.scm.check_unclean_sandbox(directory)

            name, version = self.setuptools.get_package_info(directory)
            if self.isremote:
                print 'Releasing', name, version

            if not self.skiptag:
                tagid = self.scm.get_tag_id(directory, version)
                self.scm.check_tag_exists(directory, tagid)
                print 'Tagging', name, version
                self.scm.create_tag(
                    directory, tagid, name, version, self.push)

            distfile = self.setuptools.run_sdist(
                directory, infoflags, sdistflags, scmtype, self.quiet)

            if not self.skipupload:
                for location in self.locations:
                    if self.locations.is_server(location):
                        if '--sign' in uploadflags:
                            if isfile(distfile+'.asc'):
                                os.remove(distfile+'.asc')
                        self.setuptools.run_upload(
                            directory, location, infoflags, sdistflags, uploadflags, scmtype)
                    else:
                        self.scp.run_scp(
                            distfile, location)
        finally:
            if not self.keeptemp:
                shutil.rmtree(tempdir)

    def run(self):
        self.python.check_valid_python()
        self.get_options()
        self.get_packageurl()
        self.make_release()
        print 'done'

def main(args=None):
    if args is None:
        args = sys.argv[1:]
    try:
        ReleaseMaker(args).run()
    except SystemExit, e:
        return e.code
    return 0

if __name__ == '__main__':
    sys.exit(main())

<commit_msg>Make -e short for --develop.<commit_after>import sys
import os
import getopt
import tempfile
import shutil
import ConfigParser

from os.path import abspath, join, expanduser, isfile

from python import Python
from setuptools import Setuptools
from scp import SCP
from scm import SCMContainer
from exit import msg_exit, err_exit

pypiurl = "http://pypi.python.org/pypi"
maxaliasdepth = 23

version = "jarn.mkrelease 2.0b1"
usage = "Try 'mkrelease --help' for more information."
help = """\
Usage: mkrelease [options] [scm-url|scm-sandbox]

Release sdist eggs

Options:
  -C, --skip-checkin  Do not checkin modified files from the sandbox.
  -T, --skip-tag      Do not tag the release in SCM.
  -S, --skip-upload   Do not upload the release to dist-location.
  -D, --dry-run       Dry-run; equivalent to -CTS.

  --svn, --hg, --git  Select the SCM type. Only required if the SCM type
                      cannot be guessed from the argument.

  -s, --sign          Sign the release with GnuPG.
  -i identity, --identity=identity
                      The GnuPG identity to sign with.

  -d dist-location, --dist-location=dist-location
                      An scp destination specification, or an index server
                      configured in ~/.pypirc, or an alias name for either.
                      This option may be specified more than once.

  -p, --push          Push changes upstream (hg and git only).
  -q, --quiet         Suppress output of setuptools commands.
  -k, --keep-temp     Keep the temporary build directory.
  -h, --help          Print this help message and exit.
  -v, --version       Print the version string and exit.

  scm-url             The URL of a remote SCM repository.
  scm-sandbox         A local SCM sandbox; defaults to the current working
                      directory.
"""

class Defaults(object):

    def __init__(self):
        """Read config files.
        """
        self.parser = ConfigParser.ConfigParser()
        self.parser.read((expanduser('~/.pypirc'), '/etc/mkrelease',
                          expanduser('~/.mkrelease')))

        def get(section, key, default=None):
            if self.parser.has_option(section, key):
                return self.parser.get(section, key)
            return default

        self.python = sys.executable
        self.distbase = get('defaults', 'distbase', '')
        self.distdefault = get('defaults', 'distdefault', '')

        self.aliases = {}
        if self.parser.has_section('aliases'):
            for key, value in self.parser.items('aliases'):
                self.aliases[key] = value.split()

        self.servers = {}
        for server in get('distutils', 'index-servers', '').split():
            self.servers[server] = True
            url = get(server, 'repository', pypiurl)
            self.servers[url] = True

class Locations(object):

    def __init__(self, defaults):
        self.distbase = defaults.distbase
        self.distdefault = defaults.distdefault
        self.aliases = defaults.aliases
        self.servers = defaults.servers
        self.locations = []

    def __len__(self):
        """Return number of locations.
        """
        return len(self.locations)

    def __iter__(self):
        """Iterate over locations.
        """
        return iter(self.locations)

    def extend(self, location):
        """Extend list of locations.
        """
        self.locations.extend(location)

    def is_server(self, location):
        """Return True if 'location' is an index server.
        """
        return location in self.servers

    def has_host(self, location):
        """Return True if 'location' contains a host part.
        """
        colon = location.find(':')
        slash = location.find('/')
        return colon > 0 and (slash < 0 or slash > colon)

    def get_location(self, location, depth=0):
        """Resolve aliases and apply distbase.
        """
        if not location:
            return []
        if location in self.aliases:
            res = []
            if depth > maxaliasdepth:
                err_exit('Maximum alias depth exceeded: %(location)s' % locals())
            for loc in self.aliases[location]:
                res.extend(self.get_location(loc, depth+1))
            return res
        if self.is_server(location):
            return [location]
        if not self.has_host(location) and self.distbase:
            sep = '/'
            if self.distbase[-1] in (':', '/'):
                sep = ''
            return [self.distbase + sep + location]
        return [location]

    def default_location(self):
        """Return the default location.
        """
        return self.get_location(self.distdefault)

    def check_valid_locations(self, locations=None):
        """Fail if 'locations' is empty or contains bad scp destinations.
        """
        if locations is None:
            locations = self.locations
        if not locations:
            err_exit('mkrelease: option -d is required\n%s' % usage)
        for location in locations:
            if not self.is_server(location) and not self.has_host(location):
                err_exit('Scp destination must contain host part: %(location)s' % locals())

class ReleaseMaker(object):

    def __init__(self, args):
        """Set defaults.
        """
        self.skipcheckin = False
        self.skiptag = False
        self.skipupload = False
        self.keeptemp = False
        self.push = False
        self.quiet = False
        self.infoflags = ['--no-svn-revision', '--no-date', '--tag-build=""']
        self.sdistflags = ['--formats="zip"']
        self.uploadflags = []
        self.directory = os.curdir
        self.defaults = Defaults()
        self.locations = Locations(self.defaults)
        self.python = Python(self.defaults)
        self.setuptools = Setuptools(self.defaults)
        self.scp = SCP()
        self.scmcontainer = SCMContainer()
        self.scm = None
        self.scmtype = ''
        self.args = args

    def get_options(self):
        """Parse command line.
        """
        try:
            options, args = getopt.getopt(self.args, 'CDSTd:ehi:kpqsv',
                ('skip-checkin', 'skip-tag', 'skip-upload', 'dry-run', 'keep-temp',
                 'sign', 'identity=', 'dist-location=', 'version', 'help',
                 'push', 'quiet', 'svn', 'hg', 'git', 'develop'))
        except getopt.GetoptError, e:
            err_exit('mkrelease: %s\n%s' % (e.msg, usage))

        for name, value in options:
            if name in ('-C', '--skip-checkin'):
                self.skipcheckin = True
            elif name in ('-T', '--skip-tag'):
                self.skiptag = True
            elif name in ('-S', '--skip-upload'):
                self.skipupload = True
            elif name in ('-D', '--dry-run'):
                self.skipcheckin = self.skiptag = self.skipupload = True
            elif name in ('-k', '--keep-temp'):
                self.keeptemp = True
            elif name in ('-p', '--push'):
                self.push = True
            elif name in ('-q', '--quiet'):
                self.quiet = True
            elif name in ('-s', '--sign'):
                self.uploadflags.append('--sign')
            elif name in ('-i', '--identity'):
                self.uploadflags.append('--identity="%s"' % value)
            elif name in ('-d', '--dist-location'):
                self.locations.extend(self.locations.get_location(value))
            elif name in ('-v', '--version'):
                msg_exit(version)
            elif name in ('-h', '--help'):
                msg_exit(help)
            elif name in ('--svn', '--hg', '--git'):
                self.scmtype = name[2:]
            elif name in ('-e', '--develop'):
                self.infoflags = []

        if self.uploadflags and '--sign' not in self.uploadflags:
            self.uploadflags.append('--sign')

        if not self.locations:
            self.locations.extend(self.locations.default_location())

        if not self.skipupload:
            self.locations.check_valid_locations()

        if len(args) > 1:
            err_exit('mkrelease: too many arguments\n%s' % usage)

        if args:
            self.directory = args[0]

    def get_packageurl(self):
        """Get URL to release.
        """
        directory = self.directory
        self.scm = self.scmcontainer.guess_scm(self.scmtype, directory)

        if self.scm.is_valid_url(directory):
            self.remoteurl = directory
            self.isremote = True
            self.push = True

            if directory.startswith('file://') and not directory.startswith('file:///'):
                err_exit('File URLs must be absolute: %(directory)s' % locals())
        else:
            directory = abspath(directory)
            self.scm.check_valid_sandbox(directory)
            self.setuptools.check_valid_package(directory)

            self.remoteurl = self.scm.get_url_from_sandbox(directory)
            self.isremote = False

            name, version = self.setuptools.get_package_info(directory)
            print 'Releasing', name, version

            if not self.skipcheckin:
                if self.scm.is_dirty_sandbox(directory):
                    self.scm.checkin_sandbox(
                        directory, name, version, self.push)

    def make_release(self):
        """Build and distribute the egg.
        """
        tempdir = abspath(tempfile.mkdtemp(prefix='mkrelease-'))
        directory = join(tempdir, 'checkout')
        scmtype = self.scm.name
        infoflags = self.infoflags
        sdistflags = self.sdistflags
        uploadflags = self.uploadflags

        try:
            if self.isremote:
                self.scm.checkout_url(self.remoteurl, directory)
            else:
                directory = abspath(self.directory)

            self.scm.check_valid_sandbox(directory)
            self.setuptools.check_valid_package(directory)
            self.scm.check_dirty_sandbox(directory)
            self.scm.check_unclean_sandbox(directory)

            name, version = self.setuptools.get_package_info(directory)
            if self.isremote:
                print 'Releasing', name, version

            if not self.skiptag:
                tagid = self.scm.get_tag_id(directory, version)
                self.scm.check_tag_exists(directory, tagid)
                print 'Tagging', name, version
                self.scm.create_tag(
                    directory, tagid, name, version, self.push)

            distfile = self.setuptools.run_sdist(
                directory, infoflags, sdistflags, scmtype, self.quiet)

            if not self.skipupload:
                for location in self.locations:
                    if self.locations.is_server(location):
                        if '--sign' in uploadflags:
                            if isfile(distfile+'.asc'):
                                os.remove(distfile+'.asc')
                        self.setuptools.run_upload(
                            directory, location, infoflags, sdistflags, uploadflags, scmtype)
                    else:
                        self.scp.run_scp(
                            distfile, location)
        finally:
            if not self.keeptemp:
                shutil.rmtree(tempdir)

    def run(self):
        self.python.check_valid_python()
        self.get_options()
        self.get_packageurl()
        self.make_release()
        print 'done'

def main(args=None):
    if args is None:
        args = sys.argv[1:]
    try:
        ReleaseMaker(args).run()
    except SystemExit, e:
        return e.code
    return 0

if __name__ == '__main__':
    sys.exit(main())

<|endoftext|>


<commit_before>import signal
from cmd import Cmd
from functools import partial
import sys
import traceback

from colorama import Fore
from PluginManager import PluginManager

from utilities import schedule
from utilities.voice import create_voice
from utilities.notification import notify
from utilities.GeneralUtilities import print_say

from packages.memory.memory import Memory
from utilities.animations import SpinnerThread

class JarvisAPI(object):
    """
    Jarvis interface for plugins.

    Plugins will receive a instance of this as the second (non-self) parameter
    of the exec()-method.

    Everything Jarvis-related that can't be implemented as a stateless-function
    in the utilities-package should be implemented here.
    """

    _CONNECTION_ERROR_MSG = "You are not connected to Internet"

    def __init__(self, jarvis):
        self._jarvis = jarvis
        self.spinner_running = False

    def say(self, text, color="", speak=True):
        """
        This method give the jarvis the ability to print a text
        and talk when sound is enable.
        :param text: the text to print (or talk)
        :param color: for text - use colorama (https://pypi.org/project/colorama/)
                      e.g. Fore.BLUE
        :param speak: False-, if text shouldn't be spoken even if speech is enabled
        """
        print(color + text + Fore.RESET)
        if speak:
            self._jarvis.speak(text)

    def input(self, prompt="", color=""):
        """
        Get user input
        """
        # we can't use input because for some reason input() and color codes do not work on
        # windows cmd
        sys.stdout.write(color + prompt + Fore.RESET)
        sys.stdout.flush()
        text = sys.stdin.readline()
        # return without newline
        return text.rstrip()

    def input_number(self, prompt="", color="", rtype=float, rmin=None, rmax=None):
        """
        Get user input: As number.

        Guaranteed only returns number - ask user till correct number entered.

        :param prompt: Printed to console
        :param color: Color of prompot
        :param rtype: type of return value; e.g. float (default) or int
        :param rmin: Minum of values returned
        :param rmax: Maximum of values returned
        """
        while True:
            try:
                value = rtype(self.input(prompt, color).replace(',', '.'))
                if (rmin is not None and value < rmin) or (rmax is not None and value > rmax):
                    prompt = "Sorry, needs to be between {} and {}. Try again: ".format(rmin, rmax)
                else:
                    return value
            except ValueError:
                prompt = 'Sorry, needs to be a number. Try again: '
                continue

    def connection_error(self):
        """Print generic connection error"""

        if self.is_spinner_running():
            self.spinner_stop('')

        self.say(JarvisAPI._CONNECTION_ERROR_MSG)

    def exit(self):
        """Immediately exit Jarvis"""
        self._jarvis.close()

    def notification(self, msg, time_seconds=0):
        """
        Sends notification msg in time_in milliseconds
        :param msg: Message. Either String (message body) or tuple (headline, message body)
        :param time_seconds: Time in seconds to wait before showing notification
        """
        if isinstance(msg, tuple):
            headline, message = msg
        elif isinstance(msg, str):
            headline = "Jarvis"
            message = msg
        else:
            raise ValueError("msg not a string or tuple")

        if time_seconds == 0:
            notify(headline, message)
        else:
            schedule(time_seconds, notify, headline, message)

    def schedule(self, time_seconds, function, *args):
        """
        Schedules function
        After time_seconds call function with these parameter:
           - reference to this JarvisAPI instance
           - schedule_id (return value of this function)
           - *args
        :return: integer, id - use with cancel
        """
        return self._jarvis.scheduler.create_event(
            time_seconds, function, self, *args)

    def cancel(self, schedule_id):
        """
        Cancel event scheduled with schedule
        :param schedule_id: id returned by schedule
        """
        spinner = SpinnerThread('Cancelling', 0.15)
        spinner.start()

        self._jarvis.scheduler.cancel(schedule_id)

        spinner.stop()
        jarvis.say('Cancellation successful', Fore.GREEN)

    # Voice wrapper
    def enable_voice(self):
        """
        Use text to speech for every text passed to jarvis.say()
        """
        self._jarvis.enable_voice = True

    def disable_voice(self):
        """
        Stop text to speech output for every text passed to jarvis.say()
        """
        self._jarvis.enable_voice = False

    def is_voice_enabled(self):
        """
        Returns True/False if voice is enabled/disabled with
        enable_voice or disable_voice
        Default: False (disabled)
        """
        return self._jarvis.enable_voice

    # MEMORY WRAPPER
    def get_data(self, key):
        """
        Get a specific key from memory
        """
        return self._jarvis.memory.get_data(key)

    def add_data(self, key, value):
        """
        Add a key and value to memory
        """
        self._jarvis.memory.add_data(key, value)
        self._jarvis.memory.save()

    def update_data(self, key, value):
        """
        Updates a key with supplied value.
        """
        self._jarvis.memory.update_data(key, value)
        self._jarvis.memory.save()

    def del_data(self, key):
        """
        Delete a key from memory
        """
        self._jarvis.memory.del_data(key)
        self._jarvis.memory.save()

    def eval(self, s):
        """
        Simulates typing 's' in Jarvis prompt
        """
        line = self._jarvis.precmd(s)
        stop = self._jarvis.onecmd(line)
        stop = self._jarvis.postcmd(stop, line)

    def spinner_start(self, message="Starting "):
        """
        Function for starting a spinner when prompted from a plugin
        and a default message for performing the task
        """
        self.spinner_running = True
        self.spinner = SpinnerThread(message, 0.15)
        self.spinner.start()

    def spinner_stop(self, message="Task executed successfully! ", color=Fore.GREEN):
        """
        Function for stopping the spinner when prompted from a plugin
        and displaying the message after completing the task
        """
        self.spinner.stop()
        self.say(message, color)
        self.spinner_running = False

    def is_spinner_running(self):
        return self.spinner_running

def catch_all_exceptions(do, pass_self=True):
    def try_do(self, s):
        try:
            if pass_self:
                do(self, s)
            else:
                do(s)
        except Exception:
            if self.is_spinner_running():
                self.spinner_stop("It seems some error has occured")
            print(
                Fore.RED
                + "Some error occurred, please open an issue on github!")
            print("Here is error:")
            print('')
            traceback.print_exc()
            print(Fore.RESET)
    return try_do

class CmdInterpreter(Cmd):
    # We use this variable at Breakpoint #1.
    # We use this in order to allow Jarvis say "Hi", only at the first
    # interaction.

    # This can be used to store user specific data

    def __init__(
            self,
            first_reaction_text,
            prompt,
            directories=[],
            first_reaction=True,
            enable_voice=False):
        """
        This constructor contains a dictionary with Jarvis Actions (what Jarvis can do).
        In alphabetically order.
        """
        Cmd.__init__(self)
        self.first_reaction = first_reaction
        self.first_reaction_text = first_reaction_text
        self.prompt = prompt
        self.enable_voice = enable_voice
        # Register do_quit() function to SIGINT signal (Ctrl-C)
        signal.signal(signal.SIGINT, self.interrupt_handler)

        self.memory = Memory()
        self.scheduler = schedule.Scheduler()
        # what if the platform does not have any engines, travis doesn't have sapi5 acc to me
        try:
            self.speech = create_voice()
        except Exception as e:
            print_say("Voice not supported", self, Fore.RED)
            print_say(str(e), self, Fore.RED)

        self.fixed_responses = {"what time is it": "clock",
                                "where am i": "pinpoint",
                                }

        self._api = JarvisAPI(self)
        self._plugin_manager = PluginManager()

        for directory in directories:
            self._plugin_manager.add_directory(directory)

        self._activate_plugins()
        self._init_plugin_info()

    def _init_plugin_info(self):
        plugin_status_formatter = {
            "disabled": len(self._plugin_manager.get_disabled()),
            "enabled": self._plugin_manager.get_number_plugins_loaded(),
            "red": Fore.RED,
            "blue": Fore.BLUE,
            "reset": Fore.RESET
        }

        plugin_status = "{red}{enabled} {blue}plugins loaded"
        if plugin_status_formatter['disabled'] > 0:
            plugin_status += " {red}{disabled} {blue}plugins disabled. More information: {red}status\n"
        plugin_status += Fore.RESET

        self.first_reaction_text += plugin_status.format(
            **plugin_status_formatter)

    def _activate_plugins(self):
        """Generate do_XXX, help_XXX and (optionally) complete_XXX functions"""
        for (plugin_name, plugin) in self._plugin_manager.get_plugins().items():
            self._plugin_update_completion(plugin, plugin_name)

            run_catch = catch_all_exceptions(plugin.run)
            setattr(
                CmdInterpreter,
                "do_"
                + plugin_name,
                partial(
                    run_catch,
                    self))
            setattr(
                CmdInterpreter,
                "help_" + plugin_name,
                partial(
                    self._api.say,
                    plugin.get_doc()))

            plugin.init(self._api)

    def _plugin_update_completion(self, plugin, plugin_name):
        """Return True if completion is available"""
        completions = [i for i in plugin.complete()]
        if len(completions) > 0:
            def complete(completions):
                def _complete_impl(self, text, line, begidx, endidx):
                    return [i for i in completions if i.startswith(text)]
                return _complete_impl
            setattr(
                CmdInterpreter,
                "complete_"
                + plugin_name,
                complete(completions))

    def get_api(self):
        return self._api

    def close(self):
        """Closing Jarvis."""

        '''Stop the spinner if it is already running'''
        if self._api.is_spinner_running():
            self._api.spinner_stop('Some error has occured')

        print_say("Goodbye, see you later!", self, Fore.RED)
        self.scheduler.stop_all()
        sys.exit()

    def execute_once(self, command):
        self.get_api().eval(command)
        self.close()

    def error(self):
        """Jarvis let you know if an error has occurred."""
        print_say("I could not identify your command...", self, Fore.RED)

    def interrupt_handler(self, signal, frame):
        """Closes Jarvis on SIGINT signal. (Ctrl-C)"""
        self.close()

    def do_status(self, s):
        """Prints plugin status status"""
        count_enabled = self._plugin_manager.get_number_plugins_loaded()
        count_disabled = len(self._plugin_manager.get_disabled())
        print_say(
            "{} Plugins enabled, {} Plugins disabled.".format(
                count_enabled,
                count_disabled),
            self)

        if "short" not in s and count_disabled > 0:
            print_say("", self)
            for disabled, reason in self._plugin_manager.get_disabled().items():
                print_say(
                    "{:<20}: {}".format(
                        disabled,
                        " OR ".join(reason)),
                    self)

    def help_status(self):
        print_say("Prints info about enabled or disabled plugins", self)
        print_say("Use \"status short\" to omit detailed information.", self)
<commit_msg>Fix typo in CmdInterpreter / spinner stop<commit_after>import signal
from cmd import Cmd
from functools import partial
import sys
import traceback

from colorama import Fore
from PluginManager import PluginManager

from utilities import schedule
from utilities.voice import create_voice
from utilities.notification import notify
from utilities.GeneralUtilities import print_say

from packages.memory.memory import Memory
from utilities.animations import SpinnerThread

class JarvisAPI(object):
    """
    Jarvis interface for plugins.

    Plugins will receive a instance of this as the second (non-self) parameter
    of the exec()-method.

    Everything Jarvis-related that can't be implemented as a stateless-function
    in the utilities-package should be implemented here.
    """

    _CONNECTION_ERROR_MSG = "You are not connected to Internet"

    def __init__(self, jarvis):
        self._jarvis = jarvis
        self.spinner_running = False

    def say(self, text, color="", speak=True):
        """
        This method give the jarvis the ability to print a text
        and talk when sound is enable.
        :param text: the text to print (or talk)
        :param color: for text - use colorama (https://pypi.org/project/colorama/)
                      e.g. Fore.BLUE
        :param speak: False-, if text shouldn't be spoken even if speech is enabled
        """
        print(color + text + Fore.RESET)
        if speak:
            self._jarvis.speak(text)

    def input(self, prompt="", color=""):
        """
        Get user input
        """
        # we can't use input because for some reason input() and color codes do not work on
        # windows cmd
        sys.stdout.write(color + prompt + Fore.RESET)
        sys.stdout.flush()
        text = sys.stdin.readline()
        # return without newline
        return text.rstrip()

    def input_number(self, prompt="", color="", rtype=float, rmin=None, rmax=None):
        """
        Get user input: As number.

        Guaranteed only returns number - ask user till correct number entered.

        :param prompt: Printed to console
        :param color: Color of prompot
        :param rtype: type of return value; e.g. float (default) or int
        :param rmin: Minum of values returned
        :param rmax: Maximum of values returned
        """
        while True:
            try:
                value = rtype(self.input(prompt, color).replace(',', '.'))
                if (rmin is not None and value < rmin) or (rmax is not None and value > rmax):
                    prompt = "Sorry, needs to be between {} and {}. Try again: ".format(rmin, rmax)
                else:
                    return value
            except ValueError:
                prompt = 'Sorry, needs to be a number. Try again: '
                continue

    def connection_error(self):
        """Print generic connection error"""

        if self.is_spinner_running():
            self.spinner_stop('')

        self.say(JarvisAPI._CONNECTION_ERROR_MSG)

    def exit(self):
        """Immediately exit Jarvis"""
        self._jarvis.close()

    def notification(self, msg, time_seconds=0):
        """
        Sends notification msg in time_in milliseconds
        :param msg: Message. Either String (message body) or tuple (headline, message body)
        :param time_seconds: Time in seconds to wait before showing notification
        """
        if isinstance(msg, tuple):
            headline, message = msg
        elif isinstance(msg, str):
            headline = "Jarvis"
            message = msg
        else:
            raise ValueError("msg not a string or tuple")

        if time_seconds == 0:
            notify(headline, message)
        else:
            schedule(time_seconds, notify, headline, message)

    def schedule(self, time_seconds, function, *args):
        """
        Schedules function
        After time_seconds call function with these parameter:
           - reference to this JarvisAPI instance
           - schedule_id (return value of this function)
           - *args
        :return: integer, id - use with cancel
        """
        return self._jarvis.scheduler.create_event(
            time_seconds, function, self, *args)

    def cancel(self, schedule_id):
        """
        Cancel event scheduled with schedule
        :param schedule_id: id returned by schedule
        """
        spinner = SpinnerThread('Cancelling', 0.15)
        spinner.start()

        self._jarvis.scheduler.cancel(schedule_id)

        spinner.stop()
        jarvis.say('Cancellation successful', Fore.GREEN)

    # Voice wrapper
    def enable_voice(self):
        """
        Use text to speech for every text passed to jarvis.say()
        """
        self._jarvis.enable_voice = True

    def disable_voice(self):
        """
        Stop text to speech output for every text passed to jarvis.say()
        """
        self._jarvis.enable_voice = False

    def is_voice_enabled(self):
        """
        Returns True/False if voice is enabled/disabled with
        enable_voice or disable_voice
        Default: False (disabled)
        """
        return self._jarvis.enable_voice

    # MEMORY WRAPPER
    def get_data(self, key):
        """
        Get a specific key from memory
        """
        return self._jarvis.memory.get_data(key)

    def add_data(self, key, value):
        """
        Add a key and value to memory
        """
        self._jarvis.memory.add_data(key, value)
        self._jarvis.memory.save()

    def update_data(self, key, value):
        """
        Updates a key with supplied value.
        """
        self._jarvis.memory.update_data(key, value)
        self._jarvis.memory.save()

    def del_data(self, key):
        """
        Delete a key from memory
        """
        self._jarvis.memory.del_data(key)
        self._jarvis.memory.save()

    def eval(self, s):
        """
        Simulates typing 's' in Jarvis prompt
        """
        line = self._jarvis.precmd(s)
        stop = self._jarvis.onecmd(line)
        stop = self._jarvis.postcmd(stop, line)

    def spinner_start(self, message="Starting "):
        """
        Function for starting a spinner when prompted from a plugin
        and a default message for performing the task
        """
        self.spinner_running = True
        self.spinner = SpinnerThread(message, 0.15)
        self.spinner.start()

    def spinner_stop(self, message="Task executed successfully! ", color=Fore.GREEN):
        """
        Function for stopping the spinner when prompted from a plugin
        and displaying the message after completing the task
        """
        self.spinner.stop()
        self.say(message, color)
        self.spinner_running = False

    def is_spinner_running(self):
        return self.spinner_running

def catch_all_exceptions(do, pass_self=True):
    def try_do(self, s):
        try:
            if pass_self:
                do(self, s)
            else:
                do(s)
        except Exception:
            if self._api.is_spinner_running():
                self.spinner_stop("It seems some error has occured")
            print(
                Fore.RED
                + "Some error occurred, please open an issue on github!")
            print("Here is error:")
            print('')
            traceback.print_exc()
            print(Fore.RESET)
    return try_do

class CmdInterpreter(Cmd):
    # We use this variable at Breakpoint #1.
    # We use this in order to allow Jarvis say "Hi", only at the first
    # interaction.

    # This can be used to store user specific data

    def __init__(
            self,
            first_reaction_text,
            prompt,
            directories=[],
            first_reaction=True,
            enable_voice=False):
        """
        This constructor contains a dictionary with Jarvis Actions (what Jarvis can do).
        In alphabetically order.
        """
        Cmd.__init__(self)
        self.first_reaction = first_reaction
        self.first_reaction_text = first_reaction_text
        self.prompt = prompt
        self.enable_voice = enable_voice
        # Register do_quit() function to SIGINT signal (Ctrl-C)
        signal.signal(signal.SIGINT, self.interrupt_handler)

        self.memory = Memory()
        self.scheduler = schedule.Scheduler()
        # what if the platform does not have any engines, travis doesn't have sapi5 acc to me
        try:
            self.speech = create_voice()
        except Exception as e:
            print_say("Voice not supported", self, Fore.RED)
            print_say(str(e), self, Fore.RED)

        self.fixed_responses = {"what time is it": "clock",
                                "where am i": "pinpoint",
                                }

        self._api = JarvisAPI(self)
        self._plugin_manager = PluginManager()

        for directory in directories:
            self._plugin_manager.add_directory(directory)

        self._activate_plugins()
        self._init_plugin_info()

    def _init_plugin_info(self):
        plugin_status_formatter = {
            "disabled": len(self._plugin_manager.get_disabled()),
            "enabled": self._plugin_manager.get_number_plugins_loaded(),
            "red": Fore.RED,
            "blue": Fore.BLUE,
            "reset": Fore.RESET
        }

        plugin_status = "{red}{enabled} {blue}plugins loaded"
        if plugin_status_formatter['disabled'] > 0:
            plugin_status += " {red}{disabled} {blue}plugins disabled. More information: {red}status\n"
        plugin_status += Fore.RESET

        self.first_reaction_text += plugin_status.format(
            **plugin_status_formatter)

    def _activate_plugins(self):
        """Generate do_XXX, help_XXX and (optionally) complete_XXX functions"""
        for (plugin_name, plugin) in self._plugin_manager.get_plugins().items():
            self._plugin_update_completion(plugin, plugin_name)

            run_catch = catch_all_exceptions(plugin.run)
            setattr(
                CmdInterpreter,
                "do_"
                + plugin_name,
                partial(
                    run_catch,
                    self))
            setattr(
                CmdInterpreter,
                "help_" + plugin_name,
                partial(
                    self._api.say,
                    plugin.get_doc()))

            plugin.init(self._api)

    def _plugin_update_completion(self, plugin, plugin_name):
        """Return True if completion is available"""
        completions = [i for i in plugin.complete()]
        if len(completions) > 0:
            def complete(completions):
                def _complete_impl(self, text, line, begidx, endidx):
                    return [i for i in completions if i.startswith(text)]
                return _complete_impl
            setattr(
                CmdInterpreter,
                "complete_"
                + plugin_name,
                complete(completions))

    def get_api(self):
        return self._api

    def close(self):
        """Closing Jarvis."""

        '''Stop the spinner if it is already running'''
        if self._api.is_spinner_running():
            self._api.spinner_stop('Some error has occured')

        print_say("Goodbye, see you later!", self, Fore.RED)
        self.scheduler.stop_all()
        sys.exit()

    def execute_once(self, command):
        self.get_api().eval(command)
        self.close()

    def error(self):
        """Jarvis let you know if an error has occurred."""
        print_say("I could not identify your command...", self, Fore.RED)

    def interrupt_handler(self, signal, frame):
        """Closes Jarvis on SIGINT signal. (Ctrl-C)"""
        self.close()

    def do_status(self, s):
        """Prints plugin status status"""
        count_enabled = self._plugin_manager.get_number_plugins_loaded()
        count_disabled = len(self._plugin_manager.get_disabled())
        print_say(
            "{} Plugins enabled, {} Plugins disabled.".format(
                count_enabled,
                count_disabled),
            self)

        if "short" not in s and count_disabled > 0:
            print_say("", self)
            for disabled, reason in self._plugin_manager.get_disabled().items():
                print_say(
                    "{:<20}: {}".format(
                        disabled,
                        " OR ".join(reason)),
                    self)

    def help_status(self):
        print_say("Prints info about enabled or disabled plugins", self)
        print_say("Use \"status short\" to omit detailed information.", self)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import logging
# http://docs.python.org/howto/argparse.html
import argparse
import os
import datetime
import re
import mmap
import contextlib
import multiprocessing
import itertools

import sys
sys.path.append( '.' )
sys.path.append( './lib/' )
import shutil

from patXML import XMLPatent
from patXML import uniasc
from fwork  import *

regex = re.compile(r"""
 ([<][?]xml[ ]version.*?[>]       #all XML starts with ?xml
.*?
[<][/]us[-]patent[-]grant[>])    #and here is the end tag
""", re.I+re.S+re.X)

def list_files(directories, patentroot, xmlregex):
    """
    Returns listing of all files within all directories relative to patentroot
    whose filenames match xmlregex
    """
    files = [patentroot+'/'+directory+'/'+fi for directory in directories for fi in \
            os.listdir(patentroot+'/'+directory) \
            if re.search(xmlregex, fi, re.I) != None]
    return files

def parse_file(filename):
    parsed_xmls = []
    size = os.stat(filename).st_size
    with open(filename,'r') as f:
        with contextlib.closing(mmap.mmap(f.fileno(), size, access=mmap.ACCESS_READ)) as m:
            parsed_xmls.extend(regex.findall(m))
    return parsed_xmls

def parallel_parse(filelist):
    pool = multiprocessing.Pool(multiprocessing.cpu_count())
    return list(itertools.chain(*pool.imap_unordered(parse_file, filelist)))

# setup argparse
parser = argparse.ArgumentParser(description=\
        'Specify source directory/directories for xml files to be parsed')
parser.add_argument('--directory','-d', type=str, nargs='+', default='',
        help='comma separated list of directories relative to $PATENTROOT that \
        parse.py will search for .xml files')
parser.add_argument('--patentroot','-p', type=str, nargs='?',
        default=os.environ['PATENTROOT'] \
        if os.environ.has_key('PATENTROOT') else '/',
        help='root directory of all patent files/directories')
parser.add_argument('--xmlregex','-x', type=str,
        nargs='?', default=r"ipg\d{6}.xml",
        help='regex used to match xml files in each directory')
parser.add_argument('--verbosity', '-v', type = int,
        nargs='?', default=0,
        help='Set the level of verbosity for the computation. The higher the \
        verbosity level, the less restrictive the print policy. 0 (default) \
        = error, 1 = warning, 2 = info, 3 = debug')

# double check that variables are actually set
# we ignore the verbosity argument when determining
# if any variables have been set by the user
specified = [arg for arg in sys.argv if arg.startswith('-')]
nonverbose = [opt for opt in specified if '-v' not in opt]
if len(nonverbose)==0:
    parser.print_help()
    sys.exit(1)

# parse arguments and assign values
args = parser.parse_args()
DIRECTORIES = args.directory
XMLREGEX = args.xmlregex
PATENTROOT = args.patentroot
# adjust verbosity levels based on specified input
logging_levels = {0: logging.ERROR,
                  1: logging.WARNING,
                  2: logging.INFO,
                  3: logging.DEBUG}
VERBOSITY = logging_levels[args.verbosity]

logfile = "./" + 'xml-parsing.log'
logging.basicConfig(filename=logfile, level=VERBOSITY)

t1 = datetime.datetime.now()

#get a listing of all files within the directory that follow the naming pattern
files = list_files(DIRECTORIES, PATENTROOT, XMLREGEX)
logging.info("Total files: %d" % (len(files)))

# list of parsed xml strings
parsed_xmls = parallel_parse(files)

logging.info("   - Total Patents: %d" % (len(parsed_xmls)))

tables = ["assignee", "citation", "class", "inventor", "patent",\
        "patdesc", "lawyer", "sciref", "usreldoc"]
total_count = 0
total_patents = 0

for filename in parsed_xmls:

    xmllist = []
    count = 0
    patents = 0

    for i, x in enumerate(parsed_xmls):
        try:
            xmllist.append(XMLPatent(x))   #TODO: this is the slow part (parallelize)
            patents += 1
        except Exception as inst:
            #print type(inst)
            logging.error(type(inst))
            logging.error("  - Error: %s (%d)  %s" % (filename, i, x[175:200]))
            count += 1
        #print xmllist

    logging.info("   - number of patents: %d %s ", len(xmllist), datetime.datetime.now()-t1)
    logging.info( "   - number of errors: %d", count)
    total_count += count
    total_patents += patents

    for table in tables:
        # Cut the chaining here to better parameterize the call, allowing
        # the databases to be built in place
        # (/var/share/patentdata/patents/<year>)
        # outdb = PATENTROOT + "/" + table
        x.insertcallbacks[table]()

    logging.info("   - %s", datetime.datetime.now()-t1)
    logging.info("   - total errors: %d", total_count)
    logging.info("   - total patents: %d", total_patents)
<commit_msg>use new parse.py<commit_after><|endoftext|>


<commit_before>import pandas as pd
import argparse
import sys
import re

def read_input_files():
    df_dict = {}
    file_index = 0

    for path in sys.argv[1:]:
        # need to merge molecular descriptors...
        if file_index > 0:
            df = parse_file(path)
            df_dict[file_index] = df
        file_index +=1

    df_agg = pd.concat(df_dict,axis=0)
    df_agg.shape
    df_agg.drop_duplicates(inplace=True)

    # output the aggregated file, need to find a better name...
    df_agg.to_csv('ml_features.csv',sep=' ')

    # TODO: verfy that there are no duplicates

def parse_file(filepath):
    '''
        Given a filepath, determines which parser to run in order to extract data. Returns a pandas DataFrame.
    :param filepath: path to the input file
    :return: pandas DataFrame
    '''
    data = pd.DataFrame()
    if filepath.find('.dat') != -1:
        print filepath.find('.dat')
        data = load_dat(filepath)
    elif filepath.find('.pdbqt') != -1:
        print 'Not implemented'
    elif filepath.find('mmpbsa_energy') != -1:
        data = load_mmgbsa_energy(filepath)
    elif filepath.find('docking_results') != -1:
        data = load_docking_results(filepath)
    elif filepath.find('MolecularDescriptors') != -1:
        data = load_molecular_descriptors(filepath)
    elif filepath.find('protein_features_coach_avg') != -1: # not sure
        data =  load_protein_features_coach_avg(filepath)
    elif filepath.find('protein_features_2struc') != -1: # not sure
        data =  load_protein_features_2struc(filepath)
    else:
        print 'File not supported'

    return data

# create a function for reading each input file type

def load_molecular_descriptors(filepath):
    '''
        reads input files containing molecular descriptors
    :param filepath: path to the input file
    :return: pandas DataFrame
    '''

    # no need to skip any line
    data = pd.read_csv(filepath,delimiter='\t',skiprows=2)
    # I don't know why
    data = data.set_index('MOL_ID').reset_index()

    # Fatemah Code
    data = pd.read_csv(filepath,delimiter='\t', low_memory=False)

    # rename duplicated moleculars
    for index, row in data.iterrows():
        molName = data.get_value(index,'NAME',takeable=False)
        duplicatedMol = data.loc[data['NAME'] == molName]
        rowNum = len(duplicatedMol.index)

        # there are duplicated Mol.
        # rename from second molecule _#
        molIndex = 0
        if (rowNum>1):
            for innerIndex, innerRow in duplicatedMol.iterrows():
                molIndex = molIndex + 1
                if (molIndex>1):
                    newMolName = molName + '_' + str(molIndex)
                    data.set_value(innerIndex,'NAME',newMolName)

    # rename the second column to use it as key in the merge
    descriptorsResults.rename(columns={'NAME':'moleculeName'}, inplace = True)
    return data

def load_mmgbsa_energy(filepath):
    '''
        reads output from mmgbsa calculations
    :param filepath: path to the input file
    :return: pandas DataFrame
    '''
    data = pd.read_csv(filepath,delimiter='\t')
    data.drop('Order',axis=1,inplace=True)
    data = data.set_index('Ligand').reset_index()

    # remove .pdbqt from the Ligand
    data['Ligand'] = pd.DataFrame(data.Ligand.str.replace('.pdbqt',''))

    # extract protein name from Ligand and store it in new column
    data['proteinName'] = data['Ligand'].str.extract('(...._cluster\d+)', expand=True)

    # extract molecule name from Ligand and store it in new column
    data['moleculeName'] = data['Ligand'].str.extract('((?<=cluster\d_)\w+)', expand=True)

    # rename Energy col
    data.rename(columns={'Energy':'dockingEnergy'}, inplace = True)

    data = data.drop('Ligand',1)

    return data

def load_docking_results(filepath):
    '''
        reads docking results
    :param filepath: path to the input file
    :return: pandas DataFrame
    '''
    data = pd.read_csv(filepath,delimiter='\t')
    data.drop('Order',axis=1,inplace=True)
    data = data.set_index('Ligand').reset_index()

    # remove .pdbqt from the Ligand
    data['Ligand'] = pd.DataFrame(data.Ligand.str.replace('.pdbqt',''))

    # extract protein name from Ligand and store it in new column
    data['proteinName'] = data['Ligand'].str.extract('(...._cluster\d+)', expand=True)

    # extract molecule name from Ligand and store it in new column
    data['moleculeName'] = data['Ligand'].str.extract('((?<=cluster\d_)\w+)', expand=True)

     # rename Energy col
    data.rename(columns={'Energy':'dockingEnergy'}, inplace = True)

    # do we need to drop Ligand column ??
    data = data.drop('Ligand',1)

    return data

def load_protein_features_2struc(filepath):
    '''
        reads protein_features_2struc
    :param filepath: path to the input file
    :return: pandas DataFrame
    '''
    data = pd.read_csv(filepath,delimiter=',')

    # rename the first column to use it as key in the merge
    data.rename(columns={'Cluster_Name':'proteinName'}, inplace = True)

    return data

def load_protein_features_coach_avg(filepath):
    '''
        reads protein_features_coach_avg
    :param filepath: path to the input file
    :return: pandas DataFrame
    '''
    data = pd.read_csv(filepath,delimiter=',')

    # rename the first column to use it as key in the merge
    data.rename(columns={'cluster_name':'proteinName'}, inplace = True)

    return data

def load_dat(filepath):
    '''
        Loads VINA results

    :param filepath: path to file containing data
    :return: a pandas dataframe of VINA results
    '''
    file = open(filepath)
    feature_list = {}
    i = 0
    j = 0

    for line in file.readlines():
        if line.find("TOTAL") != -1 and line.find("DELTA") == -1:
            line = line.strip('\n')
            line = line.split(" ")
            line = filter(lambda x: len(x) >0,line)
            value = float(line[1])
            line[0] = line[0]+str(i)
            col_name = line[0]
            feature_list[col_name] = value
            i +=1

        elif line.find("TOTAL") != -1 and line.find("DELTA") != -1:
            line = line.strip('\n')
            line = line.split(" ")
            line = filter(lambda x: len(x) > 0, line)
            line[1] = line[1]+str(j)
            col_name = line[0]+line[1]
            value = float(line[2])
            feature_list[col_name] = value
            j+=1

    df = pd.DataFrame([feature_list])
    return df

def get_merged_results(dfX,dfY,key):
    '''

    Merge two dataframes (datadrames assumed to have common key, does not support protein || drug and protein && drug matching).

    :param dfX: one of the two dataframes to be merged
    :param dfY: the other of the two dataframes to be merged
    :param key: a string denoting the column on which the dataframes will be merged
    :return: a pandas dataframe containing the matches between the two dataframes (maximum = least row dimenison of dfX or dfY)
    '''

    # merge_docking_mmpbsa = pd.merge(mmpbsaResults,dockingResults, how='outer' , on=['proteinName','moleculeName'])
    # merge_docking_mmpbsa_descriptors = pd.merge(merge_docking_mmpbsa,descriptorsResults, how='outer', on='moleculeName')
    # merge_protein2struc_proteinCoach = pd.merge(protein2struc , proteinCoach,on='proteinName')
    # merge_all = pd.merge(merge_docking_mmpbsa_descriptors,merge_protein2struc_proteinCoach,how='left',on='proteinName')

    merge_result = dfX.merge(dfY,left_on=key, right_on=key)
    merge_result = merge_result.set_index(key).reset_index()
    return merge_result

#TODO: write a function to merge the molecular descriptors to the docking and energy results. Will need regular expressions.
#TODO: read the filenames from stdin and then execute necessary functions to generate output datafile
#TODO: merge all resulting dataframes on id's

def merge_molec_descriptors(dfM,dfX):
    # for each m_index in dfM
        # for each x_index in dfX
            #if m_index in x_index
                #concatenate dfM[m_index] to dfX[x_index]

    #for m_index in dfM['MOL_ID'].values:
    for m_row in dfM.itertuples():
        #print m_row
    #    for x_index in dfX['Ligand'].values:
        for x_row in dfX.itertuples():
            print x_row[1]
            #if m_index in x_index:
            #    print type(pd.concat([dfM[m_index],dfX[x_index]],axis=1))

    return

read_input_files()

<commit_msg>cleaning up<commit_after>import pandas as pd
import argparse
import sys
import re

def read_input_files():
    df_dict = {}
    file_index = 0

    for path in sys.argv[1:]:
        # need to merge molecular descriptors...
        df = parse_file(path)
        df_dict[file_index] = df
        file_index +=1

    df_agg = pd.concat(df_dict,axis=0)
    df_agg.shape
    df_agg.drop_duplicates(inplace=True)

    # output the aggregated file, need to find a better name...
    df_agg.to_csv('ml_features.csv',sep=' ')

def parse_file(filepath):
    '''
        Given a filepath, determines which parser to run in order to extract data. Returns a pandas DataFrame.
    :param filepath: path to the input file
    :return: pandas DataFrame
    '''
    data = pd.DataFrame()
    if filepath.find('.dat') != -1:
        print filepath.find('.dat')
        data = load_dat(filepath)
    elif filepath.find('.pdbqt') != -1:
        print 'Not implemented'
    elif filepath.find('mmpbsa_energy') != -1:
        data = load_mmgbsa_energy(filepath)
    elif filepath.find('docking_results') != -1:
        data = load_docking_results(filepath)
    elif filepath.find('MolecularDescriptors') != -1:
        data = load_molecular_descriptors(filepath)
    elif filepath.find('protein_features_coach_avg') != -1: # not sure
        data =  load_protein_features_coach_avg(filepath)
    elif filepath.find('protein_features_2struc') != -1: # not sure
        data =  load_protein_features_2struc(filepath)
    else:
        print 'File not supported'

    return data

# create a function for reading each input file type

def load_molecular_descriptors(filepath):
    '''
        reads input files containing molecular descriptors
    :param filepath: path to the input file
    :return: pandas DataFrame
    '''

    # no need to skip any line
    data = pd.read_csv(filepath,delimiter='\t',skiprows=2)
    # I don't know why
    data = data.set_index('MOL_ID').reset_index()

    # Fatemah Code
    data = pd.read_csv(filepath,delimiter='\t', low_memory=False)

    # rename duplicated moleculars
    for index, row in data.iterrows():
        molName = data.get_value(index,'NAME',takeable=False)
        duplicatedMol = data.loc[data['NAME'] == molName]
        rowNum = len(duplicatedMol.index)

        # there are duplicated Mol.
        # rename from second molecule _#
        molIndex = 0
        if (rowNum>1):
            for innerIndex, innerRow in duplicatedMol.iterrows():
                molIndex = molIndex + 1
                if (molIndex>1):
                    newMolName = molName + '_' + str(molIndex)
                    data.set_value(innerIndex,'NAME',newMolName)

    # rename the second column to use it as key in the merge
    #descriptorsResults.rename(columns={'NAME':'moleculeName'}, inplace = True)
    data.rename(columns={'NAME':'moleculeName'}, inplace = True)
    return data

def load_mmgbsa_energy(filepath):
    '''
        reads output from mmgbsa calculations
    :param filepath: path to the input file
    :return: pandas DataFrame
    '''
    data = pd.read_csv(filepath,delimiter='\t')
    data.drop('Order',axis=1,inplace=True)
    data = data.set_index('Ligand').reset_index()

    # remove .pdbqt from the Ligand
    data['Ligand'] = pd.DataFrame(data.Ligand.str.replace('.pdbqt',''))

    # extract protein name from Ligand and store it in new column
    data['proteinName'] = data['Ligand'].str.extract('(...._cluster\d+)', expand=True)

    # extract molecule name from Ligand and store it in new column
    data['moleculeName'] = data['Ligand'].str.extract('((?<=cluster\d_)\w+)', expand=True)

    # rename Energy col
    data.rename(columns={'Energy':'dockingEnergy'}, inplace = True)

    data = data.drop('Ligand',1)

    return data

def load_docking_results(filepath):
    '''
        reads docking results
    :param filepath: path to the input file
    :return: pandas DataFrame
    '''
    data = pd.read_csv(filepath,delimiter='\t')
    data.drop('Order',axis=1,inplace=True)
    data = data.set_index('Ligand').reset_index()

    # remove .pdbqt from the Ligand
    data['Ligand'] = pd.DataFrame(data.Ligand.str.replace('.pdbqt',''))

    # extract protein name from Ligand and store it in new column
    data['proteinName'] = data['Ligand'].str.extract('(...._cluster\d+)', expand=True)

    # extract molecule name from Ligand and store it in new column
    data['moleculeName'] = data['Ligand'].str.extract('((?<=cluster\d_)\w+)', expand=True)

     # rename Energy col
    data.rename(columns={'Energy':'dockingEnergy'}, inplace = True)

    # do we need to drop Ligand column ??
    data = data.drop('Ligand',1)

    return data

def load_protein_features_2struc(filepath):
    '''
        reads protein_features_2struc
    :param filepath: path to the input file
    :return: pandas DataFrame
    '''
    data = pd.read_csv(filepath,delimiter=',')

    # rename the first column to use it as key in the merge
    data.rename(columns={'Cluster_Name':'proteinName'}, inplace = True)

    return data

def load_protein_features_coach_avg(filepath):
    '''
        reads protein_features_coach_avg
    :param filepath: path to the input file
    :return: pandas DataFrame
    '''
    data = pd.read_csv(filepath,delimiter=',')

    # rename the first column to use it as key in the merge
    data.rename(columns={'cluster_name':'proteinName'}, inplace = True)

    return data

def load_dat(filepath):
    '''
        Loads VINA results

    :param filepath: path to file containing data
    :return: a pandas dataframe of VINA results
    '''
    file = open(filepath)
    feature_list = {}
    i = 0
    j = 0

    for line in file.readlines():
        if line.find("TOTAL") != -1 and line.find("DELTA") == -1:
            line = line.strip('\n')
            line = line.split(" ")
            line = filter(lambda x: len(x) >0,line)
            value = float(line[1])
            line[0] = line[0]+str(i)
            col_name = line[0]
            feature_list[col_name] = value
            i +=1

        elif line.find("TOTAL") != -1 and line.find("DELTA") != -1:
            line = line.strip('\n')
            line = line.split(" ")
            line = filter(lambda x: len(x) > 0, line)
            line[1] = line[1]+str(j)
            col_name = line[0]+line[1]
            value = float(line[2])
            feature_list[col_name] = value
            j+=1

    df = pd.DataFrame([feature_list])
    return df

def get_merged_results(dfX,dfY,key):
    '''

    Merge two dataframes (datadrames assumed to have common key, does not support protein || drug and protein && drug matching).

    :param dfX: one of the two dataframes to be merged
    :param dfY: the other of the two dataframes to be merged
    :param key: a string denoting the column on which the dataframes will be merged
    :return: a pandas dataframe containing the matches between the two dataframes (maximum = least row dimenison of dfX or dfY)
    '''

    # merge_docking_mmpbsa = pd.merge(mmpbsaResults,dockingResults, how='outer' , on=['proteinName','moleculeName'])
    # merge_docking_mmpbsa_descriptors = pd.merge(merge_docking_mmpbsa,descriptorsResults, how='outer', on='moleculeName')
    # merge_protein2struc_proteinCoach = pd.merge(protein2struc , proteinCoach,on='proteinName')
    # merge_all = pd.merge(merge_docking_mmpbsa_descriptors,merge_protein2struc_proteinCoach,how='left',on='proteinName')

    merge_result = dfX.merge(dfY,left_on=key, right_on=key)
    merge_result = merge_result.set_index(key).reset_index()
    return merge_result

#TODO: write a function to merge the molecular descriptors to the docking and energy results. Will need regular expressions.
#TODO: read the filenames from stdin and then execute necessary functions to generate output datafile
#TODO: merge all resulting dataframes on id's

'''def merge_molec_descriptors(dfM,dfX):
    # for each m_index in dfM
        # for each x_index in dfX
            #if m_index in x_index
                #concatenate dfM[m_index] to dfX[x_index]

    #for m_index in dfM['MOL_ID'].values:
    for m_row in dfM.itertuples():
        #print m_row
    #    for x_index in dfX['Ligand'].values:
        for x_row in dfX.itertuples():
            print x_row[1]
            #if m_index in x_index:
            #    print type(pd.concat([dfM[m_index],dfX[x_index]],axis=1))

    return'''

read_input_files()

<|endoftext|>


<commit_before>from parser.parse_ja import generate_translation_tuples as ja_parser
from parser.parse_vi import generate_translation_tuples as vi_parser
from parser.parse_tr import generate_translation_tuples as tr_parser
from parser.parse_fr import generate_translation_tuples as fr_parser

import sys
import argparse
import parser.lang_code_conversion as languages
from parser.helper import infer_edition_from_url, get_html_tree_from_string, get_html_tree_from_url

if sys.version_info[0:3] >= (3, 0, 0):  # python 3 (tested)
    from zim.zimpy_p3 import ZimFile
else:  # python 2 (not tested)
    from zim.zimpy_p2 import ZimFile

tested_url = [
    "https://vi.wiktionary.org/wiki/kh%C3%B4ng#Ti.E1.BA.BFng_Vi.E1.BB.87t",
    "https://vi.wiktionary.org/wiki/c%C3%A1m_%C6%A1n#Ti.E1.BA.BFng_Vi.E1.BB.87t",
    "https://ja.wiktionary.org/wiki/%E3%81%AA%E3%81%84",
    "https://ja.wiktionary.org/wiki/%E9%81%BA%E4%BC%9D%E5%AD%90",
    "https://tr.wiktionary.org/wiki/ev",
    "https://tr.wiktionary.org/wiki/abartmak",
    "https://fr.wiktionary.org/wiki/amour",
    "https://fr.wiktionary.org/wiki/ouvrir",
]

parsers = {'ja': ja_parser, 'vi': vi_parser, 'tr': tr_parser, 'fr': fr_parser}

def read_zim_file(file):
    # print(file.metadata())
    # we only need main articles. They are in namespace 'A'.
    namespace = b'A'
    for article in file.articles():
        if article['namespace'] != namespace:
            continue
        body = file.get_article_by_index(
            article['index'], follow_redirect=False)[0]
        if not body:
            continue
        else:
            yield (body.decode('utf-8'))

def test_zim(filename, edition=None):
    file = ZimFile(filename=filename)
    file.list_articles_by_url()
    edition_lang_code = file.metadata()['language'].decode('utf-8')
    edition_wikt_code = edition or languages.get_wikt_code_from_iso639_3(edition_lang_code)
    print(edition_wikt_code)
    if edition_wikt_code not in parsers:
        print("We don't have a parser for {}/{} language yet.".format(edition_lang_code, edition_wikt_code))
        return
    parser = parsers[edition_wikt_code]
    page_generator = read_zim_file(file)

    for page in page_generator:
        soup = get_html_tree_from_string(page)
        for tup in parser(soup):
            print(",".join(tup))

def test_html():
    for url in tested_url:
        edition = infer_edition_from_url(url)
        soup = get_html_tree_from_url(url)
        for tup in parsers[edition](soup):
            print(",".join(tup))

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--zim', '-z', help='use zim file instead of html')
    parser.add_argument('--edition', '-e', help='explicitly specify the language edition')

    args = parser.parse_args()
    if args.zim:
        test_zim(args.zim, args.edition)
    else:
        test_html()

if __name__ == '__main__':
    main()<commit_msg>refactor imports statements and urls for testing<commit_after>import argparse
import sys

from parser.helper import infer_edition_from_url, get_html_tree_from_string, get_html_tree_from_url

if sys.version_info[0:3] >= (3, 0, 0):  # python 3 (tested)
    from zim.zimpy_p3 import ZimFile
else:  # python 2 (not tested)
    from zim.zimpy_p2 import ZimFile

parsers = {}
tested_url = []

def import_parsers():
    parser_list = ['ja', 'vi', 'tr', 'fr']
    import importlib
    for parser_name in parser_list:
        module_to_import = '.parse_' + parser_name
        module = importlib.import_module(module_to_import, package='parser')
        parsers[parser_name] = module.generate_translation_tuples
        tested_url.extend(module.tested_url)

def read_zim_file(file):
    # print(file.metadata())
    # we only need main articles. They are in namespace 'A'.
    namespace = b'A'
    for article in file.articles():
        if article['namespace'] != namespace:
            continue
        body = file.get_article_by_index(
            article['index'], follow_redirect=False)[0]
        if not body:
            continue
        else:
            yield (body.decode('utf-8'))

def test_zim(filename, edition=None):
    file = ZimFile(filename=filename)
    file.list_articles_by_url()
    edition_lang_code = file.metadata()['language'].decode('utf-8')

    if edition:
        edition_wikt_code = edition
    else:
        import parser.lang_code_conversion as languages
        edition_wikt_code = languages.get_wikt_code_from_iso639_3(edition_lang_code)
    # print(edition_wikt_code)
    if edition_wikt_code not in parsers:
        print("We don't have a parser for {}/{} language yet.".format(edition_lang_code, edition_wikt_code))
        return
    parser = parsers[edition_wikt_code]
    page_generator = read_zim_file(file)

    for page in page_generator:
        soup = get_html_tree_from_string(page)
        for tup in parser(soup):
            print(",".join(tup))

def test_html():
    for url in tested_url:
        edition = infer_edition_from_url(url)
        soup = get_html_tree_from_url(url)
        for tup in parsers[edition](soup):
            print(",".join(tup))

def main():
    import_parsers()

    parser = argparse.ArgumentParser()
    parser.add_argument('--zim', '-z', help='use zim file instead of html')
    parser.add_argument('--edition', '-e', help='explicitly specify the language edition')

    args = parser.parse_args()
    if args.zim:
        test_zim(args.zim, args.edition)
    else:
        test_html()

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>#!/usr/bin/python
import sys, getopt, exceptions
import nltk

def processStopWords(text):
    stopWordsList = []
    for line in text:
        stopWordsList.extend(line.strip('\n').split(' '))
    return stopWordsList

def processRawData(text, stopWords = None):
    rawDataList = []
    for line in text:
        segments = line.split('\t')
        rawDataList.append(segments[2])
    return rawDataList

def readArgsFromInput(argv):
    rawDataName = stopWordsName = outputName = directory = None
    # Record the flags that user are using
    usedOpts = []
    try:
        opts, args = getopt.getopt(argv,"hr:s:i1o",["rawdata=","stopwords=","ouput="])
    except getopt.GetoptError:
        print 'run parser.py with flag -h to get help doc'
        sys.exit()
    for opt, arg in opts:
        if opt == '-h':
            print 'Flags List:'
            print '-h Help'
            print '-r <rawdata_filename> Import raw data file'
            print '-s <stopwords_filename> Import stopwords file'
            print "-o <outputfile_filename> Define the name of output file.(It's named as no_stopwords.txt by default)"
            print '-1 Mode 1: Process raw data without stripping the stopwords'
            sys.exit()

        elif opt in ("-r","--rawdata"):
            rawDataName = arg
            usedOpts.append('-r')

        elif opt in ("-s","--stopwords"):
            stopWordsName = arg
            usedOpts.append('-s')

        elif opt in ("-o","--output"):
            outputName = arg
            usedOpts.append('-o')

        elif opt in ("-i"):
            usedOpts.append('-i')
            # TODO:export stats images
            pass
        elif opt in ("-1"):
            usedOpts.append('-1')
            # TODO:Mode 1 -- Process raw data without stripping the stopwords
            pass
        else:
            sys.exit("Invalid Flag(s)")

    # pareser wihout flag -o, then the output will be named as this by default
    if '-o' not in usedOpts:
        outputName = 'no_stopwords.txt'

    # cannot proceed without filename of raw data
    if rawDataName == None:
        sys.exit("Cannot proceed wihout the filename of raw data")

    return rawDataName, stopWordsName, outputName, usedOpts

def main(argv):

    rawDataName, stopWordsName, outputName, usedOpts = readArgsFromInput(argv)
    # load raw data
    try:
        rawDataFile = open(rawDataName,'r')

        # parser is in mode 1 then prcoess raw data without stripping the stopwords
        if '-1' in usedOpts:
            pass
            #processRawData(rawDataFile)
        else:
            stopWordsFile = open(stopWordsName,'r')
            stopWordsList = processStopWords(stopWordsFile)
            #processRawData(rawDataFile,stopWordsList)

    except IOError as e:
        print "Cannot read:",fileName

if __name__ == "__main__":
    main(sys.argv[1:])<commit_msg>update process rawdata<commit_after>#!/usr/bin/python
import sys, getopt, exceptions, re
import nltk

def processStopWords(text):
    stopWordsList = []
    for line in text:
        stopWordsList.extend(line.strip('\n').split(' '))
    return stopWordsList

def processRawData(text, stopWords = None):
    invertedRawData = {}

    for line in text:
        # get sentence
        sentenceStr = line.split('\t')[2]
        # get sentiment
        sentiment = int(line.strip('\n').split('\t')[3])
        # replace one or more spaces by single space
        # then split
        sentenceTokens = re.sub("\s+"," ",entenceStr).split(' ')

        # if stopwords are required, do the following
        if stopwords:
            sentenceTokens = stripWords(sentenceTokens,stopwords)

        # create a sentiment key if doesn't exist
        if sentiment in invertedRawData:
            # append the sentence to its corresponding sentiment list
            invertedRawData[sentiment].append(sentenceTokens)
        else
            invertedRawData[sentiment] = []
            invertedRawData[sentiment].append(sentenceTokens)

    return invertedRawData

def stripWords(sentenceTokens,wordsList):

    for word in wordsList:
        if word in sentenceTokens:
            sentenceTokens.remove(word)
    return sentenceTokens

def readArgsFromInput(argv):
    rawDataName = stopWordsName = outputName = directory = None
    # Record the flags that user are using
    usedOpts = []
    try:
        opts, args = getopt.getopt(argv,"hr:s:i1o",["rawdata=","stopwords=","ouput="])
    except getopt.GetoptError:
        print 'run parser.py with flag -h to get help doc'
        sys.exit()
    for opt, arg in opts:
        if opt == '-h':
            print 'Flags List:'
            print '-h Help'
            print '-r <rawdata_filename> Import raw data file'
            print '-s <stopwords_filename> Import stopwords file'
            print "-o <outputfile_filename> Define the name of output file.(It's named as no_stopwords.txt by default)"
            print '-1 Mode 1: Process raw data without stripping the stopwords'
            sys.exit()

        elif opt in ("-r","--rawdata"):
            rawDataName = arg
            usedOpts.append('-r')

        elif opt in ("-s","--stopwords"):
            stopWordsName = arg
            usedOpts.append('-s')

        elif opt in ("-o","--output"):
            outputName = arg
            usedOpts.append('-o')

        elif opt in ("-i"):
            usedOpts.append('-i')
            # TODO:export stats images
            pass
        elif opt in ("-1"):
            usedOpts.append('-1')
            # TODO:Mode 1 -- Process raw data without stripping the stopwords
            pass
        else:
            sys.exit("Invalid Flag(s)")

    # pareser wihout flag -o, then the output will be named as this by default
    if '-o' not in usedOpts:
        outputName = 'no_stopwords.txt'

    # cannot proceed without filename of raw data
    if rawDataName == None:
        sys.exit("Cannot proceed wihout the filename of raw data")

    return rawDataName, stopWordsName, outputName, usedOpts

def main(argv):

    rawDataName, stopWordsName, outputName, usedOpts = readArgsFromInput(argv)
    # load raw data
    try:
        rawDataFile = open(rawDataName,'r')

        # parser is in mode 1 then prcoess raw data without stripping the stopwords
        if '-1' in usedOpts:
            pass
            #processRawData(rawDataFile)
        else:
            stopWordsFile = open(stopWordsName,'r')
            stopWordsList = processStopWords(stopWordsFile)
            processRawData(rawDataFile,stopWordsList)

    except IOError as e:
        print "Cannot read:",fileName

if __name__ == "__main__":
    main(sys.argv[1:])<|endoftext|>