<commit_before># -*- coding: utf-8 -*-

import json
import os
import glob
import base64
from string import Template
import time
import logging
import unittest
import tempfile, tarfile
import StringIO
import re

import utils
from utils import run

logger = logging.getLogger(__name__)

available_hosts = set()
available_profiles = set()
avialable_testcases = set()

class UpdateableObject(object):
    def __init__(self, **kwargs):
        self.__dict__.update(kwargs)

class Host(UpdateableObject):
    session = None

    def prepare(self, session):
        raise Exception("Not implemented.")

    def submit_testsuite(self, testsuite):
        raise Exception("Not implemented.")

    def get_name(self):
        raise Exception("Not implemented.")

    def get_mac_address(self):
        raise Exception("Not implemented.")

    def purge(self):
        raise Exception("Not implemented.")

class Profile(UpdateableObject):

    def get_name(self):
        raise Exception("Not implemented.")

    def assign_to(self, host):
        raise Exception("Not implemented.")

    def revoke_from(self, host):
        raise Exception("Not implemented.")

class Factory:
    @staticmethod
    def from_file(filename, per_line_cb):
        fdir = os.path.dirname(filename)
        objlist = []
        with open(filename, "r") as f:
            for line in f:
                line = re.sub("\s*#.*$", "", line).strip()
                if line == "":
                    continue
                objlist.append(per_line_cb(os.path.join(fdir, line)))
        return objlist

    @staticmethod
    def testsuites_from_path(path, suffix=".suite"):
        suites = {}
        pat = os.path.join(path, "*%s" % suffix)
        logger.debug("Trying to load from %s" % pat)
        for f in glob.glob(pat):
            suite = Factory.testsuite_from_file(f)
            suites[suite.name] = suite
        return suites

    @staticmethod
    def testsuite_from_file(filename, suffix=".suite"):
        name = os.path.basename(filename).replace(suffix, "")
        sets = Factory.from_file(filename, Factory.testset_from_file)
        return Testsuite(name=name, testsets=sets)

    @staticmethod
    def testset_from_file(filename, suffix=".set"):
        name = os.path.basename(filename).replace(suffix, "")
        cases = Factory.from_file(filename, Testcase.from_line)
        return Testset(name=name, testcases=cases)

class Testsuite(object):
    name = None
    testsets = None

    def __init__(self, name, testsets=[]):
        self.name = name
        self.testsets = testsets

    def testcases(self):
        cases = []
        for tset in self.testsets:
            cases += tset.testcases()
        return cases

    def timeout(self):
        return sum([int(c.timeout) for c in self.testcases()])
    def __str__(self):
        testsets_str = "\n".join([str(ts) for ts in self.testsets])
        return "Suite: %s\nTestsets:\n%s" % (self.name, testsets_str)
    def __json__(self):
        return { \
            "name": self.name,
            "testsets": [t.__json__() for t in self.testsets]
            }
    def get_archive(self, subdir="testcases"):
        r = StringIO.StringIO()
        logger.debug("Preparing archive for testsuite %s" % self.name)
        with tarfile.open(fileobj=r,mode="w:bz2") as archive:
            stepn = 0
            for testcase in self.testcases():
                logger.debug("Adding testcase #%s: %s" % (stepn, \
                                                          testcase.name))
                if testcase.filename is None:
                    logger.warning("Empty testcase: %s" % testcase.name)
                    continue

                arcname = os.path.join(subdir, "%d-%s" % (stepn, \
                                          os.path.basename(testcase.filename)))
                self.__add_testcase_to_archive(archive, arcname, testcase)
                stepn += 1
        return r

    def __add_testcase_to_archive(self, archive, arcname, testcase):
        srcobj = StringIO.StringIO(testcase.source())
        info = tarfile.TarInfo(name=arcname)
        info.size = len(srcobj.buf)
        archive.addfile(tarinfo=info, fileobj=srcobj)
        testcaseextradir = testcase.filename + ".d"
        if os.path.exists(testcaseextradir):
            logger.debug("Adding extra dir: %s" % testcaseextradir)
            archive.add(testcaseextradir, arcname=arcname + ".d", \
                        recursive=True)

class Testset(object):
    name = None
    _testcases = None

    def __init__(self, name, testcases=[]):
        self.name = name
        self._testcases = []
        self.add(testcases)

    def testcases(self):
        return self._testcases

    def timeout(self):
        return sum([int(c.timeout) for c in self.testcases()])

    def add(self, cs):
        for c in cs:
            if not isinstance(c, Testcase):
                c = Testcase(c)
            self._testcases.append(c)

    def __str__(self):
        return "%s: %s" % (self.name, str(["%s: %s" % (n, c) for n, c in enumerate(self.testcases())]))

    def __json__(self):
        return {
            "testcases": [c.__json__() for c in self.testcases()]
        }

class Testcase(object):
    name = None
    filename = None
    source = None
    timeout = 60

    def __init__(self, filename=None, name=None):
        if name is None and filename is None:
            raise Exception("At least a filename must be given")
        if name is None:
            self.name = os.path.basename(filename)
        else:
            self.name = name
        self.filename = filename

    @staticmethod
    def from_line(line):
        token = line.split()
        token.reverse()
        assert len(token) > 0, "Testcase filename is mandatory"
        filename = token.pop()
        c = Testcase(filename)
        for k, v in utils.cmdline_to_dict(" ".join(token)).items():
            if k == "timeout":
                c.timeout = int(v)
        return c

    def source(self):
        src = None
        with open(self.filename, "r") as f:
            src = f.read()
        return src
    def __str__(self):
        return "%s (%s, %s)" % (self.name, self.filename, self.timeout)
    def __json__(self):
        return self.__dict__

class TestSession(UpdateableObject):
    dirname = None
    cookie = None

    do_cleanup = False

    def __init__(self, cookie, session_path, cleanup=True):
        assert session_path is not None, "session path can not be None"

        self.do_cleanup = cleanup
        self.cookie = cookie
        self.dirname = run("mktemp -d '%s/test-session-XXXXX-%s'" % \
                           (session_path, self.cookie))
        run("chmod a+X '%s'" % self.dirname)
        logger.info("Starting session %s in %s" % (self.cookie, self.dirname))

    def remove(self):
        logger.debug("Removing testdir '%s'" % self.dirname)
        for artifact in self.artifacts():
            logger.debug("Removing artifact '%s'" % artifact)
            os.remove(os.path.join(self.dirname, artifact))
        os.removedirs(self.dirname)

    def add_artifact(self, name, data):
        assert("/" not in name and "\\" not in name)
        afilename = os.path.join(self.dirname, name)
        # fixme collsisions
        with open(afilename, "w") as afile:
            afile.write(data)

    def artifacts(self, use_abs = False):
        fns = os.listdir(self.dirname)
        if use_abs:
            fns = [os.path.join(self.dirname, fn) \
                   for fn in fns]
        return fns

    def get_artifacts_archive(self, selection=None):
        selection = selection or self.artifacts()
        r = StringIO.StringIO()
        logger.debug("Preparing artifacts archive for session %s" % self.cookie)
        with tarfile.open(fileobj=r,mode="w:bz2") as archive:
            for artifact in selection:
                if artifact not in self.artifacts():
                    logger.debug("Artifact not here: %s" % artifact)
                logger.debug("Adding artifact %s" % artifact)
                archive.add(os.path.join(self.dirname, artifact), artifact)
        return r

    def __enter__(self):
        logger.debug("With session '%s'" % self.cookie)
        return self

    def __exit__(self, type, value, traceback):
        logger.debug("Ending session %s" % self.cookie)
        if self.do_cleanup:
            self.remove()
        logger.info("Session '%s' ended." % self.cookie)

if __name__ == "__main__":
    suites = Factory.testsuites_from_path("../testcases/")
    print suites
<commit_msg>Fix tempdir creation<commit_after># -*- coding: utf-8 -*-

import json
import os
import glob
import base64
from string import Template
import time
import logging
import unittest
import tempfile, tarfile
import StringIO
import re

import utils
from utils import run

logger = logging.getLogger(__name__)

available_hosts = set()
available_profiles = set()
avialable_testcases = set()

class UpdateableObject(object):
    def __init__(self, **kwargs):
        self.__dict__.update(kwargs)

class Host(UpdateableObject):
    session = None

    def prepare(self, session):
        raise Exception("Not implemented.")

    def submit_testsuite(self, testsuite):
        raise Exception("Not implemented.")

    def get_name(self):
        raise Exception("Not implemented.")

    def get_mac_address(self):
        raise Exception("Not implemented.")

    def purge(self):
        raise Exception("Not implemented.")

class Profile(UpdateableObject):

    def get_name(self):
        raise Exception("Not implemented.")

    def assign_to(self, host):
        raise Exception("Not implemented.")

    def revoke_from(self, host):
        raise Exception("Not implemented.")

class Factory:
    @staticmethod
    def from_file(filename, per_line_cb):
        fdir = os.path.dirname(filename)
        objlist = []
        with open(filename, "r") as f:
            for line in f:
                line = re.sub("\s*#.*$", "", line).strip()
                if line == "":
                    continue
                objlist.append(per_line_cb(os.path.join(fdir, line)))
        return objlist

    @staticmethod
    def testsuites_from_path(path, suffix=".suite"):
        suites = {}
        pat = os.path.join(path, "*%s" % suffix)
        logger.debug("Trying to load from %s" % pat)
        for f in glob.glob(pat):
            suite = Factory.testsuite_from_file(f)
            suites[suite.name] = suite
        return suites

    @staticmethod
    def testsuite_from_file(filename, suffix=".suite"):
        name = os.path.basename(filename).replace(suffix, "")
        sets = Factory.from_file(filename, Factory.testset_from_file)
        return Testsuite(name=name, testsets=sets)

    @staticmethod
    def testset_from_file(filename, suffix=".set"):
        name = os.path.basename(filename).replace(suffix, "")
        cases = Factory.from_file(filename, Testcase.from_line)
        return Testset(name=name, testcases=cases)

class Testsuite(object):
    name = None
    testsets = None

    def __init__(self, name, testsets=[]):
        self.name = name
        self.testsets = testsets

    def testcases(self):
        cases = []
        for tset in self.testsets:
            cases += tset.testcases()
        return cases

    def timeout(self):
        return sum([int(c.timeout) for c in self.testcases()])
    def __str__(self):
        testsets_str = "\n".join([str(ts) for ts in self.testsets])
        return "Suite: %s\nTestsets:\n%s" % (self.name, testsets_str)
    def __json__(self):
        return { \
            "name": self.name,
            "testsets": [t.__json__() for t in self.testsets]
            }
    def get_archive(self, subdir="testcases"):
        r = StringIO.StringIO()
        logger.debug("Preparing archive for testsuite %s" % self.name)
        with tarfile.open(fileobj=r,mode="w:bz2") as archive:
            stepn = 0
            for testcase in self.testcases():
                logger.debug("Adding testcase #%s: %s" % (stepn, \
                                                          testcase.name))
                if testcase.filename is None:
                    logger.warning("Empty testcase: %s" % testcase.name)
                    continue

                arcname = os.path.join(subdir, "%d-%s" % (stepn, \
                                          os.path.basename(testcase.filename)))
                self.__add_testcase_to_archive(archive, arcname, testcase)
                stepn += 1
        return r

    def __add_testcase_to_archive(self, archive, arcname, testcase):
        srcobj = StringIO.StringIO(testcase.source())
        info = tarfile.TarInfo(name=arcname)
        info.size = len(srcobj.buf)
        archive.addfile(tarinfo=info, fileobj=srcobj)
        testcaseextradir = testcase.filename + ".d"
        if os.path.exists(testcaseextradir):
            logger.debug("Adding extra dir: %s" % testcaseextradir)
            archive.add(testcaseextradir, arcname=arcname + ".d", \
                        recursive=True)

class Testset(object):
    name = None
    _testcases = None

    def __init__(self, name, testcases=[]):
        self.name = name
        self._testcases = []
        self.add(testcases)

    def testcases(self):
        return self._testcases

    def timeout(self):
        return sum([int(c.timeout) for c in self.testcases()])

    def add(self, cs):
        for c in cs:
            if not isinstance(c, Testcase):
                c = Testcase(c)
            self._testcases.append(c)

    def __str__(self):
        return "%s: %s" % (self.name, str(["%s: %s" % (n, c) for n, c in enumerate(self.testcases())]))

    def __json__(self):
        return {
            "testcases": [c.__json__() for c in self.testcases()]
        }

class Testcase(object):
    name = None
    filename = None
    source = None
    timeout = 60

    def __init__(self, filename=None, name=None):
        if name is None and filename is None:
            raise Exception("At least a filename must be given")
        if name is None:
            self.name = os.path.basename(filename)
        else:
            self.name = name
        self.filename = filename

    @staticmethod
    def from_line(line):
        token = line.split()
        token.reverse()
        assert len(token) > 0, "Testcase filename is mandatory"
        filename = token.pop()
        c = Testcase(filename)
        for k, v in utils.cmdline_to_dict(" ".join(token)).items():
            if k == "timeout":
                c.timeout = int(v)
        return c

    def source(self):
        src = None
        with open(self.filename, "r") as f:
            src = f.read()
        return src
    def __str__(self):
        return "%s (%s, %s)" % (self.name, self.filename, self.timeout)
    def __json__(self):
        return self.__dict__

class TestSession(UpdateableObject):
    dirname = None
    cookie = None

    do_cleanup = False

    def __init__(self, cookie, session_path, cleanup=True):
        assert session_path is not None, "session path can not be None"

        self.do_cleanup = cleanup
        self.cookie = cookie
        self.dirname = run("mktemp -d '%s/test-session-%s-XXXXXX'" % \
                           (session_path, self.cookie))
        run("chmod a+X '%s'" % self.dirname)
        logger.info("Starting session %s in %s" % (self.cookie, self.dirname))

    def remove(self):
        logger.debug("Removing testdir '%s'" % self.dirname)
        for artifact in self.artifacts():
            logger.debug("Removing artifact '%s'" % artifact)
            os.remove(os.path.join(self.dirname, artifact))
        os.removedirs(self.dirname)

    def add_artifact(self, name, data):
        assert("/" not in name and "\\" not in name)
        afilename = os.path.join(self.dirname, name)
        # fixme collsisions
        with open(afilename, "w") as afile:
            afile.write(data)

    def artifacts(self, use_abs = False):
        fns = os.listdir(self.dirname)
        if use_abs:
            fns = [os.path.join(self.dirname, fn) \
                   for fn in fns]
        return fns

    def get_artifacts_archive(self, selection=None):
        selection = selection or self.artifacts()
        r = StringIO.StringIO()
        logger.debug("Preparing artifacts archive for session %s" % self.cookie)
        with tarfile.open(fileobj=r,mode="w:bz2") as archive:
            for artifact in selection:
                if artifact not in self.artifacts():
                    logger.debug("Artifact not here: %s" % artifact)
                logger.debug("Adding artifact %s" % artifact)
                archive.add(os.path.join(self.dirname, artifact), artifact)
        return r

    def __enter__(self):
        logger.debug("With session '%s'" % self.cookie)
        return self

    def __exit__(self, type, value, traceback):
        logger.debug("Ending session %s" % self.cookie)
        if self.do_cleanup:
            self.remove()
        logger.info("Session '%s' ended." % self.cookie)

if __name__ == "__main__":
    suites = Factory.testsuites_from_path("../testcases/")
    print suites
<|endoftext|>


<commit_before>"""
"""

import os
import platform
import socket
import sys

from datetime import timedelta
from time import time

start_time = time()

def error_report_extra_provider(request):
    ts = os.times()
    e = request.environ
    if 'CONTENT_LENGTH' in e and e['CONTENT_LENGTH'] and request.form:
        form = filter_names(request.form, ignore=(
            'password',
            'confirm_password'
        ))
    else:
        form = {}
    return {
        'HTTP_ACCEPT_LANGUAGE': e['HTTP_ACCEPT_LANGUAGE'],
        'HTTP_REFERER': e.get('HTTP_REFERER', '?'),
        'HTTP_USER_AGENT': e['HTTP_USER_AGENT'],
        'PATH_INFO': e['PATH_INFO'],
        'REMOTE_ADDR': e['REMOTE_ADDR'],
        'REQUEST_METHOD': e['REQUEST_METHOD'],
        'executable': sys.executable,
        'hostname': socket.gethostname(),
        'http_cookies': request.cookies,
        'http_form': form,
        'machine': platform.machine(),
        'modules': modules_info(),
        'process_uptime': timedelta(seconds=time() - start_time),
        'python_compiler': platform.python_compiler(),
        'python_version': platform.python_version(),
        'release': platform.release(),
        'route_args': dict(e['route_args']),
        'stime': timedelta(seconds=ts[1]),
        'system': platform.system(),
        'utime': timedelta(seconds=ts[0]),
        'uwsgi.version': e.get('uwsgi.version', '?'),
    }

def filter_names(d, ignore):
    return dict((name, d[name]) for name in d if name not in ignore)

def modules_info():
    def predicate(m):
        return (hasattr(m, '__version__') and
                not (m.__name__.startswith('_') or '._' in m.__name__))
    return sorted([(m.__name__, m.__version__) for m in sys.modules.values()
                   if predicate(m)])

ERROR_REPORT_FORMAT = """
%(message)s

Environ Variables
-----------------
PATH_INFO: %(PATH_INFO)s
REQUEST_METHOD: %(REQUEST_METHOD)s
REMOTE_ADDR: %(REMOTE_ADDR)s
HTTP_REFERER: %(HTTP_REFERER)s
HTTP_ACCEPT_LANGUAGE: %(HTTP_ACCEPT_LANGUAGE)s
HTTP_USER_AGENT: %(HTTP_USER_AGENT)s

HTTP Request
------------
Route: %(route_args)s
Cookies: %(http_cookies)s
Form: %(http_form)s

Hosting Process
---------------
Process Id: %(process)d
Up Time: %(process_uptime)s
User Time: %(utime)s
System Time: %(stime)s

Machine Platform
----------------
Host: %(hostname)s
OS: %(system)s %(release)s %(machine)s
Python Version: %(python_version)s [%(python_compiler)s]
uWSGI Version: %(uwsgi.version)s
Executable: %(executable)s
Timestamp: %(asctime)s

Loaded Modules
--------------
%(modules)s

"""
<commit_msg>Use get for HTTP environment variables.<commit_after>"""
"""

import os
import platform
import socket
import sys

from datetime import timedelta
from time import time

start_time = time()

def error_report_extra_provider(request):
    ts = os.times()
    e = request.environ
    if 'CONTENT_LENGTH' in e and e['CONTENT_LENGTH'] and request.form:
        form = filter_names(request.form, ignore=(
            'password',
            'confirm_password'
        ))
    else:
        form = {}
    return {
        'HTTP_ACCEPT_LANGUAGE': e.get('HTTP_ACCEPT_LANGUAGE', '?'),
        'HTTP_REFERER': e.get('HTTP_REFERER', '?'),
        'HTTP_USER_AGENT': e.get('HTTP_USER_AGENT', '?'),
        'PATH_INFO': e['PATH_INFO'],
        'REMOTE_ADDR': e['REMOTE_ADDR'],
        'REQUEST_METHOD': e['REQUEST_METHOD'],
        'executable': sys.executable,
        'hostname': socket.gethostname(),
        'http_cookies': request.cookies,
        'http_form': form,
        'machine': platform.machine(),
        'modules': modules_info(),
        'process_uptime': timedelta(seconds=time() - start_time),
        'python_compiler': platform.python_compiler(),
        'python_version': platform.python_version(),
        'release': platform.release(),
        'route_args': dict(e['route_args']),
        'stime': timedelta(seconds=ts[1]),
        'system': platform.system(),
        'utime': timedelta(seconds=ts[0]),
        'uwsgi.version': e.get('uwsgi.version', '?'),
    }

def filter_names(d, ignore):
    return dict((name, d[name]) for name in d if name not in ignore)

def modules_info():
    def predicate(m):
        return (hasattr(m, '__version__') and
                not (m.__name__.startswith('_') or '._' in m.__name__))
    return sorted([(m.__name__, m.__version__) for m in sys.modules.values()
                   if predicate(m)])

ERROR_REPORT_FORMAT = """
%(message)s

Environ Variables
-----------------
PATH_INFO: %(PATH_INFO)s
REQUEST_METHOD: %(REQUEST_METHOD)s
REMOTE_ADDR: %(REMOTE_ADDR)s
HTTP_REFERER: %(HTTP_REFERER)s
HTTP_ACCEPT_LANGUAGE: %(HTTP_ACCEPT_LANGUAGE)s
HTTP_USER_AGENT: %(HTTP_USER_AGENT)s

HTTP Request
------------
Route: %(route_args)s
Cookies: %(http_cookies)s
Form: %(http_form)s

Hosting Process
---------------
Process Id: %(process)d
Up Time: %(process_uptime)s
User Time: %(utime)s
System Time: %(stime)s

Machine Platform
----------------
Host: %(hostname)s
OS: %(system)s %(release)s %(machine)s
Python Version: %(python_version)s [%(python_compiler)s]
uWSGI Version: %(uwsgi.version)s
Executable: %(executable)s
Timestamp: %(asctime)s

Loaded Modules
--------------
%(modules)s

"""
<|endoftext|>


<commit_before>import csv
import datetime
import os

import numpy as np

from .wattbike_importer import HubClient

BASE_TEMPLATE = os.path.join(os.path.dirname(os.path.realpath(__file__)), './templates/tcx_base')
TRACKPOINT_TEMPLATE = os.path.join(os.path.dirname(os.path.realpath(__file__)), './templates/tcx_trackpoint')

class Workout:
    def __init__(self, session_ids):
        self.sessions = list()
        for session_id in session_ids:
            self.sessions.append(HubClient().get_session(session_id))

    def __repr__(self):
        return '<Workout instance containing {} sessions>'.format(len(self.session))

    def export_to_tcx(self, fname=None, session=None):
        if session:
            sessions = [session]
        else:
            sessions = self.sessions

        trackpoints = str()
        min_starttime = datetime.datetime.now()
        total_time = 0.0
        total_hr = 0.0
        hr_max = 0
        total_cad = 0
        total_pwr = 0

        for session in sessions:
            trackpoints += self._tcx_render_trackpoints(session)

            session_start = datetime.datetime.strptime(
                session['headers']['startDate'], '%Y-%m-%d %H:%M:%S'
            )
            if session_start < min_starttime:
                min_starttime = session_start
            duration = float(session['headers']['timeTotal'])
            total_time += duration
            total_hr += float(session['headers']['hrAvg'])*duration
            if float(session['headers']['hrPeak']) > hr_max:
                hr_max = float(session['headers']['hrPeak'])
            total_cad += float(session['headers']['cadence'])*duration
            total_pwr += float(session['headers']['powerAvg'])*duration

        tcx_string = self._tcx_render_base(
            trackpoints=trackpoints,
            start_time=min_starttime.isoformat(),
            total_time=int(total_time),
            avg_hr=int(total_hr/total_time),
            max_hr=int(hr_max),
            avg_cad=int(total_cad/total_time),
            avg_pwr=int(total_pwr/total_time)
        )

        fname = '{}_{}.tcx'.format(
            self.sessions[0]['headers']['title'],
            min_starttime
        )

        with open(fname, 'w') as output_file:
            output_file.write(tcx_string)

    def export_to_csv(self, fname=None):
        pass

    def _tcx_render_trackpoints(self, session=None, trackpoint_template=None):
        trackpoint_template = trackpoint_template if trackpoint_template\
            else TRACKPOINT_TEMPLATE
        with open(trackpoint_template, 'r') as tcx_trackpoint_file:
            tcx_trackpoint = tcx_trackpoint_file.read()

        session = session if session else self.session[0]

        start_ts = session['xAxis'][0]
        timestamps = [(ts - start_ts)/1000. for ts in session['xAxis']]
        x_axis = range(max(session['time']))
        heartrate = np.interp(x_axis, timestamps, session['hr'])
        cadence = np.interp(x_axis, timestamps, session['cadence'])
        power = np.interp(x_axis, timestamps, session['power'])

        trackpoints = str()
        start = datetime.datetime.strptime(
            session['headers']['startDate'], '%Y-%m-%d %H:%M:%S'
        )

        for t, hr, cad, pwr in zip(x_axis, heartrate, cadence, power):
            timestamp = start + datetime.timedelta(0, t)
            trkp = tcx_trackpoint.replace('{TIMESTAMP}', timestamp.isoformat()+'Z')
            trkp = trkp.replace('{HEARTRATE}', str(int(hr)))
            trkp = trkp.replace('{CADENCE}', str(int(cad)))
            trkp = trkp.replace('{POWER}', str(int(pwr)))
            trackpoints += trkp

        return trackpoints

    def _tcx_render_base(self, trackpoints, start_time, total_time, avg_hr,
                         max_hr, avg_cad, avg_pwr, dist=0.0,
                         cal=0, avg_spd=0.0, max_spd=0.0,
                         base_template=None):

        base_template = base_template if base_template\
            else BASE_TEMPLATE

        with open(base_template, 'r') as tcx_base_file:
            tcx_base = tcx_base_file.read()
        repl_dict = {
            '{TRACKPOINTS}': trackpoints,
            '{ACTIVITY_ID}': start_time+'Z',
            '{LAP_STARTTIME}': '"'+start_time+'Z"',
            '{TOTAL_TIME_SECONDS}': str(total_time),
            '{DISTANCE_METERS}': str(dist),
            '{MAXIMUMSPEED}': str(max_spd),
            '{CALORIES}': str(cal),
            '{AVERAGE_HEARTRATE}': str(avg_hr),
            '{MAXIMUM_HEARTRATE}': str(max_hr),
            '{AVERAGE_CADENCE}': str(avg_cad),
            '{AVERAGE_WATTS}': str(avg_pwr),
            '{AVERAGE_SPEED}': str(avg_spd)
        }

        for key, val in repl_dict.items():
            tcx_base = tcx_base.replace(key, val)

        return tcx_base
<commit_msg>NEW - Merge multiple sessions into one file<commit_after>import csv
import datetime
import os

import numpy as np

from .wattbike_importer import HubClient

BASE_TEMPLATE = os.path.join(os.path.dirname(os.path.realpath(__file__)), './templates/tcx_base')
TRACKPOINT_TEMPLATE = os.path.join(os.path.dirname(os.path.realpath(__file__)), './templates/tcx_trackpoint')

class Workout:
    def __init__(self, session_ids):
        self.sessions = list()
        for session_id in session_ids:
            self.sessions.append(HubClient().get_session(session_id))

    def __repr__(self):
        return '<Workout instance containing {} sessions>'.format(len(self.session))

    def export_to_tcx(self, fname=None, session=None):
        if session:
            sessions = [session]
        else:
            sessions = self.sessions

        trackpoints = dict()
        min_starttime = datetime.datetime.now()
        total_time = 0.0
        total_hr = 0.0
        hr_max = 0
        total_cad = 0
        total_pwr = 0

        for session in sessions:
            session_start, session_trackpoints = self._tcx_render_trackpoints(session)
            trackpoints[session_start] = session_trackpoints

            session_start = datetime.datetime.strptime(
                session['headers']['startDate'], '%Y-%m-%d %H:%M:%S'
            )
            if session_start < min_starttime:
                min_starttime = session_start
            duration = float(session['headers']['timeTotal'])
            total_time += duration
            total_hr += float(session['headers']['hrAvg'])*duration
            if float(session['headers']['hrPeak']) > hr_max:
                hr_max = float(session['headers']['hrPeak'])
            total_cad += float(session['headers']['cadence'])*duration
            total_pwr += float(session['headers']['powerAvg'])*duration

        # Multiple session trackpoint ordering
        trackpoints_string = str()
        while trackpoints:
            min_session = min(trackpoints.keys())
            trackpoints_string += trackpoints.pop(min_session)

        tcx_string = self._tcx_render_base(
            trackpoints=trackpoints_string,
            start_time=min_starttime.isoformat(),
            total_time=int(total_time),
            avg_hr=int(total_hr/total_time),
            max_hr=int(hr_max),
            avg_cad=int(total_cad/total_time),
            avg_pwr=int(total_pwr/total_time)
        )

        fname = '{}_{}.tcx'.format(
            self.sessions[0]['headers']['title'],
            min_starttime
        )

        with open(fname, 'w') as output_file:
            output_file.write(tcx_string)

    def export_to_csv(self, fname=None):
        pass

    def _tcx_render_trackpoints(self, session=None, trackpoint_template=None):
        trackpoint_template = trackpoint_template if trackpoint_template\
            else TRACKPOINT_TEMPLATE
        with open(trackpoint_template, 'r') as tcx_trackpoint_file:
            tcx_trackpoint = tcx_trackpoint_file.read()

        session = session if session else self.session[0]

        start_ts = session['xAxis'][0]
        timestamps = [(ts - start_ts)/1000. for ts in session['xAxis']]
        x_axis = range(max(session['time']))
        heartrate = np.interp(x_axis, timestamps, session['hr'])
        cadence = np.interp(x_axis, timestamps, session['cadence'])
        power = np.interp(x_axis, timestamps, session['power'])

        # Some padding to prevent interpolation of data after merging
        x_axis = [-2, -1] + list(x_axis) + [len(list(x_axis)), len(list(x_axis))+1]
        heartrate = [0, 0] + list(heartrate) + [0, 0]
        cadence = [0, 0] + list(cadence) + [0, 0]
        power = [0, 0] + list(power) + [0, 0]

        trackpoints = str()
        start = datetime.datetime.strptime(
            session['headers']['startDate'], '%Y-%m-%d %H:%M:%S'
        )

        for t, hr, cad, pwr in zip(x_axis, heartrate, cadence, power):
            timestamp = start + datetime.timedelta(0, t)
            trkp = tcx_trackpoint.replace('{TIMESTAMP}', timestamp.isoformat()+'Z')
            trkp = trkp.replace('{HEARTRATE}', str(int(hr)))
            trkp = trkp.replace('{CADENCE}', str(int(cad)))
            trkp = trkp.replace('{POWER}', str(int(pwr)))
            trackpoints += trkp

        return start, trackpoints

    def _tcx_render_base(self, trackpoints, start_time, total_time, avg_hr,
                         max_hr, avg_cad, avg_pwr, dist=0.0,
                         cal=0, avg_spd=0.0, max_spd=0.0,
                         base_template=None):

        base_template = base_template if base_template\
            else BASE_TEMPLATE

        with open(base_template, 'r') as tcx_base_file:
            tcx_base = tcx_base_file.read()
        repl_dict = {
            '{TRACKPOINTS}': trackpoints,
            '{ACTIVITY_ID}': start_time+'Z',
            '{LAP_STARTTIME}': '"'+start_time+'Z"',
            '{TOTAL_TIME_SECONDS}': str(total_time),
            '{DISTANCE_METERS}': str(dist),
            '{MAXIMUMSPEED}': str(max_spd),
            '{CALORIES}': str(cal),
            '{AVERAGE_HEARTRATE}': str(avg_hr),
            '{MAXIMUM_HEARTRATE}': str(max_hr),
            '{AVERAGE_CADENCE}': str(avg_cad),
            '{AVERAGE_WATTS}': str(avg_pwr),
            '{AVERAGE_SPEED}': str(avg_spd)
        }

        for key, val in repl_dict.items():
            tcx_base = tcx_base.replace(key, val)

        return tcx_base
<|endoftext|>


<commit_before>#!/usr/bin/env python
from copy import deepcopy
import os
import cPickle
import re
import sys
import logging
import ConfigParser

from hunmisc.utils.huntool_wrapper import Hundisambig, Ocamorph, MorphAnalyzer

from construction import VerbConstruction
from sentence_parser import SentenceParser
from lexicon import Lexicon
from pymachine.src.machine import MachineGraph
from pymachine.src.machine import Machine
from pymachine.src.control import ConceptControl
from spreading_activation import SpreadingActivation
from definition_parser import read as read_defs
from sup_dic import supplementary_dictionary_reader as sdreader
from dep_map import dep_map_reader
#from demo_misc import add_verb_constructions, add_avm_constructions
import np_grammar

class KeyDefaultDict(dict):
    def __missing__(self, key):
        return key

def jaccard(s1, s2):
    try:
        return float(len(s1 & s2)) / len(s1 | s2)
    except ZeroDivisionError:
        return 0.0

class Wrapper:

    dep_regex = re.compile("([a-z_]*)\((.*?)-([0-9]*)'*, (.*?)-([0-9]*)'*\)")
    num_re = re.compile(r'^[0-9.,]+$', re.UNICODE)

    def get_lemma(self, word):
        if word in self.tok2lemma:
            return self.tok2lemma[word]
        elif Wrapper.num_re.match(word):
            return word
        for char in ('.', ',', '=', '"', "'", '/'):
            for part in word.split(char):
                if part in self.tok2lemma:
                    logging.info('returning lemma of {0} instead of {1}')
                    return self.tok2lemma[part]

        logging.info(u'analyzing {0}'.format(word))
        self.tok2lemma[word] = list(self.analyzer.analyze(
            [[word]]))[0][0][1].split('||')[0].split('<')[0]
        logging.info(u'got this: {0}'.format(self.tok2lemma[word]))
        return self.tok2lemma[word]

    @staticmethod
    def get_analyzer():
        hunmorph_dir = os.environ['HUNMORPH_DIR']
        return MorphAnalyzer(
            Ocamorph(
                os.path.join(hunmorph_dir, "ocamorph"),
                os.path.join(hunmorph_dir, "morphdb_en.bin")),
            Hundisambig(
                os.path.join(hunmorph_dir, "hundisambig"),
                os.path.join(hunmorph_dir, "en_wsj.model")))

    @staticmethod
    def get_tok2lemma(tok2lemma_fn):
        tok2lemma = {}
        if tok2lemma_fn is None:
            return tok2lemma
        for line in file(tok2lemma_fn):
            try:
                tok, lemma = line.decode('utf-8').strip().split('\t')
            except (ValueError, UnicodeDecodeError), e:
                raise Exception(
                    'error parsing line in tok2lemma file: {0}\n{1}'.format(
                        e, line))
            tok2lemma[tok] = lemma

        return tok2lemma

    def __init__(self, cf, include_longman=False, batch=False):
        self.cfn = cf
        self.__read_config()
        self.batch = batch

        self.tok2lemma = Wrapper.get_tok2lemma(self.tok2lemma_fn)
        self.wordlist = set()
        self.dep_to_op = dep_map_reader(self.dep_map_fn)
        self.__read_definitions()
        if include_longman:
            self.get_longman_definitions()
        self.__read_supp_dict()
        self.reset_lexicon()

    def reset_lexicon(self, load_from=None, save_to=None):
        if load_from:
            self.lexicon = cPickle.load(open(load_from))
        else:
            self.lexicon = Lexicon()
            self.__add_definitions()
            self.__add_constructions()
        if save_to:
            cPickle.dump(self.lexicon, open(save_to, 'w'))

    def __read_config(self):
        machinepath = os.path.realpath(__file__).rsplit("/", 2)[0]
        if "MACHINEPATH" in os.environ:
            machinepath = os.environ["MACHINEPATH"]
        config = ConfigParser.SafeConfigParser({"machinepath": machinepath})
        logging.info('reading machine config from {0}'.format(self.cfn))
        config.read(self.cfn)
        items = dict(config.items("machine"))
        self.def_files = [(s.split(":")[0].strip(), int(s.split(":")[1]))
                          for s in items["definitions"].split(",")]
        self.dep_map_fn = items.get("dep_map")
        self.tok2lemma_fn = items.get("tok2lemma")
        self.longman_deps_path = items.get("longman_deps")
        self.supp_dict_fn = items.get("supp_dict")
        self.plural_fn = items.get("plurals")

    def __read_definitions(self):
        self.definitions = {}
        for file_name, printname_index in self.def_files:
            # TODO HACK makefile needed
            if (file_name.endswith("generated") and
                    not os.path.exists(file_name)):
                raise Exception(
                    "A definition file that should be generated" +
                    " by pymachine/scripts/generate_translation_dict.sh" +
                    " does not exist: {0}".format(file_name))

            if file_name.endswith('pickle'):
                logging.debug('loading 4lang definitions...')
                definitions = cPickle.load(file(file_name))
            else:
                logging.info('parsing 4lang definitions...')
                definitions = read_defs(
                    file(file_name), self.plural_fn, printname_index,
                    three_parts=True)

                logging.info('dumping 4lang definitions to file...')
                f = open('{0}.pickle'.format(file_name), 'w')
                cPickle.dump(definitions, f)

            self.definitions.update(definitions)

    def __add_definitions(self):
            definitions = deepcopy(self.definitions)
            self.lexicon.add_static(definitions.itervalues())
            self.lexicon.finalize_static()

    def __read_supp_dict(self):
        self.supp_dict = sdreader(
            file(self.supp_dict_fn)) if self.supp_dict_fn else {}

    def __add_constructions(self):
        for construction in np_grammar.np_rules:
            self.lexicon.add_construction(construction)
        #add_verb_constructions(self.lexicon, self.supp_dict)
        #add_avm_constructions(self.lexicon, self.supp_dict)

    def get_longman_definitions(self):
        #logging.info('adding Longman definitions')
        self.analyzer = Wrapper.get_analyzer()
        if self.longman_deps_path.endswith('pickle'):
            logging.info('loading Longman definitions...')
            definitions = cPickle.load(file(self.longman_deps_path))

        else:
            files = os.listdir(self.longman_deps_path)

            logging.info('only parsing first meanings for now')
            files = filter(lambda fn: '_' not in fn, files)
            #TODO

            logging.info('will now parse {0} definitions'.format(len(files)))
            definitions = {}
            for c, fn in enumerate(files):
                if c % 1000 == 0:
                    logging.info('{0}...'.format(c))
                word, _ = fn.split('.')
                deps = [
                    line.strip() for line in open(
                        os.path.join(self.longman_deps_path, fn))]
                machine = self.get_dep_definition(word, deps)
                if machine is None:
                    continue
                definitions[word] = machine

            logging.info('pickling Longman definitions...')
            f = open('{0}.pickle'.format(self.longman_deps_path), 'w')
            cPickle.dump(definitions, f)

        for word, machine in definitions.iteritems():
            if word not in self.definitions:
                self.definitions[word] = machine

        logging.info('done')

    @staticmethod
    def parse_dependency(string):
        dep_match = Wrapper.dep_regex.match(string)
        if not dep_match:
            raise Exception('cannot parse dependency: {0}'.format(string))
        dep, word1, id1, word2, id2 = dep_match.groups()
        return dep, (word1, id1), (word2, id2)

    def get_dep_definition(self, word, dep_strings):
        deps = map(Wrapper.parse_dependency, dep_strings)
        root_deps = filter(lambda d: d[0] == 'root', deps)
        root_word, root_id = root_deps[0][2]
        root_lemma = self.get_lemma(root_word)
        if len(root_deps) != 1:
            logging.warning(
                'no unique root dependency, skipping word "{0}"'.format(word))
            return None

        word2machine = {}
        for dep, (word1, id1), (word2, id2) in deps:
            lemma1 = self.get_lemma(word1)
            lemma2 = self.get_lemma(word2)
            #logging.info('lemma1: {0}, lemma2: {1}'.format(lemma1, lemma2))
            machine1, machine2 = self._add_dependency(
                dep, (lemma1, id1), (lemma2, id2), word2machine=word2machine)
            word2machine[lemma1] = machine1
            word2machine[lemma2] = machine2

        root_machine = word2machine[root_lemma]
        machine = Machine(word, ConceptControl())
        machine.append(root_machine, 0)
        return machine

    def add_dependency(self, string):
        #e.g. nsubjpass(pushed-7, salesman-5)
        logging.debug('processing dependency: {}'.format(string))
        dep, (word1, id1), (word2, id2) = Wrapper.parse_dependency(string)
        lemma1 = self.get_lemma(word1)
        lemma2 = self.get_lemma(word2)
        self._add_dependency(dep, (lemma1, id1), (lemma2, id2),
                             use_lexicon=True, activate_machines=True)

    def _add_dependency(self, dep, (word1, id1), (word2, id2),
                        word2machine=None, use_lexicon=False,
                        activate_machines=False):
        """Given a triplet from Stanford Dep.: D(w1,w2), we create and activate
        machines for w1 and w2, then run all operators associated with D on the
        sequence of the new machines (m1, m2)"""
        machines = []
        for word in (word1, word2):
            if use_lexicon:
                machine = self.lexicon.get_machine(word)
            else:
                machine = word2machine.get(word,
                                           Machine(word, ConceptControl()))

            if activate_machines:
                logging.debug('activating {}'.format(machine))
                self.lexicon.add_active(machine)
                self.lexicon.expand(machine)
            machines.append(machine)

        machine1, machine2 = machines
        for operator in self.dep_to_op.get(dep, []):
            logging.info('operator {0} acting on machines {1} and {2}'.format(
                operator, machine1, machine2))
            operator.act((machine1, machine2))

        return machine1, machine2

    def draw_word_graphs(self):
        for word, machine in self.definitions.iteritems():
            graph = MachineGraph.create_from_machines([machine])
            clean_word = Machine.d_clean(word)
            f = open('graphs/words/{0}.dot'.format(clean_word), 'w')
            f.write(graph.to_dot().encode('utf-8'))

    def word_similarity(self, word1, word2, pos1, pos2):
        lemma1, lemma2 = map(self.get_lemma, (word1, word2))
        #logging.info(u'lemma1: {0}, lemma2: {1}'.format(lemma1, lemma2))
        if lemma1 == lemma2:
            return 1
        oov = filter(lambda l: l not in self.definitions, (lemma1, lemma2))
        if oov:
            logging.debug(u'OOV: {0}, no machine similarity')
            return None

        machine1, machine2 = map(self.definitions.get, (lemma1, lemma2))
        #map(self.lexicon.add_active, (machine1, machine2))
        #map(self.lexicon.expand, (machine1, machine2))
        zero_links_1 = set(filter(
            lambda s: not s.isupper(),
            [m.printname() for m in machine1.partitions[0]]))
        zero_links_2 = set(filter(
            lambda s: not s.isupper(),
            [m.printname() for m in machine2.partitions[0]]))
        #logging.info('machine1 0-links: {0}, machine2 0-links: {1}'.format(
        #    zero_links_1, zero_links_2))
        #sim = jaccard(zero_links_1, zero_links_2)
        union = zero_links_1 | zero_links_2
        intersection = zero_links_1 & zero_links_2
        if not intersection:
            sim = 0
        else:
            sim = float(len(intersection)) / len(union)
            #sim = float(len(intersection)) / min(len(zero_links_1),
            #                                     len(zero_links_2))
            logging.info(u'lemma1: {0}, lemma2: {1}'.format(lemma1, lemma2))
            logging.info(u'shared: {0}'.format(intersection))
            logging.info('sim: {0}'.format(sim))

        draw_graphs = True
        if draw_graphs and not self.batch:
            graph = MachineGraph.create_from_machines(
                [machine1, machine2], max_depth=1)
            f = open('graphs/{0}_{1}.dot'.format(lemma1, lemma2), 'w')
            f.write(graph.to_dot().encode('utf-8'))
        return sim

    def run(self, sentence):
        """Parses a sentence, runs the spreading activation and returns the
        messages that have to be sent to the active plugins."""
        try:
            sp = SentenceParser()
            sa = SpreadingActivation(self.lexicon)
            machines = sp.parse(sentence)
            logging.debug('machines: {}'.format(machines))
            logging.debug('machines: {}'.format(
                [m for m in machines]))
            for machine_list in machines:
                for machine in machine_list:
                    if machine.control.kr['CAT'] == 'VERB':
                        logging.debug('adding verb construction for {}'.format(
                            machine))
                        self.lexicon.add_construction(VerbConstruction(
                            machine.printname(), self.lexicon, self.supp_dict))
            logging.info('constructions: {}'.format(
                self.lexicon.constructions))

            # results is a list of (url, data) tuples
            results = sa.activation_loop(machines)
            print 'results:', results
            print 'machines:', machines

            graph = MachineGraph.create_from_machines(
                [m[0] for m in machines], max_depth=1)
            f = open('machines.dot', 'w')
            f.write(graph.to_dot().encode('utf-8'))

            self.lexicon.clear_active()
        except Exception, e:
            import traceback
            traceback.print_exc(e)
            raise(e)

        return results

def test_plain():
    print 'building wrapper...'
    w = Wrapper(sys.argv[1])
    test_sen = [
        ([
            ("The", "the/ART"),
            ("snake", "snake/NOUN")], 'NP'),
        ("ate", "eat/VERB<PAST>"),
        ([
            ("the", "the/ART"),
            ("elephant", "elephant/NOUN")], 'NP')]

    print 'running...'
    w.run(test_sen)

def test_dep():
    print 'building wrapper...'
    w = Wrapper(sys.argv[1])
    for line in sys.stdin:
        w.add_dependency(line)

    active_machines = w.lexicon.active_machines()
    logging.debug('active machines: {}'.format(active_machines))
    graph = MachineGraph.create_from_machines(active_machines)
    f = open('machines.dot', 'w')
    f.write(graph.to_dot().encode('utf-8'))

def build_ext_defs():
    return Wrapper(sys.argv[1], include_longman=True)

if __name__ == "__main__":
    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s : " +
        "%(module)s (%(lineno)s) - %(levelname)s - %(message)s")
    w = build_ext_defs()
    w.draw_word_graphs()
    #f = open('wrapper.pickle', 'w')
    #cPickle.dump(w, f)
<commit_msg>new functions for drawing graphs, moved similarity functions to separate file<commit_after>#!/usr/bin/env python
from copy import deepcopy
import os
import cPickle
import re
import sys
import logging
import ConfigParser

from hunmisc.utils.huntool_wrapper import Hundisambig, Ocamorph, MorphAnalyzer

from construction import VerbConstruction
from sentence_parser import SentenceParser
from lexicon import Lexicon
from pymachine.src.machine import MachineGraph
from pymachine.src.machine import Machine
from pymachine.src.control import ConceptControl
from spreading_activation import SpreadingActivation
from definition_parser import read as read_defs
from sup_dic import supplementary_dictionary_reader as sdreader
from dep_map import dep_map_reader
#from demo_misc import add_verb_constructions, add_avm_constructions
import np_grammar

class KeyDefaultDict(dict):
    def __missing__(self, key):
        return key

def jaccard(s1, s2):
    try:
        return float(len(s1 & s2)) / len(s1 | s2)
    except ZeroDivisionError:
        return 0.0

class Wrapper:

    dep_regex = re.compile("([a-z_]*)\((.*?)-([0-9]*)'*, (.*?)-([0-9]*)'*\)")
    num_re = re.compile(r'^[0-9.,]+$', re.UNICODE)

    def get_lemma(self, word):
        if word in self.tok2lemma:
            return self.tok2lemma[word]
        elif Wrapper.num_re.match(word):
            return word
        for char in ('.', ',', '=', '"', "'", '/'):
            for part in word.split(char):
                if part in self.tok2lemma:
                    logging.info('returning lemma of {0} instead of {1}')
                    return self.tok2lemma[part]

        logging.info(u'analyzing {0}'.format(word))
        self.tok2lemma[word] = list(self.analyzer.analyze(
            [[word]]))[0][0][1].split('||')[0].split('<')[0]
        logging.info(u'got this: {0}'.format(self.tok2lemma[word]))
        return self.tok2lemma[word]

    @staticmethod
    def get_analyzer():
        hunmorph_dir = os.environ['HUNMORPH_DIR']
        return MorphAnalyzer(
            Ocamorph(
                os.path.join(hunmorph_dir, "ocamorph"),
                os.path.join(hunmorph_dir, "morphdb_en.bin")),
            Hundisambig(
                os.path.join(hunmorph_dir, "hundisambig"),
                os.path.join(hunmorph_dir, "en_wsj.model")))

    @staticmethod
    def get_tok2lemma(tok2lemma_fn):
        tok2lemma = {}
        if tok2lemma_fn is None:
            return tok2lemma
        for line in file(tok2lemma_fn):
            try:
                tok, lemma = line.decode('utf-8').strip().split('\t')
            except (ValueError, UnicodeDecodeError), e:
                raise Exception(
                    'error parsing line in tok2lemma file: {0}\n{1}'.format(
                        e, line))
            tok2lemma[tok] = lemma

        return tok2lemma

    def __init__(self, cf, include_longman=False, batch=False):
        self.cfn = cf
        self.__read_config()
        self.batch = batch

        self.tok2lemma = Wrapper.get_tok2lemma(self.tok2lemma_fn)
        self.wordlist = set()
        self.dep_to_op = dep_map_reader(self.dep_map_fn)
        self.__read_definitions()
        if include_longman:
            self.get_longman_definitions()
        self.__read_supp_dict()
        self.reset_lexicon()

    def reset_lexicon(self, load_from=None, save_to=None):
        if load_from:
            self.lexicon = cPickle.load(open(load_from))
        else:
            self.lexicon = Lexicon()
            self.__add_definitions()
            self.__add_constructions()
        if save_to:
            cPickle.dump(self.lexicon, open(save_to, 'w'))

    def __read_config(self):
        machinepath = os.path.realpath(__file__).rsplit("/", 2)[0]
        if "MACHINEPATH" in os.environ:
            machinepath = os.environ["MACHINEPATH"]
        config = ConfigParser.SafeConfigParser({"machinepath": machinepath})
        logging.info('reading machine config from {0}'.format(self.cfn))
        config.read(self.cfn)
        items = dict(config.items("machine"))
        self.def_files = [(s.split(":")[0].strip(), int(s.split(":")[1]))
                          for s in items["definitions"].split(",")]
        self.dep_map_fn = items.get("dep_map")
        self.tok2lemma_fn = items.get("tok2lemma")
        self.longman_deps_path = items.get("longman_deps")
        self.supp_dict_fn = items.get("supp_dict")
        self.plural_fn = items.get("plurals")

    def __read_definitions(self):
        self.definitions = {}
        for file_name, printname_index in self.def_files:
            # TODO HACK makefile needed
            if (file_name.endswith("generated") and
                    not os.path.exists(file_name)):
                raise Exception(
                    "A definition file that should be generated" +
                    " by pymachine/scripts/generate_translation_dict.sh" +
                    " does not exist: {0}".format(file_name))

            if file_name.endswith('pickle'):
                logging.debug('loading 4lang definitions...')
                definitions = cPickle.load(file(file_name))
            else:
                logging.info('parsing 4lang definitions...')
                definitions = read_defs(
                    file(file_name), self.plural_fn, printname_index,
                    three_parts=True)

                logging.info('dumping 4lang definitions to file...')
                f = open('{0}.pickle'.format(file_name), 'w')
                cPickle.dump(definitions, f)

            self.definitions.update(definitions)

    def __add_definitions(self):
            definitions = deepcopy(self.definitions)
            self.lexicon.add_static(definitions.itervalues())
            self.lexicon.finalize_static()

    def __read_supp_dict(self):
        self.supp_dict = sdreader(
            file(self.supp_dict_fn)) if self.supp_dict_fn else {}

    def __add_constructions(self):
        for construction in np_grammar.np_rules:
            self.lexicon.add_construction(construction)
        #add_verb_constructions(self.lexicon, self.supp_dict)
        #add_avm_constructions(self.lexicon, self.supp_dict)

    def get_longman_definitions(self):
        #logging.info('adding Longman definitions')
        self.analyzer = Wrapper.get_analyzer()
        if self.longman_deps_path.endswith('pickle'):
            logging.info('loading Longman definitions...')
            definitions = cPickle.load(file(self.longman_deps_path))

        else:
            files = os.listdir(self.longman_deps_path)

            logging.info('only parsing first meanings for now')
            files = filter(lambda fn: '_' not in fn, files)
            #TODO

            logging.info('will now parse {0} definitions'.format(len(files)))
            definitions = {}
            for c, fn in enumerate(files):
                if c % 1000 == 0:
                    logging.info('{0}...'.format(c))
                word, _ = fn.split('.')
                deps = [
                    line.strip() for line in open(
                        os.path.join(self.longman_deps_path, fn))]
                machine = self.get_dep_definition(word, deps)
                if machine is None:
                    continue
                definitions[word] = machine

            logging.info('pickling Longman definitions...')
            f = open('{0}.pickle'.format(self.longman_deps_path), 'w')
            cPickle.dump(definitions, f)

        for word, machine in definitions.iteritems():
            if word not in self.definitions:
                self.definitions[word] = machine

        logging.info('done')

    @staticmethod
    def parse_dependency(string):
        dep_match = Wrapper.dep_regex.match(string)
        if not dep_match:
            raise Exception('cannot parse dependency: {0}'.format(string))
        dep, word1, id1, word2, id2 = dep_match.groups()
        return dep, (word1, id1), (word2, id2)

    def get_dep_definition(self, word, dep_strings):
        deps = map(Wrapper.parse_dependency, dep_strings)
        root_deps = filter(lambda d: d[0] == 'root', deps)
        root_word, root_id = root_deps[0][2]
        root_lemma = self.get_lemma(root_word)
        if len(root_deps) != 1:
            logging.warning(
                'no unique root dependency, skipping word "{0}"'.format(word))
            return None

        word2machine = {}
        for dep, (word1, id1), (word2, id2) in deps:
            lemma1 = self.get_lemma(word1)
            lemma2 = self.get_lemma(word2)
            #logging.info('lemma1: {0}, lemma2: {1}'.format(lemma1, lemma2))
            machine1, machine2 = self._add_dependency(
                dep, (lemma1, id1), (lemma2, id2), word2machine=word2machine)
            word2machine[lemma1] = machine1
            word2machine[lemma2] = machine2

        root_machine = word2machine[root_lemma]
        machine = Machine(word, ConceptControl())
        machine.append(root_machine, 0)
        return machine

    def add_dependency(self, string):
        #e.g. nsubjpass(pushed-7, salesman-5)
        logging.debug('processing dependency: {}'.format(string))
        dep, (word1, id1), (word2, id2) = Wrapper.parse_dependency(string)
        lemma1 = self.get_lemma(word1)
        lemma2 = self.get_lemma(word2)
        self._add_dependency(dep, (lemma1, id1), (lemma2, id2),
                             use_lexicon=True, activate_machines=True)

    def _add_dependency(self, dep, (word1, id1), (word2, id2),
                        word2machine=None, use_lexicon=False,
                        activate_machines=False):
        """Given a triplet from Stanford Dep.: D(w1,w2), we create and activate
        machines for w1 and w2, then run all operators associated with D on the
        sequence of the new machines (m1, m2)"""
        machines = []
        for word in (word1, word2):
            if use_lexicon:
                machine = self.lexicon.get_machine(word)
            else:
                machine = word2machine.get(word,
                                           Machine(word, ConceptControl()))

            if activate_machines:
                logging.debug('activating {}'.format(machine))
                self.lexicon.add_active(machine)
                self.lexicon.expand(machine)
            machines.append(machine)

        machine1, machine2 = machines
        for operator in self.dep_to_op.get(dep, []):
            logging.info('operator {0} acting on machines {1} and {2}'.format(
                operator, machine1, machine2))
            operator.act((machine1, machine2))

        return machine1, machine2

    def draw_single_graph(self, word):
        for w, machine in self.definitions.iteritems():
            if w != word:
                continue
            graph = MachineGraph.create_from_machines([machine])
            clean_word = Machine.d_clean(w)
            f = open('graphs/words/{0}.dot'.format(clean_word), 'w')
            f.write(graph.to_dot().encode('utf-8'))

    def draw_word_graphs(self):
        for c, (word, machine) in enumerate(self.definitions.iteritems()):
            if c % 1000 == 0:
                logging.info("{0}...".format(c))
            graph = MachineGraph.create_from_machines([machine])
            clean_word = Machine.d_clean(word)
            f = open('graphs/words/{0}.dot'.format(clean_word), 'w')
            f.write(graph.to_dot().encode('utf-8'))

    def run(self, sentence):
        """Parses a sentence, runs the spreading activation and returns the
        messages that have to be sent to the active plugins."""
        try:
            sp = SentenceParser()
            sa = SpreadingActivation(self.lexicon)
            machines = sp.parse(sentence)
            logging.debug('machines: {}'.format(machines))
            logging.debug('machines: {}'.format(
                [m for m in machines]))
            for machine_list in machines:
                for machine in machine_list:
                    if machine.control.kr['CAT'] == 'VERB':
                        logging.debug('adding verb construction for {}'.format(
                            machine))
                        self.lexicon.add_construction(VerbConstruction(
                            machine.printname(), self.lexicon, self.supp_dict))
            logging.info('constructions: {}'.format(
                self.lexicon.constructions))

            # results is a list of (url, data) tuples
            results = sa.activation_loop(machines)
            print 'results:', results
            print 'machines:', machines

            graph = MachineGraph.create_from_machines(
                [m[0] for m in machines], max_depth=1)
            f = open('machines.dot', 'w')
            f.write(graph.to_dot().encode('utf-8'))

            self.lexicon.clear_active()
        except Exception, e:
            import traceback
            traceback.print_exc(e)
            raise(e)

        return results

def test_plain():
    print 'building wrapper...'
    w = Wrapper(sys.argv[1])
    test_sen = [
        ([
            ("The", "the/ART"),
            ("snake", "snake/NOUN")], 'NP'),
        ("ate", "eat/VERB<PAST>"),
        ([
            ("the", "the/ART"),
            ("elephant", "elephant/NOUN")], 'NP')]

    print 'running...'
    w.run(test_sen)

def test_dep():
    print 'building wrapper...'
    w = Wrapper(sys.argv[1])
    for line in sys.stdin:
        w.add_dependency(line)

    active_machines = w.lexicon.active_machines()
    logging.debug('active machines: {}'.format(active_machines))
    graph = MachineGraph.create_from_machines(active_machines)
    f = open('machines.dot', 'w')
    f.write(graph.to_dot().encode('utf-8'))

def build_ext_defs():
    return Wrapper(sys.argv[1], include_longman=True)

if __name__ == "__main__":
    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s : " +
        "%(module)s (%(lineno)s) - %(levelname)s - %(message)s")
    w = build_ext_defs()
    #w.draw_single_graph('inmate')
    #w.draw_single_graph('prisoner')
    w.draw_word_graphs()
    #f = open('wrapper.pickle', 'w')
    #cPickle.dump(w, f)
<|endoftext|>


<commit_before>import os
import sys
import datetime
import os.path as op
import glob
import pandas as pd
import contextlib
import logging
import distutils.spawn
import subprocess
import shutil
from collections import OrderedDict
from collections import Callable
try:
    from IPython.display import clear_output
    have_ipython = True
except ImportError:
    have_ipython = False

log = logging.getLogger(__name__)

class Date():
    """Various methods to return formatted dates for today.
    """
    def __init__(self):
        self.short_date = self.short_date_prefix()

    def short_date_prefix(self):
        today = datetime.date.today()
        return today.strftime('%y%m%d')

def todays_short_date():
    today = Date()
    return today.short_date

class DefaultOrderedDict(OrderedDict):
    """Class to combine defaultdict and OrderedDict.
    Source: http://stackoverflow.com/a/6190500/562769

    """
    def __init__(self, default_factory=None, *a, **kw):
        if (default_factory is not None and
                not isinstance(default_factory, Callable)):
            raise TypeError('first argument must be callable')
        OrderedDict.__init__(self, *a, **kw)
        self.default_factory = default_factory

    def __getitem__(self, key):
        try:
            return OrderedDict.__getitem__(self, key)
        except KeyError:
            return self.__missing__(key)

    def __missing__(self, key):
        if self.default_factory is None:
            raise KeyError(key)
        self[key] = value = self.default_factory()
        return value

    def __reduce__(self):
        if self.default_factory is None:
            args = tuple()
        else:
            args = self.default_factory,
        return type(self), args, None, None, self.items()

    def copy(self):
        return self.__copy__()

    def __copy__(self):
        return type(self)(self.default_factory, self)

    def __deepcopy__(self, memo):
        import copy
        return type(self)(self.default_factory,
                          copy.deepcopy(self.items()))

    def __repr__(self):
        return 'OrderedDefaultDict(%s, %s)' % (self.default_factory,
                                               OrderedDict.__repr__(self))

@contextlib.contextmanager
def suppress_stdout():
    """Suppress the stdout of any function.

    Usage:
    with ssbio.utils.suppress_stdout():
        my_function_here()
    """
    with open(os.devnull, "w") as devnull:
        old_stdout = sys.stdout
        sys.stdout = devnull
        try:
            yield
        finally:
            sys.stdout = old_stdout

def split_folder_and_path(filepath):
    """Split a file path into its folder, filename, and extension

    Args:
        path (str): Path to a file

    Returns:
        tuple: of (folder, filename (without extension), extension)

    """
    dirname = op.dirname(filepath)
    filename = op.basename(filepath)
    splitext = op.splitext(filename)
    filename_without_extension = splitext[0]
    extension = splitext[1]

    return dirname, filename_without_extension, extension

def outfile_maker(inname, outext='.out', outname='', outdir='', append_to_name=''):
    """Create a default name for an output file based on the inname name, unless a output name is specified.

    Args:
        inname: Path to input file
        outext: Optional specified extension for output file (with the "."). Default is ".out".
        outfile: Optional specified name of output file.
        outdir: Optional path to output directory.

    Returns:
        str: Path to final output destination.

    Examples:

        >>> outfile_maker(inname='P00001.fasta')
        'P00001.out'

        >>> outfile_maker(inname='P00001.fasta', append_to_name='_new')
        'P00001_new.out'

        >>> outfile_maker(inname='P00001.fasta', outext='.mao')
        'P00001.mao'

        >>> outfile_maker(inname='P00001.fasta', outext='.mao', append_to_name='_new')
        'P00001_new.mao'

        >>> outfile_maker(inname='P00001.fasta', outext='.new', outname='P00001_aligned')
        'P00001_aligned.new'

        >>> outfile_maker(inname='P00001.fasta', outname='P00001_aligned')
        'P00001_aligned.out'

        >>> outfile_maker(inname='P00001.fasta', outname='P00001_aligned', append_to_name='_new')
        'P00001_aligned_new.out'

        >>> outfile_maker(inname='P00001.fasta', outname='P00001_aligned', outdir='/my/dir/')
        '/my/dir/P00001_aligned.out'

        >>> outfile_maker(inname='/test/other/dir/P00001.fasta', append_to_name='_new')
        '/test/other/dir/P00001_new.out'

        >>> outfile_maker(inname='/test/other/dir/P00001.fasta', outname='P00001_aligned')
        '/test/other/dir/P00001_aligned.out'

        >>> outfile_maker(inname='/test/other/dir/P00001.fasta', outname='P00001_aligned', outdir='/my/dir/')
        '/my/dir/P00001_aligned.out'

    """

    orig_dir, orig_name, orig_ext = split_folder_and_path(inname)

    # If output filename not provided, default is to take name of inname
    if not outname:
        outname = orig_name

    # Create new path in the same directory of old path if a new one isn't specified
    if not outdir:
        outdir = orig_dir

    # Append additional stuff to the filename if specified
    if append_to_name:
        outname = outname + append_to_name

    # Join the output filename and output extension
    final_outfile = op.join(outdir, '{}{}'.format(outname, outext))

    return final_outfile

def force_rerun(flag, outfile):
    """Check if we should force rerunning of a command if an output file exists.

    Args:
        flag (bool): Flag to force rerun.
        outfile (str): Path to output file which may already exist.

    Returns:
        bool: If we should force rerunning of a command

    Examples:
        >>> force_rerun(flag=True, outfile='/not/existing/file.txt')
        True

        >>> force_rerun(flag=False, outfile='/not/existing/file.txt')
        True

        >>> force_rerun(flag=True, outfile='/existing/file.txt')
        True

        >>> force_rerun(flag=False, outfile='/existing/file.txt')
        False

    """
    # If flag is True, always run
    if flag:
        return True
    # If flag is False but file doesn't exist, also run
    elif not flag and not op.exists(outfile):
        return True
    # If flag is False but filesize of output is 0, also run
    elif not flag and not os.stat(outfile).st_size != 0:
        return True
    # Otherwise, do not run
    else:
        return False

def program_exists(prog_name):
    """Check if a program is available as a command line executable on a system.

    Args:
        prog_name: Name of the program.

    Returns:
        bool: True if the program is available.

    """
    if distutils.spawn.find_executable(prog_name):
        return True
    else:
        return False

def command_runner(program, args, force_rerun_flag, outfile):
    """Run a program with command-line arguments.

    Args:
        program: Name of the program.
        args: Any program flags as they would be formatted in the command-line (ie. "-i test.in -o test.out").
        force_rerun_flag: If the program should be rerun even if the output file exists.
        outfile: Name out the output file which may have been generated.
            This does not specify what the outfile will be, that should be done in the args.

    Returns:
        bool: If the program ran successfully.

    """
    # Check if pepstats is installed
    if not program_exists(program):
        raise OSError('{}: program not installed'.format(program))

    # Check for force rerunning
    if force_rerun(flag=force_rerun_flag, outfile=outfile):
        cmd = '{} {}'.format(program, args)
        command = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        out, err = command.communicate()
        log.debug('{}: Ran program, output to {}'.format(program, outfile))
    else:
        log.debug('{}: Output already exists'.format(outfile))

def dict_head(d, N=5):
    """Return the head of a dictionary. It will be random!

    Default is to return the first 5 key/value pairs in a dictionary.

    Args:
        d: Dictionary to get head.
        N: Number of elements to display.

    Returns:
        dict: the first N items of the dictionary.

    """
    return {k: d[k] for k in list(d.keys())[:N]}

def rank_dated_files(pattern, dir, descending=True):
    """Search a directory for files that match a pattern. Return an ordered list of these files by filename.

    Args:
        pattern: The glob pattern to search for.
        dir: Path to directory where the files will be searched for.
        descending: Default True, will sort alphabetically by descending order.

    Returns:
        list: Rank-ordered list by filename.

    """
    files = glob.glob(op.join(dir, pattern))
    return sorted(files, reverse=descending)

def find(lst, a, case_sensitive=True):
    """Return indices of a list which have elements that match an object or list of objects

    Args:
        lst: list of values
        a: object(s) to check equality
        case_sensitive: if the search should be case sensitive

    Returns:
        list: list of indicies of lst which equal a

    """
    a = force_list(a)

    if not case_sensitive:
        lst = [x.lower() for x in lst]
        a = [y.lower() for y in a]

    return [i for i, x in enumerate(lst) if x in a]

def not_find(lst, a, case_sensitive=True):
    """Return indices of a list which have elements that DO NOT match an object or list of objects

    Args:
        lst: list of values
        a: object(s) to check inequality
        case_sensitive: if the search should be case sensitive

    Returns:
        list: list of indices of lst which do not equal a

    """
    a = force_list(a)

    if not case_sensitive:
        lst = [x.lower() for x in lst]
        a = [y.lower() for y in a]

    return [i for i, x in enumerate(lst) if x not in a]

def filter_list(lst, takeout, case_sensitive=True):
    """Return a modified list removing items specified.

    Args:
        lst: Original list of values
        takeout: Object or objects to remove from lst
        case_sensitive: if the search should be case sensitive

    Returns:
        list: Filtered list of values

    """
    takeout = force_list(takeout)

    if not case_sensitive:
        lst = [x.lower() for x in lst]
        takeout = [y.lower() for y in takeout]

    return [x for x in lst if x not in takeout]

def filter_list_by_indices(lst, indices):
    """Return a modified list containing only the indices indicated.

    Args:
        lst: Original list of values
        indices: List of indices to keep from the original list

    Returns:
        list: Filtered list of values

    """
    return [x for i, x in enumerate(lst) if i in indices]

def force_string(val=None):
    """Force a string representation of an object

    Args:
        val (str): object to parse into a string

    Returns:
        String

    """
    if val is None:
        return ''
    return val if isinstance(val, str) else ';'.join(val)

def force_list(val=None):
    """Force a list representation of an object

    Args:
        val: object to parse into a list

    Returns:

    """
    if val is None:
        return []
    if isinstance(val, pd.Series):
        return val.tolist()
    return val if isinstance(val, list) else [val]

def force_lower_list(val=None):
    """Force a lowercase list representation of strings

    Args:
        val: string or strings to parse into a list

    Returns:
        list: with lowercase values

    """
    return [x.lower() for x in force_list(val)]

def force_upper_list(val=None):
    """Force a UPPERCASE list representation of strings

    Args:
        val: string or strings to parse into a list

    Returns:
        list: with UPPERCASE values

    """
    return [x.upper() for x in force_list(val)]

def split_list_by_n(l, n):
    """Split a list into lists of size n.

    Args:
        l: List of stuff.
        n: Size of new lists.

    Returns:
        list: List of lists each of size n derived from l.

    """
    n = max(1, n)
    return list(l[i:i+n] for i in range(0, len(l), n))

def split_list_into_n_lists(l, n):
    """Split a list into n lists.

    Args:
        l: List of stuff.
        n: Number of new lists to generate.

    Returns:
        list: List of n lists.

    """
    return [l[i::n] for i in range(n)]

def input_list_parser(infile_list):
    """Always return a list of files with varying input.

    >>> input_list_parser(['/path/to/folder/'])
    ['/path/to/folder/file1.txt', '/path/to/folder/file2.txt', '/path/to/folder/file3.txt']

    >>> input_list_parser(['/path/to/file.txt'])
    ['/path/to/file.txt']

    >>> input_list_parser(['file1.txt'])
    ['file1.txt']

    Args:
        infile_list: List of arguments

    Returns:
        list: Standardized list of files

    """

    final_list_of_files = []

    for x in infile_list:

        # If the input is a folder
        if op.isdir(x):
            os.chdir(x)
            final_list_of_files.extend(glob.glob('*'))

        # If the input is a file
        if op.isfile(x):
            final_list_of_files.append(x)

    return final_list_of_files

def flatlist_dropdup(list_of_lists):
    """Make a single list out of a list of lists, and drop all duplicates.

    Args:
        list_of_lists: List of lists.

    Returns:
        list: List of single objects.

    """
    return list(set([str(item) for sublist in list_of_lists for item in sublist]))

def combinations(iterable, r):
    """Calculate combinations

    >>> list(combinations('ABCD',2))
    [['A', 'B'], ['A', 'C'], ['A', 'D'], ['B', 'C'], ['B', 'D'], ['C', 'D']]

    >>> list(combinations(range(4), 3))
    [[0, 1, 2], [0, 1, 3], [0, 2, 3], [1, 2, 3]]

    Args:
        iterable: Any iterable object.
        r: Size of combination.

    Yields:
        list: Combination of size r.

    """
    pool = tuple(iterable)
    n = len(pool)
    if r > n:
        return

    indices = list(range(r))
    yield list(pool[i] for i in indices)

    while True:
        for i in reversed(range(r)):
            if indices[i] != i + n - r:
                break
        else:
            return
        indices[i] += 1
        for j in range(i + 1, r):
            indices[j] = indices[j - 1] + 1
        yield list(pool[i] for i in indices)

def percentage_to_float(x):
    """Convert a string representation of a percentage to float.

    >>> percentage_to_float('55%')
    0.55

    Args:
        x: String representation of a percentage

    Returns:
        float: Percentage in decimal form

    """
    return float(x.strip('%')) / 100
<commit_msg>Add silent flag to command_runner<commit_after>import os
import sys
import datetime
import os.path as op
import glob
import pandas as pd
import contextlib
import logging
import distutils.spawn
import subprocess
import shutil
from collections import OrderedDict
from collections import Callable
try:
    from IPython.display import clear_output
    have_ipython = True
except ImportError:
    have_ipython = False

log = logging.getLogger(__name__)

class Date():
    """Various methods to return formatted dates for today.
    """
    def __init__(self):
        self.short_date = self.short_date_prefix()

    def short_date_prefix(self):
        today = datetime.date.today()
        return today.strftime('%y%m%d')

def todays_short_date():
    today = Date()
    return today.short_date

class DefaultOrderedDict(OrderedDict):
    """Class to combine defaultdict and OrderedDict.
    Source: http://stackoverflow.com/a/6190500/562769

    """
    def __init__(self, default_factory=None, *a, **kw):
        if (default_factory is not None and
                not isinstance(default_factory, Callable)):
            raise TypeError('first argument must be callable')
        OrderedDict.__init__(self, *a, **kw)
        self.default_factory = default_factory

    def __getitem__(self, key):
        try:
            return OrderedDict.__getitem__(self, key)
        except KeyError:
            return self.__missing__(key)

    def __missing__(self, key):
        if self.default_factory is None:
            raise KeyError(key)
        self[key] = value = self.default_factory()
        return value

    def __reduce__(self):
        if self.default_factory is None:
            args = tuple()
        else:
            args = self.default_factory,
        return type(self), args, None, None, self.items()

    def copy(self):
        return self.__copy__()

    def __copy__(self):
        return type(self)(self.default_factory, self)

    def __deepcopy__(self, memo):
        import copy
        return type(self)(self.default_factory,
                          copy.deepcopy(self.items()))

    def __repr__(self):
        return 'OrderedDefaultDict(%s, %s)' % (self.default_factory,
                                               OrderedDict.__repr__(self))

@contextlib.contextmanager
def suppress_stdout():
    """Suppress the stdout of any function.

    Usage:
    with ssbio.utils.suppress_stdout():
        my_function_here()
    """
    with open(os.devnull, "w") as devnull:
        old_stdout = sys.stdout
        sys.stdout = devnull
        try:
            yield
        finally:
            sys.stdout = old_stdout

def split_folder_and_path(filepath):
    """Split a file path into its folder, filename, and extension

    Args:
        path (str): Path to a file

    Returns:
        tuple: of (folder, filename (without extension), extension)

    """
    dirname = op.dirname(filepath)
    filename = op.basename(filepath)
    splitext = op.splitext(filename)
    filename_without_extension = splitext[0]
    extension = splitext[1]

    return dirname, filename_without_extension, extension

def outfile_maker(inname, outext='.out', outname='', outdir='', append_to_name=''):
    """Create a default name for an output file based on the inname name, unless a output name is specified.

    Args:
        inname: Path to input file
        outext: Optional specified extension for output file (with the "."). Default is ".out".
        outfile: Optional specified name of output file.
        outdir: Optional path to output directory.

    Returns:
        str: Path to final output destination.

    Examples:

        >>> outfile_maker(inname='P00001.fasta')
        'P00001.out'

        >>> outfile_maker(inname='P00001.fasta', append_to_name='_new')
        'P00001_new.out'

        >>> outfile_maker(inname='P00001.fasta', outext='.mao')
        'P00001.mao'

        >>> outfile_maker(inname='P00001.fasta', outext='.mao', append_to_name='_new')
        'P00001_new.mao'

        >>> outfile_maker(inname='P00001.fasta', outext='.new', outname='P00001_aligned')
        'P00001_aligned.new'

        >>> outfile_maker(inname='P00001.fasta', outname='P00001_aligned')
        'P00001_aligned.out'

        >>> outfile_maker(inname='P00001.fasta', outname='P00001_aligned', append_to_name='_new')
        'P00001_aligned_new.out'

        >>> outfile_maker(inname='P00001.fasta', outname='P00001_aligned', outdir='/my/dir/')
        '/my/dir/P00001_aligned.out'

        >>> outfile_maker(inname='/test/other/dir/P00001.fasta', append_to_name='_new')
        '/test/other/dir/P00001_new.out'

        >>> outfile_maker(inname='/test/other/dir/P00001.fasta', outname='P00001_aligned')
        '/test/other/dir/P00001_aligned.out'

        >>> outfile_maker(inname='/test/other/dir/P00001.fasta', outname='P00001_aligned', outdir='/my/dir/')
        '/my/dir/P00001_aligned.out'

    """

    orig_dir, orig_name, orig_ext = split_folder_and_path(inname)

    # If output filename not provided, default is to take name of inname
    if not outname:
        outname = orig_name

    # Create new path in the same directory of old path if a new one isn't specified
    if not outdir:
        outdir = orig_dir

    # Append additional stuff to the filename if specified
    if append_to_name:
        outname = outname + append_to_name

    # Join the output filename and output extension
    final_outfile = op.join(outdir, '{}{}'.format(outname, outext))

    return final_outfile

def force_rerun(flag, outfile):
    """Check if we should force rerunning of a command if an output file exists.

    Args:
        flag (bool): Flag to force rerun.
        outfile (str): Path to output file which may already exist.

    Returns:
        bool: If we should force rerunning of a command

    Examples:
        >>> force_rerun(flag=True, outfile='/not/existing/file.txt')
        True

        >>> force_rerun(flag=False, outfile='/not/existing/file.txt')
        True

        >>> force_rerun(flag=True, outfile='/existing/file.txt')
        True

        >>> force_rerun(flag=False, outfile='/existing/file.txt')
        False

    """
    # If flag is True, always run
    if flag:
        return True
    # If flag is False but file doesn't exist, also run
    elif not flag and not op.exists(outfile):
        return True
    # If flag is False but filesize of output is 0, also run
    elif not flag and not os.stat(outfile).st_size != 0:
        return True
    # Otherwise, do not run
    else:
        return False

def program_exists(prog_name):
    """Check if a program is available as a command line executable on a system.

    Args:
        prog_name: Name of the program.

    Returns:
        bool: True if the program is available.

    """
    if distutils.spawn.find_executable(prog_name):
        return True
    else:
        return False

def command_runner(program, args, force_rerun_flag, outfile, silent=False):
    """Run a program with command-line arguments.

    Args:
        program: Name of the program.
        args: Any program flags as they would be formatted in the command-line (ie. "-i test.in -o test.out").
        force_rerun_flag: If the program should be rerun even if the output file exists.
        outfile: Name out the output file which may have been generated.
            This does not specify what the outfile will be, that should be done in the args.

    Returns:
        bool: If the program ran successfully.

    """
    # Check if pepstats is installed
    if not program_exists(program):
        raise OSError('{}: program not installed'.format(program))

    # Check for force rerunning
    if force_rerun(flag=force_rerun_flag, outfile=outfile):
        cmd = '{} {}'.format(program, args)
        command = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        out, err = command.communicate()
        if not silent:
            print(out.decode('ascii'))
        log.debug('{}: Ran program, output to {}'.format(program, outfile))
    else:
        log.debug('{}: Output already exists'.format(outfile))

def dict_head(d, N=5):
    """Return the head of a dictionary. It will be random!

    Default is to return the first 5 key/value pairs in a dictionary.

    Args:
        d: Dictionary to get head.
        N: Number of elements to display.

    Returns:
        dict: the first N items of the dictionary.

    """
    return {k: d[k] for k in list(d.keys())[:N]}

def rank_dated_files(pattern, dir, descending=True):
    """Search a directory for files that match a pattern. Return an ordered list of these files by filename.

    Args:
        pattern: The glob pattern to search for.
        dir: Path to directory where the files will be searched for.
        descending: Default True, will sort alphabetically by descending order.

    Returns:
        list: Rank-ordered list by filename.

    """
    files = glob.glob(op.join(dir, pattern))
    return sorted(files, reverse=descending)

def find(lst, a, case_sensitive=True):
    """Return indices of a list which have elements that match an object or list of objects

    Args:
        lst: list of values
        a: object(s) to check equality
        case_sensitive: if the search should be case sensitive

    Returns:
        list: list of indicies of lst which equal a

    """
    a = force_list(a)

    if not case_sensitive:
        lst = [x.lower() for x in lst]
        a = [y.lower() for y in a]

    return [i for i, x in enumerate(lst) if x in a]

def not_find(lst, a, case_sensitive=True):
    """Return indices of a list which have elements that DO NOT match an object or list of objects

    Args:
        lst: list of values
        a: object(s) to check inequality
        case_sensitive: if the search should be case sensitive

    Returns:
        list: list of indices of lst which do not equal a

    """
    a = force_list(a)

    if not case_sensitive:
        lst = [x.lower() for x in lst]
        a = [y.lower() for y in a]

    return [i for i, x in enumerate(lst) if x not in a]

def filter_list(lst, takeout, case_sensitive=True):
    """Return a modified list removing items specified.

    Args:
        lst: Original list of values
        takeout: Object or objects to remove from lst
        case_sensitive: if the search should be case sensitive

    Returns:
        list: Filtered list of values

    """
    takeout = force_list(takeout)

    if not case_sensitive:
        lst = [x.lower() for x in lst]
        takeout = [y.lower() for y in takeout]

    return [x for x in lst if x not in takeout]

def filter_list_by_indices(lst, indices):
    """Return a modified list containing only the indices indicated.

    Args:
        lst: Original list of values
        indices: List of indices to keep from the original list

    Returns:
        list: Filtered list of values

    """
    return [x for i, x in enumerate(lst) if i in indices]

def force_string(val=None):
    """Force a string representation of an object

    Args:
        val (str): object to parse into a string

    Returns:
        String

    """
    if val is None:
        return ''
    return val if isinstance(val, str) else ';'.join(val)

def force_list(val=None):
    """Force a list representation of an object

    Args:
        val: object to parse into a list

    Returns:

    """
    if val is None:
        return []
    if isinstance(val, pd.Series):
        return val.tolist()
    return val if isinstance(val, list) else [val]

def force_lower_list(val=None):
    """Force a lowercase list representation of strings

    Args:
        val: string or strings to parse into a list

    Returns:
        list: with lowercase values

    """
    return [x.lower() for x in force_list(val)]

def force_upper_list(val=None):
    """Force a UPPERCASE list representation of strings

    Args:
        val: string or strings to parse into a list

    Returns:
        list: with UPPERCASE values

    """
    return [x.upper() for x in force_list(val)]

def split_list_by_n(l, n):
    """Split a list into lists of size n.

    Args:
        l: List of stuff.
        n: Size of new lists.

    Returns:
        list: List of lists each of size n derived from l.

    """
    n = max(1, n)
    return list(l[i:i+n] for i in range(0, len(l), n))

def split_list_into_n_lists(l, n):
    """Split a list into n lists.

    Args:
        l: List of stuff.
        n: Number of new lists to generate.

    Returns:
        list: List of n lists.

    """
    return [l[i::n] for i in range(n)]

def input_list_parser(infile_list):
    """Always return a list of files with varying input.

    >>> input_list_parser(['/path/to/folder/'])
    ['/path/to/folder/file1.txt', '/path/to/folder/file2.txt', '/path/to/folder/file3.txt']

    >>> input_list_parser(['/path/to/file.txt'])
    ['/path/to/file.txt']

    >>> input_list_parser(['file1.txt'])
    ['file1.txt']

    Args:
        infile_list: List of arguments

    Returns:
        list: Standardized list of files

    """

    final_list_of_files = []

    for x in infile_list:

        # If the input is a folder
        if op.isdir(x):
            os.chdir(x)
            final_list_of_files.extend(glob.glob('*'))

        # If the input is a file
        if op.isfile(x):
            final_list_of_files.append(x)

    return final_list_of_files

def flatlist_dropdup(list_of_lists):
    """Make a single list out of a list of lists, and drop all duplicates.

    Args:
        list_of_lists: List of lists.

    Returns:
        list: List of single objects.

    """
    return list(set([str(item) for sublist in list_of_lists for item in sublist]))

def combinations(iterable, r):
    """Calculate combinations

    >>> list(combinations('ABCD',2))
    [['A', 'B'], ['A', 'C'], ['A', 'D'], ['B', 'C'], ['B', 'D'], ['C', 'D']]

    >>> list(combinations(range(4), 3))
    [[0, 1, 2], [0, 1, 3], [0, 2, 3], [1, 2, 3]]

    Args:
        iterable: Any iterable object.
        r: Size of combination.

    Yields:
        list: Combination of size r.

    """
    pool = tuple(iterable)
    n = len(pool)
    if r > n:
        return

    indices = list(range(r))
    yield list(pool[i] for i in indices)

    while True:
        for i in reversed(range(r)):
            if indices[i] != i + n - r:
                break
        else:
            return
        indices[i] += 1
        for j in range(i + 1, r):
            indices[j] = indices[j - 1] + 1
        yield list(pool[i] for i in indices)

def percentage_to_float(x):
    """Convert a string representation of a percentage to float.

    >>> percentage_to_float('55%')
    0.55

    Args:
        x: String representation of a percentage

    Returns:
        float: Percentage in decimal form

    """
    return float(x.strip('%')) / 100
<|endoftext|>