<commit_before>from __future__ import absolute_import, division, print_function
import re
from datetime import datetime
from datetime import timedelta

import numpy as np
import pandas
from sunpy.extern import six

import astropy.time

__all__ = ['find_time', 'extract_time', 'parse_time', 'is_time',
           'day_of_year', 'break_time', 'get_day', 'is_time_in_given_format']

# Mapping of time format codes to regular expressions.
REGEX = {
    '%Y': '(?P<year>\d{4})',
    '%j': '(?P<dayofyear>\d{3})',
    '%m': '(?P<month>\d{1,2})',
    '%d': '(?P<day>\d{1,2})',
    '%H': '(?P<hour>\d{1,2})',
    '%M': '(?P<minute>\d{1,2})',
    '%S': '(?P<second>\d{1,2})',
    '%f': '(?P<microsecond>\d+)',
    '%b': '(?P<month_str>[a-zA-Z]+)',
}

TIME_FORMAT_LIST = [
    "%Y-%m-%dT%H:%M:%S.%f",    # Example 2007-05-04T21:08:12.999999
    "%Y/%m/%dT%H:%M:%S.%f",    # Example 2007/05/04T21:08:12.999999
    "%Y-%m-%dT%H:%M:%S.%fZ",   # Example 2007-05-04T21:08:12.999Z
    "%Y-%m-%dT%H:%M:%S",       # Example 2007-05-04T21:08:12
    "%Y/%m/%dT%H:%M:%S",       # Example 2007/05/04T21:08:12
    "%Y%m%dT%H%M%S.%f",        # Example 20070504T210812.999999
    "%Y%m%dT%H%M%S",           # Example 20070504T210812
    "%Y/%m/%d %H:%M:%S",       # Example 2007/05/04 21:08:12
    "%Y/%m/%d %H:%M",          # Example 2007/05/04 21:08
    "%Y/%m/%d %H:%M:%S.%f",    # Example 2007/05/04 21:08:12.999999
    "%Y-%m-%d %H:%M:%S.%f",    # Example 2007-05-04 21:08:12.999999
    "%Y-%m-%d %H:%M:%S",       # Example 2007-05-04 21:08:12
    "%Y-%m-%d %H:%M",          # Example 2007-05-04 21:08
    "%Y-%b-%d %H:%M:%S",       # Example 2007-May-04 21:08:12
    "%Y-%b-%d %H:%M",          # Example 2007-May-04 21:08
    "%Y-%b-%d",                # Example 2007-May-04
    "%Y-%m-%d",                # Example 2007-05-04
    "%Y/%m/%d",                # Example 2007/05/04
    "%d-%b-%Y",                # Example 04-May-2007
    "%d-%b-%Y %H:%M:%S.%f",    # Example 04-May-2007 21:08:12.999999
    "%Y%m%d_%H%M%S",           # Example 20070504_210812
    "%Y:%j:%H:%M:%S",          # Example 2012:124:21:08:12
    "%Y:%j:%H:%M:%S.%f",       # Example 2012:124:21:08:12.999999
    "%Y%m%d%H%M%S",            # Example 20140101000001 (JSOC / VSO)
    "%Y.%m.%d_%H:%M:%S_TAI",   # Example 2016.05.04_21:08:12_TAI
]

def _group_or_none(match, group, fun):
    try:
        ret = match.group(group)
    except IndexError:
        return None
    else:
        return fun(ret)

def _n_or_eq(a, b):
    return a is None or a == b

def _regex_parse_time(inp, format):
    # Parser for finding out the minute value so we can adjust the string
    # from 24:00:00 to 00:00:00 the next day because strptime does not
    # understand the former.
    for key, value in six.iteritems(REGEX):
        format = format.replace(key, value)
    match = re.match(format, inp)
    if match is None:
        return None, None
    try:
        hour = match.group("hour")
    except IndexError:
        return inp, timedelta(days=0)
    if match.group("hour") == "24":
        if not all(
                   _n_or_eq(_group_or_none(match, g, int), 00)
                   for g in ["minute", "second", "microsecond"]
                  ):
            raise ValueError
        from_, to = match.span("hour")
        return inp[:from_] + "00" + inp[to:], timedelta(days=1)
    return inp, timedelta(days=0)

def find_time(string, format):
    """ Return iterator of occurrences of date formatted with format
    in string. Currently supported format codes: """
    re_format = format
    for key, value in six.iteritems(REGEX):
        re_format = re_format.replace(key, value)
    matches = re.finditer(re_format, string)
    for match in matches:
        try:
            matchstr = string[slice(*match.span())]
            dt = datetime.strptime(matchstr, format)
        except ValueError:
            continue
        else:
            yield dt

find_time.__doc__ += ', '.join(list(REGEX.keys()))

def _iter_empty(iter):
    try:
        next(iter)
    except StopIteration:
        return True
    return False

def parse_time(time_string, time_format='', **kwargs):
    """Given a time string will parse and return a datetime object.
    Similar to the anytim function in IDL.
    utime -- Time since epoch 1 Jan 1979

    Parameters
    ----------
    time_string : [ int, float, time_string, datetime ]
        Date to parse which can be either time_string, int, datetime object.
    time_format : [ basestring, utime, datetime ]
        Specifies the format user has provided the time_string in.

    Returns
    -------
    out : datetime
        DateTime corresponding to input date string

    Note:
    If time_string is an instance of float, then it is assumed to be in utime format.

    Examples
    --------
    >>> import sunpy.time
    >>> sunpy.time.parse_time('2012/08/01')
    datetime.datetime(2012, 8, 1, 0, 0)
    >>> sunpy.time.parse_time('2005-08-04T00:01:02.000Z')
    datetime.datetime(2005, 8, 4, 0, 1, 2)
    """
    if isinstance(time_string, pandas.tslib.Timestamp):
        return time_string.to_pydatetime()
    elif isinstance(time_string, datetime) or time_format == 'datetime':
        return time_string
    elif isinstance(time_string, tuple):
        return datetime(*time_string)
    elif time_format == 'utime' or isinstance(time_string, (int, float)):
        return datetime(1979, 1, 1) + timedelta(0, time_string)
    elif isinstance(time_string, pandas.tseries.index.DatetimeIndex):
        return time_string._mpl_repr()
    elif isinstance(time_string, np.ndarray) and 'datetime64' in str(time_string.dtype):
        ii = [ss.astype(datetime) for ss in time_string]
        # Validate (in an agnostic way) that we are getting a datetime rather than a date
        return np.array([datetime(*(dt.timetuple()[:6])) for dt in ii])
    elif time_string is 'now':
        return datetime.utcnow()
    elif isinstance(time_string, astropy.time.Time):
        return time_string.datetime
    else:
        # remove trailing zeros and the final dot to allow any
        # number of zeros. This solves issue #289
        if '.' in time_string:
            time_string = time_string.rstrip("0").rstrip(".")
        for time_format in TIME_FORMAT_LIST:
            try:
                try:
                    ts, time_delta = _regex_parse_time(time_string,
                                                       time_format)
                except TypeError:
                    break
                if ts is None:
                    continue
                return datetime.strptime(ts, time_format) + time_delta
            except ValueError:
                pass

        time_string_parse_format = kwargs.pop('_time_string_parse_format', None)
        if time_string_parse_format is not None:
            try:
                ts, time_delta = _regex_parse_time(time_string,
                                                   time_string_parse_format)
                if ts and time_delta:
                    return datetime.strptime(ts, time_string_parse_format) + time_delta
                else:
                    return datetime.strptime(time_string, time_string_parse_format)
            except:
                pass
        raise ValueError("{tstr!s} is not a valid time string!".format(tstr=time_string))

def is_time(time_string, time_format=''):
    """
    Returns true if the input is a valid date/time representation

    Parameters
    ----------
    time_string : [ int, float, time_string, datetime ]
        Date to parse which can be either time_string, int, datetime object.
    time_format : [ basestring, utime, datetime ]
        Specifies the format user has provided the time_string in.

    Returns
    -------
    out : bool
        True if can be parsed by parse_time

    Notes
    -----
        If time_string is an instance of float, then it is assumed to be in
        unix time format.

    Examples
    --------
    >>> import sunpy.time
    >>> sunpy.time.parse_time('2012/08/01')
    datetime.datetime(2012, 8, 1, 0, 0)
    >>> sunpy.time.parse_time('2005-08-04T00:01:02.000Z')
    datetime.datetime(2005, 8, 4, 0, 1, 2)

    .. todo::

        add ability to parse tai (International Atomic Time seconds
        since Jan 1, 1958)

    """
    if time_string is None:
        return False
    elif isinstance(time_string, datetime):
        return True

    try:
        parse_time(time_string, time_format)
    except ValueError:
        return False
    else:
        return True

def day_of_year(time_string):
    """Returns the (fractional) day of year.

    Parameters
    ----------
    time_string : string
        A parse_time compatible string

    Returns
    -------
    out : float
        The fractional day of year (where Jan 1st is 1).

    Examples
    --------
    >>> import sunpy.time
    >>> sunpy.time.day_of_year('2012/01/01')
    1.0
    >>> sunpy.time.day_of_year('2012/08/01')
    214.0
    >>> sunpy.time.day_of_year('2005-08-04T00:18:02.000Z')
    216.01252314814815

    """
    SECONDS_IN_DAY = 60 * 60 * 24.0
    time = parse_time(time_string)
    time_diff = time - datetime(time.year, 1, 1, 0, 0, 0)
    return time_diff.days + time_diff.seconds / SECONDS_IN_DAY + 1

def break_time(t='now', time_format=''):
    """Given a time returns a string. Useful for naming files."""
    # TODO: should be able to handle a time range
    return parse_time(t, time_format).strftime("%Y%m%d_%H%M%S")

def get_day(dt):
    """ Return datetime for the beginning of the day of given datetime. """
    return datetime(dt.year, dt.month, dt.day)

def is_time_in_given_format(time_string, time_format):
    """Tests whether a time string is formatted according to the given time
    format."""
    try:
        datetime.strptime(time_string, time_format)
        return True
    except ValueError:
        return False
<commit_msg>Update time.py<commit_after>from __future__ import absolute_import, division, print_function
import re
from datetime import datetime
from datetime import timedelta

import numpy as np
import pandas
from sunpy.extern import six

import astropy.time

__all__ = ['find_time', 'parse_time', 'is_time',
           'day_of_year', 'break_time', 'get_day', 'is_time_in_given_format']

# Mapping of time format codes to regular expressions.
REGEX = {
    '%Y': '(?P<year>\d{4})',
    '%j': '(?P<dayofyear>\d{3})',
    '%m': '(?P<month>\d{1,2})',
    '%d': '(?P<day>\d{1,2})',
    '%H': '(?P<hour>\d{1,2})',
    '%M': '(?P<minute>\d{1,2})',
    '%S': '(?P<second>\d{1,2})',
    '%f': '(?P<microsecond>\d+)',
    '%b': '(?P<month_str>[a-zA-Z]+)',
}

TIME_FORMAT_LIST = [
    "%Y-%m-%dT%H:%M:%S.%f",    # Example 2007-05-04T21:08:12.999999
    "%Y/%m/%dT%H:%M:%S.%f",    # Example 2007/05/04T21:08:12.999999
    "%Y-%m-%dT%H:%M:%S.%fZ",   # Example 2007-05-04T21:08:12.999Z
    "%Y-%m-%dT%H:%M:%S",       # Example 2007-05-04T21:08:12
    "%Y/%m/%dT%H:%M:%S",       # Example 2007/05/04T21:08:12
    "%Y%m%dT%H%M%S.%f",        # Example 20070504T210812.999999
    "%Y%m%dT%H%M%S",           # Example 20070504T210812
    "%Y/%m/%d %H:%M:%S",       # Example 2007/05/04 21:08:12
    "%Y/%m/%d %H:%M",          # Example 2007/05/04 21:08
    "%Y/%m/%d %H:%M:%S.%f",    # Example 2007/05/04 21:08:12.999999
    "%Y-%m-%d %H:%M:%S.%f",    # Example 2007-05-04 21:08:12.999999
    "%Y-%m-%d %H:%M:%S",       # Example 2007-05-04 21:08:12
    "%Y-%m-%d %H:%M",          # Example 2007-05-04 21:08
    "%Y-%b-%d %H:%M:%S",       # Example 2007-May-04 21:08:12
    "%Y-%b-%d %H:%M",          # Example 2007-May-04 21:08
    "%Y-%b-%d",                # Example 2007-May-04
    "%Y-%m-%d",                # Example 2007-05-04
    "%Y/%m/%d",                # Example 2007/05/04
    "%d-%b-%Y",                # Example 04-May-2007
    "%d-%b-%Y %H:%M:%S.%f",    # Example 04-May-2007 21:08:12.999999
    "%Y%m%d_%H%M%S",           # Example 20070504_210812
    "%Y:%j:%H:%M:%S",          # Example 2012:124:21:08:12
    "%Y:%j:%H:%M:%S.%f",       # Example 2012:124:21:08:12.999999
    "%Y%m%d%H%M%S",            # Example 20140101000001 (JSOC / VSO)
    "%Y.%m.%d_%H:%M:%S_TAI",   # Example 2016.05.04_21:08:12_TAI
]

def _group_or_none(match, group, fun):
    try:
        ret = match.group(group)
    except IndexError:
        return None
    else:
        return fun(ret)

def _n_or_eq(a, b):
    return a is None or a == b

def _regex_parse_time(inp, format):
    # Parser for finding out the minute value so we can adjust the string
    # from 24:00:00 to 00:00:00 the next day because strptime does not
    # understand the former.
    for key, value in six.iteritems(REGEX):
        format = format.replace(key, value)
    match = re.match(format, inp)
    if match is None:
        return None, None
    try:
        hour = match.group("hour")
    except IndexError:
        return inp, timedelta(days=0)
    if match.group("hour") == "24":
        if not all(
                   _n_or_eq(_group_or_none(match, g, int), 00)
                   for g in ["minute", "second", "microsecond"]
                  ):
            raise ValueError
        from_, to = match.span("hour")
        return inp[:from_] + "00" + inp[to:], timedelta(days=1)
    return inp, timedelta(days=0)

def find_time(string, format):
    """ Return iterator of occurrences of date formatted with format
    in string. Currently supported format codes: """
    re_format = format
    for key, value in six.iteritems(REGEX):
        re_format = re_format.replace(key, value)
    matches = re.finditer(re_format, string)
    for match in matches:
        try:
            matchstr = string[slice(*match.span())]
            dt = datetime.strptime(matchstr, format)
        except ValueError:
            continue
        else:
            yield dt

find_time.__doc__ += ', '.join(list(REGEX.keys()))

def _iter_empty(iter):
    try:
        next(iter)
    except StopIteration:
        return True
    return False

def parse_time(time_string, time_format='', **kwargs):
    """Given a time string will parse and return a datetime object.
    Similar to the anytim function in IDL.
    utime -- Time since epoch 1 Jan 1979

    Parameters
    ----------
    time_string : [ int, float, time_string, datetime ]
        Date to parse which can be either time_string, int, datetime object.
    time_format : [ basestring, utime, datetime ]
        Specifies the format user has provided the time_string in.

    Returns
    -------
    out : datetime
        DateTime corresponding to input date string

    Note:
    If time_string is an instance of float, then it is assumed to be in utime format.

    Examples
    --------
    >>> import sunpy.time
    >>> sunpy.time.parse_time('2012/08/01')
    datetime.datetime(2012, 8, 1, 0, 0)
    >>> sunpy.time.parse_time('2005-08-04T00:01:02.000Z')
    datetime.datetime(2005, 8, 4, 0, 1, 2)
    """
    if isinstance(time_string, pandas.tslib.Timestamp):
        return time_string.to_pydatetime()
    elif isinstance(time_string, datetime) or time_format == 'datetime':
        return time_string
    elif isinstance(time_string, tuple):
        return datetime(*time_string)
    elif time_format == 'utime' or isinstance(time_string, (int, float)):
        return datetime(1979, 1, 1) + timedelta(0, time_string)
    elif isinstance(time_string, pandas.tseries.index.DatetimeIndex):
        return time_string._mpl_repr()
    elif isinstance(time_string, np.ndarray) and 'datetime64' in str(time_string.dtype):
        ii = [ss.astype(datetime) for ss in time_string]
        # Validate (in an agnostic way) that we are getting a datetime rather than a date
        return np.array([datetime(*(dt.timetuple()[:6])) for dt in ii])
    elif time_string is 'now':
        return datetime.utcnow()
    elif isinstance(time_string, astropy.time.Time):
        return time_string.datetime
    else:
        # remove trailing zeros and the final dot to allow any
        # number of zeros. This solves issue #289
        if '.' in time_string:
            time_string = time_string.rstrip("0").rstrip(".")
        for time_format in TIME_FORMAT_LIST:
            try:
                try:
                    ts, time_delta = _regex_parse_time(time_string,
                                                       time_format)
                except TypeError:
                    break
                if ts is None:
                    continue
                return datetime.strptime(ts, time_format) + time_delta
            except ValueError:
                pass

        time_string_parse_format = kwargs.pop('_time_string_parse_format', None)
        if time_string_parse_format is not None:
            try:
                ts, time_delta = _regex_parse_time(time_string,
                                                   time_string_parse_format)
                if ts and time_delta:
                    return datetime.strptime(ts, time_string_parse_format) + time_delta
                else:
                    return datetime.strptime(time_string, time_string_parse_format)
            except:
                pass
        raise ValueError("{tstr!s} is not a valid time string!".format(tstr=time_string))

def is_time(time_string, time_format=''):
    """
    Returns true if the input is a valid date/time representation

    Parameters
    ----------
    time_string : [ int, float, time_string, datetime ]
        Date to parse which can be either time_string, int, datetime object.
    time_format : [ basestring, utime, datetime ]
        Specifies the format user has provided the time_string in.

    Returns
    -------
    out : bool
        True if can be parsed by parse_time

    Notes
    -----
        If time_string is an instance of float, then it is assumed to be in
        unix time format.

    Examples
    --------
    >>> import sunpy.time
    >>> sunpy.time.parse_time('2012/08/01')
    datetime.datetime(2012, 8, 1, 0, 0)
    >>> sunpy.time.parse_time('2005-08-04T00:01:02.000Z')
    datetime.datetime(2005, 8, 4, 0, 1, 2)

    .. todo::

        add ability to parse tai (International Atomic Time seconds
        since Jan 1, 1958)

    """
    if time_string is None:
        return False
    elif isinstance(time_string, datetime):
        return True

    try:
        parse_time(time_string, time_format)
    except ValueError:
        return False
    else:
        return True

def day_of_year(time_string):
    """Returns the (fractional) day of year.

    Parameters
    ----------
    time_string : string
        A parse_time compatible string

    Returns
    -------
    out : float
        The fractional day of year (where Jan 1st is 1).

    Examples
    --------
    >>> import sunpy.time
    >>> sunpy.time.day_of_year('2012/01/01')
    1.0
    >>> sunpy.time.day_of_year('2012/08/01')
    214.0
    >>> sunpy.time.day_of_year('2005-08-04T00:18:02.000Z')
    216.01252314814815

    """
    SECONDS_IN_DAY = 60 * 60 * 24.0
    time = parse_time(time_string)
    time_diff = time - datetime(time.year, 1, 1, 0, 0, 0)
    return time_diff.days + time_diff.seconds / SECONDS_IN_DAY + 1

def break_time(t='now', time_format=''):
    """Given a time returns a string. Useful for naming files."""
    # TODO: should be able to handle a time range
    return parse_time(t, time_format).strftime("%Y%m%d_%H%M%S")

def get_day(dt):
    """ Return datetime for the beginning of the day of given datetime. """
    return datetime(dt.year, dt.month, dt.day)

def is_time_in_given_format(time_string, time_format):
    """Tests whether a time string is formatted according to the given time
    format."""
    try:
        datetime.strptime(time_string, time_format)
        return True
    except ValueError:
        return False
<|endoftext|>


<commit_before>"""
At a job interview, you are challenged to write an algorithm to check if a given stringing, s, can be formed from two other stringings, part1 and part2.
The restringiction is that the characters in part1 and part2 are in the same order as in s.
The interviewer gives you the following example and tells you to figure out the rest from the given test cases.
'codewars' is a merge from 'cdw' and 'oears':
s:  c o d e w a r s   = codewars
part1:  c   d   w         = cdw
part2:    o   e   a r s   = oears
"""

# Recursive Solution
def is_merge(s, part1, part2):
    if not part1:
      return s == part2
    if not part2:
      return s == part1
    if not s:
      return part1 + part2 == ''
    if s[0] == part1[0] and is_merge(s[1:], part1[1:], part2):
      return True
    if s[0] == part2[0] and is_merge(s[1:], part1, part2[1:]):
      return True
    return False

# An iterative approach
def is_merge(s, part1, part2):
    tuple_list = [(s, part1, part2)]
    while tuple_list:
        string, p1, p2 = tuple_list.pop()
        if string:
            if p1 and string[0] == p1[0]:
                tuple_list.append((string[1:], p1[1:], p2))
            if p2 and string[0] == p2[0]:
                tuple_list.append((string[1:], p1, p2[1:]))
        else:
            if not p1 and not p2:
                return True
    return False
<commit_msg>Minor fixes.<commit_after>"""
At a job interview, you are challenged to write an algorithm to check if a given string, s, can be formed from two other strings, part1 and part2.
The restriction is that the characters in part1 and part2 are in the same order as in s.
The interviewer gives you the following example and tells you to figure out the rest from the given test cases.
'codewars' is a merge from 'cdw' and 'oears':
s:  c o d e w a r s   = codewars
part1:  c   d   w         = cdw
part2:    o   e   a r s   = oears
"""

# Recursive Solution
def is_merge(s, part1, part2):
    if not part1:
      return s == part2
    if not part2:
      return s == part1
    if not s:
      return part1 + part2 == ''
    if s[0] == part1[0] and is_merge(s[1:], part1[1:], part2):
      return True
    if s[0] == part2[0] and is_merge(s[1:], part1, part2[1:]):
      return True
    return False

# An iterative approach
def is_merge(s, part1, part2):
    tuple_list = [(s, part1, part2)]
    while tuple_list:
        string, p1, p2 = tuple_list.pop()
        if string:
            if p1 and string[0] == p1[0]:
                tuple_list.append((string[1:], p1[1:], p2))
            if p2 and string[0] == p2[0]:
                tuple_list.append((string[1:], p1, p2[1:]))
        else:
            if not p1 and not p2:
                return True
    return False
<|endoftext|>


<commit_before>#!/usr/bin/python2
# Network monitor using iwlist
import subprocess, sys, time, re, json, os, Queue
import manufacturer.parse_manufacturer as manufacturer
import modules.detectors.evil_twin_detector as detector1

captured_aps = []
manufacturer_table = "manufacturer/manufacturer_table.txt"
table_of_manufacturers = {}

class colors:
        HEADER = '\033[95m'
        OKBLUE = '\033[94m'
        OKGREEN = '\033[92m'
        WHITE = '\033[37m'
        WARNING = '\033[93m'
        FAIL = '\033[91m'
        FAIL2 = '\033[41m'
        ENDC = '\033[0m'
        BOLD = '\033[1m'
        GRAY = '\033[90m'
        UNDERLINE = '\033[4m'

def scan(interface, output, queue):

	global table_of_manufacturers
	table_of_manufacturers = manufacturer.MacParser(manufacturer_table).refresh()

	table = ['AP Name','BSSID', 'CH', 'Brand','Signal','Quality','Encryption','Cipher', 'Pairwise','Authentication']
	print colors.WARNING + '{:^22s}|{:^19s}|{:^9s}|{:^24s}|{:^8s}|{:^9s}|{:^16s}|{:^8s}|{:^11s}|{:^16s}'.format(table[0],table[1],table[2],table[3],table[4],table[5],table[6],table[7],table[8],table[9]) + colors.ENDC
	while True:
		ap_list = get_results(interface)
		try:
			with open(output,'a') as output_file:
				for line in ap_list:
					# filter to check if APs already exists
					if filter_aps(line):
						limited = False
						if len(line['essid'])>21:
							limited = True

						if limited:
							if detector1.rogueAP_detector(line,captured_aps):
								print colors.FAIL2 + '{:<21s}  {:^19s} {:^9s} {:^24s} {:^8s} {:^9s} {:^16s} {:^8s}  {:^10s} {:^16s}'.format(line['essid'][0:21],line['mac'],line['channel'], line['signal'], line['comment'],line['signal'],line['quality'],line['key type'],line['group cipher'],line['pairwise cipher'], line['authentication suites']) + colors.ENDC
							else:
								print '{:<21s}  {:^19s} {:^9s} {:^24s} {:^8s} {:^9s} {:^16s} {:^8s}  {:^10s} {:^16s}'.format(line['essid'][0:21],line['mac'],line['channel'], line['signal'], line['comment'],line['signal'],line['quality'],line['key type'],line['group cipher'],line['pairwise cipher'], line['authentication suites'])
						else:
							if detector1.rogueAP_detector(line,captured_aps):
								print colors.FAIL2 + '{:<21s}  {:^19s} {:^9s} {:^24s} {:^8s} {:^9s} {:^16s} {:^8s}  {:^10s} {:^16s}'.format(line['essid'],line['mac'],line['channel'], line['comment'],line['signal'],line['quality'],line['key type'],line['group cipher'],line['pairwise cipher'], line['authentication suites']) + colors.ENDC
							else:
								print '{:<21s}  {:^19s} {:^9s} {:^24s} {:^8s} {:^9s} {:^16s} {:^8s}  {:^10s} {:^16s}'.format(line['essid'],line['mac'],line['channel'], line['comment'],line['signal'],line['quality'],line['key type'],line['group cipher'],line['pairwise cipher'], line['authentication suites'])

						captured_aps.append(line)
						queue.put(line)
						json.dump(line,output_file)
						output_file.write("\n")
			time.sleep(1)
		except:
			pass

def get_results(interface):
    list_of_results=[]
    try:
		#Call the process to get the output to parse
        proc = subprocess.check_output("iwlist "+interface+" scan",shell=True)
		#Break the output making an array containing the info of each Access Point
        list_of_results = re.split(r'\bCell \d{2}\b - ',proc)[1:]
    except subprocess.CalledProcessError:
 		print 'Error...'

    return parse(list_of_results)

def filter_aps(access_point):
    for ap in captured_aps:
		try:
			if ap['mac'] == access_point['mac'] and ap['essid']==access_point['essid'] and ap['channel'] == access_point['channel']:
				return False
		except Exception as e:
			print "Exception found: \n"+e
			pass
    return True

def parse(list):
    parsed_list = []

    for network in list:
		try:
			ap={}
			network = network.strip()
			essid=""
			address=""
			quality=""
			signal=""
			channel=""
			encryption_key=""
			key_type=""
			group_cipher=""
			pairwise_cipher=""
			authentication_suites=""

			#Get the name of the AP
			match = re.search('ESSID:"(([ ]*(\S+)*)*)"',network)
			if match:
			    essid = match.group(1)
			    ap.update({"essid":essid})

			#Get the BSSID of the AP
			match = re.search('Address: (\S+)',network)
			if match:
			    address = match.group(1)
			    ap.update({"mac":address})

			#Get the Channel of the AP
			match = re.search('Channel:(\S+)',network)
			if match:
			    channel = match.group(1)
			    ap.update({"channel":channel})

			#Find the brand of the AP
			global table_of_manufacturers
			manufacturer_data = manufacturer.search(table_of_manufacturers,str(address))
			if(len(manufacturer_data)>0):
				ap.update({"manufacturer":manufacturer_data[0].manuf})
				ap.update({"comment":manufacturer_data[0].comment})
			else:
				ap.update({"manufacturer":"Null"})
				ap.update({"comment":"Null"})

			#Get the quality of the signal and the signal level
			match = re.search('Quality=(\d+/\d+)  Signal level=(-\d+) dBm',network)
			if match:
			    quality = match.group(1)
			    #quality = str(int(round(float(quality[0]) / float(quality[1]) * 100))).rjust(3) + " %"
			    signal = match.group(2)
			    ap.update({"quality":quality})
			    ap.update({"signal":signal})

			#Check if there is an Encryption key on the AP
			match = re.search('Encryption key:(\S+)',network)
			if match:
			    encryption_key = match.group(1)
			    ap.update({"encryption":encryption_key})

			#Find the encryption type (WEP, WPA, WPA2 or Open)
			match = re.search(r'(?<=802.11i/)[a-zA-Z0-9_ ]*',network)
			if match and match != "Unknown" and match != "IEEE 802":
			    key_type=match.group(0)
			    ap.update({"key type":key_type})
			elif ap['encryption']=='on':
			    key_type="WEP"
			    ap.update({"key type":key_type})
			else:
			    key_type="Open"
			    ap.update({"key type":key_type})

			#Get the Cipher being used
			match = re.search(r'Group Cipher : ([a-zA-Z0-9_ ]*)',network)
			if match:
			    group_cipher=match.group(1)
			    ap.update({"group cipher":group_cipher})
			elif ap['encryption']=='on':
			    group_cipher="WEP"
			    ap.update({"group cipher":group_cipher})
			else:
			    group_cipher=""
			    ap.update({"group cipher":group_cipher})

			#Get the Pairwise Cipher being used
			match = re.search('Pairwise Ciphers ([(\d+)]*) : ([a-zA-Z0-9_ ]*)',network)
			if match:
			    pairwise_cipher=match.group(2)
			    ap.update({"pairwise cipher":pairwise_cipher})
			elif ap['encryption']=='on':
			    pairwise_cipher="WEP"
			    ap.update({"pairwise cipher":pairwise_cipher})
			else:
			    pairwise_cipher=""
			    ap.update({"pairwise cipher":pairwise_cipher})

			#Get the Authentication Suites
			match = re.search('Authentication Suites ([(\d+)]*) : ([a-zA-Z0-9_ ]*)',network)
			if match:
				authentication_suites=match.group(2)
				ap.update({"authentication suites":authentication_suites})
			elif ap['encryption']=='on':
			    authentication_suites=""
			    ap.update({"authentication suites":authentication_suites})
			else:
			    authentication_suites=""
			    ap.update({"authentication suites":authentication_suites})

			parsed_list.append(ap)
		except:
			pass

    return parsed_list
<commit_msg>auxiliary file removed<commit_after>#!/usr/bin/python2
# Network monitor using iwlist
import subprocess, sys, time, re, json, os, Queue
import manufacturer.parse_manufacturer as manufacturer
import modules.detectors.evil_twin_detector as detector1

captured_aps = []
manufacturer_table = "manufacturer/manufacturer_table.txt"
table_of_manufacturers = {}

class colors:
        HEADER = '\033[95m'
        OKBLUE = '\033[94m'
        OKGREEN = '\033[92m'
        WHITE = '\033[37m'
        WARNING = '\033[93m'
        FAIL = '\033[91m'
        FAIL2 = '\033[41m'
        ENDC = '\033[0m'
        BOLD = '\033[1m'
        GRAY = '\033[90m'
        UNDERLINE = '\033[4m'

def scan(interface, queue):

	global table_of_manufacturers
	table_of_manufacturers = manufacturer.MacParser(manufacturer_table).refresh()

	table = ['AP Name','BSSID', 'CH', 'Brand','Signal','Quality','Encryption','Cipher', 'Pairwise','Authentication']
	print colors.WARNING + '{:^22s}|{:^19s}|{:^9s}|{:^24s}|{:^8s}|{:^9s}|{:^16s}|{:^8s}|{:^11s}|{:^16s}'.format(table[0],table[1],table[2],table[3],table[4],table[5],table[6],table[7],table[8],table[9]) + colors.ENDC
	while True:
		ap_list = get_results(interface)
		#print ap_list
		try:
			for line in ap_list:
				#print line
				# filter to check if APs already exists
				if filter_aps(line):
					limited = False
					if len(line['essid'])>21:
						limited = True

					if limited:
						if detector1.rogueAP_detector(line,captured_aps):
							print colors.FAIL2 + '{:<21s}  {:^19s} {:^9s} {:^24s} {:^8s} {:^9s} {:^16s} {:^8s}  {:^10s} {:^16s}'.format(line['essid'][0:21],line['mac'],line['channel'], line['signal'], line['comment'],line['signal'],line['quality'],line['key type'],line['group cipher'],line['pairwise cipher'], line['authentication suites']) + colors.ENDC
						else:
							print '{:<21s}  {:^19s} {:^9s} {:^24s} {:^8s} {:^9s} {:^16s} {:^8s}  {:^10s} {:^16s}'.format(line['essid'][0:21],line['mac'],line['channel'], line['signal'], line['comment'],line['signal'],line['quality'],line['key type'],line['group cipher'],line['pairwise cipher'], line['authentication suites'])
					else:
						if detector1.rogueAP_detector(line,captured_aps):
							print colors.FAIL2 + '{:<21s}  {:^19s} {:^9s} {:^24s} {:^8s} {:^9s} {:^16s} {:^8s}  {:^10s} {:^16s}'.format(line['essid'],line['mac'],line['channel'], line['comment'],line['signal'],line['quality'],line['key type'],line['group cipher'],line['pairwise cipher'], line['authentication suites']) + colors.ENDC
						else:
							print '{:<21s}  {:^19s} {:^9s} {:^24s} {:^8s} {:^9s} {:^16s} {:^8s}  {:^10s} {:^16s}'.format(line['essid'],line['mac'],line['channel'], line['comment'],line['signal'],line['quality'],line['key type'],line['group cipher'],line['pairwise cipher'], line['authentication suites'])

					captured_aps.append(line)
					#print captured_aps
					queue.put(line)

			time.sleep(1)
		except:
			pass

def get_results(interface):
    list_of_results=[]
    try:
		#Call the process to get the output to parse
        proc = subprocess.check_output("iwlist "+interface+" scan",shell=True)
		#Break the output making an array containing the info of each Access Point
        list_of_results = re.split(r'\bCell \d{2}\b - ',proc)[1:]
    except subprocess.CalledProcessError:
 		print 'Error...'

    return parse(list_of_results)

def filter_aps(access_point):
    for ap in captured_aps:
		try:
			if ap['mac'] == access_point['mac'] and ap['essid']==access_point['essid'] and ap['channel'] == access_point['channel']:
				return False
		except Exception as e:
			print "Exception found: \n"+e
			pass
    return True

def parse(list):
    parsed_list = []

    for network in list:
		try:
			ap={}
			network = network.strip()
			essid=""
			address=""
			quality=""
			signal=""
			channel=""
			encryption_key=""
			key_type=""
			group_cipher=""
			pairwise_cipher=""
			authentication_suites=""

			#Get the name of the AP
			match = re.search('ESSID:"(([ ]*(\S+)*)*)"',network)
			if match:
			    essid = match.group(1)
			    ap.update({"essid":essid})

			#Get the BSSID of the AP
			match = re.search('Address: (\S+)',network)
			if match:
			    address = match.group(1)
			    ap.update({"mac":address})

			#Get the Channel of the AP
			match = re.search('Channel:(\S+)',network)
			if match:
			    channel = match.group(1)
			    ap.update({"channel":channel})

			#Find the brand of the AP
			global table_of_manufacturers
			manufacturer_data = manufacturer.search(table_of_manufacturers,str(address))
			if(len(manufacturer_data)>0):
				ap.update({"manufacturer":manufacturer_data[0].manuf})
				ap.update({"comment":manufacturer_data[0].comment})
			else:
				ap.update({"manufacturer":"Null"})
				ap.update({"comment":"Null"})

			#Get the quality of the signal and the signal level
			match = re.search('Quality=(\d+/\d+)  Signal level=(-\d+) dBm',network)
			if match:
			    quality = match.group(1)
			    #quality = str(int(round(float(quality[0]) / float(quality[1]) * 100))).rjust(3) + " %"
			    signal = match.group(2)
			    ap.update({"quality":quality})
			    ap.update({"signal":signal})

			#Check if there is an Encryption key on the AP
			match = re.search('Encryption key:(\S+)',network)
			if match:
			    encryption_key = match.group(1)
			    ap.update({"encryption":encryption_key})

			#Find the encryption type (WEP, WPA, WPA2 or Open)
			match = re.search(r'(?<=802.11i/)[a-zA-Z0-9_ ]*',network)
			if match and match != "Unknown" and match != "IEEE 802":
			    key_type=match.group(0)
			    ap.update({"key type":key_type})
			elif ap['encryption']=='on':
			    key_type="WEP"
			    ap.update({"key type":key_type})
			else:
			    key_type="Open"
			    ap.update({"key type":key_type})

			#Get the Cipher being used
			match = re.search(r'Group Cipher : ([a-zA-Z0-9_ ]*)',network)
			if match:
			    group_cipher=match.group(1)
			    ap.update({"group cipher":group_cipher})
			elif ap['encryption']=='on':
			    group_cipher="WEP"
			    ap.update({"group cipher":group_cipher})
			else:
			    group_cipher=""
			    ap.update({"group cipher":group_cipher})

			#Get the Pairwise Cipher being used
			match = re.search('Pairwise Ciphers ([(\d+)]*) : ([a-zA-Z0-9_ ]*)',network)
			if match:
			    pairwise_cipher=match.group(2)
			    ap.update({"pairwise cipher":pairwise_cipher})
			elif ap['encryption']=='on':
			    pairwise_cipher="WEP"
			    ap.update({"pairwise cipher":pairwise_cipher})
			else:
			    pairwise_cipher=""
			    ap.update({"pairwise cipher":pairwise_cipher})

			#Get the Authentication Suites
			match = re.search('Authentication Suites ([(\d+)]*) : ([a-zA-Z0-9_ ]*)',network)
			if match:
				authentication_suites=match.group(2)
				ap.update({"authentication suites":authentication_suites})
			elif ap['encryption']=='on':
			    authentication_suites=""
			    ap.update({"authentication suites":authentication_suites})
			else:
			    authentication_suites=""
			    ap.update({"authentication suites":authentication_suites})

			parsed_list.append(ap)
		except:
			pass

    return parsed_list
<|endoftext|>


<commit_before>"""
A python wrapper around EFMTool, a java-based library for calculating
elementary flux modes. See http://www.csb.ethz.ch/tools/software/efmtool.html;
Terzer M, Stelling J (2008) Large-scale computation of elementary flux modes
with bit pattern trees. Bioinformatics 24: 2229-2235.
"""

import os
from cobra.util import create_stoichiometric_matrix

import numpy as np

try:
    import pandas as pd
    pandas = True
except ImportError:
    pandas = False

from pyefm.utils import make_temp_directory, run_process

class EFMToolWrapper(object):

    def __init__(self, cobra_model, opts=None, verbose=True, java_args=None):

        if opts is None:
            opts = {}

        self.opts = opts
        self.model = cobra_model
        self.verbose = verbose

        self.efm_lib_dir = (os.path.dirname(os.path.abspath(__file__)) +
                            '/efmtool')
        self.efm_command = ['java', '-jar', self.efm_lib_dir +
                            '/metabolic-efm-all.jar']

        # Insert a string with extra options to the java virtual machine.
        if java_args:
            self.efm_command.insert(1, java_args)

    def create_model_files(self, temp_dir):
        """ Write stochiometry data, reaction reversibilities, metabolite, and
        reaction names to temporary files in preparation for calling efmtool

        """

        stoich_mat = create_stoichiometric_matrix(self.model)

        # Stoichiometric Matrix
        np.savetxt(temp_dir + '/stoich.txt', stoich_mat, delimiter='\t')

        # Reaction reversibilities
        np.savetxt(temp_dir + '/revs.txt',
                   np.array([r.reversibility for r in self.model.reactions]),
                   delimiter='\t', fmt='%d', newline='\t')

        # Reaction Names
        with open(temp_dir + '/rnames.txt', 'w') as f:
            f.write('\t'.join(('"{}"'.format(r.id)
                               for r in self.model.reactions)))

        # Metabolite Names
        with open(temp_dir + '/mnames.txt', 'w') as f:
            f.write('\t'.join(('"{}"'.format(m.id)
                               for m in self.model.metabolites)))

    def read_double_out(self, out_file):
        """ Read the output file generated from EMFTool. Returns a numpy array or
        pandas dataframe (if pandas can be loaded)

        """
        with open(out_file, 'rb') as f:
            out_arr = np.fromstring(f.read()[13:], dtype='>d').reshape(
                (-1, len(self.model.reactions))).T
            out_arr = np.array(out_arr, dtype=np.float64)

        if pandas:
            out_arr = pd.DataFrame(
                out_arr, index=(r.id for r in self.model.reactions),
                columns=('EM{}'.format(i) for i in
                         range(1, out_arr.shape[1] + 1)))

        return out_arr.T

    def __call__(self):

        dir_ = self.opts.pop('dir_', None)

        with make_temp_directory('efmtool', dir_) as temp_dir:

            self.create_model_files(temp_dir)

            try:
                out_file = self.opts.pop('out_file')
            except KeyError:
                out_file = temp_dir + '/out.bin'

            # Default options for the EMFtool. I don't recommend changing any
            # of these, even though the function is set up for this. This tool
            # hasn't been tested for any options other than these
            default_opts = {
                'kind': 'stoichiometry',
                'stoich': temp_dir + '/stoich.txt',
                'rev': temp_dir + '/revs.txt',
                'reac': temp_dir + '/rnames.txt',
                'meta': temp_dir + '/mnames.txt',
                'arithmetic': 'double',
                'zero': 1E-10,
                'out': 'binary-doubles',  # TODO support binary output?
                'compression': 'default',
                'log': 'console',
                'level': 'INFO',
                'maxthreads': -1,
                'normalize': 'max',
                'adjacency-method': 'pattern-tree-minzero',
                'rowordering': 'MostZerosOrAbsLexMin',
                'tmpdir': temp_dir
            }

            default_opts.update(self.opts)

            # Create a list of arguments to pass to the python subprocess
            # module
            def opt_gen():
                for opt, val in default_opts.items():
                    yield '-' + opt
                    yield str(val)

                    # This kw takes two arguments
                    if opt == 'out':
                        yield out_file

            # Run the EFMtool, outputting STDOUT to python.
            run_process(self.efm_command + list(opt_gen()), verbose=self.verbose)

            if 'binary-doubles' in default_opts['out']:
                return self.read_double_out(out_file)

            else:
                raise RuntimeError('Other output options not supported')

def calculate_elementary_modes(cobra_model, opts=None, verbose=True,
                               java_args=None):
    """ A python wrapper around EFMTool, a java-based library for calculating
    elementary flux modes.

    Parameters:
    ===========

    cobra_model: cobra.Model or similar
        The model for which to calculate elementary flux modes. Reaction
        reversibilities are inferred from their corresponding bounds.

    opts: dict or None
        Additional dictionary options to pass to the emftool solver. See Option
        Descriptions below for additional information on arguments.

    verbose: bool
        Whether or not to redirect stdout to the python shell.

    java_args: string
        Extra command-line options to pass to the java virtual machine.
        Eg. '-Xmx1g' will set the heap space to 1 GB.

    Option Descriptions from the MATLAB Documentation:
    ==================================================

    The following options are supported (any others are silently ignored):

   'arithmetic'     Which arithmetic to use, one of
                      'double'     Double precision floating point (default)
                      'fractional' Fraction numbers, fixed or infinite
                                   precision
   'precision'      Bits to use for fraction numerator/denominator,
                    -1 for infinite precision, or 64, 128, ... bits for
                    fixed precision (only for arithmetic=fractional)
   'zero'           Values to be treated as zero
                    defaults are 1e-10 for double arithmetic and 0 for
                    fractional arithmetic
   'compression'    Which compression techniques should be applied to the
                    network before efm computation, one of
                      'default'                    Default compression
                      'default-no-combine'         No reaction merging
                      'default-no-couple-combine'  Reduced react. merging
                      'default-no-duplicate'       No duplicate gene com.
                      'default-all-duplicate'      Extended duplicate
                                                   gene compression
                      'off'                        No compression at all
                      'unique'                     only unique flows
                                                   compression
                      'nullspace'                  Only nullspace based
                                                   compression
                      'unique-no-recursion'        Only one round of
                                                   unique flows compr.
                      'nullspace-no-recursion'     Only one round of
                                                   nullspace compression

   'level'          The log level, default is INFO, supported:
                      'WARNING', 'INFO', 'FINE', 'FINER', 'FINEST'

   'maxthreads'     Maximum threads to use, default is 1 for single core
                    systems and k for multi core systems with k cores.

   'normalize'      Normalization method of the output efms, one of
                      'min'     default, minimum absolute value is 1
                      'max'     maximum absolute value is 1
                      'norm2'   norm 2, that is, vector length is 1
                      'squared' like norm 2, but all values are squared,
                                possibly negative. To get the original
                                value, take the square root of the
                                absolute value and keep the sign, i.e.
                                val = sign(valsq) * sqrt(abs(valsq)).
                                This method allows error free values
                                if fraction number arithmetic is used.
                      'none'    no normalization

   'count-only'     If true, the (binary) modes are computed and the
                      number of efms is returned, but not the efm vectors
   'sign-only'      If true, the (binary) modes are computed, but only
                      sign values of reaction fluxes are returned as EFMs.
                      The values +/-1 stand for forward/reverse flux, and 0
                      for no flux, respectively. Such sign-valued EFMs can
                      be converted back into double valued EFMs by using
                      the SignToDouble function.
   'parse-only'     If true, input data is processed and parsed, but no
                      elementary modes are computed. The returned
                      structure contains no efms field
   'convert-only'   If true, input data is processed and converted into
                      the program's internal format. The efm computation
                      itself is not started, but call options (for manual
                      invocation of the Java program) are reported as
                      return string. The converted files are stored in
                      the tmp directory.
                      Depending on the input format, these files are:
                          - stoich.txt    the stoichiometric matrix
                          - revs.txt      the reaction reversibilities
                          - mnames.txt    the metabolite names
                          - rnames.txt    the reaction names
                          - rlist.txt     the reaction list file
                          - xbml.xml      the sbml input file

   'suppress'       List of reaction names to suppress, separated by
                      whitespace. All resulting flux modes will have a
                      zero flux value for suppressed reactions

   'enforce'       List of reaction names to enforce, separated by
                      whitespace. All resulting flux modes will have a
                      non-zero flux value for enforced reactions

   'adjacency-method' Many different methods, all using pattern trees,
                      the fastest methods are
                      'pattern-tree-minzero'   default, combinatorial
                                               test with pattern trees
                      'pattern-tree-rank'      standard rank test with
                                               doubles precision
                      'pattern-tree-mod-rank'  rank test with residue
                                               arithmetic
                      'rankup-modpi-incore'    rank updating with residue
                                               int32 arithmetic, prime
                                               p <= sqrt((2^31-1)/2),
                                               pattern tree is stored in
                                               memory (i.e. in-core)
                      'rankup-modpi-outcore'   rank updating with residue
                                               int32 arithmetic, prime
                                               p <= sqrt((2^31-1)/2),
                                               pattern tree is stored out
                                               of the main memory (i.e.
                                               out-of-core)
                      'pattern-tree-rank-update-modpi' rank updating with
                                                       residue int32
                                                       arithmetic, prime
                                                       p <= sqrt((2^31-1)/2)
                      'pattern-tree-rank-update-modp'  rank updating with
                                                       residue int64
                                                       arithmetic, prime
                                                       p <= 2^31-1
                      'pattern-tree-rank-update-frac'  rank updating with
                                                       fraction numbers
                      'pattern-tree-rank-update-frac2' rank updating with
                                                       fraction numbers
                                                       and copying

   'rowordering'    Nullspace row ordering, affecting growth of
                    intermediary modes during the iteration phase.
                    The fastest row orderings are
                      'MostZerosOrAbsLexMin'    default, most zeros in a
                                                row, or absoulte
                                                lexicographical if equal
                      'MostZeros'               most zeros in a row
                      'AbsLexMin'               absoulte lexicographical
                      'LexMin'                  lexicographical
                      'FewestNegPos'            lowest product of
                                                negative/positive counts
                      'MostZerosOrFewestNegPos' the two orderings, the
                                                second ordering is used
                                                if sorting equal with 1st
                      'MostZerosOrLexMin'       the two orderings, the
                                                second ordering is used
                                                if sorting equal with 1st
                      'FewestNegPosOrMostZeros' the two orderings, the
                                                second ordering is used
                                                if sorting equal with 1st
                      'FewestNegPosOrAbsLexMin' the two orderings, the
                                                second ordering is used
                                                if sorting equal with 1st
                      'Random'                  random sorting

   'model'          Algorithm variant for EFM computation. Default model
                    is a nullspace-model, which uses the binary nullspace
                    approach (J. Gagneur & S. Klamt / C. Wagner).
                    Supported algorithm model variants are:
                      'nullspace'              default, binary nullspace
                                               approach
                      'canonical'              canonical approach

   'memory'         Memory type to use, which is usually in-core (main
                    memory, RAM) or out-of-core (store intermediary
                    results on disk).
                    Supported memory variants are:
                      'in-core'                default, keep all modes in
                                               main memory
                      'out-core'               store intermediary results
                                               in files on disk, in the
                                               temp directory (see
                                               'tmpdir' option). For
                                               extreme cases, the bit
                                               pattern trees should also
                                               stored on disk, which is
                                               currently supported by the
                                               'adjacency-method' option
                                               'rankup-modpi-outcore'
                      'sort-out-core'          like 'out-core', but also
                                               sorting to create the
                                               pattern trees is performed
                                               out of core memory

   'impl'           Algorithm implementation for EFM computation. Default
                    is a sequential double description implementation.
                    Supported algorithm implementations:
                      'SequentialDoubleDescriptionImpl'	default

   'tmpdir'         Directory for temporary files, e.g. data files for
                    intermediary results if out core implementation is
                    used (see memory option). Fast, non-server
                    directories with large storage capacity are
                    preferable. Default is tmp subdirectory in the
                    installation path

    For more details on the algorithm or implementation, see:
    http://www.csb.ethz.ch/tools/software/efmtool.html; Terzer M, Stelling J
    (2008) Large-scale computation of elementary flux modes with bit pattern
    trees. Bioinformatics 24: 2229-2235.

    """

    return EFMToolWrapper(cobra_model, opts, verbose, java_args=java_args)()

if __name__ == "__main__":

    # TODO: These should get moved to the test suite.
    # This just builds a test model (from
    # http://www.csb.ethz.ch/tools/software/efmtool/documentation.html) and
    # calculates the elementary flux modes.

    from cobra.core import Metabolite, Reaction, Model

    model = Model('simple_model')

    A = Metabolite('A')
    B = Metabolite('B')
    C = Metabolite('C')
    D = Metabolite('D')
    E = Metabolite('E')
    P = Metabolite('P')

    R1 = Reaction('R1')
    R2 = Reaction('R2')
    R3 = Reaction('R3')
    R4 = Reaction('R4')
    R5 = Reaction('R5')
    R6 = Reaction('R6')
    R7 = Reaction('R7')
    R8 = Reaction('R8')
    R9 = Reaction('R9')
    R10 = Reaction('R10')

    model.add_metabolites([A, B, C, D, E, P])
    model.add_reactions([R1, R2, R3, R4, R5, R6, R7, R8, R9, R10])

    model.reactions.R1.build_reaction_from_string('--> A')
    model.reactions.R2.build_reaction_from_string('<--> B')
    model.reactions.R3.build_reaction_from_string('P -->')
    model.reactions.R4.build_reaction_from_string('E -->')
    model.reactions.R5.build_reaction_from_string('A --> B')
    model.reactions.R6.build_reaction_from_string('A --> C')
    model.reactions.R7.build_reaction_from_string('A --> D')
    model.reactions.R8.build_reaction_from_string('B <--> C')
    model.reactions.R9.build_reaction_from_string('B --> P')
    model.reactions.R10.build_reaction_from_string('C + D --> E + P')

    out = calculate_elementary_modes(model, verbose=True,
                                     java_args='-Xmx1G')

    print(out)
<commit_msg>testing dir_<commit_after>"""
A python wrapper around EFMTool, a java-based library for calculating
elementary flux modes. See http://www.csb.ethz.ch/tools/software/efmtool.html;
Terzer M, Stelling J (2008) Large-scale computation of elementary flux modes
with bit pattern trees. Bioinformatics 24: 2229-2235.
"""

import os
from cobra.util import create_stoichiometric_matrix

import numpy as np

try:
    import pandas as pd
    pandas = True
except ImportError:
    pandas = False

from pyefm.utils import make_temp_directory, run_process

class EFMToolWrapper(object):

    def __init__(self, cobra_model, opts=None, verbose=True, java_args=None):

        if opts is None:
            opts = {}

        self.opts = opts
        self.model = cobra_model
        self.verbose = verbose

        self.efm_lib_dir = (os.path.dirname(os.path.abspath(__file__)) +
                            '/efmtool')
        self.efm_command = ['java', '-jar', self.efm_lib_dir +
                            '/metabolic-efm-all.jar']

        # Insert a string with extra options to the java virtual machine.
        if java_args:
            self.efm_command.insert(1, java_args)

    def create_model_files(self, temp_dir):
        """ Write stochiometry data, reaction reversibilities, metabolite, and
        reaction names to temporary files in preparation for calling efmtool

        """

        stoich_mat = create_stoichiometric_matrix(self.model)

        # Stoichiometric Matrix
        np.savetxt(temp_dir + '/stoich.txt', stoich_mat, delimiter='\t')

        # Reaction reversibilities
        np.savetxt(temp_dir + '/revs.txt',
                   np.array([r.reversibility for r in self.model.reactions]),
                   delimiter='\t', fmt='%d', newline='\t')

        # Reaction Names
        with open(temp_dir + '/rnames.txt', 'w') as f:
            f.write('\t'.join(('"{}"'.format(r.id)
                               for r in self.model.reactions)))

        # Metabolite Names
        with open(temp_dir + '/mnames.txt', 'w') as f:
            f.write('\t'.join(('"{}"'.format(m.id)
                               for m in self.model.metabolites)))

    def read_double_out(self, out_file):
        """ Read the output file generated from EMFTool. Returns a numpy array or
        pandas dataframe (if pandas can be loaded)

        """
        with open(out_file, 'rb') as f:
            out_arr = np.fromstring(f.read()[13:], dtype='>d').reshape(
                (-1, len(self.model.reactions))).T
            out_arr = np.array(out_arr, dtype=np.float64)

        if pandas:
            out_arr = pd.DataFrame(
                out_arr, index=(r.id for r in self.model.reactions),
                columns=('EM{}'.format(i) for i in
                         range(1, out_arr.shape[1] + 1)))

        return out_arr.T

    def __call__(self):

        dir_ = self.opts.pop('dir_', None)

        with make_temp_directory('efmtool', dir_) as temp_dir:

            self.create_model_files(temp_dir)

            try:
                out_file = self.opts.pop('out_file')
            except KeyError:
                out_file = temp_dir + '/out.bin'

            # Default options for the EMFtool. I don't recommend changing any
            # of these, even though the function is set up for this. This tool
            # hasn't been tested for any options other than these
            default_opts = {
                'kind': 'stoichiometry',
                'stoich': temp_dir + '/stoich.txt',
                'rev': temp_dir + '/revs.txt',
                'reac': temp_dir + '/rnames.txt',
                'meta': temp_dir + '/mnames.txt',
                'arithmetic': 'double',
                'zero': 1E-10,
                'out': 'binary-doubles',  # TODO support binary output?
                'compression': 'default',
                'log': 'console',
                'level': 'INFO',
                'maxthreads': -1,
                'normalize': 'max',
                'adjacency-method': 'pattern-tree-minzero',
                'rowordering': 'MostZerosOrAbsLexMin',
                'tmpdir': temp_dir
            }

            default_opts.update(self.opts)

            # Create a list of arguments to pass to the python subprocess
            # module
            def opt_gen():
                for opt, val in default_opts.items():
                    yield '-' + opt
                    yield str(val)

                    # This kw takes two arguments
                    if opt == 'out':
                        yield out_file

            # Run the EFMtool, outputting STDOUT to python.
            run_process(self.efm_command + list(opt_gen()), verbose=self.verbose)

            if 'binary-doubles' in default_opts['out']:
                return self.read_double_out(out_file)

            else:
                raise RuntimeError('Other output options not supported')

def calculate_elementary_modes(cobra_model, opts=None, verbose=True,
                               java_args=None):
    """ A python wrapper around EFMTool, a java-based library for calculating
    elementary flux modes.

    Parameters:
    ===========

    cobra_model: cobra.Model or similar
        The model for which to calculate elementary flux modes. Reaction
        reversibilities are inferred from their corresponding bounds.

    opts: dict or None
        Additional dictionary options to pass to the emftool solver. See Option
        Descriptions below for additional information on arguments.

    verbose: bool
        Whether or not to redirect stdout to the python shell.

    java_args: string
        Extra command-line options to pass to the java virtual machine.
        Eg. '-Xmx1g' will set the heap space to 1 GB.

    Option Descriptions from the MATLAB Documentation:
    ==================================================

    The following options are supported (any others are silently ignored):

   'arithmetic'     Which arithmetic to use, one of
                      'double'     Double precision floating point (default)
                      'fractional' Fraction numbers, fixed or infinite
                                   precision
   'precision'      Bits to use for fraction numerator/denominator,
                    -1 for infinite precision, or 64, 128, ... bits for
                    fixed precision (only for arithmetic=fractional)
   'zero'           Values to be treated as zero
                    defaults are 1e-10 for double arithmetic and 0 for
                    fractional arithmetic
   'compression'    Which compression techniques should be applied to the
                    network before efm computation, one of
                      'default'                    Default compression
                      'default-no-combine'         No reaction merging
                      'default-no-couple-combine'  Reduced react. merging
                      'default-no-duplicate'       No duplicate gene com.
                      'default-all-duplicate'      Extended duplicate
                                                   gene compression
                      'off'                        No compression at all
                      'unique'                     only unique flows
                                                   compression
                      'nullspace'                  Only nullspace based
                                                   compression
                      'unique-no-recursion'        Only one round of
                                                   unique flows compr.
                      'nullspace-no-recursion'     Only one round of
                                                   nullspace compression

   'level'          The log level, default is INFO, supported:
                      'WARNING', 'INFO', 'FINE', 'FINER', 'FINEST'

   'maxthreads'     Maximum threads to use, default is 1 for single core
                    systems and k for multi core systems with k cores.

   'normalize'      Normalization method of the output efms, one of
                      'min'     default, minimum absolute value is 1
                      'max'     maximum absolute value is 1
                      'norm2'   norm 2, that is, vector length is 1
                      'squared' like norm 2, but all values are squared,
                                possibly negative. To get the original
                                value, take the square root of the
                                absolute value and keep the sign, i.e.
                                val = sign(valsq) * sqrt(abs(valsq)).
                                This method allows error free values
                                if fraction number arithmetic is used.
                      'none'    no normalization

   'count-only'     If true, the (binary) modes are computed and the
                      number of efms is returned, but not the efm vectors
   'sign-only'      If true, the (binary) modes are computed, but only
                      sign values of reaction fluxes are returned as EFMs.
                      The values +/-1 stand for forward/reverse flux, and 0
                      for no flux, respectively. Such sign-valued EFMs can
                      be converted back into double valued EFMs by using
                      the SignToDouble function.
   'parse-only'     If true, input data is processed and parsed, but no
                      elementary modes are computed. The returned
                      structure contains no efms field
   'convert-only'   If true, input data is processed and converted into
                      the program's internal format. The efm computation
                      itself is not started, but call options (for manual
                      invocation of the Java program) are reported as
                      return string. The converted files are stored in
                      the tmp directory.
                      Depending on the input format, these files are:
                          - stoich.txt    the stoichiometric matrix
                          - revs.txt      the reaction reversibilities
                          - mnames.txt    the metabolite names
                          - rnames.txt    the reaction names
                          - rlist.txt     the reaction list file
                          - xbml.xml      the sbml input file

   'suppress'       List of reaction names to suppress, separated by
                      whitespace. All resulting flux modes will have a
                      zero flux value for suppressed reactions

   'enforce'       List of reaction names to enforce, separated by
                      whitespace. All resulting flux modes will have a
                      non-zero flux value for enforced reactions

   'adjacency-method' Many different methods, all using pattern trees,
                      the fastest methods are
                      'pattern-tree-minzero'   default, combinatorial
                                               test with pattern trees
                      'pattern-tree-rank'      standard rank test with
                                               doubles precision
                      'pattern-tree-mod-rank'  rank test with residue
                                               arithmetic
                      'rankup-modpi-incore'    rank updating with residue
                                               int32 arithmetic, prime
                                               p <= sqrt((2^31-1)/2),
                                               pattern tree is stored in
                                               memory (i.e. in-core)
                      'rankup-modpi-outcore'   rank updating with residue
                                               int32 arithmetic, prime
                                               p <= sqrt((2^31-1)/2),
                                               pattern tree is stored out
                                               of the main memory (i.e.
                                               out-of-core)
                      'pattern-tree-rank-update-modpi' rank updating with
                                                       residue int32
                                                       arithmetic, prime
                                                       p <= sqrt((2^31-1)/2)
                      'pattern-tree-rank-update-modp'  rank updating with
                                                       residue int64
                                                       arithmetic, prime
                                                       p <= 2^31-1
                      'pattern-tree-rank-update-frac'  rank updating with
                                                       fraction numbers
                      'pattern-tree-rank-update-frac2' rank updating with
                                                       fraction numbers
                                                       and copying

   'rowordering'    Nullspace row ordering, affecting growth of
                    intermediary modes during the iteration phase.
                    The fastest row orderings are
                      'MostZerosOrAbsLexMin'    default, most zeros in a
                                                row, or absoulte
                                                lexicographical if equal
                      'MostZeros'               most zeros in a row
                      'AbsLexMin'               absoulte lexicographical
                      'LexMin'                  lexicographical
                      'FewestNegPos'            lowest product of
                                                negative/positive counts
                      'MostZerosOrFewestNegPos' the two orderings, the
                                                second ordering is used
                                                if sorting equal with 1st
                      'MostZerosOrLexMin'       the two orderings, the
                                                second ordering is used
                                                if sorting equal with 1st
                      'FewestNegPosOrMostZeros' the two orderings, the
                                                second ordering is used
                                                if sorting equal with 1st
                      'FewestNegPosOrAbsLexMin' the two orderings, the
                                                second ordering is used
                                                if sorting equal with 1st
                      'Random'                  random sorting

   'model'          Algorithm variant for EFM computation. Default model
                    is a nullspace-model, which uses the binary nullspace
                    approach (J. Gagneur & S. Klamt / C. Wagner).
                    Supported algorithm model variants are:
                      'nullspace'              default, binary nullspace
                                               approach
                      'canonical'              canonical approach

   'memory'         Memory type to use, which is usually in-core (main
                    memory, RAM) or out-of-core (store intermediary
                    results on disk).
                    Supported memory variants are:
                      'in-core'                default, keep all modes in
                                               main memory
                      'out-core'               store intermediary results
                                               in files on disk, in the
                                               temp directory (see
                                               'tmpdir' option). For
                                               extreme cases, the bit
                                               pattern trees should also
                                               stored on disk, which is
                                               currently supported by the
                                               'adjacency-method' option
                                               'rankup-modpi-outcore'
                      'sort-out-core'          like 'out-core', but also
                                               sorting to create the
                                               pattern trees is performed
                                               out of core memory

   'impl'           Algorithm implementation for EFM computation. Default
                    is a sequential double description implementation.
                    Supported algorithm implementations:
                      'SequentialDoubleDescriptionImpl'	default

   'tmpdir'         Directory for temporary files, e.g. data files for
                    intermediary results if out core implementation is
                    used (see memory option). Fast, non-server
                    directories with large storage capacity are
                    preferable. Default is tmp subdirectory in the
                    installation path

    For more details on the algorithm or implementation, see:
    http://www.csb.ethz.ch/tools/software/efmtool.html; Terzer M, Stelling J
    (2008) Large-scale computation of elementary flux modes with bit pattern
    trees. Bioinformatics 24: 2229-2235.

    """

    return EFMToolWrapper(cobra_model, opts, verbose, java_args=java_args)()

if __name__ == "__main__":

    # TODO: These should get moved to the test suite.
    # This just builds a test model (from
    # http://www.csb.ethz.ch/tools/software/efmtool/documentation.html) and
    # calculates the elementary flux modes.

    from cobra.core import Metabolite, Reaction, Model

    model = Model('simple_model')

    A = Metabolite('A')
    B = Metabolite('B')
    C = Metabolite('C')
    D = Metabolite('D')
    E = Metabolite('E')
    P = Metabolite('P')

    R1 = Reaction('R1')
    R2 = Reaction('R2')
    R3 = Reaction('R3')
    R4 = Reaction('R4')
    R5 = Reaction('R5')
    R6 = Reaction('R6')
    R7 = Reaction('R7')
    R8 = Reaction('R8')
    R9 = Reaction('R9')
    R10 = Reaction('R10')

    model.add_metabolites([A, B, C, D, E, P])
    model.add_reactions([R1, R2, R3, R4, R5, R6, R7, R8, R9, R10])

    model.reactions.R1.build_reaction_from_string('--> A')
    model.reactions.R2.build_reaction_from_string('<--> B')
    model.reactions.R3.build_reaction_from_string('P -->')
    model.reactions.R4.build_reaction_from_string('E -->')
    model.reactions.R5.build_reaction_from_string('A --> B')
    model.reactions.R6.build_reaction_from_string('A --> C')
    model.reactions.R7.build_reaction_from_string('A --> D')
    model.reactions.R8.build_reaction_from_string('B <--> C')
    model.reactions.R9.build_reaction_from_string('B --> P')
    model.reactions.R10.build_reaction_from_string('C + D --> E + P')

    out = calculate_elementary_modes(model, verbose=True,
                                     java_args='-Xmx1G', opts={'dir_': 'test'})

    print(out)
<|endoftext|>


<commit_before>#!python3

###############################################################################
# Copyright (c) 2012-2013 Evgeny Proydakov <lord.tiran@gmail.com>
###############################################################################
 # -*- coding: utf-8 -*-
###############################################################################

import os
import sys
import uuid
import shutil
import zipfile
import argparse
import traceback
import subprocess
import xml.dom.minidom

import utils
import compare

###############################################################################
#                                 constant                                    #
###############################################################################

APP_VERSION = '0.7.0'

# data about rdo apps
app_directory      = '.'
model_directory    = '.'
test_expansion     = '.rtestx'
trace_expansion    = '.trc'
result_expansion   = '.pmv'
build_dir_substr   = 'build'
rdo_ex_substr      = 'rdo'
rdo_test_ex_substr = 'rdo_check_results'

if sys.platform == 'win32':
    rdo_ex_substr = 'rdo.exe'

RDO_CONSOLE_COMPILE_LOG_FILE_NAME = 'log.txt'

# test target type
TARGET_CONSOLE   = 'CONSOLE'
TAGRET_CONVERTER = 'CONVERTOR'
TARGET_GUI       = 'uGUI'

DEFAULT_EXIT_CODE = 0

# python script exit code
APP_CODE_TERMINATION_NORMAL = 0
APP_CODE_TERMINATION_ERROR  = 1

# rdo console exit code
RDO_CONSOLE_TERMINATION_NORMAL                         = 0
RDO_CONSOLE_TERMINATION_WITH_AN_ERROR_NO_MODEL         = 1
RDO_CONSOLE_TERMINATION_WITH_AN_ERROR_OPEN_MODEL_ERROR = 2
RDO_CONSOLE_TERMINATION_WITH_AN_ERROR_PARSE_ERROR      = 3
RDO_CONSOLE_TERMINATION_WITH_AN_ERROR_RUNTIME_ERROR    = 4

IGNORE_PATTERNS = ('^.git','.svn')

CONSOLE_PARAM_ENCODING = sys.getfilesystemencoding()

DIR_LINE = '/'
if sys.platform == 'win32':
    DIR_LINE = '\\'

# ancillary data
dividing_line = '-------------------------------------------------------------------------------'

###############################################################################
#                                 functions                                   #
###############################################################################

def safe_exit():
    utils.enc_print ('\n', 'PYTHON EXIT CODE :', G_EXIT_CODE, '\n')
    sys.exit(G_EXIT_CODE)

###############################################################################

def get_test_files(dir):
    files = utils.get_files_list(dir)

    test_files = []

    for file in files:
        res = file.endswith(test_expansion)
        if res == True:
            test_files.append(file)

    return test_files

###############################################################################

def get_executables(dir):
    files = utils.get_files_list(dir)

    rdo_ex = ''

    for file in files:
        res = file.endswith(rdo_ex_substr)
        if res == True:
            rdo_ex = file

        if rdo_ex:
           break

    return rdo_ex

###############################################################################

def get_node_attribute_from_dom(dom, node_text, attribute_text):

    nodes = dom.getElementsByTagName(node_text)
    if not len(nodes):
        return ''

    node = nodes[0]
    attribute_text_data = node.getAttribute(attribute_text)

    return attribute_text_data

###############################################################################

def compare_etalons(etalons, basedir):
    cycle_exit_code = APP_CODE_TERMINATION_NORMAL
    for etalon in etalons:
        source_file = basedir + etalon['source']
        target_file = basedir + etalon['target']
        type = etalon['type']
        res = compare.files(source_file, target_file, type)

        compare_string = 'ERROR'

        if res == True:
            compare_string = 'OK'
        else:
            cycle_exit_code = APP_CODE_TERMINATION_ERROR

        utils.enc_print ('COMPARE:  ', etalon['source'], '  AND  ', etalon['target'], ':  ', compare_string)

    return cycle_exit_code

###############################################################################

def test_console(dirname, model):
    # run rdo_console app on test model
    model_file = '' + dirname + model['name']
    command = (rdo_ex + ' -i ' + utils.wrap_the_string_in_quotes(model_file))
    utils.enc_print('Run:', command, '\n')
    simulation_code = subprocess.call(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    utils.enc_print ('SIMULATION EXIT CODE :', simulation_code)

    # check simulation exit code
    simulation_exit_code_string = 'ERROR'
    if simulation_code == exit_code:
        cycle_exit_code = APP_CODE_TERMINATION_NORMAL
        simulation_exit_code_string = 'OK'
    else:
        cycle_exit_code = APP_CODE_TERMINATION_ERROR

    utils.enc_print ('CHECK SIM EXIT CODE  :', simulation_exit_code_string, '\n')

    # nornal simulation check
    if simulation_code == RDO_CONSOLE_TERMINATION_NORMAL:
        try:
            # compare etalons
            cycle_exit_code = compare_etalons(etalons, dirname)
        except:
            traceback.print_exc(file=sys.stdout)
            cycle_exit_code = APP_CODE_TERMINATION_ERROR

    # check compile error log
    elif simulation_code == RDO_CONSOLE_TERMINATION_WITH_AN_ERROR_PARSE_ERROR:
        try:
            simulation_log_file = dirname + RDO_CONSOLE_COMPILE_LOG_FILE_NAME
            simulation_log_file_etalon = dirname + model['log_compilation']
            res = compare.full(simulation_log_file, simulation_log_file_etalon)
            check_message_cmp_string = 'ERROR'
            if res:
                cycle_exit_code = APP_CODE_TERMINATION_NORMAL
                check_message_cmp_string = 'OK'
            utils.enc_print ('CHECK ERROR LIST     :', check_message_cmp_string)

        except:
            traceback.print_exc(file=sys.stdout)
            cycle_exit_code = APP_CODE_TERMINATION_ERROR

    return cycle_exit_code

###############################################################################

def test_convertor(dirname, model):
    try:
        text_uuid = str(uuid.uuid4())
        temp_directory_name = dirname + text_uuid + DIR_LINE
        shutil.copytree(dirname, temp_directory_name, ignore=shutil.ignore_patterns(*IGNORE_PATTERNS))
        model_file = '' + temp_directory_name + model['name']
        command = (rdo_ex + ' -i ' + utils.wrap_the_string_in_quotes(model_file) + ' -c')
        utils.enc_print ('Run:', command, '\n')
        convertor_exit_code = subprocess.call(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        utils.enc_print ('CONVERT EXIT CODE :', convertor_exit_code, '\n')
        if convertor_exit_code == exit_code:
            cycle_exit_code = compare_etalons(etalons, temp_directory_name)
        if cycle_exit_code != APP_CODE_TERMINATION_NORMAL:
            arc_name = text_uuid + '.zip'
            utils.enc_print('Make zip archive: ' + arc_name)
            zipf = zipfile.ZipFile(arc_name, 'w')
            zipdir(temp_directory_name, zipf)
            zipf.close()
        shutil.rmtree(temp_directory_name)
    except:
        traceback.print_exc(file=sys.stdout)
        cycle_exit_code = APP_CODE_TERMINATION_ERROR

    return cycle_exit_code

###############################################################################

def zipdir(path, zip):
    for root, dirs, files in os.walk(path):
        for file in files:
            zip.write(os.path.join(root, file))

###############################################################################
#                                 main code                                   #
###############################################################################

# parse console options
parser = argparse.ArgumentParser(description = 'rdo executor of system tests: ' + str(APP_VERSION))
parser.add_argument('-ad', action = 'store', dest = 'app_directory',   default = app_directory,   help = 'application directory')
parser.add_argument('-md', action = 'store', dest = 'model_directory', default = model_directory, help = 'model directory')
parser.add_argument('-enc', action='store', default = '')
parser.add_argument('-dec', action='store', default = '')

args = parser.parse_args()

app_directory   = '' + args.app_directory
model_directory = '' + args.model_directory

utils.ENC = args.enc
utils.DEC = args.dec

# global exit code variable
G_EXIT_CODE = APP_CODE_TERMINATION_NORMAL

utils.enc_print (dividing_line)
utils.enc_print ('STARTED SCRIPT :', sys.argv[0])
utils.enc_print ('SYSTEM ENCODING', sys.getdefaultencoding())
utils.enc_print ('FILESYSTEM ENCODING', sys.getfilesystemencoding())

# search rdo and rdo_test executables
rdo_ex = get_executables(app_directory)
if not os.path.exists(rdo_ex):
    utils.enc_print ('Build app not found. Critical error !!!')
    sys.exit(APP_CODE_TERMINATION_ERROR)

# search .rtestx files
files = get_test_files(model_directory)
files.sort()

utils.enc_print ('\nDEBUG INFO')
utils.enc_print ('\nFind RDO executables    :')
utils.enc_print (rdo_ex)

utils.enc_print ('\nFind test project files :')
utils.print_list_of_line(files)

# parse xml and start tests
utils.enc_print ('\nSTARTED TEST CYCLE\n')
bad_models = []

# check model cycle
for task in files:

    utils.enc_print (dividing_line)

    utask   = task
    dirname = os.path.dirname(utask) + DIR_LINE

    text_task = open(utask, 'r', encoding = 'utf-8').read()

    parse_result = False

    try:
        dom = xml.dom.minidom.parseString(text_task)
        parse_result = True

    except:
        utils.enc_print ('PARSE XML ERROR:', utask)
        traceback.print_exc(file=sys.stdout)
        G_EXIT_CODE = APP_CODE_TERMINATION_ERROR
        safe_exit()

    if parse_result:
        model = {}
        model['name']            = get_node_attribute_from_dom(dom, 'model', 'name'           )
        model['target']          = get_node_attribute_from_dom(dom, 'model', 'target'         )
        model['exit_code']       = get_node_attribute_from_dom(dom, 'model', 'exit_code'      )
        model['log_compilation'] = get_node_attribute_from_dom(dom, 'model', 'log_compilation')

        utils.enc_print ('Project              :', task)
        utils.enc_print ('Model file           :', model['name'])
        utils.enc_print ('Target               :', model['target'])
        utils.enc_print ('Exit code            :', model['exit_code'])
        utils.enc_print ('Log compilation file :', model['log_compilation'])
        sys.stdout.flush()

        etalons = []
        node_etalons = dom.getElementsByTagName('etalons')
        if len(node_etalons):
            files = node_etalons[0].getElementsByTagName('file')
            for file in files:
                etalon = {}
                etalon['source'] = file.getAttribute('source')
                etalon['target'] = file.getAttribute('target')
                etalon['type']   = file.getAttribute('type')
                etalons.append(etalon)
        try:
            exit_code = int(model['exit_code'])
        except:
            traceback.print_exc(file=sys.stdout)
            exit_code = DEFAULT_EXIT_CODE

        if len(etalons):
            utils.enc_print ('\n', 'Etalons :', '\n')
        for etalon in etalons:
            utils.enc_print ('SOURCE:  ', etalon['source'], '  TARGET:  ', etalon['target'], '  TYPE:  ',   etalon['type'])
        utils.enc_print ('')

        cycle_exit_code = APP_CODE_TERMINATION_ERROR

        # select console target
        if model['target'] == TARGET_CONSOLE:
            cycle_exit_code = test_console(dirname, model)
        elif model['target'] == TAGRET_CONVERTER:
            cycle_exit_code = test_convertor(dirname, model)
        else:
            utils.enc_print ('INVALID TARGET')

        if cycle_exit_code == APP_CODE_TERMINATION_ERROR:
            bad_models.append(task)
            G_EXIT_CODE = APP_CODE_TERMINATION_ERROR

utils.enc_print ('\n', 'MODEL WITH ERRORS:')
utils.print_list_of_line(bad_models)

if(len(bad_models) < 1):
    utils.enc_print ('(empty)')

safe_exit()
<commit_msg> - updated executor of system tests : fixed logic error<commit_after>#!python3

###############################################################################
# Copyright (c) 2012-2013 Evgeny Proydakov <lord.tiran@gmail.com>
###############################################################################
 # -*- coding: utf-8 -*-
###############################################################################

import os
import sys
import uuid
import shutil
import zipfile
import argparse
import traceback
import subprocess
import xml.dom.minidom

import utils
import compare

###############################################################################
#                                 constant                                    #
###############################################################################

APP_VERSION = '0.7.0'

# data about rdo apps
app_directory      = '.'
model_directory    = '.'
test_expansion     = '.rtestx'
trace_expansion    = '.trc'
result_expansion   = '.pmv'
build_dir_substr   = 'build'
rdo_ex_substr      = 'rdo'
rdo_test_ex_substr = 'rdo_check_results'

if sys.platform == 'win32':
    rdo_ex_substr = 'rdo.exe'

RDO_CONSOLE_COMPILE_LOG_FILE_NAME = 'log.txt'

# test target type
TARGET_CONSOLE   = 'CONSOLE'
TAGRET_CONVERTER = 'CONVERTOR'
TARGET_GUI       = 'uGUI'

DEFAULT_EXIT_CODE = 0

# python script exit code
APP_CODE_TERMINATION_NORMAL = 0
APP_CODE_TERMINATION_ERROR  = 1

# rdo console exit code
RDO_CONSOLE_TERMINATION_NORMAL                         = 0
RDO_CONSOLE_TERMINATION_WITH_AN_ERROR_NO_MODEL         = 1
RDO_CONSOLE_TERMINATION_WITH_AN_ERROR_OPEN_MODEL_ERROR = 2
RDO_CONSOLE_TERMINATION_WITH_AN_ERROR_PARSE_ERROR      = 3
RDO_CONSOLE_TERMINATION_WITH_AN_ERROR_RUNTIME_ERROR    = 4

IGNORE_PATTERNS = ('^.git','.svn')

CONSOLE_PARAM_ENCODING = sys.getfilesystemencoding()

DIR_LINE = '/'
if sys.platform == 'win32':
    DIR_LINE = '\\'

# ancillary data
dividing_line = '-------------------------------------------------------------------------------'

###############################################################################
#                                 functions                                   #
###############################################################################

def safe_exit():
    utils.enc_print ('\n', 'PYTHON EXIT CODE :', G_EXIT_CODE, '\n')
    sys.exit(G_EXIT_CODE)

###############################################################################

def get_test_files(dir):
    files = utils.get_files_list(dir)

    test_files = []

    for file in files:
        res = file.endswith(test_expansion)
        if res == True:
            test_files.append(file)

    return test_files

###############################################################################

def get_executables(dir):
    files = utils.get_files_list(dir)

    rdo_ex = ''

    for file in files:
        res = file.endswith(rdo_ex_substr)
        if res == True:
            rdo_ex = file

        if rdo_ex:
           break

    return rdo_ex

###############################################################################

def get_node_attribute_from_dom(dom, node_text, attribute_text):

    nodes = dom.getElementsByTagName(node_text)
    if not len(nodes):
        return ''

    node = nodes[0]
    attribute_text_data = node.getAttribute(attribute_text)

    return attribute_text_data

###############################################################################

def compare_etalons(etalons, basedir):
    cycle_exit_code = APP_CODE_TERMINATION_NORMAL
    for etalon in etalons:
        source_file = basedir + etalon['source']
        target_file = basedir + etalon['target']
        type = etalon['type']
        res = compare.files(source_file, target_file, type)

        compare_string = 'ERROR'

        if res == True:
            compare_string = 'OK'
        else:
            cycle_exit_code = APP_CODE_TERMINATION_ERROR

        utils.enc_print ('COMPARE:  ', etalon['source'], '  AND  ', etalon['target'], ':  ', compare_string)

    return cycle_exit_code

###############################################################################

def test_console(dirname, model):
    # run rdo_console app on test model
    model_file = '' + dirname + model['name']
    command = (rdo_ex + ' -i ' + utils.wrap_the_string_in_quotes(model_file))
    utils.enc_print('Run:', command, '\n')
    simulation_code = subprocess.call(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    utils.enc_print ('SIMULATION EXIT CODE :', simulation_code)

    # check simulation exit code
    simulation_exit_code_string = 'ERROR'
    if simulation_code == exit_code:
        cycle_exit_code = APP_CODE_TERMINATION_NORMAL
        simulation_exit_code_string = 'OK'
    else:
        cycle_exit_code = APP_CODE_TERMINATION_ERROR

    utils.enc_print ('CHECK SIM EXIT CODE  :', simulation_exit_code_string, '\n')

    # nornal simulation check
    if simulation_code == RDO_CONSOLE_TERMINATION_NORMAL:
        try:
            # compare etalons
            cycle_exit_code = compare_etalons(etalons, dirname)
        except:
            traceback.print_exc(file=sys.stdout)
            cycle_exit_code = APP_CODE_TERMINATION_ERROR

    # check compile error log
    elif simulation_code == RDO_CONSOLE_TERMINATION_WITH_AN_ERROR_PARSE_ERROR:
        try:
            simulation_log_file = dirname + RDO_CONSOLE_COMPILE_LOG_FILE_NAME
            simulation_log_file_etalon = dirname + model['log_compilation']
            res = compare.full(simulation_log_file, simulation_log_file_etalon)
            check_message_cmp_string = 'ERROR'
            if res:
                cycle_exit_code = APP_CODE_TERMINATION_NORMAL
                check_message_cmp_string = 'OK'
            utils.enc_print ('CHECK ERROR LIST     :', check_message_cmp_string)

        except:
            traceback.print_exc(file=sys.stdout)
            cycle_exit_code = APP_CODE_TERMINATION_ERROR

    return cycle_exit_code

###############################################################################

def test_convertor(dirname, model):
    try:
        text_uuid = str(uuid.uuid4())
        temp_directory_name = dirname + text_uuid + DIR_LINE
        shutil.copytree(dirname, temp_directory_name, ignore=shutil.ignore_patterns(*IGNORE_PATTERNS))
        model_file = '' + temp_directory_name + model['name']
        command = (rdo_ex + ' -i ' + utils.wrap_the_string_in_quotes(model_file) + ' -c')
        utils.enc_print ('Run:', command, '\n')
        convertor_exit_code = subprocess.call(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        utils.enc_print ('CONVERT EXIT CODE :', convertor_exit_code, '\n')
        if convertor_exit_code == exit_code:
            cycle_exit_code = compare_etalons(etalons, temp_directory_name)
        if cycle_exit_code != APP_CODE_TERMINATION_NORMAL:
            arc_name = dirname + text_uuid + '.zip'
            utils.enc_print('Make zip archive: ' + arc_name)
            zipf = zipfile.ZipFile(arc_name, 'w')
            zipdir(temp_directory_name, zipf)
            zipf.close()
        shutil.rmtree(temp_directory_name)
    except:
        traceback.print_exc(file=sys.stdout)
        cycle_exit_code = APP_CODE_TERMINATION_ERROR

    return cycle_exit_code

###############################################################################

def zipdir(path, zip):
    for root, dirs, files in os.walk(path):
        for file in files:
            zip.write(os.path.join(root, file))

###############################################################################
#                                 main code                                   #
###############################################################################

# parse console options
parser = argparse.ArgumentParser(description = 'rdo executor of system tests: ' + str(APP_VERSION))
parser.add_argument('-ad', action = 'store', dest = 'app_directory',   default = app_directory,   help = 'application directory')
parser.add_argument('-md', action = 'store', dest = 'model_directory', default = model_directory, help = 'model directory')
parser.add_argument('-enc', action='store', default = '')
parser.add_argument('-dec', action='store', default = '')

args = parser.parse_args()

app_directory   = '' + args.app_directory
model_directory = '' + args.model_directory

utils.ENC = args.enc
utils.DEC = args.dec

# global exit code variable
G_EXIT_CODE = APP_CODE_TERMINATION_NORMAL

utils.enc_print (dividing_line)
utils.enc_print ('STARTED SCRIPT :', sys.argv[0])
utils.enc_print ('SYSTEM ENCODING', sys.getdefaultencoding())
utils.enc_print ('FILESYSTEM ENCODING', sys.getfilesystemencoding())

# search rdo and rdo_test executables
rdo_ex = get_executables(app_directory)
if not os.path.exists(rdo_ex):
    utils.enc_print ('Build app not found. Critical error !!!')
    sys.exit(APP_CODE_TERMINATION_ERROR)

# search .rtestx files
files = get_test_files(model_directory)
files.sort()

utils.enc_print ('\nDEBUG INFO')
utils.enc_print ('\nFind RDO executables    :')
utils.enc_print (rdo_ex)

utils.enc_print ('\nFind test project files :')
utils.print_list_of_line(files)

# parse xml and start tests
utils.enc_print ('\nSTARTED TEST CYCLE\n')
bad_models = []

# check model cycle
for task in files:

    utils.enc_print (dividing_line)

    utask   = task
    dirname = os.path.dirname(utask) + DIR_LINE

    text_task = open(utask, 'r', encoding = 'utf-8').read()

    parse_result = False

    try:
        dom = xml.dom.minidom.parseString(text_task)
        parse_result = True

    except:
        utils.enc_print ('PARSE XML ERROR:', utask)
        traceback.print_exc(file=sys.stdout)
        G_EXIT_CODE = APP_CODE_TERMINATION_ERROR
        safe_exit()

    if parse_result:
        model = {}
        model['name']            = get_node_attribute_from_dom(dom, 'model', 'name'           )
        model['target']          = get_node_attribute_from_dom(dom, 'model', 'target'         )
        model['exit_code']       = get_node_attribute_from_dom(dom, 'model', 'exit_code'      )
        model['log_compilation'] = get_node_attribute_from_dom(dom, 'model', 'log_compilation')

        utils.enc_print ('Project              :', task)
        utils.enc_print ('Model file           :', model['name'])
        utils.enc_print ('Target               :', model['target'])
        utils.enc_print ('Exit code            :', model['exit_code'])
        utils.enc_print ('Log compilation file :', model['log_compilation'])
        sys.stdout.flush()

        etalons = []
        node_etalons = dom.getElementsByTagName('etalons')
        if len(node_etalons):
            files = node_etalons[0].getElementsByTagName('file')
            for file in files:
                etalon = {}
                etalon['source'] = file.getAttribute('source')
                etalon['target'] = file.getAttribute('target')
                etalon['type']   = file.getAttribute('type')
                etalons.append(etalon)
        try:
            exit_code = int(model['exit_code'])
        except:
            traceback.print_exc(file=sys.stdout)
            exit_code = DEFAULT_EXIT_CODE

        if len(etalons):
            utils.enc_print ('\n', 'Etalons :', '\n')
        for etalon in etalons:
            utils.enc_print ('SOURCE:  ', etalon['source'], '  TARGET:  ', etalon['target'], '  TYPE:  ',   etalon['type'])
        utils.enc_print ('')

        cycle_exit_code = APP_CODE_TERMINATION_ERROR

        # select console target
        if model['target'] == TARGET_CONSOLE:
            cycle_exit_code = test_console(dirname, model)
        elif model['target'] == TAGRET_CONVERTER:
            cycle_exit_code = test_convertor(dirname, model)
        else:
            utils.enc_print ('INVALID TARGET')

        if cycle_exit_code == APP_CODE_TERMINATION_ERROR:
            bad_models.append(task)
            G_EXIT_CODE = APP_CODE_TERMINATION_ERROR

utils.enc_print ('\n', 'MODEL WITH ERRORS:')
utils.print_list_of_line(bad_models)

if(len(bad_models) < 1):
    utils.enc_print ('(empty)')

safe_exit()
<|endoftext|>