<commit_before>#!/usr/bin/env python

import re
import sys
import argparse
import logging

mrc_header = """[COURSE HEADER]
VERSION = 2
UNITS = ENGLISH
DESCRIPTION = Sample
FILE NAME = sample.mrc
MINUTES PERCENT
[END COURSE HEADER]
[COURSE DATA]
"""

mrc_trailer = """[END COURSE DATA]"""

zone_to_percent = {
    'lz1':45, 'z1':50, 'hz1':55,
    'lz2':58, 'z2':65, 'hz2':73,
    'lz3':78, 'z3':82, 'hz3':88,
    'lz4':93, 'z4':98, 'hz4':103,
    'lz5':107, 'z5':108, 'hz5':118,
    'z5+':130, 'max':150
    }

range_to_zones = {
    'z1>z2':('z1','z2'),
    'z1>z3':('z1','z3'),
    'z1>z4':('z1','z4'),
    'z2>z1':('z2','z1'),
    'z3>z1':('z3','z1')
    }

class WorkoutParser:
    def __init__(self):
        self.sm = {
            'SINGLE_STEP': self.parse_single_step,
            'REPEAT_BLOCK': self.parse_repeat_block
        }
        self.cur_state = 'SINGLE_STEP'
        self.steps = []
        self.repeat_count = 0
        self.torepeat = []

    def parse(self, line):
        self.sm[self.cur_state](line)

    def parse_single_step(self, line):
        mobj = re.search('repeat\s+([\d]+):', line)
        if mobj:
            self.repeat_count = int(mobj.group(1))
            self.cur_state = 'REPEAT_BLOCK'
            return

        mobj = re.search('^([\d.]+)\s+([\w><]+)', line)
        if mobj:
            self.steps.append((mobj.group(1), mobj.group(2)))

    def parse_repeat_block(self, line):
        mobj = re.search('\s+([\d.]+)\s+([\w><]+)', line)
        if mobj:
            self.torepeat.append((mobj.group(1), mobj.group(2)))
            return

        mobj = re.search('^([\d.]+)\s+([\w><]+)', line)
        if mobj:
            self.insert_repeat_steps()
            self.cur_state = 'SINGLE_STEP'
            self.parse_single_step(line)

    def insert_repeat_steps(self):
        self.steps += self.torepeat * self.repeat_count
        self.torepeat = []
        self.repeat_count = 0

def parse_workout(workout):
    parser = WorkoutParser()

    steps = []
    lines = [line.lower() for line in workout.split('\n') if line]
    for line in lines:
        if re.search('^#', line):
            continue
        parser.parse(line)
    return parser.steps

def body_line(start, percent):
    return "%0.2f\t%d\n" % (start, percent)

def body_block(start, duration, start_zone, end_zone):
    body = ''
    body += body_line(start, zone_to_percent[start_zone])
    body += body_line(start+duration, zone_to_percent[end_zone])
    return body

def generate_mrc_body(workout):
    steps = parse_workout(workout)

    body = ''
    start = 0
    for step in steps:
        duration = float(step[0])
        if range_to_zones.has_key(step[1]):
            zones = range_to_zones[step[1]]
            body += body_block(start, duration, zones[0], zones[1])
            start += duration
        else:
            body += body_block(start, duration, step[1], step[1])
            start += duration
    return body

def generate_mrc(name, workout):
    body = ''
    body += mrc_header
    body += generate_mrc_body(workout)
    body += mrc_trailer
    return body

def main():
    parser = argparse.ArgumentParser(description="MRC File Generator")
    parser.add_argument('-v', '--verbose', action='store_true',
                        help="Verbose output")
    parser.add_argument('workout', type=argparse.FileType('r'))

    args = parser.parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.INFO)

    mrc = generate_mrc('Sample', args.workout.read())
    print mrc

if __name__ == "__main__":
    main()

<commit_msg>Refactoring MRC generator into a class<commit_after>#!/usr/bin/env python

import re
import sys
import argparse
import logging

mrc_header = """[COURSE HEADER]
VERSION = 2
UNITS = ENGLISH
DESCRIPTION = Sample
FILE NAME = sample.mrc
MINUTES PERCENT
[END COURSE HEADER]
[COURSE DATA]
"""

mrc_trailer = """[END COURSE DATA]"""

zone_to_percent = {
    'lz1':45, 'z1':50, 'hz1':55,
    'lz2':58, 'z2':65, 'hz2':73,
    'lz3':78, 'z3':82, 'hz3':88,
    'lz4':93, 'z4':98, 'hz4':103,
    'lz5':107, 'z5':108, 'hz5':118,
    'z5+':130, 'max':150
    }

range_to_zones = {
    'z1>z2':('z1','z2'),
    'z1>z3':('z1','z3'),
    'z1>z4':('z1','z4'),
    'z2>z1':('z2','z1'),
    'z3>z1':('z3','z1')
    }

class WorkoutParser:
    def __init__(self, workout):
        self.workout = workout
        self.sm = {
            'SINGLE_STEP': self.parse_single_step,
            'REPEAT_BLOCK': self.parse_repeat_block
        }
        self.cur_state = 'SINGLE_STEP'
        self.steps = []
        self.repeat_count = 0
        self.torepeat = []

    def parse(self):
        lines = [line.lower() for line in self.workout.split('\n') if line]
        for line in lines:
            if re.search('^#', line):
                continue
            self.parseline(line)
        return self.steps

    def parseline(self, line):
        self.sm[self.cur_state](line)

    def parse_single_step(self, line):
        mobj = re.search('repeat\s+([\d]+):', line)
        if mobj:
            self.repeat_count = int(mobj.group(1))
            self.cur_state = 'REPEAT_BLOCK'
            return

        mobj = re.search('^([\d.]+)\s+([\w><]+)', line)
        if mobj:
            self.steps.append((mobj.group(1), mobj.group(2)))

    def parse_repeat_block(self, line):
        mobj = re.search('\s+([\d.]+)\s+([\w><]+)', line)
        if mobj:
            self.torepeat.append((mobj.group(1), mobj.group(2)))
            return

        mobj = re.search('^([\d.]+)\s+([\w><]+)', line)
        if mobj:
            self.insert_repeat_steps()
            self.cur_state = 'SINGLE_STEP'
            self.parse_single_step(line)

    def insert_repeat_steps(self):
        self.steps += self.torepeat * self.repeat_count
        self.torepeat = []
        self.repeat_count = 0

class MrcGenerator:
    def __init__(self, workout):
        self.body = ''
        self.workout = workout

    def generate(self, workout):
        self.generate_header()
        self.generate_body()
        self.generate_trailer()
        return self.body

    def generate_header(self):
        self.body += mrc_header

    def generate_trailer(self):
        self.body += mrc_trailer

    def generate_body(self):
        start = 0
        for step in self.workout:
            duration = float(step[0])
            if range_to_zones.has_key(step[1]):
                zones = range_to_zones[step[1]]
                self.body_block(start, duration, zones[0], zones[1])
                start += duration
            else:
                self.body_block(start, duration, step[1], step[1])
                start += duration

    def body_line(self, start, percent):
        return "%0.2f\t%d\n" % (start, percent)

    def body_block(self, start, duration, start_zone, end_zone):
        self.body += self.body_line(start, zone_to_percent[start_zone])
        self.body += self.body_line(start+duration, zone_to_percent[end_zone])

def generate_mrc(name, workout):
    p = WorkoutParser(workout)
    g = MrcGenerator(p.parse())
    return g.generate(workout)

def main():
    parser = argparse.ArgumentParser(description="MRC File Generator")
    parser.add_argument('-v', '--verbose', action='store_true',
                        help="Verbose output")
    parser.add_argument('workout', type=argparse.FileType('r'))

    args = parser.parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.INFO)

    mrc = generate_mrc('Sample', args.workout.read())
    print mrc

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# mtm = myke's 'time manager
# 2016-04-11 1.29

# use:
# mkm <cmd> params
# writes to file $MTM (or ./mtm.log in current dir if $MTM is absent):
#     dt args
# commands: on off slept gone back stat
# see full docs below
# may be launched at computer start/ stop? restart
# maybe electron/ nw.js versioon will be added
# text stats + graph reports to be done

import sys, datetime, os
# pprint

version = "2016-04-11 1.29"

dt = str(datetime.datetime.now())[:-7]
dtdir = dt[:7]  # just YYYY-MM
fout = os.getenv('MTM', os.getcwd()) + "/mtm-" + dtdir + '.log'

LASTS = 20  # lines to show by 'last' command

# categories and keywords
grocc = {
    "status":     "on off out away with at" .split(),
    "state":      "up down ill well" .split(),
    "busy":       "busy meet conf work prog study trans" .split(),
    "home":       "home shop life talk build child wash clean self doctor" .split(),
    "arts":       "art book bards concert museum cinema theater" .split(),
    "rest":       "eat tea coffee drink read write walk bed sleep" .split(),
    "exter":      "chat fun tv inet watch" .split(),
    "active":     "travel sport byke hike boat drive ride" .split(),
    "result":     "stat report" .split(),
    "info":       "? help query log last" .split()
}

occs = []
for x in grocc.values():
    occs.extend(x)
occs.sort()

# how we print them when in text mode; others aren't printed or are shown in other way
todisplay = {"busy": "b", "home": "h", "arts": "a", "rest": "r", "exter": "x", "active": "v"}

def help():
    """ print list of options """
    loks = sorted(grocc.keys())
    for ok in loks:
        print (ok + ":", end="")
        tou = 0
        for ov in sorted(grocc[ok]):
            tou += 1
            if tou % 10 == 1:
                print ("\n\t", end="")
            print (ov + ", ", end="")
        print ()

def main(args):
    """ main routine """
    print ("This is MTM myke's Time Management routine ver. {}" .format (version))
    print ("Log goes to", fout)
    #~ print (dt, sys.argv, fout)

    if len(sys.argv) > 1 and sys.argv[1] in occs:

        if sys.argv[1] == "?":
            sys.argv[1] = "help"

        cmd = sys.argv[1]

        # print entire file
        if cmd == "log":
            with open (fout) as flog:
                txt = flog.read()
                print (txt)
            return

        # print last lines
        if cmd == "last":
            amt = LASTS
            try:
                if len(sys.argv) == 3:
                    amt = int(sys.argv[2])
            except:
                amt = 10
            with open (fout) as flog:
                txt = flog.readlines()
            print ("showing {} last lines" .format(amt))
            for lino in txt[-amt:]:
                print (lino, end="")
            return

        # perform log record
        print (dt, *sys.argv[1:])

        with open (fout, 'a') as mtm:
            print (dt, *sys.argv[1:], file=mtm)

    else:
        # default output w/o parameters
        print ("available commands are:")
        #~ pprint.pprint (grocc)
        help()

    return 0

if __name__ == '__main__':
    import sys
    sys.exit(main(sys.argv))
<commit_msg>better help correct actions list<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# mtm = myke's 'time manager
# 2016-04-11 1.30

# use:
# mkm <cmd> params
# writes to file $MTM (or ./mtm.log in current dir if $MTM is absent):
#     dt args
# commands: on off slept gone back stat
# see full docs below
# may be launched at computer start/ stop? restart
# maybe electron/ nw.js versioon will be added
# text stats + graph reports to be done

import sys, datetime, os
# pprint

version = "2016-04-11 1.30"

dt = str(datetime.datetime.now())[:-7]
dtdir = dt[:7]  # just YYYY-MM
fout = os.getenv('MTM', os.getcwd()) + "/mtm-" + dtdir + '.log'

LASTS = 20  # lines to show by 'last' command

# categories and keywords
grocc = {
    "status":     "on off out away with at" .split(),
    "state":      "up down ill well" .split(),
    "busy":       "busy meet conf work prog study trans" .split(),
    "home":       "home shop life talk build child wash clean self doctor" .split(),
    "arts":       "art book bards concert museum cinema theater" .split(),
    "rest":       "eat tea coffee drink read write walk bed sleep" .split(),
    "exter":      "chat fun tv inet watch" .split(),
    "active":     "travel sport ski byke run hike boat drive ride" .split(),
    "result":     "stat report" .split(),
    "info":       "? help query log last" .split()
}

occs = []
for x in grocc.values():
    occs.extend(x)
occs.sort()

# how we print them when in text mode; others aren't printed or are shown in other way
todisplay = {"busy": "b", "home": "h", "arts": "a", "rest": "r", "exter": "x", "active": "v"}

def help():
    """ print list of options """
    loks = sorted(grocc.keys())
    for ok in loks:
        print (ok + ":", end="")
        tou = 0
        for ov in sorted(grocc[ok]):
            tou += 1
            if tou % 10 == 1:
                print ("\n\t", end="")
            print (ov + ", ", end="")
        print ()

def main(args):
    """ main routine """
    print ("This is MTM myke's Time Management routine ver. {}" .format (version))
    print ("Log goes to", fout)
    #~ print (dt, sys.argv, fout)

    if len(sys.argv) > 1 and sys.argv[1] in occs:

        if sys.argv[1] == "?":
            sys.argv[1] = "help"

        cmd = sys.argv[1]

        # print entire file
        if cmd == "log":
            with open (fout) as flog:
                txt = flog.read()
                print (txt)
            return

        # print last lines
        if cmd == "last":
            amt = LASTS
            try:
                if len(sys.argv) == 3:
                    amt = int(sys.argv[2])
            except:
                amt = 10
            with open (fout) as flog:
                txt = flog.readlines()
            print ("showing {} last lines" .format(amt))
            for lino in txt[-amt:]:
                print (lino, end="")
            return

        # perform log record
        print (dt, *sys.argv[1:])

        with open (fout, 'a') as mtm:
            print (dt, *sys.argv[1:], file=mtm)

    else:
        # default output w/o parameters
        print ("available commands are:")
        #~ pprint.pprint (grocc)
        help()

    return 0

if __name__ == '__main__':
    import sys
    sys.exit(main(sys.argv))
<|endoftext|>


<commit_before>class Multiplication():
    def __init__(self, first, second, rows, answer, env=None):
        assert len(second) == len(rows)
        if env is None:
            self.first = list(first)
            self.second = list(second)
            self.rows = [list(row) for row in rows]
            self.answer = list(answer)
        else:
            f = self.get_full_row_int(first, env["first"])
            s = self.get_full_row_int(second, env["second"])
            self.first = list(str(f))
            self.second = list(str(s))
            self.rows = []
            for c in reversed(str(s)):
                self.rows.append(list(str(int(c) * f)))
            self.answer = list(str(f * s))
        self.nb_col = len(self.answer)

    def __str__(self):
        """
        Returns:
            multiplication in str

        Examples:
            >>> mul = Multiplication('x7x', 'x5', ['xx9x', '1x56'], '16xxx')
            >>> print mul
                       X01    7  X02
                            X03    5
            -------------------------
                  X04  X05    9  X06
               1  X07    5    6
            -------------------------
               1    6  X08  X09  X10
        """
        s = ''
        counter = 0
        row_str, counter = self.get_row_str(self.first, counter, 0)
        s += row_str
        row_str, counter = self.get_row_str(self.second, counter, 0)
        s += row_str
        s += (5 * self.nb_col) * "-"
        s += '\n'
        for i, row in enumerate(self.rows):
            row_str, counter = self.get_row_str(row, counter, i)
            s += row_str
        s += (5 * self.nb_col) * "-"
        s += '\n'
        row_str, counter = self.get_row_str(self.answer, counter, 0)
        s += row_str
        return s

    def get_row_str(self, row, counter, shift):
        """
        Args:
            row: a list of characters consisting of '0', '1', ... '9', or 'x'
            counter: the number of 'x'es in rows above
            shift: the length of padding in the left

        Returns:
            s: row converted to str
            counter: the number of encountered 'x'es so far
        """
        s = ''
        row_str = ['' for _ in range(self.nb_col - len(row) - shift)] + row + list(shift * '')
        for c in row_str:
            if c == 'x':
                counter += 1
                s += '{:>4}'.format("X" + "{:02}".format(counter))
            else:
                s += '{:>4}'.format(c)
            s += ' '
        s += '\n'
        return s, counter

    def configurable_env(self):
        """
        Returns:
            a hash h where h["first" or "second"][an index at which "first" or "second" row is 'x'] is None
        """
        return {"first": {i: None for i, c in enumerate(self.first) if c == 'x'},
                "second": {i: None for i, c in enumerate(self.second) if c == 'x'}}

    @staticmethod
    def get_full_row_int(row, row_env):
        """
        Args:
            row: a list of characters consisting of '0', '1', ... '9', or 'x'
            row_env: environment that substitutes 'x' in row

        Returns:
            the largest available integer available in the row with row_env applied.

        Raises:
            ValueError: if 'x' in row is not supplemented by row_env and unknown
        """
        a = 0
        for i, c in enumerate(row):
            a *= 10
            if c == 'x':
                if row_env[i] is not None:
                    a += row_env[i]
                else:
                    raise ValueError("Evaluating a row with an incomplete environment.")
            else:
                a += int(c)
        return a

    @staticmethod
    def get_partial_row_int(row, row_env):
        """
        Args:
            row: a list of characters consisting of '0', '1', ... '9', or 'x'
            row_env: environment that substitutes 'x' in row

        Returns:
            the largest available integer available in the row with row_env applied.

        Examples:
            >>> mul.get_partial_row_int(['x', '7', 'x'], {0: None, 2: None})
            None

            >>> mul.get_partial_row_int(['x', '7', 'x'], {0: None, 2: 5}
            75

            >>> mul.get_partial_row_int(['x', '7', 'x'], {0: 1, 2: 5}
            175
        """
        if row[-1] == 'x' and row_env[len(row)-1] is None:
            return None
        a = 0
        for i, c in enumerate(row):
            a *= 10
            if c == 'x':
                if row_env[i] is not None:
                    a += row_env[i]
                else:
                    a = 0
            else:
                a += int(c)
        return a

    @staticmethod
    def match_rows(r1, r2):
        """
        Args:
            r1: a list of characters consisting of '0', '1', ... '9', or 'x'
            r2: a list of characters consisting of '0', '1', ... '9'

        Returns:
            True if r1 and r2 matches, False otherwise
        """
        for c1, c2 in zip(r1, r2):
            if c1 == 'x' or c1 == c2:
                continue
            else:
                return False
        return True

    def is_sufficient(self, env):
        """
        Args:
            env: a hash environment

        Returns:
            False if "nogood" True otherwise
        """
        if env["first"].get(0) == 0 or env["second"].get(0) == 0:
            return False

        first_row = self.get_partial_row_int(self.first, env["first"])
        if first_row is None:
            return True

        for i, a in enumerate(reversed(self.second)):
            if a == 'x':
                if env["second"][len(self.second)-i-1] is None:
                    continue
                else:
                    a = env["second"][len(self.second)-i-1]
            else:
                a = int(a)
            row = str(first_row * a)
            if len(str(first_row)) == len(self.first):
                if len(row) != len(self.rows[i]):
                    return False
                if not self.match_rows(self.rows[i][-len(row):], row):
                    return False
            else:
                # The following line extract the only relevant part of the row.
                # If first_row is 70 and a is 5, then row evaluates to 350.
                # The leftmost digit in row 3, however, is a carry out,
                # and isn't an actual result of multiplication, and thus dropped.
                relevant = row[-len(str(first_row)):]
                if not self.match_rows(self.rows[i][-len(relevant):], relevant):
                    return False

        if all([v is not None for v in env["first"].values()]) and \
                all([v is not None for v in env["second"].values()]):
            first = self.get_full_row_int(self.first, env["first"])
            second = self.get_full_row_int(self.second, env["second"])
            answer = str(first * second)
            for c1, c2 in zip(answer, self.answer):
                if c2 == 'x':
                    continue
                elif c1 != c2:
                    return False
        return True
<commit_msg>Utilizes match_rows method for comparing rows<commit_after>class Multiplication():
    def __init__(self, first, second, rows, answer, env=None):
        assert len(second) == len(rows)
        if env is None:
            self.first = list(first)
            self.second = list(second)
            self.rows = [list(row) for row in rows]
            self.answer = list(answer)
        else:
            f = self.get_full_row_int(first, env["first"])
            s = self.get_full_row_int(second, env["second"])
            self.first = list(str(f))
            self.second = list(str(s))
            self.rows = []
            for c in reversed(str(s)):
                self.rows.append(list(str(int(c) * f)))
            self.answer = list(str(f * s))
        self.nb_col = len(self.answer)

    def __str__(self):
        """
        Returns:
            multiplication in str

        Examples:
            >>> mul = Multiplication('x7x', 'x5', ['xx9x', '1x56'], '16xxx')
            >>> print mul
                       X01    7  X02
                            X03    5
            -------------------------
                  X04  X05    9  X06
               1  X07    5    6
            -------------------------
               1    6  X08  X09  X10
        """
        s = ''
        counter = 0
        row_str, counter = self.get_row_str(self.first, counter, 0)
        s += row_str
        row_str, counter = self.get_row_str(self.second, counter, 0)
        s += row_str
        s += (5 * self.nb_col) * "-"
        s += '\n'
        for i, row in enumerate(self.rows):
            row_str, counter = self.get_row_str(row, counter, i)
            s += row_str
        s += (5 * self.nb_col) * "-"
        s += '\n'
        row_str, counter = self.get_row_str(self.answer, counter, 0)
        s += row_str
        return s

    def get_row_str(self, row, counter, shift):
        """
        Args:
            row: a list of characters consisting of '0', '1', ... '9', or 'x'
            counter: the number of 'x'es in rows above
            shift: the length of padding in the left

        Returns:
            s: row converted to str
            counter: the number of encountered 'x'es so far
        """
        s = ''
        row_str = ['' for _ in range(self.nb_col - len(row) - shift)] + row + list(shift * '')
        for c in row_str:
            if c == 'x':
                counter += 1
                s += '{:>4}'.format("X" + "{:02}".format(counter))
            else:
                s += '{:>4}'.format(c)
            s += ' '
        s += '\n'
        return s, counter

    def configurable_env(self):
        """
        Returns:
            a hash h where h["first" or "second"][an index at which "first" or "second" row is 'x'] is None
        """
        return {"first": {i: None for i, c in enumerate(self.first) if c == 'x'},
                "second": {i: None for i, c in enumerate(self.second) if c == 'x'}}

    @staticmethod
    def get_full_row_int(row, row_env):
        """
        Args:
            row: a list of characters consisting of '0', '1', ... '9', or 'x'
            row_env: environment that substitutes 'x' in row

        Returns:
            the largest available integer available in the row with row_env applied.

        Raises:
            ValueError: if 'x' in row is not supplemented by row_env and unknown
        """
        a = 0
        for i, c in enumerate(row):
            a *= 10
            if c == 'x':
                if row_env[i] is not None:
                    a += row_env[i]
                else:
                    raise ValueError("Evaluating a row with an incomplete environment.")
            else:
                a += int(c)
        return a

    @staticmethod
    def get_partial_row_int(row, row_env):
        """
        Args:
            row: a list of characters consisting of '0', '1', ... '9', or 'x'
            row_env: environment that substitutes 'x' in row

        Returns:
            the largest available integer available in the row with row_env applied.

        Examples:
            >>> mul.get_partial_row_int(['x', '7', 'x'], {0: None, 2: None})
            None

            >>> mul.get_partial_row_int(['x', '7', 'x'], {0: None, 2: 5}
            75

            >>> mul.get_partial_row_int(['x', '7', 'x'], {0: 1, 2: 5}
            175
        """
        if row[-1] == 'x' and row_env[len(row)-1] is None:
            return None
        a = 0
        for i, c in enumerate(row):
            a *= 10
            if c == 'x':
                if row_env[i] is not None:
                    a += row_env[i]
                else:
                    a = 0
            else:
                a += int(c)
        return a

    @staticmethod
    def match_rows(r1, r2, full=False):
        """
        Args:
            r1: a list of characters consisting of '0', '1', ... '9', or 'x'
            r2: a list of characters consisting of '0', '1', ... '9'
            full: if True, r1 and r2 have to be fully matched

        Returns:
            True if r1 and r2 matches, False otherwise
        """
        if full and len(r1) != len(r2):
            return False
        for c1, c2 in zip(r1, r2):
            if c1 == 'x' or c1 == c2:
                continue
            else:
                return False
        return True

    def is_sufficient(self, env):
        """
        Args:
            env: a hash environment

        Returns:
            False if "nogood" True otherwise
        """
        if env["first"].get(0) == 0 or env["second"].get(0) == 0:
            return False

        first_row = self.get_partial_row_int(self.first, env["first"])
        if first_row is None:
            return True

        for i, a in enumerate(reversed(self.second)):
            if a == 'x':
                if env["second"][len(self.second)-i-1] is None:
                    continue
                else:
                    a = env["second"][len(self.second)-i-1]
            else:
                a = int(a)
            row = str(first_row * a)
            if len(str(first_row)) == len(self.first):
                if not self.match_rows(self.rows[i], row, full=True):
                    return False
            else:
                # The following line extract the only relevant part of the row.
                # If first_row is 70 and a is 5, then row evaluates to 350.
                # The leftmost digit in row 3, however, is a carry out,
                # and isn't an actual result of multiplication, and thus dropped.
                relevant = row[-len(str(first_row)):]
                if not self.match_rows(self.rows[i][-len(relevant):], relevant):
                    return False

        if all([v is not None for v in env["first"].values()]) and \
                all([v is not None for v in env["second"].values()]):
            first = self.get_full_row_int(self.first, env["first"])
            second = self.get_full_row_int(self.second, env["second"])
            answer = str(first * second)
            if not self.match_rows(self.answer, answer, full=True):
                return False
        return True
<|endoftext|>


<commit_before>#!/usr/bin/python3
#
# Email: Hagen Paul Pfeifer <hagen@jauu.net>

import sys
import os
import optparse
import subprocess
import pprint
import re
import ctypes
import math
import time
import importlib.util
import json
import subprocess

pp = pprint.PrettyPrinter(indent=4)

__programm__ = "net-applet-shuffler"
__version__  = "1"

class Printer:

    STD = 0
    VERBOSE = 1

    def __init__(self, verbose=False):
        self.verbose = verbose

    def err(self, msg):
        sys.stderr.write(msg)

    def msg(self, msg, level=VERBOSE, underline=False):
        if level == VERBOSE and self.verbose == False:
            return
        if underline == False:
            return sys.stdout.write(msg) - 1
        else:
            str_len = len(msg)
            sys.stdout.write(msg)
            self.msg("\n" + '=' * str_len + "\n")

class Ssh():

    def __init__(self):
        pass

    def exec(self, ip, user, cmd):
        full = "ssh {}@{} sudo {}".format(user, ip, cmd)
        p = subprocess.Popen(full.split(), stdout=subprocess.PIPE)
        stdout, stderr = p.communicate()
        return stdout, stderr, p.returncode

    def copy(self, remote_user, remote_ip, remote_path, local_path, to_local):
        command = str()
        if to_local:
            command = "scp {}@{}:{} {}".format(remote_user, remote_ip,
                                               remote_path, local_path)
        else:
            command = "scp {} {}@{}:{}".format(local_path, remote_user,
                                               remote_ip, remote_path)
        p = subprocess.Popen(command.split(), stdout=subprocess.PIPE)
        stdout, stderr = p.communicate()
        return stdout, stderr, p.returncode

class Exchange():

    def __init__(self):
        self.ssh = Ssh()

    def ping(self, host):
        cmd = "ping -c 1 {} 1>/dev/null 2>&1 ".format(host)
        response = os.system(cmd)
        if response == 0:
            return True
        return False

class AppletExecuter():

    def __init__(self, external_controlled=False, verbose=False):
        self.verbose = verbose
        self.applet_name = False
        self.p = Printer(verbose=self.verbose)
        if not external_controlled:
            self.parse_local_options()
        self.load_conf()

    def applet_path(self, name):
        hp = os.path.dirname(os.path.realpath(__file__))
        fp = os.path.join(hp, "applets", name)
        if not os.path.exists(fp):
            return None, False
        ffp = os.path.join(fp, "applet.py")
        return ffp, True

    def load_conf(self):
        hp = os.path.dirname(os.path.realpath(__file__))
        fp = os.path.join(hp, "conf.json")
        with open(fp, 'r') as fd:
            data = fd.read()
            self.conf = json.loads(data)

    def parse_local_options(self):
        parser = optparse.OptionParser()
        parser.usage = "Executer"
        parser.add_option( "-v", "--verbose", dest="verbose", default=False,
                          action="store_true", help="show verbose")
        self.opts, args = parser.parse_args(sys.argv[0:])

        if len(args) < 3:
            self.p.err("No <applet> argument given, exiting\n")
            sys.exit(1)

        if self.opts.verbose:
            self.p.set_verbose()
        self.applet_name = args[2]
        self.applet_args = args[3:]

    def set_applet_data(self, applet_name, applet_args):
        self.applet_name = applet_name
        self.applet_args = applet_args

    def import_applet_module(self):
        ffp, ok = self.applet_path(self.applet_name)
        if not ok:
            self.p.err("Applet ({}) not avaiable, call list\n".format(self.applet_name))
            sys.exit(1)

        spec = importlib.util.spec_from_file_location("applet", ffp)
        self.applet = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(self.applet)

    def run(self):
        if not self.applet_name:
            print("You want to execute an applet but I don't know what applet!")
            print("This is an internal error, see self.external_controlled")
            return False
        self.import_applet_module()
        xchange = Exchange()
        xchange.p = self.p
        # the status is used later for campaigns:
        # if the status is false the campaing must be
        # stopped, if true everything is fine!
        print("  execute applet \"{} {}\"".format(self.applet_name, self.applet_args))
        status = self.applet.main(xchange, self.conf, self.applet_args)
        if status == True:
            return True
        elif status == False:
            return False
        else:
            print("Applet defect: MUST return True or False")
            return False

class AppletLister():

    def __init__(self, verbose=False):
        self.verbose = verbose
        self.p = Printer(verbose=self.verbose)

    def subdirs(self, d):
        return [name for name in os.listdir(d) if os.path.isdir(os.path.join(d, name))]

    def run(self):
        hp = os.path.dirname(os.path.realpath(__file__))
        fp = os.path.join(hp, "applets")
        dirs = self.subdirs(fp)
        sys.stdout.write("Available applets:\n")
        for d in dirs:
            sys.stdout.write("  {}\n".format(d))

class CampaignExecuter():

    OPCODE_CMD_EXEC = 1
    OPCODE_CMD_SLEEP = 2

    def __init__(self, verbose=False):
        self.verbose = verbose
        self.p = Printer(verbose=self.verbose)
        self.parse_local_options()

    def campaign_path(self, name):
        hp = os.path.dirname(os.path.realpath(__file__))
        fp = os.path.join(hp, "campaigns", name)
        if not os.path.exists(fp):
            print("Campaign path \"{}\" do not exist".format(fp))
            return None, False
        ffp = os.path.join(fp, "run.cmd")
        return ffp, True

    def parse_local_options(self):
        parser = optparse.OptionParser()
        parser.usage = "Executer"
        parser.add_option( "-v", "--verbose", dest="verbose", default=False,
                          action="store_true", help="show verbose")
        self.opts, args = parser.parse_args(sys.argv[0:])

        if len(args) < 3:
            self.p.err("No <campaign> argument given, exiting\n")
            sys.exit(1)

        if self.opts.verbose:
            self.p.set_verbose()
        self.campaign_name = args[2]

    def execute_applet(self, applet_name, applet_args):
        app_executer = AppletExecuter(external_controlled=True)
        app_executer.set_applet_data(applet_name, applet_args)
        return app_executer.run()

    def read_campaign_file(self, path):
        data = list()
        with open(path, "rt") as fd:
            while True:
                line = fd.readline()
                if not line:
                    break
                line = line.strip()
                if not line:
                    # ignore lines with only withspaces
                    continue
                if line.startswith("#"):
                    # ignore comment lines
                    continue
                data.append(line)
        return data

    def transform_exec_statement(self, chunks, ret):
        d = dict()
        if not len(chunks) > 0:
            # need a second value
            return
        d['cmd'] = CampaignExecuter.OPCODE_CMD_EXEC
        d['name'] = chunks[0]
        d['args'] = chunks[1:]
        ret.append(d)

    def transform_sleep_statement(self, chunks, ret):
        d = dict()
        if not len(chunks) > 0:
            # need a second value
            return
        d['cmd'] = CampaignExecuter.OPCODE_CMD_SLEEP
        d['time'] = chunks[0]
        ret.append(d)

    def transform(self, content):
        data = list()
        for c in content:
            chunks = c.split()
            if len(chunks) < 1:
                print("Command ({}) not known, only exec and sleep allowed".format(c))
                return None, False
            if chunks[0] == "exec":
                self.transform_exec_statement(chunks[1:], data)
            elif chunks[0] == "sleep":
                self.transform_sleep_statement(chunks[1:], data)
            else:
                print("Command \"{}\" not known, only exec and sleep allowed".format(chunks[0]))
                return None, False
        return data, True

    def execute_campaign(self, data):
        for d in data:
            ret = True
            if d['cmd'] == CampaignExecuter.OPCODE_CMD_EXEC:
                ret = self.execute_applet(d['name'], d['args'])
            if d['cmd'] == CampaignExecuter.OPCODE_CMD_SLEEP:
                time.sleep(int(d['time']))
            if ret == False:
                print("Applet returned negative return code, stop campaign now")
                return

    def parse_campaign(self):
        path, ok = self.campaign_path(self.campaign_name)
        if not ok:
            print("Campaign \"{}\" not valid".format(self.campaign_name))
            return None, False
        file_data = self.read_campaign_file(path)
        return self.transform(file_data)

    def run(self):
        data, ok = self.parse_campaign()
        if not ok:
            sys.exit(1)
        print("Execute Campaign \"{}\"".format(self.campaign_name))
        self.execute_campaign(data)

class CampaignLister():

    def __init__(self, verbose=False):
        self.verbose = verbose
        self.p = Printer(verbose=self.verbose)

    def subdirs(self, d):
        return [name for name in os.listdir(d) if os.path.isdir(os.path.join(d, name))]

    def run(self):
        hp = os.path.dirname(os.path.realpath(__file__))
        fp = os.path.join(hp, "campaigns")
        dirs = self.subdirs(fp)
        sys.stdout.write("Available campaigns:\n")
        for d in dirs:
            sys.stdout.write("  {}\n".format(d))

class NetAppletShuffler:

    modes = {
       "exec-applet":    [ "AppletExecuter",   "Execute applets" ],
       "list-applets":   [ "AppletLister",     "List all applets" ],
       "exec-campaign":  [ "CampaignExecuter", "Execute campaign" ],
       "list-campaigns": [ "CampaignLister",   "List all campaigns" ]
            }

    def __init__(self):
        self.verbose = False

    def which(self, program):
        for path in os.environ["PATH"].split(os.pathsep):
            path = path.strip('"')
            full_path = os.path.join(path, program)
            if os.path.isfile(full_path) and os.access(full_path, os.X_OK):
                return full_path
        return None

    def print_version(self):
        sys.stdout.write("%s\n" % (__version__))

    def print_usage(self):
        sys.stderr.write("Usage: nas [-h | --help]" +
                         " [--version]" +
                         " <applet> [<applet-options>]\n")

    def print_modules(self):
        for i in NetAppletShuffler.modes.keys():
            sys.stderr.write("   %-15s - %s\n" % (i, NetAppletShuffler.modes[i][1]))

    def args_contains(self, argv, *cmds):
        for cmd in cmds:
            for arg in argv:
                if arg == cmd: return True
        return False

    def parse_global_otions(self):
        if len(sys.argv) <= 1:
            self.print_usage()
            sys.stderr.write("Available applets:\n")
            self.print_modules()
            return None

        self.binary_path = sys.argv[-1]

        # --version can be placed somewhere in the
        # command line and will evalutated always: it is
        # a global option
        if self.args_contains(sys.argv, "--version"):
            self.print_version()
            return None

        # handle verbose flag
        if "--verbose" in sys.argv:
            sys.argv.remove("--verbose")
            self.verbose = True
        if "-v" in sys.argv:
            self.verbose = True
            sys.argv.remove("-v")

        # -h | --help as first argument is treated special
        # and has other meaning as a submodule
        if self.args_contains(sys.argv[1:2], "-h", "--help"):
            self.print_usage()
            sys.stderr.write("Available modules:\n")
            self.print_modules()
            return None

        submodule = sys.argv[1].lower()
        if submodule not in NetAppletShuffler.modes:
            self.print_usage()
            sys.stderr.write("Modules \"%s\" not known, available modules are:\n" %
                             (submodule))
            self.print_modules()
            return None

        classname = NetAppletShuffler.modes[submodule][0]
        return classname

    def run(self):
        classtring = self.parse_global_otions()
        if not classtring:
            return 1

        classinstance = globals()[classtring](verbose=self.verbose)
        ok = classinstance.run()
        if not ok:
            return 1
        return 0

if __name__ == "__main__":
    try:
        mca = NetAppletShuffler()
        sys.exit(mca.run())
    except KeyboardInterrupt:
        sys.stderr.write("SIGINT received, exiting\n")
<commit_msg>migrate print/write calls to internal msg function<commit_after>#!/usr/bin/python3
#
# Email: Hagen Paul Pfeifer <hagen@jauu.net>

import sys
import os
import optparse
import subprocess
import pprint
import re
import ctypes
import math
import time
import importlib.util
import json
import subprocess

pp = pprint.PrettyPrinter(indent=4)

__programm__ = "net-applet-shuffler"
__version__  = "1"

class Printer:

    STD = 0
    VERBOSE = 1

    def __init__(self, verbose=False):
        self.verbose = verbose

    def err(self, msg):
        sys.stderr.write(msg)

    def msg(self, msg, level=VERBOSE, underline=False):
        if level == Printer.VERBOSE and self.verbose == False:
            return
        if underline == False:
            return sys.stdout.write(msg) - 1
        else:
            str_len = len(msg)
            sys.stdout.write(msg)
            self.msg("\n" + '=' * str_len + "\n")

class Ssh():

    def __init__(self):
        pass

    def exec(self, ip, user, cmd):
        full = "ssh {}@{} sudo {}".format(user, ip, cmd)
        p = subprocess.Popen(full.split(), stdout=subprocess.PIPE)
        stdout, stderr = p.communicate()
        return stdout, stderr, p.returncode

    def copy(self, remote_user, remote_ip, remote_path, local_path, to_local):
        command = str()
        if to_local:
            command = "scp {}@{}:{} {}".format(remote_user, remote_ip,
                                               remote_path, local_path)
        else:
            command = "scp {} {}@{}:{}".format(local_path, remote_user,
                                               remote_ip, remote_path)
        p = subprocess.Popen(command.split(), stdout=subprocess.PIPE)
        stdout, stderr = p.communicate()
        return stdout, stderr, p.returncode

class Exchange():

    def __init__(self):
        self.ssh = Ssh()

    def ping(self, host):
        cmd = "ping -c 1 {} 1>/dev/null 2>&1 ".format(host)
        response = os.system(cmd)
        if response == 0:
            return True
        return False

class AppletExecuter():

    def __init__(self, external_controlled=False, verbose=False):
        self.verbose = verbose
        self.applet_name = False
        self.p = Printer(verbose=self.verbose)
        if not external_controlled:
            self.parse_local_options()
        self.load_conf()

    def applet_path(self, name):
        hp = os.path.dirname(os.path.realpath(__file__))
        fp = os.path.join(hp, "applets", name)
        if not os.path.exists(fp):
            return None, False
        ffp = os.path.join(fp, "applet.py")
        return ffp, True

    def load_conf(self):
        hp = os.path.dirname(os.path.realpath(__file__))
        fp = os.path.join(hp, "conf.json")
        with open(fp, 'r') as fd:
            data = fd.read()
            self.conf = json.loads(data)

    def parse_local_options(self):
        parser = optparse.OptionParser()
        parser.usage = "Executer"
        parser.add_option( "-v", "--verbose", dest="verbose", default=False,
                          action="store_true", help="show verbose")
        self.opts, args = parser.parse_args(sys.argv[0:])

        if len(args) < 3:
            self.p.err("No <applet> argument given, exiting\n")
            sys.exit(1)

        if self.opts.verbose:
            self.p.set_verbose()
        self.applet_name = args[2]
        self.applet_args = args[3:]

    def set_applet_data(self, applet_name, applet_args):
        self.applet_name = applet_name
        self.applet_args = applet_args

    def import_applet_module(self):
        ffp, ok = self.applet_path(self.applet_name)
        if not ok:
            self.p.err("Applet ({}) not avaiable, call list\n".format(self.applet_name))
            sys.exit(1)

        spec = importlib.util.spec_from_file_location("applet", ffp)
        self.applet = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(self.applet)

    def run(self):
        if not self.applet_name:
            print("You want to execute an applet but I don't know what applet!")
            print("This is an internal error, see self.external_controlled")
            return False
        self.import_applet_module()
        xchange = Exchange()
        xchange.p = self.p
        # the status is used later for campaigns:
        # if the status is false the campaing must be
        # stopped, if true everything is fine!
        self.p.msg("  execute applet \"{} {}\"".format(self.applet_name, self.applet_args),                   level=Printer.STD)
        status = self.applet.main(xchange, self.conf, self.applet_args)
        if status == True:
            return True
        elif status == False:
            return False
        else:
            print("Applet defect: MUST return True or False")
            return False

class AppletLister():

    def __init__(self, verbose=False):
        self.verbose = verbose
        self.p = Printer(verbose=self.verbose)

    def subdirs(self, d):
        return [name for name in os.listdir(d) if os.path.isdir(os.path.join(d, name))]

    def run(self):
        hp = os.path.dirname(os.path.realpath(__file__))
        fp = os.path.join(hp, "applets")
        dirs = self.subdirs(fp)
        self.p.msg("Available applets:\n", level=Printer.STD)
        for d in dirs:
            self.p.msg("  {}\n".format(d), level=Printer.STD)

class CampaignExecuter():

    OPCODE_CMD_EXEC = 1
    OPCODE_CMD_SLEEP = 2

    def __init__(self, verbose=False):
        self.verbose = verbose
        self.p = Printer(verbose=self.verbose)
        self.parse_local_options()

    def campaign_path(self, name):
        hp = os.path.dirname(os.path.realpath(__file__))
        fp = os.path.join(hp, "campaigns", name)
        if not os.path.exists(fp):
            self.p.msg("Campaign path \"{}\" do not exist".format(fp),
                       level=Printer.STD)
            return None, False
        ffp = os.path.join(fp, "run.cmd")
        return ffp, True

    def parse_local_options(self):
        parser = optparse.OptionParser()
        parser.usage = "Executer"
        parser.add_option( "-v", "--verbose", dest="verbose", default=False,
                          action="store_true", help="show verbose")
        self.opts, args = parser.parse_args(sys.argv[0:])

        if len(args) < 3:
            self.p.err("No <campaign> argument given, exiting\n")
            sys.exit(1)

        if self.opts.verbose:
            self.p.set_verbose()
        self.campaign_name = args[2]

    def execute_applet(self, applet_name, applet_args):
        app_executer = AppletExecuter(external_controlled=True)
        app_executer.set_applet_data(applet_name, applet_args)
        return app_executer.run()

    def read_campaign_file(self, path):
        data = list()
        with open(path, "rt") as fd:
            while True:
                line = fd.readline()
                if not line:
                    break
                line = line.strip()
                if not line:
                    # ignore lines with only withspaces
                    continue
                if line.startswith("#"):
                    # ignore comment lines
                    continue
                data.append(line)
        return data

    def transform_exec_statement(self, chunks, ret):
        d = dict()
        if not len(chunks) > 0:
            # need a second value
            return
        d['cmd'] = CampaignExecuter.OPCODE_CMD_EXEC
        d['name'] = chunks[0]
        d['args'] = chunks[1:]
        ret.append(d)

    def transform_sleep_statement(self, chunks, ret):
        d = dict()
        if not len(chunks) > 0:
            # need a second value
            return
        d['cmd'] = CampaignExecuter.OPCODE_CMD_SLEEP
        d['time'] = chunks[0]
        ret.append(d)

    def transform(self, content):
        data = list()
        for c in content:
            chunks = c.split()
            if len(chunks) < 1:
                print("Command ({}) not known, only exec and sleep allowed".format(c))
                return None, False
            if chunks[0] == "exec":
                self.transform_exec_statement(chunks[1:], data)
            elif chunks[0] == "sleep":
                self.transform_sleep_statement(chunks[1:], data)
            else:
                self.p.err("Command \"{}\" not known, only exec and sleep allowed".format(chunks[0]))
                return None, False
        return data, True

    def execute_campaign(self, data):
        for d in data:
            ret = True
            if d['cmd'] == CampaignExecuter.OPCODE_CMD_EXEC:
                ret = self.execute_applet(d['name'], d['args'])
            if d['cmd'] == CampaignExecuter.OPCODE_CMD_SLEEP:
                time.sleep(int(d['time']))
            if ret == False:
                print("Applet returned negative return code, stop campaign now")
                return

    def parse_campaign(self):
        path, ok = self.campaign_path(self.campaign_name)
        if not ok:
            print("Campaign \"{}\" not valid".format(self.campaign_name))
            return None, False
        file_data = self.read_campaign_file(path)
        return self.transform(file_data)

    def run(self):
        data, ok = self.parse_campaign()
        if not ok:
            sys.exit(1)
        print("Execute Campaign \"{}\"".format(self.campaign_name))
        self.execute_campaign(data)

class CampaignLister():

    def __init__(self, verbose=False):
        self.verbose = verbose
        self.p = Printer(verbose=self.verbose)

    def subdirs(self, d):
        return [name for name in os.listdir(d) if os.path.isdir(os.path.join(d, name))]

    def run(self):
        hp = os.path.dirname(os.path.realpath(__file__))
        fp = os.path.join(hp, "campaigns")
        dirs = self.subdirs(fp)
        sys.stdout.write("Available campaigns:\n")
        for d in dirs:
            sys.stdout.write("  {}\n".format(d))

class NetAppletShuffler:

    modes = {
       "exec-applet":    [ "AppletExecuter",   "Execute applets" ],
       "list-applets":   [ "AppletLister",     "List all applets" ],
       "exec-campaign":  [ "CampaignExecuter", "Execute campaign" ],
       "list-campaigns": [ "CampaignLister",   "List all campaigns" ]
            }

    def __init__(self):
        self.verbose = False

    def which(self, program):
        for path in os.environ["PATH"].split(os.pathsep):
            path = path.strip('"')
            full_path = os.path.join(path, program)
            if os.path.isfile(full_path) and os.access(full_path, os.X_OK):
                return full_path
        return None

    def print_version(self):
        sys.stdout.write("%s\n" % (__version__))

    def print_usage(self):
        sys.stderr.write("Usage: nas [-h | --help]" +
                         " [--version]" +
                         " <applet> [<applet-options>]\n")

    def print_modules(self):
        for i in NetAppletShuffler.modes.keys():
            sys.stderr.write("   %-15s - %s\n" % (i, NetAppletShuffler.modes[i][1]))

    def args_contains(self, argv, *cmds):
        for cmd in cmds:
            for arg in argv:
                if arg == cmd: return True
        return False

    def parse_global_otions(self):
        if len(sys.argv) <= 1:
            self.print_usage()
            sys.stderr.write("Available applets:\n")
            self.print_modules()
            return None

        self.binary_path = sys.argv[-1]

        # --version can be placed somewhere in the
        # command line and will evalutated always: it is
        # a global option
        if self.args_contains(sys.argv, "--version"):
            self.print_version()
            return None

        # handle verbose flag
        if "--verbose" in sys.argv:
            sys.argv.remove("--verbose")
            self.verbose = True
        if "-v" in sys.argv:
            self.verbose = True
            sys.argv.remove("-v")

        # -h | --help as first argument is treated special
        # and has other meaning as a submodule
        if self.args_contains(sys.argv[1:2], "-h", "--help"):
            self.print_usage()
            sys.stderr.write("Available modules:\n")
            self.print_modules()
            return None

        submodule = sys.argv[1].lower()
        if submodule not in NetAppletShuffler.modes:
            self.print_usage()
            sys.stderr.write("Modules \"%s\" not known, available modules are:\n" %
                             (submodule))
            self.print_modules()
            return None

        classname = NetAppletShuffler.modes[submodule][0]
        return classname

    def run(self):
        classtring = self.parse_global_otions()
        if not classtring:
            return 1

        classinstance = globals()[classtring](verbose=self.verbose)
        ok = classinstance.run()
        if not ok:
            return 1
        return 0

if __name__ == "__main__":
    try:
        mca = NetAppletShuffler()
        sys.exit(mca.run())
    except KeyboardInterrupt:
        sys.stderr.write("SIGINT received, exiting\n")
<|endoftext|>


<commit_before>"""Parses the given root directory and composes a complete OpenAPI specification file."""

import getopt
import os
import sys
import yaml

def build_openapi_docs(root_file, parse_dir, dest_dir):
    """Parses a directory of YAML files for OpenAPI chunks (paths, components, etc.) and injects them into
    the given root file and writes to the desired location.

    notes:
    - The given root_file MUST be prepopulated with `info` and `server` chunks
    - Only files ending in .yml or .yaml will be parsed
    - The following OpenAPI sections will be detected: paths, components, security, tags, externalDocs

    inputs:
    - root_file - File path to the openapi.yaml file that parsed OpenAPI chunks will be attached to
    - parse_dir - The directory to parse for OpenAPI chunks
    - dest_dir - The directory that the resulting openapi.yml will be written to

    outputs:
    - A complete OpenAPI spec written to the desired directory
    """

    chunks = {
        'paths': [],
        'components': [],
        'security': [],
        'tags': [],
        'externalDocs': []
    }

    for file in os.listdir(parse_dir):
        if file.endswith(".yml") or file.endswith(".yaml"):
            with open(os.path.join(parse_dir, file), 'r') as f:
                try:
                    yml = yaml.load(f)
                except yaml.scanner.ScannerError:
                    print ' '.join(["Error parsing file :", os.path.join(parse_dir, file)])
                except yaml.parser.ParserError:
                    print ' '.join(["Error parsing file :", os.path.join(parse_dir, file)])
                print(type(yml))

if __name__ == '__main__':
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'f:p:d', ['root_file=', 'parse_dir=', 'dest_dir'])
    except getopt.GetoptError:
        sys.exit(2)
    for opt, arg in opts:
        if opt in ('-f', '--root_file'):
            root_file = arg
        elif opt in ('-p', '--parse_dir'):
            parse_dir = arg
        elif opt in ('-d', '--dest_dir'):
            dest_dir = arg
        else:
            sys.exit(2)

    build_openapi_docs(root_file, parse_dir, dest_dir)

    sys.exit(0)<commit_msg>:hammer: updated stiching logic<commit_after>"""Parses the given root directory and composes a complete OpenAPI specification file."""

import getopt
import os
import sys
import yaml

def build_openapi_docs(root_file, parse_dir):
    """Parses a directory of YAML files for OpenAPI chunks (paths, components, etc.) and injects them into
    the given root file and writes to the desired location.

    notes:
    - The given root_file MUST be prepopulated with `info` and `server` chunks
    - Only files ending in .yml or .yaml will be parsed
    - The following OpenAPI sections will be detected: paths, components, security, tags, externalDocs
    - If your root_file is in the parse_dir it will NOT be parsed

    inputs:
    - root_file - File path to the openapi.yaml file that parsed OpenAPI chunks will be attached to
    - parse_dir - The directory to parse for OpenAPI chunks

    outputs:
    - A complete OpenAPI spec written to the desired root_file
    """

    chunks = {
        'paths': {},
        'components': {
            'schemas': {}
        },
        'security': {},
        'tags': {},
        'externalDocs': {}
    }

    for _file in os.listdir(parse_dir):
        if _file == os.path.basename(root_file):
            continue

        if _file.endswith(".yml") or _file.endswith(".yaml"):
            with open(os.path.join(parse_dir, _file), 'r') as f:
                try:
                    yml = yaml.load(f)
                except yaml.scanner.ScannerError:
                    print ' '.join(["Error parsing file :", os.path.join(parse_dir, _file)])
                except yaml.parser.ParserError:
                    print ' '.join(["Error parsing file :", os.path.join(parse_dir, _file)])

                for chunk_type in chunks:
                    if chunk_type in yml:
                        c = yml[chunk_type]
                        for k, v in c.items():
                            if k == 'schemas':
                                s = yml[chunk_type][k]
                                for k, v in s.items():
                                    chunks[chunk_type]['schemas'][k] = v
                            else:
                                chunks[chunk_type][k] = v

    chunks = dict((k, v) for k, v in chunks.iteritems() if v)

    with open(root_file, 'a') as f:
        f.write(yaml.dump(chunks))

if __name__ == '__main__':
    try:
        opts, _ = getopt.getopt(sys.argv[1:], 'f:p:', ['root_file=', 'parse_dir='])
    except getopt.GetoptError:
        sys.exit(2)
    for opt, arg in opts:
        if opt in ('-f', '--root_file'):
            root_file = arg
        elif opt in ('-p', '--parse_dir'):
            parse_dir = arg
        else:
            sys.exit(2)

    build_openapi_docs(root_file, parse_dir)

    sys.exit(0)
<|endoftext|>


<commit_before>"""Defines a command message that purges jobs"""
from __future__ import unicode_literals

import logging

from django.db import transaction
from django.utils.timezone import now

from batch.models import BatchJob
from job.models import Job, JobExecution, JobExecutionEnd, JobExecutionOutput, JobInputFile, TaskUpdate
from product.models import FileAncestryLink
from queue.models import Queue
from recipe.models import RecipeNode
from messaging.messages.message import CommandMessage
from util.parse import datetime_to_string, parse_datetime

# This is the maximum number of job models that can fit in one message. This maximum ensures that every message of this
# type is less than 25 KiB long.
MAX_NUM = 100

logger = logging.getLogger(__name__)

def create_purge_jobs_messages(purge_job_ids, when):
    """Creates messages to remove the given job IDs

    :param purge_job_ids: The job IDs
    :type purge_job_ids: list
    :param when: The current time
    :type when: :class:`datetime.datetime`
    :return: The list of messages
    :rtype: list
    """

    messages = []

    message = None
    for job_id in purge_job_ids:
        if not message:
            message = PurgeJobs()
            message.status_change = when
        elif not message.can_fit_more():
            messages.append(message)
            message = PurgeJobs()
            message.status_change = when
        message.add_job(job_id)
    if message:
        messages.append(message)

    return messages

class PurgeJobs(CommandMessage):
    """Command message that removes job models
    """

    def __init__(self):
        """Constructor
        """

        super(PurgeJobs, self).__init__('purge_jobs')

        self._count = 0
        self._purge_job_ids = []
        self.status_change = None

    def add_job(self, job_id):
        """Adds the given job ID to this message

        :param job_id: The job ID
        :type job_id: int
        """

        self._count += 1
        self._purge_job_ids.append(job_id)

    def can_fit_more(self):
        """Indicates whether more jobs can fit in this message

        :return: True if more jobs can fit, False otherwise
        :rtype: bool
        """

        return self._count < MAX_NUM

    def to_json(self):
        """See :meth:`messaging.messages.message.CommandMessage.to_json`
        """

        return {'status_change': datetime_to_string(self.status_change), 'job_ids': self._purge_job_ids}

    @staticmethod
    def from_json(json_dict):
        """See :meth:`messaging.messages.message.CommandMessage.from_json`
        """

        status_change = parse_datetime(json_dict['status_change'])

        message = PurgeJobs()
        message.status_change = status_change
        for job_id in json_dict['job_ids']:
            message.add_job(job_id)

        return message

    def execute(self):
        """See :meth:`messaging.messages.message.CommandMessage.execute`
        """

        with transaction.atomic():
            FileAncestryLink.objects.filter(job__in=self._purge_job_ids).delete()
            TaskUpdate.objects.filter().delete() # sub filter based off job_exe?
            JobExecutionOutput.objects.filter().delete()
            JobExecutionEnd.objects.filter().delete()
            JobExecution.objects.filter(job__in=self._purge_job_ids).delete()
            BatchJob.filter(job__in=self._purge_job_ids).delete()
            RecipeNode.objects.filter(job__in=self._purge_job_ids).delete()
            JobInputFile.objects.filter(job__in=self._purge_job_ids).delete()
            Queue.objects.filter(job__in=self._purge_job_ids).delete()
            Job.objects.filter(id__in=self._purge_job_ids).delete()

        return True
<commit_msg>added proper filtering<commit_after>"""Defines a command message that purges jobs"""
from __future__ import unicode_literals

import logging

from django.db import transaction
from django.utils.timezone import now

from batch.models import BatchJob
from job.models import Job, JobExecution, JobExecutionEnd, JobExecutionOutput, JobInputFile, TaskUpdate
from product.models import FileAncestryLink
from queue.models import Queue
from recipe.models import RecipeNode
from messaging.messages.message import CommandMessage
from util.parse import datetime_to_string, parse_datetime

# This is the maximum number of job models that can fit in one message. This maximum ensures that every message of this
# type is less than 25 KiB long.
MAX_NUM = 100

logger = logging.getLogger(__name__)

def create_purge_jobs_messages(purge_job_ids, when):
    """Creates messages to remove the given job IDs

    :param purge_job_ids: The job IDs
    :type purge_job_ids: list
    :param when: The current time
    :type when: :class:`datetime.datetime`
    :return: The list of messages
    :rtype: list
    """

    messages = []

    message = None
    for job_id in purge_job_ids:
        if not message:
            message = PurgeJobs()
            message.status_change = when
        elif not message.can_fit_more():
            messages.append(message)
            message = PurgeJobs()
            message.status_change = when
        message.add_job(job_id)
    if message:
        messages.append(message)

    return messages

class PurgeJobs(CommandMessage):
    """Command message that removes job models
    """

    def __init__(self):
        """Constructor
        """

        super(PurgeJobs, self).__init__('purge_jobs')

        self._count = 0
        self._purge_job_ids = []
        self.status_change = None

    def add_job(self, job_id):
        """Adds the given job ID to this message

        :param job_id: The job ID
        :type job_id: int
        """

        self._count += 1
        self._purge_job_ids.append(job_id)

    def can_fit_more(self):
        """Indicates whether more jobs can fit in this message

        :return: True if more jobs can fit, False otherwise
        :rtype: bool
        """

        return self._count < MAX_NUM

    def to_json(self):
        """See :meth:`messaging.messages.message.CommandMessage.to_json`
        """

        return {'status_change': datetime_to_string(self.status_change), 'job_ids': self._purge_job_ids}

    @staticmethod
    def from_json(json_dict):
        """See :meth:`messaging.messages.message.CommandMessage.from_json`
        """

        status_change = parse_datetime(json_dict['status_change'])

        message = PurgeJobs()
        message.status_change = status_change
        for job_id in json_dict['job_ids']:
            message.add_job(job_id)

        return message

    def execute(self):
        """See :meth:`messaging.messages.message.CommandMessage.execute`
        """

        with transaction.atomic():
            job_exe_queryset = JobExecution.objects.filter(job__in=self._purge_job_ids)
            TaskUpdate.objects.filter(job_exe__in=job_exe_queryset).delete()
            JobExecutionOutput.objects.filter(job_exe__in=job_exe_queryset).delete()
            JobExecutionEnd.objects.filter(job_exe__in=job_exe_queryset).delete()
            job_exe_queryset.delete()
            FileAncestryLink.objects.filter(job__in=self._purge_job_ids).delete()
            BatchJob.filter(job__in=self._purge_job_ids).delete()
            RecipeNode.objects.filter(job__in=self._purge_job_ids).delete()
            JobInputFile.objects.filter(job__in=self._purge_job_ids).delete()
            Queue.objects.filter(job__in=self._purge_job_ids).delete()
            Job.objects.filter(id__in=self._purge_job_ids).delete()

        return True
<|endoftext|>


<commit_before>def printHi():
	print("HI")
def printWhee():
	print("Whee")

<commit_msg>updated new file in develop branch<commit_after>def printHi():
	print("HI")
def printWhee():
	print("Wheee")

<|endoftext|>


<commit_before># Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import fnmatch
import os

import nox

def _list_files(folder, pattern):
    """Lists all files below the given folder that match the pattern."""
    for root, folders, files in os.walk(folder):
        for filename in files:
            if fnmatch.fnmatch(filename, pattern):
                yield os.path.join(root, filename)

@nox.session
def check_requirements(session):
    """Checks for out of date requirements and optionally updates them."""
    session.install('gcp-devrel-py-tools')

    if 'update' in session.posargs:
        command = 'update-requirements'
    else:
        command = 'check-requirements'

    reqfiles = list(_list_files('.', 'requirements*.txt'))

    for reqfile in reqfiles:
        session.run('gcp-devrel-py-tools', command, reqfile)

@nox.session
def lint(session):
    session.interpreter = 'python3'  # So it understands Python3 syntax
    session.install('flake8', 'flake8-import-order')
    session.run(
        'flake8',
        '--import-order-style=google',
        '--application-import-names',
        'gen_dockerfile,local_cloudbuild,validation_utils',
        'scripts',
        'nox.py',
    )

@nox.session
@nox.parametrize('version', ['3.4', '3.5', '3.6'])
def tests(session, version):
    session.interpreter = 'python' + version
    session.install('-r', 'scripts/requirements-test.txt')
    session.run(
        'py.test',
        '--ignore=scripts/testdata',
        '--cov=scripts',
        '--cov-append',
        '--cov-config=.coveragerc',
        '--cov-report=',  # Report generated below
        'scripts',
        env={'PYTHONPATH': ''}
    )

@nox.session
def cover(session):
    """Run the final coverage report.

    This outputs the coverage report aggregating coverage from the unit
    test runs (not system test runs), and then erases coverage data.
    """
    session.interpreter = 'python3.6'
    session.install('coverage', 'pytest-cov')
    session.run('coverage', 'report', '--show-missing', '--fail-under=97')
    session.run('coverage', 'erase')
<commit_msg>Fix nit in nox.py<commit_after># Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import fnmatch
import os

import nox

def _list_files(folder, pattern):
    """Lists all files below the given folder that match the pattern."""
    for root, folders, files in os.walk(folder):
        for filename in files:
            if fnmatch.fnmatch(filename, pattern):
                yield os.path.join(root, filename)

@nox.session
def check_requirements(session):
    """Checks for out of date requirements and optionally updates them."""
    session.install('gcp-devrel-py-tools')

    if 'update' in session.posargs:
        command = 'update-requirements'
    else:
        command = 'check-requirements'

    reqfiles = list(_list_files('.', 'requirements*.txt'))

    for reqfile in reqfiles:
        session.run('gcp-devrel-py-tools', command, reqfile)

@nox.session
def lint(session):
    session.interpreter = 'python3'  # So it understands Python3 syntax
    session.install('flake8', 'flake8-import-order')
    session.run(
        'flake8',
        '--import-order-style', 'google',
        '--application-import-names',
        'gen_dockerfile,local_cloudbuild,validation_utils',
        'scripts',
        'nox.py',
    )

@nox.session
@nox.parametrize('version', ['3.4', '3.5', '3.6'])
def tests(session, version):
    session.interpreter = 'python' + version
    session.install('-r', 'scripts/requirements-test.txt')
    session.run(
        'py.test',
        '--ignore=scripts/testdata',
        '--cov=scripts',
        '--cov-append',
        '--cov-config=.coveragerc',
        '--cov-report=',  # Report generated below
        'scripts',
        env={'PYTHONPATH': ''}
    )

@nox.session
def cover(session):
    """Run the final coverage report.

    This outputs the coverage report aggregating coverage from the unit
    test runs (not system test runs), and then erases coverage data.
    """
    session.interpreter = 'python3.6'
    session.install('coverage', 'pytest-cov')
    session.run('coverage', 'report', '--show-missing', '--fail-under=97')
    session.run('coverage', 'erase')
<|endoftext|>


<commit_before>__all__ =['approximate_spectral_radius','infinity_norm','diag_sparse',
          'hstack_csr','vstack_csr','expand_into_blocks']

import numpy
import scipy
from scipy import ravel,arange,concatenate,tile,asarray,sqrt,diff, \
                  rand,zeros,empty,asmatrix,dot
from scipy.linalg import norm,eigvals
from scipy.sparse import isspmatrix,isspmatrix_csr,isspmatrix_csc, \
                        csr_matrix,csc_matrix,extract_diagonal, \
                        coo_matrix

def approximate_spectral_radius(A,tol=0.1,maxiter=10,symmetric=None):
    """approximate the spectral radius of a matrix

    *Parameters*:
        A : dense or sparse matrix
            E.g. csr_matrix, csc_matrix, ndarray, etc.

        tol : {scalar}
            Tolerance of approximation

        maxiter : {integer}
            Maximum number of iterations to perform

        symmetric : {None,boolean}
            True  - if A is symmetric
                    Lanczos iteration is used (more efficient)
            False - if A is non-symmetric
                    Arnoldi iteration is used (less efficient)
            None  - symmetry of A unknown
                    Method chosen automatically (default)
    *Returns*:
        An approximation to the spectral radius of A (scalar value)

    """
    #from scipy.sandbox.arpack import eigen
    #return norm(eigen(A, k=1, ncv=10, which='LM', maxiter=maxiter, tol=tol, return_eigenvectors=False))

    if not isspmatrix(A):
        A = asmatrix(A) #convert dense arrays to matrix type

    if A.shape[0] != A.shape[1]:
        raise ValueError,'expected square matrix'

    #TODO make method adaptive

    numpy.random.seed(0)  #make results deterministic

    v0  = rand(A.shape[1],1)
    v0 /= norm(v0)

    H  = zeros((maxiter+1,maxiter))
    V = [v0]

    #save past estimates
    #estimates = []

    for j in range(maxiter):
        w = A * V[-1]

        if symmetric:
            if j >= 1:
                H[j-1,j] = beta
                w -= beta * V[-2]

            alpha = dot(ravel(w),ravel(V[-1]))
            H[j,j] = alpha
            w -= alpha * V[-1]

            beta = norm(w)
            if (H[j+1,j] < 1e-10): break

            w /= beta
            H[j+1,j] = beta

            V.append(w)
            V = V[-2:] #retain only last two vectors
        else:
            #orthogonalize against Vs
            for i,v in enumerate(V):
                H[i,j] = dot(ravel(w),ravel(v))
                w -= H[i,j]*v
            H[j+1,j] = norm(w)
            if (H[j+1,j] < 1e-10): break

            w /= H[j+1,j]
            V.append(w)

            # if upper 2x2 block of Hessenberg matrix H is almost symmetric,
            # and the user has not explicitly specified symmetric=False,
            # then switch to symmetric Lanczos algorithm
            if symmetric is not False and j == 1:
                if abs(H[1,0] - H[0,1]) < 1e-12:
                    symmetric = True
                    V = V[1:]
                    H[1,0] = H[0,1]
                    beta = H[2,1]

        #estimates.append( max( [norm(x) for x in eigvals(H[:j+1,:j+1])] ) )

    return norm(H[:j+1,:j+1],2)

def infinity_norm(A):
    """
    Infinity norm of a sparse matrix (maximum absolute row sum).  This serves
    as an upper bound on spectral radius.
    """

    if isspmatrix_csr(A) or isspmatrix_csc(A):
        #avoid copying index and ptr arrays
        abs_A = A.__class__((abs(A.data),A.indices,A.indptr),dims=A.shape)
        return (abs_A * numpy.ones(A.shape[1],dtype=A.dtype)).max()
    else:
        return (abs(A) * numpy.ones(A.shape[1],dtype=A.dtype)).max()

def diag_sparse(A):
    """
    If A is a sparse matrix (e.g. csr_matrix or csc_matrix)
       - return the diagonal of A as an array

    Otherwise
       - return a csr_matrix with A on the diagonal
    """

    #TODO integrate into SciPy?
    if isspmatrix(A):
        return extract_diagonal(A)
    else:
        return csr_matrix((asarray(A),arange(len(A)),arange(len(A)+1)),(len(A),len(A)))

def symmetric_rescaling(A):
    if not (isspmatrix_csr(A) or isspmatrix_csc(A)):
        raise TypeError,'expected csr_matrix or csc_matrix'

    if A.shape[0] != A.shape[1]:
        raise ValueError,'expected square matrix'

    D = diag_sparse(A)
    mask = D == 0

    #D[mask] = 0
    D_sqrt = sqrt(abs(D))
    D_sqrt_inv = 1.0/D_sqrt
    D_sqrt_inv[mask] = 0

    #TODO time this against simple implementation
    data = A.data * D_sqrt_inv[A.indices]
    data *= D_sqrt_inv[arange(A.shape[0]).repeat(diff(A.indptr))]

    DAD = A.__class__((data,A.indices,A.indptr),dims=A.shape)

    return D_sqrt,D_sqrt_inv,DAD

def hstack_csr(A,B):
    if not isspmatrix(A) or not isspmatrix(B):
        raise TypeError,'expected sparse matrix'

    if A.shape[0] != B.shape[0]:
        raise ValueError,'row dimensions must agree'

    A = A.tocoo()
    B = B.tocoo()
    I = concatenate((A.row,B.row))
    J = concatenate((A.col,B.col+A.shape[1]))
    V = concatenate((A.data,B.data))
    return coo_matrix((V,(I,J)),dims=(A.shape[0],A.shape[1]+B.shape[1])).tocsr()

def vstack_csr(A,B):
    #TODO OPTIMIZE THIS
    if not isspmatrix(A) or not isspmatrix(B):
        raise TypeError,'expected sparse matrix'

    if A.shape[1] != B.shape[1]:
        raise ValueError,'column dimensions must agree'

    A = A.tocoo()
    B = B.tocoo()
    I = concatenate((A.row,B.row+A.shape[0]))
    J = concatenate((A.col,B.col))
    V = concatenate((A.data,B.data))
    return coo_matrix((V,(I,J)),dims=(A.shape[0]+B.shape[0],A.shape[1])).tocsr()

def expand_into_blocks(A,m,n):
    """Expand each element in a sparse matrix A into an m-by-n block.

          Example:
          >>> A.todense()
          matrix([[ 1.,  2.],
                  [ 4.,  5.]])

          >>> expand_into_blocks(A,2,2).todense()
          matrix([[ 1.,  1.,  2.,  2.],
                  [ 1.,  1.,  2.,  2.],
                  [ 4.,  4.,  5.,  5.],
                  [ 4.,  4.,  5.,  5.]])

    """
    #TODO EXPLAIN MORE
    #TODO use spkron instead, time for compairson

    if m == 1 and n == 1:
        return A #nothing to do

    A = A.tocoo()

    # expand 1x1 -> mxn
    row  = ( m*A.row ).repeat(m*n).reshape(-1,m,n)
    col  = ( n*A.col ).repeat(m*n).reshape(-1,m,n)

    # increment indices
    row += tile(arange(m).reshape(-1,1),(1,n))
    col += tile(arange(n).reshape(1,-1),(m,1))

    # flatten
    row = row.reshape(-1)
    col = col.reshape(-1)

    data = A.data.repeat(m*n)

    return coo_matrix((data,(row,col)),dims=(m*A.shape[0],n*A.shape[1]))
<commit_msg>make symmetric_rescaling work for general csr mats<commit_after>__all__ =['approximate_spectral_radius','infinity_norm','diag_sparse',
          'hstack_csr','vstack_csr','expand_into_blocks']

import numpy
import scipy
from scipy import ravel,arange,concatenate,tile,asarray,sqrt,diff, \
                  rand,zeros,empty,asmatrix,dot
from scipy.linalg import norm,eigvals
from scipy.sparse import isspmatrix,isspmatrix_csr,isspmatrix_csc, \
                        csr_matrix,csc_matrix,extract_diagonal, \
                        coo_matrix

def approximate_spectral_radius(A,tol=0.1,maxiter=10,symmetric=None):
    """approximate the spectral radius of a matrix

    *Parameters*:
        A : dense or sparse matrix
            E.g. csr_matrix, csc_matrix, ndarray, etc.

        tol : {scalar}
            Tolerance of approximation

        maxiter : {integer}
            Maximum number of iterations to perform

        symmetric : {None,boolean}
            True  - if A is symmetric
                    Lanczos iteration is used (more efficient)
            False - if A is non-symmetric
                    Arnoldi iteration is used (less efficient)
            None  - symmetry of A unknown
                    Method chosen automatically (default)
    *Returns*:
        An approximation to the spectral radius of A (scalar value)

    """
    #from scipy.sandbox.arpack import eigen
    #return norm(eigen(A, k=1, ncv=10, which='LM', maxiter=maxiter, tol=tol, return_eigenvectors=False))

    if not isspmatrix(A):
        A = asmatrix(A) #convert dense arrays to matrix type

    if A.shape[0] != A.shape[1]:
        raise ValueError,'expected square matrix'

    #TODO make method adaptive

    numpy.random.seed(0)  #make results deterministic

    v0  = rand(A.shape[1],1)
    v0 /= norm(v0)

    H  = zeros((maxiter+1,maxiter))
    V = [v0]

    #save past estimates
    #estimates = []

    for j in range(maxiter):
        w = A * V[-1]

        if symmetric:
            if j >= 1:
                H[j-1,j] = beta
                w -= beta * V[-2]

            alpha = dot(ravel(w),ravel(V[-1]))
            H[j,j] = alpha
            w -= alpha * V[-1]

            beta = norm(w)
            if (H[j+1,j] < 1e-10): break

            w /= beta
            H[j+1,j] = beta

            V.append(w)
            V = V[-2:] #retain only last two vectors
        else:
            #orthogonalize against Vs
            for i,v in enumerate(V):
                H[i,j] = dot(ravel(w),ravel(v))
                w -= H[i,j]*v
            H[j+1,j] = norm(w)
            if (H[j+1,j] < 1e-10): break

            w /= H[j+1,j]
            V.append(w)

            # if upper 2x2 block of Hessenberg matrix H is almost symmetric,
            # and the user has not explicitly specified symmetric=False,
            # then switch to symmetric Lanczos algorithm
            if symmetric is not False and j == 1:
                if abs(H[1,0] - H[0,1]) < 1e-12:
                    symmetric = True
                    V = V[1:]
                    H[1,0] = H[0,1]
                    beta = H[2,1]

        #estimates.append( max( [norm(x) for x in eigvals(H[:j+1,:j+1])] ) )

    return norm(H[:j+1,:j+1],2)

def infinity_norm(A):
    """
    Infinity norm of a sparse matrix (maximum absolute row sum).  This serves
    as an upper bound on spectral radius.
    """

    if isspmatrix_csr(A) or isspmatrix_csc(A):
        #avoid copying index and ptr arrays
        abs_A = A.__class__((abs(A.data),A.indices,A.indptr),dims=A.shape)
        return (abs_A * numpy.ones(A.shape[1],dtype=A.dtype)).max()
    else:
        return (abs(A) * numpy.ones(A.shape[1],dtype=A.dtype)).max()

def diag_sparse(A):
    """
    If A is a sparse matrix (e.g. csr_matrix or csc_matrix)
       - return the diagonal of A as an array

    Otherwise
       - return a csr_matrix with A on the diagonal
    """

    #TODO integrate into SciPy?
    if isspmatrix(A):
        return extract_diagonal(A)
    else:
        return csr_matrix((asarray(A),arange(len(A)),arange(len(A)+1)),(len(A),len(A)))

def symmetric_rescaling(A):
    if not (isspmatrix_csr(A) or isspmatrix_csc(A)):
        raise TypeError,'expected csr_matrix or csc_matrix'

    if A.shape[0] != A.shape[1]:
        raise ValueError,'expected square matrix'

    D = diag_sparse(A)
    mask = D == 0

    #D[mask] = 0
    D_sqrt = sqrt(abs(D))
    D_sqrt_inv = 1.0/D_sqrt
    D_sqrt_inv[mask] = 0

    Acoo = A.tocoo(copy=False)
    data = A.data[:A.nnz] * D_sqrt_inv[Acoo.row]
    data *= D_sqrt_inv[Acoo.col]

    DAD = A.__class__((data,A.indices[:A.nnz],A.indptr),dims=A.shape)

    return D_sqrt,D_sqrt_inv,DAD

def hstack_csr(A,B):
    if not isspmatrix(A) or not isspmatrix(B):
        raise TypeError,'expected sparse matrix'

    if A.shape[0] != B.shape[0]:
        raise ValueError,'row dimensions must agree'

    A = A.tocoo()
    B = B.tocoo()
    I = concatenate((A.row,B.row))
    J = concatenate((A.col,B.col+A.shape[1]))
    V = concatenate((A.data,B.data))
    return coo_matrix((V,(I,J)),dims=(A.shape[0],A.shape[1]+B.shape[1])).tocsr()

def vstack_csr(A,B):
    #TODO OPTIMIZE THIS
    if not isspmatrix(A) or not isspmatrix(B):
        raise TypeError,'expected sparse matrix'

    if A.shape[1] != B.shape[1]:
        raise ValueError,'column dimensions must agree'

    A = A.tocoo()
    B = B.tocoo()
    I = concatenate((A.row,B.row+A.shape[0]))
    J = concatenate((A.col,B.col))
    V = concatenate((A.data,B.data))
    return coo_matrix((V,(I,J)),dims=(A.shape[0]+B.shape[0],A.shape[1])).tocsr()

def expand_into_blocks(A,m,n):
    """Expand each element in a sparse matrix A into an m-by-n block.

          Example:
          >>> A.todense()
          matrix([[ 1.,  2.],
                  [ 4.,  5.]])

          >>> expand_into_blocks(A,2,2).todense()
          matrix([[ 1.,  1.,  2.,  2.],
                  [ 1.,  1.,  2.,  2.],
                  [ 4.,  4.,  5.,  5.],
                  [ 4.,  4.,  5.,  5.]])

    """
    #TODO EXPLAIN MORE
    #TODO use spkron instead, time for compairson

    if m == 1 and n == 1:
        return A #nothing to do

    A = A.tocoo()

    # expand 1x1 -> mxn
    row  = ( m*A.row ).repeat(m*n).reshape(-1,m,n)
    col  = ( n*A.col ).repeat(m*n).reshape(-1,m,n)

    # increment indices
    row += tile(arange(m).reshape(-1,1),(1,n))
    col += tile(arange(n).reshape(1,-1),(m,1))

    # flatten
    row = row.reshape(-1)
    col = col.reshape(-1)

    data = A.data.repeat(m*n)

    return coo_matrix((data,(row,col)),dims=(m*A.shape[0],n*A.shape[1]))
<|endoftext|>


<commit_before># This code is licensed under the MIT License (see LICENSE file for details)

import time
import datetime

from PyQt5 import Qt
from  ris_widget import image
from ris_widget import ris_widget
import freeimage

from .. import scope_client

# TODO: Should live_target be a flipbook entry instead of just self.image?

class ScopeViewerWidget(ris_widget.RisWidgetQtObject):
    NEW_IMAGE_EVENT = Qt.QEvent.registerEventType()
    OVEREXPOSURE_GETCOLOR_EXPRESSION = 's.r < 1.0f ? vec4(s.rrr, 1.0f) : vec4(1.0f, 0.0f, 0.0f, 1.0f)'

    @staticmethod
    def can_run(scope):
        return hasattr(scope, 'camera')

    def __init__(self, scope, scope_properties, window_title='Viewer', fps_max=None, app_prefs_name='scope-viewer', parent=None):
        super().__init__(window_title=window_title, app_prefs_name=app_prefs_name, parent=parent)
        self.main_view_toolbar.removeAction(self.snapshot_action)

        self.scope_toolbar = self.addToolBar('Scope')
        self.show_over_exposed_action = Qt.QAction('Show Over-Exposed', self)
        self.show_over_exposed_action.setCheckable(True)
        self.show_over_exposed_action.setChecked(True)
        self.show_over_exposed_action.toggled.connect(self.on_show_over_exposed_action_toggled)
        self.scope_toolbar.addAction(self.show_over_exposed_action)

        self.flipbook.pages.append(image.Image([[0]], name='Live Image'))
        self.flipbook_dock_widget.hide()
        self.image = None

        self.camera = scope.camera
        self.snap_action = Qt.QAction('Snap Image', self)
        self.snap_action.triggered.connect(self.snap_image)
        self.scope_toolbar.addAction(self.snap_action)

        self.save_action = Qt.QAction('Save Image', self)
        self.save_action.triggered.connect(self.save_image)
        self.scope_toolbar.addAction(self.save_action)

        self.closing = False
        self.live_streamer = scope_client.LiveStreamer(scope, scope_properties, self.post_new_image_event)
        if fps_max is None:
            self.interval_min = None
        else:
            self.interval_min = 1/fps_max
        self.last_image = 0

    def closeEvent(self, e):
        if self.closing:
            # sometimes closeEvent gets fired off twice (2018-01, PyQt 5.9). Why? TODO: verify if problem goes away in later Qt
            return
        self.closing = True
        self.live_streamer.detach()
        super().closeEvent(e)

    def event(self, e):
        # This is called by the main QT event loop to service the event posted in post_new_image_event().
        if e.type() == self.NEW_IMAGE_EVENT and self.live_streamer.image_ready():
            if self.interval_min is not None:
                t = time.time()
                if t - self.last_image < self.interval_min:
                    return True
                self.last_image = t
            self.flipbook.current_page_idx = 0
            image_data, timestamp, frame_no = self.live_streamer.get_image()
            image_bits = 12 if self.live_streamer.bit_depth == '12 Bit' else 16
            self.image = image.Image(image_data, image_bits=image_bits)
            if self.show_over_exposed_action.isChecked() and self.layer.image.type == 'G':
                self.layer.getcolor_expression = self.OVEREXPOSURE_GETCOLOR_EXPRESSION
            else:
                del self.layer.getcolor_expression
            return True
        return super().event(e)

    def post_new_image_event(self):
        # posting an event does not require calling thread to have an event loop,
        # unlike sending a signal
        if not self.closing and self.isVisible():
            Qt.QCoreApplication.postEvent(self, Qt.QEvent(self.NEW_IMAGE_EVENT))

    def on_show_over_exposed_action_toggled(self, show_over_exposed):
        if show_over_exposed:
            if self.layer.image is not None and self.layer.image.type == 'G':
                self.layer.getcolor_expression = self.OVEREXPOSURE_GETCOLOR_EXPRESSION
        else:
            # Revert to default getcolor_expression
            del self.layer.getcolor_expression

    def snap_image(self):
        self.flipbook.pages.append_named(self.camera.acquire_image(), datetime.datetime.now().isoformat(' ', 'seconds'))

    def save_image(self):
        fn, _ = Qt.QFileDialog.getSaveFileName(self, 'Save Image', filter='Images (*.png *.tiff *.tif)')
        if fn:
            freeimage.write(self.image.data, fn)
<commit_msg>bugfix<commit_after># This code is licensed under the MIT License (see LICENSE file for details)

import time
import datetime

from PyQt5 import Qt
from  ris_widget import image
from ris_widget import ris_widget
import freeimage

from .. import scope_client

# TODO: Should live_target be a flipbook entry instead of just self.image?

class ScopeViewerWidget(ris_widget.RisWidgetQtObject):
    NEW_IMAGE_EVENT = Qt.QEvent.registerEventType()
    OVEREXPOSURE_GETCOLOR_EXPRESSION = 's.r < 1.0f ? vec4(s.rrr, 1.0f) : vec4(1.0f, 0.0f, 0.0f, 1.0f)'

    @staticmethod
    def can_run(scope):
        return hasattr(scope, 'camera')

    def __init__(self, scope, scope_properties, window_title='Viewer', fps_max=None, app_prefs_name='scope-viewer', parent=None):
        super().__init__(window_title=window_title, app_prefs_name=app_prefs_name, parent=parent)
        self.main_view_toolbar.removeAction(self.snapshot_action)

        self.scope_toolbar = self.addToolBar('Scope')
        self.show_over_exposed_action = Qt.QAction('Show Over-Exposed', self)
        self.show_over_exposed_action.setCheckable(True)
        self.show_over_exposed_action.setChecked(True)
        self.show_over_exposed_action.toggled.connect(self.on_show_over_exposed_action_toggled)
        self.scope_toolbar.addAction(self.show_over_exposed_action)

        self.flipbook.pages.append(image.Image([[0]], name='Live Image'))
        self.flipbook_dock_widget.hide()
        self.image = None

        self.camera = scope.camera
        self.snap_action = Qt.QAction('Snap Image', self)
        self.snap_action.triggered.connect(self.snap_image)
        self.scope_toolbar.addAction(self.snap_action)

        self.save_action = Qt.QAction('Save Image', self)
        self.save_action.triggered.connect(self.save_image)
        self.scope_toolbar.addAction(self.save_action)

        self.closing = False
        self.live_streamer = scope_client.LiveStreamer(scope, scope_properties, self.post_new_image_event)
        if fps_max is None:
            self.interval_min = None
        else:
            self.interval_min = 1/fps_max
        self.last_image = 0

    def closeEvent(self, e):
        if self.closing:
            # sometimes closeEvent gets fired off twice (2018-01, PyQt 5.9). Why? TODO: verify if problem goes away in later Qt
            return
        self.closing = True
        self.live_streamer.detach()
        super().closeEvent(e)

    def event(self, e):
        # This is called by the main QT event loop to service the event posted in post_new_image_event().
        if e.type() == self.NEW_IMAGE_EVENT and self.live_streamer.image_ready():
            if self.interval_min is not None:
                t = time.time()
                if t - self.last_image < self.interval_min:
                    return True
                self.last_image = t
            self.flipbook.current_page_idx = 0
            image_data, timestamp, frame_no = self.live_streamer.get_image()
            image_bits = 12 if self.live_streamer.bit_depth == '12 Bit' else 16
            self.image = image.Image(image_data, image_bits=image_bits)
            if self.show_over_exposed_action.isChecked() and self.layer.image.type == 'G':
                self.layer.getcolor_expression = self.OVEREXPOSURE_GETCOLOR_EXPRESSION
            else:
                del self.layer.getcolor_expression
            return True
        return super().event(e)

    def post_new_image_event(self):
        # posting an event does not require calling thread to have an event loop,
        # unlike sending a signal
        if not self.closing and self.isVisible():
            Qt.QCoreApplication.postEvent(self, Qt.QEvent(self.NEW_IMAGE_EVENT))

    def on_show_over_exposed_action_toggled(self, show_over_exposed):
        if show_over_exposed:
            if self.layer.image is not None and self.layer.image.type == 'G':
                self.layer.getcolor_expression = self.OVEREXPOSURE_GETCOLOR_EXPRESSION
        else:
            # Revert to default getcolor_expression
            del self.layer.getcolor_expression

    def snap_image(self):
        self.flipbook.pages.append_named(self.camera.acquire_image(), datetime.datetime.now().isoformat(' ', 'seconds'))
        self.flipbook.current_page_idx = -1

    def save_image(self):
        fn, _ = Qt.QFileDialog.getSaveFileName(self, 'Save Image', self.flipbook.current_page.name, filter='Images (*.png *.tiff *.tif)')
        if fn:
            freeimage.write(self.image.data, fn)
<|endoftext|>