<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

"""
List ufm tags.

A simple script to generate a list of usfm tags that were used in one
or more utf8 encoded usfm files. Compatible with python2 and python3.

This script is public domain.

"""

from __future__ import print_function, unicode_literals
import argparse
import re
import collections
import glob

# -------------------------------------------------------------------------- #

VERSION = "1.1.1"

# -------------------------------------------------------------------------- #

# set of known usfm tags in version 3.0 of usfm specification
# I think I have all the tags here.
KNOWNTAGS = {
    # identification
    r"\id",
    r"\ide",
    r"\sts",
    r"\rem",
    r"\h",
    r"\h1",
    r"\h2",
    r"\h3",
    r"\toc1",
    r"\toc2",
    r"\toc3",
    # introductions
    r"\imt",
    r"\imt1",
    r"\imt2",
    r"\imt3",
    r"\imt4",
    r"\is",
    r"\is1",
    r"\is2",
    r"\ip",
    r"\ipi",
    r"\im",
    r"\imi",
    r"\ipq",
    r"\imq",
    r"\ipr",
    r"\iq",
    r"\iq1",
    r"\iq2",
    r"\iq3",
    r"\ib",
    r"\ili",
    r"\ili1",
    r"\ili2",
    r"\iot",
    r"\io",
    r"\io1",
    r"\io2",
    r"\io3",
    r"\io4",
    r"\ior",
    r"\ior*",
    r"\iex",
    r"\iqt",
    r"\iqt*",
    r"\imte",
    r"\imte1",
    r"\imte2",
    r"\ie",
    # titles, headings, labels
    r"\mt",
    r"\mt1",
    r"\mt2",
    r"\mt3",
    r"\mt4",
    r"\mte",
    r"\mte1",
    r"\mte2",
    r"\ms",
    r"\ms1",
    r"\ms2",
    r"\ms3",
    r"\mr",
    r"\s",
    r"\s1",
    r"\s2",
    r"\s3",
    r"\s4",
    r"\sr",
    r"\r",
    r"\rq",
    r"\rq*",
    r"\d",
    r"\sp",
    # chapters and verses
    r"\c",
    r"\ca",
    r"\ca*",
    r"\cl",
    r"\cp",
    r"\cd",
    r"\v",
    r"\va",
    r"\va*",
    r"\vp",
    r"\vp*",
    # paragraphs/poetry
    r"\p",
    r"\po",
    r"\m",
    r"\pmo",
    r"\pm",
    r"\pmc",
    r"\pmr",
    r"\pi",
    r"\pi1",
    r"\pi2",
    r"\pi3",
    r"\mi",
    r"\nb",
    r"\cls",
    r"\lh",
    r"\lf",
    r"\li",
    r"\li1",
    r"\li2",
    r"\li3",
    r"\li4",
    r"\lim",
    r"\lim1",
    r"\lim2",
    r"\lim3",
    r"\lim4",
    r"\lim5",
    r"\pc",
    r"\pr",
    r"\ph",
    r"\ph1",
    r"\ph2",
    r"\ph3",
    r"\q",
    r"\q1",
    r"\q2",
    r"\q3",
    r"\q4",
    r"\qr",
    r"\qc",
    r"\qs",
    r"\qs*",
    r"\qa",
    r"\qac",
    r"\qac*",
    r"\qm",
    r"\qm1",
    r"\qm2",
    r"\qm3",
    r"\qd",
    r"\lik",
    r"\lik*",
    r"\litl",
    r"\litl*",
    r"\ts",
    r"\sd",
    r"\sd1",
    r"\sd2",
    r"\sd3",
    r"\sd4",
    r"\sd5",
    r"\b",
    # tables
    r"\tr",
    r"\th",
    r"\th1",
    r"\th2",
    r"\th3",
    r"\th4",
    r"\th5",
    r"\thr",
    r"\thr1",
    r"\thr2",
    r"\thr3",
    r"\thr4",
    r"\thr5",
    r"\tc",
    r"\tc1",
    r"\tc2",
    r"\tc3",
    r"\tc4",
    r"\tc5",
    r"\tcr",
    r"\tcr1",
    r"\tcr2",
    r"\tcr3",
    r"\tcr4",
    r"\tcr5",
    # footnotes
    r"\f",
    r"\f*",
    r"\fe",
    r"\fe*",
    r"\fr",
    r"\fk",
    r"\fq",
    r"\fqa",
    r"\fl",
    r"\fp",
    r"\fv",
    r"\fv*",
    r"\ft",
    r"\fdc",
    r"\fdc*",
    r"\fm",
    r"\fm*",
    # cross references
    r"\x",
    r"\x*",
    r"\xo",
    r"\xk",
    r"\xq",
    r"\xt",
    r"\xot",
    r"\xot*",
    r"\xnt",
    r"\xnt*",
    r"\xdc",
    r"\xdc*",
    r"\xta",
    r"\xta*",
    r"\xop",
    r"\xop*",
    # special text
    r"\add",
    r"\add*",
    r"\bk",
    r"\bk*",
    r"\dc",
    r"\dc*",
    r"\k",
    r"\k*",
    r"\lit",
    r"\nd",
    r"\nd*",
    r"\ord",
    r"\ord*",
    r"\pn",
    r"\pn*",
    r"\qt",
    r"\qt*",
    r"\sig",
    r"\sig*",
    r"\sls",
    r"\sls*",
    r"\tl",
    r"\tl*",
    r"\wj",
    r"\wj*",
    r"\addpn",
    r"\addpn*",
    r"\+add",
    r"\+add*",
    r"\+bk",
    r"\+bk*",
    r"\+dc",
    r"\+dc*",
    r"\+k",
    r"\+k*",
    r"\+nd",
    r"\+nd*",
    r"\+ord",
    r"\+ord*",
    r"\+pn",
    r"\+pn*",
    r"\+qt",
    r"\+qt*",
    r"\+sig",
    r"\+sig*",
    r"\+sls",
    r"\+sls*",
    r"\+tl",
    r"\+tl*",
    r"\+wj",
    r"\+wj*",
    r"\+addpn",
    r"\+addpn*",
    # character styles
    r"\em",
    r"\em*",
    r"\bd",
    r"\bd*",
    r"\it",
    r"\it*",
    r"\bdit",
    r"\bdit*",
    r"\no",
    r"\no*",
    r"\sc",
    r"\sc*",
    r"\+em",
    r"\+em*",
    r"\+bd",
    r"\+bd*",
    r"\+it",
    r"\+it*",
    r"\+bdit",
    r"\+bdit*",
    r"\+no",
    r"\+no*",
    r"\+sc",
    r"\+sc*",
    # spacing
    r"\pb",
    # special features
    r"\fig",
    r"\fig*",
    r"\ndx",
    r"\ndx*",
    r"\pro",
    r"\pro*",
    r"\w",
    r"\w*",
    r"\wg",
    r"\wg*",
    r"\wh",
    r"\wh*",
    r"\wa",
    r"\wa*",
    r"\png",
    r"\png*",
    r"\jmp",
    r"\jmp*",
    r"\rb",
    r"\rb*",
    r"\rt",
    r"\rt*",
    # peripherals
    r"\periph",
    # study bible content
    r"\ef",
    r"\ef*",
    r"\ex",
    r"\ex*",
    r"\esb",
    r"\esbe",
    r"\cat",
    r"\cat*",
    # milestone quotations
    r"\qt-s",
    r"\qt1-s",
    r"\qt2-s",
    r"\qt3-s",
    r"\qt4-s",
    r"\qt5-s",
    r"\qt-e",
    r"\qt1-e",
    r"\qt2-e",
    r"\qt3-e",
    r"\qt4-e",
    r"\qt5-e",
    r"\*",
}

# regex for finding usfm tags
USFMRE = re.compile(
    r"""
    # handle usfm tags other than special spacing...
    (?:
        # the first character of a usfm tag is always a backslash
        \\

        # a plus symbol marks the start of a nested character style.
        # this may or may not be present.
        \+?

        # tag names are ascii letters
        [A-Za-z]+

        # tags may or may not be numbered
        [0-9]?

        # a word boundary to mark the end of our tags.
        \b

        # character style closing tags end with an asterisk.
        \*?
    )

    # OR...
    |

    # one of the two special spacing tags.
    (?:
        (?:~|//)
    )
""",
    re.U + re.VERBOSE,
)

# -------------------------------------------------------------------------- #

def processtags(args):
    """Process usfm tags in all files."""
    count = 0
    counttags = collections.Counter()
    knownset = set()

    filenames = []
    for _ in args.file:
        if "*" in _:
            filenames.extend(glob.glob(_))
        else:
            filenames.append(_)

    for fname in filenames:
        with open(fname, "rb") as infile:
            intext = infile.read()

            # build tag set
            knownset.update(USFMRE.findall(intext.decode("utf8")))

            # build usage counts
            for i in USFMRE.findall(intext.decode("utf8")):
                count += 1
                counttags[i] += 1

    # split tags into known and unknown sets
    unknownset = knownset.difference(KNOWNTAGS)
    knownset = knownset.intersection(KNOWNTAGS)

    # output results.
    print()
    if len(knownset) > 0:
        print("Known USFM Tags: {}\n".format(", ".join(sorted(knownset))))
    if len(unknownset) > 0:
        print("Unknown USFM Tags: {}\n".format(", ".join(sorted(unknownset))))

    # print tag usage counts
    if args.c:
        print("\nTag usage count:\n")
        for i in sorted(counttags):
            print("{: 8} - {}".format(counttags[i], i))
        print("\nTotal number of usfm tags found:   {}\n".format(count))

def main():
    """Process command line arguments and then process files."""
    parser = argparse.ArgumentParser(
        description="""
            A simple script to generate a list of usfm tags that were used in
           one or more utf8 encoded usfm files.
        """,
        epilog="""
            * Version: {} * This script is public domain *
        """.format(
            VERSION
        ),
    )
    parser.add_argument(
        "-c", help="include usage counts for tags", action="store_true"
    )
    parser.add_argument(
        "file", help="name of file to process (wildcards allowed)", nargs="+"
    )
    args = parser.parse_args()

    processtags(args)

# -------------------------------------------------------------------------- #

if __name__ == "__main__":
    main()
<commit_msg>Minor code fix.<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

"""
List ufm tags.

A simple script to generate a list of usfm tags that were used in one
or more utf8 encoded usfm files. Compatible with python2 and python3.

This script is public domain.

"""

from __future__ import print_function, unicode_literals
import argparse
import re
import collections
import glob

# -------------------------------------------------------------------------- #

VERSION = "1.1.1"

# -------------------------------------------------------------------------- #

# set of known usfm tags in version 3.0 of usfm specification
# I think I have all the tags here.
KNOWNTAGS = {
    # identification
    r"\id",
    r"\ide",
    r"\sts",
    r"\rem",
    r"\h",
    r"\h1",
    r"\h2",
    r"\h3",
    r"\toc1",
    r"\toc2",
    r"\toc3",
    # introductions
    r"\imt",
    r"\imt1",
    r"\imt2",
    r"\imt3",
    r"\imt4",
    r"\is",
    r"\is1",
    r"\is2",
    r"\ip",
    r"\ipi",
    r"\im",
    r"\imi",
    r"\ipq",
    r"\imq",
    r"\ipr",
    r"\iq",
    r"\iq1",
    r"\iq2",
    r"\iq3",
    r"\ib",
    r"\ili",
    r"\ili1",
    r"\ili2",
    r"\iot",
    r"\io",
    r"\io1",
    r"\io2",
    r"\io3",
    r"\io4",
    r"\ior",
    r"\ior*",
    r"\iex",
    r"\iqt",
    r"\iqt*",
    r"\imte",
    r"\imte1",
    r"\imte2",
    r"\ie",
    # titles, headings, labels
    r"\mt",
    r"\mt1",
    r"\mt2",
    r"\mt3",
    r"\mt4",
    r"\mte",
    r"\mte1",
    r"\mte2",
    r"\ms",
    r"\ms1",
    r"\ms2",
    r"\ms3",
    r"\mr",
    r"\s",
    r"\s1",
    r"\s2",
    r"\s3",
    r"\s4",
    r"\sr",
    r"\r",
    r"\rq",
    r"\rq*",
    r"\d",
    r"\sp",
    # chapters and verses
    r"\c",
    r"\ca",
    r"\ca*",
    r"\cl",
    r"\cp",
    r"\cd",
    r"\v",
    r"\va",
    r"\va*",
    r"\vp",
    r"\vp*",
    # paragraphs/poetry
    r"\p",
    r"\po",
    r"\m",
    r"\pmo",
    r"\pm",
    r"\pmc",
    r"\pmr",
    r"\pi",
    r"\pi1",
    r"\pi2",
    r"\pi3",
    r"\mi",
    r"\nb",
    r"\cls",
    r"\lh",
    r"\lf",
    r"\li",
    r"\li1",
    r"\li2",
    r"\li3",
    r"\li4",
    r"\lim",
    r"\lim1",
    r"\lim2",
    r"\lim3",
    r"\lim4",
    r"\lim5",
    r"\pc",
    r"\pr",
    r"\ph",
    r"\ph1",
    r"\ph2",
    r"\ph3",
    r"\q",
    r"\q1",
    r"\q2",
    r"\q3",
    r"\q4",
    r"\qr",
    r"\qc",
    r"\qs",
    r"\qs*",
    r"\qa",
    r"\qac",
    r"\qac*",
    r"\qm",
    r"\qm1",
    r"\qm2",
    r"\qm3",
    r"\qd",
    r"\lik",
    r"\lik*",
    r"\litl",
    r"\litl*",
    r"\ts",
    r"\sd",
    r"\sd1",
    r"\sd2",
    r"\sd3",
    r"\sd4",
    r"\sd5",
    r"\b",
    # tables
    r"\tr",
    r"\th",
    r"\th1",
    r"\th2",
    r"\th3",
    r"\th4",
    r"\th5",
    r"\thr",
    r"\thr1",
    r"\thr2",
    r"\thr3",
    r"\thr4",
    r"\thr5",
    r"\tc",
    r"\tc1",
    r"\tc2",
    r"\tc3",
    r"\tc4",
    r"\tc5",
    r"\tcr",
    r"\tcr1",
    r"\tcr2",
    r"\tcr3",
    r"\tcr4",
    r"\tcr5",
    # footnotes
    r"\f",
    r"\f*",
    r"\fe",
    r"\fe*",
    r"\fr",
    r"\fk",
    r"\fq",
    r"\fqa",
    r"\fl",
    r"\fp",
    r"\fv",
    r"\fv*",
    r"\ft",
    r"\fdc",
    r"\fdc*",
    r"\fm",
    r"\fm*",
    # cross references
    r"\x",
    r"\x*",
    r"\xo",
    r"\xk",
    r"\xq",
    r"\xt",
    r"\xot",
    r"\xot*",
    r"\xnt",
    r"\xnt*",
    r"\xdc",
    r"\xdc*",
    r"\xta",
    r"\xta*",
    r"\xop",
    r"\xop*",
    # special text
    r"\add",
    r"\add*",
    r"\bk",
    r"\bk*",
    r"\dc",
    r"\dc*",
    r"\k",
    r"\k*",
    r"\lit",
    r"\nd",
    r"\nd*",
    r"\ord",
    r"\ord*",
    r"\pn",
    r"\pn*",
    r"\qt",
    r"\qt*",
    r"\sig",
    r"\sig*",
    r"\sls",
    r"\sls*",
    r"\tl",
    r"\tl*",
    r"\wj",
    r"\wj*",
    r"\addpn",
    r"\addpn*",
    r"\+add",
    r"\+add*",
    r"\+bk",
    r"\+bk*",
    r"\+dc",
    r"\+dc*",
    r"\+k",
    r"\+k*",
    r"\+nd",
    r"\+nd*",
    r"\+ord",
    r"\+ord*",
    r"\+pn",
    r"\+pn*",
    r"\+qt",
    r"\+qt*",
    r"\+sig",
    r"\+sig*",
    r"\+sls",
    r"\+sls*",
    r"\+tl",
    r"\+tl*",
    r"\+wj",
    r"\+wj*",
    r"\+addpn",
    r"\+addpn*",
    # character styles
    r"\em",
    r"\em*",
    r"\bd",
    r"\bd*",
    r"\it",
    r"\it*",
    r"\bdit",
    r"\bdit*",
    r"\no",
    r"\no*",
    r"\sc",
    r"\sc*",
    r"\+em",
    r"\+em*",
    r"\+bd",
    r"\+bd*",
    r"\+it",
    r"\+it*",
    r"\+bdit",
    r"\+bdit*",
    r"\+no",
    r"\+no*",
    r"\+sc",
    r"\+sc*",
    # spacing
    r"\pb",
    # special features
    r"\fig",
    r"\fig*",
    r"\ndx",
    r"\ndx*",
    r"\pro",
    r"\pro*",
    r"\w",
    r"\w*",
    r"\wg",
    r"\wg*",
    r"\wh",
    r"\wh*",
    r"\wa",
    r"\wa*",
    r"\png",
    r"\png*",
    r"\jmp",
    r"\jmp*",
    r"\rb",
    r"\rb*",
    r"\rt",
    r"\rt*",
    # peripherals
    r"\periph",
    # study bible content
    r"\ef",
    r"\ef*",
    r"\ex",
    r"\ex*",
    r"\esb",
    r"\esbe",
    r"\cat",
    r"\cat*",
    # milestone quotations
    r"\qt-s",
    r"\qt1-s",
    r"\qt2-s",
    r"\qt3-s",
    r"\qt4-s",
    r"\qt5-s",
    r"\qt-e",
    r"\qt1-e",
    r"\qt2-e",
    r"\qt3-e",
    r"\qt4-e",
    r"\qt5-e",
    r"\*",
}

# regex for finding usfm tags
USFMRE = re.compile(
    r"""
    # handle usfm tags other than special spacing...
    (?:
        # the first character of a usfm tag is always a backslash
        \\

        # a plus symbol marks the start of a nested character style.
        # this may or may not be present.
        \+?

        # tag names are ascii letters
        [A-Za-z]+

        # tags may or may not be numbered
        [0-9]?

        # a word boundary to mark the end of our tags.
        \b

        # character style closing tags end with an asterisk.
        \*?
    )

    # OR...
    |

    # one of the two special spacing tags.
    (?:
        (?:~|//)
    )
""",
    re.U + re.VERBOSE,
)

# -------------------------------------------------------------------------- #

def processtags(args):
    """Process usfm tags in all files."""
    count = 0
    counttags = collections.Counter()
    knownset = set()

    filenames = []
    for _ in args.file:
        if "*" in _:
            filenames.extend(glob.glob(_))
        else:
            filenames.append(_)

    for fname in filenames:
        with open(fname, "rb") as infile:
            intext = infile.read()

            # build tag set
            knownset.update(USFMRE.findall(intext.decode("utf8")))

            # build usage counts
            for i in USFMRE.findall(intext.decode("utf8")):
                count += 1
                counttags[i] += 1

    # split tags into known and unknown sets
    unknownset = knownset.difference(KNOWNTAGS)
    knownset = knownset.intersection(KNOWNTAGS)

    # output results.
    print()
    if knownset:
        print("Known USFM Tags: {}\n".format(", ".join(sorted(knownset))))
    if unknownset:
        print("Unknown USFM Tags: {}\n".format(", ".join(sorted(unknownset))))

    # print tag usage counts
    if args.c:
        print("\nTag usage count:\n")
        for i in sorted(counttags):
            print("{: 8} - {}".format(counttags[i], i))
        print("\nTotal number of usfm tags found:   {}\n".format(count))

def main():
    """Process command line arguments and then process files."""
    parser = argparse.ArgumentParser(
        description="""
            A simple script to generate a list of usfm tags that were used in
           one or more utf8 encoded usfm files.
        """,
        epilog="""
            * Version: {} * This script is public domain *
        """.format(
            VERSION
        ),
    )
    parser.add_argument(
        "-c", help="include usage counts for tags", action="store_true"
    )
    parser.add_argument(
        "file", help="name of file to process (wildcards allowed)", nargs="+"
    )
    args = parser.parse_args()

    processtags(args)

# -------------------------------------------------------------------------- #

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import json
from time import sleep

try:
    from urllib.request import urlopen
    from urllib.parse import urlencode
except ImportError:
    from urllib import urlopen, urlencode

TIME_SLEEP_TOO_MANY_REQUESTS = 1.2

class InvalidToken(Exception):
    pass

class MissingToken(Exception):
    pass

class MessageNotFound(Exception):
    pass

class RateLimiterReached(Exception):
    pass

class Message(object):
    """Message Object for Mailinator Email API
    """
    _baseURL = 'http://api.mailinator.com/api/email'

    def __init__(self, token, data):
        self.token = token
        self.id = data['id']
        self.subject = data['subject']
        self.time = data['time']
        self.to = data['to']
        self.seconds_ago = data['seconds_ago']
        self.fromfull = data['fromfull']
        self.been_read = data['been_read']
        self.fromshort = data['from']
        self.ip = data['ip']
        self.headers = {}
        self.body = ""

    def get_message(self):
        query_string = {'token': self.token, 'msgid': self.id}
        url = self._baseURL + "?" + urlencode(query_string)
        request = get_request(url)
        if request.getcode() == 404:
            raise MessageNotFound
        response = request.read()
        data = json.loads(clean_response(response), strict=False)
        if data.get('error', False):
            if data.get('error').lower() is "rate limiter reached":
                raise RateLimiterReached
        self.headers = data.get('data').get('headers')
        self.body = "".join([part['body'] for part in data['data']['parts']])

class Inbox(object):
    """Inbox Object for retrieving an inbox

    Args:
        token: An api token, from http://www.mailinator.com/settings.jsp
    Methods:
        get:
            Args:
                mailbox: optional argument for the name of an inbox
    Returns:

    Raises:
        InvalidToken: When the api token is invalid
        MissingToken: When run without a token

    """
    _baseURL = "http://api.mailinator.com/api/inbox"

    def __init__(self, token):
        self.token = token
        self.messages = []

    def get(self, mailbox=None, private_domain=False):
        """Retrieves email from inbox"""
        if not self.token:
            raise MissingToken
        query_string = {'token': self.token}
        if mailbox:
            query_string.update({'to': mailbox})
        if private_domain:
            query_string.update({'private_domain': json.dumps(private_domain)})
        url = self._baseURL + '?' + urlencode(query_string)
        request = get_request(url)
        if request.getcode() == 400:
            raise InvalidToken
        response = request.read()
        return self._parse(response)

    def count(self):
        """returns the number of emails in inbox"""
        return len(self.messages)

    def view_subjects(self):
        """returns a list of messages subjects"""
        return [message.subject for message in self.messages]

    def view_message_ids(self):
        """returns a list of message ids"""
        return [message.id for message in self.messages]

    def get_message_by_subject(self, subject):
        """returns a message object with a subject line"""
        messages = [message for message in self.messages if message.subject.lower() == subject.lower()]
        if len(messages) == 0:
            return None
        elif len(messages) == 1:
            return messages[0]
        else:
            return messages

    def get_message_by_id(self, msgid):
        """returns a message object by id"""
        messages = [message for message in self.messages if message.id == msgid]
        if len(messages) == 0:
            return None
        elif len(messages) == 1:
            return messages[0]

    def filter(self, field, value):
        """returns a filtered list of messages by where message.field = value"""
        return [message for message in self.messages if getattr(message, field) == value]

    def _parse(self, data):
        self.messages = []
        parsed = json.loads(clean_response(data), strict=False)
        for message in parsed['messages']:
            email = Message(self.token, message)
            self.messages.append(email)
        return self.messages

def clean_response(response):
    if not isinstance(response, str):
        return response.decode('utf-8', 'ignore')
    else:
        return response

def get_request(url, sleep_time=.7):
    while True:
        request = urlopen(url)
        if request.getcode() != 429:
            break
        sleep(TIME_SLEEP_TOO_MANY_REQUESTS)
    return request
<commit_msg>edited time sleep const<commit_after>import json
from time import sleep

try:
    from urllib.request import urlopen
    from urllib.parse import urlencode
except ImportError:
    from urllib import urlopen, urlencode

TIME_SLEEP_TOO_MANY_REQUESTS = .7

class InvalidToken(Exception):
    pass

class MissingToken(Exception):
    pass

class MessageNotFound(Exception):
    pass

class RateLimiterReached(Exception):
    pass

class Message(object):
    """Message Object for Mailinator Email API
    """
    _baseURL = 'http://api.mailinator.com/api/email'

    def __init__(self, token, data):
        self.token = token
        self.id = data['id']
        self.subject = data['subject']
        self.time = data['time']
        self.to = data['to']
        self.seconds_ago = data['seconds_ago']
        self.fromfull = data['fromfull']
        self.been_read = data['been_read']
        self.fromshort = data['from']
        self.ip = data['ip']
        self.headers = {}
        self.body = ""

    def get_message(self):
        query_string = {'token': self.token, 'msgid': self.id}
        url = self._baseURL + "?" + urlencode(query_string)
        request = get_request(url)
        if request.getcode() == 404:
            raise MessageNotFound
        response = request.read()
        data = json.loads(clean_response(response), strict=False)
        if data.get('error', False):
            if data.get('error').lower() is "rate limiter reached":
                raise RateLimiterReached
        self.headers = data.get('data').get('headers')
        self.body = "".join([part['body'] for part in data['data']['parts']])

class Inbox(object):
    """Inbox Object for retrieving an inbox

    Args:
        token: An api token, from http://www.mailinator.com/settings.jsp
    Methods:
        get:
            Args:
                mailbox: optional argument for the name of an inbox
    Returns:

    Raises:
        InvalidToken: When the api token is invalid
        MissingToken: When run without a token

    """
    _baseURL = "http://api.mailinator.com/api/inbox"

    def __init__(self, token):
        self.token = token
        self.messages = []

    def get(self, mailbox=None, private_domain=False):
        """Retrieves email from inbox"""
        if not self.token:
            raise MissingToken
        query_string = {'token': self.token}
        if mailbox:
            query_string.update({'to': mailbox})
        if private_domain:
            query_string.update({'private_domain': json.dumps(private_domain)})
        url = self._baseURL + '?' + urlencode(query_string)
        request = get_request(url)
        if request.getcode() == 400:
            raise InvalidToken
        response = request.read()
        return self._parse(response)

    def count(self):
        """returns the number of emails in inbox"""
        return len(self.messages)

    def view_subjects(self):
        """returns a list of messages subjects"""
        return [message.subject for message in self.messages]

    def view_message_ids(self):
        """returns a list of message ids"""
        return [message.id for message in self.messages]

    def get_message_by_subject(self, subject):
        """returns a message object with a subject line"""
        messages = [message for message in self.messages if message.subject.lower() == subject.lower()]
        if len(messages) == 0:
            return None
        elif len(messages) == 1:
            return messages[0]
        else:
            return messages

    def get_message_by_id(self, msgid):
        """returns a message object by id"""
        messages = [message for message in self.messages if message.id == msgid]
        if len(messages) == 0:
            return None
        elif len(messages) == 1:
            return messages[0]

    def filter(self, field, value):
        """returns a filtered list of messages by where message.field = value"""
        return [message for message in self.messages if getattr(message, field) == value]

    def _parse(self, data):
        self.messages = []
        parsed = json.loads(clean_response(data), strict=False)
        for message in parsed['messages']:
            email = Message(self.token, message)
            self.messages.append(email)
        return self.messages

def clean_response(response):
    if not isinstance(response, str):
        return response.decode('utf-8', 'ignore')
    else:
        return response

def get_request(url, sleep_time=.7):
    while True:
        request = urlopen(url)
        if request.getcode() != 429:
            break
        sleep(TIME_SLEEP_TOO_MANY_REQUESTS)
    return request
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
Created on May 5, 2011

@author: Radoslaw Palczynski, Grzegorz Bilewski et al.
'''

import sys
import argparse
import tempfile
import Image
import ImageDraw
from selenium import webdriver
from selenium.webdriver.remote.webdriver import WebDriver
from selenium.common.exceptions import NoSuchElementException
from types import MethodType

def create(driver = None):
    # hiding everything from the world, buahaha ^_^

    def check_url(url):
        if not isinstance(url, basestring):
            raise ValueError("i don't understand your url :(")

        if url[:7] != "http://":
            raise ValueError("http protocol is required")

        return url

    def get_web_element_by_id(driver, id):
        element = None
        try:
            element = driver.find_element_by_id(id)

            if not element.is_displayed() or element.size['width'] == 0 or element.size['height'] == 0:
                return None
        except NoSuchElementException:
            pass

        return element

    def get_web_element_by_xpath(driver, xpath):
        element = None
        try:
            element = driver.find_element_by_xpath(xpath)

            if not element.is_displayed() or element.size['width'] == 0 or element.size['height'] == 0:
                return None
        except NoSuchElementException:
            pass

        return element

    def get_web_element_box_size(web_element):
        location = web_element.location
        size = web_element.size
        left = location['x']
        right = location['x'] + size['width']
        top = location['y']
        down = location['y'] + size['height']
        # box of region to crop
        box = (left, top, right, down)
        return box

    class ScreenShot(object):
        def __init__(self, driver):
            self.driver = driver

        def get_screen(self, url = None):
            """
            Get specified screen(s)

            @param url: web page to capture (including http protocol, None to reuse loaded webpage)
            """

            if url is not None:
                url = check_url(url)
                self.driver.get(url)
            elif self.driver.current_url == "about:blank":
                raise Exception("No page loaded")

            return get_screen(self.driver)

        def close(self):
            self.driver.close()

    class ImageContainer(object):

        def __init__(self, image, driver, cut = False):
            self.cut = cut
            self.driver = driver
            if image is None:
                raise Exception("Please provide an image.")
            elif isinstance(image, Image.Image):
                self.image = image
            else:
                self.tempfd = image
                self.image = Image.open(self.tempfd)

        def cut_element(self, id = None, xpath = None):
            """
            Crop one element by id or xpath
            return ImageContainer
            :param id:
            :param xpath:
            """
            if self.cut is True:
                raise Exception('You cannot cut more elements')
            elif (id is None) and (xpath is None):
                raise Exception("Please provide id or xpath.")
            elif id is not None:
                my_element = get_web_element_by_id(self.driver, id)
                box = get_web_element_box_size(my_element)
                new_image = self.image.crop(box)
                return ImageContainer(new_image, self.driver, True)
            elif xpath is not None:
                my_element = get_web_element_by_xpath(self.driver, xpath)
                box = get_web_element_box_size(my_element)
                new_image = self.image.crop(box)
                return ImageContainer(new_image, self.driver, True)

        def cut_area(self, x = 0, y = 0, height = None, width = None):
            """
            Crop page vertically from a given point to a given size (in px)
            return ImageContainer
            :param x:
            :param y:
            :param height:
            :param width:
            """
            if height is None:
                height = self.image.size[1] - y
            if width is None:
                width = self.image.size[0] - x

            box = (x, y, width + x, height + y)
            new_image = self.image.crop(box)
            return ImageContainer(new_image, self.driver, True)

        def draw_dot(self, id = None, xpath = None, coordinates = None, padding = 0, color = None, size = None):
            """
            Draw a red dot on the left of a given element (resize image to add space on left if required)
            coordinates = (x, y) - center of a dot
            Use PIL to draw elements, no JavaScript allowed.
            return ImageContainer
            :param id:
            :param xpath:
            :param coordinates:
            :param padding:
            :param color:
            :param size:
            """
            if color is None:
                color = 'red'
            if size is None:
                size = 1
            new_image = self.image.copy()
            draw = ImageDraw.Draw(new_image)
            if id is None and xpath is None and coordinates is None:
                raise Exception("Please provide id or xpath.")
            elif id is not None and self.cut is False:
                my_element = get_web_element_by_id(self.driver, id)
                box = get_web_element_box_size(my_element)
                x = box[0] - 1
                y = box[1] + int((box[3] - box[1]) / 2)
                dot_box = (x - size - size - padding,
                           y - size,
                           x - padding,
                           y + size)
                if dot_box[0] < 0:
                    additional_space = 2
                    difference = -dot_box[0]
                    bigger_image = Image.new('RGB',
                                             (new_image.size[0] + difference + additional_space, new_image.size[1]),
                                             "white")
                    bigger_image.paste(new_image, (difference + additional_space, 0))

                    dot_box = (dot_box[0] + difference + additional_space,
                               dot_box[1],
                               dot_box[2] + difference + additional_space,
                               dot_box[3])
                    draw = ImageDraw.Draw(bigger_image)
                    draw.ellipse(dot_box, fill = color, outline = color)
                    return ImageContainer(bigger_image, self.driver)
                else:
                    draw.ellipse(dot_box, fill = color, outline = color)
                    return ImageContainer(new_image, self.driver)

            elif xpath is not None and self.cut is False:
                my_element = get_web_element_by_xpath(self.driver, xpath)
                box = get_web_element_box_size(my_element)
                x = box[0] - 1
                y = box[1] + int((box[3] - box[1]) / 2)
                dot_box = (x - size - size - padding,
                           y - size,
                           x - padding,
                           y + size)
                if dot_box[0] < 0:
                    additional_space = 2
                    difference = -dot_box[0]
                    bigger_image = Image.new('RGB',
                                             (new_image.size[0] + difference + additional_space, new_image.size[1]),
                                             "white")
                    bigger_image.paste(new_image, (difference + additional_space, 0))

                    dot_box = (dot_box[0] + difference + additional_space,
                               dot_box[1],
                               dot_box[2] + difference + additional_space,
                               dot_box[3])
                    draw = ImageDraw.Draw(bigger_image)
                    draw.ellipse(dot_box, fill = color, outline = color)
                    return ImageContainer(bigger_image, self.driver)
                else:
                    draw.ellipse(dot_box, fill = color, outline = color)
                    return ImageContainer(new_image, self.driver)

            elif coordinates is not None:
                box = (coordinates[0] - size - padding,
                       coordinates[1] - size,
                       coordinates[0] + size - padding,
                       coordinates[1] + size)

                draw.ellipse(box, fill = color, outline = color)

                return ImageContainer(new_image, self.driver)
            del draw

        def draw_frame(self, id = None, xpath = None, coordinates = None, padding = None, color = None, size = None):
            """
            Draw a frame around a given element
            coordinates = (x, y, width, height) - middle of a dot
            Use PIL to draw elements, no JavaScript allowed.
            return ImageContainer
            :param id:
            :param xpath:
            :param coordinates:
            :param padding:
            :param color:
            :param size:
            """
            color = color if color is not None else "red"
            size = size if size is not None else 0
            new_image = self.image.copy()
            draw = ImageDraw.Draw(new_image)
            if id is None and xpath is None and coordinates is None:
                raise Exception("Please provide id or xpath.")
            elif id is not None and self.cut is False:
                my_element = get_web_element_by_id(self.driver, id)
                box = [i for i in get_web_element_box_size(my_element)]
                if padding is not None:
                    box[0] = box[0] - padding
                    box[1] = box[1] - padding
                    box[2] = box[2] + padding
                    box[3] = box[3] + padding
                frame = ((box[0], box[1]), (box[2], box[1]), (box[2], box[3]), (box[0], box[3]), (box[0], box[1]))
                draw.line(frame, fill = color, width = size)
                return ImageContainer(new_image, self.driver)
            elif xpath is not None and self.cut is False:
                my_element = get_web_element_by_xpath(self.driver, xpath)
                box = [i for i in get_web_element_box_size(my_element)]
                if padding is not None:
                    box[0] = box[0] - padding
                    box[1] = box[1] - padding
                    box[2] = box[2] + padding
                    box[3] = box[3] + padding
                frame = ((box[0], box[1]), (box[2], box[1]), (box[2], box[3]), (box[0], box[3]), (box[0], box[1]))
                draw.line(frame, fill = color, width = size)

                return ImageContainer(new_image, self.driver)
            elif coordinates is not None:
                box = [
                    coordinates[0] - int(coordinates[2] / 2),
                    coordinates[1] - int(coordinates[3] / 2),
                    coordinates[0] + int(coordinates[2] / 2),
                    coordinates[1] + int(coordinates[3] / 2)
                ]

                if padding is not None:
                    box[0] = box[0] - padding
                    box[1] = box[1] - padding
                    box[2] = box[2] + padding
                    box[3] = box[3] + padding
                frame = ((box[0], box[1]), (box[2], box[1]), (box[2], box[3]), (box[0], box[3]), (box[0], box[1]))
                draw.line(frame, fill = color, width = size)
                return ImageContainer(new_image, self.driver)
            del draw

        def save(self, filename):
            """
            Save to a filename
            :param filename:
            """
            self.image.save(filename, "PNG")
            return self

        def close(self):
            self.driver.close()

    def get_screen(driver):
        """
        :param driver:
        :return:
        """
        tempfd = tempfile.NamedTemporaryFile(mode = 'w+t', delete = False)
        driver.save_screenshot(tempfd.name)
        temp_filename = tempfd.name
        tempfd.close()
        return ImageContainer(temp_filename, driver)

    #########################
    #          body         #
    #########################

    if driver is None:
        # no parameter provided, create the default driver

        return ScreenShot(webdriver.Firefox())
    elif isinstance(driver, WebDriver):
        # an instance of a class/webdriver
        # will add get_screen to it

        if "get_screen" not in dir(driver):
            driver.get_screen = MethodType(get_screen, driver, driver.__class__)

        return driver
    elif isinstance(driver, WebDriver) is False and isinstance(driver, type) is True:
        # a class
        # will create an instance and rerun create function to add get_screen function

        return create(driver())
    else:
        raise Exception("There is something strange with the driver, will you check it?")

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description = 'Takes a screen shot of a web page.')
    parser.add_argument('-u', '--url', dest = "url", help = "url to web page (including http protocol)",
                        required = True)
    parser.add_argument('-i', '--ids', dest = "ids",
                        help = "list of ids on the web page separated by a space character",
                        nargs = '+')
    parser.add_argument('-x', '--xpath', dest = "xpath",
                        help = "list of xpath on the web page separated by a space character", nargs = '+')
    parser.add_argument('-d', '--path', dest = "path", help = "path to save directory; default as run script",
                        default = ".")
    parser.add_argument('-r', '--remoteUrl', dest = "remoteUrl", help = "url of selenium-server-standalone")
    # parser.add_argument('-f', '--format', dest="format", help="choose a code's output [opt: xml, json]", default=None)

    args = parser.parse_args()

    if args.url[:7] != "http://":
        print sys.argv[0] + ": error: argument -u/--url requires http protocol"
        sys.exit(2)

    if args.remoteUrl:
        s = create(webdriver.Remote(command_executor = args.remoteUrl, desired_capabilities = {
            "browserName": "firefox",
            "platform": "ANY",
        }))
        s.get(args.url)
        s.get_screen(args.ids, args.xpath, args.path)
    else:
        s = create()
        s.get_screen(args.url, args.ids, args.xpath, args.path)
    s.close()

<commit_msg>refactoring<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
Created on May 5, 2011

@author: Radoslaw Palczynski, Grzegorz Bilewski et al.
'''

import sys
import argparse
import tempfile
import Image
import ImageDraw
from selenium import webdriver
from selenium.webdriver.remote.webdriver import WebDriver
from selenium.common.exceptions import NoSuchElementException
from types import MethodType

def create(driver = None):
    # hiding everything from the world, buahaha ^_^

    def check_url(url):
        if not isinstance(url, basestring):
            raise ValueError("i don't understand your url :(")

        if url[:7] != "http://":
            raise ValueError("http protocol is required")

        return url

    def get_web_element_by_id(driver, id):
        element = None
        try:
            element = driver.find_element_by_id(id)

            if not element.is_displayed() or element.size['width'] == 0 or element.size['height'] == 0:
                return None
        except NoSuchElementException:
            pass

        return element

    def get_web_element_by_xpath(driver, xpath):
        element = None
        try:
            element = driver.find_element_by_xpath(xpath)

            if not element.is_displayed() or element.size['width'] == 0 or element.size['height'] == 0:
                return None
        except NoSuchElementException:
            pass

        return element

    def get_web_element_box_size(web_element):
        location = web_element.location
        size = web_element.size
        left = location['x']
        right = location['x'] + size['width']
        top = location['y']
        down = location['y'] + size['height']
        # box of region to crop
        box = (left, top, right, down)
        return box

    class ScreenShot(object):
        def __init__(self, driver):
            self.driver = driver

        def get_screen(self, url = None):
            """
            Get specified screen(s)

            @param url: web page to capture (including http protocol, None to reuse loaded webpage)
            """

            if url is not None:
                url = check_url(url)
                self.driver.get(url)
            elif self.driver.current_url == "about:blank":
                raise Exception("No page loaded")

            return get_screen(self.driver)

        def close(self):
            self.driver.close()

    class ImageContainer(object):

        def __init__(self, image, driver, cut = False):
            self.cut = cut
            self.driver = driver
            if image is None:
                raise Exception("Please provide an image.")
            elif isinstance(image, Image.Image):
                self.image = image
            else:
                self.filename = image
                self.image = Image.open(self.filename)

        def cut_element(self, id = None, xpath = None):
            """
            Crop one element by id or xpath
            return ImageContainer
            :param id:
            :param xpath:
            """
            if self.cut is True:
                raise Exception('You cannot cut more elements')

            elif id is not None:
                my_element = get_web_element_by_id(self.driver, id)
                box = get_web_element_box_size(my_element)
                new_image = self.image.crop(box)
                return ImageContainer(new_image, self.driver, True)
            elif xpath is not None:
                my_element = get_web_element_by_xpath(self.driver, xpath)
                box = get_web_element_box_size(my_element)
                new_image = self.image.crop(box)
                return ImageContainer(new_image, self.driver, True)
            else:
                raise ValueError("Please provide id or xpath.")

        def cut_area(self, x = 0, y = 0, height = None, width = None):
            """
            Crop page vertically from a given point to a given size (in px)
            return ImageContainer
            :param x:
            :param y:
            :param height:
            :param width:
            """
            height = height if height is not None else self.image.size[1] - y
            width = width if width is not None else self.image.size[0] - x
            box = (x, y, width + x, height + y)
            new_image = self.image.crop(box)
            return ImageContainer(new_image, self.driver, True)

        def draw_dot(self, id = None, xpath = None, coordinates = None, padding = 0, color = None, size = None):
            """
            Draw a red dot on the left of a given element (resize image to add space on left if required)
            coordinates = (x, y) - center of a dot
            Use PIL to draw elements, no JavaScript allowed.
            return ImageContainer
            :param id:
            :param xpath:
            :param coordinates:
            :param padding:
            :param color:
            :param size:
            """
            color = color if color is not None else "red"
            size = size if size is not None else 1
            new_image = self.image.copy()
            draw = ImageDraw.Draw(new_image)

            if id is not None and self.cut is False:
                my_element = get_web_element_by_id(self.driver, id)
                box = get_web_element_box_size(my_element)
            elif xpath is not None and self.cut is False:
                my_element = get_web_element_by_xpath(self.driver, xpath)
                box = get_web_element_box_size(my_element)
            elif coordinates is not None:
                box = (coordinates[0] - size - padding,
                       coordinates[1] - size,
                       coordinates[0] + size - padding,
                       coordinates[1] + size)

                draw.ellipse(box, fill = color, outline = color)

                return ImageContainer(new_image, self.driver)
            else:
                del draw
                raise ValueError("Please provide id or xpath or coordinates")
            x = box[0] - 1
            y = box[1] + int((box[3] - box[1]) / 2)
            dot_box = (x - size - size - padding,
                       y - size,
                       x - padding,
                       y + size)
            if dot_box[0] < 0:
                additional_space = 2
                difference = -dot_box[0]
                bigger_image = Image.new('RGB',
                                         (new_image.size[0] + difference + additional_space, new_image.size[1]),
                                         "white")
                bigger_image.paste(new_image, (difference + additional_space, 0))

                dot_box = (dot_box[0] + difference + additional_space,
                           dot_box[1],
                           dot_box[2] + difference + additional_space,
                           dot_box[3])
                draw = ImageDraw.Draw(bigger_image)
                draw.ellipse(dot_box, fill = color, outline = color)
                return ImageContainer(bigger_image, self.driver)
            else:
                draw.ellipse(dot_box, fill = color, outline = color)
                return ImageContainer(new_image, self.driver)

        def draw_frame(self, id = None, xpath = None, coordinates = None, padding = None, color = None, size = None):
            """
            Draw a frame around a given element
            coordinates = (x, y, width, height) - middle of a dot
            Use PIL to draw elements, no JavaScript allowed.
            return ImageContainer
            :param id:
            :param xpath:
            :param coordinates:
            :param padding:
            :param color:
            :param size:
            """
            color = color if color is not None else "red"
            size = size if size is not None else 0
            new_image = self.image.copy()
            draw = ImageDraw.Draw(new_image)
            if id is not None and self.cut is False:
                my_element = get_web_element_by_id(self.driver, id)
                box = [i for i in get_web_element_box_size(my_element)]
            elif xpath is not None and self.cut is False:
                my_element = get_web_element_by_xpath(self.driver, xpath)
                box = [i for i in get_web_element_box_size(my_element)]
            elif coordinates is not None:
                box = [
                    coordinates[0] - int(coordinates[2] / 2),
                    coordinates[1] - int(coordinates[3] / 2),
                    coordinates[0] + int(coordinates[2] / 2),
                    coordinates[1] + int(coordinates[3] / 2)
                ]
            else:
                del draw
                raise ValueError("Please provide id or xpath or coordinates")
            if padding is not None:
                box[0] = box[0] - padding
                box[1] = box[1] - padding
                box[2] = box[2] + padding
                box[3] = box[3] + padding
            frame = ((box[0], box[1]), (box[2], box[1]), (box[2], box[3]), (box[0], box[3]), (box[0], box[1]))
            draw.line(frame, fill = color, width = size)
            return ImageContainer(new_image, self.driver)

        def save(self, filename):
            """
            Save to a filename
            :param filename:
            """
            self.image.save(filename, "PNG")
            return self

        def close(self):
            self.driver.close()

    def get_screen(driver):
        """
        :param driver:
        :return:
        """
        tempfd = tempfile.NamedTemporaryFile(mode = 'w+t', delete = False)
        driver.save_screenshot(tempfd.name)
        temp_filename = tempfd.name
        tempfd.close()
        return ImageContainer(temp_filename, driver)

    #########################
    #          body         #
    #########################

    if driver is None:
        # no parameter provided, create the default driver

        return ScreenShot(webdriver.Firefox())
    elif isinstance(driver, WebDriver):
        # an instance of a class/webdriver
        # will add get_screen to it

        if "get_screen" not in dir(driver):
            driver.get_screen = MethodType(get_screen, driver, driver.__class__)

        return driver
    elif isinstance(driver, WebDriver) is False and isinstance(driver, type) is True:
        # a class
        # will create an instance and rerun create function to add get_screen function

        return create(driver())
    else:
        raise Exception("There is something strange with the driver, will you check it?")

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description = 'Takes a screen shot of a web page.')
    parser.add_argument('-u', '--url', dest = "url", help = "url to web page (including http protocol)",
                        required = True)
    parser.add_argument('-i', '--ids', dest = "ids",
                        help = "list of ids on the web page separated by a space character",
                        nargs = '+')
    parser.add_argument('-x', '--xpath', dest = "xpath",
                        help = "list of xpath on the web page separated by a space character", nargs = '+')
    parser.add_argument('-d', '--path', dest = "path", help = "path to save directory; default as run script",
                        default = ".")
    parser.add_argument('-r', '--remoteUrl', dest = "remoteUrl", help = "url of selenium-server-standalone")
    # parser.add_argument('-f', '--format', dest="format", help="choose a code's output [opt: xml, json]", default=None)

    args = parser.parse_args()

    if args.url[:7] != "http://":
        print sys.argv[0] + ": error: argument -u/--url requires http protocol"
        sys.exit(2)

    if args.remoteUrl:
        s = create(webdriver.Remote(command_executor = args.remoteUrl, desired_capabilities = {
            "browserName": "firefox",
            "platform": "ANY",
        }))
        s.get(args.url)
        s.get_screen(args.ids, args.xpath, args.path)
    else:
        s = create()
        s.get_screen(args.url, args.ids, args.xpath, args.path)
    s.close()

<|endoftext|>


<commit_before>from os import path
_PATHROOT = path.abspath(path.dirname(__file__))
__version__ = '0.2.1'
__author__ = "nickmilon"
__test__ = 1
<commit_msg>v 0.2.2<commit_after>from os import path
_PATHROOT = path.abspath(path.dirname(__file__))
__version__ = '0.2.2'
__author__ = "nickmilon"
__test__ = 2

<|endoftext|>


<commit_before>#encoding: utf8
from datetime import date, datetime

id = lambda x:x

integer = lambda x: int(x)
canonize_integer = lambda x: int(x.replace(",",""))
canonize_float = lambda x: float(x.replace(",",""))

def canonize_budget_code(code):
    code = str(code).replace("-","")
    if len(code) % 2 == 1:
        code = "0"+code
    code = "00"+code
    return code

def canonize_date(datestr):
    datestr = datestr.replace('ינו','1').replace('פבר','2').replace('מרץ','3').replace('אפר','4')\
                         .replace('מאי','5').replace('יונ','6').replace('יול','7').replace('אוג','8').replace('ספט','9')\
                         .replace('אוק','10').replace('נוב','11').replace('דצמ','12')
    datestr = datestr.replace('B1','').replace('Jan','1').replace('Feb','2').replace('Mar','3').replace('Apr','4')\
                       .replace('May','5').replace('Jun','6').replace('Jul','7').replace('Aug','8').replace('Sep','9')\
                       .replace('Oct','10').replace('Nov','11').replace('Dec','12')
    datestr = datestr.replace('B1','')
    if '-' in datestr:
        datestr = datestr.split('-')
        d = [ int(x) for x in datestr ]
        if d[2] < 100:
            d[2] += 2000
        datestr = "%s/%s/%s" % (d[0],d[1],d[2])
    elif "/" in datestr and int(datestr[-2:])<12:
        datestr = datestr.split('/')
        d = [ int(x) for x in datestr ]
        d[2] += 2000
        datestr = "%s/%s/%s" % (d[1],d[0],d[2])
    if datestr[-4:-2]=="20":
        out = datetime.strptime(datestr,"%d/%m/%Y").date()
    out = datetime.strftime(out,"%d/%m/%Y")
    return out

ngo_kind = lambda x:{u'חל"ץ':'cic','עמותה':'association'}[x]
<commit_msg>fix amutot convert to jsons<commit_after>#encoding: utf8
from datetime import date, datetime

id = lambda x:x

integer = lambda x: int(x)
canonize_integer = lambda x: int(x.replace(",",""))
canonize_float = lambda x: float(x.replace(",",""))

def canonize_budget_code(code):
    code = str(code).replace("-","")
    if len(code) % 2 == 1:
        code = "0"+code
    code = "00"+code
    return code

def canonize_date(datestr):
    datestr = datestr.replace('ינו','1').replace('פבר','2').replace('מרץ','3').replace('אפר','4')\
                         .replace('מאי','5').replace('יונ','6').replace('יול','7').replace('אוג','8').replace('ספט','9')\
                         .replace('אוק','10').replace('נוב','11').replace('דצמ','12')
    datestr = datestr.replace('B1','').replace('Jan','1').replace('Feb','2').replace('Mar','3').replace('Apr','4')\
                       .replace('May','5').replace('Jun','6').replace('Jul','7').replace('Aug','8').replace('Sep','9')\
                       .replace('Oct','10').replace('Nov','11').replace('Dec','12')
    datestr = datestr.replace('B1','')
    if '-' in datestr:
        datestr = datestr.split('-')
        d = [ int(x) for x in datestr ]
        if d[2] < 100:
            d[2] += 2000
        datestr = "%s/%s/%s" % (d[0],d[1],d[2])
    elif "/" in datestr and int(datestr[-2:])<12:
        datestr = datestr.split('/')
        d = [ int(x) for x in datestr ]
        d[2] += 2000
        datestr = "%s/%s/%s" % (d[1],d[0],d[2])
    if datestr[-4:-2]=="20":
        out = datetime.strptime(datestr,"%d/%m/%Y").date()
    out = datetime.strftime(out,"%d/%m/%Y")
    return out

ngo_kind = lambda x:{u'חל"ץ':'cic','עמותה':'association'}[x.decode('utf8')]
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import io
import pexpect
import string
import sys
import time

output_to=sys.stdout

args = list(sys.argv[1:])
logfile = open(args.pop(0), "w")
child = pexpect.spawn(' '.join(args))

def output_line(line_bits, last_skip):
  line = "".join(line_bits)
  sline = line.strip()

  skip = True
  if line.startswith(" ") and not sline.startswith('/'):
    skip = False

  if "Entering directory" in sline:
    skip = False

  if len(sline) > 1:
    if sline[0] in string.ascii_uppercase and sline[1] not in string.ascii_uppercase:
      skip = False
    if sline[0] in ('[', '=', '!', '+'):
      skip = False

  if skip != last_skip:
    output_to.write('\n')

  if skip:
    output_to.write('.')
  else:
    output_to.write(line)

  output_to.flush()
  line_bits.clear()
  return skip

def find_newline(data):
  fulldata = b"".join(data)
  newlinechar = fulldata.find(b'\n')
  retlinechar = fulldata.find(b'\r')

  if newlinechar == -1:
    newlinechar = len(fulldata)+1
  if retlinechar == -1:
    retlinechar = len(fulldata)+1

  if retlinechar+1 == newlinechar:
    splitpos = newlinechar
  else:
    splitpos = min(newlinechar, retlinechar)

  if splitpos > len(fulldata):
    return

  newline = fulldata[:splitpos+1]
  leftover = fulldata[splitpos+1:]

  data.clear()
  data.append(leftover)
  return newline

last_skip = False
cont = []
data = [b'']
while True:
  line = None
  while len(data) > 1 or len(data[0]) > 0 or child.isalive():
    line = find_newline(data)
    if line is not None:
      break
    try:
      data.append(child.read_nonblocking(100))
    except pexpect.TIMEOUT:
      pass
    except pexpect.EOF as e:
      data.append(b'\n')

  if not line:
    break

  line = line.decode('utf-8')
  logfile.write(line)
  logfile.flush()

  if line.endswith('\r'):
    cont.append(line[:-1])
    last_skip = output_line(cont, last_skip)
    cont.append('\r')
    continue

  sline = line.strip('\n\r')
  cont.append(sline)
  if sline.endswith('\\'):
    continue

  cont.append('\n')
  last_skip = output_line(cont, last_skip)

sys.exit(child.exitstatus)
<commit_msg>Skip some more stuff...<commit_after>#!/usr/bin/env python3

import io
import pexpect
import string
import sys
import time

output_to=sys.stdout

args = list(sys.argv[1:])
logfile = open(args.pop(0), "w")
child = pexpect.spawn(' '.join(args))

def output_line(line_bits, last_skip):
  line = "".join(line_bits)
  sline = line.strip()

  skip = True
  if line.startswith(" ") and not sline.startswith('/'):
    skip = False

  if "da es fi" in sline:
    skip = True
  if "setting rpath" in sline:
    skip = True

  if "Entering directory" in sline:
    skip = False
    sline = sline.split('make')[-1]

  if len(sline) > 1:
    if sline[0] in string.ascii_uppercase and sline[1] not in string.ascii_uppercase:
      skip = False
    if sline[0] in ('[', '=', '!', '+'):
      skip = False

  if skip != last_skip:
    output_to.write('\n')

  if skip:
    output_to.write('.')
  else:
    output_to.write(line)

  output_to.flush()
  line_bits.clear()
  return skip

def find_newline(data):
  fulldata = b"".join(data)
  newlinechar = fulldata.find(b'\n')
  retlinechar = fulldata.find(b'\r')

  if newlinechar == -1:
    newlinechar = len(fulldata)+1
  if retlinechar == -1:
    retlinechar = len(fulldata)+1

  if retlinechar+1 == newlinechar:
    splitpos = newlinechar
  else:
    splitpos = min(newlinechar, retlinechar)

  if splitpos > len(fulldata):
    return

  newline = fulldata[:splitpos+1]
  leftover = fulldata[splitpos+1:]

  data.clear()
  data.append(leftover)
  return newline

last_skip = False
cont = []
data = [b'']
while True:
  line = None
  while len(data) > 1 or len(data[0]) > 0 or child.isalive():
    line = find_newline(data)
    if line is not None:
      break
    try:
      data.append(child.read_nonblocking(100))
    except pexpect.TIMEOUT:
      pass
    except pexpect.EOF as e:
      data.append(b'\n')

  if not line:
    break

  line = line.decode('utf-8')
  logfile.write(line)
  logfile.flush()

  if line.endswith('\r'):
    cont.append(line[:-1])
    last_skip = output_line(cont, last_skip)
    cont.append('\r')
    continue

  sline = line.strip('\n\r')
  cont.append(sline)
  if sline.endswith('\\'):
    continue

  cont.append('\n')
  last_skip = output_line(cont, last_skip)

sys.exit(child.exitstatus)
<|endoftext|>


<commit_before>#=======================================================================
# Bits.py
#=======================================================================
# Module containing the Bits class.

from SignalValue import SignalValue

# NOTE: circular imports between Bits and helpers, using 'import helpers'
#       instead of 'from helpers import' ensures pydoc still works
import helpers
import copy

#-----------------------------------------------------------------------
# Bits
#-----------------------------------------------------------------------
# Class emulating limited precision values of a set bitwidth.
class Bits( SignalValue ):

  #---------------------------------------------------------------------
  # __init__
  #---------------------------------------------------------------------
  def __init__( self, nbits, value = 0, trunc = False ):

    value = int( value )
    nbits = int( nbits )

    # Make sure width is non-zero and that we have space for the value
    assert nbits > 0

    # Set the nbits and bitmask (_mask) attributes
    self.nbits = nbits
    self._max  = (2**nbits)- 1
    self._min  = -2**(nbits- 1) if nbits > 1 else 0
    self._mask = ( 1 << self.nbits ) - 1
    self.slice = slice( None )

    if not trunc:
      #assert nbits >= helpers.get_nbits( value )
      assert self._min <= value <= self._max

    # Convert negative values into unsigned ints and store them
    value_uint = value if ( value >= 0 ) else ( ~(-value) + 1 )
    self._uint = value_uint & self._mask

    self._target_bits = self

  #---------------------------------------------------------------------
  # __call__
  #---------------------------------------------------------------------
  # Allow Bits to act like a type that can be instantiated.
  def __call__( self ):
    return Bits( self.nbits )

  #---------------------------------------------------------------------
  # __int__
  #---------------------------------------------------------------------
  # Type conversion to an int.
  def __int__( self ):
    return int( self._uint )

  #---------------------------------------------------------------------
  # __long__
  #---------------------------------------------------------------------
  # Type conversion to a long.
  def __long__( self ):
    return long( self._uint )

  #---------------------------------------------------------------------
  # __index__
  #---------------------------------------------------------------------
  # Behave like an int() when used as an index to slices.
  def __index__( self ):
    return self._uint

  #---------------------------------------------------------------------
  # uint
  #---------------------------------------------------------------------
  # Return the unsigned integer representation of the bits.
  def uint( self ):
    return self._uint

  #---------------------------------------------------------------------
  # int
  #---------------------------------------------------------------------
  # Return the integer representation of the bits.
  def int( self ):
    if self[ self.nbits - 1]:
      twos_complement = ~self + 1
      return -twos_complement._uint
    else:
      return self._uint

  #---------------------------------------------------------------------
  # write_value
  #---------------------------------------------------------------------
  # Implementing abstract write_value method defined by SignalValue.
  def write_value( self, value ):
    value = int( value )
    assert self._min <= value <= self._max
    self._uint = (value & self._mask)

  #---------------------------------------------------------------------
  # write_next
  #---------------------------------------------------------------------
  # Implementing abstract write_next method defined by SignalValue.
  def write_next( self, value ):
    value = int( value )
    assert self._min <= value <= self._max
    self._next._uint = (value & self._mask)

  #---------------------------------------------------------------------
  # bit_length
  #---------------------------------------------------------------------
  # Implement bit_length method provided by the int built-in. Simplifies
  # the implementation of get_nbits().
  def bit_length( self ):
    return self._uint.bit_length()

  #---------------------------------------------------------------------
  # Print Methods
  #---------------------------------------------------------------------

  def __repr__(self):
    return "Bits( {0}, {1} )".format(self.nbits, self.hex())

  def __str__(self):
    num_chars = (((self.nbits-1)/4)+1)
    str = "{:x}".format(self._uint).zfill(num_chars)
    return str

  def __oct__( self ):
    print "DEPRECATED: Please use .oct()!"
    return self.oct()

  def __hex__( self ):
    print "DEPRECATED: Please use .oct()!"
    return self.hex()

  def bin(self):
    str = "{:b}".format(self._uint).zfill(self.nbits)
    return "0b"+str

  def oct( self ):
    num_chars = (((self.nbits-1)/2)+1)
    str = "{:o}".format(self._uint).zfill(num_chars)
    return "0o"+str

  def hex( self ):
    num_chars = (((self.nbits-1)/4)+1)
    str = "{:x}".format(self._uint).zfill(num_chars)
    return "0x"+str

  #----------------------------------------------------------------------
  # __getitem__
  #----------------------------------------------------------------------
  # Read a subset of bits in the Bits object.
  def __getitem__( self, addr ):

    # TODO: optimize this logic?

    # Handle slices
    if isinstance( addr, slice ):

      # Parse address range
      start = addr.start
      stop  = addr.stop

      # Open-ended range ( [:] ), return a copy of self
      if start is None and stop is None:
        return copy.copy( self )

      # Open-ended range on left ( [:N] )
      elif start is None:
        start = 0

      # Open-ended range on right ( [N:] )
      elif stop is None:
        stop = self.nbits

      stop  = int( stop  )
      start = int( start )

      # Verify our ranges are sane
      if not (start < stop):
        raise IndexError('Start index is not less than stop index')
      if not (0 <= start < stop <= self.nbits):
        raise IndexError('Bits index out of range')

      # Create a new Bits object containing the slice value and return it
      nbits = stop - start
      mask  = (1 << nbits) - 1
      value = (self._uint & (mask << start)) >> start
      return BitSlice( nbits, value, target_bits = self, offset = start )

    # Handle integers
    else:

      addr = int(addr)

      # Verify the index is sane
      if not (0 <= addr < self.nbits):
        raise IndexError('Bits index out of range')

      # Create a new Bits object containing the bit value and return it
      value = (self._uint & (1 << addr)) >> addr
      return BitSlice( 1 , value, target_bits = self, offset = addr )

  #----------------------------------------------------------------------
  # __setitem__
  #----------------------------------------------------------------------
  # Write a subset of bits in the Bits object.
  def __setitem__( self, addr, value ):

    # TODO: optimize this logic?

    value = int( value )

    # Handle slices
    if isinstance( addr, slice ):

      # Parse address range
      start = addr.start
      stop  = addr.stop

      # Open-ended range ( [:] )
      if start is None and stop is None:
        assert self._min <= value <= self._max
        self._uint = value
        return

      # Open-ended range on left ( [:N] )
      elif start is None:
        start = 0

      # Open-ended range on right ( [N:] )
      elif stop is None:
        stop = self.nbits

      # Verify our ranges are sane
      if not (0 <= start < stop <= self.nbits):
        raise IndexError('Bits index out of range')

      nbits = stop - start

      # This assert fires if the value you are trying to store is wider
      # than the bitwidth of the slice you are writing to!
      assert nbits >= helpers.get_nbits( value )

      # Clear the bits we want to set
      ones  = (1 << nbits) - 1
      mask = ~(ones << start)
      cleared_val = self._uint & mask

      # Set the bits, anding with ones to ensure negative value assign
      # works that way you would expect.
      self._uint = cleared_val | ((value & ones) << start)

    # Handle integers
    else:

      # Verify the index and values are sane
      if not (0 <= addr < self.nbits):
        raise IndexError('Bits index out of range')
      assert 0 <= value <= 1

      # Clear the bits we want to set
      mask = ~(1 << addr)
      cleared_val = self._uint & mask

      # Set the bits
      self._uint = cleared_val | (value << addr)

  #----------------------------------------------------------------------
  # Arithmetic Operators
  #----------------------------------------------------------------------
  # For now, let's make the width equal to the max of the widths of the
  # two operands. These semantics match Verilog:
  # http://www1.pldworld.com/@xilinx/html/technote/TOOL/MANUAL/21i_doc/data/fndtn/ver/ver4_4.htm

  def __invert__( self ):
    return Bits( self.nbits, ~self._uint, trunc=True )

  def __add__( self, other ):
    try:    return Bits( max( self.nbits, other.nbits), self._uint + other._uint, trunc=True )
    except: return Bits( self.nbits,                    self._uint + other,       trunc=True )

  def __sub__( self, other ):
    try:    return Bits( max( self.nbits, other.nbits), self._uint - other._uint, trunc=True )
    except: return Bits( self.nbits,                    self._uint - other,       trunc=True )

  # TODO: what about multiplying Bits object with an object of other type
  # where the bitwidth of the other type is larger than the bitwidth of the
  # Bits object? ( applies to every other operator as well.... )
  def __mul__( self, other ):
    try:    return Bits( 2*max( self.nbits, other.nbits), self._uint * other._uint, trunc=True )
    except: return Bits( 2*self.nbits,                    self._uint * other,       trunc=True )

  def __radd__( self, other ):
    return self.__add__( other )

  def __rsub__( self, other ):
    return Bits( helpers.get_nbits( other ), other ) - self

  def __rmul__( self, other ):
    return self.__mul__( other )

  def __div__(self, other):
    try:    return Bits( 2*max( self.nbits, other.nbits), self._uint / other._uint, trunc=True )
    except: return Bits( 2*self.nbits,                    self._uint / other,       trunc=True )

  def __floordiv__(self, other):
    try:    return Bits( 2*max( self.nbits, other.nbits), self._uint / other._uint, trunc=True )
    except: return Bits( 2*self.nbits,                    self._uint / other,       trunc=True )

  def __mod__(self, other):
    try:    return Bits( 2*max( self.nbits, other.nbits), self._uint % other._uint, trunc=True )
    except: return Bits( 2*self.nbits,                    self._uint % other,       trunc=True )

  # TODO: implement these?
  # def __divmod__(self, other)
  # def __pow__(self, other[, modulo])

  #----------------------------------------------------------------------
  # Shift Operators
  #----------------------------------------------------------------------

  def __lshift__( self, other ):
    # Optimization to return 0 if shift amount is greater than self.nbits
    if int( other ) >= self.nbits: return Bits( self.nbits, 0 )
    return Bits( self.nbits, self._uint << int( other ), trunc=True )

  def __rshift__( self, other ):
    return Bits( self.nbits, self._uint >> int( other ) )

  # TODO: Not implementing reflective operators because its not clear
  #       how to determine width of other object in case of lshift
  #def __rlshift__(self, other):
  #  return self.__lshift__( other )
  #def __rrshift__(self, other):
  #  return self.__rshift__( other )

  #----------------------------------------------------------------------
  # Bitwise Operators
  #----------------------------------------------------------------------

  def __and__( self, other ):
    assert other >= 0
    try:    return Bits( max( self.nbits, other.nbits), self._uint & other._uint, trunc=True )
    except: return Bits( self.nbits,                    self._uint & other,       trunc=True )

  def __xor__( self, other ):
    assert other >= 0
    try:    return Bits( max( self.nbits, other.nbits), self._uint ^ other._uint, trunc=True )
    except: return Bits( self.nbits,                    self._uint ^ other,       trunc=True )

  def __or__( self, other ):
    assert other >= 0
    try:    return Bits( max( self.nbits, other.nbits), self._uint | other._uint, trunc=True )
    except: return Bits( self.nbits,                    self._uint | other,       trunc=True )

  def __rand__( self, other ):
    return self.__and__( other )

  def __rxor__( self, other ):
    return self.__xor__( other )

  def __ror__( self, other ):
    return self.__or__( other )

  #----------------------------------------------------------------------
  # Comparison Operators
  #----------------------------------------------------------------------

  def __nonzero__( self ):
    return self._uint != 0

  # TODO: allow comparison with negative numbers?
  def __eq__( self, other ):
    if other is None: return False
    assert other >= 0
    return self._uint == other

  def __ne__( self, other ):
    if other is None: return True
    assert other >= 0
    return self._uint != other

  def __lt__( self, other ):
    assert other >= 0
    return self._uint <  other

  def __le__( self, other ):
    assert other >= 0
    return self._uint <= other

  def __gt__( self, other ):
    assert other >= 0
    return self._uint >  other

  def __ge__( self, other ):
    assert other >= 0
    return self._uint >= other

  #----------------------------------------------------------------------
  # Extension
  #----------------------------------------------------------------------
  # TODO: make abstract method in SignalValue, or implement differently?

  def _zext( self, new_width ):
    return Bits( new_width, self._uint )

  def _sext( self, new_width ):
    return Bits( new_width, self.int() )

#-----------------------------------------------------------------------
# BitSlice
#-----------------------------------------------------------------------
# Class created when slicing a Bits object. Maintains a reference to the
# original Bits instance (target_bits) so that writing the BitSlice also
# updates the value on the original Bits object. Note that this does not
# work the other way around: updating the value of target_bits will not
# update the value of BitSlices that point to it!
class BitSlice( Bits ):

  #---------------------------------------------------------------------
  # __init__
  #---------------------------------------------------------------------
  def __init__( self, nbits, value, target_bits, offset ):

    # Create the BitSlice object using the Bits constructor.
    super( BitSlice, self ).__init__( nbits, value )

    # Extra fields specific to BitSlices: _target_bits points to the
    # Bits object we are slicing, _offset provides the position of the
    # specific bits we are slicing.
    self._target_bits = target_bits
    self._offset      = offset
    self.slice        = slice( offset, offset + nbits )

    # Take the notify_sim_* methods and the _slices function pointer list
    # from the original Bits instance. This ensures writes to the BitSlice
    # object made in a simulator will trigger the appropriate callbacks
    # attached to the Bits instance.
    self.notify_sim_comb_update = self._target_bits.notify_sim_comb_update
    self.notify_sim_seq_update  = self._target_bits.notify_sim_seq_update
    self._slices                = self._target_bits._slices

  #---------------------------------------------------------------------
  # write_value
  #---------------------------------------------------------------------
  # Implementing abstract write_value method defined by SignalValue.
  def write_value( self, value ):

    # Get the updated value and update self.
    value = int( value )
    assert self._min <= value <= self._max
    self._uint = (value & self._mask)

    # Update target we are slicing. First clear the bits we want to set.
    shifted_mask = ~( self._mask << self._offset )
    cleared_val  = self._target_bits._uint & shifted_mask

    # Set the bits, write to the target.
    new_val      = cleared_val | ( self._uint << self._offset )
    self._target_bits.write_value( new_val )

  #---------------------------------------------------------------------
  # write_next
  #---------------------------------------------------------------------
  # Implementing abstract write_next method defined by SignalValue.
  def write_next( self, value ):

    # Get the updated value, but no don't update self (BitSlices contain
    # no shadow state).
    value = int( value )
    assert self._min <= value <= self._max
    value = (value & self._mask)

    # Update target we are slicing. First clear the bits we want to set.
    shifted_mask = ~( self._mask << self._offset )
    cleared_val  = self._target_bits._next._uint & shifted_mask

    # Set the bits, write to the target's shadow state.
    new_val      = cleared_val | ( value << self._offset )
    self._target_bits.write_next( new_val )
<commit_msg>[pymtl.tools.sim] fix for issue #79 (details)<commit_after>#=======================================================================
# Bits.py
#=======================================================================
# Module containing the Bits class.

from SignalValue import SignalValue

# NOTE: circular imports between Bits and helpers, using 'import helpers'
#       instead of 'from helpers import' ensures pydoc still works
import helpers
import copy

#-----------------------------------------------------------------------
# Bits
#-----------------------------------------------------------------------
# Class emulating limited precision values of a set bitwidth.
class Bits( SignalValue ):

  #---------------------------------------------------------------------
  # __init__
  #---------------------------------------------------------------------
  def __init__( self, nbits, value = 0, trunc = False ):

    value = int( value )
    nbits = int( nbits )

    # Make sure width is non-zero and that we have space for the value
    assert nbits > 0

    # Set the nbits and bitmask (_mask) attributes
    self.nbits = nbits
    self._max  = (2**nbits)- 1
    self._min  = -2**(nbits- 1) if nbits > 1 else 0
    self._mask = ( 1 << self.nbits ) - 1
    self.slice = slice( None )

    if not trunc:
      #assert nbits >= helpers.get_nbits( value )
      assert self._min <= value <= self._max

    # Convert negative values into unsigned ints and store them
    value_uint = value if ( value >= 0 ) else ( ~(-value) + 1 )
    self._uint = value_uint & self._mask

    self._target_bits = self

  #---------------------------------------------------------------------
  # __call__
  #---------------------------------------------------------------------
  # Allow Bits to act like a type that can be instantiated.
  def __call__( self ):
    return Bits( self.nbits )

  #---------------------------------------------------------------------
  # __int__
  #---------------------------------------------------------------------
  # Type conversion to an int.
  def __int__( self ):
    return int( self._uint )

  #---------------------------------------------------------------------
  # __long__
  #---------------------------------------------------------------------
  # Type conversion to a long.
  def __long__( self ):
    return long( self._uint )

  #---------------------------------------------------------------------
  # __index__
  #---------------------------------------------------------------------
  # Behave like an int() when used as an index to slices.
  def __index__( self ):
    return self._uint

  #---------------------------------------------------------------------
  # uint
  #---------------------------------------------------------------------
  # Return the unsigned integer representation of the bits.
  def uint( self ):
    return self._uint

  #---------------------------------------------------------------------
  # int
  #---------------------------------------------------------------------
  # Return the integer representation of the bits.
  def int( self ):
    if self[ self.nbits - 1]:
      twos_complement = ~self + 1
      return -twos_complement._uint
    else:
      return self._uint

  #---------------------------------------------------------------------
  # write_value
  #---------------------------------------------------------------------
  # Implementing abstract write_value method defined by SignalValue.
  def write_value( self, value ):
    value = int( value )
    assert self._min <= value <= self._max
    self._uint = (value & self._mask)

  #---------------------------------------------------------------------
  # write_next
  #---------------------------------------------------------------------
  # Implementing abstract write_next method defined by SignalValue.
  def write_next( self, value ):
    value = int( value )
    assert self._min <= value <= self._max
    self._next._uint = (value & self._mask)

  #---------------------------------------------------------------------
  # bit_length
  #---------------------------------------------------------------------
  # Implement bit_length method provided by the int built-in. Simplifies
  # the implementation of get_nbits().
  def bit_length( self ):
    return self._uint.bit_length()

  #---------------------------------------------------------------------
  # Print Methods
  #---------------------------------------------------------------------

  def __repr__(self):
    return "Bits( {0}, {1} )".format(self.nbits, self.hex())

  def __str__(self):
    num_chars = (((self.nbits-1)/4)+1)
    str = "{:x}".format(self._uint).zfill(num_chars)
    return str

  def __oct__( self ):
    print "DEPRECATED: Please use .oct()!"
    return self.oct()

  def __hex__( self ):
    print "DEPRECATED: Please use .oct()!"
    return self.hex()

  def bin(self):
    str = "{:b}".format(self._uint).zfill(self.nbits)
    return "0b"+str

  def oct( self ):
    num_chars = (((self.nbits-1)/2)+1)
    str = "{:o}".format(self._uint).zfill(num_chars)
    return "0o"+str

  def hex( self ):
    num_chars = (((self.nbits-1)/4)+1)
    str = "{:x}".format(self._uint).zfill(num_chars)
    return "0x"+str

  #----------------------------------------------------------------------
  # __getitem__
  #----------------------------------------------------------------------
  # Read a subset of bits in the Bits object.
  def __getitem__( self, addr ):

    # TODO: optimize this logic?

    # Handle slices
    if isinstance( addr, slice ):

      # Parse address range
      start = addr.start
      stop  = addr.stop

      # Open-ended range ( [:] ), return a copy of self
      if start is None and stop is None:
        return copy.copy( self )

      # Open-ended range on left ( [:N] )
      elif start is None:
        start = 0

      # Open-ended range on right ( [N:] )
      elif stop is None:
        stop = self.nbits

      stop  = int( stop  )
      start = int( start )

      # Verify our ranges are sane
      if not (start < stop):
        raise IndexError('Start index is not less than stop index')
      if not (0 <= start < stop <= self.nbits):
        raise IndexError('Bits index out of range')

      # Create a new Bits object containing the slice value and return it
      nbits = stop - start
      mask  = (1 << nbits) - 1
      value = (self._uint & (mask << start)) >> start
      return BitSlice( nbits, value, target_bits = self, offset = start )

    # Handle integers
    else:

      addr = int(addr)

      # Verify the index is sane
      if not (0 <= addr < self.nbits):
        raise IndexError('Bits index out of range')

      # Create a new Bits object containing the bit value and return it
      value = (self._uint & (1 << addr)) >> addr
      return BitSlice( 1 , value, target_bits = self, offset = addr )

  #----------------------------------------------------------------------
  # __setitem__
  #----------------------------------------------------------------------
  # Write a subset of bits in the Bits object.
  def __setitem__( self, addr, value ):

    # TODO: optimize this logic?

    value = int( value )

    # Handle slices
    if isinstance( addr, slice ):

      # Parse address range
      start = addr.start
      stop  = addr.stop

      # Open-ended range ( [:] )
      if start is None and stop is None:
        assert self._min <= value <= self._max
        self._uint = value
        return

      # Open-ended range on left ( [:N] )
      elif start is None:
        start = 0

      # Open-ended range on right ( [N:] )
      elif stop is None:
        stop = self.nbits

      # Verify our ranges are sane
      if not (0 <= start < stop <= self.nbits):
        raise IndexError('Bits index out of range')

      nbits = stop - start

      # This assert fires if the value you are trying to store is wider
      # than the bitwidth of the slice you are writing to!
      assert nbits >= helpers.get_nbits( value )

      # Clear the bits we want to set
      ones  = (1 << nbits) - 1
      mask = ~(ones << start)
      cleared_val = self._uint & mask

      # Set the bits, anding with ones to ensure negative value assign
      # works that way you would expect.
      self._uint = cleared_val | ((value & ones) << start)

    # Handle integers
    else:

      # Verify the index and values are sane
      if not (0 <= addr < self.nbits):
        raise IndexError('Bits index out of range')
      assert 0 <= value <= 1

      # Clear the bits we want to set
      mask = ~(1 << addr)
      cleared_val = self._uint & mask

      # Set the bits
      self._uint = cleared_val | (value << addr)

  #----------------------------------------------------------------------
  # Arithmetic Operators
  #----------------------------------------------------------------------
  # For now, let's make the width equal to the max of the widths of the
  # two operands. These semantics match Verilog:
  # http://www1.pldworld.com/@xilinx/html/technote/TOOL/MANUAL/21i_doc/data/fndtn/ver/ver4_4.htm

  def __invert__( self ):
    return Bits( self.nbits, ~self._uint, trunc=True )

  def __add__( self, other ):
    try:    return Bits( max( self.nbits, other.nbits), self._uint + other._uint, trunc=True )
    except: return Bits( self.nbits,                    self._uint + other,       trunc=True )

  def __sub__( self, other ):
    try:    return Bits( max( self.nbits, other.nbits), self._uint - other._uint, trunc=True )
    except: return Bits( self.nbits,                    self._uint - other,       trunc=True )

  # TODO: what about multiplying Bits object with an object of other type
  # where the bitwidth of the other type is larger than the bitwidth of the
  # Bits object? ( applies to every other operator as well.... )
  def __mul__( self, other ):
    try:    return Bits( 2*max( self.nbits, other.nbits), self._uint * other._uint, trunc=True )
    except: return Bits( 2*self.nbits,                    self._uint * other,       trunc=True )

  def __radd__( self, other ):
    return self.__add__( other )

  def __rsub__( self, other ):
    return Bits( helpers.get_nbits( other ), other ) - self

  def __rmul__( self, other ):
    return self.__mul__( other )

  def __div__(self, other):
    try:    return Bits( 2*max( self.nbits, other.nbits), self._uint / other._uint, trunc=True )
    except: return Bits( 2*self.nbits,                    self._uint / other,       trunc=True )

  def __floordiv__(self, other):
    try:    return Bits( 2*max( self.nbits, other.nbits), self._uint / other._uint, trunc=True )
    except: return Bits( 2*self.nbits,                    self._uint / other,       trunc=True )

  def __mod__(self, other):
    try:    return Bits( 2*max( self.nbits, other.nbits), self._uint % other._uint, trunc=True )
    except: return Bits( 2*self.nbits,                    self._uint % other,       trunc=True )

  # TODO: implement these?
  # def __divmod__(self, other)
  # def __pow__(self, other[, modulo])

  #----------------------------------------------------------------------
  # Shift Operators
  #----------------------------------------------------------------------

  def __lshift__( self, other ):
    # Optimization to return 0 if shift amount is greater than self.nbits
    if int( other ) >= self.nbits: return Bits( self.nbits, 0 )
    return Bits( self.nbits, self._uint << int( other ), trunc=True )

  def __rshift__( self, other ):
    return Bits( self.nbits, self._uint >> int( other ) )

  # TODO: Not implementing reflective operators because its not clear
  #       how to determine width of other object in case of lshift
  #def __rlshift__(self, other):
  #  return self.__lshift__( other )
  #def __rrshift__(self, other):
  #  return self.__rshift__( other )

  #----------------------------------------------------------------------
  # Bitwise Operators
  #----------------------------------------------------------------------

  def __and__( self, other ):
    assert other >= 0
    try:    return Bits( max( self.nbits, other.nbits), self._uint & other._uint, trunc=True )
    except: return Bits( self.nbits,                    self._uint & other,       trunc=True )

  def __xor__( self, other ):
    assert other >= 0
    try:    return Bits( max( self.nbits, other.nbits), self._uint ^ other._uint, trunc=True )
    except: return Bits( self.nbits,                    self._uint ^ other,       trunc=True )

  def __or__( self, other ):
    assert other >= 0
    try:    return Bits( max( self.nbits, other.nbits), self._uint | other._uint, trunc=True )
    except: return Bits( self.nbits,                    self._uint | other,       trunc=True )

  def __rand__( self, other ):
    return self.__and__( other )

  def __rxor__( self, other ):
    return self.__xor__( other )

  def __ror__( self, other ):
    return self.__or__( other )

  #----------------------------------------------------------------------
  # Comparison Operators
  #----------------------------------------------------------------------

  def __nonzero__( self ):
    return self._uint != 0

  # TODO: allow comparison with negative numbers?
  def __eq__( self, other ):
    if other is None: return False
    assert other >= 0
    return self._uint == other

  def __ne__( self, other ):
    if other is None: return True
    assert other >= 0
    return self._uint != other

  def __lt__( self, other ):
    assert other >= 0
    return self._uint <  other

  def __le__( self, other ):
    assert other >= 0
    return self._uint <= other

  def __gt__( self, other ):
    assert other >= 0
    return self._uint >  other

  def __ge__( self, other ):
    assert other >= 0
    return self._uint >= other

  #----------------------------------------------------------------------
  # Extension
  #----------------------------------------------------------------------
  # TODO: make abstract method in SignalValue, or implement differently?

  def _zext( self, new_width ):
    return Bits( new_width, self._uint )

  def _sext( self, new_width ):
    return Bits( new_width, self.int() )

#-----------------------------------------------------------------------
# BitSlice
#-----------------------------------------------------------------------
# Class created when slicing a Bits object. Maintains a reference to the
# original Bits instance (target_bits) so that writing the BitSlice also
# updates the value on the original Bits object. Note that this does not
# work the other way around: updating the value of target_bits will not
# update the value of BitSlices that point to it!
class BitSlice( Bits ):

  #---------------------------------------------------------------------
  # __init__
  #---------------------------------------------------------------------
  def __init__( self, nbits, value, target_bits, offset ):

    # Create the BitSlice object using the Bits constructor.
    super( BitSlice, self ).__init__( nbits, value )

    # Extra fields specific to BitSlices: _target_bits points to the
    # Bits object we are slicing, _offset provides the position of the
    # specific bits we are slicing.
    self._target_bits = target_bits
    self._offset      = offset
    self.slice        = slice( offset, offset + nbits )

    # Take the notify_sim_* methods and the _slices function pointer list
    # from the original Bits instance. This ensures writes to the BitSlice
    # object made in a simulator will trigger the appropriate callbacks
    # attached to the Bits instance.
    self.notify_sim_comb_update = self._target_bits.notify_sim_comb_update
    self.notify_sim_seq_update  = self._target_bits.notify_sim_seq_update

  @property
  def _slices( self ):
    return self._target_bits._slices

  #---------------------------------------------------------------------
  # write_value
  #---------------------------------------------------------------------
  # Implementing abstract write_value method defined by SignalValue.
  def write_value( self, value ):

    # Get the updated value and update self.
    value = int( value )
    assert self._min <= value <= self._max
    self._uint = (value & self._mask)

    # Update target we are slicing. First clear the bits we want to set.
    shifted_mask = ~( self._mask << self._offset )
    cleared_val  = self._target_bits._uint & shifted_mask

    # Set the bits, write to the target.
    new_val      = cleared_val | ( self._uint << self._offset )
    self._target_bits.write_value( new_val )

  #---------------------------------------------------------------------
  # write_next
  #---------------------------------------------------------------------
  # Implementing abstract write_next method defined by SignalValue.
  def write_next( self, value ):

    # Get the updated value, but no don't update self (BitSlices contain
    # no shadow state).
    value = int( value )
    assert self._min <= value <= self._max
    value = (value & self._mask)

    # Update target we are slicing. First clear the bits we want to set.
    shifted_mask = ~( self._mask << self._offset )
    cleared_val  = self._target_bits._next._uint & shifted_mask

    # Set the bits, write to the target's shadow state.
    new_val      = cleared_val | ( value << self._offset )
    self._target_bits.write_next( new_val )
<|endoftext|>