<commit_before>#!/usr/bin/env python3

""" BibTeX Normalization Tool

    An example of a normalized entry:
    @article{greenwade93,
        author     = {George D. Greenwade},
        title      = {The {C}omprehensive {T}ex {A}rchive {N}etwork ({CTAN})},
        year       = {1993},
        journal    = {TUGBoat},
        volume     = {14},
        number     = {3},
        pages      = {342--351},
    }

    To do:
        1) DBLP sync
        2) automatically add location for conference papers
        http://dblp.uni-trier.de/rec/bibtex/conf/sp/1996
"""

__author__ = "Xiaokui Shu"
__copyright__ = "Copyright 2014, Xiaokui Shu"
__license__ = "Apache"
__version__ = "1.0.0"
__maintainer__ = "Xiaokui Shu"
__email__ = "subx@cs.vt.edu"
__status__ = "Testing"

import sys
import os
import argparse
import time
import datetime
import logging
import string
import re

logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)
logger = logging.getLogger(__name__)

NOTCITED_FILE = "notcited.bib"

CATEGORIES = ("article", "book", "booklet", "inbook", "incollection",
"inproceedings", "manual", "mastersthesis", "phdthesis", "misc", "techreport",
"unpublished")

# attributes in order
ATTRIBUTES = ("author", "title", "journal", "booktitle", "institution",
"school", "key", "year", "month", "series", "volume", "number", "pages",
"publisher", "edition", "note", "howpublished", "url")

ATTR_ONLY_IN = {
        "url":("misc"),
        "publisher": ("article", "book", "inbook", "incollection")}

ATTRIBUTES_DROP = ("location", "address", "organization", "ee",
"doi", "crossref", "bibsource", "isbn", "issn", "acmid", "numpages",
"issue_date", "keywords")

SYNTAX_CORRECTION_MONTH = {"Jnu": "Jun"}

re_pages_one = re.compile("^\d+$")
re_pages_two = re.compile("^(\d+)\s*-+\s*(\d+)$")

re_aux_citation = re.compile("\\\\bibcite\{([^\}]+)\}")

def process_entry(oneline_entry, if_shorten_entry):
    """
    process each entry in a line

    @param oneline_entry: the entry in one line string
    """

    def check_int_attr(attr_name):
        """
        check integer type attribute

        @param attr_name: atribute name in string
        """
        if attr_name in final_attrs:
            try:
                int(final_attrs[attr_name])
            except:
                logger.warn('format error in "{0}" for entry {1}'.format(
                    attr_name, anchor_word))

    # Step 0: init
    final_attrs = {}
    final_entry_lines = []
    oneline_entry = oneline_entry.strip()

    # Stpe 1: check entry
    if oneline_entry[0] != "@" or \
            oneline_entry.find("{") < 0 or \
            oneline_entry[-1] != "}" or \
            oneline_entry.count("{") != oneline_entry.count("}"):
                raise ErrorParsedEntry()

    # Step 2: extract category
    cap_left_bracket_index = oneline_entry.find("{")

    category = oneline_entry[1:cap_left_bracket_index].lower().strip()

    if category not in CATEGORIES:
        summary = oneline_entry[:32].replace('\n', '')
        logger.info("drop comment:{0}".format(summary))
        raise DropComment()

    # Step 3: find content and extract anchor word
    content = oneline_entry[cap_left_bracket_index+1: -1]
    attrs_raw = content.split(",")
    attrs = []
    attr_merged_seg = ""

    for item in attrs_raw:
        if not attr_merged_seg:
            attr_merged_seg = item
        else:
            attr_merged_seg = attr_merged_seg + ", " + item

        if attr_merged_seg.count("{") == attr_merged_seg.count("}") and \
                attr_merged_seg.count('"') % 2 == 0:
                    attrs.append(attr_merged_seg)
                    attr_merged_seg = ""

    if attr_merged_seg:
        logger.warning("unmatched bracket, potential program logic error")

    anchor_word = attrs[0].strip()
    attrs = attrs[1:]

    # Step 4: extract attributes in content
    for attrline in attrs:
        attrline = attrline.strip()
        if attrline == "":
            continue

        eq_index = attrline.find("=")
        attribute = attrline[:eq_index].strip().lower()

        if attribute in ATTRIBUTES_DROP:
            continue

        if attribute not in ATTRIBUTES:
            logger.info("drop unknown attribute:\n"
                    + "    {0}".format(attrline))
            continue

        # drop attribute if not required
        if attribute in ATTR_ONLY_IN:
            if category not in ATTR_ONLY_IN[attribute]:
                continue

        # get rid of double quotations or brackets
        value = attrline[eq_index+1:].strip()

        # sometimes year is not surrounded by quatation or bracket
        if value[0] == '"' or value[0] == "{":
            value = value[1:-1].strip()

        # merge multiple spaces into one
        value = " ".join(value.split())

        # drop empty attributes
        if value == "":
            continue

        if value.isupper() and len(value.split()) > 2:
            value = string.capwords(value)
            logger.warn("ALL UPPER LETTER VALUE, Turned To Capwords:\n"
                    + "    {0}".format(value))

        final_attrs[attribute] = value

    # Step 5: correct attributes
    if "title" not in final_attrs:
        if category == "misc" and "note" in final_attrs:
            final_attrs["title"] = final_attrs.pop("note")
        else:
            logger.warning("entry has no title:\n"
                    + "    {0}".format(oneline_entry))

    if "url" in final_attrs or "howpublished" in final_attrs:
        if "note" in final_attrs:
            logger.warning('entry has "note", replace it with url')
        try:
            value_to_move = final_attrs.pop("url")
        except:
            value_to_move = final_attrs.pop("howpublished")
        finally:
            final_attrs["note"] = "\\url{{{0}}}, accessed {1}".format(
                    value_to_move, time.strftime("%B %Y"))

    if "pages" in final_attrs:
        pages_value = final_attrs["pages"]
        if not re_pages_one.match(pages_value):
            re_match = re_pages_two.match(pages_value)
            if re_match:
                final_attrs["pages"] = "--".join(re_match.groups())
            else:
                logger.warn('format error in "pages" ' + \
                        "for entry {0}".format(anchor_word))

    if "month" in final_attrs:
        month_value = final_attrs["month"][:3]
        if month_value in SYNTAX_CORRECTION_MONTH:
            month_value = SYNTAX_CORRECTION_MONTH[month_value]
        mon = datetime.datetime.strptime(month_value, "%b")
        if if_shorten_entry:
            final_attrs["month"] = mon.strftime("%b")
        else:
            final_attrs["month"] = mon.strftime("%B")

    for attr_name in ("number", "volume", "edition"):
        check_int_attr(attr_name)

    # Step 6: write entry into the new format
    final_entry_lines.append("@{0}{{{1},".format(category, anchor_word))
    for attribute in ATTRIBUTES:
        if attribute in final_attrs:
            final_entry_lines.append("    {0:10} = {{{1}}},".format(
                attribute, final_attrs[attribute]))
    final_entry_lines.append("}")

    # Step 7: return normalized entry in a line and the title
    return "\n".join(final_entry_lines), anchor_word, final_attrs["title"]

def process_bib_files(in_descriptors, out_descriptor, \
        if_print_titles, if_shorten_entries, if_dedup, cited):
    """
    @param in_descriptors: file descriptors for all input files
    @param cited: list of anchors cited, empty means not care
    """

    oneline_content = "".join("".join(f.readlines()) for f in in_descriptors)

    # cited entries
    final_entries = []

    # not cited
    abandoned_entries = []

    # print titles and entry deduplication
    title2anchor = {}

    # how many half brackets
    half_bracket = 0
    # record the index of entry start point, where the "@" is
    index_of_entry_start = 0

    for index, char in enumerate(oneline_content):

        if char == "@":
            if half_bracket != 0:
                raise ErrorBracketNotMatch()
            else:
                index_of_entry_start = index
        elif char == "{":
            half_bracket += 1
        elif char == "}":
            half_bracket -= 1
            if half_bracket == 0:
                try:
                    final_entry, anchor, title = process_entry(
                            oneline_content[index_of_entry_start: index+1],
                            if_shorten_entries)
                except DropComment:
                    continue

                if not cited or anchor in cited:
                    final_entries.append(final_entry)
                else:
                    abandoned_entries.append(final_entry)
                title2anchor[title] = anchor

    if if_dedup:
        dedup(title2anchor)

    out_descriptor.write("\n\n".join(final_entries))

    if abandoned_entries:
        with open(NOTCITED_FILE, "w") as ncf:
            ncf.write("\n\n".join(abandoned_entries))

    if if_print_titles:
        print("\n#### Print All Titles ####")
        for title in title2anchor.keys():
            print("{0}".format(title))

    print()
    logger.info("{0} entries processed: {1} cited, {2} not cited ({3}).".format(
        len(final_entries) + len(abandoned_entries),
        len(final_entries), len(abandoned_entries), NOTCITED_FILE))

def dedup(title2anchor):
    pass

def analyze_aux(auxfile):
    anchors = []
    with open(auxfile, "r") as f:
        for line in f.readlines():
            re_match = re_aux_citation.search(line.strip())
            if re_match:
                anchors.append(re_match.groups()[0])
    return anchors

class ErrorParsedEntry(Exception):
    pass

class ErrorBracketNotMatch(Exception):
    pass

class InvalidOutputFile(Exception):
    pass

class ErrorMultipleFilesInplace(Exception):
    pass

class DropComment(Exception):
    pass

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="BibTeX Normalization Tool")
    parser.add_argument("bibfile", nargs="+",
            help="bibfiles to read", metavar="FILE")
    parser.add_argument("-t", "--titles",
            help="print all titles", action="store_true")
    parser.add_argument("-s", "--short",
            help="print months in three letters", action="store_true")
    parser.add_argument("-d", "--deduplicate",
            help="warn duplicate entries via edit distances between titles",
            action="store_true")
    parser.add_argument("-i", "--inplace",
            help="edit the file in-place, invalid for multiple files",
            action="store_true")
    parser.add_argument("-c", "--cited",
            help="analyze aux file and output cited/non-cited in two files",
            metavar="AUX FILE")
    parser.add_argument("-o", "--output",
            help="output to file, default to stdout", metavar="NEWFILE")
    args = parser.parse_args()

    if args.inplace:
        if len(args.bibfile) == 1:
            ori_bib_file = args.bibfile[0]
            old_file_name = ori_bib_file + ".bak"
            os.rename(ori_bib_file, old_file_name)
            in_files = [open(old_file_name, "r")]
            out_file = open(ori_bib_file, "w")
        else:
            raise ErrorMultipleFilesInplace()
    else:
        in_files = [open(f, "r") for f in args.bibfile]

        if args.output:
            if args.output in args.bibfile:
                raise InvalidOutputFile()
            out_file = open(args.output, "w")
        else:
            out_file = sys.stdout

    if args.cited:
        cited_anchors = analyze_aux(args.cited)
    else:
        cited_anchors = []

    process_bib_files(in_files, out_file,
            args.titles, args.short, args.deduplicate, cited_anchors)

    for f in in_files:
        f.close()

    if out_file != sys.stdout:
        out_file.close()
<commit_msg>recognize " inside title<commit_after>#!/usr/bin/env python3

""" BibTeX Normalization Tool

    An example of a normalized entry:
    @article{greenwade93,
        author     = {George D. Greenwade},
        title      = {The {C}omprehensive {T}ex {A}rchive {N}etwork ({CTAN})},
        year       = {1993},
        journal    = {TUGBoat},
        volume     = {14},
        number     = {3},
        pages      = {342--351},
    }

    To do:
        1) DBLP sync
        2) automatically add location for conference papers
        http://dblp.uni-trier.de/rec/bibtex/conf/sp/1996
"""

__author__ = "Xiaokui Shu"
__copyright__ = "Copyright 2014, Xiaokui Shu"
__license__ = "Apache"
__version__ = "1.0.0"
__maintainer__ = "Xiaokui Shu"
__email__ = "subx@cs.vt.edu"
__status__ = "Testing"

import sys
import os
import argparse
import time
import datetime
import logging
import string
import re

logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)
logger = logging.getLogger(__name__)

NOTCITED_FILE = "notcited.bib"

CATEGORIES = ("article", "book", "booklet", "inbook", "incollection",
"inproceedings", "manual", "mastersthesis", "phdthesis", "misc", "techreport",
"unpublished")

# attributes in order
ATTRIBUTES = ("author", "title", "journal", "booktitle", "institution",
"school", "key", "year", "month", "series", "volume", "number", "pages",
"publisher", "edition", "note", "howpublished", "url")

ATTR_ONLY_IN = {
        "url":("misc"),
        "publisher": ("article", "book", "inbook", "incollection")}

ATTRIBUTES_DROP = ("location", "address", "organization", "ee",
"doi", "crossref", "bibsource", "isbn", "issn", "acmid", "numpages",
"issue_date", "keywords")

SYNTAX_CORRECTION_MONTH = {"Jnu": "Jun"}

re_pages_one = re.compile("^\d+$")
re_pages_two = re.compile("^(\d+)\s*-+\s*(\d+)$")

re_aux_citation = re.compile("\\\\bibcite\{([^\}]+)\}")

def process_entry(oneline_entry, if_shorten_entry):
    """
    process each entry in a line

    @param oneline_entry: the entry in one line string
    """

    def check_int_attr(attr_name):
        """
        check integer type attribute

        @param attr_name: atribute name in string
        """
        if attr_name in final_attrs:
            try:
                int(final_attrs[attr_name])
            except:
                logger.warn('format error in "{0}" for entry {1}'.format(
                    attr_name, anchor_word))

    # Step 0: init
    final_attrs = {}
    final_entry_lines = []
    oneline_entry = oneline_entry.strip()

    # Stpe 1: check entry
    if oneline_entry[0] != "@" or \
            oneline_entry.find("{") < 0 or \
            oneline_entry[-1] != "}" or \
            oneline_entry.count("{") != oneline_entry.count("}"):
                raise ErrorParsedEntry()

    # Step 2: extract category
    cap_left_bracket_index = oneline_entry.find("{")

    category = oneline_entry[1:cap_left_bracket_index].lower().strip()

    if category not in CATEGORIES:
        summary = oneline_entry[:32].replace('\n', '')
        logger.info("drop comment:{0}".format(summary))
        raise DropComment()

    # Step 3: find content and extract anchor word
    content = oneline_entry[cap_left_bracket_index+1: -1]
    attrs_raw = content.split(",")
    attrs = []
    attr_merged_seg = ""

    for item in attrs_raw:
        if not attr_merged_seg:
            attr_merged_seg = item
        else:
            attr_merged_seg = attr_merged_seg + ", " + item

        # test if "{" and "}" match
        if attr_merged_seg.count("{") == attr_merged_seg.count("}"):

            # test if '"' and '"' match
            quote_splited_pieces = attr_merged_seg.split('"')
            quotecnt = 0
            for piece in quote_splited_pieces[:-1]:
                if piece[-1] != "\\":
                    quotecnt += 1
            if quotecnt % 2 == 0:

                # obtain attr if "{}" and '"' match
                attrs.append(attr_merged_seg)

                # for warning on the next line
                attr_merged_seg = ""

    if attr_merged_seg:
        logger.warning("unmatched bracket, potential program logic error")

    anchor_word = attrs[0].strip()
    attrs = attrs[1:]

    # Step 4: extract attributes in content
    for attrline in attrs:
        attrline = attrline.strip()
        if attrline == "":
            continue

        eq_index = attrline.find("=")
        attribute = attrline[:eq_index].strip().lower()

        if attribute in ATTRIBUTES_DROP:
            continue

        if attribute not in ATTRIBUTES:
            logger.info("drop unknown attribute:\n"
                    + "    {0}".format(attrline))
            continue

        # drop attribute if not required
        if attribute in ATTR_ONLY_IN:
            if category not in ATTR_ONLY_IN[attribute]:
                continue

        # get rid of double quotations or brackets
        value = attrline[eq_index+1:].strip()

        # sometimes year is not surrounded by quatation or bracket
        if value[0] == '"' or value[0] == "{":
            value = value[1:-1].strip()

        # merge multiple spaces into one
        value = " ".join(value.split())

        # drop empty attributes
        if value == "":
            continue

        if value.isupper() and len(value.split()) > 2:
            value = string.capwords(value)
            logger.warn("ALL UPPER LETTER VALUE, Turned To Capwords:\n"
                    + "    {0}".format(value))

        final_attrs[attribute] = value

    # Step 5: correct attributes
    if "title" not in final_attrs:
        if category == "misc" and "note" in final_attrs:
            final_attrs["title"] = final_attrs.pop("note")
        else:
            logger.warning("entry has no title:\n"
                    + "    {0}".format(oneline_entry))

    if "url" in final_attrs or "howpublished" in final_attrs:
        if "note" in final_attrs:
            logger.warning('entry has "note", replace it with url')
        try:
            value_to_move = final_attrs.pop("url")
        except:
            value_to_move = final_attrs.pop("howpublished")
        finally:
            final_attrs["note"] = "\\url{{{0}}}, accessed {1}".format(
                    value_to_move, time.strftime("%B %Y"))

    if "pages" in final_attrs:
        pages_value = final_attrs["pages"]
        if not re_pages_one.match(pages_value):
            re_match = re_pages_two.match(pages_value)
            if re_match:
                final_attrs["pages"] = "--".join(re_match.groups())
            else:
                logger.warn('format error in "pages" ' + \
                        "for entry {0}".format(anchor_word))

    if "month" in final_attrs:
        month_value = final_attrs["month"][:3]
        if month_value in SYNTAX_CORRECTION_MONTH:
            month_value = SYNTAX_CORRECTION_MONTH[month_value]
        mon = datetime.datetime.strptime(month_value, "%b")
        if if_shorten_entry:
            final_attrs["month"] = mon.strftime("%b")
        else:
            final_attrs["month"] = mon.strftime("%B")

    for attr_name in ("number", "volume", "edition"):
        check_int_attr(attr_name)

    # Step 6: write entry into the new format
    final_entry_lines.append("@{0}{{{1},".format(category, anchor_word))
    for attribute in ATTRIBUTES:
        if attribute in final_attrs:
            final_entry_lines.append("    {0:10} = {{{1}}},".format(
                attribute, final_attrs[attribute]))
    final_entry_lines.append("}")

    # Step 7: return normalized entry in a line and the title
    return "\n".join(final_entry_lines), anchor_word, final_attrs["title"]

def process_bib_files(in_descriptors, out_descriptor, \
        if_print_titles, if_shorten_entries, if_dedup, cited):
    """
    @param in_descriptors: file descriptors for all input files
    @param cited: list of anchors cited, empty means not care
    """

    oneline_content = "".join("".join(f.readlines()) for f in in_descriptors)

    # cited entries
    final_entries = []

    # not cited
    abandoned_entries = []

    # print titles and entry deduplication
    title2anchor = {}

    # how many half brackets
    half_bracket = 0
    # record the index of entry start point, where the "@" is
    index_of_entry_start = 0

    for index, char in enumerate(oneline_content):

        if char == "@":
            if half_bracket != 0:
                raise ErrorBracketNotMatch()
            else:
                index_of_entry_start = index
        elif char == "{":
            half_bracket += 1
        elif char == "}":
            half_bracket -= 1
            if half_bracket == 0:
                try:
                    final_entry, anchor, title = process_entry(
                            oneline_content[index_of_entry_start: index+1],
                            if_shorten_entries)
                except DropComment:
                    continue

                if not cited or anchor in cited:
                    final_entries.append(final_entry)
                else:
                    abandoned_entries.append(final_entry)
                title2anchor[title] = anchor

    if if_dedup:
        dedup(title2anchor)

    out_descriptor.write("\n\n".join(final_entries))

    if abandoned_entries:
        with open(NOTCITED_FILE, "w") as ncf:
            ncf.write("\n\n".join(abandoned_entries))

    if if_print_titles:
        print("\n#### Print All Titles ####")
        for title in title2anchor.keys():
            print("{0}".format(title))

    print()
    logger.info("{0} entries processed: {1} cited, {2} not cited ({3}).".format(
        len(final_entries) + len(abandoned_entries),
        len(final_entries), len(abandoned_entries), NOTCITED_FILE))

def dedup(title2anchor):
    pass

def analyze_aux(auxfile):
    anchors = []
    with open(auxfile, "r") as f:
        for line in f.readlines():
            re_match = re_aux_citation.search(line.strip())
            if re_match:
                anchors.append(re_match.groups()[0])
    print(anchors[-1])
    return anchors

class ErrorParsedEntry(Exception):
    pass

class ErrorBracketNotMatch(Exception):
    pass

class InvalidOutputFile(Exception):
    pass

class ErrorMultipleFilesInplace(Exception):
    pass

class DropComment(Exception):
    pass

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="BibTeX Normalization Tool")
    parser.add_argument("bibfile", nargs="+",
            help="bibfiles to read", metavar="FILE")
    parser.add_argument("-t", "--titles",
            help="print all titles", action="store_true")
    parser.add_argument("-s", "--short",
            help="print months in three letters", action="store_true")
    parser.add_argument("-d", "--deduplicate",
            help="warn duplicate entries via edit distances between titles",
            action="store_true")
    parser.add_argument("-i", "--inplace",
            help="edit the file in-place, invalid for multiple files",
            action="store_true")
    parser.add_argument("-c", "--cited",
            help="analyze aux file and output cited/non-cited in two files",
            metavar="AUX FILE")
    parser.add_argument("-o", "--output",
            help="output to file, default to stdout", metavar="NEWFILE")
    args = parser.parse_args()

    if args.inplace:
        if len(args.bibfile) == 1:
            ori_bib_file = args.bibfile[0]
            old_file_name = ori_bib_file + ".bak"
            os.rename(ori_bib_file, old_file_name)
            in_files = [open(old_file_name, "r")]
            out_file = open(ori_bib_file, "w")
        else:
            raise ErrorMultipleFilesInplace()
    else:
        in_files = [open(f, "r") for f in args.bibfile]

        if args.output:
            if args.output in args.bibfile:
                raise InvalidOutputFile()
            out_file = open(args.output, "w")
        else:
            out_file = sys.stdout

    if args.cited:
        cited_anchors = analyze_aux(args.cited)
    else:
        cited_anchors = []

    process_bib_files(in_files, out_file,
            args.titles, args.short, args.deduplicate, cited_anchors)

    for f in in_files:
        f.close()

    if out_file != sys.stdout:
        out_file.close()
<|endoftext|>


<commit_before>VERSION = (0, 1, 0, "a", 1)  # following PEP 386
DEV_N = "2"

def get_version():
    version = "%s.%s" % (VERSION[0], VERSION[1])
    if VERSION[2]:
        version = "%s.%s" % (version, VERSION[2])
    if VERSION[3] != "f":
        version = "%s%s%s" % (version, VERSION[3], VERSION[4])
        if DEV_N:
            version = "%s.dev%s" % (version, DEV_N)
    return version

__version__ = get_version()<commit_msg>Changed version number.<commit_after>VERSION = (0, 1, 0, "a", 1)  # following PEP 386
DEV_N = "3"

def get_version():
    version = "%s.%s" % (VERSION[0], VERSION[1])
    if VERSION[2]:
        version = "%s.%s" % (version, VERSION[2])
    if VERSION[3] != "f":
        version = "%s%s%s" % (version, VERSION[3], VERSION[4])
        if DEV_N:
            version = "%s.dev%s" % (version, DEV_N)
    return version

__version__ = get_version()<|endoftext|>


<commit_before>'''Draw polygon regions of interest (ROIs) in matplotlib images,
similar to Matlab's roipoly function.

See the file example.py for an application.

Created by Joerg Doepfert 2014 based on code posted by Daniel
Kornhauser.

'''

import numpy as np
import sys
import matplotlib.pyplot as plt
import matplotlib.path as mplPath

class roipoly:

    def __init__(self, fig=[], ax=[], roicolor='b'):
        if fig == []:
            fig = plt.gcf()

        if ax == []:
            ax = plt.gca()

        self.previous_point = []
        self.allxpoints = []
        self.allypoints = []
        self.start_point = []
        self.end_point = []
        self.line = None
        self.roicolor = roicolor
        self.fig = fig
        self.ax = ax
        #self.fig.canvas.draw()

        self.__ID1 = self.fig.canvas.mpl_connect(
            'motion_notify_event', self.__motion_notify_callback)
        self.__ID2 = self.fig.canvas.mpl_connect(
            'button_press_event', self.__button_press_callback)

        if sys.flags.interactive:
            plt.show(block=False)
        else:
            plt.show()

    def getMask(self, currentImage):
        nx, ny = np.shape(currentImage)
        poly_verts = [(self.allxpoints[0], self.allypoints[0])]
        for i in range(len(self.allxpoints)-1, -1, -1):
            poly_verts.append((self.allxpoints[i], self.allypoints[i]))

        # Create vertex coordinates for each grid cell...
        # (<0,0> is at the top left of the grid in this system)
        x, y = np.meshgrid(np.arange(nx), np.arange(ny))
        x, y = x.flatten(), y.flatten()
        points = np.vstack((x,y)).T

        ROIpath = mplPath.Path(poly_verts)
        grid = ROIpath.contains_points(points).reshape((nx,ny))
        return grid

    def displayROI(self,**linekwargs):
        l = plt.Line2D(self.allxpoints +
                     [self.allxpoints[0]],
                     self.allypoints +
                     [self.allypoints[0]],
                     color=self.roicolor, **linekwargs)
        ax = plt.gca()
        ax.add_line(l)
        plt.draw()

    def displayMean(self,currentImage, **textkwargs):
        mask = self.getMask(currentImage)
        meanval = np.mean(np.extract(mask, currentImage))
        stdval = np.std(np.extract(mask, currentImage))
        string = "%.3f +- %.3f" % (meanval, stdval)
        plt.text(self.allxpoints[0], self.allypoints[0],
                 string, color=self.roicolor,
                 bbox=dict(facecolor='w', alpha=0.6), **textkwargs)

    def __motion_notify_callback(self, event):
        if event.inaxes:
            ax = event.inaxes
            x, y = event.xdata, event.ydata
            if (event.button == None or event.button == 1) and self.line != None: # Move line around
                self.line.set_data([self.previous_point[0], x],
                                   [self.previous_point[1], y])
                self.fig.canvas.draw()

    def __button_press_callback(self, event):
        if event.inaxes:
            x, y = event.xdata, event.ydata
            ax = event.inaxes
            if event.button == 1 and event.dblclick == False:  # If you press the left button, single click
                if self.line == None: # if there is no line, create a line
                    self.line = plt.Line2D([x, x],
                                           [y, y],
                                           marker='o',
                                           color=self.roicolor)
                    self.start_point = [x,y]
                    self.previous_point =  self.start_point
                    self.allxpoints=[x]
                    self.allypoints=[y]

                    ax.add_line(self.line)
                    self.fig.canvas.draw()
                    # add a segment
                else: # if there is a line, create a segment
                    self.line = plt.Line2D([self.previous_point[0], x],
                                           [self.previous_point[1], y],
                                           marker = 'o',color=self.roicolor)
                    self.previous_point = [x,y]
                    self.allxpoints.append(x)
                    self.allypoints.append(y)

                    event.inaxes.add_line(self.line)
                    self.fig.canvas.draw()
            elif ((event.button == 1 and event.dblclick==True) or
                  (event.button == 3 and event.dblclick==False)) and self.line != None: # close the loop and disconnect
                self.fig.canvas.mpl_disconnect(self.__ID1) #joerg
                self.fig.canvas.mpl_disconnect(self.__ID2) #joerg

                self.line.set_data([self.previous_point[0],
                                    self.start_point[0]],
                                   [self.previous_point[1],
                                    self.start_point[1]])
                ax.add_line(self.line)
                self.fig.canvas.draw()
                self.line = None

                if sys.flags.interactive:
                    pass
                else:
                    #figure has to be closed so that code can continue
                    plt.close(self.fig) <commit_msg>Bugfix: getMask()<commit_after>'''Draw polygon regions of interest (ROIs) in matplotlib images,
similar to Matlab's roipoly function.

See the file example.py for an application.

Created by Joerg Doepfert 2014 based on code posted by Daniel
Kornhauser.

'''

import numpy as np
import sys
import matplotlib.pyplot as plt
import matplotlib.path as mplPath

class roipoly:

    def __init__(self, fig=[], ax=[], roicolor='b'):
        if fig == []:
            fig = plt.gcf()

        if ax == []:
            ax = plt.gca()

        self.previous_point = []
        self.allxpoints = []
        self.allypoints = []
        self.start_point = []
        self.end_point = []
        self.line = None
        self.roicolor = roicolor
        self.fig = fig
        self.ax = ax
        #self.fig.canvas.draw()

        self.__ID1 = self.fig.canvas.mpl_connect(
            'motion_notify_event', self.__motion_notify_callback)
        self.__ID2 = self.fig.canvas.mpl_connect(
            'button_press_event', self.__button_press_callback)

        if sys.flags.interactive:
            plt.show(block=False)
        else:
            plt.show()

    def getMask(self, currentImage):
        ny, nx = np.shape(currentImage)
        poly_verts = [(self.allxpoints[0], self.allypoints[0])]
        for i in range(len(self.allxpoints)-1, -1, -1):
            poly_verts.append((self.allxpoints[i], self.allypoints[i]))

        # Create vertex coordinates for each grid cell...
        # (<0,0> is at the top left of the grid in this system)
        x, y = np.meshgrid(np.arange(nx), np.arange(ny))
        x, y = x.flatten(), y.flatten()
        points = np.vstack((x,y)).T

        ROIpath = mplPath.Path(poly_verts)
        grid = ROIpath.contains_points(points).reshape((ny,nx))
        return grid

    def displayROI(self,**linekwargs):
        l = plt.Line2D(self.allxpoints +
                     [self.allxpoints[0]],
                     self.allypoints +
                     [self.allypoints[0]],
                     color=self.roicolor, **linekwargs)
        ax = plt.gca()
        ax.add_line(l)
        plt.draw()

    def displayMean(self,currentImage, **textkwargs):
        mask = self.getMask(currentImage)
        meanval = np.mean(np.extract(mask, currentImage))
        stdval = np.std(np.extract(mask, currentImage))
        string = "%.3f +- %.3f" % (meanval, stdval)
        plt.text(self.allxpoints[0], self.allypoints[0],
                 string, color=self.roicolor,
                 bbox=dict(facecolor='w', alpha=0.6), **textkwargs)

    def __motion_notify_callback(self, event):
        if event.inaxes:
            ax = event.inaxes
            x, y = event.xdata, event.ydata
            if (event.button == None or event.button == 1) and self.line != None: # Move line around
                self.line.set_data([self.previous_point[0], x],
                                   [self.previous_point[1], y])
                self.fig.canvas.draw()

    def __button_press_callback(self, event):
        if event.inaxes:
            x, y = event.xdata, event.ydata
            ax = event.inaxes
            if event.button == 1 and event.dblclick == False:  # If you press the left button, single click
                if self.line == None: # if there is no line, create a line
                    self.line = plt.Line2D([x, x],
                                           [y, y],
                                           marker='o',
                                           color=self.roicolor)
                    self.start_point = [x,y]
                    self.previous_point =  self.start_point
                    self.allxpoints=[x]
                    self.allypoints=[y]

                    ax.add_line(self.line)
                    self.fig.canvas.draw()
                    # add a segment
                else: # if there is a line, create a segment
                    self.line = plt.Line2D([self.previous_point[0], x],
                                           [self.previous_point[1], y],
                                           marker = 'o',color=self.roicolor)
                    self.previous_point = [x,y]
                    self.allxpoints.append(x)
                    self.allypoints.append(y)

                    event.inaxes.add_line(self.line)
                    self.fig.canvas.draw()
            elif ((event.button == 1 and event.dblclick==True) or
                  (event.button == 3 and event.dblclick==False)) and self.line != None: # close the loop and disconnect
                self.fig.canvas.mpl_disconnect(self.__ID1) #joerg
                self.fig.canvas.mpl_disconnect(self.__ID2) #joerg

                self.line.set_data([self.previous_point[0],
                                    self.start_point[0]],
                                   [self.previous_point[1],
                                    self.start_point[1]])
                ax.add_line(self.line)
                self.fig.canvas.draw()
                self.line = None

                if sys.flags.interactive:
                    pass
                else:
                    #figure has to be closed so that code can continue
                    plt.close(self.fig) <|endoftext|>


<commit_before>#!/usr/bin/env python
# @file        rolling.py
# @author      Michael Foukarakis
# @version     0.1
# @date        Created:     Tue Feb 14, 2012 13:19 GTB Standard Time
#              Last Update: Thu Nov 13, 2014 19:21 SAST
#------------------------------------------------------------------------
# Description: Rolling window generator over an iterator.
#------------------------------------------------------------------------
# History:     0.1 - First implementation
# TODO:        <+missing features+>
#------------------------------------------------------------------------
# -*- coding: utf-8 -*-
#------------------------------------------------------------------------
from itertools import tee, islice, count
from collections import deque

def rolling_window(iterator, length, step = 1):
    """Returns an iterator of length LENGTH over ITERATOR, which advances by STEP after
    each call.
    >>> set(rolling_window(range(10), 3)) == {(0,1,2), (1,2,3), (2,3,4), (3,4,5), (4,5,6), (5,6,7), (6,7,8), (7,8,9)}
    True
    """
    streams = tee(iterator, length)
    return zip(*[islice(s, i, None, step) for s,i in zip(streams, count())])

def rolling_average(sequence, length):
    for win in rolling_window(sequence, length):
        yield sum(win) / length
<commit_msg>Removes unused deque import<commit_after>#!/usr/bin/env python
# @file        rolling.py
# @author      Michael Foukarakis
# @version     0.1
# @date        Created:     Tue Feb 14, 2012 13:19 GTB Standard Time
#              Last Update: Thu Nov 13, 2014 20:19 SAST
#------------------------------------------------------------------------
# Description: Rolling window generator over an iterator.
#------------------------------------------------------------------------
# History:     0.1 - First implementation
# TODO:        <+missing features+>
#------------------------------------------------------------------------
# -*- coding: utf-8 -*-
#------------------------------------------------------------------------
from itertools import tee, islice, count

def rolling_window(iterator, length, step = 1):
    """Returns an iterator of length LENGTH over ITERATOR, which advances by STEP after
    each call.
    >>> set(rolling_window(range(10), 3)) == {(0,1,2), (1,2,3), (2,3,4), (3,4,5), (4,5,6), (5,6,7), (6,7,8), (7,8,9)}
    True
    """
    streams = tee(iterator, length)
    return zip(*[islice(s, i, None, step) for s,i in zip(streams, count())])

def rolling_average(sequence, length):
    for win in rolling_window(sequence, length):
        yield sum(win) / length
<|endoftext|>


<commit_before>from pprint import pprint

import django.utils.timezone
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.db import IntegrityError, models
from django.utils.text import slugify

from utils.classes import LoggedInUser
from utils.functions.cadastro import CNPJ

class PosicaoCarga(models.Model):
    nome = models.CharField(
        max_length=20,
        db_index=True, unique=True)

    def __str__(self):
        return self.nome

    class Meta:
        db_table = "fo2_pos_carga"
        verbose_name = "Posição de carga(NF)"
        verbose_name_plural = "Posições de carga(NF)"

class NotaFiscal(models.Model):
    # campos importados
    numero = models.IntegerField(
        db_index=True, unique=True, verbose_name='número')
    ativa = models.BooleanField(
        db_index=True, default=True)
    faturamento = models.DateTimeField(
        db_index=True, null=True, blank=True)
    cod_status = models.IntegerField(
        null=True, blank=True,
        verbose_name='status')
    msg_status = models.CharField(
        max_length=200, null=True, blank=True,
        verbose_name='status (descr.)')
    dest_cnpj = models.CharField(
        max_length=20, null=True, blank=True,
        verbose_name='CNPJ')
    dest_nome = models.CharField(
        max_length=100, null=True, blank=True,
        verbose_name='Destinatário')
    natu_venda = models.BooleanField(
        db_index=True, default=False, verbose_name='venda')
    uf = models.CharField(
        max_length=2, null=True, blank=True,
        verbose_name='UF')
    natu_descr = models.CharField(
        max_length=100, null=True, blank=True,
        verbose_name='natureza')
    transp_nome = models.CharField(
        max_length=100, null=True, blank=True,
        verbose_name='transportadora')
    valor = models.DecimalField(
        decimal_places=2, max_digits=10,
        null=True, blank=True)
    volumes = models.IntegerField(
        null=True, blank=True)
    pedido = models.IntegerField(
        null=True, blank=True,
        verbose_name='pedido')
    ped_cliente = models.CharField(
        max_length=30, null=True, blank=True,
        verbose_name='pedido cliente')
    nf_devolucao = models.IntegerField(
        null=True, blank=True, verbose_name='Devolução')
    trail = models.CharField(
        db_index=True, max_length=32, null=True, blank=True, default='')
    posicao = models.ForeignKey(
        PosicaoCarga, default=1,
        verbose_name='Posição',
        on_delete=models.PROTECT)

    # campos editáveis
    saida = models.DateField(null=True, blank=True, verbose_name='saída')
    entrega = models.DateField(
        null=True, blank=True,
        verbose_name='agendamento')
    confirmada = models.BooleanField(
        default=False,
        verbose_name='entregue')
    observacao = models.TextField(
        null=True, blank=True,
        verbose_name='observação')

    def save(self, *args, **kwargs):
        if self.id:
            if self.saida:
                if self.posicao.id != 3:
                    self.posicao = PosicaoCarga.objects.get(id=3)
            else:
                if self.posicao.id == 3:
                    self.posicao = PosicaoCarga.objects.get(id=1)
        if self.id:
            if self.confirmada:
                if self.posicao.id != 5:
                    self.posicao = PosicaoCarga.objects.get(id=5)
            else:
                if self.posicao.id == 5:
                    if self.saida:
                        self.posicao = PosicaoCarga.objects.get(id=3)
                    else:
                        self.posicao = PosicaoCarga.objects.get(id=1)
        super(NotaFiscal, self).save(*args, **kwargs)

    class Meta:
        db_table = "fo2_fat_nf"
        verbose_name = "Nota Fiscal"
        verbose_name_plural = "Notas Fiscais"
        permissions = (("can_beep_shipment", "Can beep shipment"),
                       )

class RotinaLogistica(models.Model):
    nome = models.CharField(
        max_length=30,
        db_index=True, unique=True)
    slug = models.SlugField()

    def __str__(self):
        return self.nome

    class Meta:
        db_table = "fo2_logist_rotina"
        verbose_name = "Rotina ligada à app Logistica"
        verbose_name_plural = "Rotinas ligadas à app Logistica"

    def save(self, *args, **kwargs):
        self.slug = slugify(self.nome)
        super(RotinaLogistica, self).save(*args, **kwargs)

class PosicaoCargaAlteracao(models.Model):
    inicial = models.ForeignKey(
        PosicaoCarga,
        verbose_name='Estado inicial',
        related_name='posicao_inicial_set',
        on_delete=models.PROTECT)
    ordem = models.IntegerField(default=0)
    descricao = models.CharField(
        'descrição', max_length=200)
    efeito = models.ForeignKey(
        RotinaLogistica, default=1,
        on_delete=models.PROTECT)
    final = models.ForeignKey(
        PosicaoCarga,
        verbose_name='Estado Final',
        related_name='posicao_final_set',
        on_delete=models.PROTECT)
    so_nfs_ativas = models.BooleanField(
        verbose_name='Só NFs ativas',
        default=True)

    class Meta:
        db_table = "fo2_pos_carga_alt"
        verbose_name = "Alteração de posição de carga(NF)"
        verbose_name_plural = "Alterações de Posição de carga(NF)"

class PosicaoCargaAlteracaoLog(models.Model):
    numero = models.IntegerField(
        db_index=True, verbose_name='número')
    time = models.DateTimeField(
        db_index=True,
        auto_now_add=True, verbose_name='Hora')
    user = models.CharField(
        db_index=True,
        max_length=64, verbose_name='Usuário')
    inicial = models.ForeignKey(
        PosicaoCarga,
        verbose_name='Estado inicial',
        related_name='log_posicao_inicial_set',
        on_delete=models.PROTECT)
    final = models.ForeignKey(
        PosicaoCarga,
        verbose_name='Estado Final',
        related_name='log_posicao_final_set',
        on_delete=models.PROTECT)
    saida = models.DateField(null=True, blank=True, verbose_name='saída')

    class Meta:
        db_table = "fo2_pos_carga_alt_log"
        verbose_name = "Log de alteração de posição de carga(NF)"

class NfEntrada(models.Model):
    cadastro = models.CharField(
        'CNPJ', max_length=20, default='')
    emissor = models.CharField(
        max_length=200)
    numero = models.IntegerField(
        'número')
    descricao = models.CharField(
        'descrição', max_length=200)
    qtd = models.DecimalField(
        'quantidade', max_digits=13, decimal_places=4)
    hora_entrada = models.TimeField(
        'hora de entrada', default=django.utils.timezone.now)
    transportadora = models.CharField(
        max_length=100)
    motorista = models.CharField(
        max_length=100)
    placa = models.CharField(
        max_length=10)
    responsavel = models.CharField(
        'responsável', max_length=100)
    usuario = models.ForeignKey(
        User, models.PROTECT,
        verbose_name='usuário')
    quando = models.DateTimeField(
        null=True, auto_now_add=True)

    def __str__(self):
        cnpj = CNPJ()
        return f'{cnpj.mask(self.cadastro)} NF {self.numero}'

    def save(self, *args, **kwargs):
        val_cnpj = CNPJ()
        if val_cnpj.validate(self.cadastro):
            self.cadastro = val_cnpj.cnpj
        else:
            raise ValidationError(f"Cadastro nacional inválido.")
        logged_in = LoggedInUser()
        self.usuario = logged_in.user
        try:
            super(NfEntrada, self).save(*args, **kwargs)
        except IntegrityError as e:
            if '(cadastro, numero)' in str(e):
                raise IntegrityError(
                    f"CNPJ '{self.cadastro}' com número de NF '{self.numero}' já gravado"
                )
            else:
                raise e

    class Meta:
        db_table = "fo2_nf_entrada"
        verbose_name = "Nota fiscal de entrada"
        verbose_name_plural = "Notas fiscais de entrada"
        unique_together = [["cadastro", "numero"]]
<commit_msg>passei validações para o método clean; Dessa forma a mensagem de erro do unique_together ficou automaticamente amigável<commit_after>from pprint import pprint

import django.utils.timezone
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.db import IntegrityError, models
from django.utils.text import slugify

from utils.classes import LoggedInUser
from utils.functions.cadastro import CNPJ

class PosicaoCarga(models.Model):
    nome = models.CharField(
        max_length=20,
        db_index=True, unique=True)

    def __str__(self):
        return self.nome

    class Meta:
        db_table = "fo2_pos_carga"
        verbose_name = "Posição de carga(NF)"
        verbose_name_plural = "Posições de carga(NF)"

class NotaFiscal(models.Model):
    # campos importados
    numero = models.IntegerField(
        db_index=True, unique=True, verbose_name='número')
    ativa = models.BooleanField(
        db_index=True, default=True)
    faturamento = models.DateTimeField(
        db_index=True, null=True, blank=True)
    cod_status = models.IntegerField(
        null=True, blank=True,
        verbose_name='status')
    msg_status = models.CharField(
        max_length=200, null=True, blank=True,
        verbose_name='status (descr.)')
    dest_cnpj = models.CharField(
        max_length=20, null=True, blank=True,
        verbose_name='CNPJ')
    dest_nome = models.CharField(
        max_length=100, null=True, blank=True,
        verbose_name='Destinatário')
    natu_venda = models.BooleanField(
        db_index=True, default=False, verbose_name='venda')
    uf = models.CharField(
        max_length=2, null=True, blank=True,
        verbose_name='UF')
    natu_descr = models.CharField(
        max_length=100, null=True, blank=True,
        verbose_name='natureza')
    transp_nome = models.CharField(
        max_length=100, null=True, blank=True,
        verbose_name='transportadora')
    valor = models.DecimalField(
        decimal_places=2, max_digits=10,
        null=True, blank=True)
    volumes = models.IntegerField(
        null=True, blank=True)
    pedido = models.IntegerField(
        null=True, blank=True,
        verbose_name='pedido')
    ped_cliente = models.CharField(
        max_length=30, null=True, blank=True,
        verbose_name='pedido cliente')
    nf_devolucao = models.IntegerField(
        null=True, blank=True, verbose_name='Devolução')
    trail = models.CharField(
        db_index=True, max_length=32, null=True, blank=True, default='')
    posicao = models.ForeignKey(
        PosicaoCarga, default=1,
        verbose_name='Posição',
        on_delete=models.PROTECT)

    # campos editáveis
    saida = models.DateField(null=True, blank=True, verbose_name='saída')
    entrega = models.DateField(
        null=True, blank=True,
        verbose_name='agendamento')
    confirmada = models.BooleanField(
        default=False,
        verbose_name='entregue')
    observacao = models.TextField(
        null=True, blank=True,
        verbose_name='observação')

    def save(self, *args, **kwargs):
        if self.id:
            if self.saida:
                if self.posicao.id != 3:
                    self.posicao = PosicaoCarga.objects.get(id=3)
            else:
                if self.posicao.id == 3:
                    self.posicao = PosicaoCarga.objects.get(id=1)
        if self.id:
            if self.confirmada:
                if self.posicao.id != 5:
                    self.posicao = PosicaoCarga.objects.get(id=5)
            else:
                if self.posicao.id == 5:
                    if self.saida:
                        self.posicao = PosicaoCarga.objects.get(id=3)
                    else:
                        self.posicao = PosicaoCarga.objects.get(id=1)
        super(NotaFiscal, self).save(*args, **kwargs)

    class Meta:
        db_table = "fo2_fat_nf"
        verbose_name = "Nota Fiscal"
        verbose_name_plural = "Notas Fiscais"
        permissions = (("can_beep_shipment", "Can beep shipment"),
                       )

class RotinaLogistica(models.Model):
    nome = models.CharField(
        max_length=30,
        db_index=True, unique=True)
    slug = models.SlugField()

    def __str__(self):
        return self.nome

    class Meta:
        db_table = "fo2_logist_rotina"
        verbose_name = "Rotina ligada à app Logistica"
        verbose_name_plural = "Rotinas ligadas à app Logistica"

    def save(self, *args, **kwargs):
        self.slug = slugify(self.nome)
        super(RotinaLogistica, self).save(*args, **kwargs)

class PosicaoCargaAlteracao(models.Model):
    inicial = models.ForeignKey(
        PosicaoCarga,
        verbose_name='Estado inicial',
        related_name='posicao_inicial_set',
        on_delete=models.PROTECT)
    ordem = models.IntegerField(default=0)
    descricao = models.CharField(
        'descrição', max_length=200)
    efeito = models.ForeignKey(
        RotinaLogistica, default=1,
        on_delete=models.PROTECT)
    final = models.ForeignKey(
        PosicaoCarga,
        verbose_name='Estado Final',
        related_name='posicao_final_set',
        on_delete=models.PROTECT)
    so_nfs_ativas = models.BooleanField(
        verbose_name='Só NFs ativas',
        default=True)

    class Meta:
        db_table = "fo2_pos_carga_alt"
        verbose_name = "Alteração de posição de carga(NF)"
        verbose_name_plural = "Alterações de Posição de carga(NF)"

class PosicaoCargaAlteracaoLog(models.Model):
    numero = models.IntegerField(
        db_index=True, verbose_name='número')
    time = models.DateTimeField(
        db_index=True,
        auto_now_add=True, verbose_name='Hora')
    user = models.CharField(
        db_index=True,
        max_length=64, verbose_name='Usuário')
    inicial = models.ForeignKey(
        PosicaoCarga,
        verbose_name='Estado inicial',
        related_name='log_posicao_inicial_set',
        on_delete=models.PROTECT)
    final = models.ForeignKey(
        PosicaoCarga,
        verbose_name='Estado Final',
        related_name='log_posicao_final_set',
        on_delete=models.PROTECT)
    saida = models.DateField(null=True, blank=True, verbose_name='saída')

    class Meta:
        db_table = "fo2_pos_carga_alt_log"
        verbose_name = "Log de alteração de posição de carga(NF)"

class NfEntrada(models.Model):
    cadastro = models.CharField(
        'CNPJ', max_length=20, default='')
    emissor = models.CharField(
        max_length=200)
    numero = models.IntegerField(
        'número')
    descricao = models.CharField(
        'descrição', max_length=200)
    qtd = models.DecimalField(
        'quantidade', max_digits=13, decimal_places=4)
    hora_entrada = models.TimeField(
        'hora de entrada', default=django.utils.timezone.now)
    transportadora = models.CharField(
        max_length=100)
    motorista = models.CharField(
        max_length=100)
    placa = models.CharField(
        max_length=10)
    responsavel = models.CharField(
        'responsável', max_length=100)
    usuario = models.ForeignKey(
        User, models.PROTECT,
        verbose_name='usuário')
    quando = models.DateTimeField(
        null=True, auto_now_add=True)

    def __str__(self):
        cnpj = CNPJ()
        return f'{cnpj.mask(self.cadastro)} NF {self.numero}'

    def clean_cadastro(self):
        val_cnpj = CNPJ()
        if val_cnpj.validate(self.cadastro):
            cadastro = val_cnpj.cnpj
        else:
            raise ValidationError(f"Cadastro nacional inválido.")
        return cadastro

    def clean_usuario(self):
        logged_in = LoggedInUser()
        return logged_in.user

    def clean(self):
        self.cadastro = self.clean_cadastro()
        self.usuario = self.clean_usuario()

    class Meta:
        db_table = "fo2_nf_entrada"
        verbose_name = "Nota fiscal de entrada"
        verbose_name_plural = "Notas fiscais de entrada"
        unique_together = [["cadastro", "numero"]]
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

print """
 ____ ____                     _____
/ ___|___ / ____  _ __  _   _ /__   /_____
\___ \ |_ \|  __|| '__|| | | |   / /| ____|
 ___) ___) | |__ | |   | |_| | / /_ | ____|
|____/____/|____||_|    \__,_|/____/|_____|

S3 bucket enumeration and file upload
Release v1.0
By JR0ch17
"""

import sys, os, commands, requests, random
from argparse import ArgumentParser
from random import randrange

randomNumber = randrange(100000, 999999)
uploadFile = "BugBounty-%s.txt" % randomNumber
inputFile = ""
targetBucket = ""
upload = ""

file = open(uploadFile, 'w+')
file.write("This is a file upload test for bug bounty purposes")
file.close()

parser = ArgumentParser()
parser.add_argument("-t", "--target", dest="targetBucket", help="Select a target bucket name (e.g. 'shopify')", metavar="targetBucket", required="True")
parser.add_argument("-f", "--file", dest="inputFile", help="Select a bucket brute-forcing file (default: bucket-names.txt)", default="bucket-names.txt", metavar="inputFile")
parser.add_argument("-u", "--upload", dest="upload", help="File to upload will be automatically generated (e.g. 'BugBounty-[######].txt')", default=False, action="store_true")

args = parser.parse_args()

with open(args.inputFile, 'r') as f:
    bucketName = [line.strip() for line in f]
    lineCount = len(bucketName)

print "[*] Starting enumeration of the '%s' bucket, reading %i lines from '%s'. \n" % (args.targetBucket, lineCount, f.name)

for name in bucketName:
        r = requests.head("http://%s%s.s3.amazonaws.com" % (args.targetBucket, name))
        if r.status_code != 404:
                print "\n [+] Checking potential match: %s%s --> %s." % (args.targetBucket, name, r.status_code)
                ls = commands.getoutput("/usr/local/bin/aws s3 ls s3://%s%s" % (args.targetBucket, name))
                print ls

		if args.upload:
				    print "[+] Uploading file: %s." % (uploadFile)
				    cp = commands.getoutput("/usr/local/bin/aws s3 cp %s s3://%s%s" % (uploadFile, args.targetBucket, name))
                		    print "%s \n" % (cp)
		else:
				    sys.stdout.write('')
        else:
                sys.stdout.write('')

os.remove("%s" % (uploadFile))

print "\n [*] S3Cruze is now complete on %s." % (args.targetBucket)

<commit_msg>Update s3cruze.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

print """
 ____ ____                     _____
/ ___|___ / ____  _ __  _   _ /__   /_____
\___ \ |_ \|  __|| '__|| | | |   / /| ____|
 ___) ___) | |__ | |   | |_| | / /_ | ____|
|____/____/|____||_|    \__,_|/____/|_____|

S3 bucket enumeration and file upload
Release v1.0
By JR0ch17
"""

import sys, os, commands, requests, random
from argparse import ArgumentParser
from random import randrange

randomNumber = randrange(100000, 999999)
uploadFile = "BugBounty-%s.txt" % randomNumber
inputFile = ""
targetBucket = ""
upload = ""
delete = ""

file = open(uploadFile, 'w+')
file.write("This is a file upload test for bug bounty purposes")
file.close()

parser = ArgumentParser()
parser.add_argument("-t", "--target", dest="targetBucket", help="Select a target bucket name (e.g. 'shopify')", metavar="targetBucket", required="True")
parser.add_argument("-f", "--file", dest="inputFile", help="Select a bucket brute-forcing file (default: bucket-names.txt)", default="bucket-names.txt", metavar="inputFile")
parser.add_argument("-u", "--upload", dest="upload", help="File to upload will be automatically generated (e.g. 'BugBounty-[######].txt')", default=False, action="store_true")
parser.add_argument("-d", "--delete", dest="delete", help="Delete file from bucket after uploading it", default=False, action="store_true")

args = parser.parse_args()

with open(args.inputFile, 'r') as f:
    bucketName = [line.strip() for line in f]
    lineCount = len(bucketName)

print "[*] Starting enumeration of '%s' buckets, reading %i lines from '%s'. \n" % (args.targetBucket, lineCount, f.name)

for name in bucketName:
        r = requests.head("http://%s%s.s3.amazonaws.com" % (args.targetBucket, name))
        if r.status_code != 404:
                print "\n [+] Checking potential match: %s%s --> %s." % (args.targetBucket, name, r.status_code)
                ls = commands.getoutput("/usr/local/bin/aws s3 ls s3://%s%s" % (args.targetBucket, name))
                print ls

                if args.upload:
                                    print "[+] Uploading file: %s." % (uploadFile)
                                    cp = commands.getoutput("/usr/local/bin/aws s3 cp %s s3://%s%s" % (uploadFile, args.targetBucket, name))
                                    print "%s \n" % (cp)

                                    if args.remove:
                                            print "[+] Delete file: %s." % (uploadFile)
                                            rm = commands.getoutput("/usr/local/bin/aws s3 rm s3://%s%s/%s" % (args.targetBucket, name, uploadFile))
                                            print "%s \n" % (rm)
                                    else:
                                            sys.stdout.write('')
                else:
                                    sys.stdout.write('')
        else:
                sys.stdout.write('')

os.remove("%s" % (uploadFile))

print "\n [*] S3Cruze is now complete on %s." % (args.targetBucket)
<|endoftext|>


<commit_before>#!/usr/bin/python

from dawg import IntDAWG
from dawg import DAWG

import struct, json, bz2
from helper import *
from honeyvault_config import *
from scanner_helper import *

class Scanner:
    d_w_d = re.compile(r'(?P<pre>[^a-zA-Z]*)(?P<word>[a-zA-Z]*)(?P<post>[^a-zA-Z]*)')
    non_alphabet = re.compile(r'^[^a-zA-Z]+$')

    def __init__( self, dawg=None, tweaker=None):
        self.dawg = dawg
        # TODO add these facilities later
        #self.keyboard = KeyBoard()
        #self.date = Date()
        #self.standard_english = None;
        self.M = tweaker or Tweaker()
        if not self.dawg and os.path.exists(DICTIONARY_DAWG):
            # print 'Using the old dicrionary already stored!'
            self.dawg = DAWG()
            self.dawg.load(DICTIONARY_DAWG)
        if not self.dawg:
            print "Oh My God!! You dont have the dictionary({DICTIONARY_DAWG}). I can't take it. Exiting!".format(**locals())
            exit(0);
        self.infix = '@._*'

    # scores how good is the chunking
    def score(self, orig, wlist):
        return sum([len(w)*len(w) for w in wlist])
        dic_words = filter(lambda x: unicode(x) in self.standard_english, wlist )
        num_words = filter(lambda x : re.match(self.non_alphabet, x) is not None, wlist)
        # t_d = float(sum([len(x) for x in dic_words]))/(0.1+len(dic_words))+1
        # t_all = len(orig)/len(wlist)
        # t_last = (0 if re.match(self.non_alphabet, wlist[-1]) is None else 1)
        # t_last+= (0 if re.match(self.non_alphabet, wlist[0] ) is None else 1)
        non_dic_words = 2*len(wlist)-len(num_words)-len(dic_words)
        scr = - non_dic_words #float(t_d)+t_all + t_last + (len(wlist) - len(dic_words) - len(num_words))
        #print 'Score:', wlist, scr
        return scr

    # It converts the password into unmnagled version and
    # break it into 'best' possible chunks
    def unmangle_word(self, w, pre=''):
        if re.match(self.non_alphabet, w): return [w]
        if len(w)>6 and len(set(w))<=2: return None

        #nw = w
        w = w.lower();
        nw = self.M.tweak(w) if len(w)>1 else w
        P = self.dawg.prefixes(unicode(nw))
        if not P:
            # print 'Failed!!', w;
            return [w]
        if len(P)<=1 and P[0].isalpha() and (len(w) - len(P[-1]))*2>len(w):
            # print "Couldnot find!", w, '-->', P
            return None
        #P.sort(key = lambda x: len(x)*(1+(x in self.standard_english)), reverse=True)
        P.sort(key = lambda x: len(x), reverse = True)
        #print pre+w+str(P)
        nWlist=[]
        oWlist=[[], -99]
        test = 0;
        for p in P:
            npw = '<3>';
            if len(p) < len(w):
                npw = self.unmangle_word(w[len(p):], pre+'++')
            if npw:
                nWlist.append(p)
                if npw != '<3>': nWlist.extend(npw)
                s = self.score(w, nWlist)
                if s>oWlist[1]:
                    oWlist = [nWlist, s]
                    #print pre+"Changing: "+str(oWlist)
                nWlist = []
                if test>2 or len(p) == len(w): break
                test+=1
        #if oWlist[0]: print pre+"returning==>", oWlist, [nWlist, s]
        return oWlist[0];

    # Splits the word into parts,
    # tokenize works like the following
    # Input: Comput3rS3cret@1302 -->
    # Output:
    # (['L8', 'L6', 'Y1', 'D4'],
    #  [u'computer', u'secret', u'@', u'1302'],
    #  [u'Comput3r', u'S3cret', u'@', u'1302'])
    def tokenize(self, w, isMangling=False):
        #s = self.keyboard.IsKeyboardSeq(w)
        # if s: return ['K%d'%len(s)], s, [w]
        #grp = self.date.IsDate(w)
        #if grp:
        #    print grp
        #    # w1,w2=grp[0], grp[1]
        #    # dt, mob=grp[2], grp[3]
        #    # if dt: 'T'
        #    # R1, R2=[[],[],[]], [[],[],[]]
        #    # if w1: R1 = tokenize(w1, isMangling)
        #    # if w2: R2 = tokenize(w2, isMangling)
        #    # print 'Mobmatch:', w1, dt, mob, w2
        #else:
        #    pass
        #    #print "not Date"
        save_w=w
        T = [w]
        # print self.unmangle_word(w)
        #m = re.match(self.d_w_d, w)
        if isMangling:
            # Dont do any fansy unmangling now..:P
            T = self.unmangle_word(w) # unmangled breakup
            #T = [ sym for w in T for list_match in re.findall(regex, w)
            #      for sym in list_match if sym ]
            if not T: T = [w]
            # print 'TOkenize:', T

        # Dumbest Grammar one can think of,. But just to complete the pipeline
        # will come an reiterate later
        # if len(T) > 5:
        #     print "Say something, I am giving upon you:", w, T;
        #     return None, None
        Tags = [None for i in range(len(T)+2)]
        Tags[0] = '^'
        Tags[-1] = '$'
        for i, w in enumerate(T):
            if len(w)>4: Tags[i+1] = 'W'

        if None in Tags:
            if len(Tags) == 3: Tags[1] = 'W'
            elif not all(Tags):
                for i, w in enumerate(T):
                    if len(w)>3: Tags[i+1] = 'W'

            conversion_rules = { '^' : { 'W' : 'P', None : 'W' },
                                 'W' : { 'P' : 'S', 'W' : 'S', '$' : 'S', None: 'S' },
                                 'S' : { 'W' : 'P', 'P' : 'I', None: 'W', '$': 'P'}
                                 }
            for i, t in enumerate(Tags[1:-1]):
                if t: continue;
                if not i:
                    if len(T[i]) >= 4: Tags[i+1] = 'W'
                try:
                    Tags[i+1] = conversion_rules[Tags[i]][Tags[i+2]]
                except KeyError or IndexError:
                    print Tags, T

        # for i, t in enumerate(Tags):
        # W = [ sym for w in T for list_match in re.findall(regex, w)
        #       for sym in list_match if sym ]
        # P = [ "%s%d" % ( whatchar(x[0]), len(x)) for x in W ]
        U,s,w = [], 0, save_w
        for p in T:
             U.append(w[s:s+len(p)])
             s += len(p)
        return Tags[1:-1], T, U

class Grammar:
    inversemap = {}
    def __init__(self, config_fl=None, scanner=None):
        self.scanner = scanner;
        if not scanner:
            self.scanner = Scanner()
        self.grammar_structure = GrammarStructure().G
        self.G={}
        for k,v in self.grammar_structure.items():
            if len(v)!=1:
                keys = v
                freq = [[0, (NONTERMINAL if x.isalpha() else TERMINAL)] for x in v]
                self.G[k] = [keys, freq, 0]

        # self.addDotStarRules();
        # TODO: make it better
        from string import ascii_letters, digits, punctuation
        for typ, characters in zip('LDY', [ascii_letters, digits, punctuation]):
            self.G[typ] = [[x for x in characters], [[MIN_COUNT-1, TERMINAL] for x in characters], 0]
            self.G['G'][0].append('%s,G' % typ)
            self.G['G'][1].append([MIN_COUNT-1, NONTERMINAL])
            self.G['G'][2] += MIN_COUNT-1

    def addRule_lite( self, lhs, rhs, freq, typ, ignore_addingfreq = False): # typ is NONTERMINAL or NERMINAL
        G = self.G
        try:
            i = G[lhs][0].index(rhs)
            if not ignore_addingfreq:
                G[lhs][1][i][0] += freq
                G[lhs][2] += freq
        except ValueError:
            self.G[lhs][0].append(rhs)
            self.G[lhs][1].append([freq, typ])
            self.G[lhs][2] += freq
        except KeyError:
            G[lhs]=[[rhs], [[freq,  typ]], freq]

    def addRule(self, lhs, rhs, isNonT, freq):
        G = self.G
        # is lhs exist?
        try: x = G[lhs]
        except KeyError: # insert the lhs first
            G[lhs] = []
        try: # found in the inversemap => previous entry exist
            if len(rhs)>1:
                pos = Grammar.inversemap[rhs]
            else:
                pos = -1;
                for i,x in enumerate(G[lhs]):
                    if rhs == x[0]:
                        pos = i;
                        break
            if pos==-1: G[lhs].append([rhs, isNonT, freq])
            else: G[lhs][pos][2] += freq
        except KeyError: #first time in this place
            Grammar.inversemap[rhs] = len(G[lhs])
            G[lhs].append([rhs,isNonT,freq])
        except IndexError:
            print 'IndexERROR:', lhs, '--->', rhs, freq
            for k,v in G.items(): print k, '=>', v
            exit()

    # Completely messed up now
    def insert(self, w, freq):
        P, W, U = self.scanner.tokenize(w, True)
        # TODO Add mangling
        # Improve the grammar
        rule = ','.join(P)
        self.addRule_lite('G', rule, freq, NONTERMINAL)
        # print w, freq, P, W, U
        # S -> P
        # self.addRule('S', ','.join(P), 1, freq)
        # for i,p in enumerate(P):
        #     if len(W[i])>30: continue; # too big password
        #     # first add the nonterminal word
        #     if W[i].isalpha():
        #         self.addRule(p, W[i], 2, freq) # word NT- Complicated!
        #         for w,u in zip(W[i],U[i]):
        #             self.addRule(w, u, 0, freq)
        #     else:
        #         self.addRule(p, W[i], 0, freq)
        # print 'insert: ', U

    def addDotStarRules(self):
        """
        This is to support parsing all possible passwords.
        Artifical rules like, S -> L,S | D,S | Y,S
        and L -> a|b|c|d..
        D -> 1|3|4 etc
        """
        self.addRule('G', 'L,G', 1, MIN_COUNT-1);
        self.addRule('G', 'D,G', 1, MIN_COUNT-1);
        self.addRule('G', 'Y,G', 1, MIN_COUNT-1);
        import string
        for c in string.ascii_letters:
            self.addRule('L', c, 0, MIN_COUNT-1 )
        for d in string.digits:
            self.addRule('D', d, 0, MIN_COUNT-1 )
        for s in string.punctuation:
            self.addRule('Y', s, 0, MIN_COUNT-1 )
        #for e, s in self.scanner.M.rules.items():
        #    self.addRule(s, e, 0, MIN_COUNT-1)
        # mangling_rule={'a':'@', 's':'$', 'o':'0', 'i':'!'}

    def save(self, out_file):
        #for rule in self.G:
        #    self.G[rule].sort(key = lambda x: x[0])
        json.dump(self.G, out_file)

    def load(self, in_file):
        self.G = json.load(open_(in_file))
        # TODO - TOCHECK wheather required or not
        # for rule in self.G:
        #     for i,w in enumerate(self.G[rule]):
        #         if w[0] != rule:
        #             inversemap[w[0]] = i

    # find the left most gegneration tree for a password
    # TODO -  NOT impelemetned

    def __iter__(self):
        return self.G.iterator

    def __getitem__(self, key):
        return self.G[key]

    def __contains__(self, x):
        return x in self.G

    def __str__(self):
        return "Grammar: " + str(self.G)

    def findPath(self, w):
        # scanner cannont be null,
        P, W, U = self.scanner.tokenize(w)
        # start path
        try: pos = inversemap[','.join(P)]
        except KeyError:
            print "Falling back to all-match Rule!"
            return [] # Not implemented till now
        print pos

<commit_msg>temporary<commit_after>#!/usr/bin/python

from dawg import IntDAWG
from dawg import DAWG

import struct, json, bz2
from helper import *
from honeyvault_config import *
from scanner_helper import *

class Scanner:
    d_w_d = re.compile(r'(?P<pre>[^a-zA-Z]*)(?P<word>[a-zA-Z]*)(?P<post>[^a-zA-Z]*)')
    non_alphabet = re.compile(r'^[^a-zA-Z]+$')

    def __init__( self, dawg=None, tweaker=None):
        self.dawg = dawg
        # TODO add these facilities later
        #self.keyboard = KeyBoard()
        #self.date = Date()
        #self.standard_english = None;
        self.M = tweaker or Tweaker()
        if not self.dawg and os.path.exists(DICTIONARY_DAWG):
            # print 'Using the old dicrionary already stored!'
            self.dawg = DAWG()
            self.dawg.load(DICTIONARY_DAWG)
        if not self.dawg:
            print "Oh My God!! You dont have the dictionary({DICTIONARY_DAWG}). I can't take it. Exiting!".format(**locals())
            exit(0);
        self.infix = '@._*'

    # scores how good is the chunking
    def score(self, orig, wlist):
        return sum([len(w)*len(w) for w in wlist])
        dic_words = filter(lambda x: unicode(x) in self.standard_english, wlist )
        num_words = filter(lambda x : re.match(self.non_alphabet, x) is not None, wlist)
        # t_d = float(sum([len(x) for x in dic_words]))/(0.1+len(dic_words))+1
        # t_all = len(orig)/len(wlist)
        # t_last = (0 if re.match(self.non_alphabet, wlist[-1]) is None else 1)
        # t_last+= (0 if re.match(self.non_alphabet, wlist[0] ) is None else 1)
        non_dic_words = 2*len(wlist)-len(num_words)-len(dic_words)
        scr = - non_dic_words #float(t_d)+t_all + t_last + (len(wlist) - len(dic_words) - len(num_words))
        #print 'Score:', wlist, scr
        return scr

    # It converts the password into unmnagled version and
    # break it into 'best' possible chunks
    def unmangle_word(self, w, pre=''):
        if re.match(self.non_alphabet, w): return [w]
        if len(w)>6 and len(set(w))<=2: return None

        #nw = w
        w = w.lower();
        nw = self.M.tweak(w) if len(w)>1 else w
        P = self.dawg.prefixes(unicode(nw))
        if not P:
            # print 'Failed!!', w;
            return [w]
        if len(P)<=1 and P[0].isalpha() and (len(w) - len(P[-1]))*2>len(w):
            # print "Couldnot find!", w, '-->', P
            return None
        #P.sort(key = lambda x: len(x)*(1+(x in self.standard_english)), reverse=True)
        P.sort(key = lambda x: len(x), reverse = True)
        #print pre+w+str(P)
        nWlist=[]
        oWlist=[[], -99]
        test = 0;
        for p in P:
            npw = '<3>';
            if len(p) < len(w):
                npw = self.unmangle_word(w[len(p):], pre+'++')
            if npw:
                nWlist.append(p)
                if npw != '<3>': nWlist.extend(npw)
                s = self.score(w, nWlist)
                if s>oWlist[1]:
                    oWlist = [nWlist, s]
                    #print pre+"Changing: "+str(oWlist)
                nWlist = []
                if test>2 or len(p) == len(w): break
                test+=1
        #if oWlist[0]: print pre+"returning==>", oWlist, [nWlist, s]
        return oWlist[0];

    # Splits the word into parts,
    # tokenize works like the following
    # Input: Comput3rS3cret@1302 -->
    # Output:
    # (['L8', 'L6', 'Y1', 'D4'],
    #  [u'computer', u'secret', u'@', u'1302'],
    #  [u'Comput3r', u'S3cret', u'@', u'1302'])
    def tokenize(self, w, isMangling=False):
        #s = self.keyboard.IsKeyboardSeq(w)
        # if s: return ['K%d'%len(s)], s, [w]
        #grp = self.date.IsDate(w)
        #if grp:
        #    print grp
        #    # w1,w2=grp[0], grp[1]
        #    # dt, mob=grp[2], grp[3]
        #    # if dt: 'T'
        #    # R1, R2=[[],[],[]], [[],[],[]]
        #    # if w1: R1 = tokenize(w1, isMangling)
        #    # if w2: R2 = tokenize(w2, isMangling)
        #    # print 'Mobmatch:', w1, dt, mob, w2
        #else:
        #    pass
        #    #print "not Date"
        save_w=w
        T = [w]
        # print self.unmangle_word(w)
        #m = re.match(self.d_w_d, w)
        if isMangling:
            # Dont do any fansy unmangling now..:P
            T = self.unmangle_word(w) # unmangled breakup
            #T = [ sym for w in T for list_match in re.findall(regex, w)
            #      for sym in list_match if sym ]
            if not T: T = [w]
            # print 'TOkenize:', T

        # The Dumbest Grammar one can think of,. But just to complete the pipeline
        # will come an reiterate later
        # if len(T) > 5:
        #     print "Say something, I am giving upon you:", w, T;
        #     return None, None
        Tags = [None for i in range(len(T)+2)]
        Tags[0] = '^'
        Tags[-1] = '$'
        for i, w in enumerate(T):
            if len(w)>4: Tags[i+1] = 'W'

        if None in Tags:
            if len(Tags) == 3: Tags[1] = 'W'
            elif not all(Tags):
                for i, w in enumerate(T):
                    if len(w)>3: Tags[i+1] = 'W'

            conversion_rules = { '^' : { 'W' : 'P', None : 'W' },
                                 'W' : { 'P' : 'S', 'W' : 'S', '$' : 'S', None: 'S' },
                                 'S' : { 'W' : 'P', 'P' : 'I', None: 'W', '$': 'P'}
                                 }
            for i, t in enumerate(Tags[1:-1]):
                if t: continue;
                if not i:
                    if len(T[i]) >= 4: Tags[i+1] = 'W'
                try:
                    Tags[i+1] = conversion_rules[Tags[i]][Tags[i+2]]
                except KeyError or IndexError:
                    print Tags, T

        # for i, t in enumerate(Tags):
        # W = [ sym for w in T for list_match in re.findall(regex, w)
        #       for sym in list_match if sym ]
        # P = [ "%s%d" % ( whatchar(x[0]), len(x)) for x in W ]
        U,s,w = [], 0, save_w
        for p in T:
             U.append(w[s:s+len(p)])
             s += len(p)
        return Tags[1:-1], T, U

    def get_parse_tree(self, w):
        T, W, U = self.tokenize(v, True)
        rule = ','.join(T)
        G = {}
        G['G'] = rule
        for (l,r) in zip(T,W):
            try:
                index = G[l][0].find(r)
                if index<0:
                    G[l][0].append(r)
                    G[l][1].append([0,TERMINAL])
            except:
                G[l] = [[r],[[0,TERMINAL]]]

class Grammar:
    inversemap = {}
    def __init__(self, config_fl=None, scanner=None):
        self.scanner = scanner;
        if not scanner:
            self.scanner = Scanner()
        self.grammar_structure = GrammarStructure().G
        self.G={}
        for k,v in self.grammar_structure.items():
            if len(v)!=1:
                keys = v
                freq = [[0, (NONTERMINAL if x.isalpha() else TERMINAL)] for x in v]
                self.G[k] = [keys, freq, 0]

        # self.addDotStarRules();
        # TODO: make it better
        from string import ascii_letters, digits, punctuation
        for typ, characters in zip('LDY', [ascii_letters, digits, punctuation]):
            self.G[typ] = [[x for x in characters], [[MIN_COUNT-1, TERMINAL] for x in characters], 0]
            self.G['G'][0].append('%s,G' % typ)
            self.G['G'][1].append([MIN_COUNT-1, NONTERMINAL])
            self.G['G'][2] += MIN_COUNT-1

    def addRule_lite( self, lhs, rhs, freq, typ, ignore_addingfreq = False): # typ is NONTERMINAL or NERMINAL
        G = self.G
        try:
            i = G[lhs][0].index(rhs)
            if not ignore_addingfreq:
                G[lhs][1][i][0] += freq
                G[lhs][2] += freq
        except ValueError:
            self.G[lhs][0].append(rhs)
            self.G[lhs][1].append([freq, typ])
            self.G[lhs][2] += freq
        except KeyError:
            G[lhs]=[[rhs], [[freq,  typ]], freq]

    def addRule(self, lhs, rhs, isNonT, freq):
        G = self.G
        # is lhs exist?
        try: x = G[lhs]
        except KeyError: # insert the lhs first
            G[lhs] = []
        try: # found in the inversemap => previous entry exist
            if len(rhs)>1:
                pos = Grammar.inversemap[rhs]
            else:
                pos = -1;
                for i,x in enumerate(G[lhs]):
                    if rhs == x[0]:
                        pos = i;
                        break
            if pos==-1: G[lhs].append([rhs, isNonT, freq])
            else: G[lhs][pos][2] += freq
        except KeyError: #first time in this place
            Grammar.inversemap[rhs] = len(G[lhs])
            G[lhs].append([rhs,isNonT,freq])
        except IndexError:
            print 'IndexERROR:', lhs, '--->', rhs, freq
            for k,v in G.items(): print k, '=>', v
            exit()

    # Completely messed up now
    def insert(self, w, freq):
        P, W, U = self.scanner.tokenize(w, True)
        # TODO Add mangling
        # Improve the grammar
        rule = ','.join(P)
        self.addRule_lite('G', rule, freq, NONTERMINAL)
        # print w, freq, P, W, U
        # S -> P
        # self.addRule('S', ','.join(P), 1, freq)
        # for i,p in enumerate(P):
        #     if len(W[i])>30: continue; # too big password
        #     # first add the nonterminal word
        #     if W[i].isalpha():
        #         self.addRule(p, W[i], 2, freq) # word NT- Complicated!
        #         for w,u in zip(W[i],U[i]):
        #             self.addRule(w, u, 0, freq)
        #     else:
        #         self.addRule(p, W[i], 0, freq)
        # print 'insert: ', U

    def addDotStarRules(self):
        """
        This is to support parsing all possible passwords.
        Artifical rules like, S -> L,S | D,S | Y,S
        and L -> a|b|c|d..
        D -> 1|3|4 etc
        """
        self.addRule('G', 'L,G', 1, MIN_COUNT-1);
        self.addRule('G', 'D,G', 1, MIN_COUNT-1);
        self.addRule('G', 'Y,G', 1, MIN_COUNT-1);
        import string
        for c in string.ascii_letters:
            self.addRule('L', c, 0, MIN_COUNT-1 )
        for d in string.digits:
            self.addRule('D', d, 0, MIN_COUNT-1 )
        for s in string.punctuation:
            self.addRule('Y', s, 0, MIN_COUNT-1 )
        #for e, s in self.scanner.M.rules.items():
        #    self.addRule(s, e, 0, MIN_COUNT-1)
        # mangling_rule={'a':'@', 's':'$', 'o':'0', 'i':'!'}

    def save(self, out_file):
        #for rule in self.G:
        #    self.G[rule].sort(key = lambda x: x[0])
        json.dump(self.G, out_file)

    def load(self, in_file):
        self.G = json.load(open_(in_file))
        # TODO - TOCHECK wheather required or not
        # for rule in self.G:
        #     for i,w in enumerate(self.G[rule]):
        #         if w[0] != rule:
        #             inversemap[w[0]] = i

    # find the left most gegneration tree for a password
    # TODO -  NOT impelemetned

    def __iter__(self):
        return self.G.iterator

    def __getitem__(self, key):
        return self.G[key]

    def __contains__(self, x):
        return x in self.G

    def __str__(self):
        return "Grammar: " + str(self.G)

    def findPath(self, w):
        # scanner cannont be null,
        P, W, U = self.scanner.tokenize(w)
        # start path
        try: pos = inversemap[','.join(P)]
        except KeyError:
            print "Falling back to all-match Rule!"
            return [] # Not implemented till now
        print pos

<|endoftext|>


<commit_before>#!/usr/bin/env python3
# coding=utf-8
"""
This is the scanner daemon, basically just a filesystem walker that checks if the database is up-to-date
and requests others to do the updates if necessary.
"""

import argparse
from datetime import datetime
from genericpath import isfile, getmtime, getsize, isdir
from os import listdir
from os.path import join
import socket
from time import sleep
import sys

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

from base import set_session, Host, HashRequest, session

__author__ = 'konsti'

import logging

logger = logging.getLogger(__name__)

def scan(folder):
    """
    Main scanner function
    @param folder: the folder you want to scan
    @raise AttributeError: if folder is not local
    """
    if not folder.host.is_local:
        raise AttributeError('%r is not local' % folder)

    for name in listdir(folder.path):
        archive = folder.child_by_name(name)
        path = join(folder.path, name)
        if isfile(path):
            if archive is None or archive.mtime != datetime.fromtimestamp(getmtime(path)) or archive.size != getsize(
                    path):
                if archive is not None:
                    archive.delete()
                request_hash(name=name, folder=folder, mtime=datetime.fromtimestamp(getmtime(path)),
                             size=getsize(path))
        elif isdir(path):
            if archive is None:
                archive = folder.add_folder(name)
            scan(archive)

    for file in folder.files:
        if not isfile(file.path):
            file.delete()

    for child in folder.folders:
        if not isdir(child.path):
            child.delete()

def request_hash(name, folder, mtime, size):
    """
    Creates a HashRequest for the given file
    @param name: The filename
    @param folder: The parent Folder
    @param mtime: timestamp as a datetime.datetime object
    @param size: Filesize as Integer in Bytes
    """
    request = session().query(HashRequest).filter(HashRequest.name == name, HashRequest.folder == folder,
                                                  HashRequest.size == size, HashRequest.mtime == mtime).first()
    if request is None:
        request = HashRequest(name=name, folder=folder, mtime=mtime, size=size, host=folder.host)
        session().add(request)
        session().flush()

def daemon(interval):
    """
    @param interval: Integer, seconds between to scan runs
    """
    host = Host.by_name(socket.gethostname())
    while True:
        logger.debug('Stating scanner round')
        for root in host.roots:
            scan(root)
            session().commit()
        sleep(interval)

def main(args=sys.argv[1:]):
    """
    main :)
    @param args: the args used for the parser, just useful for testing
    """
    parser = argparse.ArgumentParser(description='Scanner Tool')
    parser.add_argument('-d', '--database', type=str, metavar='"Connection String"', required=True)
    parser.add_argument('--debug', action='store_true')
    parser.add_argument('-i', '--interval', type=int, metavar='SECONDS', default=360,
                        help='Interval between two Scan runs, defaults to 1 hour')
    args = parser.parse_args(args)
    if args.debug:
        logging.basicConfig(level=logging.DEBUG)
        logger.debug('Scanner started with following arguments: %s' % args)
    else:
        logging.basicConfig(level=logging.INFO)

    database = create_engine(args.database, echo=False)
    s = sessionmaker(bind=database)()
    set_session(s)
    daemon(args.interval)

if __name__ == '__main__':
    exit(main())
<commit_msg>Added debug argument to scenner<commit_after>#!/usr/bin/env python3
# coding=utf-8
"""
This is the scanner daemon, basically just a filesystem walker that checks if the database is up-to-date
and requests others to do the updates if necessary.
"""

import argparse
from datetime import datetime
from genericpath import isfile, getmtime, getsize, isdir
from os import listdir
from os.path import join
import socket
from time import sleep
import sys

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

from base import set_session, Host, HashRequest, session

__author__ = 'konsti'

import logging

logger = logging.getLogger(__name__)

def scan(folder):
    """
    Main scanner function
    @param folder: the folder you want to scan
    @raise AttributeError: if folder is not local
    """
    if not folder.host.is_local:
        raise AttributeError('%r is not local' % folder)

    for name in listdir(folder.path):
        archive = folder.child_by_name(name)
        path = join(folder.path, name)
        if isfile(path):
            if archive is None or archive.mtime != datetime.fromtimestamp(getmtime(path)) or archive.size != getsize(
                    path):
                if archive is not None:
                    archive.delete()
                request_hash(name=name, folder=folder, mtime=datetime.fromtimestamp(getmtime(path)),
                             size=getsize(path))
        elif isdir(path):
            if archive is None:
                archive = folder.add_folder(name)
            scan(archive)

    for file in folder.files:
        if not isfile(file.path):
            file.delete()

    for child in folder.folders:
        if not isdir(child.path):
            child.delete()

def request_hash(name, folder, mtime, size):
    """
    Creates a HashRequest for the given file
    @param name: The filename
    @param folder: The parent Folder
    @param mtime: timestamp as a datetime.datetime object
    @param size: Filesize as Integer in Bytes
    """
    request = session().query(HashRequest).filter(HashRequest.name == name, HashRequest.folder == folder,
                                                  HashRequest.size == size, HashRequest.mtime == mtime).first()
    if request is None:
        request = HashRequest(name=name, folder=folder, mtime=mtime, size=size, host=folder.host)
        session().add(request)
        session().flush()

def daemon(interval):
    """
    @param interval: Integer, seconds between to scan runs
    """
    host = Host.by_name(socket.gethostname())
    while True:
        logger.debug('Stating scanner round')
        for root in host.roots:
            scan(root)
            session().commit()
        sleep(interval)

def main(args=sys.argv[1:]):
    """
    main :)
    @param args: the args used for the parser, just useful for testing
    """
    parser = argparse.ArgumentParser(description='Scanner Tool')
    parser.add_argument('-d', '--database', type=str, metavar='"Connection String"', required=True)
    parser.add_argument('--debug', action='store_true')
    parser.add_argument('-i', '--interval', type=int, metavar='SECONDS', default=360,
                        help='Interval between two Scan runs, defaults to 1 hour')
    args = parser.parse_args(args)
    if args.debug:
        logging.basicConfig(level=logging.DEBUG)
        logger.debug('Scanner started with following arguments: %s' % args)
    else:
        logging.basicConfig(level=logging.INFO)

    database = create_engine(args.database, echo=False, assert_unicode=True)
    s = sessionmaker(bind=database)()
    set_session(s)
    daemon(args.interval)

if __name__ == '__main__':
    exit(main())
<|endoftext|>