<commit_before>
import re, mx.DateTime, string
from support import ugc, vtec, hvtec, reference

#TORNADO = re.compile(r"(STORM\W+CAPABLE\W+OF\W+PRODUCING|REPORTED|INDICATED?)\W+A\W+TORNADO")
TORNADO = re.compile(r"^AT |^\* AT")
WINDHAIL = re.compile(".*WIND\.\.\.HAIL (?P<winddir>[><]?)(?P<wind>[0-9]+)MPH (?P<haildir>[><]?)(?P<hail>[0-9\.]+)IN")
class TextProduct:

    def __init__(self, raw, bypass=False):
        self.idd = None
        self.wmo = None
        self.source = None
        self.issueTime = None
        self.afos = None
        self.segments = []
        self.fcster = None
        self.z = None
        self.product_header = None

        self.raw = raw.strip().replace("\015\015\012", "\n") # to unix
        self.sections = self.raw.split("\n\n")

        if not bypass:
            parseCallbacks = [self.findIssueTime, self.findSegments,
                self.findIDD, self.findWMO, self.findAFOS, self.figureFcster,
                self.find_pheader]
            for cb in parseCallbacks:
                apply( cb )

    def find_pheader(self):
        """ Figure out what the WMO AND MND area and save it for database
            updates where we don't want to save the entire SVS! """
        if (len(self.sections) < 2):
            return

        if len( self.sections[0].split("\n") ) > 3:
            self.product_header = self.sections[0]
        else:
            self.product_header = "\n\n".join(self.sections[0:2])

    def get_iembot_source(self):
        if (self.source is None or len(self.source) != 4):
            return None
        # This is a hack for now, will have to think some more about
        # what to do with the conflict between PABR and KABR
        if (self.source == "PABR"):
            return "XXX"
        return self.source[1:]

    def get_product_id(self):
        s = "%s-%s-%s-%s" % (self.issueTime.strftime("%Y%m%d%H%M"),\
                self.source, self.wmo, self.afos)
        return s.strip()

    def sqlraw(self):
        return re.sub("'", "\\'", self.raw)

    def figureFcster(self):
        self.fcster = string.strip( (self.sections[-1]).replace("\n","") )[:24]

    def findWarnedCounties(self):
        return re.findall("\n  (.*) COUNTY IN (.*)", self.raw)

    def findAFOS(self):
        """ Determine the AFOS header from the NWS text product """
        data = "\n".join([line.strip()
                         for line in self.sections[0].split("\n")])
        tokens = re.findall("^([A-Z0-9 ]{4,6})$", data, re.M)
        if (len(tokens) > 0):
            self.afos = tokens[0]

    def findWMO(self,):
        t = re.findall("^([A-Z]{4}[0-9][0-9]) ([A-Z]{4})", self.sections[0], re.M)
        if (len(t) == 1):
            self.wmo = t[0][0]
            self.source = t[0][1]

    def findIDD(self,):
        t = re.findall("^([0-9]+)", self.sections[0], re.M)
        if (len(t) > 0):
            self.idd = int(t[0])

    def findSegments(self,):
        meat = "\n\n".join( self.sections)
        segs = meat.split("$$")
        for s in segs[:-1]:
            self.segments.append(TextProductSegment(s, self.issueTime))

    def __str__(self,):
        s = """
=================================
idd: %(idd)s
wmo: %(wmo)s
source: %(source)s
issueTime: %(issueTime)s
afos: %(afos)s
--------------
""" % vars(self)
        i = 0
        for seg in self.segments:
            s += "+--- segment %s |||| %s" % (i, seg,)
            i += 1
        return s

    def generate_fake_vtec(self):
        """ I generate a fake VTEC ...
            is for non VTEC products prior to 2005, I think :) """
        r = self.raw.replace("NOON", "1200 PM").replace("MIDNIGHT", "1200 AM").replace("VALID UNTIL", "UNTIL")
        dre = "\*\s+UNTIL\s+([0-9]{1,4})\s+(AM|PM)\s+([A-Z]{3,4})"
        tokens = re.findall(dre, r)
        if (len(tokens[0][0]) < 3):
            h = int(tokens[0][0])
            m = 0
        else:
            h = int(tokens[0][0][:-2])
            m = int(tokens[0][0][-2:])
        if tokens[0][1] == "AM" and h == 12:
            h = 0
        if tokens[0][1] == "PM" and h < 12:
            h += 12
        # Figure local issued time
        issue = self.issueTime - mx.DateTime.RelativeDateTime(hours=reference.offsets[self.z])
        expire = issue + mx.DateTime.RelativeDateTime(hour=h, minute=m)
        if (expire < issue):
            expire += mx.DateTime.RelativeDateTime(days=1)
        expireZ = expire + mx.DateTime.RelativeDateTime(hours=reference.offsets[self.z])
        print issue, expire, expireZ, self.z

        if (self.afos[:3] == "SVR"):
            typ = "SV"
        elif (self.afos[:3] == "TOR"):
            typ = "TO"
        elif (self.afos[:3] == "FFW"):
            typ = "FF"

        fake = "/O.NEW.%s.%s.W.0000.%s-%s/" % (self.source, typ, \
               self.issueTime.strftime("%y%m%dT%H%MZ"), \
               expireZ.strftime("%y%m%dT%H%MZ") )
        print 'Fake VTEC', fake
        self.segments[0].vtec.append( vtec.vtec(fake) )

    def findIssueTime(self):
        """ Too much fun, we need to determine when this text product was
            issued.  We'll first look to see if it is explicitly said."""

        # Search out the WMO header first, this had better always be there
        # We only care about the first hit in the file, searching from top
        dRE = "^[A-Z0-9]{6} [A-Z]{4} ([0-3][0-9])([0-2][0-9])([0-5][0-9])"
        tokens = re.findall(dRE , self.raw, re.M)
        iDay = int(tokens[0][0])
        iHour = int(tokens[0][1])
        iMinute = int(tokens[0][2])

        # Now lets look for a local timestamp in the product MND or elsewhere
        time_re = "^([0-9]+) (AM|PM) ([A-Z][A-Z][A-Z]?T) [A-Z][A-Z][A-Z] ([A-Z][A-Z][A-Z]) ([0-9]+) ([1-2][0-9][0-9][0-9])$"
        tokens = re.findall(time_re, self.raw, re.M)
        # If we don't find anything, lets default to now, its the best
        if (len(tokens) == 0):
            hack_time = mx.DateTime.utc().strftime("%I%M %p GMT %a %b %d %Y")
            tokens = re.findall(time_re, hack_time.upper(), re.M)
        # [('1249', 'AM', 'EDT', 'JUL', '1', '2005')]
        self.z = tokens[0][2]
        if (len(tokens[0][0]) < 3):
            h = tokens[0][0]
            m = 0
        else:
            h = tokens[0][0][:-2]
            m = tokens[0][0][-2:]
        dstr = "%s:%s %s %s %s %s" % (h, m, tokens[0][1], tokens[0][3], tokens[0][4], tokens[0][5])
        now = mx.DateTime.strptime(dstr, "%I:%M %p %b %d %Y")
        if reference.offsets.has_key(self.z):
            now += mx.DateTime.RelativeDateTime(hours=reference.offsets[self.z])
        else:
            print "Unknown TZ: %s " % (self.z,)

            # Now we need to see if we are changing months!
        addOn = 0
        if (iDay < now.day):
            addOn = mx.DateTime.RelativeDateTime(months=+1)
        self.issueTime = now + mx.DateTime.RelativeDateTime(day=iDay,
                                hour=iHour, minute=iMinute, second=0)

class TextProductSegment:

    def __init__(self, raw, issueTime):
        self.issueTime = issueTime
        self.raw = raw
        self.headlines = []
        self.ugc = []
        self.vtec = []
        self.hvtec = []
        self.bullets = []
        self.giswkt = None
        self.ugcExpire = None
        self.windtag = None
        self.hailtag = None
        self.haildirtag = None
        self.winddirtag = None
        self.parse()

    def bullet_splitter(self):
        self.bullets = re.findall("\* ([^\*]*)", self.raw.replace("\n"," ") )

    def get_hvtec_nwsli(self):
        if (len(self.hvtec) == 0):
            return ""
        return self.hvtec[0].nwsli

    def svs_search(self):
        """ Special search the product for special text """
        sections = self.raw.split("\n\n")
        for s in sections:
            if len(TORNADO.findall(s)) > 0:
                return " ..."+ re.sub("\s+", " ", s.replace("\n", " "))
        return ""

    def parse(self):
        u = ugc.ugc( self.raw )
        self.ugc = u.ugc
        if (u.rawexpire is not None and self.issueTime is not None):
          day = int(u.rawexpire[:2])
          hr = int(u.rawexpire[2:4])
          mi = int(u.rawexpire[4:])
          offset = 0
          if (day < self.issueTime.day):
              offset = mx.DateTime.RelativeDateTime(months=+1)
          self.ugcExpire = self.issueTime + offset + mx.DateTime.RelativeDateTime(hour=hr, day=day, minute=mi)

        # Lets look for headlines
        self.headlines = re.findall("^\.\.\.(.*?)\.\.\.[ ]?\n\n", self.raw, re.M | re.S)
        for h in range(len(self.headlines)):
          self.headlines[h] = self.headlines[h].replace("...",", ").replace("\n", " ")

        # Lets look for VTEC!
        tokens = re.findall(vtec._re, self.raw)
        for t in tokens:
          self.vtec.append( vtec.vtec(t[0]) )

        # Lets look for HVTEC!
        tokens = re.findall(hvtec._re, self.raw)
        for t in tokens:
          self.hvtec.append( hvtec.hvtec(t[0]) )

        s = self.raw.replace("\n", " ")
        rend = re.split("LAT\.\.\.LON", s)
        if (len(rend) == 1):
            return None
        # Hack!
        rend2 = re.split("TIME...MOT...LOC", rend[1])
        if (len(rend2) > 1):
            rend[1] = rend2[0]
        pairs = re.findall("([0-9]+)\s+([0-9]+)", rend[1])

        g = "SRID=4326;MULTIPOLYGON((("
        for pr in pairs:
            lat = float(pr[0]) / 100.00
            lon = float(pr[1]) / 100.00
            g += "-%s %s," % (lon, lat)
        g += "-%s %s" % ( float(pairs[0][1]) / 100.00, float(pairs[0][0]) / 100.00)
        g += ")))"
        self.giswkt = g
        # Look for new WIND...HAIL stuff
        if len(rend2) > 1:
            m = WINDHAIL.match( rend2[1] )
            if m:
                d = m.groupdict()
                self.windtag = d['wind']
                self.haildirtag = d['haildir']
                self.winddirtag = d['winddir']
                self.hailtag = d['hail']

    def __str__(self,):
        s = """
headlines: %(headlines)s
ugc: %(ugc)s
giswkt: %(giswkt)s
""" % vars(self)
        i = 0
        for v in self.vtec:
            s += "vtec %s: %s\n" % (i, v)
            i += 1
        i = 0
        for h in self.hvtec:
            s += "hvtec %s: %s\n" % (i, h)
            i += 1
        return s

if (__name__ == "__main__"):
    import sys
    o = open( sys.argv[1], 'r').read()
    t = TextProduct(o)
    print t
<commit_msg>make findWMO more useful<commit_after>
import re, mx.DateTime, string
from support import ugc, vtec, hvtec, reference

#TORNADO = re.compile(r"(STORM\W+CAPABLE\W+OF\W+PRODUCING|REPORTED|INDICATED?)\W+A\W+TORNADO")
TORNADO = re.compile(r"^AT |^\* AT")
WINDHAIL = re.compile(".*WIND\.\.\.HAIL (?P<winddir>[><]?)(?P<wind>[0-9]+)MPH (?P<haildir>[><]?)(?P<hail>[0-9\.]+)IN")
class TextProduct:

    def __init__(self, raw, bypass=False):
        self.idd = None
        self.wmo = None
        self.source = None
        self.issueTime = None
        self.afos = None
        self.segments = []
        self.fcster = None
        self.z = None
        self.product_header = None

        self.raw = raw.strip().replace("\015\015\012", "\n") # to unix
        self.sections = self.raw.split("\n\n")

        if not bypass:
            parseCallbacks = [self.findIssueTime, self.findSegments,
                self.findIDD, self.findWMO, self.findAFOS, self.figureFcster,
                self.find_pheader]
            for cb in parseCallbacks:
                apply( cb )

    def find_pheader(self):
        """ Figure out what the WMO AND MND area and save it for database
            updates where we don't want to save the entire SVS! """
        if (len(self.sections) < 2):
            return

        if len( self.sections[0].split("\n") ) > 3:
            self.product_header = self.sections[0]
        else:
            self.product_header = "\n\n".join(self.sections[0:2])

    def get_iembot_source(self):
        if (self.source is None or len(self.source) != 4):
            return None
        # This is a hack for now, will have to think some more about
        # what to do with the conflict between PABR and KABR
        if (self.source == "PABR"):
            return "XXX"
        return self.source[1:]

    def get_product_id(self):
        s = "%s-%s-%s-%s" % (self.issueTime.strftime("%Y%m%d%H%M"),\
                self.source, self.wmo, self.afos)
        return s.strip()

    def sqlraw(self):
        return re.sub("'", "\\'", self.raw)

    def figureFcster(self):
        self.fcster = string.strip( (self.sections[-1]).replace("\n","") )[:24]

    def findWarnedCounties(self):
        return re.findall("\n  (.*) COUNTY IN (.*)", self.raw)

    def findAFOS(self):
        """ Determine the AFOS header from the NWS text product """
        data = "\n".join([line.strip()
                         for line in self.sections[0].split("\n")])
        tokens = re.findall("^([A-Z0-9 ]{4,6})$", data, re.M)
        if (len(tokens) > 0):
            self.afos = tokens[0]

    def findWMO(self,):
        t = re.findall("^([A-Z]{4}[0-9][0-9]) ([A-Z]{4})", self.sections[0], re.M)
        if len(t) > 0:
            self.wmo = t[0][0]
            self.source = t[0][1]

    def findIDD(self,):
        t = re.findall("^([0-9]+)", self.sections[0], re.M)
        if (len(t) > 0):
            self.idd = int(t[0])

    def findSegments(self,):
        meat = "\n\n".join( self.sections)
        segs = meat.split("$$")
        for s in segs[:-1]:
            self.segments.append(TextProductSegment(s, self.issueTime))

    def __str__(self,):
        s = """
=================================
idd: %(idd)s
wmo: %(wmo)s
source: %(source)s
issueTime: %(issueTime)s
afos: %(afos)s
--------------
""" % vars(self)
        i = 0
        for seg in self.segments:
            s += "+--- segment %s |||| %s" % (i, seg,)
            i += 1
        return s

    def generate_fake_vtec(self):
        """ I generate a fake VTEC ...
            is for non VTEC products prior to 2005, I think :) """
        r = self.raw.replace("NOON", "1200 PM").replace("MIDNIGHT", "1200 AM").replace("VALID UNTIL", "UNTIL")
        dre = "\*\s+UNTIL\s+([0-9]{1,4})\s+(AM|PM)\s+([A-Z]{3,4})"
        tokens = re.findall(dre, r)
        if (len(tokens[0][0]) < 3):
            h = int(tokens[0][0])
            m = 0
        else:
            h = int(tokens[0][0][:-2])
            m = int(tokens[0][0][-2:])
        if tokens[0][1] == "AM" and h == 12:
            h = 0
        if tokens[0][1] == "PM" and h < 12:
            h += 12
        # Figure local issued time
        issue = self.issueTime - mx.DateTime.RelativeDateTime(hours=reference.offsets[self.z])
        expire = issue + mx.DateTime.RelativeDateTime(hour=h, minute=m)
        if (expire < issue):
            expire += mx.DateTime.RelativeDateTime(days=1)
        expireZ = expire + mx.DateTime.RelativeDateTime(hours=reference.offsets[self.z])
        print issue, expire, expireZ, self.z

        if (self.afos[:3] == "SVR"):
            typ = "SV"
        elif (self.afos[:3] == "TOR"):
            typ = "TO"
        elif (self.afos[:3] == "FFW"):
            typ = "FF"

        fake = "/O.NEW.%s.%s.W.0000.%s-%s/" % (self.source, typ, \
               self.issueTime.strftime("%y%m%dT%H%MZ"), \
               expireZ.strftime("%y%m%dT%H%MZ") )
        print 'Fake VTEC', fake
        self.segments[0].vtec.append( vtec.vtec(fake) )

    def findIssueTime(self):
        """ Too much fun, we need to determine when this text product was
            issued.  We'll first look to see if it is explicitly said."""

        # Search out the WMO header first, this had better always be there
        # We only care about the first hit in the file, searching from top
        dRE = "^[A-Z0-9]{6} [A-Z]{4} ([0-3][0-9])([0-2][0-9])([0-5][0-9])"
        tokens = re.findall(dRE , self.raw, re.M)
        iDay = int(tokens[0][0])
        iHour = int(tokens[0][1])
        iMinute = int(tokens[0][2])

        # Now lets look for a local timestamp in the product MND or elsewhere
        time_re = "^([0-9]+) (AM|PM) ([A-Z][A-Z][A-Z]?T) [A-Z][A-Z][A-Z] ([A-Z][A-Z][A-Z]) ([0-9]+) ([1-2][0-9][0-9][0-9])$"
        tokens = re.findall(time_re, self.raw, re.M)
        # If we don't find anything, lets default to now, its the best
        if (len(tokens) == 0):
            hack_time = mx.DateTime.utc().strftime("%I%M %p GMT %a %b %d %Y")
            tokens = re.findall(time_re, hack_time.upper(), re.M)
        # [('1249', 'AM', 'EDT', 'JUL', '1', '2005')]
        self.z = tokens[0][2]
        if (len(tokens[0][0]) < 3):
            h = tokens[0][0]
            m = 0
        else:
            h = tokens[0][0][:-2]
            m = tokens[0][0][-2:]
        dstr = "%s:%s %s %s %s %s" % (h, m, tokens[0][1], tokens[0][3], tokens[0][4], tokens[0][5])
        now = mx.DateTime.strptime(dstr, "%I:%M %p %b %d %Y")
        if reference.offsets.has_key(self.z):
            now += mx.DateTime.RelativeDateTime(hours=reference.offsets[self.z])
        else:
            print "Unknown TZ: %s " % (self.z,)

            # Now we need to see if we are changing months!
        addOn = 0
        if (iDay < now.day):
            addOn = mx.DateTime.RelativeDateTime(months=+1)
        self.issueTime = now + mx.DateTime.RelativeDateTime(day=iDay,
                                hour=iHour, minute=iMinute, second=0)

class TextProductSegment:

    def __init__(self, raw, issueTime):
        self.issueTime = issueTime
        self.raw = raw
        self.headlines = []
        self.ugc = []
        self.vtec = []
        self.hvtec = []
        self.bullets = []
        self.giswkt = None
        self.ugcExpire = None
        self.windtag = None
        self.hailtag = None
        self.haildirtag = None
        self.winddirtag = None
        self.parse()

    def bullet_splitter(self):
        self.bullets = re.findall("\* ([^\*]*)", self.raw.replace("\n"," ") )

    def get_hvtec_nwsli(self):
        if (len(self.hvtec) == 0):
            return ""
        return self.hvtec[0].nwsli

    def svs_search(self):
        """ Special search the product for special text """
        sections = self.raw.split("\n\n")
        for s in sections:
            if len(TORNADO.findall(s)) > 0:
                return " ..."+ re.sub("\s+", " ", s.replace("\n", " "))
        return ""

    def parse(self):
        u = ugc.ugc( self.raw )
        self.ugc = u.ugc
        if (u.rawexpire is not None and self.issueTime is not None):
          day = int(u.rawexpire[:2])
          hr = int(u.rawexpire[2:4])
          mi = int(u.rawexpire[4:])
          offset = 0
          if (day < self.issueTime.day):
              offset = mx.DateTime.RelativeDateTime(months=+1)
          self.ugcExpire = self.issueTime + offset + mx.DateTime.RelativeDateTime(hour=hr, day=day, minute=mi)

        # Lets look for headlines
        self.headlines = re.findall("^\.\.\.(.*?)\.\.\.[ ]?\n\n", self.raw, re.M | re.S)
        for h in range(len(self.headlines)):
          self.headlines[h] = self.headlines[h].replace("...",", ").replace("\n", " ")

        # Lets look for VTEC!
        tokens = re.findall(vtec._re, self.raw)
        for t in tokens:
          self.vtec.append( vtec.vtec(t[0]) )

        # Lets look for HVTEC!
        tokens = re.findall(hvtec._re, self.raw)
        for t in tokens:
          self.hvtec.append( hvtec.hvtec(t[0]) )

        s = self.raw.replace("\n", " ")
        rend = re.split("LAT\.\.\.LON", s)
        if (len(rend) == 1):
            return None
        # Hack!
        rend2 = re.split("TIME...MOT...LOC", rend[1])
        if (len(rend2) > 1):
            rend[1] = rend2[0]
        pairs = re.findall("([0-9]+)\s+([0-9]+)", rend[1])

        g = "SRID=4326;MULTIPOLYGON((("
        for pr in pairs:
            lat = float(pr[0]) / 100.00
            lon = float(pr[1]) / 100.00
            g += "-%s %s," % (lon, lat)
        g += "-%s %s" % ( float(pairs[0][1]) / 100.00, float(pairs[0][0]) / 100.00)
        g += ")))"
        self.giswkt = g
        # Look for new WIND...HAIL stuff
        if len(rend2) > 1:
            m = WINDHAIL.match( rend2[1] )
            if m:
                d = m.groupdict()
                self.windtag = d['wind']
                self.haildirtag = d['haildir']
                self.winddirtag = d['winddir']
                self.hailtag = d['hail']

    def __str__(self,):
        s = """
headlines: %(headlines)s
ugc: %(ugc)s
giswkt: %(giswkt)s
""" % vars(self)
        i = 0
        for v in self.vtec:
            s += "vtec %s: %s\n" % (i, v)
            i += 1
        i = 0
        for h in self.hvtec:
            s += "hvtec %s: %s\n" % (i, h)
            i += 1
        return s

if (__name__ == "__main__"):
    import sys
    o = open( sys.argv[1], 'r').read()
    t = TextProduct(o)
    print t
<|endoftext|>


<commit_before>import os
import re
import tempfile
import zipfile
import csv
import json
from django.template.loader import render_to_string

class UniqueHeaderGenerator(object):
    def __init__(self, max_column_size=None):
        self.used = set()
        self.max_column_size = max_column_size or 2000

    def next_unique(self, header):
        header = self._next_unique(header)
        self.used.add(header)
        return header

    def _next_unique(self, string):
        counter = 1
        if len(string) > self.max_column_size:
            # truncate from the beginning since the end has more specific information
            string = string[-self.max_column_size:]
        orig_string = string
        while string in self.used:
            string = "%s%s" % (orig_string, counter)
            if len(string) > self.max_column_size:
                counterlen = len(str(counter))
                string = "%s%s" % (orig_string[-(self.max_column_size - counterlen):], counter)
            counter += 1

        return string

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        pass

class ExportWriter(object):
    max_table_name_size = 500

    def open(self, header_table, file, max_column_size=2000, table_titles=None):
        """
        Create any initial files, headings, etc necessary.
        """
        table_titles = table_titles or {}

        self._isopen = True
        self.max_column_size = max_column_size
        self._current_primary_id = 0
        self.file = file

        self._init()
        self.table_name_generator = UniqueHeaderGenerator(
            self.max_table_name_size
        )
        for table_index, table in header_table:
            self.add_table(
                table_index,
                table[0],
                table_title=table_titles.get(table_index)
            )

    def add_table(self, table_index, headers, table_title=None):
        def _clean_name(name):
            return re.sub(r"[[\\?*/:\]]", "-", name)
        table_title_truncated = _clean_name(
            self.table_name_generator.next_unique(table_title or table_index)
        )

        # make sure we trim the headers
        with UniqueHeaderGenerator(self.max_column_size) as g:
            try:
                headers.data = [g.next_unique(header) for header in headers.data]
            except AttributeError:
                headers = [g.next_unique(header) for header in headers]

        self._init_table(table_index, table_title_truncated)
        self.write_row(table_index, headers)

    def write(self, document_table, skip_first=False):
        """
        Given a document that's been parsed into the appropriate
        rows, write those rows to the resulting files.
        """
        assert self._isopen
        for table_index, table in document_table:
            for i, row in enumerate(table):
                if skip_first and i is 0:
                    continue
                # update the primary component of the ID to match
                # how many docs we've seen
                try:
                    row_has_id = row.has_id()
                except AttributeError:
                    row_has_id = False
                if row_has_id:
                    row.id = (self._current_primary_id,) + tuple(row.id[1:])

                self.write_row(table_index, row)

        self._current_primary_id += 1

    def write_row(self, table_index, headers):
        """
        Currently just calls the subclass's implementation
        but if we were to add a universal validation step,
        such a thing would happen here.
        """
        return self._write_row(table_index, headers)

    def close(self):
        """
        Close any open file references, do any cleanup.
        """
        assert(self._isopen)
        self._close()
        self._isopen = False

    def _init(self):
        raise NotImplementedError

    def _init_table(self, table_index, table_title):
        raise NotImplementedError

    def _write_row(self, sheet_index, row):
        raise NotImplementedError

    def _close(self):
        raise NotImplementedError

    @classmethod
    def get_data(cls, row):
        """
        Get around the fact that row can be either an iterable or
        a FormattedRow

        """
        try:
            return row.get_data()
        except AttributeError:
            return row

class CsvExportWriter(ExportWriter):

    def _init(self):
        self.tables = {}
        self.table_names = {}
        self.table_files = {}

    def _init_table(self, table_index, table_title):
        fd, path = tempfile.mkstemp()
        file = os.fdopen(fd, 'wb')
        writer = csv.writer(file, dialect=csv.excel)
        self.tables[table_index] = writer
        self.table_files[table_index] = (file, path)
        self.table_names[table_index] = table_title

    def _write_row(self, sheet_index, row):
        def _encode_if_needed(val):
            return val.encode("utf8") if isinstance(val, unicode) else val

        row = map(_encode_if_needed, self.get_data(row))
        writer = self.tables[sheet_index]
        writer.writerow(row)

    def _close(self):
        """
        Close any open file references, do any cleanup.
        """
        archive = zipfile.ZipFile(self.file, 'w', zipfile.ZIP_DEFLATED)
        for index, name in self.table_names.items():
            tmpfile, path = self.table_files[index]
            tmpfile.close()
            archive.write(path, "%s.csv" % name)
        archive.close()
        self.file.seek(0)

class Excel2007ExportWriter(ExportWriter):
    max_table_name_size = 31

    def _init(self):
        try:
            import openpyxl
        except ImportError:
            raise Exception("It doesn't look like this machine is configured for "
                            "excel export. To export to excel you have to run the "
                            "command:  easy_install openpyxl")

        self.book = openpyxl.workbook.Workbook(optimized_write=True)
        self.tables = {}
        self.table_indices = {}

    def _init_table(self, table_index, table_title):
        sheet = self.book.create_sheet()
        sheet.title = table_title
        self.tables[table_index] = sheet
        self.table_indices[table_index] = 0

    def _write_row(self, sheet_index, row):
        sheet = self.tables[sheet_index]
        # Source: http://stackoverflow.com/questions/1707890/fast-way-to-filter-illegal-xml-unicode-chars-in-python
        dirty_chars = re.compile(
            u'[\x00-\x08\x0b-\x1f\x7f-\x84\x86-\x9f\ud800-\udfff\ufdd0-\ufddf\ufffe-\uffff]')
        # NOTE: don't touch this. changing anything like formatting in the
        # row by referencing the cells will cause huge memory issues.
        # see: http://packages.python.org/openpyxl/optimized.html
        sheet.append([dirty_chars.sub(unicode('?'), unicode(v)) for v in self.get_data(row)])

    def _close(self):
        """
        Close any open file references, do any cleanup.
        """
        self.book.save(self.file)

class Excel2003ExportWriter(ExportWriter):
    max_table_name_size = 31

    def _init(self):
        try:
            import xlwt
        except ImportError:
            raise Exception("It doesn't look like this machine is configured for "
                            "excel export. To export to excel you have to run the "
                            "command:  easy_install xlutils")
        self.book = xlwt.Workbook()
        self.tables = {}
        self.table_indices = {}

    def _init_table(self, table_index, table_title):
        sheet = self.book.add_sheet(table_title)
        self.tables[table_index] = sheet
        self.table_indices[table_index] = 0

    def _write_row(self, sheet_index, row):
        row_index = self.table_indices[sheet_index]
        sheet = self.tables[sheet_index]
        # have to deal with primary ids
        for i, val in enumerate(self.get_data(row)):
            sheet.write(row_index,i,unicode(val))
        self.table_indices[sheet_index] = row_index + 1

    def _close(self):
        self.book.save(self.file)

class InMemoryExportWriter(ExportWriter):
    """
    Keeps tables in memory. Subclassed by other export writers.
    """

    def _init(self):
        self.tables = {}
        self.table_names = {}

    def _init_table(self, table_index, table_title):
        self.table_names[table_index] = table_title
        self.tables[table_index] = []

    def _write_row(self, sheet_index, row):
        table = self.tables[sheet_index]
        # have to deal with primary ids
        row_data = [val for val in self.get_data(row)]
        table.append(row_data)

    def _close(self):
        pass

class JsonExportWriter(InMemoryExportWriter):
    """
    Write tables to JSON
    """

    class ConstantEncoder(json.JSONEncoder):

        def default(self, obj):
            from dimagi.utils.web import json_handler
            from couchexport.export import Constant
            if isinstance(obj, Constant):
                return obj.message
            else:
                return json_handler(obj)

    def _close(self):
        new_tables = {}
        for tablename, data in self.tables.items():
            new_tables[self.table_names[tablename]] = {"headers":data[0], "rows": data[1:]}

        self.file.write(json.dumps(new_tables, cls=self.ConstantEncoder))

class HtmlExportWriter(InMemoryExportWriter):
    """
    Write tables to HTML
    """

    def _close(self):
        self.file.write(render_to_string("couchexport/html_export.html", {'tables': self.tables}))

<commit_msg>remove unnecessary import check (is in requirements in setup.py)<commit_after>import os
import re
import tempfile
import zipfile
import csv
import json
from django.template.loader import render_to_string
import xlwt

class UniqueHeaderGenerator(object):
    def __init__(self, max_column_size=None):
        self.used = set()
        self.max_column_size = max_column_size or 2000

    def next_unique(self, header):
        header = self._next_unique(header)
        self.used.add(header)
        return header

    def _next_unique(self, string):
        counter = 1
        if len(string) > self.max_column_size:
            # truncate from the beginning since the end has more specific information
            string = string[-self.max_column_size:]
        orig_string = string
        while string in self.used:
            string = "%s%s" % (orig_string, counter)
            if len(string) > self.max_column_size:
                counterlen = len(str(counter))
                string = "%s%s" % (orig_string[-(self.max_column_size - counterlen):], counter)
            counter += 1

        return string

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        pass

class ExportWriter(object):
    max_table_name_size = 500

    def open(self, header_table, file, max_column_size=2000, table_titles=None):
        """
        Create any initial files, headings, etc necessary.
        """
        table_titles = table_titles or {}

        self._isopen = True
        self.max_column_size = max_column_size
        self._current_primary_id = 0
        self.file = file

        self._init()
        self.table_name_generator = UniqueHeaderGenerator(
            self.max_table_name_size
        )
        for table_index, table in header_table:
            self.add_table(
                table_index,
                table[0],
                table_title=table_titles.get(table_index)
            )

    def add_table(self, table_index, headers, table_title=None):
        def _clean_name(name):
            return re.sub(r"[[\\?*/:\]]", "-", name)
        table_title_truncated = _clean_name(
            self.table_name_generator.next_unique(table_title or table_index)
        )

        # make sure we trim the headers
        with UniqueHeaderGenerator(self.max_column_size) as g:
            try:
                headers.data = [g.next_unique(header) for header in headers.data]
            except AttributeError:
                headers = [g.next_unique(header) for header in headers]

        self._init_table(table_index, table_title_truncated)
        self.write_row(table_index, headers)

    def write(self, document_table, skip_first=False):
        """
        Given a document that's been parsed into the appropriate
        rows, write those rows to the resulting files.
        """
        assert self._isopen
        for table_index, table in document_table:
            for i, row in enumerate(table):
                if skip_first and i is 0:
                    continue
                # update the primary component of the ID to match
                # how many docs we've seen
                try:
                    row_has_id = row.has_id()
                except AttributeError:
                    row_has_id = False
                if row_has_id:
                    row.id = (self._current_primary_id,) + tuple(row.id[1:])

                self.write_row(table_index, row)

        self._current_primary_id += 1

    def write_row(self, table_index, headers):
        """
        Currently just calls the subclass's implementation
        but if we were to add a universal validation step,
        such a thing would happen here.
        """
        return self._write_row(table_index, headers)

    def close(self):
        """
        Close any open file references, do any cleanup.
        """
        assert(self._isopen)
        self._close()
        self._isopen = False

    def _init(self):
        raise NotImplementedError

    def _init_table(self, table_index, table_title):
        raise NotImplementedError

    def _write_row(self, sheet_index, row):
        raise NotImplementedError

    def _close(self):
        raise NotImplementedError

    @classmethod
    def get_data(cls, row):
        """
        Get around the fact that row can be either an iterable or
        a FormattedRow

        """
        try:
            return row.get_data()
        except AttributeError:
            return row

class CsvExportWriter(ExportWriter):

    def _init(self):
        self.tables = {}
        self.table_names = {}
        self.table_files = {}

    def _init_table(self, table_index, table_title):
        fd, path = tempfile.mkstemp()
        file = os.fdopen(fd, 'wb')
        writer = csv.writer(file, dialect=csv.excel)
        self.tables[table_index] = writer
        self.table_files[table_index] = (file, path)
        self.table_names[table_index] = table_title

    def _write_row(self, sheet_index, row):
        def _encode_if_needed(val):
            return val.encode("utf8") if isinstance(val, unicode) else val

        row = map(_encode_if_needed, self.get_data(row))
        writer = self.tables[sheet_index]
        writer.writerow(row)

    def _close(self):
        """
        Close any open file references, do any cleanup.
        """
        archive = zipfile.ZipFile(self.file, 'w', zipfile.ZIP_DEFLATED)
        for index, name in self.table_names.items():
            tmpfile, path = self.table_files[index]
            tmpfile.close()
            archive.write(path, "%s.csv" % name)
        archive.close()
        self.file.seek(0)

class Excel2007ExportWriter(ExportWriter):
    max_table_name_size = 31

    def _init(self):
        try:
            import openpyxl
        except ImportError:
            raise Exception("It doesn't look like this machine is configured for "
                            "excel export. To export to excel you have to run the "
                            "command:  easy_install openpyxl")

        self.book = openpyxl.workbook.Workbook(optimized_write=True)
        self.tables = {}
        self.table_indices = {}

    def _init_table(self, table_index, table_title):
        sheet = self.book.create_sheet()
        sheet.title = table_title
        self.tables[table_index] = sheet
        self.table_indices[table_index] = 0

    def _write_row(self, sheet_index, row):
        sheet = self.tables[sheet_index]
        # Source: http://stackoverflow.com/questions/1707890/fast-way-to-filter-illegal-xml-unicode-chars-in-python
        dirty_chars = re.compile(
            u'[\x00-\x08\x0b-\x1f\x7f-\x84\x86-\x9f\ud800-\udfff\ufdd0-\ufddf\ufffe-\uffff]')
        # NOTE: don't touch this. changing anything like formatting in the
        # row by referencing the cells will cause huge memory issues.
        # see: http://packages.python.org/openpyxl/optimized.html
        sheet.append([dirty_chars.sub(unicode('?'), unicode(v)) for v in self.get_data(row)])

    def _close(self):
        """
        Close any open file references, do any cleanup.
        """
        self.book.save(self.file)

class Excel2003ExportWriter(ExportWriter):
    max_table_name_size = 31

    def _init(self):
        self.book = xlwt.Workbook()
        self.tables = {}
        self.table_indices = {}

    def _init_table(self, table_index, table_title):
        sheet = self.book.add_sheet(table_title)
        self.tables[table_index] = sheet
        self.table_indices[table_index] = 0

    def _write_row(self, sheet_index, row):
        row_index = self.table_indices[sheet_index]
        sheet = self.tables[sheet_index]
        # have to deal with primary ids
        for i, val in enumerate(self.get_data(row)):
            sheet.write(row_index,i,unicode(val))
        self.table_indices[sheet_index] = row_index + 1

    def _close(self):
        self.book.save(self.file)

class InMemoryExportWriter(ExportWriter):
    """
    Keeps tables in memory. Subclassed by other export writers.
    """

    def _init(self):
        self.tables = {}
        self.table_names = {}

    def _init_table(self, table_index, table_title):
        self.table_names[table_index] = table_title
        self.tables[table_index] = []

    def _write_row(self, sheet_index, row):
        table = self.tables[sheet_index]
        # have to deal with primary ids
        row_data = [val for val in self.get_data(row)]
        table.append(row_data)

    def _close(self):
        pass

class JsonExportWriter(InMemoryExportWriter):
    """
    Write tables to JSON
    """

    class ConstantEncoder(json.JSONEncoder):

        def default(self, obj):
            from dimagi.utils.web import json_handler
            from couchexport.export import Constant
            if isinstance(obj, Constant):
                return obj.message
            else:
                return json_handler(obj)

    def _close(self):
        new_tables = {}
        for tablename, data in self.tables.items():
            new_tables[self.table_names[tablename]] = {"headers":data[0], "rows": data[1:]}

        self.file.write(json.dumps(new_tables, cls=self.ConstantEncoder))

class HtmlExportWriter(InMemoryExportWriter):
    """
    Write tables to HTML
    """

    def _close(self):
        self.file.write(render_to_string("couchexport/html_export.html", {'tables': self.tables}))

<|endoftext|>


<commit_before>from svtools.vcf.genotype import Genotype
import sys

class Variant(object):
    '''
    Class for storing information stored in a VCF line
    '''
    def __init__(self, var_list, vcf):
        '''
        Initialize values.

        If fixed_genotypes is True then a string corresponding to the
        genotype portion of the line is cached for printing later.
        '''
        self.chrom = var_list[0]
        self.pos = int(var_list[1])
        self.var_id = var_list[2]
        self.ref = var_list[3]
        self.alt = var_list[4]
        self.qual = var_list[5]
        self.filter = var_list[6]
        self.sample_list = vcf.sample_list
        self.info_list = vcf.info_list
        self.info = dict()
        self.format_list = vcf.format_list
        self.format_set = {i.id for i in vcf.format_list}
        self.gts = None

        # fill in empty sample genotypes
        if len(var_list) < 8:
            sys.stderr.write('\nError: VCF file must have at least 8 columns\n')
            exit(1)

        # make a genotype for each sample at variant
        self.format_string = var_list[8]
        self.format_dict = { key: index for index, key in enumerate(self.format_string.split(':')) }
        self.format_dict.setdefault('GT', len(self.format_dict)) #add GT if it doesn't exist
        self.gts_string = '\t'.join(var_list[9:])

        self.info = dict()
        i_split = [a.split('=') for a in var_list[7].split(';')] # temp list of split info column
        for i in i_split:
            if len(i) == 1:
                i.append(True)
            self.info[i[0]] = i[1]

    def _parse_genotypes(self, genotype_array):
        '''
        Parse the genotype strings
        '''
        gts = dict()
        for index, sample_string in enumerate(genotype_array):
            sample_name = self.sample_list[index]
            sample_field = sample_string.split(':')
            g = Genotype(self, sample_field)
            gts[sample_name] = g
        return gts

    def set_info(self, field, value):
        '''
        Set value of the specified field in the INFO section.
        The INFO field must exist already.
        '''
        if field in [i.id for i in self.info_list]:
            self.info[field] = value
        else:
            sys.stderr.write('\nError: invalid INFO field, \"' + field + '\"\n')
            exit(1)

    def get_info(self, field):
        '''
        Get a value for the given INFO field
        '''
        return self.info[field]

    def get_info_string(self):
        '''
        Construct the INFO string for printing. Order is matched to the header.
        '''
        i_list = list()
        for info_field in self.info_list:
            if info_field.id in self.info.keys():
                if info_field.type == 'Flag':
                    if self.info[info_field.id]:
                        i_list.append(info_field.id)
                else:
                    i_list.append('%s=%s' % (info_field.id, self.info[info_field.id]))
        return ';'.join(i_list)

    def get_format_string(self):
        '''
        Construct the FORMAT field containing the names of the fields in the Genotype columns
        '''
        f_list = list()
        for f in self.format_list:
            if f.id in self.format_dict:
                f_list.append(f.id)
        return ':'.join(f_list)

    def get_gt_string(self, use_cached_gt_string=False):
        '''
        Construct the genotype string.
        '''
        if self.gts:
            if use_cached_gt_string:
                return self.gts_string
            else:
                return '\t'.join(self.genotype(s).get_gt_string() for s in self.sample_list)
        else:
            return self.gts_string

    def _uncache_gts(self):
        '''
        Parse genotypes if they are requested
        '''
        if self.gts is None:
            self.gts = self._parse_genotypes(self.gts_string.split('\t'))
            self.format_string = None

    def genotypes(self):
        '''
        Return a list of all genotype data in the Variant line
        '''
        self._uncache_gts()
        return self.gts.values()

    def genotype(self, sample_name):
        '''
        Return the Genotype object for the requested sample
        '''
        self._uncache_gts()
        try:
            return self.gts[sample_name]
        except KeyError:
            sys.stderr.write('\nError: invalid sample name, \"' + sample_name + '\"\n')
            sys.exit(1)

    def get_var_string(self, use_cached_gt_string=False):
        '''
        Return the String representation for this line
        '''
        fields = [
                self.chrom,
                self.pos,
                self.var_id,
                self.ref,
                self.alt,
                self.qual,
                self.filter,
                self.get_info_string()
                ]

        if self.format_dict:
            gts_string = self.get_gt_string(use_cached_gt_string)
            if gts_string is None:
                sys.stderr.write("Unable to construct or retrieve genotype string\n")
                sys.exit(1)
            else:
                fields += [
                        self.get_format_string(),
                        gts_string
                    ]
        return '\t'.join(map(str, fields))
<commit_msg>make sure to invalidate genotype strings if we didn't have a GT field<commit_after>from svtools.vcf.genotype import Genotype
import sys

class Variant(object):
    '''
    Class for storing information stored in a VCF line
    '''
    def __init__(self, var_list, vcf):
        '''
        Initialize values.

        If fixed_genotypes is True then a string corresponding to the
        genotype portion of the line is cached for printing later.
        '''
        self.chrom = var_list[0]
        self.pos = int(var_list[1])
        self.var_id = var_list[2]
        self.ref = var_list[3]
        self.alt = var_list[4]
        self.qual = var_list[5]
        self.filter = var_list[6]
        self.sample_list = vcf.sample_list
        self.info_list = vcf.info_list
        self.info = dict()
        self.format_list = vcf.format_list
        self.format_set = {i.id for i in vcf.format_list}
        self.gts = None

        # fill in empty sample genotypes
        if len(var_list) < 8:
            sys.stderr.write('\nError: VCF file must have at least 8 columns\n')
            exit(1)

        # make a genotype for each sample at variant
        self.format_string = var_list[8]
        self.format_dict = { key: index for index, key in enumerate(self.format_string.split(':')) }
        self.gts_string = '\t'.join(var_list[9:])

        if 'GT' not in self.format_dict:
            self.format_dict['GT'] = len(self.format_dict) #add GT if it doesn't exist
            self._uncache_gts()

        self.info = dict()
        i_split = [a.split('=') for a in var_list[7].split(';')] # temp list of split info column
        for i in i_split:
            if len(i) == 1:
                i.append(True)
            self.info[i[0]] = i[1]

    def _parse_genotypes(self, genotype_array):
        '''
        Parse the genotype strings
        '''
        gts = dict()
        for index, sample_string in enumerate(genotype_array):
            sample_name = self.sample_list[index]
            sample_field = sample_string.split(':')
            g = Genotype(self, sample_field)
            gts[sample_name] = g
        return gts

    def set_info(self, field, value):
        '''
        Set value of the specified field in the INFO section.
        The INFO field must exist already.
        '''
        if field in [i.id for i in self.info_list]:
            self.info[field] = value
        else:
            sys.stderr.write('\nError: invalid INFO field, \"' + field + '\"\n')
            exit(1)

    def get_info(self, field):
        '''
        Get a value for the given INFO field
        '''
        return self.info[field]

    def get_info_string(self):
        '''
        Construct the INFO string for printing. Order is matched to the header.
        '''
        i_list = list()
        for info_field in self.info_list:
            if info_field.id in self.info.keys():
                if info_field.type == 'Flag':
                    if self.info[info_field.id]:
                        i_list.append(info_field.id)
                else:
                    i_list.append('%s=%s' % (info_field.id, self.info[info_field.id]))
        return ';'.join(i_list)

    def get_format_string(self):
        '''
        Construct the FORMAT field containing the names of the fields in the Genotype columns
        '''
        f_list = list()
        for f in self.format_list:
            if f.id in self.format_dict:
                f_list.append(f.id)
        return ':'.join(f_list)

    def get_gt_string(self, use_cached_gt_string=False):
        '''
        Construct the genotype string.
        '''
        if self.gts:
            if use_cached_gt_string:
                return self.gts_string
            else:
                return '\t'.join(self.genotype(s).get_gt_string() for s in self.sample_list)
        else:
            return self.gts_string

    def _uncache_gts(self):
        '''
        Parse genotypes if they are requested
        '''
        if self.gts is None:
            self.gts = self._parse_genotypes(self.gts_string.split('\t'))
            self.format_string = None

    def genotypes(self):
        '''
        Return a list of all genotype data in the Variant line
        '''
        self._uncache_gts()
        return self.gts.values()

    def genotype(self, sample_name):
        '''
        Return the Genotype object for the requested sample
        '''
        self._uncache_gts()
        try:
            return self.gts[sample_name]
        except KeyError:
            sys.stderr.write('\nError: invalid sample name, \"' + sample_name + '\"\n')
            sys.exit(1)

    def get_var_string(self, use_cached_gt_string=False):
        '''
        Return the String representation for this line
        '''
        fields = [
                self.chrom,
                self.pos,
                self.var_id,
                self.ref,
                self.alt,
                self.qual,
                self.filter,
                self.get_info_string()
                ]

        if self.format_dict:
            gts_string = self.get_gt_string(use_cached_gt_string)
            if gts_string is None:
                sys.stderr.write("Unable to construct or retrieve genotype string\n")
                sys.exit(1)
            else:
                fields += [
                        self.get_format_string(),
                        gts_string
                    ]
        return '\t'.join(map(str, fields))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import sys
import logging
import openerp
import openerp.netsvc as netsvc
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, osv, expression, orm
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from openerp import SUPERUSER_ID, api
from openerp import tools
from openerp.tools.translate import _
from openerp.tools.float_utils import float_round as round
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)

_logger = logging.getLogger(__name__)

class SaleOrder(orm.Model):
    """ Model name: SaleOrder
    """
    _inherit = 'sale.order'

    #def get_write_close_line(self, line):
    #    ''' Internal procedure for close (overrided for mrp line)
    #    '''
    def unlock_closed_order(self, cr, uid, ids, context=None):
        ''' Unlock order
        '''
        # TODO:
        assert len(ids) == 1, 'Works only with one order!'

        self.write(cr, uid, ids, {
            'mx_closed': False,
            }, context=context)

        return True

    # Scheduled events:
    def scheduled_check_close_order(self, cr, uid, context=None):
        ''' Check closed order (completely delivered)
        '''
        logfile = '/home/administrator/photo/log/order/close.log'
        sol_pool = self.pool.get('sale.order.line')

        log = []

        # ---------------------------------------------------------------------
        # Close all pricelist confirmed:
        # ---------------------------------------------------------------------
        log.append('Start setup order as closed')
        _logger.info(log[-1])

        order_ids = self.search(cr, uid, [
            ('state', 'not in', ('cancel', 'sent', 'draft')),
            ('mx_closed', '=', False),
            ('pricelist_order', '=', True),
            ], context=context)
        if order_ids:
            self.write(cr, uid, order_ids, {
                'mx_closed': True,
                #'all_produced': True,
                }, context=context)
        log.append('Update order not closed but pricelist (# %s)' % len(
            order_ids))
        _logger.info(log[-1])

        # ---------------------------------------------------------------------
        # Close order line of order closed:
        # ---------------------------------------------------------------------
        #TODO temp?!?
        line_ids = sol_pool.search(cr, uid, [
            ('order_id.state', 'not in', ('cancel', 'sent', 'draft')),
            ('order_id.mx_closed', '=', True),
            ('mx_closed', '=', False),
            ], context=context)
        if line_ids:
            sol_pool.write(cr, uid, line_ids, {
                'mx_closed': True,
                #'all_produced': True,
                }, context=context)
        log.append('Close lines of closed order: %s' % len(line_ids))
        _logger.info(log[-1])

        # --------------------------
        # TODO check Forecast order:
        # --------------------------

        # ---------------------------------------------------------------------
        # Check line in order open (qty VS delivered):
        # ---------------------------------------------------------------------
        order_ids = self.search(cr, uid, [
            ('state', 'not in', ('cancel', 'sent', 'draft')),
            ('mx_closed', '=', False),
            ], context=context)
        log.append('Check open order: %s' % len(order_ids))
        _logger.info(log[-1])

        close_ids = []
        for order in self.browse(cr, uid, order_ids, context=context):
            to_close = True
            for line in order.order_line:
                if line.product_uom_qty - line.delivered_qty > 0.0:
                    to_close = False
                    break
            if to_close:
               close_ids.append(order.id)

        if close_ids:
            # Close order header:
            self.write(cr, uid, close_ids, {
                'mx_closed': True,
                }, context=context)
            log.append('Closed all delivered order: %s' % len(close_ids))
            _logger.info(log[-1])

            # Close order line:
            line_ids = sol_pool.search(cr, uid, [
                ('order_id', 'in', close_ids),
                ('mx_closed', '=', False),
                ], context=context)
            if line_ids:
                sol_pool.write(cr, uid, line_ids, {
                    'mx_closed': True,
                    }, context=context)
                log.append(
                    'Closed all order line delivered: %s' % len(line_ids))
                _logger.info(log[-1])

        # Log event:
        log_f = open(logfile, 'a')
        log_f.write('\n'.join(log))
        log_f.close()
        return True

    _columns = {
        'mx_closed': fields.boolean('MX closed'),
        }

class SaleOrderLine(orm.Model):
    """ Model name: SaleOrderLine
    """
    _inherit = 'sale.order.line'

    _columns = {
        'mx_closed': fields.boolean('Line closed'),
        }

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>start log procedure<commit_after># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import sys
import logging
import openerp
import openerp.netsvc as netsvc
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, osv, expression, orm
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from openerp import SUPERUSER_ID, api
from openerp import tools
from openerp.tools.translate import _
from openerp.tools.float_utils import float_round as round
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)

_logger = logging.getLogger(__name__)

class SaleOrder(orm.Model):
    """ Model name: SaleOrder
    """
    _inherit = 'sale.order'

    _logfile = '/home/administrator/photo/log/order/close.log'

    #def get_write_close_line(self, line):
    #    ''' Internal procedure for close (overrided for mrp line)
    #    '''
    def unlock_closed_order(self, cr, uid, ids, context=None):
        ''' Unlock order
        '''
        # TODO:
        assert len(ids) == 1, 'Works only with one order!'

        self.write(cr, uid, ids, {
            'mx_closed': False,
            }, context=context)

        return True

    # Scheduled events:
    def scheduled_check_close_order(self, cr, uid, context=None):
        ''' Check closed order (completely delivered)
        '''
        logfile = self._logfile
        sol_pool = self.pool.get('sale.order.line')

        log = []

        # ---------------------------------------------------------------------
        # Close all pricelist confirmed:
        # ---------------------------------------------------------------------
        log.append('Start setup order as closed')
        _logger.info(log[-1])

        order_ids = self.search(cr, uid, [
            ('state', 'not in', ('cancel', 'sent', 'draft')),
            ('mx_closed', '=', False),
            ('pricelist_order', '=', True),
            ], context=context)
        if order_ids:
            self.write(cr, uid, order_ids, {
                'mx_closed': True,
                #'all_produced': True,
                }, context=context)
        log.append('Update order not closed but pricelist (# %s)' % len(
            order_ids))
        _logger.info(log[-1])

        # ---------------------------------------------------------------------
        # Close order line of order closed:
        # ---------------------------------------------------------------------
        #TODO temp?!?
        line_ids = sol_pool.search(cr, uid, [
            ('order_id.state', 'not in', ('cancel', 'sent', 'draft')),
            ('order_id.mx_closed', '=', True),
            ('mx_closed', '=', False),
            ], context=context)
        if line_ids:
            sol_pool.write(cr, uid, line_ids, {
                'mx_closed': True,
                #'all_produced': True,
                }, context=context)
        log.append('Close lines of closed order: %s' % len(line_ids))
        _logger.info(log[-1])

        # --------------------------
        # TODO check Forecast order:
        # --------------------------

        # ---------------------------------------------------------------------
        # Check line in order open (qty VS delivered):
        # ---------------------------------------------------------------------
        order_ids = self.search(cr, uid, [
            ('state', 'not in', ('cancel', 'sent', 'draft')),
            ('mx_closed', '=', False),
            ], context=context)
        log.append('Check open order: %s' % len(order_ids))
        _logger.info(log[-1])

        close_ids = []
        for order in self.browse(cr, uid, order_ids, context=context):
            to_close = True
            for line in order.order_line:
                if line.product_uom_qty - line.delivered_qty > 0.0:
                    to_close = False
                    break
                # TODO order line remain not closed?
            if to_close:
               close_ids.append(order.id)

        if close_ids:
            # Close order header:
            self.write(cr, uid, close_ids, {
                'mx_closed': True,
                }, context=context)
            log.append('Closed all delivered order: %s' % len(close_ids))
            _logger.info(log[-1])

            # Close order line:
            line_ids = sol_pool.search(cr, uid, [
                ('order_id', 'in', close_ids),
                ('mx_closed', '=', False),
                ], context=context)
            if line_ids:
                sol_pool.write(cr, uid, line_ids, {
                    'mx_closed': True,
                    }, context=context)
                log.append(
                    'Closed all order line delivered: %s' % len(line_ids))
                _logger.info(log[-1])

        # Log event:
        log_f = open(logfile, 'a')
        log_f.write('\n'.join(log))
        log_f.close()
        return True

    _columns = {
        'mx_closed': fields.boolean('MX closed'),
        }

class SaleOrderLine(orm.Model):
    """ Model name: SaleOrderLine
    """
    _inherit = 'sale.order.line'

    _columns = {
        'mx_closed': fields.boolean('Line closed'),
        }

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>import datetime
import re

class Timelogs:
    def __init__(self, file):
        self.file = file
        self.linenumber = 0
        self.line = None
        self.data = {}
        self.process()

    def processing_error(self, error_message):
        return Exception('%s at line %d: %s' % (error_message, self.linenumber, self.line))

    def process_date(self):
        tasks = []
        while True:
            line = unicode(self.readline().strip())
            if line.strip() == '':
                break
            else:
                match = re.match('\d+\. (.*)', line)
                if match:
                    tasks.append(match.group(1))
                else:
                    tasks.append(line)
        return tasks

    def readline(self):
        self.line = self.file.readline()
        self.linenumber = self.linenumber + 1
        return self.line

    def process(self):
        self.task = self.readline().strip()
        self.rate = float(self.readline().strip())

        while True:
            line = self.readline()
            if line == '':
                print 'Finished processing file'
                return
            if line.strip() == '':
                continue
            match = re.match('(\w+ \d+,?( \d+)?)(( - )|\s+)(\d+)( .*)?', self.line)
            if match:
                try:
                    date = datetime.datetime.strptime(match.group(1), '%b %d, %Y')
                except ValueError:
                    try:
                        current_year = datetime.date.today().year
                        date = datetime.datetime.strptime("%s, %d" % (match.group(1), current_year), '%b %d, %Y')
                    except ValueError:
                        raise self.processing_error('cannot parse date for header')
                total = float(match.group(5))
                key = datetime.datetime.strftime(date, '%Y-%b-%d')
                if key in self.data:
                    raise self.processing_error('duplicate date')
                try:
                    tasks = self.process_date()
                except UnicodeDecodeError:
                    raise self.processing_error('non-unicode compatible character')
                else:
                    self.data[key] = {'total': total, 'tasks': tasks}
            else:
                raise self.processing_error('malformed header')

    def pretty_print(self):
        total = 0
        keys = sorted(self.data.keys())
        print self.task
        for date in keys:
            log = self.data[date]
            total = total + log['total']
            if log['total'] > 0:
                print '%s: %.2f hours' % (date, log['total'])
                for task in log['tasks']:
                    print task
                print
        print "Total (%d days): %.2f hours" % (len(keys), total)
<commit_msg>Handling of floating point hours.<commit_after>import datetime
import re

class Timelogs:
    def __init__(self, file):
        self.file = file
        self.linenumber = 0
        self.line = None
        self.data = {}
        self.process()

    def processing_error(self, error_message):
        return Exception('%s at line %d: %s' % (error_message, self.linenumber, self.line))

    def process_date(self):
        tasks = []
        while True:
            line = unicode(self.readline().strip())
            if line.strip() == '':
                break
            else:
                match = re.match('\d+\. (.*)', line)
                if match:
                    tasks.append(match.group(1))
                else:
                    tasks.append(line)
        return tasks

    def readline(self):
        self.line = self.file.readline()
        self.linenumber = self.linenumber + 1
        return self.line

    def process(self):
        self.task = self.readline().strip()
        self.rate = float(self.readline().strip())

        while True:
            line = self.readline()
            if line == '':
                print 'Finished processing file'
                return
            if line.strip() == '':
                continue
            match = re.match('(\w+ \d+,?( \d+)?)(( - )|\s+)((\d+)(\.\d+)?)( .*)?', self.line)
            if match:
                try:
                    date = datetime.datetime.strptime(match.group(1), '%b %d, %Y')
                except ValueError:
                    try:
                        current_year = datetime.date.today().year
                        date = datetime.datetime.strptime("%s, %d" % (match.group(1), current_year), '%b %d, %Y')
                    except ValueError:
                        raise self.processing_error('cannot parse date for header')
                total = float(match.group(5))
                key = datetime.datetime.strftime(date, '%Y-%b-%d')
                if key in self.data:
                    raise self.processing_error('duplicate date')
                try:
                    tasks = self.process_date()
                except UnicodeDecodeError:
                    raise self.processing_error('non-unicode compatible character')
                else:
                    self.data[key] = {'total': total, 'tasks': tasks}
            else:
                raise self.processing_error('malformed header')

    def pretty_print(self):
        total = 0
        keys = sorted(self.data.keys())
        print self.task
        for date in keys:
            log = self.data[date]
            total = total + log['total']
            if log['total'] > 0:
                print '%s: %.2f hours' % (date, log['total'])
                for task in log['tasks']:
                    print task
                print
        print "Total (%d days): %.2f hours" % (len(keys), total)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#  Copyright (c) 2001-2013 Python Software Foundation; All Rights Reserved
#  Copyright 2013 Takeshi KOMIYA
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import re
from hgext import color
from mercurial import extensions
from difflib import SequenceMatcher

INSERT_NORM = 'diff.inserted'
INSERT_EMPH = 'diff.inserted_highlight'
DELETE_NORM = 'diff.deleted'
DELETE_EMPH = 'diff.deleted_highlight'

class colorui(color.colorui):
    hunk = None

    def __init__(self, src=None):
        super(colorui, self).__init__(src)
        self.hunk = []

    def write(self, *args, **opts):
        label = opts.get('label')
        if label in (INSERT_NORM, DELETE_NORM):
            self.hunk.append(("".join(args), opts))
        elif label == 'diff.trailingwhitespace':  # merge to hunk
            change = self.hunk.pop()
            self.hunk.append((change[0] + "".join(args), change[1]))
        elif label == '' and args == ("\n",) and self.hunk:
            self.hunk.append((args, opts))
        else:
            self.flush_hunk()
            super(colorui, self).write(*args, **opts)

    def flush(self):
        self.flush_hunk()
        super(colorui, self).flush()

    def flush_hunk(self):
        if self.hunk is None:  # not initialized yet
            return

        new = [c[0] for c in self.hunk if c[1]['label'] == INSERT_NORM]
        old = [c[0] for c in self.hunk if c[1]['label'] == DELETE_NORM]

        self.pprint_hunk(new, 0, len(new), old, 0, len(old))

        self.hunk = []

    def pprint_hunk(self, new, new_lo, new_hi, old, old_lo, old_hi):
        # derived from difflib.py (Python stdlib) Differ#_fancy_replace()
        best_ratio, cutoff = 0.59, 0.60
        write = super(colorui, self).write

        cruncher = SequenceMatcher(None)
        for j in range(old_lo, old_hi):
            cruncher.set_seq2(old[j])
            for i in range(new_lo, new_hi):
                cruncher.set_seq1(new[i])
                if (cruncher.real_quick_ratio() > best_ratio and
                   cruncher.quick_ratio() > best_ratio and
                   cruncher.ratio() > best_ratio):
                    best_ratio, best_i, best_j = cruncher.ratio(), i, j

        # no non-identical "pretty close" pair
        if best_ratio < cutoff:
            for line in old[old_lo:old_hi]:
                write(line, "\n", label=DELETE_NORM)
            for line in new[new_lo:new_hi]:
                write(line, "\n", label=INSERT_NORM)

            return

        self.pprint_hunk_helper(new, new_lo, best_i,
                                old, old_lo, best_j)
        self.pprint_pair(cruncher, new[best_i], old[best_j])
        self.pprint_hunk_helper(new, best_i + 1, new_hi,
                                old, best_j + 1, old_hi)

    def pprint_hunk_helper(self, new, new_lo, new_hi, old, old_lo, old_hi):
        # derived from difflib.py (Python stdlib) Differ#_fancy_helper()
        write = super(colorui, self).write

        if new_lo < new_hi:
            if old_lo < old_hi:
                self.pprint_hunk(new, new_lo, new_hi, old, old_lo, old_hi)
            else:
                for line in new[new_lo:new_hi]:
                    write(line, "\n", label=INSERT_NORM)
        elif old_lo < old_hi:
            for line in old[old_lo:old_hi]:
                write(line, "\n", label=DELETE_NORM)

    def pprint_pair(self, cruncher, newline, oldline):
        write = super(colorui, self).write

        new = [[newline[0].decode('utf-8'), INSERT_NORM]]
        old = [[oldline[0].decode('utf-8'), DELETE_NORM]]
        cruncher.set_seqs(newline[1:].decode('utf-8'),
                          oldline[1:].decode('utf-8'))
        for tag, new1, new2, old1, old2 in cruncher.get_opcodes():
            new_piece = newline[new1 + 1:new2 + 1]
            old_piece = oldline[old1 + 1:old2 + 1]
            if tag == 'equal':
                new.append([new_piece, INSERT_NORM])
                old.append([old_piece, DELETE_NORM])
            else:
                new.append([new_piece, INSERT_EMPH])
                old.append([old_piece, DELETE_EMPH])

        # change highlighting: character base -> word base
        for i in range(len(new) - 1, 1, -1):
            if is_mergeable(new, old, i):
                new[i - 2][0] += new[i - 1][0]
                old[i - 2][0] += old[i - 1][0]
                del new[i - 1]
                del old[i - 1]

        # optimize ESC chars
        for i in range(len(new) - 1, 0, -1):
            if new[i][1] == new[i - 1][1]:
                new[i - 1][0] += new[i][0]
                del new[i]

        for i in range(len(old) - 1, 0, -1):
            if old[i][1] == old[i - 1][1]:
                old[i - 1][0] += old[i][0]
                del old[i]

        # write highlighted lines
        for string, label in old + [['\n', '']] + new + [['\n', '']]:
            write(string.encode('utf-8'), label=label)

def is_mergeable(new, old, i):
    chars = '[a-zA-Z0-9_.]'
    startswith_word = lambda s: re.match('^%s' % chars, s)
    endswith_word = lambda s: re.search('%s$' % chars, s)
    is_word = lambda s: re.match('^(%s|\s)+$' % chars, s)

    marks = '[ !"#$%&\'()*+,\-./:;<=>?@\[\\]^_{|}~]'
    startswith_mark = lambda s: re.match('^%s' % marks, s)
    endswith_mark = lambda s: re.search('%s$' % marks, s)
    is_mark = lambda s: re.match('^(%s|\s)+$' % marks, s)

    n1, n2, n3 = new[i - 2: i + 1]
    o1, o2, o3 = old[i - 2: i + 1]
    if not (n1[1] == n3[1] == INSERT_EMPH and n2[1] == INSERT_NORM):
        return False

    # WORD1 ends with word(alnum) and WORD2 is word
    if ((endswith_word(n1[0]) and is_word(n2[0])) or
       (endswith_word(o1[0]) and is_word(o2[0]))):
        return True

    # WORD2 is word and WORD3 starts with word(alnum)
    if ((is_word(n2[0]) and startswith_word(n3[0])) or
       (is_word(o2[0]) and startswith_word(o3[0]))):
        return True

    # WORD1 ends with any marks and WORD2 is any marks
    if ((endswith_mark(n1[0]) and is_mark(n2[0])) or
       (endswith_mark(o1[0]) and is_mark(o2[0]))):
        return True

    # WORD2 is any marks and WORD3 starts with any marks
    if ((is_mark(n1[0]) and startswith_mark(n2[0])) or
       (is_mark(o1[0]) and startswith_mark(o2[0]))):
        return True

    return False

def uisetup(ui):
    if ui.plain():
        return
    if not isinstance(ui, colorui):
        colorui.__bases__ = (ui.__class__,)
        ui.__class__ = colorui

    def colorconfig(orig, *args, **kwargs):
        ret = orig(*args, **kwargs)

        styles = color._styles
        if INSERT_EMPH not in styles:
            styles[INSERT_EMPH] = styles[INSERT_NORM] + ' inverse'

        if DELETE_EMPH not in styles:
            styles[DELETE_EMPH] = styles[DELETE_NORM] + ' inverse'

        return ret

    extensions.wrapfunction(color, 'configstyles', colorconfig)
<commit_msg>Change order of copyrights<commit_after># -*- coding: utf-8 -*-
#  Copyright 2013 Takeshi KOMIYA
#  Copyright (c) 2001-2013 Python Software Foundation; All Rights Reserved
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import re
from hgext import color
from mercurial import extensions
from difflib import SequenceMatcher

INSERT_NORM = 'diff.inserted'
INSERT_EMPH = 'diff.inserted_highlight'
DELETE_NORM = 'diff.deleted'
DELETE_EMPH = 'diff.deleted_highlight'

class colorui(color.colorui):
    hunk = None

    def __init__(self, src=None):
        super(colorui, self).__init__(src)
        self.hunk = []

    def write(self, *args, **opts):
        label = opts.get('label')
        if label in (INSERT_NORM, DELETE_NORM):
            self.hunk.append(("".join(args), opts))
        elif label == 'diff.trailingwhitespace':  # merge to hunk
            change = self.hunk.pop()
            self.hunk.append((change[0] + "".join(args), change[1]))
        elif label == '' and args == ("\n",) and self.hunk:
            self.hunk.append((args, opts))
        else:
            self.flush_hunk()
            super(colorui, self).write(*args, **opts)

    def flush(self):
        self.flush_hunk()
        super(colorui, self).flush()

    def flush_hunk(self):
        if self.hunk is None:  # not initialized yet
            return

        new = [c[0] for c in self.hunk if c[1]['label'] == INSERT_NORM]
        old = [c[0] for c in self.hunk if c[1]['label'] == DELETE_NORM]

        self.pprint_hunk(new, 0, len(new), old, 0, len(old))

        self.hunk = []

    def pprint_hunk(self, new, new_lo, new_hi, old, old_lo, old_hi):
        # derived from difflib.py (Python stdlib) Differ#_fancy_replace()
        best_ratio, cutoff = 0.59, 0.60
        write = super(colorui, self).write

        cruncher = SequenceMatcher(None)
        for j in range(old_lo, old_hi):
            cruncher.set_seq2(old[j])
            for i in range(new_lo, new_hi):
                cruncher.set_seq1(new[i])
                if (cruncher.real_quick_ratio() > best_ratio and
                   cruncher.quick_ratio() > best_ratio and
                   cruncher.ratio() > best_ratio):
                    best_ratio, best_i, best_j = cruncher.ratio(), i, j

        # no non-identical "pretty close" pair
        if best_ratio < cutoff:
            for line in old[old_lo:old_hi]:
                write(line, "\n", label=DELETE_NORM)
            for line in new[new_lo:new_hi]:
                write(line, "\n", label=INSERT_NORM)

            return

        self.pprint_hunk_helper(new, new_lo, best_i,
                                old, old_lo, best_j)
        self.pprint_pair(cruncher, new[best_i], old[best_j])
        self.pprint_hunk_helper(new, best_i + 1, new_hi,
                                old, best_j + 1, old_hi)

    def pprint_hunk_helper(self, new, new_lo, new_hi, old, old_lo, old_hi):
        # derived from difflib.py (Python stdlib) Differ#_fancy_helper()
        write = super(colorui, self).write

        if new_lo < new_hi:
            if old_lo < old_hi:
                self.pprint_hunk(new, new_lo, new_hi, old, old_lo, old_hi)
            else:
                for line in new[new_lo:new_hi]:
                    write(line, "\n", label=INSERT_NORM)
        elif old_lo < old_hi:
            for line in old[old_lo:old_hi]:
                write(line, "\n", label=DELETE_NORM)

    def pprint_pair(self, cruncher, newline, oldline):
        write = super(colorui, self).write

        new = [[newline[0].decode('utf-8'), INSERT_NORM]]
        old = [[oldline[0].decode('utf-8'), DELETE_NORM]]
        cruncher.set_seqs(newline[1:].decode('utf-8'),
                          oldline[1:].decode('utf-8'))
        for tag, new1, new2, old1, old2 in cruncher.get_opcodes():
            new_piece = newline[new1 + 1:new2 + 1]
            old_piece = oldline[old1 + 1:old2 + 1]
            if tag == 'equal':
                new.append([new_piece, INSERT_NORM])
                old.append([old_piece, DELETE_NORM])
            else:
                new.append([new_piece, INSERT_EMPH])
                old.append([old_piece, DELETE_EMPH])

        # change highlighting: character base -> word base
        for i in range(len(new) - 1, 1, -1):
            if is_mergeable(new, old, i):
                new[i - 2][0] += new[i - 1][0]
                old[i - 2][0] += old[i - 1][0]
                del new[i - 1]
                del old[i - 1]

        # optimize ESC chars
        for i in range(len(new) - 1, 0, -1):
            if new[i][1] == new[i - 1][1]:
                new[i - 1][0] += new[i][0]
                del new[i]

        for i in range(len(old) - 1, 0, -1):
            if old[i][1] == old[i - 1][1]:
                old[i - 1][0] += old[i][0]
                del old[i]

        # write highlighted lines
        for string, label in old + [['\n', '']] + new + [['\n', '']]:
            write(string.encode('utf-8'), label=label)

def is_mergeable(new, old, i):
    chars = '[a-zA-Z0-9_.]'
    startswith_word = lambda s: re.match('^%s' % chars, s)
    endswith_word = lambda s: re.search('%s$' % chars, s)
    is_word = lambda s: re.match('^(%s|\s)+$' % chars, s)

    marks = '[ !"#$%&\'()*+,\-./:;<=>?@\[\\]^_{|}~]'
    startswith_mark = lambda s: re.match('^%s' % marks, s)
    endswith_mark = lambda s: re.search('%s$' % marks, s)
    is_mark = lambda s: re.match('^(%s|\s)+$' % marks, s)

    n1, n2, n3 = new[i - 2: i + 1]
    o1, o2, o3 = old[i - 2: i + 1]
    if not (n1[1] == n3[1] == INSERT_EMPH and n2[1] == INSERT_NORM):
        return False

    # WORD1 ends with word(alnum) and WORD2 is word
    if ((endswith_word(n1[0]) and is_word(n2[0])) or
       (endswith_word(o1[0]) and is_word(o2[0]))):
        return True

    # WORD2 is word and WORD3 starts with word(alnum)
    if ((is_word(n2[0]) and startswith_word(n3[0])) or
       (is_word(o2[0]) and startswith_word(o3[0]))):
        return True

    # WORD1 ends with any marks and WORD2 is any marks
    if ((endswith_mark(n1[0]) and is_mark(n2[0])) or
       (endswith_mark(o1[0]) and is_mark(o2[0]))):
        return True

    # WORD2 is any marks and WORD3 starts with any marks
    if ((is_mark(n1[0]) and startswith_mark(n2[0])) or
       (is_mark(o1[0]) and startswith_mark(o2[0]))):
        return True

    return False

def uisetup(ui):
    if ui.plain():
        return
    if not isinstance(ui, colorui):
        colorui.__bases__ = (ui.__class__,)
        ui.__class__ = colorui

    def colorconfig(orig, *args, **kwargs):
        ret = orig(*args, **kwargs)

        styles = color._styles
        if INSERT_EMPH not in styles:
            styles[INSERT_EMPH] = styles[INSERT_NORM] + ' inverse'

        if DELETE_EMPH not in styles:
            styles[DELETE_EMPH] = styles[DELETE_NORM] + ' inverse'

        return ret

    extensions.wrapfunction(color, 'configstyles', colorconfig)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: UTF-8 -*-

from datetime import date, datetime, timedelta
from icalendar import Calendar, Event, vRecur
import json
import os.path
from os.path import dirname
from uuid import uuid4

class iCalSchedule(object):

    DAYS = ['MO', 'TU', 'WE', 'TH', 'FR', 'SA', 'SU']

    def __init__(self, scraper=None, startdate=None, enddate=None):
        if startdate is None or enddate is None:
            dts, dte = self._get_dates(scraper.period, scraper.year,
                    scraper.semester)

            if startdate is None:
                startdate = dts
            if enddate is None:
                enddate = dte

        self.startdate = startdate
        self.enddate = enddate
        self._first_weekdays = {} # cache
        self._init_ical()
        if scraper is not None:
            for ev in scraper.get_events():
                self.add_event(ev)

    def _init_ical(self):
        cal = Calendar()
        cal.add('version', '2.0')
        cal.add('prodid', '-//edt2ics//bfontaine.net//')
        cal.add('method', 'publish')
        self.cal = cal

    def _recur_params(self, wday):
        return {
            'freq': 'weekly',
            'wkst': self.DAYS[0],
            'byday': self.DAYS[wday],
            'until': self.enddate,
        }

    def _get_first_weekday(self, day):
        """
        Return the first date after ``self.startdate`` which is on the given
        weekday (0=Monday, 1=Tuesday, etc)
        """
        if day not in self._first_weekdays:
            start_wd = self.startdate.weekday()
            delta = (day - start_wd + 7) % 7
            self._first_weekdays[day] = self.startdate + timedelta(days=delta)
        return self._first_weekdays[day]

    def _get_dates(self, period, year, semester):
        source = os.path.join(dirname(__file__), 'dates.json')
        with open(source, 'r') as f:
            data = json.loads(f.read())
        dates = data['dates'][period][str(semester)][year]
        start, end = dates['start'], dates['end']

        return self._str2date(start), self._str2date(end)

    def _str2date(self, s):
        return date(*map(lambda e: int(e, 10), s.split('-')))

    def add_event(self, ev):
        """
        Add a new recurrent event to this schedule
        """
        day = self._get_first_weekday(ev.day)
        dtstart = datetime.combine(day, ev.tstart)
        dtend = datetime.combine(day, ev.tend)

        iev = Event()
        iev.add('uid', str(uuid4()))
        iev.add('status', 'confirmed')
        iev.add('dtstart', dtstart)
        iev.add('dtend', dtend)
        iev.add('rrule', vRecur(self._recur_params(ev.day)))
        iev.add('summary', '%s %s' % (ev.type_, ev.title))
        iev.add('location', ev.room)
        iev.add('description', ev.description)
        self.cal.add_component(iev)

    def to_ical(self):
        return self.cal.to_ical()
<commit_msg>iCalSchedule.enddate delayed one day later<commit_after>#! /usr/bin/env python
# -*- coding: UTF-8 -*-

from datetime import date, datetime, timedelta
from icalendar import Calendar, Event, vRecur
import json
import os.path
from os.path import dirname
from uuid import uuid4

class iCalSchedule(object):

    DAYS = ['MO', 'TU', 'WE', 'TH', 'FR', 'SA', 'SU']

    def __init__(self, scraper=None, startdate=None, enddate=None):
        if startdate is None or enddate is None:
            dts, dte = self._get_dates(scraper.period, scraper.year,
                    scraper.semester)

            if startdate is None:
                startdate = dts
            if enddate is None:
                enddate = dte

        self.startdate = startdate
        # The 'DTEND' property is exclusive, we then must end one day later
        self.enddate = enddate + timedelta(days=1)
        self._first_weekdays = {} # cache
        self._init_ical()
        if scraper is not None:
            for ev in scraper.get_events():
                self.add_event(ev)

    def _init_ical(self):
        cal = Calendar()
        cal.add('version', '2.0')
        cal.add('prodid', '-//edt2ics//bfontaine.net//')
        cal.add('method', 'publish')
        self.cal = cal

    def _recur_params(self, wday):
        return {
            'freq': 'weekly',
            'wkst': self.DAYS[0],
            'byday': self.DAYS[wday],
            'until': self.enddate,
        }

    def _get_first_weekday(self, day):
        """
        Return the first date after ``self.startdate`` which is on the given
        weekday (0=Monday, 1=Tuesday, etc)
        """
        if day not in self._first_weekdays:
            start_wd = self.startdate.weekday()
            delta = (day - start_wd + 7) % 7
            self._first_weekdays[day] = self.startdate + timedelta(days=delta)
        return self._first_weekdays[day]

    def _get_dates(self, period, year, semester):
        source = os.path.join(dirname(__file__), 'dates.json')
        with open(source, 'r') as f:
            data = json.loads(f.read())
        dates = data['dates'][period][str(semester)][year]
        start, end = dates['start'], dates['end']

        return self._str2date(start), self._str2date(end)

    def _str2date(self, s):
        return date(*map(lambda e: int(e, 10), s.split('-')))

    def add_event(self, ev):
        """
        Add a new recurrent event to this schedule
        """
        day = self._get_first_weekday(ev.day)
        dtstart = datetime.combine(day, ev.tstart)
        dtend = datetime.combine(day, ev.tend)

        iev = Event()
        iev.add('uid', str(uuid4()))
        iev.add('status', 'confirmed')
        iev.add('dtstart', dtstart)
        iev.add('dtend', dtend)
        iev.add('rrule', vRecur(self._recur_params(ev.day)))
        iev.add('summary', '%s %s' % (ev.type_, ev.title))
        iev.add('location', ev.room)
        iev.add('description', ev.description)
        self.cal.add_component(iev)

    def to_ical(self):
        return self.cal.to_ical()
<|endoftext|>