<commit_before>#!/usr/bin/env python2.5
import sys, re, errno, stat, tempfile, struct, mmap
import options
from helpers import *

INDEX_SIG = '!IIIIIQ20sH'
ENTLEN = struct.calcsize(INDEX_SIG)

IX_EXISTS = 0x8000
IX_HASHVALID = 0x4000

class OsFile:
    def __init__(self, path):
        self.fd = None
        self.fd = os.open(path, os.O_RDONLY|os.O_LARGEFILE|os.O_NOFOLLOW)
        #self.st = os.fstat(self.fd)

    def __del__(self):
        if self.fd:
            fd = self.fd
            self.fd = None
            os.close(fd)

    def fchdir(self):
        os.fchdir(self.fd)

class IxEntry:
    def __init__(self, name, m, ofs):
        self._m = m
        self._ofs = ofs
        self.name = str(name)
        (self.dev, self.ctime, self.mtime, self.uid, self.gid,
         self.size, self.sha,
         self.flags) = struct.unpack(INDEX_SIG, buffer(m, ofs, ENTLEN))

    def __repr__(self):
        return ("(%s,0x%04x,%d,%d,%d,%d,%d,0x%04x)"
                % (self.name, self.dev,
                   self.ctime, self.mtime, self.uid, self.gid,
                   self.size, self.flags))

    def pack(self):
        return struct.pack(INDEX_SIG, self.dev, self.ctime, self.mtime,
                           self.uid, self.gid, self.size, self.sha,
                           self.flags)

    def repack(self):
        self._m[self._ofs:self._ofs+ENTLEN] = self.pack()

    def from_stat(self, st):
        old = (self.dev, self.ctime, self.mtime,
               self.uid, self.gid, self.size)
        new = (st.st_dev, int(st.st_ctime), int(st.st_mtime),
               st.st_uid, st.st_gid, st.st_size)
        self.dev = st.st_dev
        self.ctime = int(st.st_ctime)
        self.mtime = int(st.st_mtime)
        self.uid = st.st_uid
        self.gid = st.st_gid
        self.size = st.st_size
        self.flags |= IX_EXISTS
        if old != new:
            self.flags &= ~IX_HASHVALID
            return 1  # dirty
        else:
            return 0  # not dirty

class IndexReader:
    def __init__(self, filename):
        self.filename = filename
        self.m = ''
        self.writable = False
        f = None
        try:
            f = open(filename, 'r+')
        except IOError, e:
            if e.errno == errno.ENOENT:
                pass
            else:
                raise
        if f:
            st = os.fstat(f.fileno())
        if f and st.st_size:
            self.m = mmap.mmap(f.fileno(), 0,
                               mmap.MAP_SHARED, mmap.PROT_READ|mmap.PROT_WRITE)
            f.close()  # map will persist beyond file close
            self.writable = True

    def __iter__(self):
        ofs = 0
        while ofs < len(self.m):
            eon = self.m.find('\0', ofs)
            assert(eon >= 0)
            yield IxEntry(buffer(self.m, ofs, eon-ofs),
                          self.m, eon+1)
            ofs = eon + 1 + ENTLEN

    def save(self):
        if self.writable:
            self.m.flush()

def ix_encode(st, sha, flags):
    return struct.pack(INDEX_SIG, st.st_dev, int(st.st_ctime),
                       int(st.st_mtime), st.st_uid, st.st_gid,
                       st.st_size, sha, flags)

class IndexWriter:
    def __init__(self, filename):
        self.f = None
        self.lastfile = None
        self.filename = None
        self.filename = filename = os.path.realpath(filename)
        (dir,name) = os.path.split(filename)
        (ffd,self.tmpname) = tempfile.mkstemp('.tmp', filename, dir)
        self.f = os.fdopen(ffd, 'wb', 65536)

    def __del__(self):
        self.abort()

    def abort(self):
        f = self.f
        self.f = None
        if f:
            f.close()
            os.unlink(self.tmpname)

    def close(self):
        f = self.f
        self.f = None
        if f:
            f.close()
            os.rename(self.tmpname, self.filename)

    def add(self, name, st):
        #log('ADDING %r\n' % name)
        if self.lastfile:
            assert(cmp(self.lastfile, name) > 0) # reverse order only
        self.lastfile = name
        data = name + '\0' + ix_encode(st, '\0'*20, IX_EXISTS|IX_HASHVALID)
        self.f.write(data)

    def add_ixentry(self, e):
        if self.lastfile:
            assert(cmp(self.lastfile, e.name) > 0) # reverse order only
        self.lastfile = e.name
        data = e.name + '\0' + e.pack()
        self.f.write(data)

    def new_reader(self):
        self.f.flush()
        return IndexReader(self.tmpname)

saved_errors = []
def add_error(e):
    saved_errors.append(e)
    log('\n%s\n' % e)

# the use of fchdir() and lstat() are for two reasons:
#  - help out the kernel by not making it repeatedly look up the absolute path
#  - avoid race conditions caused by doing listdir() on a changing symlink
def handle_path(ri, wi, dir, name, pst):
    dirty = 0
    path = dir + name
    #log('handle_path(%r,%r)\n' % (dir, name))
    if stat.S_ISDIR(pst.st_mode):
        if opt.verbose == 1: # log dirs only
            sys.stdout.write('%s\n' % path)
            sys.stdout.flush()
        try:
            OsFile(name).fchdir()
        except OSError, e:
            add_error(Exception('in %s: %s' % (dir, str(e))))
            return 0
        try:
            try:
                ld = os.listdir('.')
                #log('* %r: %r\n' % (name, ld))
            except OSError, e:
                add_error(Exception('in %s: %s' % (path, str(e))))
                return 0
            lds = []
            for p in ld:
                try:
                    st = os.lstat(p)
                except OSError, e:
                    add_error(Exception('in %s: %s' % (path, str(e))))
                    continue
                if stat.S_ISDIR(st.st_mode):
                    p += '/'
                lds.append((p, st))
            for p,st in reversed(sorted(lds)):
                dirty += handle_path(ri, wi, path, p, st)
        finally:
            os.chdir('..')
    #log('endloop: ri.cur:%r path:%r\n' % (ri.cur.name, path))
    while ri.cur and ri.cur.name > path:
        #log('ricur:%r path:%r\n' % (ri.cur, path))
        if dir and ri.cur.name.startswith(dir):
            #log('    --- deleting\n')
            ri.cur.flags &= ~(IX_EXISTS | IX_HASHVALID)
            ri.cur.repack()
            dirty += 1
        ri.next()
    if ri.cur and ri.cur.name == path:
        dirty += ri.cur.from_stat(pst)
        if dirty:
            #log('   --- updating %r\n' % path)
            ri.cur.repack()
        ri.next()
    else:
        wi.add(path, pst)
        dirty += 1
    if opt.verbose > 1:  # all files, not just dirs
        sys.stdout.write('%s\n' % path)
        sys.stdout.flush()
    return dirty

def _next(i):
    try:
        return i.next()
    except StopIteration:
        return None

def merge_indexes(out, r1, r2):
    log('Merging indexes.\n')
    i1 = iter(r1)
    i2 = iter(r2)

    e1 = _next(i1)
    e2 = _next(i2)
    while e1 or e2:
        if e1 and (not e2 or e2.name < e1.name):
            if e1.flags & IX_EXISTS:
                out.add_ixentry(e1)
            e1 = _next(i1)
        elif e2 and (not e1 or e1.name < e2.name):
            if e2.flags & IX_EXISTS:
                out.add_ixentry(e2)
            e2 = _next(i2)
        elif e1.name == e2.name:
            assert(0)  # duplicate name? should never happen anymore.
            if e2.flags & IX_EXISTS:
                out.add_ixentry(e2)
            e1 = _next(i1)
            e2 = _next(i2)

class MergeGetter:
    def __init__(self, l):
        self.i = iter(l)
        self.cur = None
        self.next()

    def next(self):
        try:
            self.cur = self.i.next()
        except StopIteration:
            self.cur = None
        return self.cur

def update_index(path):
    ri = IndexReader('index')
    wi = IndexWriter('index')
    rpath = os.path.realpath(path)
    st = os.lstat(rpath)
    f = OsFile('.')
    if rpath[-1] == '/':
        rpath = rpath[:-1]
    (dir, name) = os.path.split(rpath)
    if dir and dir[-1] != '/':
        dir += '/'
    if stat.S_ISDIR(st.st_mode) and (not rpath or rpath[-1] != '/'):
        name += '/'
    rig = MergeGetter(ri)
    OsFile(dir or '/').fchdir()
    dirty = handle_path(rig, wi, dir, name, st)

    # make sure all the parents of the updated path exist and are invalidated
    # if appropriate.
    while 1:
        (rpath, junk) = os.path.split(rpath)
        if not rpath:
            break
        elif rpath == '/':
            p = rpath
        else:
            p = rpath + '/'
        while rig.cur and rig.cur.name > p:
            #log('FINISHING: %r path=%r d=%r\n' % (rig.cur.name, p, dirty))
            rig.next()
        if rig.cur and rig.cur.name == p:
            if dirty:
                rig.cur.flags &= ~IX_HASHVALID
                rig.cur.repack()
        else:
            wi.add(p, os.lstat(p))
        if p == '/':
            break

    f.fchdir()
    ri.save()
    mi = IndexWriter('index')
    merge_indexes(mi, ri, wi.new_reader())
    wi.abort()
    mi.close()

optspec = """
bup index [-vp] <filenames...>
--
p,print    print index after updating
v,verbose  increase log output (can be used more than once)
"""
o = options.Options('bup index', optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])

for path in extra:
    update_index(path)

if opt['print']:
    for ent in IndexReader('index'):
        if not ent.flags & IX_EXISTS:
            print 'D ' + ent.name
        elif not ent.flags & IX_HASHVALID:
            print 'M ' + ent.name
        else:
            print '  ' + ent.name
        #print repr(ent)

if saved_errors:
    log('WARNING: %d errors encountered.\n' % len(saved_errors))
    exit(1)
<commit_msg>index: add --xdev (--one-file-system) option.<commit_after>#!/usr/bin/env python2.5
import sys, re, errno, stat, tempfile, struct, mmap
import options
from helpers import *

INDEX_SIG = '!IIIIIQ20sH'
ENTLEN = struct.calcsize(INDEX_SIG)

IX_EXISTS = 0x8000
IX_HASHVALID = 0x4000

class OsFile:
    def __init__(self, path):
        self.fd = None
        self.fd = os.open(path, os.O_RDONLY|os.O_LARGEFILE|os.O_NOFOLLOW)
        #self.st = os.fstat(self.fd)

    def __del__(self):
        if self.fd:
            fd = self.fd
            self.fd = None
            os.close(fd)

    def fchdir(self):
        os.fchdir(self.fd)

class IxEntry:
    def __init__(self, name, m, ofs):
        self._m = m
        self._ofs = ofs
        self.name = str(name)
        (self.dev, self.ctime, self.mtime, self.uid, self.gid,
         self.size, self.sha,
         self.flags) = struct.unpack(INDEX_SIG, buffer(m, ofs, ENTLEN))

    def __repr__(self):
        return ("(%s,0x%04x,%d,%d,%d,%d,%d,0x%04x)"
                % (self.name, self.dev,
                   self.ctime, self.mtime, self.uid, self.gid,
                   self.size, self.flags))

    def pack(self):
        return struct.pack(INDEX_SIG, self.dev, self.ctime, self.mtime,
                           self.uid, self.gid, self.size, self.sha,
                           self.flags)

    def repack(self):
        self._m[self._ofs:self._ofs+ENTLEN] = self.pack()

    def from_stat(self, st):
        old = (self.dev, self.ctime, self.mtime,
               self.uid, self.gid, self.size)
        new = (st.st_dev, int(st.st_ctime), int(st.st_mtime),
               st.st_uid, st.st_gid, st.st_size)
        self.dev = st.st_dev
        self.ctime = int(st.st_ctime)
        self.mtime = int(st.st_mtime)
        self.uid = st.st_uid
        self.gid = st.st_gid
        self.size = st.st_size
        self.flags |= IX_EXISTS
        if old != new:
            self.flags &= ~IX_HASHVALID
            return 1  # dirty
        else:
            return 0  # not dirty

class IndexReader:
    def __init__(self, filename):
        self.filename = filename
        self.m = ''
        self.writable = False
        f = None
        try:
            f = open(filename, 'r+')
        except IOError, e:
            if e.errno == errno.ENOENT:
                pass
            else:
                raise
        if f:
            st = os.fstat(f.fileno())
        if f and st.st_size:
            self.m = mmap.mmap(f.fileno(), 0,
                               mmap.MAP_SHARED, mmap.PROT_READ|mmap.PROT_WRITE)
            f.close()  # map will persist beyond file close
            self.writable = True

    def __iter__(self):
        ofs = 0
        while ofs < len(self.m):
            eon = self.m.find('\0', ofs)
            assert(eon >= 0)
            yield IxEntry(buffer(self.m, ofs, eon-ofs),
                          self.m, eon+1)
            ofs = eon + 1 + ENTLEN

    def save(self):
        if self.writable:
            self.m.flush()

def ix_encode(st, sha, flags):
    return struct.pack(INDEX_SIG, st.st_dev, int(st.st_ctime),
                       int(st.st_mtime), st.st_uid, st.st_gid,
                       st.st_size, sha, flags)

class IndexWriter:
    def __init__(self, filename):
        self.f = None
        self.lastfile = None
        self.filename = None
        self.filename = filename = os.path.realpath(filename)
        (dir,name) = os.path.split(filename)
        (ffd,self.tmpname) = tempfile.mkstemp('.tmp', filename, dir)
        self.f = os.fdopen(ffd, 'wb', 65536)

    def __del__(self):
        self.abort()

    def abort(self):
        f = self.f
        self.f = None
        if f:
            f.close()
            os.unlink(self.tmpname)

    def close(self):
        f = self.f
        self.f = None
        if f:
            f.close()
            os.rename(self.tmpname, self.filename)

    def add(self, name, st):
        #log('ADDING %r\n' % name)
        if self.lastfile:
            assert(cmp(self.lastfile, name) > 0) # reverse order only
        self.lastfile = name
        data = name + '\0' + ix_encode(st, '\0'*20, IX_EXISTS|IX_HASHVALID)
        self.f.write(data)

    def add_ixentry(self, e):
        if self.lastfile:
            assert(cmp(self.lastfile, e.name) > 0) # reverse order only
        self.lastfile = e.name
        data = e.name + '\0' + e.pack()
        self.f.write(data)

    def new_reader(self):
        self.f.flush()
        return IndexReader(self.tmpname)

saved_errors = []
def add_error(e):
    saved_errors.append(e)
    log('\n%s\n' % e)

# the use of fchdir() and lstat() are for two reasons:
#  - help out the kernel by not making it repeatedly look up the absolute path
#  - avoid race conditions caused by doing listdir() on a changing symlink
def handle_path(ri, wi, dir, name, pst, xdev):
    dirty = 0
    path = dir + name
    #log('handle_path(%r,%r)\n' % (dir, name))
    if stat.S_ISDIR(pst.st_mode):
        if opt.verbose == 1: # log dirs only
            sys.stdout.write('%s\n' % path)
            sys.stdout.flush()
        try:
            OsFile(name).fchdir()
        except OSError, e:
            add_error(Exception('in %s: %s' % (dir, str(e))))
            return 0
        try:
            try:
                ld = os.listdir('.')
                #log('* %r: %r\n' % (name, ld))
            except OSError, e:
                add_error(Exception('in %s: %s' % (path, str(e))))
                return 0
            lds = []
            for p in ld:
                try:
                    st = os.lstat(p)
                except OSError, e:
                    add_error(Exception('in %s: %s' % (path, str(e))))
                    continue
                if xdev != None and st.st_dev != xdev:
                    log('Skipping %r: different filesystem.\n'
                        % os.path.realpath(p))
                    continue
                if stat.S_ISDIR(st.st_mode):
                    p += '/'
                lds.append((p, st))
            for p,st in reversed(sorted(lds)):
                dirty += handle_path(ri, wi, path, p, st, xdev)
        finally:
            os.chdir('..')
    #log('endloop: ri.cur:%r path:%r\n' % (ri.cur.name, path))
    while ri.cur and ri.cur.name > path:
        #log('ricur:%r path:%r\n' % (ri.cur, path))
        if dir and ri.cur.name.startswith(dir):
            #log('    --- deleting\n')
            ri.cur.flags &= ~(IX_EXISTS | IX_HASHVALID)
            ri.cur.repack()
            dirty += 1
        ri.next()
    if ri.cur and ri.cur.name == path:
        dirty += ri.cur.from_stat(pst)
        if dirty:
            #log('   --- updating %r\n' % path)
            ri.cur.repack()
        ri.next()
    else:
        wi.add(path, pst)
        dirty += 1
    if opt.verbose > 1:  # all files, not just dirs
        sys.stdout.write('%s\n' % path)
        sys.stdout.flush()
    return dirty

def _next(i):
    try:
        return i.next()
    except StopIteration:
        return None

def merge_indexes(out, r1, r2):
    log('Merging indexes.\n')
    i1 = iter(r1)
    i2 = iter(r2)

    e1 = _next(i1)
    e2 = _next(i2)
    while e1 or e2:
        if e1 and (not e2 or e2.name < e1.name):
            if e1.flags & IX_EXISTS:
                out.add_ixentry(e1)
            e1 = _next(i1)
        elif e2 and (not e1 or e1.name < e2.name):
            if e2.flags & IX_EXISTS:
                out.add_ixentry(e2)
            e2 = _next(i2)
        elif e1.name == e2.name:
            assert(0)  # duplicate name? should never happen anymore.
            if e2.flags & IX_EXISTS:
                out.add_ixentry(e2)
            e1 = _next(i1)
            e2 = _next(i2)

class MergeGetter:
    def __init__(self, l):
        self.i = iter(l)
        self.cur = None
        self.next()

    def next(self):
        try:
            self.cur = self.i.next()
        except StopIteration:
            self.cur = None
        return self.cur

def update_index(path):
    ri = IndexReader('index')
    wi = IndexWriter('index')
    rpath = os.path.realpath(path)
    st = os.lstat(rpath)
    if opt.xdev:
        xdev = st.st_dev
    else:
        xdev = None
    f = OsFile('.')
    if rpath[-1] == '/':
        rpath = rpath[:-1]
    (dir, name) = os.path.split(rpath)
    if dir and dir[-1] != '/':
        dir += '/'
    if stat.S_ISDIR(st.st_mode) and (not rpath or rpath[-1] != '/'):
        name += '/'
    rig = MergeGetter(ri)
    OsFile(dir or '/').fchdir()
    dirty = handle_path(rig, wi, dir, name, st, xdev)

    # make sure all the parents of the updated path exist and are invalidated
    # if appropriate.
    while 1:
        (rpath, junk) = os.path.split(rpath)
        if not rpath:
            break
        elif rpath == '/':
            p = rpath
        else:
            p = rpath + '/'
        while rig.cur and rig.cur.name > p:
            #log('FINISHING: %r path=%r d=%r\n' % (rig.cur.name, p, dirty))
            rig.next()
        if rig.cur and rig.cur.name == p:
            if dirty:
                rig.cur.flags &= ~IX_HASHVALID
                rig.cur.repack()
        else:
            wi.add(p, os.lstat(p))
        if p == '/':
            break

    f.fchdir()
    ri.save()
    mi = IndexWriter('index')
    merge_indexes(mi, ri, wi.new_reader())
    wi.abort()
    mi.close()

optspec = """
bup index [-vp] <filenames...>
--
p,print    print index after updating
x,xdev,one-file-system  don't cross filesystem boundaries
v,verbose  increase log output (can be used more than once)
"""
o = options.Options('bup index', optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])

for path in extra:
    update_index(path)

if opt['print']:
    for ent in IndexReader('index'):
        if not ent.flags & IX_EXISTS:
            print 'D ' + ent.name
        elif not ent.flags & IX_HASHVALID:
            print 'M ' + ent.name
        else:
            print '  ' + ent.name
        #print repr(ent)

if saved_errors:
    log('WARNING: %d errors encountered.\n' % len(saved_errors))
    exit(1)
<|endoftext|>


<commit_before>import clodius.tiles.nplabels as ctn
import h5py

def test_tiles():
    filename = 'data/labels.h5'
    f = h5py.File(filename, 'r')

    array = f['labels']

    tiles = ctn.tiles(array, 0, 0, None)

    assert(tiles[0]['x'] == 0)
    assert(tiles[-1]['x'] == 86992)

    tiles = ctn.tiles(array, 2, 1, None)

    assert(tiles[0]['x'] == 32768)
    assert(tiles[-1]['x'] == 65535)<commit_msg>Added extra line<commit_after>import clodius.tiles.nplabels as ctn
import h5py

def test_tiles():
    filename = 'data/labels.h5'
    f = h5py.File(filename, 'r')

    array = f['labels']

    tiles = ctn.tiles(array, 0, 0, None)

    assert(tiles[0]['x'] == 0)
    assert(tiles[-1]['x'] == 86992)

    tiles = ctn.tiles(array, 2, 1, None)

    assert(tiles[0]['x'] == 32768)
    assert(tiles[-1]['x'] == 65535)
<|endoftext|>


<commit_before>from pprint import pprint

from django.contrib.auth.mixins import LoginRequiredMixin
from django.db import IntegrityError, transaction
from django.db.utils import Error

from base.views import O2BaseGetPostView
from o2.functions import csrf_used

import servico.forms
import servico.models

class CriaOrdem(LoginRequiredMixin, O2BaseGetPostView):

    def __init__(self, *args, **kwargs):
        super(CriaOrdem, self).__init__(*args, **kwargs)
        self.cleaned_data2self = True
        self.Form_class = servico.forms.CriaServicoEventoForm
        self.template_name = 'servico/cria_ordem.html'
        self.title_name = 'Cria ordem'

    def salva_evento(self):
        if csrf_used(self.request):
            self.context.update({
                'erro': 'Formulário já gravado.',
            })
            raise Error

        try:
            tipo = servico.models.TipoDocumento.objects.get(slug='os')
        except servico.models.TipoDocumento.DoesNotExist as e:
            self.context.update({
                'erro': 'Tipo de documento inválido.',
            })
            raise e

        try:
            evento = servico.models.Evento.objects.get(criar=True)
        except servico.models.Evento.DoesNotExist as e:
            self.context.update({
                'erro': 'Evento de criação não encontrado.',
            })
            raise e

        try:
            self.doc = servico.models.NumeroDocumento(tipo=tipo)
            self.doc.save()
        except Exception as e:
            self.context.update({
                'erro': 'Não foi possível gerar um número de documento.',
            })
            raise e

        try:
            evento = servico.models.ServicoEvento(
                numero=self.doc,
                evento=evento,
                nivel=self.nivel,
                equipe=self.equipe,
                descricao=self.descricao,
            )
            evento.save()
        except Exception as e:
            self.context.update({
                'erro': 'Não foi possível gerar o evento de requisição.',
            })
            raise e

        self.doc.ativo = True
        self.doc.save()

    def mount_context(self):
        try:
            with transaction.atomic():
                self.salva_evento()
        except Exception:
            return
        numero = self.doc.id
        self.redirect = ('servico:ordem__get', numero)
<commit_msg>Model Evento codigo no lugar de ação<commit_after>from pprint import pprint

from django.contrib.auth.mixins import LoginRequiredMixin
from django.db import IntegrityError, transaction
from django.db.utils import Error

from base.views import O2BaseGetPostView
from o2.functions import csrf_used

import servico.forms
import servico.models

class CriaOrdem(LoginRequiredMixin, O2BaseGetPostView):

    def __init__(self, *args, **kwargs):
        super(CriaOrdem, self).__init__(*args, **kwargs)
        self.cleaned_data2self = True
        self.Form_class = servico.forms.CriaServicoEventoForm
        self.template_name = 'servico/cria_ordem.html'
        self.title_name = 'Cria ordem'

    def salva_evento(self):
        if csrf_used(self.request):
            self.context.update({
                'erro': 'Formulário já gravado.',
            })
            raise Error

        try:
            tipo = servico.models.TipoDocumento.objects.get(slug='os')
        except servico.models.TipoDocumento.DoesNotExist as e:
            self.context.update({
                'erro': 'Tipo de documento inválido.',
            })
            raise e

        try:
            evento = servico.models.Evento.objects.get(codigo='req')
        except servico.models.Evento.DoesNotExist as e:
            self.context.update({
                'erro': 'Evento de criação não encontrado.',
            })
            raise e

        try:
            self.doc = servico.models.NumeroDocumento(tipo=tipo)
            self.doc.save()
        except Exception as e:
            self.context.update({
                'erro': 'Não foi possível gerar um número de documento.',
            })
            raise e

        try:
            evento = servico.models.ServicoEvento(
                numero=self.doc,
                evento=evento,
                nivel=self.nivel,
                equipe=self.equipe,
                descricao=self.descricao,
            )
            evento.save()
        except Exception as e:
            self.context.update({
                'erro': 'Não foi possível gerar o evento de requisição.',
            })
            raise e

        self.doc.ativo = True
        self.doc.save()

    def mount_context(self):
        try:
            with transaction.atomic():
                self.salva_evento()
        except Exception:
            return
        numero = self.doc.id
        self.redirect = ('servico:ordem__get', numero)
<|endoftext|>


<commit_before>import unittest
from autoprotocol.protocol import Protocol, Ref
from autoprotocol.instruction import Instruction, Thermocycle, Incubate, Pipette, Spin
from autoprotocol.container_type import ContainerType
from autoprotocol.container import Container, WellGroup, Well
from autoprotocol.unit import Unit
import json

class ProtocolMultipleExistTestCase(unittest.TestCase):
    def runTest(self):
        p1 = Protocol()
        p2 = Protocol()

        p1.spin("dummy_ref", "2000:rpm", "560:second")
        self.assertEqual(len(p2.instructions), 0,
            "incorrect number of instructions in empty protocol")

class ProtocolBasicTestCase(unittest.TestCase):
    def runTest(self):
        protocol = Protocol()
        resource = protocol.ref("resource", None, "96-flat", discard=True)
        pcr = protocol.ref("pcr", None, "96-flat", discard=True)
        bacteria = protocol.ref("bacteria", None, "96-flat", discard=True)
        self.assertEqual(len(protocol.as_dict()['refs']), 3, 'incorrect number of refs')
        self.assertEqual(protocol.as_dict()['refs']['resource'], {"new": "96-flat",
                        "discard": True})

        bacteria_wells = WellGroup([bacteria.well("B1"), bacteria.well("C5"),
                                    bacteria.well("A5"), bacteria.well("A1")])

        protocol.distribute(resource.well("A1").set_volume("40:microliter"),
                            pcr.wells_from('A1',5), "5:microliter")
        protocol.distribute(resource.well("A1").set_volume("40:microliter"),
                            bacteria_wells, "5:microliter")

        self.assertEqual(len(protocol.instructions), 1)
        self.assertEqual(protocol.instructions[0].op, "pipette")
        self.assertEqual(len(protocol.instructions[0].groups), 2)

        protocol.incubate(bacteria, "warm_37", "30:minute")

        self.assertEqual(len(protocol.instructions), 2)
        self.assertEqual(protocol.instructions[1].op, "incubate")
        self.assertEqual(protocol.instructions[1].duration, "30:minute")

class ProtocolAppendTestCase(unittest.TestCase):
    def runTest(self):
        p = Protocol()
        self.assertEqual(len(p.instructions), 0,
            "should not be any instructions before appending to empty protocol")

        p.append(Spin("dummy_ref", "100:meter/second^2", "60:second"))
        self.assertEqual(len(p.instructions), 1,
            "incorrect number of instructions after single instruction append")
        self.assertEqual(p.instructions[0].op, "spin",
            "incorrect instruction appended")

        p.append([
                    Incubate("dummy_ref", "ambient", "30:second"),
                    Spin("dummy_ref", "2000:rpm", "120:second")
                ])
        self.assertEqual(len(p.instructions), 3,
            "incorrect number of instructions after appending instruction list")
        self.assertEqual(p.instructions[1].op, "incubate",
            "incorrect instruction order after list append")
        self.assertEqual(p.instructions[2].op, "spin",
            "incorrect instruction at end after list append.")

class RefTestCase(unittest.TestCase):
    def test_duplicates_not_allowed(self):
        p = Protocol()
        p.ref("test", None, "96-flat", discard=True)
        with self.assertRaises(RuntimeError):
            p.ref("test", None, "96-flat", storage="cold_20")
        self.assertTrue(p.refs["test"].opts["discard"])
        self.assertFalse("where" in p.refs["test"].opts)

class ThermocycleTestCase(unittest.TestCase):
    def test_thermocycle_append(self):
        t = Thermocycle("plate", [
            { "cycles": 1, "steps": [
                { "temperature": "95:celsius", "duration": "60:second" },
            ] },
            { "cycles": 30, "steps": [
                { "temperature": "95:celsius", "duration": "15:second" },
                { "temperature": "55:celsius", "duration": "15:second" },
                { "temperature": "72:celsius", "duration": "10:second" },
            ] },
            { "cycles": 1, "steps": [
                { "temperature": "72:celsius", "duration": "600:second" },
                { "temperature": "12:celsius", "duration": "120:second" },
            ] },
        ], "20:microliter")
        self.assertEqual(len(t.groups), 3, 'incorrect number of groups')
        self.assertEqual(t.volume, "20:microliter")

    def test_thermocycle_dyes_and_datarefs(self):
        self.assertRaises(ValueError,
                    Thermocycle,
                    "plate",
                    [{"cycles": 1,
                      "steps": [{
                          "temperature": "50: celsius",
                          "duration": "20:minute"
                      }]
                      }],
                    dyes={"FAM": ["A1"]})
        self.assertRaises(ValueError,
                    Thermocycle,
                    "plate",
                    [{"cycles": 1,
                      "steps": [{
                          "temperature": "50: celsius",
                          "duration": "20:minute"
                      }]
                      }],
                    dataref="test_dataref")
        self.assertRaises(ValueError,
                    Thermocycle,
                    "plate",
                    [{"cycles": 1,
                      "steps": [{
                          "temperature": "50: celsius",
                          "duration": "20:minute"
                      }]
                      }],
                    dyes={"ThisDyeIsInvalid": ["A1"]})

    def test_thermocycle_melting(self):
        self.assertRaises(ValueError,
                    Thermocycle,
                    "plate",
                    [{"cycles": 1,
                      "steps": [{
                          "temperature": "50: celsius",
                          "duration": "20:minute"
                      }]
                      }],
                    melting_start = "50:celsius")
        self.assertRaises(ValueError,
                    Thermocycle,
                    "plate",
                    [{"cycles": 1,
                      "steps": [{
                          "temperature": "50: celsius",
                          "duration": "20:minute"
                      }]
                      }],
                    melting_start = "50:celsius",
                    melting_end = "60:celsius",
                    melting_increment = "1:celsius",
                    melting_rate = "2:minute")

class DistributeTestCase(unittest.TestCase):
    def test_distribute_one_well(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        p.distribute(c.well(0).set_volume("20:microliter"),
                     c.well(1),
                     "5:microliter")
        self.assertEqual(1, len(p.instructions))
        self.assertEqual("distribute",
                         list(p.as_dict()["instructions"][0]["groups"][0].keys())[0])
        self.assertTrue(5, c.well(1).volume.value)
        self.assertTrue(15, c.well(0).volume.value)

    def test_distribute_multiple_wells(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        p.distribute(c.well(0).set_volume("20:microliter"),
                     c.wells_from(1, 3),
                     "5:microliter")
        self.assertEqual(1, len(p.instructions))
        self.assertEqual("distribute",
                         list(p.as_dict()["instructions"][0]["groups"][0].keys())[0])
        for w in c.wells_from(1, 3):
            self.assertTrue(5, w.volume.value)
        self.assertTrue(5, c.well(0).volume.value)

    def test_fill_wells(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        srcs = c.wells_from(1, 2).set_volume("100:microliter")
        dests = c.wells_from(7, 4)
        p.distribute(srcs, dests, "30:microliter", allow_carryover=True)
        self.assertEqual(2, len(p.instructions[0].groups))

        # track source vols
        self.assertEqual(10, c.well(1).volume.value)
        self.assertEqual(70, c.well(2).volume.value)

        # track dest vols
        self.assertEqual(30, c.well(7).volume.value)
        self.assertIs(None, c.well(6).volume)

        # test distribute from Well to Well
        p.distribute(c.well("A1").set_volume("20:microliter"), c.well("A2"), "5:microliter")
        self.assertTrue("distribute" in p.instructions[-1].groups[-1])

class TransferTestCase(unittest.TestCase):
    def test_single_transfer(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        p.transfer(c.well(0), c.well(1), "20:microliter")
        self.assertEqual(Unit(20, "microliter"), c.well(1).volume)
        self.assertEqual(None, c.well(0).volume)
        self.assertTrue("transfer" in p.instructions[-1].groups[-1])

    def test_max_transfer(self):
        p = Protocol()
        c = p.ref("test", None, "micro-2.0", storage="cold_4")
        p.transfer(c.well(0), c.well(0), "3050:microliter")

    def test_multiple_transfers(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        p.transfer(c.wells_from(0, 2), c.wells_from(2, 2), "20:microliter")
        self.assertEqual(c.well(2).volume, c.well(3).volume)
        self.assertEqual(2, len(p.instructions[0].groups))

    def test_one_tip(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        p.transfer(c.wells_from(0, 2), c.wells_from(2, 2), "20:microliter",
            one_tip=True)
        self.assertEqual(c.well(2).volume, c.well(3).volume)
        self.assertEqual(1, len(p.instructions[0].groups))

    def test_one_source(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        with self.assertRaises(RuntimeError):
            p.transfer(c.wells_from(0, 2),
                   c.wells_from(2, 2), "40:microliter", one_source=True)
        with self.assertRaises(RuntimeError):
            p.transfer(c.wells_from(0, 2).set_volume("1:microliter"),
                       c.wells_from(1, 5), "10:microliter", one_source=True)
        p.transfer(c.wells_from(0, 2).set_volume("50:microliter"),
                   c.wells_from(2, 2), "40:microliter", one_source=True)
        self.assertEqual(2, len(p.instructions[0].groups))
        self.assertFalse(p.instructions[0].groups[0]["transfer"][0]["from"] == p.instructions[0].groups[1]["transfer"][0]["from"])

class ConsolidateTestCase(unittest.TestCase):
    def test_multiple_sources(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        with self.assertRaises(TypeError):
            p.consolidate(c.wells_from(0, 3), c.wells_from(2, 3), "10:microliter")
        with self.assertRaises(ValueError):
            p.consolidate(c.wells_from(0, 3), c.well(4), ["10:microliter"])
        p.consolidate(c.wells_from(0, 3), c.well(4), "10:microliter")
        self.assertEqual(Unit(30, "microliter"), c.well(4).volume)
        self.assertEqual(3, len(p.instructions[0].groups[0]["consolidate"]["from"]))

    def test_one_source(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        p.consolidate(c.well(0), c.well(4), "30:microliter")
        self.assertEqual(Unit(30, "microliter"), c.well(4).volume)

class StampTestCase(unittest.TestCase):
    def test_single_transfers(self):
        p = Protocol()
        plate_96_list = []
        for plate_num in range(5):
            plate_name = ("test_96_"+str(plate_num))
            plate_96_list.append(p.ref(plate_name, None, "96-flat", discard=True))
        plate_384_list = []
        for plate_num in range(3):
            plate_name = ("test_384_"+str(plate_num))
            plate_384_list.append(p.ref(plate_name, None, "384-flat", discard=True))
        with self.assertRaises(RuntimeError):
            # Transfer 4 plates
            for pl, q in zip(plate_96_list, [0, 1, 24, 26]):
                p.stamp(pl, plate_384_list[0], "10:microliter", to_quad = q)

class RefifyTestCase(unittest.TestCase):
    def test_refifying_various(self):
        p = Protocol()
        # refify container
        refs = {"plate": p.ref("test", None, "96-flat", "cold_20")}
        self.assertEqual(p._refify(refs["plate"]), "test")
        # refify dict
        self.assertEqual(p._refify(refs), {"plate": "test"})

        # refify Well
        well = refs["plate"].well("A1")
        self.assertEqual(p._refify(well), "test/0")

        # refify WellGroup
        wellgroup = refs["plate"].wells_from("A2", 3)
        self.assertEqual(p._refify(wellgroup), ["test/1", "test/2", "test/3"])

        # refify other
        s = "randomstring"
        i = 24
        self.assertEqual("randomstring", p._refify(s))
        self.assertEqual(24, p._refify(i))

class OutsTestCase(unittest.TestCase):
    def test_outs(self):
        p = Protocol()
        self.assertEqual(p.as_dict()['outs'], {})
        plate = p.ref("plate", None, "96-pcr", discard=True)
        plate.well(0).set_name("test_well")
        self.assertTrue(plate.well(0).name == "test_well")
        self.assertTrue(list(p.as_dict()['outs'].keys()) == ['plate'])
        self.assertTrue(list(list(p.as_dict()['outs'].values())[0].keys()) == ['0'])
        self.assertTrue(list(p.as_dict()['outs'].values())[0]['0'] == {'name': 'test_well'})
<commit_msg>test: Add test for transfers greater than 750 microliters<commit_after>import unittest
from autoprotocol.protocol import Protocol, Ref
from autoprotocol.instruction import Instruction, Thermocycle, Incubate, Pipette, Spin
from autoprotocol.container_type import ContainerType
from autoprotocol.container import Container, WellGroup, Well
from autoprotocol.unit import Unit
from autoprotocol.pipette_tools import *
import json

class ProtocolMultipleExistTestCase(unittest.TestCase):
    def runTest(self):
        p1 = Protocol()
        p2 = Protocol()

        p1.spin("dummy_ref", "2000:rpm", "560:second")
        self.assertEqual(len(p2.instructions), 0,
            "incorrect number of instructions in empty protocol")

class ProtocolBasicTestCase(unittest.TestCase):
    def runTest(self):
        protocol = Protocol()
        resource = protocol.ref("resource", None, "96-flat", discard=True)
        pcr = protocol.ref("pcr", None, "96-flat", discard=True)
        bacteria = protocol.ref("bacteria", None, "96-flat", discard=True)
        self.assertEqual(len(protocol.as_dict()['refs']), 3, 'incorrect number of refs')
        self.assertEqual(protocol.as_dict()['refs']['resource'], {"new": "96-flat",
                        "discard": True})

        bacteria_wells = WellGroup([bacteria.well("B1"), bacteria.well("C5"),
                                    bacteria.well("A5"), bacteria.well("A1")])

        protocol.distribute(resource.well("A1").set_volume("40:microliter"),
                            pcr.wells_from('A1',5), "5:microliter")
        protocol.distribute(resource.well("A1").set_volume("40:microliter"),
                            bacteria_wells, "5:microliter")

        self.assertEqual(len(protocol.instructions), 1)
        self.assertEqual(protocol.instructions[0].op, "pipette")
        self.assertEqual(len(protocol.instructions[0].groups), 2)

        protocol.incubate(bacteria, "warm_37", "30:minute")

        self.assertEqual(len(protocol.instructions), 2)
        self.assertEqual(protocol.instructions[1].op, "incubate")
        self.assertEqual(protocol.instructions[1].duration, "30:minute")

class ProtocolAppendTestCase(unittest.TestCase):
    def runTest(self):
        p = Protocol()
        self.assertEqual(len(p.instructions), 0,
            "should not be any instructions before appending to empty protocol")

        p.append(Spin("dummy_ref", "100:meter/second^2", "60:second"))
        self.assertEqual(len(p.instructions), 1,
            "incorrect number of instructions after single instruction append")
        self.assertEqual(p.instructions[0].op, "spin",
            "incorrect instruction appended")

        p.append([
                    Incubate("dummy_ref", "ambient", "30:second"),
                    Spin("dummy_ref", "2000:rpm", "120:second")
                ])
        self.assertEqual(len(p.instructions), 3,
            "incorrect number of instructions after appending instruction list")
        self.assertEqual(p.instructions[1].op, "incubate",
            "incorrect instruction order after list append")
        self.assertEqual(p.instructions[2].op, "spin",
            "incorrect instruction at end after list append.")

class RefTestCase(unittest.TestCase):
    def test_duplicates_not_allowed(self):
        p = Protocol()
        p.ref("test", None, "96-flat", discard=True)
        with self.assertRaises(RuntimeError):
            p.ref("test", None, "96-flat", storage="cold_20")
        self.assertTrue(p.refs["test"].opts["discard"])
        self.assertFalse("where" in p.refs["test"].opts)

class ThermocycleTestCase(unittest.TestCase):
    def test_thermocycle_append(self):
        t = Thermocycle("plate", [
            { "cycles": 1, "steps": [
                { "temperature": "95:celsius", "duration": "60:second" },
            ] },
            { "cycles": 30, "steps": [
                { "temperature": "95:celsius", "duration": "15:second" },
                { "temperature": "55:celsius", "duration": "15:second" },
                { "temperature": "72:celsius", "duration": "10:second" },
            ] },
            { "cycles": 1, "steps": [
                { "temperature": "72:celsius", "duration": "600:second" },
                { "temperature": "12:celsius", "duration": "120:second" },
            ] },
        ], "20:microliter")
        self.assertEqual(len(t.groups), 3, 'incorrect number of groups')
        self.assertEqual(t.volume, "20:microliter")

    def test_thermocycle_dyes_and_datarefs(self):
        self.assertRaises(ValueError,
                    Thermocycle,
                    "plate",
                    [{"cycles": 1,
                      "steps": [{
                          "temperature": "50: celsius",
                          "duration": "20:minute"
                      }]
                      }],
                    dyes={"FAM": ["A1"]})
        self.assertRaises(ValueError,
                    Thermocycle,
                    "plate",
                    [{"cycles": 1,
                      "steps": [{
                          "temperature": "50: celsius",
                          "duration": "20:minute"
                      }]
                      }],
                    dataref="test_dataref")
        self.assertRaises(ValueError,
                    Thermocycle,
                    "plate",
                    [{"cycles": 1,
                      "steps": [{
                          "temperature": "50: celsius",
                          "duration": "20:minute"
                      }]
                      }],
                    dyes={"ThisDyeIsInvalid": ["A1"]})

    def test_thermocycle_melting(self):
        self.assertRaises(ValueError,
                    Thermocycle,
                    "plate",
                    [{"cycles": 1,
                      "steps": [{
                          "temperature": "50: celsius",
                          "duration": "20:minute"
                      }]
                      }],
                    melting_start = "50:celsius")
        self.assertRaises(ValueError,
                    Thermocycle,
                    "plate",
                    [{"cycles": 1,
                      "steps": [{
                          "temperature": "50: celsius",
                          "duration": "20:minute"
                      }]
                      }],
                    melting_start = "50:celsius",
                    melting_end = "60:celsius",
                    melting_increment = "1:celsius",
                    melting_rate = "2:minute")

class DistributeTestCase(unittest.TestCase):
    def test_distribute_one_well(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        p.distribute(c.well(0).set_volume("20:microliter"),
                     c.well(1),
                     "5:microliter")
        self.assertEqual(1, len(p.instructions))
        self.assertEqual("distribute",
                         list(p.as_dict()["instructions"][0]["groups"][0].keys())[0])
        self.assertTrue(5, c.well(1).volume.value)
        self.assertTrue(15, c.well(0).volume.value)

    def test_distribute_multiple_wells(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        p.distribute(c.well(0).set_volume("20:microliter"),
                     c.wells_from(1, 3),
                     "5:microliter")
        self.assertEqual(1, len(p.instructions))
        self.assertEqual("distribute",
                         list(p.as_dict()["instructions"][0]["groups"][0].keys())[0])
        for w in c.wells_from(1, 3):
            self.assertTrue(5, w.volume.value)
        self.assertTrue(5, c.well(0).volume.value)

    def test_fill_wells(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        srcs = c.wells_from(1, 2).set_volume("100:microliter")
        dests = c.wells_from(7, 4)
        p.distribute(srcs, dests, "30:microliter", allow_carryover=True)
        self.assertEqual(2, len(p.instructions[0].groups))

        # track source vols
        self.assertEqual(10, c.well(1).volume.value)
        self.assertEqual(70, c.well(2).volume.value)

        # track dest vols
        self.assertEqual(30, c.well(7).volume.value)
        self.assertIs(None, c.well(6).volume)

        # test distribute from Well to Well
        p.distribute(c.well("A1").set_volume("20:microliter"), c.well("A2"), "5:microliter")
        self.assertTrue("distribute" in p.instructions[-1].groups[-1])

class TransferTestCase(unittest.TestCase):
    def test_single_transfer(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        p.transfer(c.well(0), c.well(1), "20:microliter")
        self.assertEqual(Unit(20, "microliter"), c.well(1).volume)
        self.assertEqual(None, c.well(0).volume)
        self.assertTrue("transfer" in p.instructions[-1].groups[-1])

    def test_gt_750uL_transfer(self):
        p = Protocol()
        c = p.ref("test", None, "96-deep", discard=True)
        p.transfer(
            c.well(0),
            c.well(1),
            "1800:microliter"
            )
        self.assertEqual(3, len(p.instructions[0].groups))
        self.assertEqual(
            Unit(750, 'microliter'),
            p.instructions[0].groups[0]['transfer'][0]['volume']
            )
        self.assertEqual(
            Unit(750, 'microliter'),
            p.instructions[0].groups[1]['transfer'][0]['volume']
            )
        self.assertEqual(
            Unit(300, 'microliter'),
            p.instructions[0].groups[2]['transfer'][0]['volume']
            )

    def test_gt_750uL_wellgroup_transfer(self):
        p = Protocol()
        c = p.ref("test", None, "96-deep", discard=True)
        p.transfer(
            c.wells_from(0, 8, columnwise=True),
            c.wells_from(1, 8, columnwise=True),
            '1800:microliter'
            )
        self.assertEqual(
            24,
            len(p.instructions[0].groups)
            )

    def test_transfer_option_propagation(self):
        p = Protocol()
        c = p.ref("test", None, "96-deep", discard=True)
        p.transfer(
            c.well(0),
            c.well(1),
            "1800:microliter",
            aspirate_source=aspirate_source(
                depth("ll_bottom", distance=".004:meter")
                )
            )
        self.assertEqual(
            len(p.instructions[0].groups[0]['transfer'][0]),
            len(p.instructions[0].groups[1]['transfer'][0])
            )
        self.assertEqual(
            len(p.instructions[0].groups[0]['transfer'][0]),
            len(p.instructions[0].groups[2]['transfer'][0])
            )

    def test_max_transfer(self):
        p = Protocol()
        c = p.ref("test", None, "micro-2.0", storage="cold_4")
        p.transfer(c.well(0), c.well(0), "3050:microliter")

    def test_multiple_transfers(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        p.transfer(c.wells_from(0, 2), c.wells_from(2, 2), "20:microliter")
        self.assertEqual(c.well(2).volume, c.well(3).volume)
        self.assertEqual(2, len(p.instructions[0].groups))

    def test_one_tip(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        p.transfer(c.wells_from(0, 2), c.wells_from(2, 2), "20:microliter",
            one_tip=True)
        self.assertEqual(c.well(2).volume, c.well(3).volume)
        self.assertEqual(1, len(p.instructions[0].groups))

    def test_one_source(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        with self.assertRaises(RuntimeError):
            p.transfer(c.wells_from(0, 2),
                   c.wells_from(2, 2), "40:microliter", one_source=True)
        with self.assertRaises(RuntimeError):
            p.transfer(c.wells_from(0, 2).set_volume("1:microliter"),
                       c.wells_from(1, 5), "10:microliter", one_source=True)
        p.transfer(c.wells_from(0, 2).set_volume("50:microliter"),
                   c.wells_from(2, 2), "40:microliter", one_source=True)
        self.assertEqual(2, len(p.instructions[0].groups))
        self.assertFalse(p.instructions[0].groups[0]["transfer"][0]["from"] == p.instructions[0].groups[1]["transfer"][0]["from"])

class ConsolidateTestCase(unittest.TestCase):
    def test_multiple_sources(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        with self.assertRaises(TypeError):
            p.consolidate(c.wells_from(0, 3), c.wells_from(2, 3), "10:microliter")
        with self.assertRaises(ValueError):
            p.consolidate(c.wells_from(0, 3), c.well(4), ["10:microliter"])
        p.consolidate(c.wells_from(0, 3), c.well(4), "10:microliter")
        self.assertEqual(Unit(30, "microliter"), c.well(4).volume)
        self.assertEqual(3, len(p.instructions[0].groups[0]["consolidate"]["from"]))

    def test_one_source(self):
        p = Protocol()
        c = p.ref("test", None, "96-flat", discard=True)
        p.consolidate(c.well(0), c.well(4), "30:microliter")
        self.assertEqual(Unit(30, "microliter"), c.well(4).volume)

class StampTestCase(unittest.TestCase):
    def test_single_transfers(self):
        p = Protocol()
        plate_96_list = []
        for plate_num in range(5):
            plate_name = ("test_96_"+str(plate_num))
            plate_96_list.append(p.ref(plate_name, None, "96-flat", discard=True))
        plate_384_list = []
        for plate_num in range(3):
            plate_name = ("test_384_"+str(plate_num))
            plate_384_list.append(p.ref(plate_name, None, "384-flat", discard=True))
        with self.assertRaises(RuntimeError):
            # Transfer 4 plates
            for pl, q in zip(plate_96_list, [0, 1, 24, 26]):
                p.stamp(pl, plate_384_list[0], "10:microliter", to_quad = q)

class RefifyTestCase(unittest.TestCase):
    def test_refifying_various(self):
        p = Protocol()
        # refify container
        refs = {"plate": p.ref("test", None, "96-flat", "cold_20")}
        self.assertEqual(p._refify(refs["plate"]), "test")
        # refify dict
        self.assertEqual(p._refify(refs), {"plate": "test"})

        # refify Well
        well = refs["plate"].well("A1")
        self.assertEqual(p._refify(well), "test/0")

        # refify WellGroup
        wellgroup = refs["plate"].wells_from("A2", 3)
        self.assertEqual(p._refify(wellgroup), ["test/1", "test/2", "test/3"])

        # refify other
        s = "randomstring"
        i = 24
        self.assertEqual("randomstring", p._refify(s))
        self.assertEqual(24, p._refify(i))

class OutsTestCase(unittest.TestCase):
    def test_outs(self):
        p = Protocol()
        self.assertEqual(p.as_dict()['outs'], {})
        plate = p.ref("plate", None, "96-pcr", discard=True)
        plate.well(0).set_name("test_well")
        self.assertTrue(plate.well(0).name == "test_well")
        self.assertTrue(list(p.as_dict()['outs'].keys()) == ['plate'])
        self.assertTrue(list(list(p.as_dict()['outs'].values())[0].keys()) == ['0'])
        self.assertTrue(list(p.as_dict()['outs'].values())[0]['0'] == {'name': 'test_well'})
<|endoftext|>


<commit_before># Copyright 2019 Ecosoft Co., Ltd. (http://ecosoft.co.th)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    "name": "Purchase Work Acceptance",
    "version": "14.0.1.0.3",
    "category": "Purchase Management",
    "author": "Ecosoft, Odoo Community Association (OCA)",
    "license": "AGPL-3",
    "website": "https://github.com/OCA/purchase-workflow",
    "depends": ["purchase_stock"],
    "data": [
        "data/work_acceptance_sequence.xml",
        "security/ir.model.access.csv",
        "security/security.xml",
        "views/account_move_views.xml",
        "views/purchase_views.xml",
        "views/res_config_settings_views.xml",
        "views/stock_picking_views.xml",
        "views/work_acceptance_views.xml",
        "wizard/select_work_acceptance_wizard_views.xml",
    ],
    "maintainer": ["ps-tubtim"],
    "installable": True,
    "development_status": "Alpha",
}
<commit_msg>purchase_work_acceptance 14.0.1.0.4<commit_after># Copyright 2019 Ecosoft Co., Ltd. (http://ecosoft.co.th)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    "name": "Purchase Work Acceptance",
    "version": "14.0.1.0.4",
    "category": "Purchase Management",
    "author": "Ecosoft, Odoo Community Association (OCA)",
    "license": "AGPL-3",
    "website": "https://github.com/OCA/purchase-workflow",
    "depends": ["purchase_stock"],
    "data": [
        "data/work_acceptance_sequence.xml",
        "security/ir.model.access.csv",
        "security/security.xml",
        "views/account_move_views.xml",
        "views/purchase_views.xml",
        "views/res_config_settings_views.xml",
        "views/stock_picking_views.xml",
        "views/work_acceptance_views.xml",
        "wizard/select_work_acceptance_wizard_views.xml",
    ],
    "maintainer": ["ps-tubtim"],
    "installable": True,
    "development_status": "Alpha",
}
<|endoftext|>


<commit_before># Copyright 2019 Ecosoft Co., Ltd. (http://ecosoft.co.th)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    "name": "Purchase Work Acceptance",
    "version": "14.0.1.0.4",
    "category": "Purchase Management",
    "author": "Ecosoft, Odoo Community Association (OCA)",
    "license": "AGPL-3",
    "website": "https://github.com/OCA/purchase-workflow",
    "depends": ["purchase_stock"],
    "data": [
        "data/work_acceptance_sequence.xml",
        "security/ir.model.access.csv",
        "security/security.xml",
        "views/account_move_views.xml",
        "views/purchase_views.xml",
        "views/res_config_settings_views.xml",
        "views/stock_picking_views.xml",
        "views/work_acceptance_views.xml",
        "wizard/select_work_acceptance_wizard_views.xml",
    ],
    "maintainer": ["ps-tubtim"],
    "installable": True,
    "development_status": "Alpha",
}
<commit_msg>purchase_work_acceptance 14.0.1.0.5<commit_after># Copyright 2019 Ecosoft Co., Ltd. (http://ecosoft.co.th)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    "name": "Purchase Work Acceptance",
    "version": "14.0.1.0.5",
    "category": "Purchase Management",
    "author": "Ecosoft, Odoo Community Association (OCA)",
    "license": "AGPL-3",
    "website": "https://github.com/OCA/purchase-workflow",
    "depends": ["purchase_stock"],
    "data": [
        "data/work_acceptance_sequence.xml",
        "security/ir.model.access.csv",
        "security/security.xml",
        "views/account_move_views.xml",
        "views/purchase_views.xml",
        "views/res_config_settings_views.xml",
        "views/stock_picking_views.xml",
        "views/work_acceptance_views.xml",
        "wizard/select_work_acceptance_wizard_views.xml",
    ],
    "maintainer": ["ps-tubtim"],
    "installable": True,
    "development_status": "Alpha",
}
<|endoftext|>


<commit_before>### Ready the data for the db

import csv
import codecs
import re
import xml.etree.cElementTree as ET
import cerberus
import schema

# Set the schema according to Udacity conventions

SCHEMA = {
    'node': {
        'type': 'dict',
        'schema': {
            'id': {'required': True, 'type': 'integer', 'coerce': int},
            'lat': {'required': True, 'type': 'float', 'coerce': float},
            'lon': {'required': True, 'type': 'float', 'coerce': float},
            'user': {'required': True, 'type': 'string'},
            'uid': {'required': True, 'type': 'integer', 'coerce': int},
            'version': {'required': True, 'type': 'string'},
            'changeset': {'required': True, 'type': 'integer', 'coerce': int},
            'timestamp': {'required': True, 'type': 'string'}
        }
    },
    'node_tags': {
        'type': 'list',
        'schema': {
            'type': 'dict',
            'schema': {
                'id': {'required': True, 'type': 'integer', 'coerce': int},
                'key': {'required': True, 'type': 'string'},
                'value': {'required': True, 'type': 'string'},
                'type': {'required': True, 'type': 'string'}
            }
        }
    },
    'way': {
        'type': 'dict',
        'schema': {
            'id': {'required': True, 'type': 'integer', 'coerce': int},
            'user': {'required': True, 'type': 'string'},
            'uid': {'required': True, 'type': 'integer', 'coerce': int},
            'version': {'required': True, 'type': 'string'},
            'changeset': {'required': True, 'type': 'integer', 'coerce': int},
            'timestamp': {'required': True, 'type': 'string'}
        }
    },
    'way_nodes': {
        'type': 'list',
        'schema': {
            'type': 'dict',
            'schema': {
                'id': {'required': True, 'type': 'integer', 'coerce': int},
                'node_id': {'required': True, 'type': 'integer', 'coerce': int},
                'position': {'required': True, 'type': 'integer', 'coerce': int}
            }
        }
    },
    'way_tags': {
        'type': 'list',
        'schema': {
            'type': 'dict',
            'schema': {
                'id': {'required': True, 'type': 'integer', 'coerce': int},
                'key': {'required': True, 'type': 'string'},
                'value': {'required': True, 'type': 'string'},
                'type': {'required': True, 'type': 'string'}
            }
        }
    }
}

# Slight adjustments for Python 3 when dealing with old functions
unicode = str

### Writing CSVs from the OSM files

# Main Functions to organize data into proper CSVs files before we put into a database
OSM_PATH = 'udacity_project_openstreetdata_rjl/bozeman_MT.osm'
if not os.path.exists('./CSV_Files'):
    os.makedirs('./CSV_Files/')

NODES_PATH = "nodes.csv"
NODE_TAGS_PATH = "nodes_tags.csv"
WAYS_PATH = "ways.csv"
WAY_NODES_PATH = "ways_nodes.csv"
WAY_TAGS_PATH = "ways_tags.csv"

LOWER_COLON = re.compile(r'^([a-z]|_)+:([a-z]|_)+')
PROBLEMCHARS = re.compile(r'[=\+/&<>;\'"\?%#$@\,\. \t\r\n]')
PHONENUM = re.compile(r'\+1\s\d{3}\s\d{3}\s\d{4}')
POSTCODE = re.compile(r'[A-z]\d[A-z]\s?\d[A-z]\d')

# Make sure the fields order in the csvs matches the column order in the sql table schema
NODE_FIELDS = ['id', 'lat', 'lon', 'user', 'uid', 'version', 'changeset', 'timestamp']
NODE_TAGS_FIELDS = ['id', 'key', 'value', 'type']
WAY_FIELDS = ['id', 'user', 'uid', 'version', 'changeset', 'timestamp']
WAY_TAGS_FIELDS = ['id', 'key', 'value', 'type']
WAY_NODES_FIELDS = ['id', 'node_id', 'position']

# update Phone Number
def update_phone_num(phone_num):
    """
    Clean phone number for insertion into SQL database
    """
    # Check for valid phone number format
    m = PHONENUM.match(phone_num)
    if m is None:
        # Convert all dashes to spaces
        if "-" in phone_num:
            phone_num = re.sub("-", " ", phone_num)
        # Remove all brackets
        if "(" in phone_num or ")" in phone_num:
            phone_num = re.sub("[()]", "", phone_num)
        # Space out 10 straight numbers
        if re.match(r'\d{10}', phone_num) is not None:
            phone_num = phone_num[:3] + " " + phone_num[3:6] + " " + phone_num[6:]
        # Space out 11 straight numbers
        elif re.match(r'\d{11}', phone_num) is not None:
            phone_num = phone_num[:1] + " " + phone_num[1:4] + " " + phone_num[4:7] + " " + phone_num[7:]
        # Add full country code
        if re.match(r'\d{3}\s\d{3}\s\d{4}', phone_num) is not None:
            phone_num = "+1 " + phone_num
        # Add + in country code
        elif re.match(r'1\s\d{3}\s\d{3}\s\d{4}', phone_num) is not None:
            phone_num = "+" + phone_num
        # Ignore tag if no area code and local number (<10 digits)
        elif sum(c.isdigit() for c in phone_num) < 10:
            return None
    return phone_num

# Update tag dict for way_tags
def load_new_tag(element, secondary, default_tag_type):
    """
    Load a new tag dict to go into the list of dicts for way_tags, node_tags
    """
    new = {}
    new['id'] = element.attrib['id']
    if ":" not in secondary.attrib['k']:
        new['key'] = secondary.attrib['k']
        new['type'] = default_tag_type
    else:
        post_colon = secondary.attrib['k'].index(":") + 1
        new['key'] = secondary.attrib['k'][post_colon:]
        new['type'] = secondary.attrib['k'][:post_colon - 1]

    # Cleaning and loading values of various keys
    if is_street_name(secondary):
        # Why don't i need to use mapping, street_mapping,
        # and num_line_mapping dicts  as params?
        street_name = update_name(secondary.attrib['v'], mapping)
        new['value'] = street_name

    elif new['key'] == 'phone':
        phone_num = update_phone_num(secondary.attrib['v'])
        if phone_num is not None:
            new['value'] = phone_num
        else:
            return None

    elif new['key'] == 'province':
        # Change Ontario to ON
        province = secondary.attrib['v']
        if province == 'Ontario':
            province = 'ON'
        new['value'] = province

    elif new['key'] == 'postcode':
        post_code = secondary.attrib['v'].strip()
        m = POSTCODE.match(post_code)
        if m is not None:
            # Add space in middle if there is none
            if " " not in post_code:
                post_code = post_code[:3] + " " + post_code[3:]
            # Convert to upper case
            new['value'] = post_code.upper()
        else:
            # Keep zip code revealed in postal code audit for document deletion purposes
            if post_code[:5] == "14174":
                new['value'] = post_code
            # Ignore tag if improper postal code format
            else:
                return None

    else:
        new['value'] = secondary.attrib['v']

    return new

# Update XML elements into dicts
def shape_element(element, node_attr_fields=NODE_FIELDS, way_attr_fields=WAY_FIELDS,
                  problem_chars=PROBLEMCHARS, default_tag_type='regular'):
    """Clean and shape node or way XML element to Python dict"""

    node_attribs = {}
    way_attribs = {}
    way_nodes = []
    tags = []  # Handle secondary tags the same way for both node and way elements

    if element.tag == 'node':
        for attrib, value in element.attrib.items():
            if attrib in node_attr_fields:
                node_attribs[attrib] = value

        # For elements within the top element
        for secondary in element.iter():
            if secondary.tag == 'tag':
                if problem_chars.match(secondary.attrib['k']) is not None:
                    continue
                else:
                    new = load_new_tag(element, secondary, default_tag_type)
                    if new is not None:
                        tags.append(new)
        return {'node': node_attribs, 'node_tags': tags}

    elif element.tag == 'way':
        for attrib, value in element.attrib.items():
            if attrib in way_attr_fields:
                way_attribs[attrib] = value

        counter = 0
        for secondary in element.iter():
            if secondary.tag == 'tag':
                if problem_chars.match(secondary.attrib['k']) is not None:
                    continue
                else:
                    new = load_new_tag(element, secondary, default_tag_type)
                    if new is not None:
                        tags.append(new)
            elif secondary.tag == 'nd':
                newnd = {}
                newnd['id'] = element.attrib['id']
                newnd['node_id'] = secondary.attrib['ref']
                newnd['position'] = counter
                counter += 1
                way_nodes.append(newnd)

        return {'way': way_attribs, 'way_nodes': way_nodes, 'way_tags': tags}

### Helper Function
# If correctly tagged, element is used

def get_element(osm_file, tags=('node', 'way', 'relation')):
    """Yield element if it is the right type of tag"""

    context = ET.iterparse(osm_file, events=('start', 'end'))
    _, root = next(context)
    for event, elem in context:
        if event == 'end' and elem.tag in tags:
            yield elem
            root.clear()

# Update if element doesn't fit schema
def validate_element(element, validator, schema=SCHEMA):
    """Raise ValidationError if element does not match schema"""
    if validator.validate(element, schema) is not True:
        field, errors = next(validator.errors.items())
        message_string = "\nElement of type '{0}' has the following errors:\n{1}"
        error_strings = (
            "{0}: {1}".format(k, v if isinstance(v, str) else ", ".join(v))
            for k, v in errors.items()
        )
        raise cerberus.ValidationError(
            message_string.format(field, "\n".join(error_strings))
        )

# Update Unicode writing
class csvDictWriter(csv.DictWriter, object):
    """Extend csv.DictWriter to handle Unicode input"""

    def writerow(self, row):
        super(csv.DictWriter, self).writerow({
            k: (v.encode('utf-8') if isinstance(v, unicode) else v) for k, v in row.items()
        })

    def writerows(self, rows):
        for row in rows:
            self.writerow(row)

# Update csvs with XML data
def process_map(file_in, validate):
    """Iteratively process each XML element and write to csv(s)"""

    with codecs.open(NODES_PATH, 'w') as nodes_file, \
         codecs.open(NODE_TAGS_PATH, 'w') as nodes_tags_file, \
         codecs.open(WAYS_PATH, 'w') as ways_file, \
         codecs.open(WAY_NODES_PATH, 'w') as way_nodes_file, \
         codecs.open(WAY_TAGS_PATH, 'w') as way_tags_file:

        nodes_writer = csv.DictWriter(nodes_file, NODE_FIELDS)
        node_tags_writer = csv.DictWriter(nodes_tags_file, NODE_TAGS_FIELDS)
        ways_writer = csv.DictWriter(ways_file, WAY_FIELDS)
        way_nodes_writer = csv.DictWriter(way_nodes_file, WAY_NODES_FIELDS)
        way_tags_writer = csv.DictWriter(way_tags_file, WAY_TAGS_FIELDS)

        nodes_writer.writeheader()
        node_tags_writer.writeheader()
        ways_writer.writeheader()
        way_nodes_writer.writeheader()
        way_tags_writer.writeheader()

        validator = cerberus.Validator()

        for element in get_element(file_in, tags=('node', 'way')):
            el = shape_element(element)
            if el:
                if validate is True:
                    validate_element(el, validator)

                if element.tag == 'node':
                    nodes_writer.writerow(el['node'])
                    node_tags_writer.writerows(el['node_tags'])
                elif element.tag == 'way':
                    ways_writer.writerow(el['way'])
                    way_nodes_writer.writerows(el['way_nodes'])
                    way_tags_writer.writerows(el['way_tags'])

### Main Function
# Final step to prep the data for the DB
if __name__ == '__main__':
    # Note: Validation is ~ 10X slower. For the project consider using a small
    # sample of the map when validating.
    process_map(OSM_PATH, validate=False)
print('Processed {} Map Succesfully!'.format(OSM_PATH[:-4]))

# Next file is the preping the database file 'project_openstreetdata_inserting_db.py'
<commit_msg>Update py.project_openstreetmap_preparing_db.py<commit_after>### Ready the data for the db

import csv
import codecs
import re
import xml.etree.cElementTree as ET
import cerberus
import schema
# the left over function is from the previous file project_openstreet_data/py.project_openstreetdata_casestudy.py
# However,
from py_project_openstreetdata_casestudy import update name

# Set the schema according to Udacity conventions

SCHEMA = {
    'node': {
        'type': 'dict',
        'schema': {
            'id': {'required': True, 'type': 'integer', 'coerce': int},
            'lat': {'required': True, 'type': 'float', 'coerce': float},
            'lon': {'required': True, 'type': 'float', 'coerce': float},
            'user': {'required': True, 'type': 'string'},
            'uid': {'required': True, 'type': 'integer', 'coerce': int},
            'version': {'required': True, 'type': 'string'},
            'changeset': {'required': True, 'type': 'integer', 'coerce': int},
            'timestamp': {'required': True, 'type': 'string'}
        }
    },
    'node_tags': {
        'type': 'list',
        'schema': {
            'type': 'dict',
            'schema': {
                'id': {'required': True, 'type': 'integer', 'coerce': int},
                'key': {'required': True, 'type': 'string'},
                'value': {'required': True, 'type': 'string'},
                'type': {'required': True, 'type': 'string'}
            }
        }
    },
    'way': {
        'type': 'dict',
        'schema': {
            'id': {'required': True, 'type': 'integer', 'coerce': int},
            'user': {'required': True, 'type': 'string'},
            'uid': {'required': True, 'type': 'integer', 'coerce': int},
            'version': {'required': True, 'type': 'string'},
            'changeset': {'required': True, 'type': 'integer', 'coerce': int},
            'timestamp': {'required': True, 'type': 'string'}
        }
    },
    'way_nodes': {
        'type': 'list',
        'schema': {
            'type': 'dict',
            'schema': {
                'id': {'required': True, 'type': 'integer', 'coerce': int},
                'node_id': {'required': True, 'type': 'integer', 'coerce': int},
                'position': {'required': True, 'type': 'integer', 'coerce': int}
            }
        }
    },
    'way_tags': {
        'type': 'list',
        'schema': {
            'type': 'dict',
            'schema': {
                'id': {'required': True, 'type': 'integer', 'coerce': int},
                'key': {'required': True, 'type': 'string'},
                'value': {'required': True, 'type': 'string'},
                'type': {'required': True, 'type': 'string'}
            }
        }
    }
}

# Slight adjustments for Python 3 when dealing with old functions
unicode = str

### Writing CSVs from the OSM files

# Main Functions to organize data into proper CSVs files before we put into a database
OSM_PATH = 'udacity_project_openstreetdata_rjl/bozeman_MT.osm'
if not os.path.exists('./CSV_Files'):
    os.makedirs('./CSV_Files/')

NODES_PATH = "nodes.csv"
NODE_TAGS_PATH = "nodes_tags.csv"
WAYS_PATH = "ways.csv"
WAY_NODES_PATH = "ways_nodes.csv"
WAY_TAGS_PATH = "ways_tags.csv"

LOWER_COLON = re.compile(r'^([a-z]|_)+:([a-z]|_)+')
PROBLEMCHARS = re.compile(r'[=\+/&<>;\'"\?%#$@\,\. \t\r\n]')
PHONENUM = re.compile(r'\+1\s\d{3}\s\d{3}\s\d{4}')
POSTCODE = re.compile(r'[A-z]\d[A-z]\s?\d[A-z]\d')

# Make sure the fields order in the csvs matches the column order in the sql table schema
NODE_FIELDS = ['id', 'lat', 'lon', 'user', 'uid', 'version', 'changeset', 'timestamp']
NODE_TAGS_FIELDS = ['id', 'key', 'value', 'type']
WAY_FIELDS = ['id', 'user', 'uid', 'version', 'changeset', 'timestamp']
WAY_TAGS_FIELDS = ['id', 'key', 'value', 'type']
WAY_NODES_FIELDS = ['id', 'node_id', 'position']

# update Phone Number
def update_phone_num(phone_num):
    """
    Clean phone number for insertion into SQL database
    """
    # Check for valid phone number format
    m = PHONENUM.match(phone_num)
    if m is None:
        # Convert all dashes to spaces
        if "-" in phone_num:
            phone_num = re.sub("-", " ", phone_num)
        # Remove all brackets
        if "(" in phone_num or ")" in phone_num:
            phone_num = re.sub("[()]", "", phone_num)
        # Space out 10 straight numbers
        if re.match(r'\d{10}', phone_num) is not None:
            phone_num = phone_num[:3] + " " + phone_num[3:6] + " " + phone_num[6:]
        # Space out 11 straight numbers
        elif re.match(r'\d{11}', phone_num) is not None:
            phone_num = phone_num[:1] + " " + phone_num[1:4] + " " + phone_num[4:7] + " " + phone_num[7:]
        # Add full country code
        if re.match(r'\d{3}\s\d{3}\s\d{4}', phone_num) is not None:
            phone_num = "+1 " + phone_num
        # Add + in country code
        elif re.match(r'1\s\d{3}\s\d{3}\s\d{4}', phone_num) is not None:
            phone_num = "+" + phone_num
        # Ignore tag if no area code and local number (<10 digits)
        elif sum(c.isdigit() for c in phone_num) < 10:
            return None
    return phone_num

# Update tag dict for way_tags
def load_new_tag(element, secondary, default_tag_type):
    """
    Load a new tag dict to go into the list of dicts for way_tags, node_tags
    """
    new = {}
    new['id'] = element.attrib['id']
    if ":" not in secondary.attrib['k']:
        new['key'] = secondary.attrib['k']
        new['type'] = default_tag_type
    else:
        post_colon = secondary.attrib['k'].index(":") + 1
        new['key'] = secondary.attrib['k'][post_colon:]
        new['type'] = secondary.attrib['k'][:post_colon - 1]

    # Cleaning and loading values of various keys
    if is_street_name(secondary):
        # Why don't i need to use mapping, street_mapping,
        # and num_line_mapping dicts  as params?
        street_name = update_name(secondary.attrib['v'], mapping)
        new['value'] = street_name

    elif new['key'] == 'phone':
        phone_num = update_phone_num(secondary.attrib['v'])
        if phone_num is not None:
            new['value'] = phone_num
        else:
            return None

    elif new['key'] == 'province':
        # Change Ontario to ON
        province = secondary.attrib['v']
        if province == 'Ontario':
            province = 'ON'
        new['value'] = province

    elif new['key'] == 'postcode':
        post_code = secondary.attrib['v'].strip()
        m = POSTCODE.match(post_code)
        if m is not None:
            # Add space in middle if there is none
            if " " not in post_code:
                post_code = post_code[:3] + " " + post_code[3:]
            # Convert to upper case
            new['value'] = post_code.upper()
        else:
            # Keep zip code revealed in postal code audit for document deletion purposes
            if post_code[:5] == "14174":
                new['value'] = post_code
            # Ignore tag if improper postal code format
            else:
                return None

    else:
        new['value'] = secondary.attrib['v']

    return new

# Update XML elements into dicts
def shape_element(element, node_attr_fields=NODE_FIELDS, way_attr_fields=WAY_FIELDS,
                  problem_chars=PROBLEMCHARS, default_tag_type='regular'):
    """Clean and shape node or way XML element to Python dict"""

    node_attribs = {}
    way_attribs = {}
    way_nodes = []
    tags = []  # Handle secondary tags the same way for both node and way elements

    if element.tag == 'node':
        for attrib, value in element.attrib.items():
            if attrib in node_attr_fields:
                node_attribs[attrib] = value

        # For elements within the top element
        for secondary in element.iter():
            if secondary.tag == 'tag':
                if problem_chars.match(secondary.attrib['k']) is not None:
                    continue
                else:
                    new = load_new_tag(element, secondary, default_tag_type)
                    if new is not None:
                        tags.append(new)
        return {'node': node_attribs, 'node_tags': tags}

    elif element.tag == 'way':
        for attrib, value in element.attrib.items():
            if attrib in way_attr_fields:
                way_attribs[attrib] = value

        counter = 0
        for secondary in element.iter():
            if secondary.tag == 'tag':
                if problem_chars.match(secondary.attrib['k']) is not None:
                    continue
                else:
                    new = load_new_tag(element, secondary, default_tag_type)
                    if new is not None:
                        tags.append(new)
            elif secondary.tag == 'nd':
                newnd = {}
                newnd['id'] = element.attrib['id']
                newnd['node_id'] = secondary.attrib['ref']
                newnd['position'] = counter
                counter += 1
                way_nodes.append(newnd)

        return {'way': way_attribs, 'way_nodes': way_nodes, 'way_tags': tags}

### Helper Function
# If correctly tagged, element is used

def get_element(osm_file, tags=('node', 'way', 'relation')):
    """Yield element if it is the right type of tag"""

    context = ET.iterparse(osm_file, events=('start', 'end'))
    _, root = next(context)
    for event, elem in context:
        if event == 'end' and elem.tag in tags:
            yield elem
            root.clear()

# Update if element doesn't fit schema
def validate_element(element, validator, schema=SCHEMA):
    """Raise ValidationError if element does not match schema"""
    if validator.validate(element, schema) is not True:
        field, errors = next(validator.errors.items())
        message_string = "\nElement of type '{0}' has the following errors:\n{1}"
        error_strings = (
            "{0}: {1}".format(k, v if isinstance(v, str) else ", ".join(v))
            for k, v in errors.items()
        )
        raise cerberus.ValidationError(
            message_string.format(field, "\n".join(error_strings))
        )

# Update Unicode writing
class csvDictWriter(csv.DictWriter, object):
    """Extend csv.DictWriter to handle Unicode input"""

    def writerow(self, row):
        super(csv.DictWriter, self).writerow({
            k: (v.encode('utf-8') if isinstance(v, unicode) else v) for k, v in row.items()
        })

    def writerows(self, rows):
        for row in rows:
            self.writerow(row)

# Update csvs with XML data
def process_map(file_in, validate):
    """Iteratively process each XML element and write to csv(s)"""

    with codecs.open(NODES_PATH, 'w') as nodes_file, \
         codecs.open(NODE_TAGS_PATH, 'w') as nodes_tags_file, \
         codecs.open(WAYS_PATH, 'w') as ways_file, \
         codecs.open(WAY_NODES_PATH, 'w') as way_nodes_file, \
         codecs.open(WAY_TAGS_PATH, 'w') as way_tags_file:

        nodes_writer = csv.DictWriter(nodes_file, NODE_FIELDS)
        node_tags_writer = csv.DictWriter(nodes_tags_file, NODE_TAGS_FIELDS)
        ways_writer = csv.DictWriter(ways_file, WAY_FIELDS)
        way_nodes_writer = csv.DictWriter(way_nodes_file, WAY_NODES_FIELDS)
        way_tags_writer = csv.DictWriter(way_tags_file, WAY_TAGS_FIELDS)

        nodes_writer.writeheader()
        node_tags_writer.writeheader()
        ways_writer.writeheader()
        way_nodes_writer.writeheader()
        way_tags_writer.writeheader()

        validator = cerberus.Validator()

        for element in get_element(file_in, tags=('node', 'way')):
            el = shape_element(element)
            if el:
                if validate is True:
                    validate_element(el, validator)

                if element.tag == 'node':
                    nodes_writer.writerow(el['node'])
                    node_tags_writer.writerows(el['node_tags'])
                elif element.tag == 'way':
                    ways_writer.writerow(el['way'])
                    way_nodes_writer.writerows(el['way_nodes'])
                    way_tags_writer.writerows(el['way_tags'])

### Main Function
# Final step to prep the data for the DB
if __name__ == '__main__':
    # Note: Validation is ~ 10X slower. For the project consider using a small
    # sample of the map when validating.
    process_map(OSM_PATH, validate=False)
print('Processed {} Map Succesfully!'.format(OSM_PATH[:-4]))

# Next file is the preping the database file 'project_openstreetdata_inserting_db.py'
<|endoftext|>


<commit_before>#
# See top-level LICENSE.rst file for Copyright information
#
# -*- coding: utf-8 -*-

from __future__ import absolute_import, division, print_function

from collections import OrderedDict

from ..defs import (task_name_sep, task_state_to_int, task_int_to_state)

from ...util import option_list

from ...io import findfile

from .base import (BaseTask, task_classes)

from desiutil.log import get_logger

import sys,re,os,copy

# NOTE: only one class in this file should have a name that starts with "Task".

class TaskTraceShift(BaseTask):
    """Class containing the properties of one trace shift task.
    """
    def __init__(self):
        super(TaskTraceShift, self).__init__()
        # then put int the specifics of this class
        # _cols must have a state
        self._type = "traceshift"
        self._cols = [
            "night",
            "band",
            "spec",
            "expid",
            "state"
        ]
        self._coltypes = [
            "integer",
            "text",
            "integer",
            "integer",
            "integer"
        ]
        # _name_fields must also be in _cols
        self._name_fields  = ["night","band","spec","expid"]
        self._name_formats = ["08d","s","d","08d"]

    def _paths(self, name):
        """See BaseTask.paths.
        """
        props = self.name_split(name)
        camera = "{}{}".format(props["band"], props["spec"])
        return [ findfile("psf", night=props["night"], expid=props["expid"],
            camera=camera, groupname=None, nside=None, band=props["band"],
            spectrograph=props["spec"]) ]

    def _deps(self, name, db, inputs):
        """See BaseTask.deps.
        """
        from .base import task_classes
        props = self.name_split(name)
        deptasks = {
            "image" : task_classes["preproc"].name_join(props),
            "psf" : task_classes["psfnight"].name_join(props)
            }
        return deptasks

    def _run_max_procs(self, procs_per_node):
        """See BaseTask.run_max_procs.
        """
        return 1

    def _run_time(self, name, procs_per_node, db=None):
        """See BaseTask.run_time.
        """
        return 1

    def _run_defaults(self):
        """See BaseTask.run_defaults.
        """
        opts = {}
        opts["degxx"] = 2
        opts["degxy"] = 2
        opts["degyx"] = 2
        opts["degyy"] = 2
        opts["auto"]  = True
        return opts

    def _option_list(self, name, opts):
        """Build the full list of options.

        This includes appending the filenames and incorporating runtime
        options.
        """
        from .base import task_classes, task_type

        deps = self.deps(name)
        options = {}
        options["image"]    = task_classes["preproc"].paths(deps["image"])[0]
        options["psf"]      = task_classes["psfnight"].paths(deps["psf"])[0]
        options["outpsf"]   = self.paths(name)[0]

        options.update(opts)
        return option_list(options)

    def _run_cli(self, name, opts, procs, db):
        """See BaseTask.run_cli.
        """
        entry = "desi_compute_trace_shifts"
        optlist = self._option_list(name, opts)
        com = "{} {}".format(entry, " ".join(optlist))
        return com

    def _run(self, name, opts, comm, db):
        """See BaseTask.run.
        """
        from ...scripts import trace_shifts
        optlist = self._option_list(name, opts)
        args = trace_shifts.parse(optlist)
        if comm is None :
            trace_shifts.main(args)
        else :
            trace_shifts.main_mpi(args, comm=comm)
        return

    def postprocessing(self, db, name, cur):
        """For successful runs, postprocessing on DB"""
        # run getready for all extraction with same night,band,spec
        props = self.name_split(name)
        log  = get_logger()
        for tt in ["extract"] :
            cmd = "select name from {} where night={} and expid={} and band='{}' and spec={} and state=0".format(tt,props["night"],props["expid"],props["band"],props["spec"])
            cur.execute(cmd)
            tasks = [ x for (x,) in cur.fetchall() ]
            log.debug("checking {} {}".format(tt,tasks))
            for task in tasks :
                task_classes[tt].getready( db=db,name=task,cur=cur)
<commit_msg>change default degree for y<commit_after>#
# See top-level LICENSE.rst file for Copyright information
#
# -*- coding: utf-8 -*-

from __future__ import absolute_import, division, print_function

from collections import OrderedDict

from ..defs import (task_name_sep, task_state_to_int, task_int_to_state)

from ...util import option_list

from ...io import findfile

from .base import (BaseTask, task_classes)

from desiutil.log import get_logger

import sys,re,os,copy

# NOTE: only one class in this file should have a name that starts with "Task".

class TaskTraceShift(BaseTask):
    """Class containing the properties of one trace shift task.
    """
    def __init__(self):
        super(TaskTraceShift, self).__init__()
        # then put int the specifics of this class
        # _cols must have a state
        self._type = "traceshift"
        self._cols = [
            "night",
            "band",
            "spec",
            "expid",
            "state"
        ]
        self._coltypes = [
            "integer",
            "text",
            "integer",
            "integer",
            "integer"
        ]
        # _name_fields must also be in _cols
        self._name_fields  = ["night","band","spec","expid"]
        self._name_formats = ["08d","s","d","08d"]

    def _paths(self, name):
        """See BaseTask.paths.
        """
        props = self.name_split(name)
        camera = "{}{}".format(props["band"], props["spec"])
        return [ findfile("psf", night=props["night"], expid=props["expid"],
            camera=camera, groupname=None, nside=None, band=props["band"],
            spectrograph=props["spec"]) ]

    def _deps(self, name, db, inputs):
        """See BaseTask.deps.
        """
        from .base import task_classes
        props = self.name_split(name)
        deptasks = {
            "image" : task_classes["preproc"].name_join(props),
            "psf" : task_classes["psfnight"].name_join(props)
            }
        return deptasks

    def _run_max_procs(self, procs_per_node):
        """See BaseTask.run_max_procs.
        """
        return 1

    def _run_time(self, name, procs_per_node, db=None):
        """See BaseTask.run_time.
        """
        return 1

    def _run_defaults(self):
        """See BaseTask.run_defaults.
        """
        opts = {}
        opts["degxx"] = 2
        opts["degxy"] = 2
        opts["degyx"] = 0
        opts["degyy"] = 0
        opts["auto"]  = True
        return opts

    def _option_list(self, name, opts):
        """Build the full list of options.

        This includes appending the filenames and incorporating runtime
        options.
        """
        from .base import task_classes, task_type

        deps = self.deps(name)
        options = {}
        options["image"]    = task_classes["preproc"].paths(deps["image"])[0]
        options["psf"]      = task_classes["psfnight"].paths(deps["psf"])[0]
        options["outpsf"]   = self.paths(name)[0]

        options.update(opts)
        return option_list(options)

    def _run_cli(self, name, opts, procs, db):
        """See BaseTask.run_cli.
        """
        entry = "desi_compute_trace_shifts"
        optlist = self._option_list(name, opts)
        com = "{} {}".format(entry, " ".join(optlist))
        return com

    def _run(self, name, opts, comm, db):
        """See BaseTask.run.
        """
        from ...scripts import trace_shifts
        optlist = self._option_list(name, opts)
        args = trace_shifts.parse(optlist)
        if comm is None :
            trace_shifts.main(args)
        else :
            trace_shifts.main_mpi(args, comm=comm)
        return

    def postprocessing(self, db, name, cur):
        """For successful runs, postprocessing on DB"""
        # run getready for all extraction with same night,band,spec
        props = self.name_split(name)
        log  = get_logger()
        for tt in ["extract"] :
            cmd = "select name from {} where night={} and expid={} and band='{}' and spec={} and state=0".format(tt,props["night"],props["expid"],props["band"],props["spec"])
            cur.execute(cmd)
            tasks = [ x for (x,) in cur.fetchall() ]
            log.debug("checking {} {}".format(tt,tasks))
            for task in tasks :
                task_classes[tt].getready( db=db,name=task,cur=cur)
<|endoftext|>


<commit_before>from fol import *

if __name__ == "__main__":
  e1 = And([Predicate("P", Function("f", Variable("x"))), Predicate("P",
        Function("g", Variable("x")))])
  e2 = Or([Predicate("P", Function("h", Variable("x"))), Predicate("i",
        Function("g", Variable("x")))])
  e3 = Implication(e1, e2)
  e4 = Equivalence(e1,e2)
  e5 = ThereExists(Variable("x"), And([Predicate("P", Function("f", Variable("x"))),
        Predicate("P", Function("g", Variable("x")))]))
  e6 = ForAll(Variable("x"), And([Predicate("P", Function("f", Variable("x"))),
        Predicate("P", Function("g", Variable("x")))]))
  print("Orignal Expression:")
  print("Expression 1: ", e1)
  print("Expression 2: ", e2)
  print("Expression 3: ", e3)
  print("Expression 4: ", e4)
  print("Expression 5: ", e5)
  print("Expression 6: ", e6)

  print("\n\nNegated Expressions")
  e1.negate()
  e2.negate()
  e3.negate()
  e4.negate()
  e5.negate()
  e6.negate()
  print("Expression 7: ", e1)
  print("Expression 8: ", e2)
  print("Expression 9: ", e3)
  print("Expression 10: ", e4)
  print("Expression 11: ", e5)
  print("Expression 12: ", e6)

  print("\n\nPushed Negation Expressions")
  e13 = e1.push_negation()
  e14 = e2.push_negation()
  e15 = e3.push_negation()
  e16 = e4.push_negation()
  e17 = e5.push_negation()
  e18 = e6.push_negation()
  print("Expression 13: ", e13)
  print("Expression 14: ", e14)
  print("Expression 15: ", e15)
  print("Expression 16: ", e16)
  print("Expression 17: ", e17)
  print("Expression 18: ", e18)

  print("\n\nRemoving Equivalences")
  test1 = Equivalence(Equivalence(Equivalence(Variable('x'), Variable('x')), Variable('x')), Variable('z'))
  test2 = Or([Equivalence(Variable('x'), Variable('y')),Variable('z')])
  print(test1)
  print(remove_equivalences(test1))
  print("\n")
  print(test2)
  print(remove_equivalences(test2))

  print("\n\nRemoving Implications")
  test3 = Implication(Implication(Variable('x'), Variable('y')), Variable('z'))
  test4 = Implication(Implication(Implication(Variable('x'), Variable('x')), Variable('x')), Variable('z'))
  print(test3)
  print(remove_implications(test3))
  print("\n")
  print(test4)
  print(remove_implications(test4))

  print("\n\nPush Not")
  test5 = And([And([Predicate("P", Function("f", Variable("x"))), Predicate("P",
        Function("g", Variable("x")))]), Predicate("P",Function("g", Variable("x")))])
  test6 = ForAll(Variable("x"), And([Predicate("P", Function("f", Variable("x"))),
        Predicate("P", Function("g", Variable("x")))]))
  test5.negate()
  test6.negate()
  print(test5)
  print(push_nots_inwards(test5))
  print("\n")
  print(test6)
  print(push_nots_inwards(test6))
<commit_msg>Tried fixing some tests<commit_after>from fol import *

if __name__ == "__main__":
  e1 = And([Predicate("P", Function("f", Variable("x"))), Predicate("P",
        Function("g", Variable("x")))])
  e2 = Or([Predicate("P", Function("h", Variable("x"))), Predicate("i",
        Function("g", Variable("x")))])
  e3 = Implication(e1, e2)
  e4 = Equivalence(e1,e2)
  e5 = ThereExists("x", And([Predicate("P", Function("f", Variable("x"))),
        Predicate("P", Function("g", Variable("x")))]))
  e6 = ForAll("x", And([Predicate("P", Function("f", Variable("x"))),
        Predicate("P", Function("g", Variable("x")))]))
  print("Orignal Expression:")
  print("Expression 1: ", e1)
  print("Expression 2: ", e2)
  print("Expression 3: ", e3)
  print("Expression 4: ", e4)
  print("Expression 5: ", e5)
  print("Expression 6: ", e6)

  print("\n\nNegated Expressions")
  e1.negate()
  e2.negate()
  e3.negate()
  e4.negate()
  e5.negate()
  e6.negate()
  print("Expression 7: ", e1)
  print("Expression 8: ", e2)
  print("Expression 9: ", e3)
  print("Expression 10: ", e4)
  print("Expression 11: ", e5)
  print("Expression 12: ", e6)

  print("\n\nPushed Negation Expressions")
  e13 = e1.push_negation()
  e14 = e2.push_negation()
  e15 = e3.push_negation()
  e16 = e4.push_negation()
  e17 = e5.push_negation()
  e18 = e6.push_negation()
  print("Expression 13: ", e13)
  print("Expression 14: ", e14)
  print("Expression 15: ", e15)
  print("Expression 16: ", e16)
  print("Expression 17: ", e17)
  print("Expression 18: ", e18)

  print("\n\nRemoving Equivalences")
  test1 = Equivalence(Equivalence(Equivalence(Variable('x'), Variable('x')), Variable('x')), Variable('z'))
  test2 = Or([Equivalence(Variable('x'), Variable('y')),Variable('z')])
  print(test1)
  print(remove_equivalences(test1))
  print("\n")
  print(test2)
  print(remove_equivalences(test2))

  print("\n\nRemoving Implications")
  test3 = Implication(Implication(Predicate("P", Variable('x')), Predicate("P", Variable('y'))), Predicate("P", Variable('z')))
  test4 = Implication(Implication(Implication(Variable('x'), Variable('x')), Variable('x')), Variable('z'))
  print(test3)
  print(remove_implications(test3))
  print("\n")
  print(test4)
  print(remove_implications(test4))

  print("\n\nPush Not")
  test5 = And([And([Predicate("P", Function("f", Variable("x"))), Predicate("P",
        Function("g", Variable("x")))]), Predicate("P",Function("g", Variable("x")))])
  test6 = ForAll(Variable("x"), And([Predicate("P", Function("f", Variable("x"))),
        Predicate("P", Function("g", Variable("x")))]))
  test5.negate()
  test6.negate()
  print(test5)
  print(push_nots_inwards(test5))
  print("\n")
  print(test6)
  print(push_nots_inwards(test6))
<|endoftext|>