<commit_before>import collections
import fnmatch
import re
import subprocess
from common import *

SVNADMIN = 'svnadmin'
SVNLOOK = 'svnlook'

head_rev_rx = re.compile(r'^(?=.)(?P<head>\D[^:]*)?:?(?P<rev>\d+)?$')
mergeinfo_rx = re.compile(r'^(?P<head>.+):(?P<minrev>\d+)(?:-(?P<maxrev>\d+))$')

HistoryEntry = collections.namedtuple('HistoryEntry', 'rev path')

class SvnRepo(VCSRepo):
  @classmethod
  def create(cls, path):
    cmd = [SVNADMIN, 'create', path]
    subprocess.check_call(cmd)
    return cls(path)

  @classmethod
  def cleanPath(cls, path):
    path = multislash_rx.sub('/', path)
    if not path.startswith('/'):
      path = '/' + path
    return path

  def __init__(self, path):
    super(SvnRepo, self).__init__(path)
    self.branch_glob = ['/trunk/', '/branches/*/']
    self.tag_glob = ['/tags/*/']

  def _proplist(self, rev, path):
    cmd = [SVNLOOK, 'proplist', '-r', rev, '.', path or '--revprop']
    output = self._command(cmd)
    return [x.strip() for x in output.splitlines()]

  def proplist(self, rev, path=None):
    rev, prefix = self._maprev(rev)
    if path is None:
      return self._proplist(str(rev), None)
    else:
      path = type(self).cleanPath(prefix + path)
      return self._proplist(str(rev), path)

  def _propget(self, prop, rev, path):
    cmd = [SVNLOOK, 'propget', '-r', rev, '.', prop, path or '--revprop']
    return self._command(cmd)

  def propget(self, prop, rev, path=None):
    rev, prefix = self._maprev(rev)
    if path is None:
      return self._propget(prop, str(rev), None)
    else:
      path = type(self).cleanPath(prefix + path)
      return self._propget(prop, str(rev), path)

  def _mergeinfo(self, rev, path):
    revstr = str(rev)
    if 'svn:mergeinfo' not in self._proplist(revstr, path):
      return []
    results = []
    mergeinfo = self._propget('svn:mergeinfo', revstr, path)
    for line in mergeinfo.splitlines():
      m = mergeinfo_rx.match(line)
      assert m
      head, minrev, maxrev = m.group('head', 'minrev', 'maxrev')
      minrev = int(minrev)
      maxrev = int(maxrev or minrev)
      results.append((head, minrev, maxrev))
    return results

  def _maprev(self, rev):
    if isinstance(rev, int):
      return (rev, '/')
    m = head_rev_rx.match(rev)
    assert m, 'invalid rev'
    head, rev = m.group('head', 'rev')
    if rev:
      rev = int(rev)
    else:
      rev = self.youngest()
    if head is None:
      return (rev, '')
    elif head == 'HEAD':
      return (rev, '')
    else:
      return (rev, '/' + head)

  def ls(self, rev, path, recursive=False, recursive_dirs=False,
         directory=False, report=()):
    rev, prefix = self._maprev(rev)
    path = type(self).cleanPath(prefix + path)
    forcedir = False
    if path.endswith('/'):
      forcedir = True
      if path != '/':
        path = path.rstrip('/')
    if path == '/':
      if directory:
        return [{'type':'d'}]
      ltrim = 1
      prefix = '/'
    else:
      ltrim = len(path) + 1
      prefix = path + '/'

    cmd = [SVNLOOK, 'tree', '-r', str(rev), '--full-paths']
    if not recursive:
      cmd.append('--non-recursive')
    cmd.extend(['.', path])
    p = subprocess.Popen(cmd, cwd=self.path, stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    output, stderr = p.communicate()
    if p.returncode != 0:
      if p.returncode == 1 and 'File not found' in stderr:
        raise PathDoesNotExist(rev, path)
      raise subprocess.CalledProcessError(p.returncode, cmd, stderr)

    results = []
    lines = output.splitlines()
    if forcedir and not lines[0].endswith('/'):
      raise PathDoesNotExist(rev, path)
    if lines[0].endswith('/'):
      if directory:
        lines = lines[:1]
      else:
        lines = lines[1:]
    for name in lines:
      entry_name = name[ltrim:]
      entry = attrdict()
      if name.endswith('/'):
        if recursive and not recursive_dirs:
          continue
        entry.type = 'd'
        entry_name = entry_name.rstrip('/')
      else:
        proplist = self._proplist(str(rev), name)
        if 'svn:special' in proplist:
          link = self._cat(str(rev), name).split(None, 1)
          if len(link) == 2 and link[0] == 'link':
            entry.type = 'l'
            if 'target' in report:
              entry.target = link[1]
        if 'type' not in entry:
          entry.type = 'f'
          if 'executable' in report:
            entry.executable = 'svn:executable' in proplist
          if 'size' in report:
            entry.size = len(self._cat(str(rev), name))
      if entry_name:
        entry.name = entry_name
      results.append(entry)
    return results

  def _cat(self, rev, path):
    cmd = [SVNLOOK, 'cat', '-r', rev, '.', path]
    return self._command(cmd)

  def cat(self, rev, path):
    rev, prefix = self._maprev(rev)
    path = type(self).cleanPath(prefix + path)
    ls = self.ls(rev, path, directory=True)
    assert len(ls) == 1
    if ls[0].type != 'f':
      raise BadFileType(rev, path)
    return self._cat(str(rev), path)

  def _readlink(self, rev, path):
    output = self._cat(rev, path)
    link = output.split(None, 1)
    assert len(link) == 2 and link[0] == 'link'
    return link[1]

  def readlink(self, rev, path):
    rev, prefix = self._maprev(rev)
    path = type(self).cleanPath(prefix + path)
    ls = self.ls(rev, path, directory=True)
    assert len(ls) == 1
    if ls[0].type != 'l':
      raise BadFileType(rev, path)
    return self._readlink(str(rev), path)

  def youngest(self):
    cmd = [SVNLOOK, 'youngest', '.']
    return int(self._command(cmd))

  def _heads(self, globs):
    root = {}
    for glob in globs:
      n = root
      for p in glob.strip('/').split('/'):
        n = n.setdefault(p, {})
    youngest = self.youngest()
    results = []
    def match(n, path):
      for d in self.ls(youngest, path):
        if d.type == 'd':
          for k, v in n.iteritems():
            if fnmatch.fnmatchcase(d.name, k):
              if path:
                p = path + '/' + d.name
              else:
                p = d.name
              if v:
                match(v, p)
              else:
                results.append(p)
    match(root, '')
    return results

  def branches(self):
    return self._heads(self.branch_glob)

  def tags(self):
    return self._heads(self.tag_glob)

  def heads(self):
    return ['HEAD'] + self._heads(self.branch_glob + self.tag_glob)

  def empty(self):
    cmd = [SVNLOOK, 'history', '.', '-l2']
    output = self._command(cmd)
    return len(output.splitlines()) < 4

  def log(self, revrange=None, limit=None, firstparent=False, merges=None,
          path=None, follow=False):
    if revrange is None or revrange == (None, None):
      results = []
      for rev, path in self._history(self.youngest(), '/', limit):
        results.append(self._logentry(rev, '/'))
      return results
    elif isinstance(revrange, tuple):
      if revrange[1] is None:
        include = set()
        rev1 = self.youngest()
        for head in self.heads():
          if head == 'HEAD':
            continue
          path1 = type(self).cleanPath(head)
          include.update(self._mergehistory(rev1, path1))
      else:
        rev1, path1 = self._maprev(revrange[1])
        if firstparent:
          include = self._history(rev1, path1)
        else:
          include = self._mergehistory(rev1, path1)
      if revrange[0] is None:
        results = include
      else:
        rev0, path0 = self._maprev(revrange[0])
        exclude = self._mergehistory(rev0, path0)
        results = include - exclude
      results = sorted(results, key=lambda x: x.rev, reverse=True)
      if limit is not None:
        results = results[:limit]
      return map(lambda x: self._logentry(x.rev, x.path), results)
    else:
      rev, path = self._maprev(revrange)
      h = self._history(rev, path, 1)
      rev = h[0].rev
      return self._logentry(rev, path)

  def _logentry(self, rev, path, history=None):
    revstr = str(rev)
    cmd = [SVNLOOK, 'info', '.', '-r', revstr]
    output = self._command(cmd)
    author, date, logsize, message = output.split('\n', 3)
    date = parse_isodate(date)
    if history is None:
      history = self._history(rev, path, 2)
    parents = []
    if len(history) > 1:
      prev = history[1].rev
      if path == '/':
        parents.append(prev)
      else:
        parents.append('%s:%d' % (path, prev))
      for head, minrev, maxrev in self._mergeinfo(rev, path):
        if prev < maxrev:
          h = self._history(maxrev, head, 1)
          if head == '/':
            parents.append(h[0].rev)
          else:
            parents.append('%s:%d' % (head, h[0].rev))
    return CommitLogEntry(rev, parents, date, author, message)

  def diff(self, rev_a, rev_b, path_a, path_b=None):
    raise NotImplementedError

  def _history(self, rev, path, limit=None):
    cmd = [SVNLOOK, 'history', '.', '-r', str(rev), path]
    if limit is not None:
      cmd.extend(['-l', str(limit)])
    output = self._command(cmd)
    results = []
    for line in output.splitlines()[2:]:
      r, p = line.split(None, 1)
      results.append(HistoryEntry(int(r), p))
    return results

  def _mergehistory(self, rev, path, limit=None):
    results = set(self._history(rev, path, limit))
    for head, minrev, maxrev in self._mergeinfo(rev, path):
      l = maxrev - minrev + 1
      if limit is not None:
        l = min(l, limit)
      h = self._history(maxrev, head, l)
      for r, p in h:
        if r < minrev:
          break
        results.add(HistoryEntry(r, p))
    return results

  def ancestor(self, rev1, rev2):
    rev1, prefix1 = self._maprev(rev1)
    rev2, prefix2 = self._maprev(rev2)
    prefix1 = type(self).cleanPath(prefix1)
    if prefix1 != '/':
      prefix1 = prefix1.rstrip('/')
    prefix2 = type(self).cleanPath(prefix2)
    if prefix2 != '/':
      prefix2 = prefix2.rstrip('/')

    self.ls(rev1, prefix1, directory=True)
    self.ls(rev2, prefix2, directory=True)

    minrev = min(rev1, rev2)
    if prefix1 == prefix2:
      return '%s:%d' % (prefix1, minrev)

    history1 = self._history(minrev, prefix1)
    history2 = self._history(minrev, prefix2)

    youngest = HistoryEntry(0, '/')

    for head, minrev, maxrev in self._mergeinfo(rev1, prefix1):
      for h in history2:
        if h.rev < minrev or h.rev < youngest.rev:
          break
        if h.path == head and minrev <= h.rev <= maxrev:
          youngest = h

    for head, minrev, maxrev in self._mergeinfo(rev2, prefix2):
      for h in history1:
        if h.rev < minrev or h.rev < youngest.rev:
          break
        if h.path == head and minrev <= h.rev <= maxrev:
          youngest = h

    if youngest.rev > 0:
      return '%s:%d' % (youngest.path, youngest.rev)

    i1 = 0
    i2 = 0
    len1 = len(history1)
    len2 = len(history2)
    while i1 < len1 and i2 < len2:
      if history1[i1].rev < history2[i2].rev:
        i2 += 1
      elif history1[i1].rev > history2[i2].rev:
        i1 += 1
      else:
        if history1[i1].path == history2[i2].path:
          return '%s:%d' % (history1[i1].path, history1[i1].rev)
        else:
          i1 += 1
          i2 += 1

    return None
<commit_msg>almost correctly handle SvnRepo.log(path=something)<commit_after>import collections
import fnmatch
import re
import subprocess
from common import *

SVNADMIN = 'svnadmin'
SVNLOOK = 'svnlook'

head_rev_rx = re.compile(r'^(?=.)(?P<head>\D[^:]*)?:?(?P<rev>\d+)?$')
mergeinfo_rx = re.compile(r'^(?P<head>.+):(?P<minrev>\d+)(?:-(?P<maxrev>\d+))$')
changed_copy_info_rx = re.compile(r'^    \(from (?P<path>.+):r(?P<rev>\d+)\)')

HistoryEntry = collections.namedtuple('HistoryEntry', 'rev path')
ChangeInfo = collections.namedtuple('ChangeInfo', 'status copy')

class SvnRepo(VCSRepo):
  @classmethod
  def create(cls, path):
    cmd = [SVNADMIN, 'create', path]
    subprocess.check_call(cmd)
    return cls(path)

  @classmethod
  def cleanPath(cls, path):
    path = multislash_rx.sub('/', path)
    if not path.startswith('/'):
      path = '/' + path
    return path

  def __init__(self, path):
    super(SvnRepo, self).__init__(path)
    self.branch_glob = ['/trunk/', '/branches/*/']
    self.tag_glob = ['/tags/*/']

  def _proplist(self, rev, path):
    cmd = [SVNLOOK, 'proplist', '-r', rev, '.', path or '--revprop']
    output = self._command(cmd)
    return [x.strip() for x in output.splitlines()]

  def proplist(self, rev, path=None):
    rev, prefix = self._maprev(rev)
    if path is None:
      return self._proplist(str(rev), None)
    else:
      path = type(self).cleanPath(prefix + path)
      return self._proplist(str(rev), path)

  def _propget(self, prop, rev, path):
    cmd = [SVNLOOK, 'propget', '-r', rev, '.', prop, path or '--revprop']
    return self._command(cmd)

  def propget(self, prop, rev, path=None):
    rev, prefix = self._maprev(rev)
    if path is None:
      return self._propget(prop, str(rev), None)
    else:
      path = type(self).cleanPath(prefix + path)
      return self._propget(prop, str(rev), path)

  def _mergeinfo(self, rev, path):
    revstr = str(rev)
    if 'svn:mergeinfo' not in self._proplist(revstr, path):
      return []
    results = []
    mergeinfo = self._propget('svn:mergeinfo', revstr, path)
    for line in mergeinfo.splitlines():
      m = mergeinfo_rx.match(line)
      assert m
      head, minrev, maxrev = m.group('head', 'minrev', 'maxrev')
      minrev = int(minrev)
      maxrev = int(maxrev or minrev)
      results.append((head, minrev, maxrev))
    return results

  def _maprev(self, rev):
    if isinstance(rev, int):
      return (rev, '/')
    m = head_rev_rx.match(rev)
    assert m, 'invalid rev'
    head, rev = m.group('head', 'rev')
    if rev:
      rev = int(rev)
    else:
      rev = self.youngest()
    if head is None:
      return (rev, '')
    elif head == 'HEAD':
      return (rev, '')
    else:
      return (rev, '/' + head)

  def ls(self, rev, path, recursive=False, recursive_dirs=False,
         directory=False, report=()):
    rev, prefix = self._maprev(rev)
    path = type(self).cleanPath(prefix + path)
    forcedir = False
    if path.endswith('/'):
      forcedir = True
      if path != '/':
        path = path.rstrip('/')
    if path == '/':
      if directory:
        return [{'type':'d'}]
      ltrim = 1
      prefix = '/'
    else:
      ltrim = len(path) + 1
      prefix = path + '/'

    cmd = [SVNLOOK, 'tree', '-r', str(rev), '--full-paths']
    if not recursive:
      cmd.append('--non-recursive')
    cmd.extend(['.', path])
    p = subprocess.Popen(cmd, cwd=self.path, stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    output, stderr = p.communicate()
    if p.returncode != 0:
      if p.returncode == 1 and 'File not found' in stderr:
        raise PathDoesNotExist(rev, path)
      raise subprocess.CalledProcessError(p.returncode, cmd, stderr)

    results = []
    lines = output.splitlines()
    if forcedir and not lines[0].endswith('/'):
      raise PathDoesNotExist(rev, path)
    if lines[0].endswith('/'):
      if directory:
        lines = lines[:1]
      else:
        lines = lines[1:]
    for name in lines:
      entry_name = name[ltrim:]
      entry = attrdict()
      if name.endswith('/'):
        if recursive and not recursive_dirs:
          continue
        entry.type = 'd'
        entry_name = entry_name.rstrip('/')
      else:
        proplist = self._proplist(str(rev), name)
        if 'svn:special' in proplist:
          link = self._cat(str(rev), name).split(None, 1)
          if len(link) == 2 and link[0] == 'link':
            entry.type = 'l'
            if 'target' in report:
              entry.target = link[1]
        if 'type' not in entry:
          entry.type = 'f'
          if 'executable' in report:
            entry.executable = 'svn:executable' in proplist
          if 'size' in report:
            entry.size = len(self._cat(str(rev), name))
      if entry_name:
        entry.name = entry_name
      results.append(entry)
    return results

  def _cat(self, rev, path):
    cmd = [SVNLOOK, 'cat', '-r', rev, '.', path]
    return self._command(cmd)

  def cat(self, rev, path):
    rev, prefix = self._maprev(rev)
    path = type(self).cleanPath(prefix + path)
    ls = self.ls(rev, path, directory=True)
    assert len(ls) == 1
    if ls[0].type != 'f':
      raise BadFileType(rev, path)
    return self._cat(str(rev), path)

  def _readlink(self, rev, path):
    output = self._cat(rev, path)
    link = output.split(None, 1)
    assert len(link) == 2 and link[0] == 'link'
    return link[1]

  def readlink(self, rev, path):
    rev, prefix = self._maprev(rev)
    path = type(self).cleanPath(prefix + path)
    ls = self.ls(rev, path, directory=True)
    assert len(ls) == 1
    if ls[0].type != 'l':
      raise BadFileType(rev, path)
    return self._readlink(str(rev), path)

  def youngest(self):
    cmd = [SVNLOOK, 'youngest', '.']
    return int(self._command(cmd))

  def _heads(self, globs):
    root = {}
    for glob in globs:
      n = root
      for p in glob.strip('/').split('/'):
        n = n.setdefault(p, {})
    youngest = self.youngest()
    results = []
    def match(n, path):
      for d in self.ls(youngest, path):
        if d.type == 'd':
          for k, v in n.iteritems():
            if fnmatch.fnmatchcase(d.name, k):
              if path:
                p = path + '/' + d.name
              else:
                p = d.name
              if v:
                match(v, p)
              else:
                results.append(p)
    match(root, '')
    return results

  def branches(self):
    return self._heads(self.branch_glob)

  def tags(self):
    return self._heads(self.tag_glob)

  def heads(self):
    return ['HEAD'] + self._heads(self.branch_glob + self.tag_glob)

  def empty(self):
    cmd = [SVNLOOK, 'history', '.', '-l2']
    output = self._command(cmd)
    return len(output.splitlines()) < 4

  def log(self, revrange=None, limit=None, firstparent=False, merges=None,
          path=None, follow=False):
    if revrange is None or revrange == (None, None):
      results = []
      for rev, prefix in self._history(self.youngest(), '/', limit):
        results.append(self._logentry(rev, prefix))
      return results
    elif isinstance(revrange, tuple):
      path_filter = None
      if revrange[1] is None:
        include = set()
        rev1 = self.youngest()
        for head in self.heads():
          if head == 'HEAD':
            continue
          prefix1 = type(self).cleanPath(head)
          include.update(self._mergehistory(rev1, prefix1))
      else:
        rev1, prefix1 = self._maprev(revrange[1])
        if firstparent:
          include = self._history(rev1, prefix1)
        else:
          include = self._mergehistory(rev1, prefix1)
        if path is not None:
          path_filter = set([(type(self).cleanPath(prefix1), 0, rev1)])
      if revrange[0] is None:
        results = include
      else:
        rev0, prefix0 = self._maprev(revrange[0])
        exclude = self._mergehistory(rev0, prefix0)
        results = include - exclude
      results = sorted(results, key=lambda x: x.rev, reverse=True)
      if path_filter is not None:
        path = type(self).cleanPath(path)
        _results = []
        for r in results:
          changed = self._changed(r.rev)
          merge = set()
          for prefix, minrev, maxrev in path_filter:
            if minrev <= r.rev <= maxrev:
              fullpath = prefix + path
              if fullpath in changed:
                merges = self._mergeinfo(r.rev, prefix)
                merge.update(merges)
                _results.append(r)
                continue
          path_filter.update(merge)
        results = _results
      if limit is not None:
        results = results[:limit]
      return map(lambda x: self._logentry(x.rev, x.path), results)
    else:
      rev, prefix = self._maprev(revrange)
      h = self._history(rev, prefix, 1)
      rev = h[0].rev
      return self._logentry(rev, prefix)

  def _logentry(self, rev, path, history=None):
    revstr = str(rev)
    cmd = [SVNLOOK, 'info', '.', '-r', revstr]
    output = self._command(cmd)
    author, date, logsize, message = output.split('\n', 3)
    date = parse_isodate(date)
    if history is None:
      history = self._history(rev, path, 2)
    parents = []
    if len(history) > 1:
      prev = history[1].rev
      if path == '/':
        parents.append(prev)
      else:
        parents.append('%s:%d' % (path, prev))
      for head, minrev, maxrev in self._mergeinfo(rev, path):
        if prev < maxrev:
          h = self._history(maxrev, head, 1)
          if head == '/':
            parents.append(h[0].rev)
          else:
            parents.append('%s:%d' % (head, h[0].rev))
    return CommitLogEntry(rev, parents, date, author, message)

  def diff(self, rev_a, rev_b, path_a, path_b=None):
    raise NotImplementedError

  def _changed(self, rev):
    cmd = [SVNLOOK, 'changed', '.', '-r', str(rev), '--copy-info']
    output = self._command(cmd)
    lines = output.splitlines()
    lines.reverse()
    results = {}
    while lines:
      line = lines.pop()
      status = line[:3]
      path = '/' + line[4:].lstrip('/')
      copy = None
      if status.endswith('+'):
        line = lines.pop()
        m = changed_copy_info_rx.match(line)
        assert m
        rev, p = m.group('rev', 'path')
        p = '/' + path.lstrip('/')
        copy = HistoryEntry(int(rev), p)
      results[path] = ChangeInfo(status, copy)
    return results

  def _history(self, rev, path, limit=None):
    cmd = [SVNLOOK, 'history', '.', '-r', str(rev), path]
    if limit is not None:
      cmd.extend(['-l', str(limit)])
    output = self._command(cmd)
    results = []
    for line in output.splitlines()[2:]:
      r, p = line.split(None, 1)
      results.append(HistoryEntry(int(r), p))
    return results

  def _mergehistory(self, rev, path, limit=None):
    results = set(self._history(rev, path, limit))
    for head, minrev, maxrev in self._mergeinfo(rev, path):
      l = maxrev - minrev + 1
      if limit is not None:
        l = min(l, limit)
      h = self._history(maxrev, head, l)
      for r, p in h:
        if r < minrev:
          break
        results.add(HistoryEntry(r, p))
    return results

  def ancestor(self, rev1, rev2):
    rev1, prefix1 = self._maprev(rev1)
    rev2, prefix2 = self._maprev(rev2)
    prefix1 = type(self).cleanPath(prefix1)
    if prefix1 != '/':
      prefix1 = prefix1.rstrip('/')
    prefix2 = type(self).cleanPath(prefix2)
    if prefix2 != '/':
      prefix2 = prefix2.rstrip('/')

    self.ls(rev1, prefix1, directory=True)
    self.ls(rev2, prefix2, directory=True)

    minrev = min(rev1, rev2)
    if prefix1 == prefix2:
      return '%s:%d' % (prefix1, minrev)

    history1 = self._history(minrev, prefix1)
    history2 = self._history(minrev, prefix2)

    youngest = HistoryEntry(0, '/')

    for head, minrev, maxrev in self._mergeinfo(rev1, prefix1):
      for h in history2:
        if h.rev < minrev or h.rev < youngest.rev:
          break
        if h.path == head and minrev <= h.rev <= maxrev:
          youngest = h

    for head, minrev, maxrev in self._mergeinfo(rev2, prefix2):
      for h in history1:
        if h.rev < minrev or h.rev < youngest.rev:
          break
        if h.path == head and minrev <= h.rev <= maxrev:
          youngest = h

    if youngest.rev > 0:
      return '%s:%d' % (youngest.path, youngest.rev)

    i1 = 0
    i2 = 0
    len1 = len(history1)
    len2 = len(history2)
    while i1 < len1 and i2 < len2:
      if history1[i1].rev < history2[i2].rev:
        i2 += 1
      elif history1[i1].rev > history2[i2].rev:
        i1 += 1
      else:
        if history1[i1].path == history2[i2].path:
          return '%s:%d' % (history1[i1].path, history1[i1].rev)
        else:
          i1 += 1
          i2 += 1

    return None
<|endoftext|>


<commit_before>import re
import subprocess
from common import *

SVNADMIN = 'svnadmin'
SVNLOOK = 'svnlook'

head_rev_rx = re.compile(r'^(?=.)(?P<head>\D[^:]*)?:?(?P<rev>\d+)?$')

class SvnRepo(VCSRepo):
  @classmethod
  def create(cls, path):
    cmd = [SVNADMIN, 'create', path]
    subprocess.check_call(cmd)
    return cls(path)

  @classmethod
  def cleanPath(cls, path):
    path = multislash_rx.sub('/', path)
    if not path.startswith('/'):
      path = '/' + path
    return path

  def _proplist(self, rev, path):
    cmd = [SVNLOOK, 'proplist', '-r', rev, '.', path]
    output = self._command(cmd)
    return [x.strip() for x in output.splitlines()]

  def _maprev(self, rev, path):
    if isinstance(rev, int):
      return (rev, path)
    m = head_rev_rx.match(rev)
    assert m, 'invalid rev'
    head, rev = m.group('head', 'rev')
    if rev:
      rev = int(rev)
    else:
      rev = self.youngest()
    if head is None:
      return (rev, path)
    elif head == 'HEAD':
      return (rev, path)
    else:
      return (rev, '/' + head + '/' + path)

  def ls(self, rev, path, recursive=False, recursive_dirs=False,
         directory=False, report=()):
    rev, path = self._maprev(rev, path)
    path = type(self).cleanPath(path)
    forcedir = False
    if path.endswith('/'):
      forcedir = True
      if path != '/':
        path = path.rstrip('/')
    if path == '/':
      if directory:
        return [{'type':'d'}]
      ltrim = 1
      prefix = '/'
    else:
      ltrim = len(path) + 1
      prefix = path + '/'

    cmd = [SVNLOOK, 'tree', '-r', str(rev), '--full-paths']
    if not recursive:
      cmd.append('--non-recursive')
    cmd.extend(['.', path])
    p = subprocess.Popen(cmd, cwd=self.path, stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    output, stderr = p.communicate()
    if p.returncode != 0:
      if p.returncode == 1 and 'File not found' in stderr:
        raise PathDoesNotExist(rev, path)
      raise subprocess.CalledProcessError(p.returncode, cmd, stderr)

    results = []
    lines = output.splitlines()
    if forcedir and not lines[0].endswith('/'):
      raise PathDoesNotExist(rev, path)
    if lines[0].endswith('/'):
      if directory:
        lines = lines[:1]
      else:
        lines = lines[1:]
    for name in lines:
      entry_name = name[ltrim:]
      entry = attrdict()
      if name.endswith('/'):
        if recursive and not recursive_dirs:
          continue
        entry.type = 'd'
        entry_name = entry_name.rstrip('/')
      else:
        proplist = self._proplist(str(rev), name)
        if 'svn:special' in proplist:
          link = self._cat(str(rev), name).split(None, 1)
          if len(link) == 2 and link[0] == 'link':
            entry.type = 'l'
            if 'target' in report:
              entry.target = link[1]
        if 'type' not in entry:
          entry.type = 'f'
          if 'executable' in report:
            entry.executable = 'svn:executable' in proplist
          if 'size' in report:
            entry.size = len(self._cat(str(rev), name))
      if entry_name:
        entry.name = entry_name
      results.append(entry)
    return results

  def _cat(self, rev, path):
    cmd = [SVNLOOK, 'cat', '-r', rev, '.', path]
    return self._command(cmd)

  def cat(self, rev, path):
    rev, path = self._maprev(rev, path)
    path = type(self).cleanPath(path)
    ls = self.ls(rev, path, directory=True)
    assert len(ls) == 1
    if ls[0].type != 'f':
      raise BadFileType(rev, path)
    return self._cat(str(rev), path)

  def _readlink(self, rev, path):
    output = self._cat(rev, path)
    link = output.split(None, 1)
    assert len(link) == 2 and link[0] == 'link'
    return link[1]

  def readlink(self, rev, path):
    rev, path = self._maprev(rev, path)
    path = type(self).cleanPath(path)
    ls = self.ls(rev, path, directory=True)
    assert len(ls) == 1
    if ls[0].type != 'l':
      raise BadFileType(rev, path)
    return self._readlink(str(rev), path)

  def youngest(self):
    cmd = [SVNLOOK, 'youngest', '.']
    return int(self._command(cmd))

  def _heads(self, headdirs):
    youngest = self.youngest()
    root = self.ls(youngest, '/')
    if {'name':'trunk', 'type':'d'} in root:
      roots = [('', root)]
    else:
      roots = []
      for d in (x.name for x in filter(lambda x: x.type == 'd', root)):
        droot = self.ls(youngest, '/' + d + '/')
        if {'name':'trunk', 'type':'d'} in droot:
          roots.append((d+'/', droot))
    results = []
    for prefix, root in roots:
      results.append(prefix + 'trunk')
      for d in headdirs:
        if {'name':d, 'type':'d'} in root:
          dls = self.ls(self.youngest(), '/' + prefix + d + '/')
          results.extend(prefix + d + '/' + x.name
                         for x in filter(lambda x: x.type == 'd', dls))
    return results

  def branches(self):
    return self._heads(('branches',))

  def tags(self):
    return self._heads(('tags',))

  def heads(self):
    return ['HEAD'] + self._heads(('branches', 'tags'))

  def log(self, revrange=None, path=None, follow=False, firstparent=False,
          branchlog=False, limit=None):
    if revrange is None:
      revrange = (None, None)
    if isinstance(revrange, tuple):
      if revrange[0] is None:
        startrev = None
      else:
        startrev = int(revrange[0])
      if revrange[1] is None:
        endrev = self.youngest()
      else:
        endrev = int(revrange[1])
      cmd = [SVNLOOK, 'history', '.', '-r', str(endrev)]
      if limit is not None:
        cmd.extend(['-l', str(limit)])
      elif startrev is not None:
        cmd.extend(['-l', str(endrev - startrev)])
      output = self._command(cmd)

      revs = []
      consume = endrev is None
      for line in output.splitlines()[2:]:
        rev = int(line.split()[0])
        if not consume and rev != endrev:
          continue
        consume = True
        if rev == startrev:
          break
        revs.append(rev)
    else:
      revs = [int(revrange)]

    results = []
    for rev in revs:
      cmd = [SVNLOOK, 'info', '.', '-r', str(rev)]
      output = self._command(cmd)
      author, date, logsize, message = output.split('\n', 3)
      if rev == 0:
        parents = []
      else:
        parents = [rev - 1]
      date = parse_isodate(date)
      subject = message.split('\n', 1)[0]
      entry = CommitLogEntry(rev, parents, date, author, subject)
      results.append(entry)
    return results

  def diff(self, rev_a, rev_b, path_a, path_b=None):
    raise NotImplementedError
<commit_msg>fix signature of SvnRepo.log<commit_after>import re
import subprocess
from common import *

SVNADMIN = 'svnadmin'
SVNLOOK = 'svnlook'

head_rev_rx = re.compile(r'^(?=.)(?P<head>\D[^:]*)?:?(?P<rev>\d+)?$')

class SvnRepo(VCSRepo):
  @classmethod
  def create(cls, path):
    cmd = [SVNADMIN, 'create', path]
    subprocess.check_call(cmd)
    return cls(path)

  @classmethod
  def cleanPath(cls, path):
    path = multislash_rx.sub('/', path)
    if not path.startswith('/'):
      path = '/' + path
    return path

  def _proplist(self, rev, path):
    cmd = [SVNLOOK, 'proplist', '-r', rev, '.', path]
    output = self._command(cmd)
    return [x.strip() for x in output.splitlines()]

  def _maprev(self, rev, path):
    if isinstance(rev, int):
      return (rev, path)
    m = head_rev_rx.match(rev)
    assert m, 'invalid rev'
    head, rev = m.group('head', 'rev')
    if rev:
      rev = int(rev)
    else:
      rev = self.youngest()
    if head is None:
      return (rev, path)
    elif head == 'HEAD':
      return (rev, path)
    else:
      return (rev, '/' + head + '/' + path)

  def ls(self, rev, path, recursive=False, recursive_dirs=False,
         directory=False, report=()):
    rev, path = self._maprev(rev, path)
    path = type(self).cleanPath(path)
    forcedir = False
    if path.endswith('/'):
      forcedir = True
      if path != '/':
        path = path.rstrip('/')
    if path == '/':
      if directory:
        return [{'type':'d'}]
      ltrim = 1
      prefix = '/'
    else:
      ltrim = len(path) + 1
      prefix = path + '/'

    cmd = [SVNLOOK, 'tree', '-r', str(rev), '--full-paths']
    if not recursive:
      cmd.append('--non-recursive')
    cmd.extend(['.', path])
    p = subprocess.Popen(cmd, cwd=self.path, stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    output, stderr = p.communicate()
    if p.returncode != 0:
      if p.returncode == 1 and 'File not found' in stderr:
        raise PathDoesNotExist(rev, path)
      raise subprocess.CalledProcessError(p.returncode, cmd, stderr)

    results = []
    lines = output.splitlines()
    if forcedir and not lines[0].endswith('/'):
      raise PathDoesNotExist(rev, path)
    if lines[0].endswith('/'):
      if directory:
        lines = lines[:1]
      else:
        lines = lines[1:]
    for name in lines:
      entry_name = name[ltrim:]
      entry = attrdict()
      if name.endswith('/'):
        if recursive and not recursive_dirs:
          continue
        entry.type = 'd'
        entry_name = entry_name.rstrip('/')
      else:
        proplist = self._proplist(str(rev), name)
        if 'svn:special' in proplist:
          link = self._cat(str(rev), name).split(None, 1)
          if len(link) == 2 and link[0] == 'link':
            entry.type = 'l'
            if 'target' in report:
              entry.target = link[1]
        if 'type' not in entry:
          entry.type = 'f'
          if 'executable' in report:
            entry.executable = 'svn:executable' in proplist
          if 'size' in report:
            entry.size = len(self._cat(str(rev), name))
      if entry_name:
        entry.name = entry_name
      results.append(entry)
    return results

  def _cat(self, rev, path):
    cmd = [SVNLOOK, 'cat', '-r', rev, '.', path]
    return self._command(cmd)

  def cat(self, rev, path):
    rev, path = self._maprev(rev, path)
    path = type(self).cleanPath(path)
    ls = self.ls(rev, path, directory=True)
    assert len(ls) == 1
    if ls[0].type != 'f':
      raise BadFileType(rev, path)
    return self._cat(str(rev), path)

  def _readlink(self, rev, path):
    output = self._cat(rev, path)
    link = output.split(None, 1)
    assert len(link) == 2 and link[0] == 'link'
    return link[1]

  def readlink(self, rev, path):
    rev, path = self._maprev(rev, path)
    path = type(self).cleanPath(path)
    ls = self.ls(rev, path, directory=True)
    assert len(ls) == 1
    if ls[0].type != 'l':
      raise BadFileType(rev, path)
    return self._readlink(str(rev), path)

  def youngest(self):
    cmd = [SVNLOOK, 'youngest', '.']
    return int(self._command(cmd))

  def _heads(self, headdirs):
    youngest = self.youngest()
    root = self.ls(youngest, '/')
    if {'name':'trunk', 'type':'d'} in root:
      roots = [('', root)]
    else:
      roots = []
      for d in (x.name for x in filter(lambda x: x.type == 'd', root)):
        droot = self.ls(youngest, '/' + d + '/')
        if {'name':'trunk', 'type':'d'} in droot:
          roots.append((d+'/', droot))
    results = []
    for prefix, root in roots:
      results.append(prefix + 'trunk')
      for d in headdirs:
        if {'name':d, 'type':'d'} in root:
          dls = self.ls(self.youngest(), '/' + prefix + d + '/')
          results.extend(prefix + d + '/' + x.name
                         for x in filter(lambda x: x.type == 'd', dls))
    return results

  def branches(self):
    return self._heads(('branches',))

  def tags(self):
    return self._heads(('tags',))

  def heads(self):
    return ['HEAD'] + self._heads(('branches', 'tags'))

  def log(self, revrange=None, limit=None, branchlog=False, firstparent=False,
          merges=None, path=None, follow=False):
    if revrange is None:
      revrange = (None, None)
    if isinstance(revrange, tuple):
      if revrange[0] is None:
        startrev = None
      else:
        startrev = int(revrange[0])
      if revrange[1] is None:
        endrev = self.youngest()
      else:
        endrev = int(revrange[1])
      cmd = [SVNLOOK, 'history', '.', '-r', str(endrev)]
      if limit is not None:
        cmd.extend(['-l', str(limit)])
      elif startrev is not None:
        cmd.extend(['-l', str(endrev - startrev)])
      output = self._command(cmd)

      revs = []
      consume = endrev is None
      for line in output.splitlines()[2:]:
        rev = int(line.split()[0])
        if not consume and rev != endrev:
          continue
        consume = True
        if rev == startrev:
          break
        revs.append(rev)
    else:
      revs = [int(revrange)]

    results = []
    for rev in revs:
      cmd = [SVNLOOK, 'info', '.', '-r', str(rev)]
      output = self._command(cmd)
      author, date, logsize, message = output.split('\n', 3)
      if rev == 0:
        parents = []
      else:
        parents = [rev - 1]
      date = parse_isodate(date)
      subject = message.split('\n', 1)[0]
      entry = CommitLogEntry(rev, parents, date, author, subject)
      results.append(entry)
    return results

  def diff(self, rev_a, rev_b, path_a, path_b=None):
    raise NotImplementedError
<|endoftext|>


<commit_before>"""The setup plugin: when the meta-problem counter reaches a high level"""
import pbt
import os
from glob import glob
from os.path import join, dirname, basename
from collections import defaultdict
from xdg.BaseDirectory import save_data_path, load_data_paths
import time

def test_or_compiled_file(path):
    return (('__pycache__' in path) or
            path[-4:] == '.pyc' or
            path[-4:] == '.pyo' or
            basename(path)[:5] == 'test_')

def only_interesting_files(l):
    """Weeds out all compiled modules and test files."""
    return filter(lambda path: not test_or_compiled_file(path), l)

def recursive(root, spec):
    files = defaultdict(lambda: [])
    ans = []

    for path in only_interesting_files(glob(spec)):
        base = join(root, dirname(path))

        files[base].append(path)

    for i in files.items():
        ans.append(i)

    return ans

def writeln(file, line=""):
    "Pascal's turning in its grave."
    file.write(line)
    file.write(os.linesep)

@pbt.command(name="setup")
def main(ctx, args, p):
    """Generates a valid setup.py file.
    It uses the info found in project.pbt."""

    # NOTE: yes, short names, 'cause I'm gonna write them a gazillion times
    f = open('setup.py', 'w+')
    s = p.settings

    # TODO: support other major versions
    writeln(f, "#! /usr/bin/env python3")
    writeln(f)
    writeln(f, "# file generated by pbt.plugins.setup on %s" %
            time.strftime("%a, %d %b %Y %H:%M:%S %z"))
    writeln(f)
    writeln(f, "from distutils.core import setup")
    writeln(f)
    writeln(f, "setup(")

    # lazy, me?
    def kv(k, v):
        writeln(f, "    %s='%s'," % (k, v))

    def kvs(k, vs):
        writeln(f, "    %s=%s," % (k, vs))

    kv('name', p.name)
    kv('version', p.version)
    kv('description', p.description)
    # TODO: author
    kv('url', p.url)
    kvs('packages', s.packages)
    # TODO: run the plugin that generates the binary from the entry point
    kvs('scripts', s.scripts)
    kv('license', p.license['name'])
    # TODO: classifiers
    # TODO: requires

    # plugins
    # according to the Debian Python Policy, they should go in
    # /usr/share/<program>
    # NOTE: for the time being, we're gonna install them in the user's xdg.data
    # dir

    # first, make sure the dir exists
    # save_data_path("pbt")

    # the user's dir comes first
    # plugins = recursive(next(load_data_paths('pbt')), 'plugins/*/*')
    # kvs('data_files', plugins)

    writeln(f, "    )")
    f.close()

    print("setup.py file written.")
<commit_msg>cleanup after the output of pbt check, setp 3.1: the setup plugin.<commit_after>"""The setup plugin: when the meta-problem counter reaches a high level"""
import pbt
import os
from glob import glob
from os.path import join, dirname, basename
from collections import defaultdict
import time

def test_or_compiled_file(path):
    return (('__pycache__' in path) or
            path[-4:] == '.pyc' or
            path[-4:] == '.pyo' or
            basename(path)[:5] == 'test_')

def only_interesting_files(l):
    """Weeds out all compiled modules and test files."""
    return filter(lambda path: not test_or_compiled_file(path), l)

def recursive(root, spec):
    files = defaultdict(lambda: [])
    ans = []

    for path in only_interesting_files(glob(spec)):
        base = join(root, dirname(path))

        files[base].append(path)

    for i in files.items():
        ans.append(i)

    return ans

def writeln(file, line=""):
    "Pascal's turning in its grave."
    file.write(line)
    file.write(os.linesep)

@pbt.command(name="setup")
def main(ctx, args, p):
    """Generates a valid setup.py file.
    It uses the info found in project.pbt."""

    # NOTE: yes, short names, 'cause I'm gonna write them a gazillion times
    f = open('setup.py', 'w+')
    s = p.settings

    # TODO: support other major versions
    writeln(f, "#! /usr/bin/env python3")
    writeln(f)
    writeln(f, "# file generated by pbt.plugins.setup on %s" %
            time.strftime("%a, %d %b %Y %H:%M:%S %z"))
    writeln(f)
    writeln(f, "from distutils.core import setup")
    writeln(f)
    writeln(f, "setup(")

    # lazy, me?
    def kv(k, v):
        writeln(f, "    %s='%s'," % (k, v))

    def kvs(k, vs):
        writeln(f, "    %s=%s," % (k, vs))

    kv('name', p.name)
    kv('version', p.version)
    kv('description', p.description)
    # TODO: author
    kv('url', p.url)
    kvs('packages', s.packages)
    # TODO: run the plugin that generates the binary from the entry point
    kvs('scripts', s.scripts)
    kv('license', p.license['name'])
    # TODO: classifiers
    # TODO: requires

    # plugins
    # according to the Debian Python Policy, they should go in
    # /usr/share/<program>
    # NOTE: for the time being, we're gonna install them in the user's xdg.data
    # dir

    # first, make sure the dir exists
    # save_data_path("pbt")

    # the user's dir comes first
    # plugins = recursive(next(load_data_paths('pbt')), 'plugins/*/*')
    # kvs('data_files', plugins)

    writeln(f, "    )")
    f.close()

    print("setup.py file written.")
<|endoftext|>


<commit_before>#!/usr/bin/python -tt
# vim:set ts=4 sw=4 expandtab:
# NodeManager plugin to empower slivers to make API calls

"""
Sliver authentication support for NodeManager.

"""

import errno
import os
import random
import string
import tempfile

import logger
import tools

def start(options, conf):
    logger.log("sliverauth plugin starting up...")

def SetSliverTag(plc, slice, tagname, value):
    node_id = tools.node_id()
    slivertags=plc.GetSliceTags({"name":slice,"node_id":node_id})
    if len(slivertags)==0:
        slivertag_id=plc.AddSliceTag(slice,tagname,value,node_id)
    else:
        slivertag_id=slivertags[0]['slice_tag_id']
        plc.UpdateSliceTag(slivertag_id,value)

def GetSlivers(plc, data, conf):
    if 'slivers' not in data:
        logger.log("sliverauth: getslivers data lack's sliver information. IGNORING!")
        return

    random.seed(42)
    for sliver in data['slivers']:
        found_hmac = False
        for attribute in sliver['attributes']:
            name = attribute.get('tagname',attribute.get('name',''))
            if name == 'hmac':
                found_hmac = True
                hmac = attribute['value']
                break

        if not found_hmac:
            d = [random.choice(string.letters) for x in xrange(32)]
            hmac = "".join(d)
            SetSliverTag(plc,sliver['name'],'hmac',hmac)

        path = '/vservers/%s/etc/planetlab' % sliver['name']
        if os.path.exists(path):
            keyfile = '%s/key' % path
            oldhmac = ''
            if os.path.exists(keyfile):
                f = open(keyfile,'r')
                oldhmac = f.read()
                f.close()

            if oldhmac <> hmac:
                # create a temporary file in the vserver
                fd, name = tempfile.mkstemp('','key',path)
                os.write(fd,hmac)
                os.close(fd)
                if os.path.exists(keyfile):
                    os.unlink(keyfile)
                os.rename(name,keyfile)

            os.chmod(keyfile,0400)

<commit_msg>Disabling sliverauth<commit_after>#!/usr/bin/python -tt
# vim:set ts=4 sw=4 expandtab:
# NodeManager plugin to empower slivers to make API calls

"""
Sliver authentication support for NodeManager.

"""

import errno
import os
import random
import string
import tempfile

import logger
import tools

def start(options, conf):
    # XXX REMOVE ME
    return

    logger.log("sliverauth plugin starting up...")

def SetSliverTag(plc, slice, tagname, value):
    node_id = tools.node_id()
    slivertags=plc.GetSliceTags({"name":slice,"node_id":node_id})
    if len(slivertags)==0:
        slivertag_id=plc.AddSliceTag(slice,tagname,value,node_id)
    else:
        slivertag_id=slivertags[0]['slice_tag_id']
        plc.UpdateSliceTag(slivertag_id,value)

def GetSlivers(plc, data, conf):
    # XXX REMOVE ME
    logger.log("sliverauth: DISABLED!")
    return

    if 'slivers' not in data:
        logger.log("sliverauth: getslivers data lack's sliver information. IGNORING!")
        return

    random.seed(42)
    for sliver in data['slivers']:
        found_hmac = False
        for attribute in sliver['attributes']:
            name = attribute.get('tagname',attribute.get('name',''))
            if name == 'hmac':
                found_hmac = True
                hmac = attribute['value']
                break

        if not found_hmac:
            d = [random.choice(string.letters) for x in xrange(32)]
            hmac = "".join(d)
            SetSliverTag(plc,sliver['name'],'hmac',hmac)

        path = '/vservers/%s/etc/planetlab' % sliver['name']
        if os.path.exists(path):
            keyfile = '%s/key' % path
            oldhmac = ''
            if os.path.exists(keyfile):
                f = open(keyfile,'r')
                oldhmac = f.read()
                f.close()

            if oldhmac <> hmac:
                # create a temporary file in the vserver
                fd, name = tempfile.mkstemp('','key',path)
                os.write(fd,hmac)
                os.close(fd)
                if os.path.exists(keyfile):
                    os.unlink(keyfile)
                os.rename(name,keyfile)

            os.chmod(keyfile,0400)

<|endoftext|>


<commit_before>"""
This file demonstrates writing tests using the unittest module. These will pass
when you run "manage.py test".

Replace this with more appropriate tests for your application.
"""

from django.test import TestCase

class SimpleTest(TestCase):
    def test_basic_addition(self):
        """
        Tests that 1 + 1 always equals 2.
        """
        self.assertEqual(1 + 1, 2)
<commit_msg>Update test cases and find a bug<commit_after>"""
This file demonstrates writing tests using the unittest module. These will pass
when you run "manage.py test".

Replace this with more appropriate tests for your application.
"""

import datetime

from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.utils import timezone
from django.test import TestCase

from coffee_capsules.models import Capsule, Purchase, PurchaseItem, Request

class SimpleTest(TestCase):
    def test_basic_addition(self):
        """
        Tests that 1 + 1 always equals 2.
        """
        self.assertEqual(1 + 1, 2)

def create_capsule():
    Capsule.objects.create(name='C0', price=1000)
    Capsule.objects.create(name='C1', price=1000)
    Capsule.objects.create(name='C2', price=1000)
    Capsule.objects.create(name='C3', price=1000)
    Capsule.objects.create(name='C4', price=1000)

def create_purchase(name):
    purchase = Purchase.objects.create(
        name=name,
        begin_date=timezone.now(),
        end_date=(timezone.now() + datetime.timedelta(days=7)),
        g_unit=10, p_unit=50, u_unit=10,
    )
    all_capsule_list = Capsule.objects.order_by('pk')
    for capsule in all_capsule_list:
        pi = PurchaseItem.objects.create(
            purchase=purchase,
            capsule=capsule,
            price=capsule.price
        )
    # rof
    return purchase

class RequestTest(TestCase):
    def test_bulk_request0(self):
        """
        Test #0
        """
        create_capsule()
        purchase = create_purchase("test0")
        user = User.objects.create()
        pi_list = purchase.purchaseitem_set.all()
        self.assertEqual(len(pi_list), 5)
        request = Request(
            purchaseitem=pi_list[4],
            user=user,
            quantity_queued=10
        )
        request.save()
        request = Request(
            purchaseitem=pi_list[0],
            user=user,
            quantity_queued=10
        )
        request.save()
        request = Request(
            purchaseitem=pi_list[1],
            user=user,
            quantity_queued=10
        )
        request.save()
        request = Request(
            purchaseitem=pi_list[2],
            user=user,
            quantity_queued=10
        )
        request.save()
        request = Request(
            purchaseitem=pi_list[3],
            user=user,
            quantity_queued=10
        )
        request.save()
        pi_list = purchase.purchaseitem_set.all()  # diff with test1
        request = Request(
            purchaseitem=pi_list[4],
            user=user,
            quantity_queued=10
        )
        request.save()
        self.assertEqual(pi_list[4].quantity_accepted, 10)
        self.assertEqual(pi_list[4].quantity_grouped, 10)
        self.assertEqual(pi_list[4].quantity_queued, 0)

    def test_bulk_request1(self):
        """
        Test #1
        """
        create_capsule()
        purchase = create_purchase("test0")
        user = User.objects.create()
        pi_list = purchase.purchaseitem_set.all()
        self.assertEqual(len(pi_list), 5)
        request = Request(
            purchaseitem=pi_list[4],
            user=user,
            quantity_queued=10
        )
        request.save()
        request = Request(
            purchaseitem=pi_list[0],
            user=user,
            quantity_queued=10
        )
        request.save()
        request = Request(
            purchaseitem=pi_list[1],
            user=user,
            quantity_queued=10
        )
        request.save()
        request = Request(
            purchaseitem=pi_list[2],
            user=user,
            quantity_queued=10
        )
        request.save()
        request = Request(
            purchaseitem=pi_list[3],
            user=user,
            quantity_queued=10
        )
        request.save()
        request = Request(
            purchaseitem=pi_list[4],
            user=user,
            quantity_queued=10
        )
        request.save()
        pi_list = purchase.purchaseitem_set.all()  # diff with test0
        self.assertEqual(pi_list[4].quantity_accepted, 10)
        self.assertEqual(pi_list[4].quantity_grouped, 10)
        self.assertEqual(pi_list[4].quantity_queued, 0)

<|endoftext|>


<commit_before>#!/usr/bin/python -tt
# vim:set ts=4 sw=4 expandtab:
# NodeManager plugin to empower slivers to make API calls

"""
Sliver authentication support for NodeManager.

"""

import errno
import os
import random
import string
import tempfile

import logger
import tools

def start(options, conf):
    # XXX REMOVE ME
    return

    logger.log("sliverauth plugin starting up...")

def SetSliverTag(plc, slice, tagname, value):
    node_id = tools.node_id()
    slivertags=plc.GetSliceTags({"name":slice,"node_id":node_id,"tagname":tagname})
    if len(slivertags)==0:
        slivertag_id=plc.AddSliceTag(slice,tagname,value,node_id)
    else:
        slivertag_id=slivertags[0]['slice_tag_id']
        plc.UpdateSliceTag(slivertag_id,value)

def GetSlivers(data, config, plc):
    if 'slivers' not in data:
        logger.log("sliverauth: getslivers data lack's sliver information. IGNORING!")
        return

    random.seed(42)
    for sliver in data['slivers']:
        found_hmac = False
        for attribute in sliver['attributes']:
            name = attribute.get('tagname',attribute.get('name',''))
            if name == 'hmac':
                found_hmac = True
                hmac = attribute['value']
                break

        if not found_hmac:
            d = [random.choice(string.letters) for x in xrange(32)]
            hmac = "".join(d)
            SetSliverTag(plc,sliver['name'],'hmac',hmac)

        path = '/vservers/%s/etc/planetlab' % sliver['name']
        if os.path.exists(path):
            keyfile = '%s/key' % path
            oldhmac = ''
            if os.path.exists(keyfile):
                f = open(keyfile,'r')
                oldhmac = f.read()
                f.close()

            if oldhmac <> hmac:
                # create a temporary file in the vserver
                fd, name = tempfile.mkstemp('','key',path)
                os.write(fd,hmac)
                os.close(fd)
                if os.path.exists(keyfile):
                    os.unlink(keyfile)
                os.rename(name,keyfile)

            os.chmod(keyfile,0400)

<commit_msg>minor improvements with the hope that slivers across machines get different hmacs<commit_after>#!/usr/bin/python -tt
# vim:set ts=4 sw=4 expandtab:
# NodeManager plugin to empower slivers to make API calls

"""
Sliver authentication support for NodeManager.

"""

import errno
import os
import random
import string
import tempfile
import time

import logger
import tools

def start(options, conf):
    logger.log("sliverauth plugin starting up...")

def SetSliverTag(plc, slice, tagname, value):
    node_id = tools.node_id()
    slivertags=plc.GetSliceTags({"name":slice,"node_id":node_id,"tagname":tagname})
    if len(slivertags)==0:
        slivertag_id=plc.AddSliceTag(slice,tagname,value,node_id)
    else:
        slivertag_id=slivertags[0]['slice_tag_id']
        plc.UpdateSliceTag(slivertag_id,value)

def GetSlivers(data, config, plc):
    if 'slivers' not in data:
        logger.log("sliverauth: getslivers data lack's sliver information. IGNORING!")
        return

    for sliver in data['slivers']:
        found_hmac = False
        for attribute in sliver['attributes']:
            name = attribute.get('tagname',attribute.get('name',''))
            if name == 'hmac':
                found_hmac = True
                hmac = attribute['value']
                break

        if not found_hmac:
            # XXX need a better random seed?!
            random.seed(time.time())
            d = [random.choice(string.letters) for x in xrange(32)]
            hmac = "".join(d)
            SetSliverTag(plc,sliver['name'],'hmac',hmac)
            logger.log("sliverauth setting %s hmac" % sliver['name'])

        path = '/vservers/%s/etc/planetlab' % sliver['name']
        if os.path.exists(path):
            keyfile = '%s/key' % path
            oldhmac = ''
            if os.path.exists(keyfile):
                f = open(keyfile,'r')
                oldhmac = f.read()
                f.close()

            if oldhmac <> hmac:
                # create a temporary file in the vserver
                fd, name = tempfile.mkstemp('','key',path)
                os.write(fd,hmac)
                os.close(fd)
                if os.path.exists(keyfile):
                    os.unlink(keyfile)
                os.rename(name,keyfile)
                logger.log("sliverauth writing hmac to %s " % keyfile)

            os.chmod(keyfile,0400)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# accountant_node.py
#
# Copyright (C) 2008-2016 Veselin Penev, http://bitdust.io
#
# This file (accountant_node.py) is part of BitDust Software.
#
# BitDust is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# BitDust Software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with BitDust Software.  If not, see <http://www.gnu.org/licenses/>.
#
# Please contact us if you have any questions at bitdust.io@gmail.com

"""
.. module:: accountant_node.

.. role:: red

BitDust accountant_node() Automat

EVENTS:
    * :red:`accountant-connected`
    * :red:`coin-broadcasted`
    * :red:`coin-not-valid`
    * :red:`coin-verified`
    * :red:`connection-lost`
    * :red:`init`
    * :red:`lookup-failed`
    * :red:`new-coin-mined`
    * :red:`pending-coin`
    * :red:`shutdown`
    * :red:`start`
    * :red:`stop`
    * :red:`timer-1min`
    * :red:`timer-2min`
    * :red:`valid-coins-received`
"""

#------------------------------------------------------------------------------

_Debug = True
_DebugLevel = 6

#------------------------------------------------------------------------------

import datetime

from twisted.internet.defer import Deferred

#------------------------------------------------------------------------------

from logs import lg

from lib import utime

from automats import automat

from transport import callback

from p2p import p2p_service
from p2p import commands

from coins import coins_db
from coins import coins_io

from broadcast import broadcast_service

#------------------------------------------------------------------------------

_AccountantNode = None

#------------------------------------------------------------------------------

def A(event=None, arg=None):
    """
    Access method to interact with the state machine.
    """
    global _AccountantNode
    if event is None and arg is None:
        return _AccountantNode
    if _AccountantNode is None:
        # set automat name and starting state here
        _AccountantNode = AccountantNode('accountant_node', 'AT_STARTUP', _DebugLevel, _Debug)
    if event is not None:
        _AccountantNode.automat(event, arg)
    return _AccountantNode

#------------------------------------------------------------------------------

class AccountantNode(automat.Automat):
    """
    This class implements all the functionality of the ``accountant_node()``
    state machine.
    """
    timers = {
        'timer-2min': (120, ['ACCOUNTANTS?']),
        'timer-1min': (60, ['READ_COINS']),
    }

    def init(self):
        """
        Method to initialize additional variables and flags at creation phase
        of accountant_node() machine.
        """
        self.connected_accountants = []
        self.min_accountants_connected = 1  # TODO: read from settings
        self.max_accountants_connected = 1  # TODO: read from settings
        self.max_coins_per_packet = 100     # TODO: read from settings
        self.download_offset = datetime.datetime(2016, 1, 1)
        self.download_limit = 100
        self.pending_coins = []
        self.current_coin = None

    def state_changed(self, oldstate, newstate, event, arg):
        """
        Method to catch the moment when accountant_node() state were changed.
        """

    def state_not_changed(self, curstate, event, arg):
        """
        This method intended to catch the moment when some event was fired in
        the accountant_node() but its state was not changed.
        """

    def A(self, event, arg):
        """
        The state machine code, generated using `visio2python
        <http://bitdust.io/visio2python/>`_ tool.
        """
        #---READY---
        if self.state == 'READY':
            if event == 'shutdown':
                self.state = 'CLOSED'
                self.doDestroyMe(arg)
            elif event == 'connection-lost' or event == 'stop':
                self.state = 'OFFLINE'
            elif event == 'new-coin-mined':
                self.doPushCoin(arg)
            elif event == 'valid-coins-received':
                self.doWriteCoins(arg)
            elif event == 'pending-coin':
                self.state = 'VALID_COIN?'
                self.doPullCoin(arg)
                self.doVerifyCoin(arg)
        #---READ_COINS---
        elif self.state == 'READ_COINS':
            if event == 'shutdown':
                self.state = 'CLOSED'
                self.doDestroyMe(arg)
            elif event == 'accountant-connected':
                self.doAddAccountant(arg)
                self.doRetreiveCoins(arg)
            elif event == 'stop' or ( event == 'timer-1min' and not self.isAnyCoinsReceived(arg) ):
                self.state = 'OFFLINE'
            elif event == 'new-coin-mined':
                self.doPushCoin(arg)
            elif event == 'valid-coins-received' and self.isMoreCoins(arg):
                self.doWriteCoins(arg)
                self.doRetreiveCoins(arg)
            elif event == 'valid-coins-received' and not self.isMoreCoins(arg):
                self.state = 'READY'
                self.doWriteCoins(arg)
                self.doCheckPendingCoins(arg)
        #---AT_STARTUP---
        elif self.state == 'AT_STARTUP':
            if event == 'init':
                self.state = 'OFFLINE'
                self.doInit(arg)
        #---VALID_COIN?---
        elif self.state == 'VALID_COIN?':
            if event == 'shutdown':
                self.state = 'CLOSED'
                self.doDestroyMe(arg)
            elif event == 'stop':
                self.state = 'OFFLINE'
            elif event == 'coin-not-valid':
                self.state = 'READY'
                self.doCheckPendingCoins(arg)
            elif event == 'accountant-connected':
                self.doAddAccountant(arg)
            elif event == 'coin-verified':
                self.state = 'WRITE_COIN!'
                self.doWriteCoin(arg)
                self.doBroadcastCoin(arg)
            elif event == 'valid-coins-received':
                self.doWriteCoins(arg)
            elif event == 'new-coin-mined':
                self.doPushCoin(arg)
        #---WRITE_COIN!---
        elif self.state == 'WRITE_COIN!':
            if event == 'shutdown':
                self.state = 'CLOSED'
                self.doDestroyMe(arg)
            elif event == 'stop':
                self.state = 'OFFLINE'
            elif event == 'coin-broadcasted':
                self.state = 'READY'
                self.doCheckPendingCoins(arg)
            elif event == 'accountant-connected':
                self.doAddAccountant(arg)
            elif event == 'valid-coins-received':
                self.doWriteCoins(arg)
            elif event == 'new-coin-mined':
                self.doPushCoin(arg)
        #---OFFLINE---
        elif self.state == 'OFFLINE':
            if event == 'shutdown':
                self.state = 'CLOSED'
                self.doDestroyMe(arg)
            elif event == 'start':
                self.state = 'ACCOUNTANTS?'
                self.Attempts=0
                self.doLookupAccountants(arg)
            elif event == 'accountant-connected':
                self.state = 'ACCOUNTANTS?'
                self.doAddAccountant(arg)
                self.Attempts=2
                self.doLookupAccountants(arg)
        #---CLOSED---
        elif self.state == 'CLOSED':
            pass
        #---ACCOUNTANTS?---
        elif self.state == 'ACCOUNTANTS?':
            if event == 'shutdown':
                self.state = 'CLOSED'
                self.doDestroyMe(arg)
            elif event == 'accountant-connected' and self.isMoreNeeded(arg):
                self.doAddAccountant(arg)
                self.doLookupAccountants(arg)
            elif event == 'accountant-connected' and not self.isMoreNeeded(arg):
                self.state = 'READ_COINS'
                self.doRetreiveCoins(arg)
            elif event == 'lookup-failed' and self.Attempts < 5 and self.isAnyAccountants(arg):
                self.Attempts+=1
                self.doLookupAccountants(arg)
            elif ( event == 'lookup-failed' and ( self.Attempts>=5 or not self.isAnyAccountants(arg) ) ) or event == 'timer-2min':
                self.state = 'OFFLINE'
        return None

    def isAnyAccountants(self, arg):
        """
        Condition method.
        """
        return len(self.connected_accountants) > 0

    def isMoreNeeded(self, arg):
        """
        Condition method.
        """
        return len(self.connected_accountants) < self.max_accountants_connected

    def isMoreCoins(self, arg):
        """
        Condition method.
        """
        return len(arg) > 0

    def isAnyCoinsReceived(self, arg):
        """
        Condition method.
        """

    def doInit(self, arg):
        """
        Action method.
        """
        callback.append_inbox_callback(self._on_inbox_packet)

    def doLookupAccountants(self, arg):
        """
        Action method.
        """
        from coins import accountants_finder
        accountants_finder.A('start', (self.automat, 'join'))

    def doAddAccountant(self, arg):
        """
        Action method.
        """
        if arg in self.connected_accountants:
            if _Debug:
                lg.out(_DebugLevel, 'accountant_node.doAddAccountant SKIP, %s already connected, skip' % arg)
            return
        self.connected_accountants.append(arg)
        if _Debug:
            lg.out(_DebugLevel, 'accountant_node.doAddAccountant NEW %s connected, %d total accountants' % (
                arg, len(self.connected_accountants)))

    def doRetreiveCoins(self, arg):
        """
        Action method.
        """
        query = {'method': 'get_all',
                 'index': 'time',
                 'limit': self.download_limit,
                 'start': utime.datetime_to_sec1970(self.download_offset),
                 'end': utime.utcnow_to_sec1970(), }
        for idurl in self.connected_accountants:
            p2p_service.SendRetreiveCoin(idurl, query)

    def doWriteCoins(self, arg):
        """
        Action method.
        """
        for coin in arg:
            if not coins_db.exist(coin):
                coins_db.insert(coin)
            if coin['tm'] > utime.datetime_to_sec1970(self.download_offset):
                self.download_offset = utime.sec1970_to_datetime_utc(coin['tm'])

    def doPushCoin(self, arg):
        """
        Action method.
        """
        self.pending_coins.append(arg)

    def doPullCoin(self, arg):
        """
        Action method.
        """
        self.current_coin = self.pending_coins.pop(0)

    def doCheckPendingCoins(self, arg):
        """
        Action method.
        """
        if len(self.pending_coins) > 0:
            self.automat('pending-coin')

    def doVerifyCoin(self, arg):
        """
        Action method.
        """
        if not coins_io.validate_coin(self.current_coin):
            self.current_coin = None
            self.automat('coin-not-valid')
            return
        d = self._verify_coin(self.current_coin)
        d.addCallback(lambda coin: self.automat('coin-verified'))
        d.addErrback(lambda err: self.automat('coin-not-valid'))

    def doBroadcastCoin(self, arg):
        """
        Action method.
        """
        broadcast_service.send_broadcast_message({'type': 'coin', 'data': arg})

    def doWriteCoin(self, arg):
        """
        Action method.
        """
        coins_db.insert(arg)

    def doDestroyMe(self, arg):
        """
        Remove all references to the state machine object to destroy it.
        """
        self.pending_coins = None
        self.connected_accountants = None
        callback.remove_inbox_callback(self._on_inbox_packet)
        self.unregister()
        global _AccountantNode
        del _AccountantNode
        _AccountantNode = None

    #------------------------------------------------------------------------------

    def _verify_coin(self, acoin):
        # TODO:
        # run coin verification process
        # may require to communicate with other accountants
        d = Deferred()
        d.callback(acoin)
        return d

    def _on_inbox_packet(self, newpacket, info, status, error_message):
        if status != 'finished':
            return False
        if newpacket.Command == commands.RetreiveCoin():
            query_j = coins_io.read_query_from_packet(newpacket)
            if not query_j:
                p2p_service.SendFail(newpacket, 'incorrect query received')
                return False
            coins, error = coins_db.query_json(query_j)
            if error:
                p2p_service.SendFail(newpacket, error)
                return False
            result_coins = []
            for coin in coins:
                result_coins.append(coin)
                if len(result_coins) > self.max_coins_per_packet:
                    result_coins.append(None)
                    break
            p2p_service.SendCoin(newpacket.CreatorID, result_coins, packet_id=newpacket.PacketID)
            return True
        if newpacket.Command == commands.Coin():
            coins_list = coins_io.read_coins_from_packet(newpacket)
            if not coins_list:
                p2p_service.SendFail(newpacket, 'failed to read coins from packet')
                return True
            if coins_list[-1] is None:
                # TODO: partial result, request more coins
                lg.warn('partial response, more coins were found but not transferred')
                del coins_list[-1]
            if len(coins_list) == 1:
                acoin = coins_list[0]
                if not coins_io.validate_coin(acoin):
                    p2p_service.SendFail(newpacket, 'coin validation failed')
                    return True
                if not coins_io.verify_coin(acoin):
                    p2p_service.SendFail(newpacket, 'coin verification failed')
                    return True
                if coins_db.exist(acoin):
                    self.automat('valid-coins-received', [acoin, ])
                else:
                    self.automat('new-coin-mined', acoin)
                return True
            valid_coins = []
            for acoin in coins_list:
                if not coins_io.validate_coin(acoin):
                    continue
                if not coins_io.verify_coin(acoin):
                    continue
                valid_coins.append(acoin)
            if len(valid_coins) == len(coins_list):
                self.automat('valid-coins-received', valid_coins)
            else:
                p2p_service.SendFail(newpacket, 'some non-valid coins received')
            return True
        return False
<commit_msg>	modified:   coins/accountant_node.py<commit_after>#!/usr/bin/env python
# accountant_node.py
#
# Copyright (C) 2008-2016 Veselin Penev, http://bitdust.io
#
# This file (accountant_node.py) is part of BitDust Software.
#
# BitDust is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# BitDust Software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with BitDust Software.  If not, see <http://www.gnu.org/licenses/>.
#
# Please contact us if you have any questions at bitdust.io@gmail.com

"""
.. module:: accountant_node.

.. role:: red

BitDust accountant_node() Automat

EVENTS:
    * :red:`accountant-connected`
    * :red:`coin-broadcasted`
    * :red:`coin-not-valid`
    * :red:`coin-verified`
    * :red:`connection-lost`
    * :red:`init`
    * :red:`lookup-failed`
    * :red:`new-coin-mined`
    * :red:`pending-coin`
    * :red:`shutdown`
    * :red:`start`
    * :red:`stop`
    * :red:`timer-1min`
    * :red:`timer-2min`
    * :red:`valid-coins-received`
"""

#------------------------------------------------------------------------------

_Debug = True
_DebugLevel = 6

#------------------------------------------------------------------------------

import datetime

from twisted.internet.defer import Deferred

#------------------------------------------------------------------------------

from logs import lg

from lib import utime

from automats import automat

from transport import callback

from p2p import p2p_service
from p2p import commands

from coins import coins_db
from coins import coins_io

from broadcast import broadcast_service

#------------------------------------------------------------------------------

_AccountantNode = None

#------------------------------------------------------------------------------

def A(event=None, arg=None):
    """
    Access method to interact with the state machine.
    """
    global _AccountantNode
    if event is None and arg is None:
        return _AccountantNode
    if _AccountantNode is None:
        # set automat name and starting state here
        _AccountantNode = AccountantNode('accountant_node', 'AT_STARTUP', _DebugLevel, _Debug)
    if event is not None:
        _AccountantNode.automat(event, arg)
    return _AccountantNode

#------------------------------------------------------------------------------

class AccountantNode(automat.Automat):
    """
    This class implements all the functionality of the ``accountant_node()``
    state machine.
    """
    timers = {
        'timer-2min': (120, ['ACCOUNTANTS?']),
        'timer-1min': (60, ['READ_COINS']),
    }

    def init(self):
        """
        Method to initialize additional variables and flags at creation phase
        of accountant_node() machine.
        """
        self.connected_accountants = []
        self.min_accountants_connected = 1  # TODO: read from settings
        self.max_accountants_connected = 1  # TODO: read from settings
        self.max_coins_per_packet = 100     # TODO: read from settings
        self.download_offset = datetime.datetime(2016, 1, 1)
        self.download_limit = 100
        self.pending_coins = []
        self.current_coin = None

    def state_changed(self, oldstate, newstate, event, arg):
        """
        Method to catch the moment when accountant_node() state were changed.
        """

    def state_not_changed(self, curstate, event, arg):
        """
        This method intended to catch the moment when some event was fired in
        the accountant_node() but its state was not changed.
        """

    def A(self, event, arg):
        """
        The state machine code, generated using `visio2python
        <http://bitdust.io/visio2python/>`_ tool.
        """
        #---READY---
        if self.state == 'READY':
            if event == 'shutdown':
                self.state = 'CLOSED'
                self.doDestroyMe(arg)
            elif event == 'connection-lost' or event == 'stop':
                self.state = 'OFFLINE'
            elif event == 'new-coin-mined':
                self.doPushCoin(arg)
            elif event == 'valid-coins-received':
                self.doWriteCoins(arg)
            elif event == 'pending-coin':
                self.state = 'VALID_COIN?'
                self.doPullCoin(arg)
                self.doVerifyCoin(arg)
        #---READ_COINS---
        elif self.state == 'READ_COINS':
            if event == 'shutdown':
                self.state = 'CLOSED'
                self.doDestroyMe(arg)
            elif event == 'accountant-connected':
                self.doAddAccountant(arg)
                self.doRetreiveCoins(arg)
            elif event == 'stop' or ( event == 'timer-1min' and not self.isAnyCoinsReceived(arg) ):
                self.state = 'OFFLINE'
            elif event == 'new-coin-mined':
                self.doPushCoin(arg)
            elif event == 'valid-coins-received' and self.isMoreCoins(arg):
                self.doWriteCoins(arg)
                self.doRetreiveCoins(arg)
            elif event == 'valid-coins-received' and not self.isMoreCoins(arg):
                self.state = 'READY'
                self.doWriteCoins(arg)
                self.doCheckPendingCoins(arg)
        #---AT_STARTUP---
        elif self.state == 'AT_STARTUP':
            if event == 'init':
                self.state = 'OFFLINE'
                self.doInit(arg)
        #---VALID_COIN?---
        elif self.state == 'VALID_COIN?':
            if event == 'shutdown':
                self.state = 'CLOSED'
                self.doDestroyMe(arg)
            elif event == 'stop':
                self.state = 'OFFLINE'
            elif event == 'coin-not-valid':
                self.state = 'READY'
                self.doCheckPendingCoins(arg)
            elif event == 'accountant-connected':
                self.doAddAccountant(arg)
            elif event == 'coin-verified':
                self.state = 'WRITE_COIN!'
                self.doWriteCoin(arg)
                self.doBroadcastCoin(arg)
            elif event == 'valid-coins-received':
                self.doWriteCoins(arg)
            elif event == 'new-coin-mined':
                self.doPushCoin(arg)
        #---WRITE_COIN!---
        elif self.state == 'WRITE_COIN!':
            if event == 'shutdown':
                self.state = 'CLOSED'
                self.doDestroyMe(arg)
            elif event == 'stop':
                self.state = 'OFFLINE'
            elif event == 'coin-broadcasted':
                self.state = 'READY'
                self.doCheckPendingCoins(arg)
            elif event == 'accountant-connected':
                self.doAddAccountant(arg)
            elif event == 'valid-coins-received':
                self.doWriteCoins(arg)
            elif event == 'new-coin-mined':
                self.doPushCoin(arg)
        #---OFFLINE---
        elif self.state == 'OFFLINE':
            if event == 'shutdown':
                self.state = 'CLOSED'
                self.doDestroyMe(arg)
            elif event == 'start':
                self.state = 'ACCOUNTANTS?'
                self.Attempts=0
                self.doLookupAccountants(arg)
            elif event == 'accountant-connected':
                self.state = 'ACCOUNTANTS?'
                self.doAddAccountant(arg)
                self.Attempts=2
                self.doLookupAccountants(arg)
        #---CLOSED---
        elif self.state == 'CLOSED':
            pass
        #---ACCOUNTANTS?---
        elif self.state == 'ACCOUNTANTS?':
            if event == 'shutdown':
                self.state = 'CLOSED'
                self.doDestroyMe(arg)
            elif event == 'accountant-connected' and self.isMoreNeeded(arg):
                self.doAddAccountant(arg)
                self.doLookupAccountants(arg)
            elif event == 'accountant-connected' and not self.isMoreNeeded(arg):
                self.state = 'READ_COINS'
                self.doRetreiveCoins(arg)
            elif event == 'lookup-failed' and self.Attempts < 5 and self.isAnyAccountants(arg):
                self.Attempts+=1
                self.doLookupAccountants(arg)
            elif ( event == 'lookup-failed' and ( self.Attempts>=5 or not self.isAnyAccountants(arg) ) ) or event == 'timer-2min':
                self.state = 'OFFLINE'
        return None

    def isAnyAccountants(self, arg):
        """
        Condition method.
        """
        return len(self.connected_accountants) > 0

    def isMoreNeeded(self, arg):
        """
        Condition method.
        """
        return len(self.connected_accountants) < self.max_accountants_connected

    def isMoreCoins(self, arg):
        """
        Condition method.
        """
        return len(arg) > 0

    def isAnyCoinsReceived(self, arg):
        """
        Condition method.
        """

    def doInit(self, arg):
        """
        Action method.
        """
        callback.append_inbox_callback(self._on_inbox_packet)

    def doLookupAccountants(self, arg):
        """
        Action method.
        """
        from coins import accountants_finder
        accountants_finder.A('start', (self.automat, 'join'))

    def doAddAccountant(self, arg):
        """
        Action method.
        """
        if arg in self.connected_accountants:
            if _Debug:
                lg.out(_DebugLevel, 'accountant_node.doAddAccountant SKIP, %s already connected, skip' % arg)
            return
        self.connected_accountants.append(arg)
        if _Debug:
            lg.out(_DebugLevel, 'accountant_node.doAddAccountant NEW %s connected, %d total accountants' % (
                arg, len(self.connected_accountants)))

    def doRetreiveCoins(self, arg):
        """
        Action method.
        """
        query = {'method': 'get_all',
                 'index': 'time_mined',
                 'limit': self.download_limit,
                 'start': utime.datetime_to_sec1970(self.download_offset),
                 'end': utime.utcnow_to_sec1970(), }
        for idurl in self.connected_accountants:
            p2p_service.SendRetreiveCoin(idurl, query)

    def doWriteCoins(self, arg):
        """
        Action method.
        """
        for coin in arg:
            if not coins_db.exist(coin):
                coins_db.insert(coin)
            if coin['tm'] > utime.datetime_to_sec1970(self.download_offset):
                self.download_offset = utime.sec1970_to_datetime_utc(coin['tm'])

    def doPushCoin(self, arg):
        """
        Action method.
        """
        self.pending_coins.append(arg)

    def doPullCoin(self, arg):
        """
        Action method.
        """
        self.current_coin = self.pending_coins.pop(0)

    def doCheckPendingCoins(self, arg):
        """
        Action method.
        """
        if len(self.pending_coins) > 0:
            self.automat('pending-coin')

    def doVerifyCoin(self, arg):
        """
        Action method.
        """
        if not coins_io.validate_coin(self.current_coin):
            self.current_coin = None
            self.automat('coin-not-valid')
            return
        d = self._verify_coin(self.current_coin)
        d.addCallback(lambda coin: self.automat('coin-verified'))
        d.addErrback(lambda err: self.automat('coin-not-valid'))

    def doBroadcastCoin(self, arg):
        """
        Action method.
        """
        broadcast_service.send_broadcast_message({'type': 'coin', 'data': arg})

    def doWriteCoin(self, arg):
        """
        Action method.
        """
        coins_db.insert(arg)

    def doDestroyMe(self, arg):
        """
        Remove all references to the state machine object to destroy it.
        """
        self.pending_coins = None
        self.connected_accountants = None
        callback.remove_inbox_callback(self._on_inbox_packet)
        self.unregister()
        global _AccountantNode
        del _AccountantNode
        _AccountantNode = None

    #------------------------------------------------------------------------------

    def _verify_coin(self, acoin):
        # TODO:
        # run coin verification process
        # may require to communicate with other accountants
        d = Deferred()
        d.callback(acoin)
        return d

    def _on_inbox_packet(self, newpacket, info, status, error_message):
        if status != 'finished':
            return False
        if newpacket.Command == commands.RetreiveCoin():
            query_j = coins_io.read_query_from_packet(newpacket)
            if not query_j:
                p2p_service.SendFail(newpacket, 'incorrect query received')
                return False
            coins, error = coins_db.query_json(query_j)
            if error:
                p2p_service.SendFail(newpacket, error)
                return False
            result_coins = []
            for coin in coins:
                result_coins.append(coin)
                if len(result_coins) > self.max_coins_per_packet:
                    result_coins.append(None)
                    break
            p2p_service.SendCoin(newpacket.CreatorID, result_coins, packet_id=newpacket.PacketID)
            return True
        if newpacket.Command == commands.Coin():
            coins_list = coins_io.read_coins_from_packet(newpacket)
            if not coins_list:
                p2p_service.SendFail(newpacket, 'failed to read coins from packet')
                return True
            if coins_list[-1] is None:
                # TODO: partial result, request more coins
                lg.warn('partial response, more coins were found but not transferred')
                del coins_list[-1]
            if len(coins_list) == 1:
                acoin = coins_list[0]
                if not coins_io.validate_coin(acoin):
                    p2p_service.SendFail(newpacket, 'coin validation failed')
                    return True
                if not coins_io.verify_coin(acoin):
                    p2p_service.SendFail(newpacket, 'coin verification failed')
                    return True
                if coins_db.exist(acoin):
                    self.automat('valid-coins-received', [acoin, ])
                else:
                    self.automat('new-coin-mined', acoin)
                return True
            valid_coins = []
            for acoin in coins_list:
                if not coins_io.validate_coin(acoin):
                    continue
                if not coins_io.verify_coin(acoin):
                    continue
                valid_coins.append(acoin)
            if len(valid_coins) == len(coins_list):
                self.automat('valid-coins-received', valid_coins)
            else:
                p2p_service.SendFail(newpacket, 'some non-valid coins received')
            return True
        return False
<|endoftext|>


<commit_before>from datetime import datetime

from django.db import models, transaction
from django.contrib.auth.models import User

class Problem(models.Model):
    problemId = models.IntegerField(primary_key=True)
    problemName = models.CharField(max_length=128, db_index=True)
    problemStatement = models.TextField()

    points = models.IntegerField(db_index=True)
    tags = models.CharField(max_length=128, db_index=True)

    date = models.DateField(db_index=True)

    users = models.ManyToManyField(
        User,
        through='ProblemStar',
        through_fields=('problem', 'user'),
        related_name='starred_problems',
    )

    def __unicode__(self):
        return u'Problem: [#%d] %s' % (self.problemId, self.problemName)

class ProblemStar(models.Model):
    problem = models.ForeignKey(Problem, related_name='stars')
    user = models.ForeignKey(User, related_name='stars')
    datetime = models.DateTimeField(auto_now_add=True, db_index=True)

    def __unicode__(self):
        return u'Star: #%d <-> %s' % (self.problem.problemId, self.user.username)

    class Meta:
        ordering = ('-datetime', )
        unique_together = ('problem', 'user')

class ProblemSheet(models.Model):
    number = models.IntegerField(db_index=True)
    user = models.ForeignKey(User, related_name="sheets")
    date = models.DateField(auto_now_add=True)

    @property
    def owner(self):
        return self.user

    @property
    def is_last(self):
        return self.user.sheets.last().id == self.id

    @classmethod
    @transaction.atomic
    def add(cls, user):
        return cls.objects.create(
            number=user.sheets.count() + 1,
            user=user
        )

    @transaction.atomic
    def auto_assign_problems(self):
        if self.problems.count() > 0: return

        overdues = ProblemAssignment.objects.filter(
            sheet__user=self.user,
            type='new',
            done=False
        ).prefetch_related('originProblem').all()

        for overdue in overdues:
            ProblemAssignment.assign_problem(overdue.originProblem, self, 'overdue')

        reviews = ProblemAssignment.objects.filter(
            sheet__user=self.user,
            type='new',
            done=True,
        ).order_by('?').all()[:2]

        for review in reviews:
            ProblemAssignment.assign_problem(review.originProblem, self, 'review')

        total = len(overdues) + len(reviews)
        if total < 12:
            allassigns = ProblemAssignment.objects.filter(
                sheet__user=self.user,
                type='new',
            ).values_list('originProblem', flat=True)
            newproblems = Problem.objects.exclude(problemId__in=allassigns).order_by('?').all()[:12 - total]
            for problem in newproblems:
                ProblemAssignment.assign_problem(problem, self)

    def __unicode__(self):
        return u'Sheet: #%d for %s' % (self.number, self.user.username)

class ProblemAssignment(models.Model):
    sheet = models.ForeignKey(ProblemSheet, related_name="problems")
    originProblem = models.ForeignKey(Problem, related_name="assignments")
    problemName = models.CharField(max_length=128)
    points = models.IntegerField()
    tags = models.CharField(max_length=128)
    date = models.DateField()
    done = models.BooleanField(default=False)
    done_at = models.DateTimeField(null=True, blank=True)
    type = models.CharField(max_length=10, default='new', choices=(
        ('new', 'Unsolved problem'),
        ('overdue', 'Overdue problem'),
        ('review', 'Problem to review'),
    ))

    @property
    def owner(self):
        return self.sheet.user

    @classmethod
    def assign_problem(cls, problem, sheet, type_='new'):
        return cls.objects.create(
            sheet=sheet,
            originProblem=problem,
            problemName=problem.problemName,
            points=problem.points,
            tags=problem.tags,
            date=problem.date,
            type=type_
        )

    @transaction.atomic
    def done_problem(self):
        ProblemAssignment.objects.filter(
            sheet__user=self.sheet.user,
            originProblem=self.originProblem,
        ).update(done=True)
        ProblemAssignment.objects.filter(
            sheet__user=self.sheet.user,
            originProblem=self.originProblem,
            done_at=None,
        ).update(done_at=datetime.now())

class ProblemComment(models.Model):
    user = models.ForeignKey(User, related_name="users")
    problem = models.ForeignKey(Problem, related_name="comments")
    content = models.TextField()
    datetime = models.DateTimeField(auto_now_add=True, auto_now=True)

    @property
    def owner(self):
        return self.user

    def __unicode__(self):
        return u'Comment: %s on [#%d]' % (self.user.username, self.problem.id)

    class Meta:
        ordering = ('-datetime', )
<commit_msg>add db index for datetime<commit_after>from datetime import datetime

from django.db import models, transaction
from django.contrib.auth.models import User

class Problem(models.Model):
    problemId = models.IntegerField(primary_key=True)
    problemName = models.CharField(max_length=128, db_index=True)
    problemStatement = models.TextField()

    points = models.IntegerField(db_index=True)
    tags = models.CharField(max_length=128, db_index=True)

    date = models.DateField(db_index=True)

    users = models.ManyToManyField(
        User,
        through='ProblemStar',
        through_fields=('problem', 'user'),
        related_name='starred_problems',
    )

    def __unicode__(self):
        return u'Problem: [#%d] %s' % (self.problemId, self.problemName)

class ProblemStar(models.Model):
    problem = models.ForeignKey(Problem, related_name='stars')
    user = models.ForeignKey(User, related_name='stars')
    datetime = models.DateTimeField(auto_now_add=True, db_index=True)

    def __unicode__(self):
        return u'Star: #%d <-> %s' % (self.problem.problemId, self.user.username)

    class Meta:
        ordering = ('-datetime', )
        unique_together = ('problem', 'user')

class ProblemSheet(models.Model):
    number = models.IntegerField(db_index=True)
    user = models.ForeignKey(User, related_name="sheets")
    date = models.DateField(auto_now_add=True)

    @property
    def owner(self):
        return self.user

    @property
    def is_last(self):
        return self.user.sheets.last().id == self.id

    @classmethod
    @transaction.atomic
    def add(cls, user):
        return cls.objects.create(
            number=user.sheets.count() + 1,
            user=user
        )

    @transaction.atomic
    def auto_assign_problems(self):
        if self.problems.count() > 0: return

        overdues = ProblemAssignment.objects.filter(
            sheet__user=self.user,
            type='new',
            done=False
        ).prefetch_related('originProblem').all()

        for overdue in overdues:
            ProblemAssignment.assign_problem(overdue.originProblem, self, 'overdue')

        reviews = ProblemAssignment.objects.filter(
            sheet__user=self.user,
            type='new',
            done=True,
        ).order_by('?').all()[:2]

        for review in reviews:
            ProblemAssignment.assign_problem(review.originProblem, self, 'review')

        total = len(overdues) + len(reviews)
        if total < 12:
            allassigns = ProblemAssignment.objects.filter(
                sheet__user=self.user,
                type='new',
            ).values_list('originProblem', flat=True)
            newproblems = Problem.objects.exclude(problemId__in=allassigns).order_by('?').all()[:12 - total]
            for problem in newproblems:
                ProblemAssignment.assign_problem(problem, self)

    def __unicode__(self):
        return u'Sheet: #%d for %s' % (self.number, self.user.username)

class ProblemAssignment(models.Model):
    sheet = models.ForeignKey(ProblemSheet, related_name="problems")
    originProblem = models.ForeignKey(Problem, related_name="assignments")
    problemName = models.CharField(max_length=128)
    points = models.IntegerField()
    tags = models.CharField(max_length=128)
    date = models.DateField()
    done = models.BooleanField(default=False)
    done_at = models.DateTimeField(null=True, blank=True)
    type = models.CharField(max_length=10, default='new', choices=(
        ('new', 'Unsolved problem'),
        ('overdue', 'Overdue problem'),
        ('review', 'Problem to review'),
    ))

    @property
    def owner(self):
        return self.sheet.user

    @classmethod
    def assign_problem(cls, problem, sheet, type_='new'):
        return cls.objects.create(
            sheet=sheet,
            originProblem=problem,
            problemName=problem.problemName,
            points=problem.points,
            tags=problem.tags,
            date=problem.date,
            type=type_
        )

    @transaction.atomic
    def done_problem(self):
        ProblemAssignment.objects.filter(
            sheet__user=self.sheet.user,
            originProblem=self.originProblem,
        ).update(done=True)
        ProblemAssignment.objects.filter(
            sheet__user=self.sheet.user,
            originProblem=self.originProblem,
            done_at=None,
        ).update(done_at=datetime.now())

class ProblemComment(models.Model):
    user = models.ForeignKey(User, related_name="users")
    problem = models.ForeignKey(Problem, related_name="comments")
    content = models.TextField()
    datetime = models.DateTimeField(auto_now_add=True, auto_now=True, db_index=True)

    @property
    def owner(self):
        return self.user

    def __unicode__(self):
        return u'Comment: %s on [#%d]' % (self.user.username, self.problem.id)

    class Meta:
        ordering = ('-datetime', )
<|endoftext|>