<commit_before>import os
import re

# for random TMPDIR
import random
import string

import subprocess

###
# common functions
###

def get_filestats(filename):
    filesize = os.path.getsize(filename)
    try:
        filemd5 = subprocess.check_output(
            "md5sum " + filename + " | awk '{print $1}'", shell=True)
    except subprocess.CalledProcessError as e:
        raise e

    return filesize, filemd5.strip().decode('utf-8')

def gen_randomstr(length):
    """Create a random string
    length: int
    return str
    """
    randomstr = string.ascii_letters + string.digits
    return ''.join([random.choice(randomstr) for i in range(length)])

###
# class Source and related functions
###

def get_commitlog(name, oldversion, newversion):
    """Generate commits between old version and newversion
    name: str
    oldversion: str
    newversion: str
    return commitlog: str
    """
    REPODIR = "/home/leaeasy/git-repo/"
    # get all deepin repos
    try:
        allrepos = [f for f in os.listdir(
            REPODIR) if os.path.isdir(os.path.join(REPODIR, f))]
    except:
        # REPODIR not found
        return 9

    # not deepin packages
    if name not in allrepos:
        return 9

    # version example:
    # 3.0.1-1
    # 2:1.18.1-1
    # 10.1.0.5503~a20p2
    versionre = re.compile("(\d:)?([\d.]+).*")
    oldtag = re.findall(versionre, oldversion)[0][1]
    newtag = re.findall(versionre, newversion)[0][1]

    commitcmd = "cd " + REPODIR + name + \
        " && git log --pretty=oneline --abbrev-commit " + \
        oldtag + ".." + newtag + " && cd - >/dev/null"

    commitlog = os.popen(commitcmd).read()

    if commitlog:
        return gen_commit_url(commitlog, name)
    else:
        return 9

def gen_commit_url(data, name):
    urlbase = "https://github.com/linuxdeepin/" + name + "/commit/"
    for x in data.split('\n'):
        commitid = x.split(" ")[0]
        if commitid:
            data = data.replace(commitid, "%s" %
                                (urlbase, commitid, commitid))

    return data

class Source(object):
    """docstring for Source"""

    def __init__(self, name):
        super(Source, self).__init__()
        self.name = name
        # source version
        self.version = ''
        self.oldversion = '0'

        # source's all debs
        self.debs = {}

        # self.arch is all debs' arch
        self.arch = ''

        self.section = 'section'
        self.priority = 'priority'

        self.distribution = 'unstable'

        # source's changelog diff
        self.changelogdiff = ''
        # source's smallest deb name to extract
        self.minsize = 0
        self.mindeb = ''
        # source's commit logs
        self.commitlog = ''

    def _set_version(self, version, oldversion):
        self.version = version
        self.oldversion = oldversion

    def _set_arch(self, arch):
        if arch in self.arch:
            pass
        else:
            self.arch = self.arch + " " + arch

    def _set_details(self, section, priority):
        self.section = section
        self.priority = priority

    def _set_distro(self, distro):
        self.distribution = distro

    def _set_changelogdiff(self, diff):
        self.changelogdiff = diff

    def _set_minsize(self, deb):
        if self.minsize > deb['filesize']:
            self.minsize = deb['filesize']
            self.mindeb = deb

    def _set_commitlog(self):
        commits = get_commitlog(self.name, self.oldversion, self.version)
        if commits == 9:
            commits = "No commit logs found"

        self.commitlog = commits

    def add_debs(self, deb):
        self.debs.append({deb['name']: deb})

####
# class Deb and related functions
####

class Deb(object):
    """docstring for Deb"""

    def __init__(self, filename):
        super(Deb, self).__init__()
        self.filename = filename

        # informations from dpkg-deb -f
        self.name = ''
        self.arch = ''
        # Installed size
        self.installedsize = 0
        self.section = ''
        self.priority = ''

        # changelogs information from dpkg-deb -c
        self.changelogpath = ''

        # file's stats
        self.filesize = 0
        self.md5 = ''

        self.changelog = ''

    def init_deb(self):
        self._set_stats()
        self._set_details()

    def _set_details(self):
        # get information from dpkg-deb -f
        controlfile = os.popen('dpkg-deb -f %s' % self.filename).readlines()
        for line in controlfile:
            line = line.strip()
            if line.startswith("Package: "):
                self.name = line[9:]
            if line.startswith("Architecture: "):
                self.arch = line[14:]
            if line.startswith("Installed-Size: "):
                self.installedsize = int(line[15:])
            if line.startswith("Section: "):
                self.section = line[9:]
            if line.startswith("Priority: "):
                self.priority = line[10:]

    def _set_stats(self):
        filesize, md5 = get_filestats(self.filename)
        self.filesize = filesize
        self.md5sum = md5

    def get_changelog_path(self):
        """Get a deb's changelog path in deb.
        depends on dpkg-deb, grep, awk
        """
        # some deb files contain multi changelog files, only one is useful
        # example:
        # -rw-r--r-- root/root     38974 2016-01-13 23:09 ./usr/share/doc/python3.5/changelog.Debian.gz
        # lrwxrwxrwx root/root         0 2016-01-14 02:55 ./usr/share/doc/python3.5/changelog.gz -> NEWS.gz
        # and another
        # -rw-r--r-- root/root     23459 2015-12-10 12:32 ./usr/share/doc/vim-common/changelog.gz
        # -rw-r--r-- root/root     84110 2016-01-25 10:25 ./usr/share/doc/vim-common/changelog.Debian.gz
        # filepath="./usr/share/doc/vim-common/changelog.gz\n./usr/share/doc/vim-common/changelog.Debian.gz"

        # changelog*.gz always in usr/share/doc/
        cmd = "dpkg-deb -c " + self.filename + \
            " | grep -i changelog | grep 'usr/share/doc' | awk '$3!=0{print $6;}'"
        # strip the \n in filepath
        # TODO: another method to find out the changelog path
        filepath = os.popen(cmd).read().strip().split('\n')
        changelogpath = ''

        if filepath:
            # multi changelog files in filepath
            if len(filepath) > 1:
                for x in filepath:
                    if os.path.split(x)[1] == 'changelog.Debian.gz':
                        changelogpath = x
                        break
                # changelog.Debian.gz is not in deb file list
                if not changelogpath:
                    for x in filepath:
                        # a file has changelog and Debian may be the right one?
                        if x.find('Debian') != -1:
                            filepath = x
            else:
                changelogpath = filepath[0]
            self.changelogpath = changelogpath
        else:
            # deb file doesn't contain a changelog,
            self.changelogpath = ''

    def get_changelog(self):
        """Get changelogs of package.
        depends on dpkg-deb, zcat
        return changelogs: str
        """
        # create tmp dir
        if self.changelogpath:
            randomstr = gen_randomstr(10)
            TMPDIR = '/tmp/diffchangelog-' + randomstr

            extractcmd = "dpkg-deb -x " + self.filename + " " + TMPDIR

            extractdeb = os.system(extractcmd)
            # extract deb failed?
            if extractdeb != 0:
                raise "Extract deb file failed."

            zcatcmd = "cd " + TMPDIR + " && zcat " + self.changelogpath

            changelogs = os.popen(zcatcmd).read()

            # clean TMPDIR
            cleancmd = "rm -rf " + TMPDIR
            os.system(cleancmd)

            if changelogs:
                self.changelog = changelogs
        else:
            self.changelog = "No changlog found."
<commit_msg>Finish basic functions of Source<commit_after>import os
import re

# for random TMPDIR
import random
import string

import subprocess

###
# common functions
###

def get_filestats(filename):
    filesize = os.path.getsize(filename)
    try:
        filemd5 = subprocess.check_output(
            "md5sum " + filename + " | awk '{print $1}'", shell=True)
    except subprocess.CalledProcessError as e:
        raise e

    return filesize, filemd5.strip().decode('utf-8')

def gen_randomstr(length):
    """Create a random string
    length: int
    return str
    """
    randomstr = string.ascii_letters + string.digits
    return ''.join([random.choice(randomstr) for i in range(length)])

###
# class Source and related functions
###

def get_commitlog(name, oldversion, newversion):
    """Generate commits between old version and newversion
    name: str
    oldversion: str
    newversion: str
    return commitlog: str
    """
    REPODIR = "/home/leaeasy/git-repo/"
    # get all deepin repos
    try:
        allrepos = [f for f in os.listdir(
            REPODIR) if os.path.isdir(os.path.join(REPODIR, f))]
    except:
        # REPODIR not found
        return 9

    # not deepin packages
    if name not in allrepos:
        return 9

    # version example:
    # 3.0.1-1
    # 2:1.18.1-1
    # 10.1.0.5503~a20p2
    versionre = re.compile("(\d:)?([\d.]+).*")
    oldtag = re.findall(versionre, oldversion)[0][1]
    newtag = re.findall(versionre, newversion)[0][1]

    commitcmd = "cd " + REPODIR + name + \
        " && git log --pretty=oneline --abbrev-commit " + \
        oldtag + ".." + newtag + " && cd - >/dev/null"

    commitlog = os.popen(commitcmd).read()

    if commitlog:
        return gen_commit_url(commitlog, name)
    else:
        return 9

def gen_commit_url(data, name):
    urlbase = "https://github.com/linuxdeepin/" + name + "/commit/"
    for x in data.split('\n'):
        commitid = x.split(" ")[0]
        if commitid:
            data = data.replace(commitid, "%s" %
                                (urlbase, commitid, commitid))

    return data

class Source(object):
    """docstring for Source"""

    def __init__(self, name):
        super(Source, self).__init__()
        self.name = name
        # source version
        self.version = ''
        self.oldversion = '0'

        # source's all debs
        self.debs = {}

        # self.arch is all debs' arch
        self.arch = ''

        self.section = 'section'
        self.priority = 'priority'

        self.distribution = 'unstable'

        # source's changelog diff
        self.changelogdiff = ''
        # source's smallest deb name to extract
        self.minsize = 0
        self.mindeb = ''
        # source's commit logs
        self.commitlog = ''

    def _set_version(self, version, oldversion):
        self.version = version
        self.oldversion = oldversion

    def _set_arch(self, arch):
        if arch in self.arch:
            pass
        else:
            self.arch = self.arch + " " + arch

    def _set_details(self, section, priority):
        self.section = section
        self.priority = priority

    def _set_distro(self, distro):
        '''
        Get soruce distro from debs if distro is set to ''
        '''
        if not distro:
            if not self.mindeb.changelog:
                self.mindeb.get_changelog()

            distrore = re.compile(r'.* (\w+); urgency')
            distro = re.findall(distrore, self.mindeb.changelog)[0]

        self.distribution = distro

    # TODO: diff from versions of changelog
    # may be set distro in this function is a good choice?
    def _set_changelogdiff(self):
        # a new source
        if self.oldversion == '0':
            diff = self.mindeb.changelog
        else:
            oldstr = self.name + ' (' + self.oldversion + ') '

            # old version not in changelog, maybe step over some versions?
            header_pattern = re.compile(r'.*;.*urgency=.*\n+')
            changelogs = header_pattern.split(self.mindeb.changelog)
            headers = header_pattern.findall(self.mindeb.changelog)

            diff = ''
            # show max 5 changelog items
            for x in range(0, len(headers)):
                if oldstr in headers[x]:
                    break
                elif x > 4:
                    break
                else:
                    diff = diff + headers[x] + changelogs[x + 1]

        self.changelogdiff = diff

    def _set_minsize(self, deb):
        # some debs don't have changelogs
        if deb.changelogpath == '':
            pass
        # the first deb of this source
        elif self.minsize == 0:
            self.minsize = deb.filesize
            self.mindeb = deb
        elif self.minsize > deb.filesize:
            self.minsize = deb.filesize
            self.mindeb = deb

    def _set_commitlog(self):
        commits = get_commitlog(self.name, self.oldversion, self.version)
        if commits == 9:
            commits = "No commit logs found"

        self.commitlog = commits

    # TODO: Many things to do in this
    def add_debs(self, deb):
        self.debs[deb.name] = deb
        self._set_arch(deb.arch)
        self._set_details(deb.section, deb.priority)
        self._set_minsize(deb)

####
# class Deb and related functions
####

class Deb(object):
    """docstring for Deb"""

    def __init__(self, filename):
        super(Deb, self).__init__()
        self.filename = filename

        # informations from dpkg-deb -f
        self.name = ''
        self.arch = ''
        self.version = ''
        # Installed size
        self.installedsize = 0
        self.section = ''
        self.priority = ''

        # file's stats
        self.filesize = 0
        self.md5 = ''

        # changelogs information from dpkg-deb -c
        self.changelogpath = ''

        self.changelog = ''

    def init_deb(self):
        # initial all deb's information
        # only one deb in one Source should get changelogs
        self._set_stats()
        self._set_details()
        self.get_changelog_path()

    def _set_details(self):
        # get information from dpkg-deb -f
        controlfile = os.popen('dpkg-deb -f %s' % self.filename).readlines()
        for line in controlfile:
            line = line.strip()
            if line.startswith("Package: "):
                self.name = line[9:]
            if line.startswith("Version: "):
                self.version = line[9:]
            if line.startswith("Architecture: "):
                self.arch = line[14:]
            if line.startswith("Installed-Size: "):
                self.installedsize = int(line[15:])
            if line.startswith("Section: "):
                self.section = line[9:]
            if line.startswith("Priority: "):
                self.priority = line[10:]

    def _set_stats(self):
        filesize, md5 = get_filestats(self.filename)
        self.filesize = filesize
        self.md5sum = md5

    def get_changelog_path(self):
        """Get a deb's changelog path in deb.
        depends on dpkg-deb, grep, awk

        some deb files contain multi changelog files, only one is useful
        example:
        ./usr/share/doc/python3.5/changelog.Debian.gz
        ./usr/share/doc/python3.5/changelog.gz -> NEWS.gz
        and another
        ./usr/share/doc/vim-common/changelog.gz
        ./usr/share/doc/vim-common/changelog.Debian.gz
        filepath="./usr/share/doc/vim-common/changelog.gz\n./usr/share/doc/vim-common/changelog.Debian.gz"
        """

        # changelog*.gz always in usr/share/doc/
        cmd = "dpkg-deb -c " + self.filename + \
            " | grep -i changelog | grep 'usr/share/doc'" + \
            " | awk '$3!=0{print $6;}'"
        # strip the \n in filepath
        # TODO: another method to find out the changelog path
        filepath = os.popen(cmd).read().strip().split('\n')
        changelogpath = ''

        if filepath:
            # multi changelog files in filepath
            if len(filepath) > 1:
                for x in filepath:
                    if os.path.split(x)[1] == 'changelog.Debian.gz':
                        changelogpath = x
                        break
                # changelog.Debian.gz is not in deb file list
                if not changelogpath:
                    for x in filepath:
                        # a file has changelog and Debian may be the right one?
                        if x.find('Debian') != -1:
                            filepath = x
            else:
                changelogpath = filepath[0]
            self.changelogpath = changelogpath
        else:
            # deb file doesn't contain a changelog,
            self.changelogpath = ''

    def get_changelog(self):
        """Get changelogs of package.
        depends on dpkg-deb, zcat
        return changelogs: str
        """
        # create tmp dir
        if self.changelogpath:
            randomstr = gen_randomstr(10)
            TMPDIR = '/tmp/diffchangelog-' + randomstr

            extractcmd = "dpkg-deb -x " + self.filename + " " + TMPDIR

            extractdeb = os.system(extractcmd)
            # extract deb failed?
            if extractdeb != 0:
                raise "Extract deb file failed."

            zcatcmd = "cd " + TMPDIR + " && zcat " + self.changelogpath

            changelogs = os.popen(zcatcmd).read()

            # clean TMPDIR
            cleancmd = "rm -rf " + TMPDIR
            os.system(cleancmd)

            if changelogs:
                self.changelog = changelogs
        else:
            self.changelog = "No changlog found."
<|endoftext|>


<commit_before>"""A simple guestbook app to test parts of NDB end-to-end."""

import cgi
import logging
import sys
import time

from google.appengine.api import users
from google.appengine.datastore import entity_pb
from google.appengine.ext import webapp
from google.appengine.ext.webapp import util

from core import datastore_query
from core import datastore_rpc
from core import monkey

import bpt
from ndb import context
from ndb import eventloop
from ndb import model
from ndb import tasks

HOME_PAGE = """

  Nickname: %(nickname)s |
  login |
  logout

  <form method=POST action=/>

    <input type=text id=body name=body size=60>
    <input type=submit>
  </form>

"""

ACCOUNT_PAGE = """

  Nickname: %(nickname)s |
  logout

  <form method=POST action=/account>

    Email: %(email)s
    <input type=text name=nickname size=20 value=%(nickname)s>
    <input type=submit name=%(action)s value="%(action)s Account">
    <input type=submit name=delete value="Delete Account">
    back to home page
  </form>

"""

class Account(model.Model):
  """A user."""

  email = model.StringProperty()
  userid = model.StringProperty()
  nickname = model.StringProperty()

class Message(model.Model):
  """A guestbook message."""

  body = model.StringProperty()
  when = model.IntegerProperty()
  userid = model.StringProperty()

def WaitForRpcs():
  eventloop.run()

@tasks.task
def AsyncGetAccountByUser(context, user, create=False, nickname=None):
  """Find an account."""
  assert isinstance(user, users.User), '%r is not a User' % user
  assert user.user_id() is not None, 'user_id is None'
  prop = entity_pb.Property()
  prop.set_name(Account.userid.name)
  prop.set_multiple(False)
  pval = prop.mutable_value()
  pval.set_stringvalue(user.user_id())
  pred = datastore_query.PropertyFilter('=', prop)
  query = datastore_query.Query(kind=Account.GetKind(),
                                filter_predicate=pred)
  options = datastore_query.QueryOptions(limit=1)

  def callback(entity):
    return entity  # TODO: This should be the default callback?
  fut1, fut2 = context.map_query(query, callback, options)
  res1, res2 = yield fut1, fut2
  assert len(res1) == res2
  if res1:
    account = res1[0]
    assert isinstance(account, Account)
    raise tasks.Return(account)

  if not create:
    return  # None

  account = Account(key=model.Key(flat=['Account', user.user_id()]),
                    email=user.email(),
                    userid=user.user_id())
  if nickname:
    account.nickname = nickname
  # Write to datastore asynchronously.
  context.put(account)  # Don't wait
  raise tasks.Return(account)

def GetAccountByUser(context, user, create=False, nickname=None):
  fut = AsyncGetAccountByUser(context, user, create, nickname)
  return fut.get_result()

class HomePage(webapp.RequestHandler):

  @context.add_context
  def get(self):
    user = users.get_current_user()
    email = None
    nickname = 'Anonymous'
    if user is not None:
      email = user.email()
      account = GetAccountByUser(self.ctx, user)
      if account is None:
        nickname = 'Withdrawn'
      else:
        email = account.email
        nickname = account.nickname or account.email
    values = {'email': email,
              'nickname': nickname,
              'login': users.create_login_url('/'),
              'logout': users.create_logout_url('/'),
              }
    self.response.out.write(HOME_PAGE % values)
    order = datastore_query.PropertyOrder(
      'when',
      datastore_query.PropertyOrder.DESCENDING)
    query = datastore_query.Query(kind=Message.GetKind(), order=order)
    options = datastore_query.QueryOptions(batch_size=13, limit=50)
    fut1, fut2 = self.ctx.map_query(query, self._result_callback, options)
    futures = fut1.get_result()
    size = fut2.get_result()
    assert len(futures) == size
    results = [f.get_result() for f in futures]
    results.sort()
    for key, text in results:
      self.response.out.write(text)

  @tasks.task
  def _result_callback(self, result):
    # Task started for each query result.
    if result.userid is not None:
      key = model.Key(flat=['Account', result.userid])
      account = yield self.ctx.get(key)
      if account is None:
        author = 'Withdrawn'
      else:
        author = account.nickname or account.email
    else:
      author = 'Anonymous'
    raise tasks.Return((-result.when,
                        '%s / %s — %s
' %
                        (cgi.escape(author),
                         time.ctime(result.when),
                         cgi.escape(result.body))))

  @context.add_context
  def post(self):
    body = self.request.get('body')
    if not body.strip():
      self.redirect('/')
      return
    user = users.get_current_user()
    futs = []
    logging.info('body=%.100r', body)
    body = body.rstrip()
    if body:
      msg = Message()
      msg.body = body
      msg.when = int(time.time())
      if user is not None:
        msg.userid = user.user_id()
      # Write to datastore asynchronously.
      f = self.ctx.put(msg)
      futs.append(f)
      if user is not None:
        # Check that the account exists and create it if necessary.
        f = AsyncGetAccountByUser(self.ctx, user, create=True)
        futs.append(f)
    self.redirect('/')
    for f in futs:
      logging.info('f before: %s', f)
      f.check_success()
      logging.info('f after: %s', f)
    WaitForRpcs()  # Ensure Account gets written.

class AccountPage(webapp.RequestHandler):

  @context.add_context
  def get(self):
    user = users.get_current_user()
    if user is None:
      self.redirect(users.create_login_url('/account'))
      return
    email = user.email()
    account = GetAccountByUser(self.ctx, user)
    action = 'Create'
    nickname = 'Withdrawn'
    if account is not None:
      action = 'Update'
      nickname = account.nickname or account.email
    values = {'email': email,
              'nickname': nickname,
              'action': action,
              'login': users.create_login_url('/account'),
              'logout': users.create_logout_url('/'),
              }
    self.response.out.write(ACCOUNT_PAGE % values)

  @context.add_context
  def post(self):
    user = users.get_current_user()
    if user is None:
      self.redirect(users.create_login_url('/account'))
      return
    if self.request.get('delete'):
      account = GetAccountByUser(self.ctx, user)
      if account is not None:
        self.ctx.delete(account.key).check_success()
      self.redirect('/account')
      return
    nickname = self.request.get('nickname')
    account = GetAccountByUser(self.ctx, user, create=True)  ##, nickname=nickname)
    if nickname and account.nickname != nickname:
      account.nickname = nickname
      f = self.ctx.put(account)
      f.check_success()
    self.redirect('/account')
    WaitForRpcs()  # Ensure Account gets written.

urls = [
  ('/', HomePage),
  ('/account', AccountPage),
  ]

app = webapp.WSGIApplication(urls)

def main():
  util.run_wsgi_app(app)

if __name__ == '__main__':
  main()
<commit_msg>Restructure main.<commit_after>"""A simple guestbook app to test parts of NDB end-to-end."""

import cgi
import logging
import sys
import time

from google.appengine.api import users
from google.appengine.datastore import entity_pb
from google.appengine.ext import webapp
from google.appengine.ext.webapp import util

from core import datastore_query
from core import datastore_rpc
from core import monkey

from ndb import context
from ndb import eventloop
from ndb import model
from ndb import tasks

HOME_PAGE = """

  Nickname: %(nickname)s |
  login |
  logout

  <form method=POST action=/>

    <input type=text id=body name=body size=60>
    <input type=submit>
  </form>

"""

ACCOUNT_PAGE = """

  Nickname: %(nickname)s |
  logout

  <form method=POST action=/account>

    Email: %(email)s
    <input type=text name=nickname size=20 value=%(nickname)s>
    <input type=submit name=%(action)s value="%(action)s Account">
    <input type=submit name=delete value="Delete Account">
    back to home page
  </form>

"""

class Account(model.Model):
  """User account."""

  email = model.StringProperty()
  userid = model.StringProperty()
  nickname = model.StringProperty()

class Message(model.Model):
  """Guestbook message."""

  body = model.StringProperty()
  when = model.IntegerProperty()
  userid = model.StringProperty()

def taskify(func):
  """Decorator to run a function as a task when called.

  Use this to wrap a request handler function that will be called by
  some web application framework (e.g. a Django view function or a
  webapp.RequestHandler.get method).
  """
  taskfunc = tasks.task(func)
  def taskify_wrapper(*args):
    return taskfunc(*args).get_result()
  return taskify_wrapper

def account_key(userid):
  return model.Key(flat=['Account', userid])

def get_account(ctx, userid):
  """Return a Future for an Account."""
  return ctx.get(account_key(userid))

@tasks.task
def get_nickname(ctx, userid):
  """Return a Future for a nickname from an account."""
  account = yield get_account(ctx, userid)
  if not account:
    nickname = 'Unknown'
  else:
    nickname = account.nickname or account.email
  raise tasks.Return(nickname)

class HomePage(webapp.RequestHandler):

  @context.add_context
  @taskify
  def get(self):
    nickname = 'Anonymous'
    user = users.get_current_user()
    if user is not None:
      nickname = yield get_nickname(self.ctx, user.user_id())
    values = {'nickname': nickname,
              'login': users.create_login_url('/'),
              'logout': users.create_logout_url('/'),
              }
    self.response.out.write(HOME_PAGE % values)
    query, options = self._make_query()
    ffuts, fcount = self.ctx.map_query(query, self._callback, options)
    # TODO: Yielding as a tuple returns values in the wrong order!
    futs = yield ffuts
    count = yield fcount
    assert len(futs) == count
    pairs = [f.get_result() for f in futs]
    pairs.sort()
    for key, text in pairs:
      self.response.out.write(text)

  def _make_query(self):
    order = datastore_query.PropertyOrder(
      'when',
      datastore_query.PropertyOrder.DESCENDING)
    query = datastore_query.Query(kind=Message.GetKind(), order=order)
    options = datastore_query.QueryOptions(batch_size=13, limit=7)
    return query, options

  @tasks.task
  def _callback(self, message):
    nickname = 'Anonymous'
    if message.userid:
      nickname = yield get_nickname(self.ctx, message.userid)
    text = '%s - %s - %s
' % (cgi.escape(nickname),
                                 time.ctime(message.when),
                                 cgi.escape(message.body))
    raise tasks.Return((-message.when, text))

  @taskify
  @context.add_context
  def post(self):
    # TODO: XSRF protection.
    body = self.request.get('body', '').strip()
    if body:
      userid = None
      user = users.get_current_user()
      if user:
        userid = user.user_id()
      message = Message(body=body, when=int(time.time()), userid=userid)
      yield self.ctx.put(message)  # Synchronous.
    self.redirect('/')

class AccountPage(webapp.RequestHandler):

  @taskify
  @context.add_context
  def get(self):
    user = users.get_current_user()
    if not user:
      self.redirect(users.create_login_url('/account'))
      return
    email = user.email()
    action = 'Create'
    nickname = yield get_nickname(self.ctx, user.user_id())
    if nickname != 'Unknown':
      action = 'Update'
    values = {'email': email,
              'nickname': nickname,
              'login': users.create_login_url('/'),
              'logout': users.create_logout_url('/'),
              'action': action,
              }
    self.response.out.write(ACCOUNT_PAGE % values)

  @context.add_context
  @taskify
  def post(self):
    # TODO: XSRF protection.
    user = users.get_current_user()
    if not user:
      self.redirect(users.create_login_url('/account'))
      return
    account = yield get_account(self.ctx, user.user_id())
    if self.request.get('delete'):
      if account:
        yield self.ctx.delete(account.key)
      self.redirect('/account')
      return
    if not account:
      account = Account(key=account_key(user.user_id()),
                        email=user.email(), userid=user.user_id())
    nickname = self.request.get('nickname')
    if nickname and nickname != 'Unknown':
      account.nickname = nickname
    yield self.ctx.put(account)
    self.redirect('/account')

urls = [
  ('/', HomePage),
  ('/account', AccountPage),
  ]

app = webapp.WSGIApplication(urls)

def main():
  util.run_wsgi_app(app)

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before>from adventurelib import *

Room.items = Bag()

current_room = starting_room = Room("""
You are in a dark room.
""")

valley = starting_room.north = Room("""
You are in a beautiful valley.
""")

mallet = Item('rusty mallet', 'mallet')
valley.items = Bag({mallet,})

inventory = Bag()

@when('north', direction='north')
@when('south', direction='south')
@when('east', direction='east')
@when('west', direction='west')
def go(direction):
    global current_room
    room = current_room.exit(direction)
    if room:
        current_room = room
        say('You go %s.' % direction)
        look()

@when('take ITEM')
def take(item):
    obj = current_room.items.take(item)
    if obj:
        say('You pick up the %s.' % obj)
        inventory.add(obj)
    else:
        say('There is no %s here.' % item)

@when('drop THING')
def drop(thing):
    obj = inventory.take(thing)
    if not obj:
        say('You do not have a %s.' % thing)
    else:
        say('You drop the %s.' % obj)
        current_room.items.add(obj)

@when('look')
def look():
    say(current_room)
    if current_room.items:
        for i in current_room.items:
            say('A %s is here.' % i)

@when('inventory')
def show_inventory():
    say('You have:')
    for thing in inventory:
        say(thing)

look()
start()
<commit_msg>Small example of contextual command<commit_after>from adventurelib import *

Room.items = Bag()

current_room = starting_room = Room("""
You are in a dark room.
""")

valley = starting_room.north = Room("""
You are in a beautiful valley.
""")

magic_forest = valley.north = Room("""
You are in a enchanted forest where magic grows wildly.
""")

mallet = Item('rusty mallet', 'mallet')
valley.items = Bag({mallet,})

inventory = Bag()

@when('north', direction='north')
@when('south', direction='south')
@when('east', direction='east')
@when('west', direction='west')
def go(direction):
    global current_room
    room = current_room.exit(direction)
    if room:
        current_room = room
        say('You go %s.' % direction)
        look()
        if room == magic_forest:
            set_context('magic_aura')
        else:
            set_context('default')

@when('take ITEM')
def take(item):
    obj = current_room.items.take(item)
    if obj:
        say('You pick up the %s.' % obj)
        inventory.add(obj)
    else:
        say('There is no %s here.' % item)

@when('drop THING')
def drop(thing):
    obj = inventory.take(thing)
    if not obj:
        say('You do not have a %s.' % thing)
    else:
        say('You drop the %s.' % obj)
        current_room.items.add(obj)

@when('look')
def look():
    say(current_room)
    if current_room.items:
        for i in current_room.items:
            say('A %s is here.' % i)

@when('inventory')
def show_inventory():
    say('You have:')
    for thing in inventory:
        say(thing)

@when('cast', context='magic_aura', magic=None)
def cast(magic):
    if magic == None:
        say("Which magic you would like to spell?")

look()
start()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
lit - LLVM Integrated Tester.

See lit.pod for more information.
"""

import math, os, platform, random, re, sys, time, threading, traceback

import ProgressBar
import TestRunner
import Util

import LitConfig
import Test

import lit.discovery

class TestingProgressDisplay:
    def __init__(self, opts, numTests, progressBar=None):
        self.opts = opts
        self.numTests = numTests
        self.current = None
        self.lock = threading.Lock()
        self.progressBar = progressBar
        self.completed = 0

    def update(self, test):
        # Avoid locking overhead in quiet mode
        if self.opts.quiet and not test.result.isFailure:
            self.completed += 1
            return

        # Output lock.
        self.lock.acquire()
        try:
            self.handleUpdate(test)
        finally:
            self.lock.release()

    def finish(self):
        if self.progressBar:
            self.progressBar.clear()
        elif self.opts.quiet:
            pass
        elif self.opts.succinct:
            sys.stdout.write('\n')

    def handleUpdate(self, test):
        self.completed += 1
        if self.progressBar:
            self.progressBar.update(float(self.completed)/self.numTests,
                                    test.getFullName())

        if self.opts.succinct and not test.result.isFailure:
            return

        if self.progressBar:
            self.progressBar.clear()

        print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
                                     self.completed, self.numTests)

        if test.result.isFailure and self.opts.showOutput:
            print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
                                              '*'*20)
            print test.output
            print "*" * 20

        sys.stdout.flush()

class TestProvider:
    def __init__(self, tests, maxTime):
        self.maxTime = maxTime
        self.iter = iter(tests)
        self.lock = threading.Lock()
        self.startTime = time.time()

    def get(self):
        # Check if we have run out of time.
        if self.maxTime is not None:
            if time.time() - self.startTime > self.maxTime:
                return None

        # Otherwise take the next test.
        self.lock.acquire()
        try:
            item = self.iter.next()
        except StopIteration:
            item = None
        self.lock.release()
        return item

class Tester(threading.Thread):
    def __init__(self, litConfig, provider, display):
        threading.Thread.__init__(self)
        self.litConfig = litConfig
        self.provider = provider
        self.display = display

    def run(self):
        while 1:
            item = self.provider.get()
            if item is None:
                break
            self.runTest(item)

    def runTest(self, test):
        result = None
        startTime = time.time()
        try:
            result, output = test.config.test_format.execute(test,
                                                             self.litConfig)
        except KeyboardInterrupt:
            # This is a sad hack. Unfortunately subprocess goes
            # bonkers with ctrl-c and we start forking merrily.
            print '\nCtrl-C detected, goodbye.'
            os.kill(0,9)
        except:
            if self.litConfig.debug:
                raise
            result = Test.UNRESOLVED
            output = 'Exception during script execution:\n'
            output += traceback.format_exc()
            output += '\n'
        elapsed = time.time() - startTime

        test.setResult(result, output, elapsed)
        self.display.update(test)

def runTests(numThreads, litConfig, provider, display):
    # If only using one testing thread, don't use threads at all; this lets us
    # profile, among other things.
    if numThreads == 1:
        t = Tester(litConfig, provider, display)
        t.run()
        return

    # Otherwise spin up the testing threads and wait for them to finish.
    testers = [Tester(litConfig, provider, display)
               for i in range(numThreads)]
    for t in testers:
        t.start()
    try:
        for t in testers:
            t.join()
    except KeyboardInterrupt:
        sys.exit(2)

def main(builtinParameters = {}):
    # Bump the GIL check interval, its more important to get any one thread to a
    # blocking operation (hopefully exec) than to try and unblock other threads.
    #
    # FIXME: This is a hack.
    import sys
    sys.setcheckinterval(1000)

    global options
    from optparse import OptionParser, OptionGroup
    parser = OptionParser("usage: %prog [options] {file-or-path}")

    parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
                      help="Number of testing threads",
                      type=int, action="store", default=None)
    parser.add_option("", "--config-prefix", dest="configPrefix",
                      metavar="NAME", help="Prefix for 'lit' config files",
                      action="store", default=None)
    parser.add_option("", "--param", dest="userParameters",
                      metavar="NAME=VAL",
                      help="Add 'NAME' = 'VAL' to the user defined parameters",
                      type=str, action="append", default=[])

    group = OptionGroup(parser, "Output Format")
    # FIXME: I find these names very confusing, although I like the
    # functionality.
    group.add_option("-q", "--quiet", dest="quiet",
                     help="Suppress no error output",
                     action="store_true", default=False)
    group.add_option("-s", "--succinct", dest="succinct",
                     help="Reduce amount of output",
                     action="store_true", default=False)
    group.add_option("-v", "--verbose", dest="showOutput",
                     help="Show all test output",
                     action="store_true", default=False)
    group.add_option("", "--no-progress-bar", dest="useProgressBar",
                     help="Do not use curses based progress bar",
                     action="store_false", default=True)
    parser.add_option_group(group)

    group = OptionGroup(parser, "Test Execution")
    group.add_option("", "--path", dest="path",
                     help="Additional paths to add to testing environment",
                     action="append", type=str, default=[])
    group.add_option("", "--vg", dest="useValgrind",
                     help="Run tests under valgrind",
                     action="store_true", default=False)
    group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
                     help="Check for memory leaks under valgrind",
                     action="store_true", default=False)
    group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
                     help="Specify an extra argument for valgrind",
                     type=str, action="append", default=[])
    group.add_option("", "--time-tests", dest="timeTests",
                     help="Track elapsed wall time for each test",
                     action="store_true", default=False)
    group.add_option("", "--no-execute", dest="noExecute",
                     help="Don't execute any tests (assume PASS)",
                     action="store_true", default=False)
    parser.add_option_group(group)

    group = OptionGroup(parser, "Test Selection")
    group.add_option("", "--max-tests", dest="maxTests", metavar="N",
                     help="Maximum number of tests to run",
                     action="store", type=int, default=None)
    group.add_option("", "--max-time", dest="maxTime", metavar="N",
                     help="Maximum time to spend testing (in seconds)",
                     action="store", type=float, default=None)
    group.add_option("", "--shuffle", dest="shuffle",
                     help="Run tests in random order",
                     action="store_true", default=False)
    group.add_option("", "--filter", dest="filter", metavar="EXPRESSION",
                     help=("Only run tests with paths matching the given "
                           "regular expression"),
                     action="store", default=None)
    parser.add_option_group(group)

    group = OptionGroup(parser, "Debug and Experimental Options")
    group.add_option("", "--debug", dest="debug",
                      help="Enable debugging (for 'lit' development)",
                      action="store_true", default=False)
    group.add_option("", "--show-suites", dest="showSuites",
                      help="Show discovered test suites",
                      action="store_true", default=False)
    group.add_option("", "--repeat", dest="repeatTests", metavar="N",
                      help="Repeat tests N times (for timing)",
                      action="store", default=None, type=int)
    parser.add_option_group(group)

    (opts, args) = parser.parse_args()

    if not args:
        parser.error('No inputs specified')

    if opts.numThreads is None:
# Python <2.5 has a race condition causing lit to always fail with numThreads>1
# http://bugs.python.org/issue1731717
# I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
# threads by default there.
       if sys.hexversion >= 0x2050200:
               opts.numThreads = Util.detectCPUs()
       else:
               opts.numThreads = 1

    inputs = args

    # Create the user defined parameters.
    userParams = dict(builtinParameters)
    for entry in opts.userParameters:
        if '=' not in entry:
            name,val = entry,''
        else:
            name,val = entry.split('=', 1)
        userParams[name] = val

    # Create the global config object.
    litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
                                    path = opts.path,
                                    quiet = opts.quiet,
                                    useValgrind = opts.useValgrind,
                                    valgrindLeakCheck = opts.valgrindLeakCheck,
                                    valgrindArgs = opts.valgrindArgs,
                                    noExecute = opts.noExecute,
                                    ignoreStdErr = False,
                                    debug = opts.debug,
                                    isWindows = (platform.system()=='Windows'),
                                    params = userParams,
                                    config_prefix = opts.configPrefix)

    tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)

    if opts.showSuites:
        suitesAndTests = {}
        for t in tests:
            if t.suite not in suitesAndTests:
                suitesAndTests[t.suite] = []
            suitesAndTests[t.suite].append(t)

        print '-- Test Suites --'
        suitesAndTests = suitesAndTests.items()
        suitesAndTests.sort(key = lambda (ts,_): ts.name)
        for ts,ts_tests in suitesAndTests:
            print '  %s - %d tests' %(ts.name, len(ts_tests))
            print '    Source Root: %s' % ts.source_root
            print '    Exec Root  : %s' % ts.exec_root

    # Select and order the tests.
    numTotalTests = len(tests)

    # First, select based on the filter expression if given.
    if opts.filter:
        try:
            rex = re.compile(opts.filter)
        except:
            parser.error("invalid regular expression for --filter: %r" % (
                    opts.filter))
        tests = [t for t in tests
                 if rex.search(t.getFullName())]

    # Then select the order.
    if opts.shuffle:
        random.shuffle(tests)
    else:
        tests.sort(key = lambda t: t.getFullName())

    # Finally limit the number of tests, if desired.
    if opts.maxTests is not None:
        tests = tests[:opts.maxTests]

    # Don't create more threads than tests.
    opts.numThreads = min(len(tests), opts.numThreads)

    extra = ''
    if len(tests) != numTotalTests:
        extra = ' of %d' % numTotalTests
    header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
                                                      opts.numThreads)

    if opts.repeatTests:
        tests = [t.copyWithIndex(i)
                 for t in tests
                 for i in range(opts.repeatTests)]

    progressBar = None
    if not opts.quiet:
        if opts.succinct and opts.useProgressBar:
            try:
                tc = ProgressBar.TerminalController()
                progressBar = ProgressBar.ProgressBar(tc, header)
            except ValueError:
                print header
                progressBar = ProgressBar.SimpleProgressBar('Testing: ')
        else:
            print header

    startTime = time.time()
    display = TestingProgressDisplay(opts, len(tests), progressBar)
    provider = TestProvider(tests, opts.maxTime)
    runTests(opts.numThreads, litConfig, provider, display)
    display.finish()

    if not opts.quiet:
        print 'Testing Time: %.2fs'%(time.time() - startTime)

    # Update results for any tests which weren't run.
    for t in tests:
        if t.result is None:
            t.setResult(Test.UNRESOLVED, '', 0.0)

    # List test results organized by kind.
    hasFailures = False
    byCode = {}
    for t in tests:
        if t.result not in byCode:
            byCode[t.result] = []
        byCode[t.result].append(t)
        if t.result.isFailure:
            hasFailures = True

    # FIXME: Show unresolved and (optionally) unsupported tests.
    for title,code in (('Unexpected Passing Tests', Test.XPASS),
                       ('Failing Tests', Test.FAIL)):
        elts = byCode.get(code)
        if not elts:
            continue
        print '*'*20
        print '%s (%d):' % (title, len(elts))
        for t in elts:
            print '    %s' % t.getFullName()
        print

    if opts.timeTests:
        # Collate, in case we repeated tests.
        times = {}
        for t in tests:
            key = t.getFullName()
            times[key] = times.get(key, 0.) + t.elapsed

        byTime = list(times.items())
        byTime.sort(key = lambda (name,elapsed): elapsed)
        if byTime:
            Util.printHistogram(byTime, title='Tests')

    for name,code in (('Expected Passes    ', Test.PASS),
                      ('Expected Failures  ', Test.XFAIL),
                      ('Unsupported Tests  ', Test.UNSUPPORTED),
                      ('Unresolved Tests   ', Test.UNRESOLVED),
                      ('Unexpected Passes  ', Test.XPASS),
                      ('Unexpected Failures', Test.FAIL),):
        if opts.quiet and not code.isFailure:
            continue
        N = len(byCode.get(code,[]))
        if N:
            print '  %s: %d' % (name,N)

    # If we encountered any additional errors, exit abnormally.
    if litConfig.numErrors:
        print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
        sys.exit(2)

    # Warn about warnings.
    if litConfig.numWarnings:
        print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings

    if hasFailures:
        sys.exit(1)
    sys.exit(0)

if __name__=='__main__':
    main()
<commit_msg>lit: Shorten a metavar to make --help look nicer.<commit_after>#!/usr/bin/env python

"""
lit - LLVM Integrated Tester.

See lit.pod for more information.
"""

import math, os, platform, random, re, sys, time, threading, traceback

import ProgressBar
import TestRunner
import Util

import LitConfig
import Test

import lit.discovery

class TestingProgressDisplay:
    def __init__(self, opts, numTests, progressBar=None):
        self.opts = opts
        self.numTests = numTests
        self.current = None
        self.lock = threading.Lock()
        self.progressBar = progressBar
        self.completed = 0

    def update(self, test):
        # Avoid locking overhead in quiet mode
        if self.opts.quiet and not test.result.isFailure:
            self.completed += 1
            return

        # Output lock.
        self.lock.acquire()
        try:
            self.handleUpdate(test)
        finally:
            self.lock.release()

    def finish(self):
        if self.progressBar:
            self.progressBar.clear()
        elif self.opts.quiet:
            pass
        elif self.opts.succinct:
            sys.stdout.write('\n')

    def handleUpdate(self, test):
        self.completed += 1
        if self.progressBar:
            self.progressBar.update(float(self.completed)/self.numTests,
                                    test.getFullName())

        if self.opts.succinct and not test.result.isFailure:
            return

        if self.progressBar:
            self.progressBar.clear()

        print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
                                     self.completed, self.numTests)

        if test.result.isFailure and self.opts.showOutput:
            print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
                                              '*'*20)
            print test.output
            print "*" * 20

        sys.stdout.flush()

class TestProvider:
    def __init__(self, tests, maxTime):
        self.maxTime = maxTime
        self.iter = iter(tests)
        self.lock = threading.Lock()
        self.startTime = time.time()

    def get(self):
        # Check if we have run out of time.
        if self.maxTime is not None:
            if time.time() - self.startTime > self.maxTime:
                return None

        # Otherwise take the next test.
        self.lock.acquire()
        try:
            item = self.iter.next()
        except StopIteration:
            item = None
        self.lock.release()
        return item

class Tester(threading.Thread):
    def __init__(self, litConfig, provider, display):
        threading.Thread.__init__(self)
        self.litConfig = litConfig
        self.provider = provider
        self.display = display

    def run(self):
        while 1:
            item = self.provider.get()
            if item is None:
                break
            self.runTest(item)

    def runTest(self, test):
        result = None
        startTime = time.time()
        try:
            result, output = test.config.test_format.execute(test,
                                                             self.litConfig)
        except KeyboardInterrupt:
            # This is a sad hack. Unfortunately subprocess goes
            # bonkers with ctrl-c and we start forking merrily.
            print '\nCtrl-C detected, goodbye.'
            os.kill(0,9)
        except:
            if self.litConfig.debug:
                raise
            result = Test.UNRESOLVED
            output = 'Exception during script execution:\n'
            output += traceback.format_exc()
            output += '\n'
        elapsed = time.time() - startTime

        test.setResult(result, output, elapsed)
        self.display.update(test)

def runTests(numThreads, litConfig, provider, display):
    # If only using one testing thread, don't use threads at all; this lets us
    # profile, among other things.
    if numThreads == 1:
        t = Tester(litConfig, provider, display)
        t.run()
        return

    # Otherwise spin up the testing threads and wait for them to finish.
    testers = [Tester(litConfig, provider, display)
               for i in range(numThreads)]
    for t in testers:
        t.start()
    try:
        for t in testers:
            t.join()
    except KeyboardInterrupt:
        sys.exit(2)

def main(builtinParameters = {}):
    # Bump the GIL check interval, its more important to get any one thread to a
    # blocking operation (hopefully exec) than to try and unblock other threads.
    #
    # FIXME: This is a hack.
    import sys
    sys.setcheckinterval(1000)

    global options
    from optparse import OptionParser, OptionGroup
    parser = OptionParser("usage: %prog [options] {file-or-path}")

    parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
                      help="Number of testing threads",
                      type=int, action="store", default=None)
    parser.add_option("", "--config-prefix", dest="configPrefix",
                      metavar="NAME", help="Prefix for 'lit' config files",
                      action="store", default=None)
    parser.add_option("", "--param", dest="userParameters",
                      metavar="NAME=VAL",
                      help="Add 'NAME' = 'VAL' to the user defined parameters",
                      type=str, action="append", default=[])

    group = OptionGroup(parser, "Output Format")
    # FIXME: I find these names very confusing, although I like the
    # functionality.
    group.add_option("-q", "--quiet", dest="quiet",
                     help="Suppress no error output",
                     action="store_true", default=False)
    group.add_option("-s", "--succinct", dest="succinct",
                     help="Reduce amount of output",
                     action="store_true", default=False)
    group.add_option("-v", "--verbose", dest="showOutput",
                     help="Show all test output",
                     action="store_true", default=False)
    group.add_option("", "--no-progress-bar", dest="useProgressBar",
                     help="Do not use curses based progress bar",
                     action="store_false", default=True)
    parser.add_option_group(group)

    group = OptionGroup(parser, "Test Execution")
    group.add_option("", "--path", dest="path",
                     help="Additional paths to add to testing environment",
                     action="append", type=str, default=[])
    group.add_option("", "--vg", dest="useValgrind",
                     help="Run tests under valgrind",
                     action="store_true", default=False)
    group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
                     help="Check for memory leaks under valgrind",
                     action="store_true", default=False)
    group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
                     help="Specify an extra argument for valgrind",
                     type=str, action="append", default=[])
    group.add_option("", "--time-tests", dest="timeTests",
                     help="Track elapsed wall time for each test",
                     action="store_true", default=False)
    group.add_option("", "--no-execute", dest="noExecute",
                     help="Don't execute any tests (assume PASS)",
                     action="store_true", default=False)
    parser.add_option_group(group)

    group = OptionGroup(parser, "Test Selection")
    group.add_option("", "--max-tests", dest="maxTests", metavar="N",
                     help="Maximum number of tests to run",
                     action="store", type=int, default=None)
    group.add_option("", "--max-time", dest="maxTime", metavar="N",
                     help="Maximum time to spend testing (in seconds)",
                     action="store", type=float, default=None)
    group.add_option("", "--shuffle", dest="shuffle",
                     help="Run tests in random order",
                     action="store_true", default=False)
    group.add_option("", "--filter", dest="filter", metavar="REGEX",
                     help=("Only run tests with paths matching the given "
                           "regular expression"),
                     action="store", default=None)
    parser.add_option_group(group)

    group = OptionGroup(parser, "Debug and Experimental Options")
    group.add_option("", "--debug", dest="debug",
                      help="Enable debugging (for 'lit' development)",
                      action="store_true", default=False)
    group.add_option("", "--show-suites", dest="showSuites",
                      help="Show discovered test suites",
                      action="store_true", default=False)
    group.add_option("", "--repeat", dest="repeatTests", metavar="N",
                      help="Repeat tests N times (for timing)",
                      action="store", default=None, type=int)
    parser.add_option_group(group)

    (opts, args) = parser.parse_args()

    if not args:
        parser.error('No inputs specified')

    if opts.numThreads is None:
# Python <2.5 has a race condition causing lit to always fail with numThreads>1
# http://bugs.python.org/issue1731717
# I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
# threads by default there.
       if sys.hexversion >= 0x2050200:
               opts.numThreads = Util.detectCPUs()
       else:
               opts.numThreads = 1

    inputs = args

    # Create the user defined parameters.
    userParams = dict(builtinParameters)
    for entry in opts.userParameters:
        if '=' not in entry:
            name,val = entry,''
        else:
            name,val = entry.split('=', 1)
        userParams[name] = val

    # Create the global config object.
    litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
                                    path = opts.path,
                                    quiet = opts.quiet,
                                    useValgrind = opts.useValgrind,
                                    valgrindLeakCheck = opts.valgrindLeakCheck,
                                    valgrindArgs = opts.valgrindArgs,
                                    noExecute = opts.noExecute,
                                    ignoreStdErr = False,
                                    debug = opts.debug,
                                    isWindows = (platform.system()=='Windows'),
                                    params = userParams,
                                    config_prefix = opts.configPrefix)

    tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)

    if opts.showSuites:
        suitesAndTests = {}
        for t in tests:
            if t.suite not in suitesAndTests:
                suitesAndTests[t.suite] = []
            suitesAndTests[t.suite].append(t)

        print '-- Test Suites --'
        suitesAndTests = suitesAndTests.items()
        suitesAndTests.sort(key = lambda (ts,_): ts.name)
        for ts,ts_tests in suitesAndTests:
            print '  %s - %d tests' %(ts.name, len(ts_tests))
            print '    Source Root: %s' % ts.source_root
            print '    Exec Root  : %s' % ts.exec_root

    # Select and order the tests.
    numTotalTests = len(tests)

    # First, select based on the filter expression if given.
    if opts.filter:
        try:
            rex = re.compile(opts.filter)
        except:
            parser.error("invalid regular expression for --filter: %r" % (
                    opts.filter))
        tests = [t for t in tests
                 if rex.search(t.getFullName())]

    # Then select the order.
    if opts.shuffle:
        random.shuffle(tests)
    else:
        tests.sort(key = lambda t: t.getFullName())

    # Finally limit the number of tests, if desired.
    if opts.maxTests is not None:
        tests = tests[:opts.maxTests]

    # Don't create more threads than tests.
    opts.numThreads = min(len(tests), opts.numThreads)

    extra = ''
    if len(tests) != numTotalTests:
        extra = ' of %d' % numTotalTests
    header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
                                                      opts.numThreads)

    if opts.repeatTests:
        tests = [t.copyWithIndex(i)
                 for t in tests
                 for i in range(opts.repeatTests)]

    progressBar = None
    if not opts.quiet:
        if opts.succinct and opts.useProgressBar:
            try:
                tc = ProgressBar.TerminalController()
                progressBar = ProgressBar.ProgressBar(tc, header)
            except ValueError:
                print header
                progressBar = ProgressBar.SimpleProgressBar('Testing: ')
        else:
            print header

    startTime = time.time()
    display = TestingProgressDisplay(opts, len(tests), progressBar)
    provider = TestProvider(tests, opts.maxTime)
    runTests(opts.numThreads, litConfig, provider, display)
    display.finish()

    if not opts.quiet:
        print 'Testing Time: %.2fs'%(time.time() - startTime)

    # Update results for any tests which weren't run.
    for t in tests:
        if t.result is None:
            t.setResult(Test.UNRESOLVED, '', 0.0)

    # List test results organized by kind.
    hasFailures = False
    byCode = {}
    for t in tests:
        if t.result not in byCode:
            byCode[t.result] = []
        byCode[t.result].append(t)
        if t.result.isFailure:
            hasFailures = True

    # FIXME: Show unresolved and (optionally) unsupported tests.
    for title,code in (('Unexpected Passing Tests', Test.XPASS),
                       ('Failing Tests', Test.FAIL)):
        elts = byCode.get(code)
        if not elts:
            continue
        print '*'*20
        print '%s (%d):' % (title, len(elts))
        for t in elts:
            print '    %s' % t.getFullName()
        print

    if opts.timeTests:
        # Collate, in case we repeated tests.
        times = {}
        for t in tests:
            key = t.getFullName()
            times[key] = times.get(key, 0.) + t.elapsed

        byTime = list(times.items())
        byTime.sort(key = lambda (name,elapsed): elapsed)
        if byTime:
            Util.printHistogram(byTime, title='Tests')

    for name,code in (('Expected Passes    ', Test.PASS),
                      ('Expected Failures  ', Test.XFAIL),
                      ('Unsupported Tests  ', Test.UNSUPPORTED),
                      ('Unresolved Tests   ', Test.UNRESOLVED),
                      ('Unexpected Passes  ', Test.XPASS),
                      ('Unexpected Failures', Test.FAIL),):
        if opts.quiet and not code.isFailure:
            continue
        N = len(byCode.get(code,[]))
        if N:
            print '  %s: %d' % (name,N)

    # If we encountered any additional errors, exit abnormally.
    if litConfig.numErrors:
        print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
        sys.exit(2)

    # Warn about warnings.
    if litConfig.numWarnings:
        print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings

    if hasFailures:
        sys.exit(1)
    sys.exit(0)

if __name__=='__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python
from datatype import *
import os
import shutil

class DataMaker:
    def __init__(self,Object,**option):
        self.startidx = option.get("startidx", 0)
        self.directory = option.get("directory", "input")
        self.name = option.get("name", "input")
        self.extension = option.get("extension", "in")
        self.Object=Object
        self.option=option

        if os.path.exists(self.directory):
            shutil.rmtree(self.directory)

        os.mkdir(self.directory)

    def __call__(self,Object,**option):
        self.__init__(self,Object,**option)

    def create(self,dataset):
        for i in range(len(dataset)):
            f = open(self.directory + "/" + self.name + str(i + self.startidx) + '.' + self.extension, "w")
            f.write(str(self.Object(dataset[i][0], dataset[i][1],**dataset[i][2])))
            print dataset[i][0],dataset[i][1]
            f.close()

class LinearDataMaker(DataMaker):
    def __init__(self,Object,testcase,**option):
        DataMaker.__init__(self,Object,**option)

        self.interval=option.get("interval",0)
        self.start=option.get("start",0)
        dataset=[]
        for i in range(testcase):
            dataset.append([self.start+self.interval*i,self.start+self.interval*(i+1),option])
        self.create(dataset)

class GroupDataMaker(DataMaker):
    def __init__(self,Object,group,**option):
        DataMaker.__init__(self, Object, **option)

        dataset=[]
        for g in group:
            low=g['nlow']
            high=g['nhigh']
            num=g['num']
            for i in range(num):
                dataset.append([low,high,g])
        self.create(dataset)

DT=DataType
S=Sequence
G=Graph
BG=BipartiteGraph
DM=DataMaker
LDM=LinearDataMaker
GDM=GroupDataMaker

"""
group=[
    {
        'nlow':1,
        'nhigh':10,
        'num':5
    },
    {
        'nlow':10,
        'nhigh':100,
        'num':5
    }
]
GDM(S,group)
"""

<commit_msg>make main for example<commit_after>#!/usr/bin/python
from datatype import *
import os
import shutil

class DataMaker:
    def __init__(self,Object,**option):
        self.startidx = option.get("startidx", 0)
        self.directory = option.get("directory", "input")
        self.name = option.get("name", "input")
        self.extension = option.get("extension", "in")
        self.Object=Object
        self.option=option

        if os.path.exists(self.directory):
            shutil.rmtree(self.directory)

        os.mkdir(self.directory)

    def __call__(self,Object,**option):
        self.__init__(self,Object,**option)

    def create(self,dataset):
        for i in range(len(dataset)):
            f = open(self.directory + "/" + self.name + str(i + self.startidx) + '.' + self.extension, "w")
            f.write(str(self.Object(dataset[i][0], dataset[i][1],**dataset[i][2])))
            print dataset[i][0],dataset[i][1]
            f.close()

class LinearDataMaker(DataMaker):
    def __init__(self,Object,testcase,**option):
        DataMaker.__init__(self,Object,**option)

        self.interval=option.get("interval",0)
        self.start=option.get("start",0)
        dataset=[]
        for i in range(testcase):
            dataset.append([self.start+self.interval*i,self.start+self.interval*(i+1),option])
        self.create(dataset)

class GroupDataMaker(DataMaker):
    def __init__(self,Object,group,**option):
        DataMaker.__init__(self, Object, **option)

        dataset=[]
        for g in group:
            low=g['nlow']
            high=g['nhigh']
            num=g['num']
            for i in range(num):
                dataset.append([low,high,g])
        self.create(dataset)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2010 University of Chicago
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Demonstrate API calls.

Example run using standard globus toolkit certificate locations:

python example.py USERNAME -k ~/.globus/userkey.pem -c ~/.globus/usercert.pem
"""
import time
from datetime import datetime, timedelta
import traceback
import json, os
import re, sys

from globusonline.transfer.api_client import Transfer, create_client_from_args
from globusonline.transfer.api_client import ActivationRequirementList
from globusonline.transfer.api_client import TransferAPIClient

# TransferAPIClient instance.
api = None

def mover(username, src_site, dst_site, dst_dir):
    """
    Do a bunch of API calls and display the results. Does a small transfer
    between tutorial endpoints, but otherwise does not modify user data.

    Uses module global API client instance.
    """
    global api
    #activer=[username,"-c",os.getcwd()+"/credential.pem"]
    #api, _ = create_client_from_args(activer)
    user_credential_path=os.getcwd()+"/credential.pem"
    print "user_credential_path=",user_credential_path
    api = TransferAPIClient(username, cert_file=user_credential_path)
    #print " Here (mover): ",api.task_list()
    # See what is in the account before we make any submissions.
    print "=== Before transfer ==="
    #display_tasksummary(); print
    #display_task_list(); print
    #display_endpoint_list(); print

    print "=== Activate endpoints ==="
    print "== If asked below, insert the password of your myproxy server =="
    print "== If asked after two \"Activating EP\" strings, insert the password of your GO account =="
    site_ep1 = src_site
    site_ep2 = dst_site
    site_username = username
    dest_directory= dst_dir
    api.set_debug_print(False, False)
    conditional_activation(site_ep1,site_username)
    conditional_activation(site_ep2,site_username)

    print "=== Prepare transfer ==="
    #raw_input("Press Enter to continue...")
    # submit a transfer
    oldstdout=sys.stdout
    sys.stdout = open(os.devnull,'w')
    code, message, data = api.transfer_submission_id()
    sys.stdout = oldstdout # enable output
    #code, message, data = api.transfer_submission_id()
    submission_id = data["value"]
    deadline = datetime.utcnow() + timedelta(minutes=10)
    t = Transfer(submission_id, site_ep1, site_ep2)#, deadline)

    f=open('json_file','r')
    json_results=f.read()
    f.close
    #print json_results,type(json_results)
    results=json.loads(json_results)
    #print results,type(results)
    #print results[0],type(results[0])
    for result in results:
        #print "Result: ",result
        if result[-1]=="/":
            #print "Result: it is a directory"
            t.add_item(result, dest_directory, recursive=True)
        else:
            #print "Result: it is a file"
            file_name=re.split("/",result)[-1]
            #print "Result: "+file_name
            t.add_item(result, dest_directory+"/"+file_name)

    print "=== Submit transfer ==="
    oldstdout=sys.stdout
    sys.stdout = open(os.devnull,'w')
    code, reason, data = api.transfer(t)
    sys.stdout = oldstdout # enable output
    #code, reason, data = api.transfer(t)
    task_id = data["task_id"]
    #print " Task ID is %s " % (task_id)

    # see the new transfer show up
    #print "=== After submit ==="
    #display_tasksummary(); print
    #display_task(task_id); print
    #raw_input("Press Enter to continue...")

    # wait for the task to complete, and see the summary and lists
    # update
    print "=== Checking completion ==="
    # To save the task_id for further check could be useful.
    #wait_for_task(task_id)
    max_wait = 10*1
    if wait_for_task(task_id,max_wait):
        print " Task %s is still under process " % (task_id)
        oldstdout=sys.stdout
        sys.stdout = open(os.devnull,'w')
        display_tasksummary(); print
        sys.stdout = oldstdout # enable output
        #display_task(task_id); print
        #display_ls("cin0641a#GSI-PLX"); print
    print "=== Exiting ==="
    #display_tasksummary(); print
    print "The task ID for this operation is: "+task_id; print
    oldstdout=sys.stdout
    sys.stdout = open(os.devnull,'w')
    status, reason, result = api.task(task_id)
    sys.stdout = oldstdout # enable output
    print "Its status is "+result["status"]; print

def lookforurl(username, task_id):
    """
    Uses module global API client instance.
    """
    activer=[username,"-c",os.getcwd()+"/credential.pem"]
    global api
    api, _ = create_client_from_args(activer)
    #print " Here: ",api.task_list()
    # See what is in the account before we make any submissions.
    #print "=== Before transfer ==="
    #display_tasksummary(); print
    #display_task_list(); print
    #display_endpoint_list(); print
    #status, reason, result = api.task(task_id)
    status, reason, result = api.subtask_list(task_id)
    #print "Transfer status is: "+result["status"]
    #print "Transfer command was: "+result["command"]
    #print result; print
    #print result["subtask_link"]; print
    #print result["subtask_link"]["resource"],type(result["subtask_link"]["resource"])
    inurllist    = []
    outurllist   = []
    destendpoint = []
    for subtask in result["DATA"]:
        #print subtask
        #print subtask["source_path"], subtask["destination_path"], subtask["destination_endpoint"]
        inurllist.append(subtask["source_path"])
        outurllist.append(subtask["destination_path"])
        destendpoint.append(re.split("#",subtask["destination_endpoint"])[1])
    return inurllist, outurllist, destendpoint

def conditional_activation(endpoint_name,site_username):
    api.endpoint_autoactivate(endpoint_name)
    code, reason, endpoint_list = api.endpoint_list(limit=100)
    #print "Found %d endpoints for user %s:" % (endpoint_list["length"], api.username)
    for ep in endpoint_list["DATA"]:
        if ep["name"] == endpoint_name:
                if ep["activated"]:
                        print " %s is already active. " % (endpoint_name)
                else:
                        print " Activating %s " % (endpoint_name)
                        status, message, data = api.endpoint_autoactivate(endpoint_name)
                        data.set_requirement_value("myproxy", "username", site_username)
                        from getpass import getpass
                        passphrase = getpass()
                        data.set_requirement_value("myproxy", "passphrase", passphrase)
                        api.endpoint_activate(endpoint_name, data)

def display_activation(endpoint_name):
    print "=== Endpoint pre-activation ==="
    display_endpoint(endpoint_name)
    print
    code, reason, result = api.endpoint_autoactivate(endpoint_name,
                                                     if_expires_in=600)
    if result["code"].startswith("AutoActivationFailed"):
        print "Auto activation failed, ls and transfers will likely fail!"
    print "result: %s (%s)" % (result["code"], result["message"])
    print "=== Endpoint post-activation ==="
    display_endpoint(endpoint_name)
    print

def display_tasksummary():
    print " Here: ",api.task_list()
    code, reason, data = api.tasksummary()
    print "Task Summary for %s:" % api.username
    for k, v in data.iteritems():
        if k == "DATA_TYPE":
            continue
        print "%3d %s" % (int(v), k.upper().ljust(9))

def display_task_list(max_age=None):
    """
    @param max_age: only show tasks requested at or after now - max_age.
    @type max_age: timedelta
    """
    kwargs = {}
    if max_age:
        min_request_time = datetime.utcnow() - max_age
        # filter on request_time starting at min_request_time, with no
        # upper limit on request_time.
        kwargs["request_time"] = "%s," % min_request_time

    code, reason, task_list = api.task_list(**kwargs)
    print "task_list for %s:" % api.username
    for task in task_list["DATA"]:
        print "Task %s:" % task["task_id"]
        _print_task(task)

def _print_task(data, indent_level=0):
    """
    Works for tasks and subtasks, since both have a task_id key
    and other key/values are printed by iterating through the items.
    """
    indent = " " * indent_level
    indent += " " * 2
    for k, v in data.iteritems():
        if k in ("DATA_TYPE", "LINKS"):
            continue
        print indent + "%s: %s" % (k, v)

def display_task(task_id, show_subtasks=True):
    code, reason, data = api.task(task_id)
    print "Task %s:" % task_id
    _print_task(data, 0)

    if show_subtasks:
        code, reason, data = api.subtask_list(task_id)
        subtask_list = data["DATA"]
        for t in subtask_list:
            print "  subtask %s:" % t["task_id"]
            _print_task(t, 4)

def wait_for_task(task_id, timeout=-1):
    status = "ACTIVE"
    while timeout and status == "ACTIVE":
        code, reason, data = api.task(task_id, fields="status")
        status = data["status"]
        time.sleep(1)
        timeout -= 1

    if status != "ACTIVE":
        print "Task %s complete!" % task_id
        return True
    else:
        print "Task still not complete after %d seconds" % timeout
        return False

def display_endpoint_list():
    code, reason, endpoint_list = api.endpoint_list(limit=100)
    print "Found %d endpoints for user %s:" \
          % (endpoint_list["length"], api.username)
    for ep in endpoint_list["DATA"]:
        _print_endpoint(ep)

def display_endpoint(endpoint_name):
    code, reason, data = api.endpoint(endpoint_name)
    _print_endpoint(data)

def _print_endpoint(ep):
    name = ep["canonical_name"]
    print name
    if ep["activated"]:
        print "  activated (expires: %s)" % ep["expire_time"]
    else:
        print "  not activated"
    if ep["public"]:
        print "  public"
    else:
        print "  not public"
    if ep["myproxy_server"]:
        print "  default myproxy server: %s" % ep["myproxy_server"]
    else:
        print "  no default myproxy server"
    servers = ep.get("DATA", ())
    print "  servers:"
    for s in servers:
        uri = s["uri"]
        if not uri:
            uri = "GC endpoint, no uri available"
        print "    " + uri,
        if s["subject"]:
            print " (%s)" % s["subject"]
        else:
            print

def unicode_(data):
    """
    Coerce any type to unicode, assuming utf-8 encoding for strings.
    """
    if isinstance(data, unicode):
        return data
    if isinstance(data, str):
        return unicode(data, "utf-8")
    else:
        return unicode(data)

def display_ls(endpoint_name, path=""):
    code, reason, data = api.endpoint_ls(endpoint_name, path)
    # Server returns canonical path; "" maps to the users default path,
    # which is typically their home directory "/~/".
    path = data["path"]
    print "Contents of %s on %s:" % (path, endpoint_name)
    headers = "name, type, permissions, size, user, group, last_modified"
    headers_list = headers.split(", ")
    print headers
    for f in data["DATA"]:
        print ", ".join([unicode_(f[k]) for k in headers_list])

##################################################################################
# Old code...
##################################################################################
def tutorial():
    """
    Do a bunch of API calls and display the results. Does a small transfer
    between tutorial endpoints, but otherwise does not modify user data.

    Uses module global API client instance.
    """
    # See what is in the account before we make any submissions.
    print "=== Before transfer ==="
    display_tasksummary(); print
    #display_task_list(); print
    #display_endpoint_list(); print

    print "=== Activate endpoints ==="
    site_ep1 = "irods-dev"
    site_ep2 = "GSI-PLX"
    site_username = "cin0641a"
    dest_directory= "/~/tmp/"
    api.set_debug_print(False, True)
    conditional_activation(site_ep1,site_username)
    conditional_activation(site_ep2,site_username)

    print "=== Prepare transfer ==="
    #raw_input("Press Enter to continue...")
    # submit a transfer
    code, message, data = api.transfer_submission_id()
    submission_id = data["value"]
    deadline = datetime.utcnow() + timedelta(minutes=10)
    t = Transfer(submission_id, site_ep1, site_ep2)#, deadline)

    f=open('json_file','r')
    json_results=f.read()
    f.close
    #print json_results,type(json_results)
    results=json.loads(json_results)
    #print results,type(results)
    #print results[0],type(results[0])
    for result in results:
        print "Result:", result
        t.add_item(result, dest_directory, recursive=True)

    print "=== Submit transfer ==="
    code, reason, data = api.transfer(t)
    task_id = data["task_id"]
    print " Task ID is %s " % (task_id)

    # see the new transfer show up
    #print "=== After submit ==="
    #display_tasksummary(); print
    #display_task(task_id); print
    #raw_input("Press Enter to continue...")

    # wait for the task to complete, and see the summary and lists
    # update
    print "=== Checking completion ==="
    # To save the task_id for further check could be useful.
    #wait_for_task(task_id)
    max_wait = 60*1
    if wait_for_task(task_id,max_wait):
        print " Task %s is still under process " % (task_id)
        display_tasksummary(); print
        #display_task(task_id); print
        #display_ls("cin0641a#GSI-PLX"); print
    print "=== Exiting ==="
    display_tasksummary(); print
##################################################################################
# End of old code...
##################################################################################

if __name__ == '__main__':
    api, _ = create_client_from_args()
    #print " Here: ",api.task_list()
    #sys.exit(0)
    tutorial()
<commit_msg>Some improvements in the use of only local certificates. <commit_after>#!/usr/bin/env python

# Copyright 2010 University of Chicago
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Demonstrate API calls.

Example run using standard globus toolkit certificate locations:

python example.py USERNAME -k ~/.globus/userkey.pem -c ~/.globus/usercert.pem
"""
import time
from datetime import datetime, timedelta
import traceback
import json, os
import re, sys

from globusonline.transfer.api_client import Transfer, create_client_from_args
from globusonline.transfer.api_client import ActivationRequirementList
from globusonline.transfer.api_client import TransferAPIClient
from globusonline.transfer.api_client import x509_proxy

# TransferAPIClient instance.
api = None

def mover(username, src_site, dst_site, dst_dir):
    """
    Do a bunch of API calls and display the results. Does a small transfer
    between tutorial endpoints, but otherwise does not modify user data.

    Uses module global API client instance.
    """
    global api
    #activer=[username,"-c",os.getcwd()+"/credential.pem"]
    #api, _ = create_client_from_args(activer)
    user_credential_path=os.getcwd()+"/credential.pem"
    print "user_credential_path=",user_credential_path
    api = TransferAPIClient(username, cert_file=user_credential_path)
    api.set_debug_print(False, False)
    #print " Here (mover): ",api.task_list()
    # See what is in the account before we make any submissions.
    print "=== Before transfer ==="
    #display_tasksummary(); print
    #display_task_list(); print
    #display_endpoint_list(); print

    print "=== Activate endpoints ==="
    #print "== If asked below, insert the password of your myproxy server =="
    #print "== If asked after two \"Activating EP\" strings, insert the password of your GO account =="
    site_username = username
    dest_directory= dst_dir
    site_ep1 = src_site
    site_ep2 = dst_site

    _, _, reqs = api.endpoint_activation_requirements(site_ep1, type="delegate_proxy")
    print "site_ep1"
    print reqs
    public_key = reqs.get_requirement_value("delegate_proxy", "public_key")
    print public_key
    proxy = x509_proxy.create_proxy_from_file(user_credential_path, public_key)
    print proxy
    reqs.set_requirement_value("delegate_proxy", "proxy_chain", proxy)
    print reqs
    result = api.endpoint_activate(site_ep1, reqs)
    print result

    _, _, reqs = api.endpoint_activation_requirements(site_ep2, type="delegate_proxy")
    print "site_ep2"
    print reqs
    public_key = reqs.get_requirement_value("delegate_proxy", "public_key")
    print public_key
    proxy = x509_proxy.create_proxy_from_file(user_credential_path, public_key)
    reqs.set_requirement_value("delegate_proxy", "proxy_chain", proxy)
    print reqs
    result = api.endpoint_activate(site_ep2, reqs)
    print result

    #conditional_activation(site_ep1,site_username)
    #conditional_activation(site_ep2,site_username)

    print "=== Prepare transfer ==="
    #raw_input("Press Enter to continue...")
    # submit a transfer
    oldstdout=sys.stdout
    sys.stdout = open(os.devnull,'w')
    code, message, data = api.transfer_submission_id()
    sys.stdout = oldstdout # enable output
    #code, message, data = api.transfer_submission_id()
    submission_id = data["value"]
    deadline = datetime.utcnow() + timedelta(minutes=10)
    t = Transfer(submission_id, site_ep1, site_ep2)#, deadline)

    f=open('json_file','r')
    json_results=f.read()
    f.close
    #print json_results,type(json_results)
    results=json.loads(json_results)
    #print results,type(results)
    #print results[0],type(results[0])
    for result in results:
        #print "Result: ",result
        if result[-1]=="/":
            #print "Result: it is a directory"
            t.add_item(result, dest_directory, recursive=True)
        else:
            #print "Result: it is a file"
            file_name=re.split("/",result)[-1]
            #print "Result: "+file_name
            t.add_item(result, dest_directory+"/"+file_name)

    print "=== Submit transfer ==="
    oldstdout=sys.stdout
    sys.stdout = open(os.devnull,'w')
    code, reason, data = api.transfer(t)
    sys.stdout = oldstdout # enable output
    #code, reason, data = api.transfer(t)
    task_id = data["task_id"]
    #print " Task ID is %s " % (task_id)

    # see the new transfer show up
    #print "=== After submit ==="
    #display_tasksummary(); print
    #display_task(task_id); print
    #raw_input("Press Enter to continue...")

    # wait for the task to complete, and see the summary and lists
    # update
    print "=== Checking completion ==="
    # To save the task_id for further check could be useful.
    #wait_for_task(task_id)
    max_wait = 10*1
    if wait_for_task(task_id,max_wait):
        print " Task %s is still under process " % (task_id)
        oldstdout=sys.stdout
        sys.stdout = open(os.devnull,'w')
        display_tasksummary(); print
        sys.stdout = oldstdout # enable output
        #display_task(task_id); print
        #display_ls("cin0641a#GSI-PLX"); print
    print "=== Exiting ==="
    #display_tasksummary(); print
    print "The task ID for this operation is: "+task_id; print
    oldstdout=sys.stdout
    sys.stdout = open(os.devnull,'w')
    status, reason, result = api.task(task_id)
    sys.stdout = oldstdout # enable output
    print "Its status is "+result["status"]; print

def lookforurl(username, task_id):
    """
    Uses module global API client instance.
    """
    activer=[username,"-c",os.getcwd()+"/credential.pem"]
    global api
    api, _ = create_client_from_args(activer)
    #print " Here: ",api.task_list()
    # See what is in the account before we make any submissions.
    #print "=== Before transfer ==="
    #display_tasksummary(); print
    #display_task_list(); print
    #display_endpoint_list(); print
    #status, reason, result = api.task(task_id)
    status, reason, result = api.subtask_list(task_id)
    #print "Transfer status is: "+result["status"]
    #print "Transfer command was: "+result["command"]
    #print result; print
    #print result["subtask_link"]; print
    #print result["subtask_link"]["resource"],type(result["subtask_link"]["resource"])
    inurllist    = []
    outurllist   = []
    destendpoint = []
    for subtask in result["DATA"]:
        #print subtask
        #print subtask["source_path"], subtask["destination_path"], subtask["destination_endpoint"]
        inurllist.append(subtask["source_path"])
        outurllist.append(subtask["destination_path"])
        destendpoint.append(re.split("#",subtask["destination_endpoint"])[1])
    return inurllist, outurllist, destendpoint

def conditional_activation(endpoint_name,site_username):
    api.endpoint_autoactivate(endpoint_name)
    code, reason, endpoint_list = api.endpoint_list(limit=100)
    #print "Found %d endpoints for user %s:" % (endpoint_list["length"], api.username)
    for ep in endpoint_list["DATA"]:
        if ep["name"] == endpoint_name:
                if ep["activated"]:
                        print " %s is already active. " % (endpoint_name)
                else:
                        print " Activating %s " % (endpoint_name)
                        status, message, data = api.endpoint_autoactivate(endpoint_name)
                        data.set_requirement_value("myproxy", "username", site_username)
                        from getpass import getpass
                        passphrase = getpass()
                        data.set_requirement_value("myproxy", "passphrase", passphrase)
                        api.endpoint_activate(endpoint_name, data)

def display_activation(endpoint_name):
    print "=== Endpoint pre-activation ==="
    display_endpoint(endpoint_name)
    print
    code, reason, result = api.endpoint_autoactivate(endpoint_name,
                                                     if_expires_in=600)
    if result["code"].startswith("AutoActivationFailed"):
        print "Auto activation failed, ls and transfers will likely fail!"
    print "result: %s (%s)" % (result["code"], result["message"])
    print "=== Endpoint post-activation ==="
    display_endpoint(endpoint_name)
    print

def display_tasksummary():
    print " Here: ",api.task_list()
    code, reason, data = api.tasksummary()
    print "Task Summary for %s:" % api.username
    for k, v in data.iteritems():
        if k == "DATA_TYPE":
            continue
        print "%3d %s" % (int(v), k.upper().ljust(9))

def display_task_list(max_age=None):
    """
    @param max_age: only show tasks requested at or after now - max_age.
    @type max_age: timedelta
    """
    kwargs = {}
    if max_age:
        min_request_time = datetime.utcnow() - max_age
        # filter on request_time starting at min_request_time, with no
        # upper limit on request_time.
        kwargs["request_time"] = "%s," % min_request_time

    code, reason, task_list = api.task_list(**kwargs)
    print "task_list for %s:" % api.username
    for task in task_list["DATA"]:
        print "Task %s:" % task["task_id"]
        _print_task(task)

def _print_task(data, indent_level=0):
    """
    Works for tasks and subtasks, since both have a task_id key
    and other key/values are printed by iterating through the items.
    """
    indent = " " * indent_level
    indent += " " * 2
    for k, v in data.iteritems():
        if k in ("DATA_TYPE", "LINKS"):
            continue
        print indent + "%s: %s" % (k, v)

def display_task(task_id, show_subtasks=True):
    code, reason, data = api.task(task_id)
    print "Task %s:" % task_id
    _print_task(data, 0)

    if show_subtasks:
        code, reason, data = api.subtask_list(task_id)
        subtask_list = data["DATA"]
        for t in subtask_list:
            print "  subtask %s:" % t["task_id"]
            _print_task(t, 4)

def wait_for_task(task_id, timeout=-1):
    status = "ACTIVE"
    while timeout and status == "ACTIVE":
        code, reason, data = api.task(task_id, fields="status")
        status = data["status"]
        time.sleep(1)
        timeout -= 1

    if status != "ACTIVE":
        print "Task %s complete!" % task_id
        return True
    else:
        print "Task still not complete after %d seconds" % timeout
        return False

def display_endpoint_list():
    code, reason, endpoint_list = api.endpoint_list(limit=100)
    print "Found %d endpoints for user %s:" \
          % (endpoint_list["length"], api.username)
    for ep in endpoint_list["DATA"]:
        _print_endpoint(ep)

def display_endpoint(endpoint_name):
    code, reason, data = api.endpoint(endpoint_name)
    _print_endpoint(data)

def _print_endpoint(ep):
    name = ep["canonical_name"]
    print name
    if ep["activated"]:
        print "  activated (expires: %s)" % ep["expire_time"]
    else:
        print "  not activated"
    if ep["public"]:
        print "  public"
    else:
        print "  not public"
    if ep["myproxy_server"]:
        print "  default myproxy server: %s" % ep["myproxy_server"]
    else:
        print "  no default myproxy server"
    servers = ep.get("DATA", ())
    print "  servers:"
    for s in servers:
        uri = s["uri"]
        if not uri:
            uri = "GC endpoint, no uri available"
        print "    " + uri,
        if s["subject"]:
            print " (%s)" % s["subject"]
        else:
            print

def unicode_(data):
    """
    Coerce any type to unicode, assuming utf-8 encoding for strings.
    """
    if isinstance(data, unicode):
        return data
    if isinstance(data, str):
        return unicode(data, "utf-8")
    else:
        return unicode(data)

def display_ls(endpoint_name, path=""):
    code, reason, data = api.endpoint_ls(endpoint_name, path)
    # Server returns canonical path; "" maps to the users default path,
    # which is typically their home directory "/~/".
    path = data["path"]
    print "Contents of %s on %s:" % (path, endpoint_name)
    headers = "name, type, permissions, size, user, group, last_modified"
    headers_list = headers.split(", ")
    print headers
    for f in data["DATA"]:
        print ", ".join([unicode_(f[k]) for k in headers_list])

##################################################################################
# Old code...
##################################################################################
def tutorial():
    """
    Do a bunch of API calls and display the results. Does a small transfer
    between tutorial endpoints, but otherwise does not modify user data.

    Uses module global API client instance.
    """
    # See what is in the account before we make any submissions.
    print "=== Before transfer ==="
    display_tasksummary(); print
    #display_task_list(); print
    #display_endpoint_list(); print

    print "=== Activate endpoints ==="
    site_ep1 = "irods-dev"
    site_ep2 = "GSI-PLX"
    site_username = "cin0641a"
    dest_directory= "/~/tmp/"
    api.set_debug_print(False, True)
    conditional_activation(site_ep1,site_username)
    conditional_activation(site_ep2,site_username)

    print "=== Prepare transfer ==="
    #raw_input("Press Enter to continue...")
    # submit a transfer
    code, message, data = api.transfer_submission_id()
    submission_id = data["value"]
    deadline = datetime.utcnow() + timedelta(minutes=10)
    t = Transfer(submission_id, site_ep1, site_ep2)#, deadline)

    f=open('json_file','r')
    json_results=f.read()
    f.close
    #print json_results,type(json_results)
    results=json.loads(json_results)
    #print results,type(results)
    #print results[0],type(results[0])
    for result in results:
        print "Result:", result
        t.add_item(result, dest_directory, recursive=True)

    print "=== Submit transfer ==="
    code, reason, data = api.transfer(t)
    task_id = data["task_id"]
    print " Task ID is %s " % (task_id)

    # see the new transfer show up
    #print "=== After submit ==="
    #display_tasksummary(); print
    #display_task(task_id); print
    #raw_input("Press Enter to continue...")

    # wait for the task to complete, and see the summary and lists
    # update
    print "=== Checking completion ==="
    # To save the task_id for further check could be useful.
    #wait_for_task(task_id)
    max_wait = 60*1
    if wait_for_task(task_id,max_wait):
        print " Task %s is still under process " % (task_id)
        display_tasksummary(); print
        #display_task(task_id); print
        #display_ls("cin0641a#GSI-PLX"); print
    print "=== Exiting ==="
    display_tasksummary(); print
##################################################################################
# End of old code...
##################################################################################

if __name__ == '__main__':
    api, _ = create_client_from_args()
    #print " Here: ",api.task_list()
    #sys.exit(0)
    tutorial()
<|endoftext|>