<commit_before>from __future__ import generators
import py
from py.__.misc import rest
from py.__.apigen.linker import relpath
import os

pypkgdir = py.path.local(py.__file__).dirpath()

mypath = py.magic.autopath().dirpath()

TIMEOUT_URLOPEN = 5.0

Option = py.test.config.Option
option = py.test.config.addoptions("documentation check options",
        Option('-R', '--checkremote',
               action="store_true", dest="checkremote", default=False,
               help="urlopen() remote links found in ReST text files.",
        ),
        Option('', '--forcegen',
               action="store_true", dest="forcegen", default=False,
               help="force generation of html files even if they appear up-to-date"
        ),
)

def get_apigenpath():
    from py.__.conftest import option
    path = os.environ.get('APIGENPATH')
    if path is None:
        path = option.apigenpath
    return pypkgdir.join(path, abs=True)

def get_docpath():
    from py.__.conftest import option
    path = os.environ.get('DOCPATH')
    if path is None:
        path = option.docpath
    return pypkgdir.join(path, abs=True)

def get_apigen_relpath():
    return relpath(get_docpath().strpath + '/',
                   get_apigenpath().strpath + '/')

def deindent(s, sep='\n'):
    leastspaces = -1
    lines = s.split(sep)
    for line in lines:
        if not line.strip():
            continue
        spaces = len(line) - len(line.lstrip())
        if leastspaces == -1 or spaces < leastspaces:
            leastspaces = spaces
    if leastspaces == -1:
        return s
    for i, line in py.builtin.enumerate(lines):
        if not line.strip():
            lines[i] = ''
        else:
            lines[i] = line[leastspaces:]
    return sep.join(lines)

_initialized = False
def checkdocutils():
    global _initialized
    try:
        import docutils
    except ImportError:
        py.test.skip("docutils not importable")
    if not _initialized:
        from py.__.rest import directive
        directive.register_linkrole('api', resolve_linkrole)
        directive.register_linkrole('source', resolve_linkrole)
        _initialized = True

def restcheck(path):
    localpath = path
    if hasattr(path, 'localpath'):
        localpath = path.localpath
    checkdocutils()
    import docutils.utils

    try:
        cur = localpath
        for x in cur.parts(reverse=True):
            confrest = x.dirpath('confrest.py')
            if confrest.check(file=1):
                confrest = confrest.pyimport()
                project = confrest.Project()
                _checkskip(path, project.get_htmloutputpath(path))
                project.process(path)
                break
        else:
            # defer to default processor
            _checkskip(path)
            rest.process(path)
    except KeyboardInterrupt:
        raise
    except docutils.utils.SystemMessage:
        # we assume docutils printed info on stdout
        py.test.fail("docutils processing failed, see captured stderr")

def _checkskip(lpath, htmlpath=None):
    if not option.forcegen:
        lpath = py.path.local(lpath)
        if htmlpath is not None:
            htmlpath = py.path.local(htmlpath)
        if lpath.ext == '.txt':
            htmlpath = htmlpath or lpath.new(ext='.html')
            if htmlpath.check(file=1) and htmlpath.mtime() >= lpath.mtime():
                py.test.skip("html file is up to date, use --forcegen to regenerate")
                #return [] # no need to rebuild

class ReSTSyntaxTest(py.test.collect.Item):
    def runtest(self):
        mypath = self.fspath
        restcheck(py.path.svnwc(mypath))

class DoctestText(py.test.collect.Item):
    def runtest(self):
        s = self._normalize_linesep()
        l = []
        prefix = '.. >>> '
        mod = py.std.types.ModuleType(self.fspath.purebasename)
        skipchunk = False
        for line in deindent(s).split('\n'):
            stripped = line.strip()
            if skipchunk and line.startswith(skipchunk):
                print "skipping", line
                continue
            skipchunk = False
            if stripped.startswith(prefix):
                try:
                    exec py.code.Source(stripped[len(prefix):]).compile() in \
                        mod.__dict__
                except ValueError, e:
                    if e.args and e.args[0] == "skipchunk":
                        skipchunk = " " * (len(line) - len(line.lstrip()))
                    else:
                        raise
            else:
                l.append(line)
        docstring = "\n".join(l)
        mod.__doc__ = docstring
        failed, tot = py.compat.doctest.testmod(mod, verbose=1)
        if failed:
            py.test.fail("doctest %s: %s failed out of %s" %(
                         self.fspath, failed, tot))

    def _normalize_linesep(self):
        # XXX quite nasty... but it works (fixes win32 issues)
        s = self.fspath.read()
        linesep = '\n'
        if '\r' in s:
            if '\n' not in s:
                linesep = '\r'
            else:
                linesep = '\r\n'
        s = s.replace(linesep, '\n')
        return s

class LinkCheckerMaker(py.test.collect.Collector):
    def collect(self):
        l = []
        for call, tryfn, path, lineno in genlinkchecks(self.fspath):
            name = "%s:%d" %(tryfn, lineno)
            l.append(
                CheckLink(name, parent=self, args=(tryfn, path, lineno), callobj=call)
            )
        return l

class CheckLink(py.test.collect.Function):
    def repr_metainfo(self):
        return self.ReprMetaInfo(fspath=self.fspath, lineno=self._args[2],
            modpath="checklink: %s" % (self._args[0],))
    def setup(self):
        pass
    def teardown(self):
        pass

class DocfileTests(py.test.collect.File):
    DoctestText = DoctestText
    ReSTSyntaxTest = ReSTSyntaxTest
    LinkCheckerMaker = LinkCheckerMaker

    def collect(self):
        return [
            self.ReSTSyntaxTest(self.fspath.basename, parent=self),
            self.LinkCheckerMaker("checklinks", self),
            self.DoctestText("doctest", self),
        ]

# generating functions + args as single tests
def genlinkchecks(path):
    for lineno, line in py.builtin.enumerate(path.readlines()):
        line = line.strip()
        if line.startswith('.. _'):
            if line.startswith('.. _`'):
                delim = '`:'
            else:
                delim = ':'
            l = line.split(delim, 1)
            if len(l) != 2:
                continue
            tryfn = l[1].strip()
            if tryfn.startswith('http:') or tryfn.startswith('https'):
                if option.checkremote:
                    yield urlcheck, tryfn, path, lineno
            elif tryfn.startswith('webcal:'):
                continue
            else:
                i = tryfn.find('#')
                if i != -1:
                    checkfn = tryfn[:i]
                else:
                    checkfn = tryfn
                if checkfn.strip() and (1 or checkfn.endswith('.html')):
                    yield localrefcheck, tryfn, path, lineno

def urlcheck(tryfn, path, lineno):
    old = py.std.socket.getdefaulttimeout()
    py.std.socket.setdefaulttimeout(TIMEOUT_URLOPEN)
    try:
        try:
            print "trying remote", tryfn
            py.std.urllib2.urlopen(tryfn)
        finally:
            py.std.socket.setdefaulttimeout(old)
    except (py.std.urllib2.URLError, py.std.urllib2.HTTPError), e:
        if e.code in (401, 403): # authorization required, forbidden
            py.test.skip("%s: %s" %(tryfn, str(e)))
        else:
            py.test.fail("remote reference error %r in %s:%d\n%s" %(
                         tryfn, path.basename, lineno+1, e))

def localrefcheck(tryfn, path, lineno):
    # assume it should be a file
    i = tryfn.find('#')
    if tryfn.startswith('javascript:'):
        return # don't check JS refs
    if i != -1:
        anchor = tryfn[i+1:]
        tryfn = tryfn[:i]
    else:
        anchor = ''
    fn = path.dirpath(tryfn)
    ishtml = fn.ext == '.html'
    fn = ishtml and fn.new(ext='.txt') or fn
    print "filename is", fn
    if not fn.check(): # not ishtml or not fn.check():
        if not py.path.local(tryfn).check(): # the html could be there
            py.test.fail("reference error %r in %s:%d" %(
                          tryfn, path.basename, lineno+1))
    if anchor:
        source = unicode(fn.read(), 'latin1')
        source = source.lower().replace('-', ' ') # aehem

        anchor = anchor.replace('-', ' ')
        match2 = ".. _`%s`:" % anchor
        match3 = ".. _%s:" % anchor
        candidates = (anchor, match2, match3)
        print "candidates", repr(candidates)
        for line in source.split('\n'):
            line = line.strip()
            if line in candidates:
                break
        else:
            py.test.fail("anchor reference error %s#%s in %s:%d" %(
                tryfn, anchor, path.basename, lineno+1))

# ___________________________________________________________
#
# hooking into py.test Directory collector's chain ...

class DocDirectory(py.test.collect.Directory):
    DocfileTests = DocfileTests
    def collect(self):
        results = super(DocDirectory, self).collect()
        for x in self.fspath.listdir('*.txt', sort=True):
            results.append(self.DocfileTests(x, parent=self))
        return results

Directory = DocDirectory

def resolve_linkrole(name, text, check=True):
    apigen_relpath = get_apigen_relpath()
    if name == 'api':
        if text == 'py':
            return ('py', apigen_relpath + 'api/index.html')
        else:
            assert text.startswith('py.'), (
                'api link "%s" does not point to the py package') % (text,)
            dotted_name = text
            if dotted_name.find('(') > -1:
                dotted_name = dotted_name[:text.find('(')]
            # remove pkg root
            path = dotted_name.split('.')[1:]
            dotted_name = '.'.join(path)
            obj = py
            if check:
                for chunk in path:
                    try:
                        obj = getattr(obj, chunk)
                    except AttributeError:
                        raise AssertionError(
                            'problem with linkrole :api:`%s`: can not resolve '
                            'dotted name %s' % (text, dotted_name,))
            return (text, apigen_relpath + 'api/%s.html' % (dotted_name,))
    elif name == 'source':
        assert text.startswith('py/'), ('source link "%s" does not point '
                                        'to the py package') % (text,)
        relpath = '/'.join(text.split('/')[1:])
        if check:
            pkgroot = py.__pkg__.getpath()
            abspath = pkgroot.join(relpath)
            assert pkgroot.join(relpath).check(), (
                    'problem with linkrole :source:`%s`: '
                    'path %s does not exist' % (text, relpath))
        if relpath.endswith('/') or not relpath:
            relpath += 'index.html'
        else:
            relpath += '.html'
        return (text, apigen_relpath + 'source/%s' % (relpath,))

<commit_msg>[svn r58010] some support for old naming because many sites import ReST checker from this conftest.<commit_after>from __future__ import generators
import py
from py.__.misc import rest
from py.__.apigen.linker import relpath
import os

pypkgdir = py.path.local(py.__file__).dirpath()

mypath = py.magic.autopath().dirpath()

TIMEOUT_URLOPEN = 5.0

Option = py.test.config.Option
option = py.test.config.addoptions("documentation check options",
        Option('-R', '--checkremote',
               action="store_true", dest="checkremote", default=False,
               help="urlopen() remote links found in ReST text files.",
        ),
        Option('', '--forcegen',
               action="store_true", dest="forcegen", default=False,
               help="force generation of html files even if they appear up-to-date"
        ),
)

def get_apigenpath():
    from py.__.conftest import option
    path = os.environ.get('APIGENPATH')
    if path is None:
        path = option.apigenpath
    return pypkgdir.join(path, abs=True)

def get_docpath():
    from py.__.conftest import option
    path = os.environ.get('DOCPATH')
    if path is None:
        path = option.docpath
    return pypkgdir.join(path, abs=True)

def get_apigen_relpath():
    return relpath(get_docpath().strpath + '/',
                   get_apigenpath().strpath + '/')

def deindent(s, sep='\n'):
    leastspaces = -1
    lines = s.split(sep)
    for line in lines:
        if not line.strip():
            continue
        spaces = len(line) - len(line.lstrip())
        if leastspaces == -1 or spaces < leastspaces:
            leastspaces = spaces
    if leastspaces == -1:
        return s
    for i, line in py.builtin.enumerate(lines):
        if not line.strip():
            lines[i] = ''
        else:
            lines[i] = line[leastspaces:]
    return sep.join(lines)

_initialized = False
def checkdocutils():
    global _initialized
    try:
        import docutils
    except ImportError:
        py.test.skip("docutils not importable")
    if not _initialized:
        from py.__.rest import directive
        directive.register_linkrole('api', resolve_linkrole)
        directive.register_linkrole('source', resolve_linkrole)
        _initialized = True

def restcheck(path):
    localpath = path
    if hasattr(path, 'localpath'):
        localpath = path.localpath
    checkdocutils()
    import docutils.utils

    try:
        cur = localpath
        for x in cur.parts(reverse=True):
            confrest = x.dirpath('confrest.py')
            if confrest.check(file=1):
                confrest = confrest.pyimport()
                project = confrest.Project()
                _checkskip(path, project.get_htmloutputpath(path))
                project.process(path)
                break
        else:
            # defer to default processor
            _checkskip(path)
            rest.process(path)
    except KeyboardInterrupt:
        raise
    except docutils.utils.SystemMessage:
        # we assume docutils printed info on stdout
        py.test.fail("docutils processing failed, see captured stderr")

def _checkskip(lpath, htmlpath=None):
    if not option.forcegen:
        lpath = py.path.local(lpath)
        if htmlpath is not None:
            htmlpath = py.path.local(htmlpath)
        if lpath.ext == '.txt':
            htmlpath = htmlpath or lpath.new(ext='.html')
            if htmlpath.check(file=1) and htmlpath.mtime() >= lpath.mtime():
                py.test.skip("html file is up to date, use --forcegen to regenerate")
                #return [] # no need to rebuild

class ReSTSyntaxTest(py.test.collect.Item):
    def runtest(self):
        mypath = self.fspath
        restcheck(py.path.svnwc(mypath))

class DoctestText(py.test.collect.Item):
    def runtest(self):
        s = self._normalize_linesep()
        l = []
        prefix = '.. >>> '
        mod = py.std.types.ModuleType(self.fspath.purebasename)
        skipchunk = False
        for line in deindent(s).split('\n'):
            stripped = line.strip()
            if skipchunk and line.startswith(skipchunk):
                print "skipping", line
                continue
            skipchunk = False
            if stripped.startswith(prefix):
                try:
                    exec py.code.Source(stripped[len(prefix):]).compile() in \
                        mod.__dict__
                except ValueError, e:
                    if e.args and e.args[0] == "skipchunk":
                        skipchunk = " " * (len(line) - len(line.lstrip()))
                    else:
                        raise
            else:
                l.append(line)
        docstring = "\n".join(l)
        mod.__doc__ = docstring
        failed, tot = py.compat.doctest.testmod(mod, verbose=1)
        if failed:
            py.test.fail("doctest %s: %s failed out of %s" %(
                         self.fspath, failed, tot))

    def _normalize_linesep(self):
        # XXX quite nasty... but it works (fixes win32 issues)
        s = self.fspath.read()
        linesep = '\n'
        if '\r' in s:
            if '\n' not in s:
                linesep = '\r'
            else:
                linesep = '\r\n'
        s = s.replace(linesep, '\n')
        return s

class LinkCheckerMaker(py.test.collect.Collector):
    def collect(self):
        l = []
        for call, tryfn, path, lineno in genlinkchecks(self.fspath):
            name = "%s:%d" %(tryfn, lineno)
            l.append(
                CheckLink(name, parent=self, args=(tryfn, path, lineno), callobj=call)
            )
        return l

class CheckLink(py.test.collect.Function):
    def repr_metainfo(self):
        return self.ReprMetaInfo(fspath=self.fspath, lineno=self._args[2],
            modpath="checklink: %s" % (self._args[0],))
    def setup(self):
        pass
    def teardown(self):
        pass

class DocfileTests(py.test.collect.File):
    DoctestText = DoctestText
    ReSTSyntaxTest = ReSTSyntaxTest
    LinkCheckerMaker = LinkCheckerMaker

    def collect(self):
        return [
            self.ReSTSyntaxTest(self.fspath.basename, parent=self),
            self.LinkCheckerMaker("checklinks", self),
            self.DoctestText("doctest", self),
        ]

# generating functions + args as single tests
def genlinkchecks(path):
    for lineno, line in py.builtin.enumerate(path.readlines()):
        line = line.strip()
        if line.startswith('.. _'):
            if line.startswith('.. _`'):
                delim = '`:'
            else:
                delim = ':'
            l = line.split(delim, 1)
            if len(l) != 2:
                continue
            tryfn = l[1].strip()
            if tryfn.startswith('http:') or tryfn.startswith('https'):
                if option.checkremote:
                    yield urlcheck, tryfn, path, lineno
            elif tryfn.startswith('webcal:'):
                continue
            else:
                i = tryfn.find('#')
                if i != -1:
                    checkfn = tryfn[:i]
                else:
                    checkfn = tryfn
                if checkfn.strip() and (1 or checkfn.endswith('.html')):
                    yield localrefcheck, tryfn, path, lineno

def urlcheck(tryfn, path, lineno):
    old = py.std.socket.getdefaulttimeout()
    py.std.socket.setdefaulttimeout(TIMEOUT_URLOPEN)
    try:
        try:
            print "trying remote", tryfn
            py.std.urllib2.urlopen(tryfn)
        finally:
            py.std.socket.setdefaulttimeout(old)
    except (py.std.urllib2.URLError, py.std.urllib2.HTTPError), e:
        if e.code in (401, 403): # authorization required, forbidden
            py.test.skip("%s: %s" %(tryfn, str(e)))
        else:
            py.test.fail("remote reference error %r in %s:%d\n%s" %(
                         tryfn, path.basename, lineno+1, e))

def localrefcheck(tryfn, path, lineno):
    # assume it should be a file
    i = tryfn.find('#')
    if tryfn.startswith('javascript:'):
        return # don't check JS refs
    if i != -1:
        anchor = tryfn[i+1:]
        tryfn = tryfn[:i]
    else:
        anchor = ''
    fn = path.dirpath(tryfn)
    ishtml = fn.ext == '.html'
    fn = ishtml and fn.new(ext='.txt') or fn
    print "filename is", fn
    if not fn.check(): # not ishtml or not fn.check():
        if not py.path.local(tryfn).check(): # the html could be there
            py.test.fail("reference error %r in %s:%d" %(
                          tryfn, path.basename, lineno+1))
    if anchor:
        source = unicode(fn.read(), 'latin1')
        source = source.lower().replace('-', ' ') # aehem

        anchor = anchor.replace('-', ' ')
        match2 = ".. _`%s`:" % anchor
        match3 = ".. _%s:" % anchor
        candidates = (anchor, match2, match3)
        print "candidates", repr(candidates)
        for line in source.split('\n'):
            line = line.strip()
            if line in candidates:
                break
        else:
            py.test.fail("anchor reference error %s#%s in %s:%d" %(
                tryfn, anchor, path.basename, lineno+1))

# ___________________________________________________________
#
# hooking into py.test Directory collector's chain ...

class DocDirectory(py.test.collect.Directory):
    DocfileTests = DocfileTests
    def collect(self):
        results = super(DocDirectory, self).collect()
        for x in self.fspath.listdir('*.txt', sort=True):
            results.append(self.DocfileTests(x, parent=self))
        return results

Directory = DocDirectory

def resolve_linkrole(name, text, check=True):
    apigen_relpath = get_apigen_relpath()
    if name == 'api':
        if text == 'py':
            return ('py', apigen_relpath + 'api/index.html')
        else:
            assert text.startswith('py.'), (
                'api link "%s" does not point to the py package') % (text,)
            dotted_name = text
            if dotted_name.find('(') > -1:
                dotted_name = dotted_name[:text.find('(')]
            # remove pkg root
            path = dotted_name.split('.')[1:]
            dotted_name = '.'.join(path)
            obj = py
            if check:
                for chunk in path:
                    try:
                        obj = getattr(obj, chunk)
                    except AttributeError:
                        raise AssertionError(
                            'problem with linkrole :api:`%s`: can not resolve '
                            'dotted name %s' % (text, dotted_name,))
            return (text, apigen_relpath + 'api/%s.html' % (dotted_name,))
    elif name == 'source':
        assert text.startswith('py/'), ('source link "%s" does not point '
                                        'to the py package') % (text,)
        relpath = '/'.join(text.split('/')[1:])
        if check:
            pkgroot = py.__pkg__.getpath()
            abspath = pkgroot.join(relpath)
            assert pkgroot.join(relpath).check(), (
                    'problem with linkrole :source:`%s`: '
                    'path %s does not exist' % (text, relpath))
        if relpath.endswith('/') or not relpath:
            relpath += 'index.html'
        else:
            relpath += '.html'
        return (text, apigen_relpath + 'source/%s' % (relpath,))

# legacy
ReSTChecker = DocfileTests
<|endoftext|>


<commit_before>import json

from heatherr.views import dispatcher
from heatherr.models import SlackAccount
from heatherr.definitions.models import Acronym

definitions = dispatcher.bot('definitions')

@definitions.ambient(r'@BOTUSERID: (?P<acronym>[A-Z]+) is (?P<definition>.+)$')
def add_defintion(bot_user_id, bot_user_name, message, match):
    slackaccount = SlackAccount.objects.get(bot_user_id=bot_user_id)
    data = match.groupdict()
    Acronym.objects.get_or_create(slackaccount=slackaccount,
                                  acronym=data['acronym'],
                                  definition=data['definition'])
    slackaccount.api_call(
        'reactions.add',
        name='thumbsup',
        channel=message['channel'],
        timestamp=str(message['ts']),)

@definitions.ambient(r'@BOTUSERID: (?P<acronym>[A-Z]+)\?',
                     r'@BOTUSERID: what is (?P<acronym>[A-Z]+)\??',
                     r'@BOTUSERID: what does (?P<acronym>[A-Z]+) mean\??')
def get_definition(bot_user_id, bot_user_name, message, match):
    slackaccount = SlackAccount.objects.get(bot_user_id=bot_user_id)
    data = match.groupdict()
    acronyms = Acronym.objects.filter(slackaccount=slackaccount,
                                      acronym__icontains=data['acronym'])
    if not acronyms.exists():
        return message.reply('I don\'t have any definitions for %s.' % (
            data['acronym'],))

    slackaccount.api_call(
        'chat.postMessage',
        channel=message['channel'],
        text='Definitions for %s' % data['acronym'],
        attachments=json.dumps([{
            'text': '\n'.join([
                '%s (%s)' % (acronym.definition, acronym.pk)
                for acronym in acronyms
            ])
        }])
    )

@definitions.ambient(r'@BOTUSERID: remove (?P<pk>\d+) for (?P<acronym>[A-Z]+)$')  # noqa
def remove_definition(bot_user_id, bot_user_name, message, match):
    slackaccount = SlackAccount.objects.get(bot_user_id=bot_user_id)
    data = match.groupdict()
    deleted_rows, _ = Acronym.objects.filter(
        slackaccount=slackaccount,
        pk=data['pk'],
        acronym=data['acronym']).delete()
    if deleted_rows:
        return message.reply('Deleted %(pk)s for %(acronym)s.' % data)
    return message.reply('Sorry, don\'t know what to delete.')
<commit_msg>add pretext with removal instructions<commit_after>import json

from heatherr.views import dispatcher
from heatherr.models import SlackAccount
from heatherr.definitions.models import Acronym

definitions = dispatcher.bot('definitions')

@definitions.ambient(r'@BOTUSERID: (?P<acronym>[A-Z]+) is (?P<definition>.+)$')
def add_defintion(bot_user_id, bot_user_name, message, match):
    slackaccount = SlackAccount.objects.get(bot_user_id=bot_user_id)
    data = match.groupdict()
    Acronym.objects.get_or_create(slackaccount=slackaccount,
                                  acronym=data['acronym'],
                                  definition=data['definition'])
    slackaccount.api_call(
        'reactions.add',
        name='thumbsup',
        channel=message['channel'],
        timestamp=str(message['ts']),)

@definitions.ambient(r'@BOTUSERID: (?P<acronym>[A-Z]+)\?',
                     r'@BOTUSERID: what is (?P<acronym>[A-Z]+)\??',
                     r'@BOTUSERID: what does (?P<acronym>[A-Z]+) mean\??')
def get_definition(bot_user_id, bot_user_name, message, match):
    slackaccount = SlackAccount.objects.get(bot_user_id=bot_user_id)
    data = match.groupdict()
    acronyms = Acronym.objects.filter(slackaccount=slackaccount,
                                      acronym__icontains=data['acronym'])
    if not acronyms.exists():
        return message.reply('I don\'t have any definitions for %s.' % (
            data['acronym'],))

    slackaccount.api_call(
        'chat.postMessage',
        channel=message['channel'],
        text='Definitions for %s' % data['acronym'],
        pretext='Type `<@%s> remove <number> for %s` to remove defintions' % (
            bot_user_id, data['acronym'],),
        attachments=json.dumps([{
            'text': '\n'.join([
                '%s (%s)' % (acronym.definition, acronym.pk)
                for acronym in acronyms
            ])
        }])
    )

@definitions.ambient(r'@BOTUSERID: remove (?P<pk>\d+) for (?P<acronym>[A-Z]+)$')  # noqa
def remove_definition(bot_user_id, bot_user_name, message, match):
    slackaccount = SlackAccount.objects.get(bot_user_id=bot_user_id)
    data = match.groupdict()
    deleted_rows, _ = Acronym.objects.filter(
        slackaccount=slackaccount,
        pk=data['pk'],
        acronym=data['acronym']).delete()
    if deleted_rows:
        return message.reply('Deleted %(pk)s for %(acronym)s.' % data)
    return message.reply('Sorry, don\'t know what to delete.')
<|endoftext|>


<commit_before>"""
IA Mine
~~~~~~~

Internet Archive data mining tools.

:copyright: (c) 2015 by Internet Archive.
:license: AGPL 3, see LICENSE for more details.

"""
__title__ = 'iamine'
__version__ = '0.3.0dev1'
__author__ = 'Jacob M. Johnson'
__license__ = 'AGPL 3'
__copyright__ = 'Copyright 2015 Internet Archive'

from .api import mine_items, search, mine_urls

__all__ = ['mine_items', 'search', 'mine_urls', '__version__']
<commit_msg>Added configure() to __all__<commit_after>"""
IA Mine
~~~~~~~

Internet Archive data mining tools.

:copyright: (c) 2015 by Internet Archive.
:license: AGPL 3, see LICENSE for more details.

"""
__title__ = 'iamine'
__version__ = '0.3.0dev1'
__author__ = 'Jacob M. Johnson'
__license__ = 'AGPL 3'
__copyright__ = 'Copyright 2015 Internet Archive'

from .api import mine_items, search, mine_urls, configure

__all__ = ['mine_items', 'search', 'mine_urls', '__version__', 'configure']
<|endoftext|>


<commit_before><commit_msg>reorder functions<commit_after><|endoftext|>


<commit_before># standard library
import re
import sys
import json
import pathlib

# other modules
import requests
import click
import yaml
import pkg_resources # setuptools

class Toleo():

    def __init__(self, debug=False, collection='default',
                 path_override=None, limit=None):
        self.debug = debug
        self.collection = collection
        self.path_override = path_override
        self.limit = limit
        self.cfg_path = self.find_config()
        self.cfg = self.read_config()
        self.line = '-' * 50

    def abort(self, msg):
        formatted_msg = click.style(msg, fg='red', bold=True)
        sys.exit(formatted_msg)

    def find_config(self):
        ''' Return a pathlib object of the desired config file. '''
        dir_name = self.path_override or click.get_app_dir('toleo')
        dir_path = pathlib.Path(dir_name)
        cfg_path = ( dir_path / self.collection ).with_suffix('.yaml')
        return cfg_path

    def read_config(self):
        ''' Read config from pathlib object. '''
        if self.cfg_path.is_file():
            with self.cfg_path.open() as f:
                full_cfg = yaml.load(f)
        else:
            msg = 'cannot read {}'.format(self.cfg_path)
            self.abort(msg)
        if self.limit is None:
            cfg = full_cfg
        else:
            cfg = {}
            for key in full_cfg:
                if self.limit in key:
                    cfg[key] = full_cfg[key]
        return cfg

    def scrape(self, url, use_headers=False):
        ''' Scrape the content or headers of a website. '''
        if use_headers:
            headers = requests.head(url).headers
            result = json.dumps(dict(headers))
        else:
            result = requests.get(url).text
        return result

    def ver_compare(self, a, b, ignore_release=False):
        ''' Logically compare two versions. '''
        if ignore_release:
            a = a.split('-')[0]
            b = b.split('-')[0]
        a_ver = pkg_resources.parse_version(a)
        b_ver = pkg_resources.parse_version(b)
        if a_ver == b_ver:
            return 'eq'
        elif a_ver > b_ver:
            return 'gt'
        elif a_ver < b_ver:
            return 'lt'

    def aur_api(self, method, data):
        ''' Query the AUR RPC interface. '''
        # https://wiki.archlinux.org/index.php/AurJson
        payload = {'type': method, 'arg': data}
        data = requests.get('https://aur.archlinux.org/rpc.php', params=payload)
        return data.json()

    def upstream_version(self, pkg_name):
        ''' Return the latest version found upstream. '''
        pkg_data = self.cfg.get(pkg_name)
        upstream = pkg_data.get('upstream')
        url = upstream.get('url')
        parser = upstream.get('parser')
        pattern = upstream.get('pattern')
        use_headers = upstream.get('use_headers', False)
        result = self.scrape(url, use_headers)
        matches = re.findall(pattern, result)
        if self.debug:
            click.echo('url:\t\t{}'.format(url))
            click.echo('parser:\t\t{}'.format(parser))
            click.echo('use_headers:\t{}'.format(use_headers))
            #click.echo('\nresult:\n{}'.format(result))
            click.echo('matches:\t{}'.format(matches))
        version = ''
        for match in matches:
            if self.ver_compare(match, version, ignore_release=True) == 'gt':
                version = match
        return version

    def repo_version(self, pkg_name):
        ''' Find the version of a package in a repo. '''
        pkg_data = self.cfg.get(pkg_name)
        repo = pkg_data.get('repo')
        parser = repo.get('parser')
        if parser == 'aur':
            data = self.aur_api('info', pkg_name)
            result = data.get('results')
            version = result.get('Version')
        elif parser == 'scrape':
            url = repo.get('url')
            use_headers = repo.get('use_headers', False)
            result = self.scrape(url, use_headers)
            pattern = repo.get('pattern')
            matches = re.findall(pattern, result)
            if self.debug:
                click.echo('url:\t\t{}'.format(url))
                click.echo('parser:\t\t{}'.format(parser))
                click.echo('use_headers:\t{}'.format(use_headers))
                click.echo('pattern:\t{}'.format(pattern))
                click.echo('\nresult:\n{}'.format(result))
                click.echo('matches:\t{}'.format(matches))
            version = ''
            for match in matches:
                if self.ver_compare(match, version) == 'gt':
                    version = match
        else:
            self.abort('unknown parser')
        return version

    def action_upstream(self):
        ''' Print all upstream versions. '''
        click.echo(self.line)
        for pkg_name in self.cfg:
            click.echo('package:\t{}'.format(pkg_name))
            src_version = self.upstream_version(pkg_name)
            click.echo('upstream:\t{}'.format(src_version))
            click.echo(self.line)

    def action_repo(self):
        ''' Print all repo versions. '''
        click.echo(self.line)
        for pkg_name in self.cfg:
            click.echo('package:\t{}'.format(pkg_name))
            pkg_version = self.repo_version(pkg_name)
            click.echo('repo:\t\t{}'.format(pkg_version))
            click.echo(self.line)

    def action_compare(self):
        ''' Print report of repo versus upstream. '''
        click.echo(self.line)
        for pkg_name in self.cfg:
            click.echo('package:\t{}'.format(pkg_name))
            src_version = self.upstream_version(pkg_name)
            click.echo('upstream:\t{}'.format(src_version))
            pkg_version = self.repo_version(pkg_name)
            click.echo('repo:\t\t{}'.format(pkg_version))
            click.echo(self.line)

@click.command()
@click.argument('action')
@click.option('--debug/--no-debug', default=False)
@click.option('--collection', '-c', default='default')
@click.option('--path-override', envvar='TOLEO_CONFIG_HOME')
@click.option('--limit', '-l')
def cli(action, debug, collection, path_override, limit):
    ''' Entry point for application. '''
    app = Toleo(debug, collection, path_override, limit)
    if action == 'upstream':
        app.action_upstream()
    elif action == 'repo':
        app.action_repo()
    elif action == 'compare':
        app.action_compare()
<commit_msg>use options instead of argument to determine action<commit_after># standard library
import re
import sys
import json
import pathlib

# other modules
import requests
import click
import yaml
import pkg_resources # setuptools

class Toleo():

    def __init__(self, debug=False, collection='default',
                 path_override=None, limit=None):
        self.debug = debug
        self.collection = collection
        self.path_override = path_override
        self.limit = limit
        self.cfg_path = self.find_config()
        self.cfg = self.read_config()
        self.line = '-' * 50

    def abort(self, msg):
        formatted_msg = click.style(msg, fg='red', bold=True)
        sys.exit(formatted_msg)

    def find_config(self):
        ''' Return a pathlib object of the desired config file. '''
        dir_name = self.path_override or click.get_app_dir('toleo')
        dir_path = pathlib.Path(dir_name)
        cfg_path = ( dir_path / self.collection ).with_suffix('.yaml')
        return cfg_path

    def read_config(self):
        ''' Read config from pathlib object. '''
        if self.cfg_path.is_file():
            with self.cfg_path.open() as f:
                full_cfg = yaml.load(f)
        else:
            msg = 'cannot read {}'.format(self.cfg_path)
            self.abort(msg)
        if self.limit is None:
            cfg = full_cfg
        else:
            cfg = {}
            for key in full_cfg:
                if self.limit in key:
                    cfg[key] = full_cfg[key]
        return cfg

    def scrape(self, url, use_headers=False):
        ''' Scrape the content or headers of a website. '''
        if use_headers:
            headers = requests.head(url).headers
            result = json.dumps(dict(headers))
        else:
            result = requests.get(url).text
        return result

    def ver_compare(self, a, b, ignore_release=False):
        ''' Logically compare two versions. '''
        if ignore_release:
            a = a.split('-')[0]
            b = b.split('-')[0]
        a_ver = pkg_resources.parse_version(a)
        b_ver = pkg_resources.parse_version(b)
        if a_ver == b_ver:
            return 'eq'
        elif a_ver > b_ver:
            return 'gt'
        elif a_ver < b_ver:
            return 'lt'

    def aur_api(self, method, data):
        ''' Query the AUR RPC interface. '''
        # https://wiki.archlinux.org/index.php/AurJson
        payload = {'type': method, 'arg': data}
        data = requests.get('https://aur.archlinux.org/rpc.php', params=payload)
        return data.json()

    def upstream_version(self, pkg_name):
        ''' Return the latest version found upstream. '''
        pkg_data = self.cfg.get(pkg_name)
        upstream = pkg_data.get('upstream')
        url = upstream.get('url')
        parser = upstream.get('parser')
        pattern = upstream.get('pattern')
        use_headers = upstream.get('use_headers', False)
        result = self.scrape(url, use_headers)
        matches = re.findall(pattern, result)
        if self.debug:
            click.echo('url:\t\t{}'.format(url))
            click.echo('parser:\t\t{}'.format(parser))
            click.echo('use_headers:\t{}'.format(use_headers))
            #click.echo('\nresult:\n{}'.format(result))
            click.echo('matches:\t{}'.format(matches))
        version = ''
        for match in matches:
            if self.ver_compare(match, version, ignore_release=True) == 'gt':
                version = match
        return version

    def repo_version(self, pkg_name):
        ''' Find the version of a package in a repo. '''
        pkg_data = self.cfg.get(pkg_name)
        repo = pkg_data.get('repo')
        parser = repo.get('parser')
        if parser == 'aur':
            data = self.aur_api('info', pkg_name)
            result = data.get('results')
            version = result.get('Version')
        elif parser == 'scrape':
            url = repo.get('url')
            use_headers = repo.get('use_headers', False)
            result = self.scrape(url, use_headers)
            pattern = repo.get('pattern')
            matches = re.findall(pattern, result)
            if self.debug:
                click.echo('url:\t\t{}'.format(url))
                click.echo('parser:\t\t{}'.format(parser))
                click.echo('use_headers:\t{}'.format(use_headers))
                click.echo('pattern:\t{}'.format(pattern))
                click.echo('\nresult:\n{}'.format(result))
                click.echo('matches:\t{}'.format(matches))
            version = ''
            for match in matches:
                if self.ver_compare(match, version) == 'gt':
                    version = match
        else:
            self.abort('unknown parser')
        return version

    def action_upstream(self):
        ''' Print all upstream versions. '''
        click.echo(self.line)
        for pkg_name in self.cfg:
            click.echo('package:\t{}'.format(pkg_name))
            src_version = self.upstream_version(pkg_name)
            click.echo('upstream:\t{}'.format(src_version))
            click.echo(self.line)

    def action_repo(self):
        ''' Print all repo versions. '''
        click.echo(self.line)
        for pkg_name in self.cfg:
            click.echo('package:\t{}'.format(pkg_name))
            pkg_version = self.repo_version(pkg_name)
            click.echo('repo:\t\t{}'.format(pkg_version))
            click.echo(self.line)

    def action_compare(self):
        ''' Print report of repo versus upstream. '''
        click.echo(self.line)
        for pkg_name in self.cfg:
            click.echo('package:\t{}'.format(pkg_name))
            src_version = self.upstream_version(pkg_name)
            click.echo('upstream:\t{}'.format(src_version))
            pkg_version = self.repo_version(pkg_name)
            click.echo('repo:\t\t{}'.format(pkg_version))
            click.echo(self.line)

@click.command()
@click.option('--upstream-only', '-u', 'action', flag_value='upstream')
@click.option('--repo-only', '-r', 'action', flag_value='repo')
@click.option('--debug/--no-debug', default=False)
@click.option('--collection', '-c', default='default')
@click.option('--path-override', envvar='TOLEO_CONFIG_HOME')
@click.option('--limit', '-l')
def cli(action, debug, collection, path_override, limit):
    ''' Entry point for application. '''
    app = Toleo(debug, collection, path_override, limit)
    if action == 'upstream':
        app.action_upstream()
    elif action == 'repo':
        app.action_repo()
    else:
        app.action_compare()
<|endoftext|>


<commit_before>#Usage: python build_locations filename
#   This script will open the image then allow a user to click on a location
#   and name the location both shorthand and verbosely. This will then be
#   posted to a remote database

#When working logged points are in red, unlogged points are in blue

import Tkinter
from PIL import Image, ImageTk
from sys import argv,exit
from db import cur,SERVER_URL,DEBUG
import requests
import json

class Location(object):
    """A location corresponds to a pixel location on a map, the file name,
        a short name and a verbose name. It allows the functionality of
        'saving' which will post to a server to store in a remote database.
        Each save will post 4 times, one for each cardinal direction"""

    def __init__(self, x, y, file_name, name, verbose):
        self.x = x
        self.y = y
        self.file_name = file_name
        self.name = name
        self.verbose = verbose

    def save(self,floor_id):
        payload = {}
        payload['x'] = self.x
        payload['y'] = self.x
        payload['name'] = self.name
        payload['verbose'] = self.verbose
        payload['floor_id'] = floor_id
        for i in range(4):
            payload['d'] = i * 90
            r = requests.post(SERVER_URL + "location", params=payload)

class Rectangle(object):
    def __init__(self,name, verbose, x1, y1, x2, y2, cid1,cid2, w):
        #x1, y1 is the top left corner
        #x2, y2 is the bottom right corner
        #w is the Tkinter canvas
        self.name = name
        self.verbose = verbose
        self.cid1 = cid1
        self.cid2 = cid2
        self.x1 = x1
        self.x2 = x2
        self.y1 = y1
        self.y2 = y2
        XL = self.x1 if self.x1 < self.x2 else self.x2
        XR = self.x2 if self.x1 < self.x2 else self.x1
        YT = self.y1 if self.y1 < self.y2 else self.y2
        YB = self.y2 if self.y1 < self.y2 else self.y1
        self.id = w.create_rectangle(XL, YT, XR, YB, fill="blue")
        self.canvas = w
    def build_locations(self,locs):
        XL = self.x1 if self.x1 < self.x2 else self.x2
        XR = self.x2 if self.x1 < self.x2 else self.x1
        YT = self.y1 if self.y1 < self.y2 else self.y2
        YB = self.y2 if self.y1 < self.y2 else self.y1
        fn = argv[1]
        loc1 = Location(XL,YT,fn, self.name +" TL", self.verbose + " Top Left")
        loc2 = Location(XR,YT,fn, self.name +" TR", self.verbose + " Top Right")
        loc3 = Location(XR,YB,fn, self.name +" BR", self.verbose + " Bottom Right")
        loc4 = Location(XL,YB,fn, self.name +" BL", self.verbose + " Bottom Left")
        loc5 = Location( (XL + XR) // 2, (YB + YT) // 2,fn, self.name +" CT", self.verbose + " Center")
        locs.append(loc1)
        locs.append(loc2)
        locs.append(loc3)
        locs.append(loc4)
        locs.append(loc5)

    def pop(self):
        self.canvas.delete(self.cid2)
        self.canvas.delete(self.id)
        return (self.x1, self.y1, self.cid1)
    def delete(self):
        # deletes the rectangle from the canvas
        self.canvas.delete(self.cid2)
        self.canvas.delete(self.cid1)
        self.canvas.delete(self.id)

corners = []
rectangles = []
locations = []

def main(argv):
    window = Tkinter.Tk(className="Location Selector")
    if len(argv) != 2:
        print "Usage: python build_locations filename"
        exit(1)
    image = Image.open(argv[1])
    payload = {'path': argv[1]}
    #try:
    r = requests.get(SERVER_URL + "floor", params=payload)
    found_fid = False
    fid = 1
    if DEBUG:
        print r.text
    if r:
        json_r = r.json()
        if 'error' not in json_r:
            found_fid = True
            fid = int(json_r['floor_id'])
    if not found_fid:
        payload['building'] = raw_input("Building: ")
        payload['floor_number'] = int(raw_input("floor_number: "))
        r = requests.post(SERVER_URL + "floor", data=json.dumps(payload))
        json_r = r.json()
        if 'error' in json_r:
            raise "Server Error"
        fid = int(json_r['floor_id'])
    # except:
    #    exit("Error finding floor id")

    pixels = image.load()
    canvas = Tkinter.Canvas(window, width=image.size[0], height=image.size[1])
    canvas.pack()
    image_tk = ImageTk.PhotoImage(image)
    oncanvas = canvas.create_image(image.size[0]//2, image.size[1]//2, image=image_tk)

    def add_corner(event):
        global corners
        global rectangles
        global locations
        radius = 2 #point radius
        cid = canvas.create_oval(event.x-radius, event.y-radius,
             event.x + radius, event.y +radius, fill="blue")
        corners.append( (event.x, event.y, cid) )
        if len(corners) == 2:
            name = raw_input("Short name: ")
            verbose = raw_input("Verbose name: ")
            rectangles.append( Rectangle(name,verbose,corners[0][0],corners[0][1],corners[1][0],corners[1][1],
                corners[0][2], corners[1][2],
                canvas)  )
            corners = []

    def pop_corner(event):
        global corners
        global rectangles
        global locations
        if len(corners) != 1:
            if len(rectangles) == 0:
                return
            else:
                r = rectangles.pop(len(rectangles) - 1)
                corners.append( r.pop() )
        else:
            c = corners[0]
            canvas.delete(c[2])
            corners = []

    canvas.bind("<Button-1>", add_corner)
    canvas.bind("<Button-2>", pop_corner)

    Tkinter.mainloop()
    for rect in rectangles:
        rect.build_locations(locations)
    for loc in locations:
        loc.save(fid)

if __name__ == '__main__':
    main(argv)
<commit_msg>reinstated try block<commit_after>#Usage: python build_locations filename
#   This script will open the image then allow a user to click on a location
#   and name the location both shorthand and verbosely. This will then be
#   posted to a remote database

#When working logged points are in red, unlogged points are in blue

import Tkinter
from PIL import Image, ImageTk
from sys import argv,exit
from db import cur,SERVER_URL,DEBUG
import requests
import json

class Location(object):
    """A location corresponds to a pixel location on a map, the file name,
        a short name and a verbose name. It allows the functionality of
        'saving' which will post to a server to store in a remote database.
        Each save will post 4 times, one for each cardinal direction"""

    def __init__(self, x, y, file_name, name, verbose):
        self.x = x
        self.y = y
        self.file_name = file_name
        self.name = name
        self.verbose = verbose

    def save(self,floor_id):
        payload = {}
        payload['x'] = self.x
        payload['y'] = self.x
        payload['name'] = self.name
        payload['verbose'] = self.verbose
        payload['floor_id'] = floor_id
        for i in range(4):
            payload['d'] = i * 90
            r = requests.post(SERVER_URL + "location", params=payload)

class Rectangle(object):
    def __init__(self,name, verbose, x1, y1, x2, y2, cid1,cid2, w):
        #x1, y1 is the top left corner
        #x2, y2 is the bottom right corner
        #w is the Tkinter canvas
        self.name = name
        self.verbose = verbose
        self.cid1 = cid1
        self.cid2 = cid2
        self.x1 = x1
        self.x2 = x2
        self.y1 = y1
        self.y2 = y2
        XL = self.x1 if self.x1 < self.x2 else self.x2
        XR = self.x2 if self.x1 < self.x2 else self.x1
        YT = self.y1 if self.y1 < self.y2 else self.y2
        YB = self.y2 if self.y1 < self.y2 else self.y1
        self.id = w.create_rectangle(XL, YT, XR, YB, fill="blue")
        self.canvas = w
    def build_locations(self,locs):
        XL = self.x1 if self.x1 < self.x2 else self.x2
        XR = self.x2 if self.x1 < self.x2 else self.x1
        YT = self.y1 if self.y1 < self.y2 else self.y2
        YB = self.y2 if self.y1 < self.y2 else self.y1
        fn = argv[1]
        loc1 = Location(XL,YT,fn, self.name +" TL", self.verbose + " Top Left")
        loc2 = Location(XR,YT,fn, self.name +" TR", self.verbose + " Top Right")
        loc3 = Location(XR,YB,fn, self.name +" BR", self.verbose + " Bottom Right")
        loc4 = Location(XL,YB,fn, self.name +" BL", self.verbose + " Bottom Left")
        loc5 = Location( (XL + XR) // 2, (YB + YT) // 2,fn, self.name +" CT", self.verbose + " Center")
        locs.append(loc1)
        locs.append(loc2)
        locs.append(loc3)
        locs.append(loc4)
        locs.append(loc5)

    def pop(self):
        self.canvas.delete(self.cid2)
        self.canvas.delete(self.id)
        return (self.x1, self.y1, self.cid1)
    def delete(self):
        # deletes the rectangle from the canvas
        self.canvas.delete(self.cid2)
        self.canvas.delete(self.cid1)
        self.canvas.delete(self.id)

corners = []
rectangles = []
locations = []

def main(argv):
    window = Tkinter.Tk(className="Location Selector")
    if len(argv) != 2:
        print "Usage: python build_locations filename"
        exit(1)
    image = Image.open(argv[1])
    payload = {'path': argv[1]}
    try:
        r = requests.get(SERVER_URL + "floor", params=payload)
        found_fid = False
        fid = 1
        if DEBUG:
            print r.text
        if r:
            json_r = r.json()
            if 'error' not in json_r:
                found_fid = True
                fid = int(json_r['floor_id'])
        if not found_fid:
            payload['building'] = raw_input("Building: ")
            payload['floor_number'] = int(raw_input("floor_number: "))
            r = requests.post(SERVER_URL + "floor", data=json.dumps(payload))
            json_r = r.json()
            if 'error' in json_r:
                raise "Server Error"
            fid = int(json_r['floor_id'])
    except:
        exit("Error finding floor id")

    pixels = image.load()
    canvas = Tkinter.Canvas(window, width=image.size[0], height=image.size[1])
    canvas.pack()
    image_tk = ImageTk.PhotoImage(image)
    oncanvas = canvas.create_image(image.size[0]//2, image.size[1]//2, image=image_tk)

    def add_corner(event):
        global corners
        global rectangles
        global locations
        radius = 2 #point radius
        cid = canvas.create_oval(event.x-radius, event.y-radius,
             event.x + radius, event.y +radius, fill="blue")
        corners.append( (event.x, event.y, cid) )
        if len(corners) == 2:
            name = raw_input("Short name: ")
            verbose = raw_input("Verbose name: ")
            rectangles.append( Rectangle(name,verbose,corners[0][0],corners[0][1],corners[1][0],corners[1][1],
                corners[0][2], corners[1][2],
                canvas)  )
            corners = []

    def pop_corner(event):
        global corners
        global rectangles
        global locations
        if len(corners) != 1:
            if len(rectangles) == 0:
                return
            else:
                r = rectangles.pop(len(rectangles) - 1)
                corners.append( r.pop() )
        else:
            c = corners[0]
            canvas.delete(c[2])
            corners = []

    canvas.bind("<Button-1>", add_corner)
    canvas.bind("<Button-2>", pop_corner)

    Tkinter.mainloop()
    for rect in rectangles:
        rect.build_locations(locations)
    for loc in locations:
        loc.save(fid)

if __name__ == '__main__':
    main(argv)
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import glob
import hashlib
import optparse
import os
import posixpath
import shutil
import subprocess
import stat
import sys
import tarfile

"""A Cygwin aware version compress/extract object.

This module supports creating and unpacking a tarfile on all platforms.  For
Cygwin, Mac, and Linux, it will use the standard tarfile implementation.  For
Win32 it will detect Cygwin style symlinks as it archives and convert them to
symlinks.

For Win32, it is unfortunate that os.stat does not return a FileID in the ino
field which would allow us to correctly determine which files are hardlinks, so
instead we assume that any files in the archive that are an exact match are
hardlinks to the same data.

We know they are not Symlinks because we are using Cygwin style symlinks only,
which appear to Win32 a normal file.

All paths stored and retrieved from a TAR file are expected to be POSIX style,
Win32 style paths will be rejected.

NOTE:
  All paths represent by the tarfile and all API functions are POSIX style paths
  except for CygTar.Add which assumes a Native path.
"""

def ToNativePath(native_path):
  """Convert to a posix style path if this is win32."""
  if sys.platform == 'win32':
    return native_path.replace('/', '\\')
  return native_path

def IsCygwinSymlink(symtext):
  """Return true if the provided text looks like a Cygwin symlink."""
  return symtext[:12] == '!<symlink>\xff\xfe'

def SymDatToPath(symtext):
  """Convert a Cygwin style symlink data to a relative path."""
  return ''.join([ch for ch in symtext[12:] if ch != '\x00'])

def PathToSymDat(filepath):
  """Convert a filepath to cygwin style symlink data."""
  symtag = '!<symlink>\xff\xfe'
  unipath = ''.join([ch + '\x00' for ch in filepath])
  strterm = '\x00\x00'
  return symtag + unipath + strterm

def CreateWin32Hardlink(filepath, targpath, try_mklink, verbose):
  """Create a hardlink on Win32 if possible

  Uses mklink to create a hardlink if possible.  On failure, it will
  assume mklink is unavailible and copy the file instead, returning False
  to indicate future calls should not attempt to use mklink."""

  # Assume an error, if subprocess succeeds, then it should return 0
  err = 1
  if try_mklink:
    dst_src = ToNativePath(filepath) + ' ' + ToNativePath(targpath)
    try:
      err = subprocess.call(['cmd', '/C', 'mklink /H ' + dst_src],
                            stdout = open(os.devnull, 'wb'),
                            stderr = open(os.devnull, 'wb'))
    except EnvironmentError:
      try_mklink = False

  # If we failed to create a hardlink, then just copy it.  We wrap this in a
  # retry for Windows which often has stale file lock issues.
  if err or not os.path.isfile(filepath):
    for cnt in range(1,4):
      try:
        shutil.copyfile(targpath, filepath)
        return False
      except EnvironmentError:
        if verbose:
          print 'Try %d: Failed hardlink %s -> %s\n' % (cnt, filepath, targpath)
    if verbose:
      print 'Giving up.'
  return try_mklink

def ComputeFileHash(filepath):
  """Generate a sha1 hash for the file at the given path."""
  sha1 = hashlib.sha1()
  with open(filepath, 'rb') as fp:
    sha1.update(fp.read())
  return sha1.hexdigest()

def ReadableSizeOf(num):
  """Convert to a human readable number."""
  if num < 1024.0:
    return '[%5dB]' % num
  for x in ['B','K','M','G','T']:
     if num < 1024.0:
       return '[%5.1f%s]' % (num, x)
     num /= 1024.0
  return '[%dT]' % int(num)

class CygTar(object):
  """ CygTar is an object which represents a Win32 and Cygwin aware tarball."""
  def __init__(self, filename, mode='r', verbose=False):
    self.size_map = {}
    self.file_hashes = {}
    # Set errorlevel=1 so that fatal errors actually raise!
    if 'r' in mode:
      self.read_file = open(filename, 'rb')
      self.read_filesize = os.path.getsize(filename)
      self.tar = tarfile.open(mode=mode, fileobj=self.read_file, errorlevel=1)
    else:
      self.read_file = None
      self.read_filesize = 0
      self.tar = tarfile.open(filename, mode=mode, errorlevel=1)
    self.verbose = verbose

  def __DumpInfo(self, tarinfo):
    """Prints information on a single object in the tarball."""
    typeinfo = '?'
    lnk = ''
    if tarinfo.issym():
      typeinfo = 'S'
      lnk = '-> ' + tarinfo.linkname
    if tarinfo.islnk():
      typeinfo = 'H'
      lnk = '-> ' + tarinfo.linkname
    if tarinfo.isdir():
      typeinfo = 'D'
    if tarinfo.isfile():
      typeinfo = 'F'
    reable_size = ReadableSizeOf(tarinfo.size)
    print '%s %s : %s %s' % (reable_size, typeinfo, tarinfo.name, lnk)
    return tarinfo

  def __AddFile(self, tarinfo, fileobj=None):
    """Add a file to the archive."""
    if self.verbose:
      self.__DumpInfo(tarinfo)
    self.tar.addfile(tarinfo, fileobj)

  def __AddLink(self, tarinfo, linktype, linkpath):
    """Add a Win32 symlink or hardlink to the archive."""
    tarinfo.linkname = linkpath
    tarinfo.type = linktype
    tarinfo.size = 0
    self.__AddFile(tarinfo)

  def Add(self, filepath, prefix=None):
    """Add path filepath to the archive which may be Native style.

    Add files individually recursing on directories.  For POSIX we use
    tarfile.addfile directly on symlinks and hardlinks.  For files, we
    must check if they are duplicates which we convert to hardlinks
    or Cygwin style symlinks which we convert form a file to a symlink
    in the tarfile.  All other files are added as a standard file.
    """

    # At this point tarinfo.name will contain a POSIX style path regardless
    # of the original filepath.
    tarinfo = self.tar.gettarinfo(filepath)
    if prefix:
      tarinfo.name = posixpath.join(prefix, tarinfo.name)

    if sys.platform == 'win32':
      # On win32 os.stat() always claims that files are world writable
      # which means that unless we remove this bit here we end up with
      # world writables files in the archive, which is almost certainly
      # not intended.
      tarinfo.mode &= ~stat.S_IWOTH
      tarinfo.mode &= ~stat.S_IWGRP

      # If we want cygwin to be able to extract this archive and use
      # executables and dll files we need to mark all the archive members as
      # executable.  This is essentially what happens anyway when the
      # archive is extracted on win32.
      tarinfo.mode |= stat.S_IXUSR | stat.S_IXOTH | stat.S_IXGRP

    # If this a symlink or hardlink, add it
    if tarinfo.issym() or tarinfo.islnk():
      tarinfo.size = 0
      self.__AddFile(tarinfo)
      return True

    # If it's a directory, then you want to recurse into it
    if tarinfo.isdir():
      self.__AddFile(tarinfo)
      native_files = glob.glob(os.path.join(filepath, '*'))
      for native_file in native_files:
        if not self.Add(native_file, prefix): return False
      return True

    # At this point we only allow addition of "FILES"
    if not tarinfo.isfile():
      print 'Failed to add non real file: %s' % filepath
      return False

    # Now check if it is a Cygwin style link disguised as a file.
    # We go ahead and check on all platforms just in case we are tar'ing a
    # mount shared with windows.
    if tarinfo.size <= 524:
      with open(filepath) as fp:
        symtext = fp.read()
      if IsCygwinSymlink(symtext):
        self.__AddLink(tarinfo, tarfile.SYMTYPE, SymDatToPath(symtext))
        return True

    # Otherwise, check if its a hardlink by seeing if it matches any unique
    # hash within the list of hashed files for that file size.
    nodelist = self.size_map.get(tarinfo.size, [])

    # If that size bucket is empty, add this file, no need to get the hash until
    # we get a bucket collision for the first time..
    if not nodelist:
      self.size_map[tarinfo.size] = [filepath]
      with open(filepath, 'rb') as fp:
        self.__AddFile(tarinfo, fp)
      return True

    # If the size collides with anything, we'll need to check hashes.  We assume
    # no hash collisions for SHA1 on a given bucket, since the number of files
    # in a bucket over possible SHA1 values is near zero.
    newhash = ComputeFileHash(filepath)
    self.file_hashes[filepath] = newhash

    for oldname in nodelist:
      oldhash = self.file_hashes.get(oldname, None)
      if not oldhash:
        oldhash = ComputeFileHash(oldname)
        self.file_hashes[oldname] = oldhash

      if oldhash == newhash:
        self.__AddLink(tarinfo, tarfile.LNKTYPE, oldname)
        return True

    # Otherwise, we missed, so add it to the bucket for this size
    self.size_map[tarinfo.size].append(filepath)
    with open(filepath, 'rb') as fp:
      self.__AddFile(tarinfo, fp)
    return True

  def Extract(self):
    """Extract the tarfile to the current directory."""
    try_mklink = True

    if self.verbose:
      sys.stdout.write('|' + ('-' * 48) + '|\n')
      sys.stdout.flush()
      dots_outputted = 0

    win32_symlinks = {}
    for m in self.tar:
      if self.verbose:
        cnt = self.read_file.tell()
        curdots = cnt * 50 / self.read_filesize
        if dots_outputted < curdots:
          for dot in xrange(dots_outputted, curdots):
            sys.stdout.write('.')
          sys.stdout.flush()
          dots_outputted = curdots

      # For hardlinks in Windows, we try to use mklink, and instead copy on
      # failure.
      if m.islnk() and sys.platform == 'win32':
        try_mklink = CreateWin32Hardlink(
            m.name, m.linkname, try_mklink, self.verbose)
      # On Windows we treat symlinks as if they were hard links.
      # Proper Windows symlinks supported by everything can be made with
      # mklink, but only by an Administrator.  The older toolchains are
      # built with Cygwin, so they could use Cygwin-style symlinks; but
      # newer toolchains do not use Cygwin, and nothing else on the system
      # understands Cygwin-style symlinks, so avoid them.
      elif m.issym() and sys.platform == 'win32':
        # For a hard link, the link target (m.linkname) always appears
        # in the archive before the link itself (m.name), so the links
        # can just be made on the fly.  However, a symlink might well
        # appear in the archive before its target file, so there would
        # not yet be any file to hard-link to.  Hence, we have to collect
        # all the symlinks and create them in dependency order at the end.
        linkname = m.linkname
        if not posixpath.isabs(linkname):
          linkname = posixpath.join(posixpath.dirname(m.name), linkname)
        linkname = posixpath.normpath(linkname)
        win32_symlinks[posixpath.normpath(m.name)] = linkname
      # Otherwise, extract normally.
      else:
        self.tar.extract(m)

    win32_symlinks_left = win32_symlinks.items()
    while win32_symlinks_left:
      this_symlink = win32_symlinks_left.pop(0)
      name, linkname = this_symlink
      if linkname in win32_symlinks:
        # The target is itself a symlink not yet created.
        # Wait for it to come 'round on the guitar.
        win32_symlinks_left.append(this_symlink)
      else:
        del win32_symlinks[name]
        try_mklink = CreateWin32Hardlink(
            name, linkname, try_mklink, self.verbose)

    if self.verbose:
      sys.stdout.write('\n')
      sys.stdout.flush()

  def List(self):
    """List the set of objects in the tarball."""
    for tarinfo in self.tar:
      self.__DumpInfo(tarinfo)

  def Close(self):
    self.tar.close()
    if self.read_file is not None:
      self.read_file.close()
      self.read_file = None
      self.read_filesize = 0

def Main(args):
  parser = optparse.OptionParser()
  # Modes
  parser.add_option('-c', '--create', help='Create a tarball.',
      action='store_const', const='c', dest='action', default='')
  parser.add_option('-x', '--extract', help='Extract a tarball.',
      action='store_const', const='x', dest='action')
  parser.add_option('-t', '--list', help='List sources in tarball.',
      action='store_const', const='t', dest='action')

  # Compression formats
  parser.add_option('-j', '--bzip2', help='Create a bz2 tarball.',
      action='store_const', const=':bz2', dest='format', default='')
  parser.add_option('-z', '--gzip', help='Create a gzip tarball.',
      action='store_const', const=':gz', dest='format', )
  # Misc
  parser.add_option('-v', '--verbose', help='Use verbose output.',
      action='store_true', dest='verbose', default=False)
  parser.add_option('-f', '--file', help='Name of tarball.',
      dest='filename', default='')
  parser.add_option('-C', '--directory', help='Change directory.',
      dest='cd', default='')
  parser.add_option('--prefix', help='Subdirectory prefix for all paths')

  options, args = parser.parse_args(args[1:])
  if not options.action:
    parser.error('Expecting compress or extract')
  if not options.filename:
    parser.error('Expecting a filename')

  if options.action in ['c'] and not args:
    parser.error('Expecting list of sources to add')
  if options.action in ['x', 't'] and args:
    parser.error('Unexpected source list on extract')

  if options.action == 'c':
    mode = 'w' + options.format
  else:
    mode = 'r'+ options.format

  tar = CygTar(options.filename, mode, verbose=options.verbose)
  if options.cd:
    os.chdir(options.cd)

  if options.action == 't':
    tar.List()
    return 0

  if options.action == 'x':
    tar.Extract()
    return 0

  if options.action == 'c':
    for filepath in args:
      if not tar.Add(filepath, options.prefix):
        return -1
    tar.Close()
    return 0

  parser.error('Missing action c, t, or x.')
  return -1

if __name__ == '__main__':
  sys.exit(Main(sys.argv))
<commit_msg>cygtar: Create win32 junctions instead of hard links for directories<commit_after>#!/usr/bin/python
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import glob
import hashlib
import optparse
import os
import posixpath
import shutil
import subprocess
import stat
import sys
import tarfile

"""A Cygwin aware version compress/extract object.

This module supports creating and unpacking a tarfile on all platforms.  For
Cygwin, Mac, and Linux, it will use the standard tarfile implementation.  For
Win32 it will detect Cygwin style symlinks as it archives and convert them to
symlinks.

For Win32, it is unfortunate that os.stat does not return a FileID in the ino
field which would allow us to correctly determine which files are hardlinks, so
instead we assume that any files in the archive that are an exact match are
hardlinks to the same data.

We know they are not Symlinks because we are using Cygwin style symlinks only,
which appear to Win32 a normal file.

All paths stored and retrieved from a TAR file are expected to be POSIX style,
Win32 style paths will be rejected.

NOTE:
  All paths represent by the tarfile and all API functions are POSIX style paths
  except for CygTar.Add which assumes a Native path.
"""

def ToNativePath(native_path):
  """Convert to a posix style path if this is win32."""
  if sys.platform == 'win32':
    return native_path.replace('/', '\\')
  return native_path

def IsCygwinSymlink(symtext):
  """Return true if the provided text looks like a Cygwin symlink."""
  return symtext[:12] == '!<symlink>\xff\xfe'

def SymDatToPath(symtext):
  """Convert a Cygwin style symlink data to a relative path."""
  return ''.join([ch for ch in symtext[12:] if ch != '\x00'])

def PathToSymDat(filepath):
  """Convert a filepath to cygwin style symlink data."""
  symtag = '!<symlink>\xff\xfe'
  unipath = ''.join([ch + '\x00' for ch in filepath])
  strterm = '\x00\x00'
  return symtag + unipath + strterm

def CreateWin32Link(filepath, targpath, verbose):
  """Create a link on Win32 if possible

  Uses mklink to create a link (hardlink or junction) if possible. On failure,
  it will assume mklink is unavailible and copy the file instead. Future calls
  will not attempt to use mklink."""

  targ_is_dir = os.path.isdir(targpath)

  call_mklink = False
  if targ_is_dir and CreateWin32Link.try_junction:
    # Creating a link to a directory will fail, but a junction (which is more
    # like a symlink) will work.
    mklink_flag = '/J'
    call_mklink = True
  elif not targ_is_dir and CreateWin32Link.try_hardlink:
    mklink_flag = '/H'
    call_mklink = True

  # Assume an error, if subprocess succeeds, then it should return 0
  err = 1
  if call_mklink:
    try:
      cmd = ['cmd', '/C', 'mklink %s %s %s' % (
              mklink_flag, ToNativePath(filepath), ToNativePath(targpath))]
      err = subprocess.call(cmd,
          stdout = open(os.devnull, 'wb'),
          stderr = open(os.devnull, 'wb'))
    except EnvironmentError:
      if targ_is_dir:
        CreateWin32Link.try_junction = False
      else:
        CreateWin32Link.try_hardlink = False

  # If we failed to create a link, then just copy it.  We wrap this in a
  # retry for Windows which often has stale file lock issues.
  if err or not os.path.exists(filepath):
    if targ_is_dir and verbose:
      print 'Failed to create junction %s -> %s. Copying instead.\n' % (
          filepath, targpath)

    for cnt in range(1,4):
      try:
        if targ_is_dir:
          shutil.copytree(targpath, filepath)
        else:
          shutil.copyfile(targpath, filepath)
        return False
      except EnvironmentError:
        if verbose:
          print 'Try %d: Failed hardlink %s -> %s\n' % (cnt, filepath, targpath)
    if verbose:
      print 'Giving up.'

CreateWin32Link.try_hardlink = True
CreateWin32Link.try_junction = True

def ComputeFileHash(filepath):
  """Generate a sha1 hash for the file at the given path."""
  sha1 = hashlib.sha1()
  with open(filepath, 'rb') as fp:
    sha1.update(fp.read())
  return sha1.hexdigest()

def ReadableSizeOf(num):
  """Convert to a human readable number."""
  if num < 1024.0:
    return '[%5dB]' % num
  for x in ['B','K','M','G','T']:
     if num < 1024.0:
       return '[%5.1f%s]' % (num, x)
     num /= 1024.0
  return '[%dT]' % int(num)

class CygTar(object):
  """ CygTar is an object which represents a Win32 and Cygwin aware tarball."""
  def __init__(self, filename, mode='r', verbose=False):
    self.size_map = {}
    self.file_hashes = {}
    # Set errorlevel=1 so that fatal errors actually raise!
    if 'r' in mode:
      self.read_file = open(filename, 'rb')
      self.read_filesize = os.path.getsize(filename)
      self.tar = tarfile.open(mode=mode, fileobj=self.read_file, errorlevel=1)
    else:
      self.read_file = None
      self.read_filesize = 0
      self.tar = tarfile.open(filename, mode=mode, errorlevel=1)
    self.verbose = verbose

  def __DumpInfo(self, tarinfo):
    """Prints information on a single object in the tarball."""
    typeinfo = '?'
    lnk = ''
    if tarinfo.issym():
      typeinfo = 'S'
      lnk = '-> ' + tarinfo.linkname
    if tarinfo.islnk():
      typeinfo = 'H'
      lnk = '-> ' + tarinfo.linkname
    if tarinfo.isdir():
      typeinfo = 'D'
    if tarinfo.isfile():
      typeinfo = 'F'
    reable_size = ReadableSizeOf(tarinfo.size)
    print '%s %s : %s %s' % (reable_size, typeinfo, tarinfo.name, lnk)
    return tarinfo

  def __AddFile(self, tarinfo, fileobj=None):
    """Add a file to the archive."""
    if self.verbose:
      self.__DumpInfo(tarinfo)
    self.tar.addfile(tarinfo, fileobj)

  def __AddLink(self, tarinfo, linktype, linkpath):
    """Add a Win32 symlink or hardlink to the archive."""
    tarinfo.linkname = linkpath
    tarinfo.type = linktype
    tarinfo.size = 0
    self.__AddFile(tarinfo)

  def Add(self, filepath, prefix=None):
    """Add path filepath to the archive which may be Native style.

    Add files individually recursing on directories.  For POSIX we use
    tarfile.addfile directly on symlinks and hardlinks.  For files, we must
    check if they are duplicates which we convert to hardlinks or symlinks
    which we convert from a file to a symlink in the tarfile.  All other files
    are added as a standard file.
    """

    # At this point tarinfo.name will contain a POSIX style path regardless
    # of the original filepath.
    tarinfo = self.tar.gettarinfo(filepath)
    if prefix:
      tarinfo.name = posixpath.join(prefix, tarinfo.name)

    if sys.platform == 'win32':
      # On win32 os.stat() always claims that files are world writable
      # which means that unless we remove this bit here we end up with
      # world writables files in the archive, which is almost certainly
      # not intended.
      tarinfo.mode &= ~stat.S_IWOTH
      tarinfo.mode &= ~stat.S_IWGRP

      # If we want cygwin to be able to extract this archive and use
      # executables and dll files we need to mark all the archive members as
      # executable.  This is essentially what happens anyway when the
      # archive is extracted on win32.
      tarinfo.mode |= stat.S_IXUSR | stat.S_IXOTH | stat.S_IXGRP

    # If this a symlink or hardlink, add it
    if tarinfo.issym() or tarinfo.islnk():
      tarinfo.size = 0
      self.__AddFile(tarinfo)
      return True

    # If it's a directory, then you want to recurse into it
    if tarinfo.isdir():
      self.__AddFile(tarinfo)
      native_files = glob.glob(os.path.join(filepath, '*'))
      for native_file in native_files:
        if not self.Add(native_file, prefix): return False
      return True

    # At this point we only allow addition of "FILES"
    if not tarinfo.isfile():
      print 'Failed to add non real file: %s' % filepath
      return False

    # Now check if it is a Cygwin style link disguised as a file.
    # We go ahead and check on all platforms just in case we are tar'ing a
    # mount shared with windows.
    if tarinfo.size <= 524:
      with open(filepath) as fp:
        symtext = fp.read()
      if IsCygwinSymlink(symtext):
        self.__AddLink(tarinfo, tarfile.SYMTYPE, SymDatToPath(symtext))
        return True

    # Otherwise, check if its a hardlink by seeing if it matches any unique
    # hash within the list of hashed files for that file size.
    nodelist = self.size_map.get(tarinfo.size, [])

    # If that size bucket is empty, add this file, no need to get the hash until
    # we get a bucket collision for the first time..
    if not nodelist:
      self.size_map[tarinfo.size] = [filepath]
      with open(filepath, 'rb') as fp:
        self.__AddFile(tarinfo, fp)
      return True

    # If the size collides with anything, we'll need to check hashes.  We assume
    # no hash collisions for SHA1 on a given bucket, since the number of files
    # in a bucket over possible SHA1 values is near zero.
    newhash = ComputeFileHash(filepath)
    self.file_hashes[filepath] = newhash

    for oldname in nodelist:
      oldhash = self.file_hashes.get(oldname, None)
      if not oldhash:
        oldhash = ComputeFileHash(oldname)
        self.file_hashes[oldname] = oldhash

      if oldhash == newhash:
        self.__AddLink(tarinfo, tarfile.LNKTYPE, oldname)
        return True

    # Otherwise, we missed, so add it to the bucket for this size
    self.size_map[tarinfo.size].append(filepath)
    with open(filepath, 'rb') as fp:
      self.__AddFile(tarinfo, fp)
    return True

  def Extract(self):
    """Extract the tarfile to the current directory."""
    if self.verbose:
      sys.stdout.write('|' + ('-' * 48) + '|\n')
      sys.stdout.flush()
      dots_outputted = 0

    win32_symlinks = {}
    for m in self.tar:
      if self.verbose:
        cnt = self.read_file.tell()
        curdots = cnt * 50 / self.read_filesize
        if dots_outputted < curdots:
          for dot in xrange(dots_outputted, curdots):
            sys.stdout.write('.')
          sys.stdout.flush()
          dots_outputted = curdots

      # For hardlinks in Windows, we try to use mklink, and instead copy on
      # failure.
      if m.islnk() and sys.platform == 'win32':
        CreateWin32Link(m.name, m.linkname, self.verbose)
      # On Windows we treat symlinks as if they were hard links.
      # Proper Windows symlinks supported by everything can be made with
      # mklink, but only by an Administrator.  The older toolchains are
      # built with Cygwin, so they could use Cygwin-style symlinks; but
      # newer toolchains do not use Cygwin, and nothing else on the system
      # understands Cygwin-style symlinks, so avoid them.
      elif m.issym() and sys.platform == 'win32':
        # For a hard link, the link target (m.linkname) always appears
        # in the archive before the link itself (m.name), so the links
        # can just be made on the fly.  However, a symlink might well
        # appear in the archive before its target file, so there would
        # not yet be any file to hard-link to.  Hence, we have to collect
        # all the symlinks and create them in dependency order at the end.
        linkname = m.linkname
        if not posixpath.isabs(linkname):
          linkname = posixpath.join(posixpath.dirname(m.name), linkname)
        linkname = posixpath.normpath(linkname)
        win32_symlinks[posixpath.normpath(m.name)] = linkname
      # Otherwise, extract normally.
      else:
        self.tar.extract(m)

    win32_symlinks_left = win32_symlinks.items()
    while win32_symlinks_left:
      this_symlink = win32_symlinks_left.pop(0)
      name, linkname = this_symlink
      if linkname in win32_symlinks:
        # The target is itself a symlink not yet created.
        # Wait for it to come 'round on the guitar.
        win32_symlinks_left.append(this_symlink)
      else:
        del win32_symlinks[name]
        CreateWin32Link(name, linkname, self.verbose)

    if self.verbose:
      sys.stdout.write('\n')
      sys.stdout.flush()

  def List(self):
    """List the set of objects in the tarball."""
    for tarinfo in self.tar:
      self.__DumpInfo(tarinfo)

  def Close(self):
    self.tar.close()
    if self.read_file is not None:
      self.read_file.close()
      self.read_file = None
      self.read_filesize = 0

def Main(args):
  parser = optparse.OptionParser()
  # Modes
  parser.add_option('-c', '--create', help='Create a tarball.',
      action='store_const', const='c', dest='action', default='')
  parser.add_option('-x', '--extract', help='Extract a tarball.',
      action='store_const', const='x', dest='action')
  parser.add_option('-t', '--list', help='List sources in tarball.',
      action='store_const', const='t', dest='action')

  # Compression formats
  parser.add_option('-j', '--bzip2', help='Create a bz2 tarball.',
      action='store_const', const=':bz2', dest='format', default='')
  parser.add_option('-z', '--gzip', help='Create a gzip tarball.',
      action='store_const', const=':gz', dest='format', )
  # Misc
  parser.add_option('-v', '--verbose', help='Use verbose output.',
      action='store_true', dest='verbose', default=False)
  parser.add_option('-f', '--file', help='Name of tarball.',
      dest='filename', default='')
  parser.add_option('-C', '--directory', help='Change directory.',
      dest='cd', default='')
  parser.add_option('--prefix', help='Subdirectory prefix for all paths')

  options, args = parser.parse_args(args[1:])
  if not options.action:
    parser.error('Expecting compress or extract')
  if not options.filename:
    parser.error('Expecting a filename')

  if options.action in ['c'] and not args:
    parser.error('Expecting list of sources to add')
  if options.action in ['x', 't'] and args:
    parser.error('Unexpected source list on extract')

  if options.action == 'c':
    mode = 'w' + options.format
  else:
    mode = 'r'+ options.format

  tar = CygTar(options.filename, mode, verbose=options.verbose)
  if options.cd:
    os.chdir(options.cd)

  if options.action == 't':
    tar.List()
    return 0

  if options.action == 'x':
    tar.Extract()
    return 0

  if options.action == 'c':
    for filepath in args:
      if not tar.Add(filepath, options.prefix):
        return -1
    tar.Close()
    return 0

  parser.error('Missing action c, t, or x.')
  return -1

if __name__ == '__main__':
  sys.exit(Main(sys.argv))
<|endoftext|>


<commit_before>import uuid
from . import log
from .tools import extract_ks_tab, build_remotely, storage_id_from_name, get_istorage_attrs, generate_token_ring_ranges

class AlreadyPersistentError(RuntimeError):
    pass

class IStorage(object):

    @property
    def storage_id(self):
        return self.__storage_id

    @storage_id.setter
    def storage_id(self, st_id):
        if st_id is not None and not isinstance(st_id, uuid.UUID):
            raise TypeError("Storage ID must be an instance of UUID")
        self.__storage_id = st_id

    def __init__(self, *args, **kwargs):
        super().__init__()
        if not getattr(self, "storage_id", None):
            self.storage_id = kwargs.pop("storage_id", None)
        self._built_remotely = kwargs.pop("built_remotely", False)
        self._tokens = kwargs.pop("tokens", None)
        given_name = kwargs.pop("name", None)
        if given_name:
            try:
                self._ksp, self._table = extract_ks_tab(given_name)
                given_name = self._ksp + '.' + self._table
                # Obtain metas for 'given_name'
                #   metas are the same for all pieces of a splitted object but the storage_id and the tokens.
                #   we get the metas from the entry of the object and for the build_remotely we keep
                #   the storage_id and tokens that we receive in self
                metas = get_istorage_attrs(storage_id_from_name(given_name)) # with split ALL splitted objects have the SAME name, but only 1 has the storage_id corresponding to its name!!
                self._istorage_metas = metas[0] # To pass information retrieved from istorage to subojects to avoid further Cassandra accesses
                given_name   = metas[0].name
                if not self._built_remotely: # When build remotely due to a split, it uses the same table name but with a different storage_id! That is stored later in hecuba istorage.
                    self.storage_id = metas[0].storage_id # Name has priority
                    self._tokens = metas[0].tokens
                if not getattr(self, '_tokens', None): #Tokens are received from kwargs or IStorage otherwise...
                    self._tokens = generate_token_ring_ranges()
            except IndexError:
                pass
        elif self.storage_id:
            try:
                metas = get_istorage_attrs(self.storage_id)
                self._istorage_metas = metas[0] # To pass information retrieved from istorage to subojects to avoid further Cassandra accesses
                given_name   = metas[0].name
                self._tokens = metas[0].tokens
            except IndexError:
                pass
        if given_name :
            # Warning: In order to inherit _tokens and storage_id, they MUST be set before the next call
            IStorage.make_persistent(self, given_name)
        else:
            self._ksp = None
            self._table = None
            self._is_persistent = False

    @classmethod
    def get_by_alias(cls, alias=""):
        return cls(name=alias)

    def __eq__(self, other):
        """
        Method to compare a IStorage object with another one.
        Args:
            other: IStorage to be compared with.
        Returns:
            boolean (true - equals, false - not equals).
        """
        return self.__class__ == other.__class__ and self.storage_id == other.storage_id

    def make_persistent(self, name):
        if getattr(self, '_is_persistent', False):
            raise AlreadyPersistentError("This Object is already persistent [Before:{}.{}][After:{}]".format(
                                         self._ksp, self._table, name))

        self._ksp, self._table = extract_ks_tab(name)
        name = self._ksp + '.' + self._table
        self._set_name(name)

        if not self.storage_id:
            self.storage_id = storage_id_from_name(name)

        # If found data, replace the constructor data
        if not getattr(self,'_tokens', None) :
            metas = get_istorage_attrs(self.storage_id)
            try:
                self._tokens = metas[0].tokens
            except IndexError:
                self._tokens = generate_token_ring_ranges()

        self._is_persistent = True

    def stop_persistent(self):
        if not self._is_persistent:
            raise RuntimeError("This Object is not persistent")

        self._is_persistent = False

    def delete_persistent(self):
        if not self._is_persistent:
            raise RuntimeError("This Object is not persistent")

        self._is_persistent = False

    def _set_name(self, name):
        if name is not None and not isinstance(name, str):
            raise TypeError("Name -{}-  should be an instance of str".format(str(name)))
        self._name = name

    def _get_name(self):
        try:
            return self._name
        except AttributeError:
            return None

    def sync(self):
        if not self._is_persistent:
            raise RuntimeError("Can't send the data to storage if the object is not persistent")

    def getID(self):
        """
        Method to retrieve the storage id as string. Used by PyCOMPSs solely.
        :return: Storage_id as str
        """
        return str(self.storage_id)

    def split(self):
        """
        Method used to divide an object into sub-objects.
        Returns:
            a subobject everytime is called
        """
        from .tools import tokens_partitions
        try:
            tokens = self._build_args.tokens
        except AttributeError as ex:
            raise RuntimeError("Object {} does not have tokens".format(self._get_name()))

        self.sync()

        for token_split in tokens_partitions(self._ksp, self._table, tokens):
            storage_id = uuid.uuid4()
            log.debug('assigning to {} num tokens {}'.format(str(storage_id), len(token_split)))
            new_args = self._build_args._replace(tokens=token_split, storage_id=storage_id)
            args_dict = new_args._asdict()
            args_dict["built_remotely"] = True
            yield build_remotely(args_dict)
<commit_msg>Fix missing update in '_istorage_metas'<commit_after>import uuid
from . import log
from .tools import extract_ks_tab, build_remotely, storage_id_from_name, get_istorage_attrs, generate_token_ring_ranges

class AlreadyPersistentError(RuntimeError):
    pass

class IStorage(object):

    @property
    def storage_id(self):
        return self.__storage_id

    @storage_id.setter
    def storage_id(self, st_id):
        if st_id is not None and not isinstance(st_id, uuid.UUID):
            raise TypeError("Storage ID must be an instance of UUID")
        self.__storage_id = st_id

    def __init__(self, *args, **kwargs):
        super().__init__()
        if not getattr(self, "storage_id", None):
            self.storage_id = kwargs.pop("storage_id", None)
        self._built_remotely = kwargs.pop("built_remotely", False)
        self._tokens = kwargs.pop("tokens", None)
        given_name = kwargs.pop("name", None)
        if given_name:
            try:
                self._ksp, self._table = extract_ks_tab(given_name)
                given_name = self._ksp + '.' + self._table
                # Obtain metas for 'given_name'
                #   metas are the same for all pieces of a splitted object but the storage_id and the tokens.
                #   we get the metas from the entry of the object and for the build_remotely we keep
                #   the storage_id and tokens that we receive in self
                metas = get_istorage_attrs(storage_id_from_name(given_name)) # with split ALL splitted objects have the SAME name, but only 1 has the storage_id corresponding to its name!!
                self._istorage_metas = metas[0] # To pass information retrieved from istorage to subojects to avoid further Cassandra accesses
                given_name   = metas[0].name
                if not self._built_remotely: # When build remotely due to a split, it uses the same table name but with a different storage_id! That is stored later in hecuba istorage.
                    self.storage_id = metas[0].storage_id # Name has priority
                    self._tokens = metas[0].tokens
                if not getattr(self, '_tokens', None): #Tokens are received from kwargs or IStorage otherwise...
                    self._tokens = generate_token_ring_ranges()
            except IndexError:
                pass
        elif self.storage_id:
            try:
                metas = get_istorage_attrs(self.storage_id)
                self._istorage_metas = metas[0] # To pass information retrieved from istorage to subojects to avoid further Cassandra accesses
                given_name   = metas[0].name
                self._tokens = metas[0].tokens
            except IndexError:
                pass
        if given_name :
            # Warning: In order to inherit _tokens and storage_id, they MUST be set before the next call
            IStorage.make_persistent(self, given_name)
        else:
            self._ksp = None
            self._table = None
            self._is_persistent = False

    @classmethod
    def get_by_alias(cls, alias=""):
        return cls(name=alias)

    def __eq__(self, other):
        """
        Method to compare a IStorage object with another one.
        Args:
            other: IStorage to be compared with.
        Returns:
            boolean (true - equals, false - not equals).
        """
        return self.__class__ == other.__class__ and self.storage_id == other.storage_id

    def make_persistent(self, name):
        if getattr(self, '_is_persistent', False):
            raise AlreadyPersistentError("This Object is already persistent [Before:{}.{}][After:{}]".format(
                                         self._ksp, self._table, name))

        self._ksp, self._table = extract_ks_tab(name)
        name = self._ksp + '.' + self._table
        self._set_name(name)

        if not self.storage_id:
            self.storage_id = storage_id_from_name(name)

        # If found data, replace the constructor data
        if not getattr(self,'_tokens', None) :
            metas = get_istorage_attrs(self.storage_id)
            try:
                self._tokens = metas[0].tokens
                self._istorage_metas = metas[0] # To pass information retrieved from istorage to subojects to avoid further Cassandra accesses
            except IndexError:
                self._tokens = generate_token_ring_ranges()

        self._is_persistent = True

    def stop_persistent(self):
        if not self._is_persistent:
            raise RuntimeError("This Object is not persistent")

        self._is_persistent = False

    def delete_persistent(self):
        if not self._is_persistent:
            raise RuntimeError("This Object is not persistent")

        self._is_persistent = False

    def _set_name(self, name):
        if name is not None and not isinstance(name, str):
            raise TypeError("Name -{}-  should be an instance of str".format(str(name)))
        self._name = name

    def _get_name(self):
        try:
            return self._name
        except AttributeError:
            return None

    def sync(self):
        if not self._is_persistent:
            raise RuntimeError("Can't send the data to storage if the object is not persistent")

    def getID(self):
        """
        Method to retrieve the storage id as string. Used by PyCOMPSs solely.
        :return: Storage_id as str
        """
        return str(self.storage_id)

    def split(self):
        """
        Method used to divide an object into sub-objects.
        Returns:
            a subobject everytime is called
        """
        from .tools import tokens_partitions
        try:
            tokens = self._build_args.tokens
        except AttributeError as ex:
            raise RuntimeError("Object {} does not have tokens".format(self._get_name()))

        self.sync()

        for token_split in tokens_partitions(self._ksp, self._table, tokens):
            storage_id = uuid.uuid4()
            log.debug('assigning to {} num tokens {}'.format(str(storage_id), len(token_split)))
            new_args = self._build_args._replace(tokens=token_split, storage_id=storage_id)
            args_dict = new_args._asdict()
            args_dict["built_remotely"] = True
            yield build_remotely(args_dict)
<|endoftext|>


<commit_before><commit_msg>Linting<commit_after><|endoftext|>


<commit_before>from twisted.protocols.jabber import component
from twisted.application import service
from twisted.python import components
import backend
import pubsub
import xmpp_error

import sys

NS_DISCO = 'http://jabber.org/protocol/disco'
NS_DISCO_INFO = NS_DISCO + '#info'
NS_DISCO_ITEMS = NS_DISCO + '#items'
NS_VERSION = 'jabber:iq:version'

IQ_GET = '/iq[@type="get"]'
IQ_SET = '/iq[@type="set"]'
VERSION = IQ_GET + '/query[@xmlns="' + NS_VERSION + '"]'
DISCO_INFO = IQ_GET + '/query[@xmlns="' + NS_DISCO_INFO + '"]'
DISCO_ITEMS = IQ_GET + '/query[@xmlns="' + NS_DISCO_ITEMS + '"]'

class IdavollService(component.Service):

	def componentConnected(self, xmlstream):
		self.xmlstream = xmlstream
		xmlstream.addObserver(VERSION, self.onVersion, 1)
		xmlstream.addObserver(DISCO_INFO, self.onDiscoInfo, 1)
		xmlstream.addObserver(DISCO_ITEMS, self.onDiscoItems, 1)
		xmlstream.addObserver(IQ_GET, self.iqFallback, -1)
		xmlstream.addObserver(IQ_SET, self.iqFallback, -1)

	def getFeatures(self, node):
		if not node:
			return [NS_DISCO_INFO, NS_DISCO_ITEMS, NS_VERSION]

	def onVersion(self, iq):
		print "version?"
		iq.swapAttributeValues("to", "from")
		iq["type"] = "result"
		name = iq.addElement("name", None, 'Idavoll')
		version = iq.addElement("version", None, '0.1')
		self.send(iq)
		iq.handled = True

	def onDiscoInfo(self, iq):
		identities = []
		features = []
		node = iq.query.getAttribute("node")

		for c in self.parent:
			if components.implements(c, component.IService):
				if hasattr(c, "getIdentities"):
					identities.extend(c.getIdentities(node))
				if hasattr(c, "getFeatures"):
					features.extend(c.getFeatures(node))

		if not features and not identities and not node:
			xmpp_error.error_from_iq(iq, 'item-not-found')
		else:
			iq.swapAttributeValues("to", "from")
			iq["type"] = "result"
			for identity in identities:
				iq.query.addElement("identity", None)
				iq.query.identity.attributes = identity
			print features
			for feature in features:
				f = iq.query.addElement("feature", None)
				f["var"] = feature

		self.send(iq)
		iq.handled = True

	def onDiscoItems(self, iq):
		iq.swapAttributeValues("to", "from")
		iq["type"] = "result"
		iq.query.children = []
		self.send(iq)

	def iqFallback(self, iq):
		if iq.handled == True:
			return

		self.send(xmpp_error.error_from_iq(iq, 'feature-not-implemented'))

def makeService(config):
	serviceCollection = service.MultiService()

	pss = backend.MemoryBackendService()

	# set up Jabber Component
	c = component.buildServiceManager(config["jid"], config["secret"],
			("tcp:%s:%s" % (config["rhost"], config["rport"])))

	s = component.IService(pss)
	s.jid = config["jid"]
	s.setServiceParent(c)

	s = IdavollService()
	s.setServiceParent(c)

	c.setServiceParent(serviceCollection)

	# other stuff

	return c
<commit_msg>Don't return disco#info, that's silly. Remove left-over print statement.<commit_after>from twisted.protocols.jabber import component
from twisted.application import service
from twisted.python import components
import backend
import pubsub
import xmpp_error

import sys

NS_DISCO = 'http://jabber.org/protocol/disco'
NS_DISCO_INFO = NS_DISCO + '#info'
NS_DISCO_ITEMS = NS_DISCO + '#items'
NS_VERSION = 'jabber:iq:version'

IQ_GET = '/iq[@type="get"]'
IQ_SET = '/iq[@type="set"]'
VERSION = IQ_GET + '/query[@xmlns="' + NS_VERSION + '"]'
DISCO_INFO = IQ_GET + '/query[@xmlns="' + NS_DISCO_INFO + '"]'
DISCO_ITEMS = IQ_GET + '/query[@xmlns="' + NS_DISCO_ITEMS + '"]'

class IdavollService(component.Service):

	def componentConnected(self, xmlstream):
		self.xmlstream = xmlstream
		xmlstream.addObserver(VERSION, self.onVersion, 1)
		xmlstream.addObserver(DISCO_INFO, self.onDiscoInfo, 1)
		xmlstream.addObserver(DISCO_ITEMS, self.onDiscoItems, 1)
		xmlstream.addObserver(IQ_GET, self.iqFallback, -1)
		xmlstream.addObserver(IQ_SET, self.iqFallback, -1)

	def getFeatures(self, node):
		if not node:
			return [NS_DISCO_ITEMS, NS_VERSION]

	def onVersion(self, iq):
		iq.swapAttributeValues("to", "from")
		iq["type"] = "result"
		name = iq.addElement("name", None, 'Idavoll')
		version = iq.addElement("version", None, '0.1')
		self.send(iq)
		iq.handled = True

	def onDiscoInfo(self, iq):
		identities = []
		features = []
		node = iq.query.getAttribute("node")

		for c in self.parent:
			if components.implements(c, component.IService):
				if hasattr(c, "getIdentities"):
					identities.extend(c.getIdentities(node))
				if hasattr(c, "getFeatures"):
					features.extend(c.getFeatures(node))

		if not features and not identities and not node:
			xmpp_error.error_from_iq(iq, 'item-not-found')
		else:
			iq.swapAttributeValues("to", "from")
			iq["type"] = "result"
			for identity in identities:
				iq.query.addElement("identity", None)
				iq.query.identity.attributes = identity
			print features
			for feature in features:
				f = iq.query.addElement("feature", None)
				f["var"] = feature

		self.send(iq)
		iq.handled = True

	def onDiscoItems(self, iq):
		iq.swapAttributeValues("to", "from")
		iq["type"] = "result"
		iq.query.children = []
		self.send(iq)

	def iqFallback(self, iq):
		if iq.handled == True:
			return

		self.send(xmpp_error.error_from_iq(iq, 'feature-not-implemented'))

def makeService(config):
	serviceCollection = service.MultiService()

	pss = backend.MemoryBackendService()

	# set up Jabber Component
	c = component.buildServiceManager(config["jid"], config["secret"],
			("tcp:%s:%s" % (config["rhost"], config["rport"])))

	s = component.IService(pss)
	s.jid = config["jid"]
	s.setServiceParent(c)

	s = IdavollService()
	s.setServiceParent(c)

	c.setServiceParent(serviceCollection)

	# other stuff

	return c
<|endoftext|>