<commit_before>#!/usr/bin/env python
try:
    import json
except ImportError:
    import simplejson as json
import codecs

import optparse
import os
import sys
import hashlib
import os.path
from M2Crypto import BIO, Rand, SMIME

class imagemodel:
    interestingkeys = set([u'vmi_uid'])
    def __init__(self,
        vmi_uid=None,
        vmi_url=None,
        vmi_hash_sha512=None,
        vmi_hypervisor=None,
        vmi_description=None,
        vmi_os_version=None,
        vmi_os_architecture=None,
        vmi_disk_size=None,
        ):

        self.metadata = {u'vmi_uid' : vmi_uid,
            u'vmi_url' : vmi_url,
            u'vmi_hash_sha512' :vmi_hash_sha512,
            u'vmi_hypervisor' : vmi_hypervisor,
            u'vmi_description' : vmi_description,
            u'vmi_os_version' : vmi_os_version,
            u'vmi_os_architecture' : vmi_os_architecture,
            u'vmi_disk_size' : vmi_disk_size}

class listmodel:
    interestingkeys = set([u'vmi_uid'])
    def __init__(self,owner_real_name=None,
        owner_email=None,
        vmic_url=None,
        images=[]):
        self.metadata = {
            u'owner_real_name' : owner_real_name,
            u'owner_email' : owner_email,
            u'vmic_url' : vmic_url,
            }
        self.images = images

class VMimageListEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, imagemodel):
            return {u'metadata' : obj.metadata}
        if isinstance(obj, listmodel):
            imagelist = []
            for i in obj.images:
                tmp_data = self.default(i)
                if tmp_data == None:
                    return json.JSONEncoder.default(self, obj)
                else:
                    imagelist.append(self.default(i))
            return {u'metadata' : obj.metadata, u'images' : imagelist}
        return json.JSONEncoder.default(self, obj)

def VMimageDecoder(dct):
    if not u'metadata' in dct.keys():
        return None
    metadata = dct[u'metadata']
    if metadata == None:
        print "coding error=%s" % (dct)

        return None
    requiredmetadata = [u'vmi_uid',
        u'vmi_url',
        u'vmi_hash_sha512',
        u'vmi_hypervisor',
        u'vmi_description',
        u'vmi_os_version',
        u'vmi_os_architecture',
        u'vmi_disk_size']
    requiredmetadataset = set(requiredmetadata)
    if not requiredmetadataset.issubset(metadata.keys()):

        return None
    return imagemodel(vmi_uid=metadata[u'vmi_uid'],
        vmi_url=metadata[u'vmi_url'],
        vmi_hash_sha512=metadata[u'vmi_hash_sha512'],
        vmi_hypervisor=metadata[u'vmi_hypervisor'],
        vmi_description=metadata[u'vmi_description'],
        vmi_os_version=metadata[u'vmi_os_version'],
        vmi_os_architecture=metadata[u'vmi_os_architecture'],
        vmi_disk_size=metadata[u'vmi_disk_size']
        )

def VMimageListDecoder(dct):
    requiredkeys = [u'metadata', u'images']
    requiredkeysset = set(requiredkeys)
    if not requiredkeysset.issubset(dct.keys()):
        return False
    listmetadata = dct[u'metadata']
    requiredmetadata = [u'owner_real_name',
        u'owner_real_name',
        u'owner_email',
        u'vmic_url']
    requiredmetadataset = set(requiredmetadata)
    if not requiredmetadataset.issubset(listmetadata.keys()):
        return False
    images = dct[u'images']
    allimages = []
    for image in images:
        translatedimage = VMimageDecoder(image)
        if not translatedimage == None:
            allimages.append(translatedimage)
    # Now we generate the output
    output = listmodel(owner_real_name=listmetadata[u'owner_real_name'],
        owner_email=listmetadata[u'owner_email'],
        vmic_url=listmetadata[u'vmic_url'],
        images=allimages
        )
    return output

def file_extract_metadata(file_name):
    if file_name == None:
        return
    if not os.path.isfile(file_name):
        return None
    m = hashlib.sha512()
    filelength = 0
    for line in open(file_name,'r'):
        filelength += len(line)
        m.update(line)
    return {u'vmi_disk_size' : filelength,
            u'vmi_hash_sha512' : m.hexdigest()}

class vmlistview:
    def load_file(self,filename):
        loadedfile = None
        fp = open(filename, 'r')

        loadedfile = json.load(fp)
        decoded_image = VMimageListDecoder(loadedfile)
        if decoded_image == False:
            return False
        return decoded_image

    def save_file(self,entry,filename):
        f = open(filename, 'w')
        json.dump(entry, f, cls=VMimageListEncoder, sort_keys=True, indent=4)
        return True
    def images_list(self,entry):
        #print "frod=%s" % (entry)
        for item in entry.images:
            print item.metadata[u'vmi_uid']
    def images_list(self,entry):
        #print "frod=%s" % (entry)
        for item in entry.images:
            print item.metadata[u'vmi_uid']
    def dumps(self,entry):
        return json.dumps(entry, cls=VMimageListEncoder, sort_keys=True, indent=4)

class vmlistcontroler:
    def __init__(self):
        self.view = vmlistview()
        self.model = listmodel()
    def load(self,filename):
        try:
            candidate = self.view.load_file(filename)
        except ValueError:
            return False
        if candidate == False:
            return False
        self.model = candidate
        return True

    def save(self,filename):
        self.view.save_file(self.model,filename)

    def images_list(self):
        if self.model == None:
            return False
        return self.view.images_list(self.model)
    def image_add(self,filename):
        f = open(filename, 'r')
        json_stuff = json.load(f)
        fred= VMimageDecoder(json_stuff)
        if fred == None:
            return False
        self.model.images.append(fred)
        return True

    def image_del(self,uuid):
        matchuuid = None
        if uuid == 'null':
            matchuuid = None
        else:
            matchuuid = uuid
        todelete = []
        for i in range(len(self.model.images)):
            item = self.model.images[i]
            if item.metadata[u'vmi_uid'] == matchuuid:
                todelete.insert(0, i)
        if len(todelete) == 0:
            return False
        for i in todelete:
            del self.model.images[i]
        return True

    def verify(self):
        if 0 == len(self.model.images):
            print "No images in data no point signing"
            return False
        requiredmetadata = [u'owner_real_name',
            u'owner_email',
            u'vmic_url']
        requiredmetadataset = set(requiredmetadata)
        if not requiredmetadataset.issubset(self.model.metadata.keys()):
            print "missing metadata"
            return False

        for item in requiredmetadata:
            value = self.model.metadata[item]
            if value == None:
                print "image list metadata set to null '%s'" % (item)
                return False
        return True

    def sign(self,signer_key,signer_cert,outfile):
        content = self.view.dumps(self.model)

        self.SMIME = SMIME.SMIME()
        self.SMIME.load_key(signer_key,signer_cert)
        buf = BIO.MemoryBuffer(content)
        p7 = self.SMIME.sign(buf, SMIME.PKCS7_DETACHED)
        buf = BIO.MemoryBuffer(content)
        out = BIO.MemoryBuffer()
        self.SMIME.write(out, p7, buf)
        self.message_signed = str(out.read())
        f = open(outfile, 'w')
        f.write(self.message_signed )
        # Save the PRNG's state.
    def generate(self,filename,imagepath=None):
        output_image = None

        if imagepath!=None:

            metadata = file_extract_metadata(imagepath)
            if metadata == None:
                print "error reading file '%s'." % (imagename)
                sys.exit(1)
            output_image = imagemodel(
                vmi_hash_sha512= metadata[u'vmi_hash_sha512'],
                vmi_disk_size=metadata[u'vmi_disk_size'])
        else:
            output_image = imagemodel()
        f = open(filename, 'w')
        json.dump(output_image, f, cls=VMimageListEncoder, sort_keys=True, indent=4)

def test_things():
    view = vmlistview()
    loadedlist = view.load_file('imagelist.json')

    avmlistOutput = json.dumps(loadedlist,cls=VMimageListEncoder)
    print "vmimagelist=%s" % (avmlistOutput)
    f = imagemodel(vmi_uid='32455374',
        vmi_url='http://www.yokel.org',
        vmi_hash_sha512='bignum',
        vmi_hypervisor='kvm',
        vmi_description='easy to run vm',
        vmi_os_version='sl6.0201101',
        vmi_os_architecture='i386',
        vmi_disk_size='20e12bytes')
    imageoutput = json.dumps(f,cls=VMimageListEncoder)
    print "xdx%s" % (imageoutput)
    fred = json.loads(imageoutput,object_hook=VMimageDecoder)
    imageoutput = json.dumps(f,cls=VMimageListEncoder)
    print "xdx%s" % (imageoutput)
    avmlist = listmodel(owner_real_name="Owen Synge",
        owner_email='owen.synge@yokel.org',
        vmi_url='www.yokel.org',
        vmic_url='www.yokel.org',
        images=[f])
    avmlistOutput = json.dumps(avmlist,cls=VMimageListEncoder)
    print "vmimagelist=%s" % (avmlistOutput)
    view.save_file(avmlist,'imagelist.json')

# User interface

def pairsNnot(list_a,list_b):
    len_generate_list = len(list_a)
    len_image_list = len(list_b)
    ocupies_generate_list = set(range(len_generate_list))
    ocupies_image_list = set(range(len_image_list))
    ocupies_pairs = ocupies_image_list.intersection(ocupies_generate_list)
    diff_a = ocupies_generate_list.difference(ocupies_image_list)
    diff_b = ocupies_image_list.difference(ocupies_generate_list)
    arepairs = []
    for i in ocupies_pairs:
        arepairs.append([list_a[i],list_b[i]])
    notpairs_a = []
    for i in diff_a:
        notpairs_a.append(list_a[i])
    notpairs_b = []
    for i in diff_b:
        notpairs_b.append(list_b[i])

    return arepairs,notpairs_a,notpairs_b

def main():
    """Runs program and handles command line options"""
    actions = set([])
    listcontroler = vmlistcontroler()
    p = optparse.OptionParser()
    p.add_option('-j', '--json', action ='store', help='Path of the json output file')
    p.add_option('-t', '--template', action ='store', help='Path of the json template file')
    p.add_option('-a', '--add', action ='append', help='adds a VM image to the JSON')
    p.add_option('-d', '--delete', action ='append', help='del a VM image to the JSON')
    p.add_option('-g', '--generate', action ='append', help='generates a VM image metadata for image')
    p.add_option('-i', '--image', action ='append', help='Sets the image to generates a VM image metadata')
    p.add_option('-l', '--list', action ='store_true', help='lists VM images in the JSON')
    p.add_option('-k', '--signer_key', action ='store', help='path to signer key')
    p.add_option('-c', '--signer_certificate', action ='store', help='path to signer certificate')
    p.add_option('-s', '--sign', action ='store', help='returns verbose output')
    p.add_option('-f', '--format', action ='store', help='Set the format valid values are JSON and XML')

    options, arguments = p.parse_args()
    template = 'imagelist.json'
    json_output = 'imagelist.json'
    generate_list = []
    add_image_file = []
    del_image_metadata = []
    format = None
    signer_key = os.environ['HOME'] + '/.globus/userkey.pem'
    signer_cert = os.environ['HOME'] + '/.globus/usercert.pem'
    signed_output = None
    list_images = False
    imagelist = []
    if options.template:
        template = options.template
        actions.add('load_template')
    if options.json:
        json_output = options.json
        actions.add('save')
    if options.add:
        add_image_file = options.add
        actions.add('image_add')
    if options.delete:
        del_image_file = options.delete
        actions.add('image_del')
    if options.list:
        list_images = True
        actions.add('image_list')
    if options.signer_key:
        signer_key = options.signer_key
    if options.signer_certificate:
        signer_cert = options.signer_certificate
    if options.sign:
        actions.add('verify')
        actions.add('sign')
        signed_output = options.sign
    if options.generate:
        generate_list = options.generate
        actions.add('generate')
    if options.format:
        print "Currently only supports JSON output RDF XML output may come later"
    if options.image:
        imagelist = options.image
        actions.add('generate')

    # Now process the actions.

    if actions.__contains__('generate'):
        pairs, extra_gens ,extra_images = pairsNnot(generate_list,imagelist)
        if len(extra_images) > 0:
            print "error images and no target"
        for paired_items in pairs:
            listcontroler.generate(paired_items[0],paired_items[1])
        for gen_it in extra_gens:
            listcontroler.generate(gen_it)
    if actions.__contains__('load_template'):
        listcontroler.load(template)
    if actions.__contains__('image_add'):
        for item in add_image_file:
            listcontroler.image_add(item)
    if actions.__contains__('image_del'):
        for item in del_image_file:
            success = listcontroler.image_del(item)
            if success == False:
                print "Failed to delete image '%s'" % (item)
                sys.exit(1)
    if actions.__contains__('image_list'):
        listcontroler.images_list()
    if actions.__contains__('verify'):
        success = listcontroler.verify()
        if success == False:
            print "Failed to verify valid meta data for image."
            sys.exit(1)
    if actions.__contains__('sign'):
        listcontroler.sign(signer_key,signer_cert,signed_output)
    if actions.__contains__('save'):
        listcontroler.save(json_output)
    #test_things()
if __name__ == "__main__":
    main()
<commit_msg>Small update accounting for the difficulties and confusion including json or simplejson on different operating systems<commit_after>#!/usr/bin/env python

import codecs
import optparse
import os
import sys
import hashlib
import os.path
# needed to the signing of images.
from M2Crypto import BIO, Rand, SMIME

# simplejson is included with Python 2.6 and above
# with the name json
if float(sys.version[:3]) >= 2.6:
    import json
else:
    # python 2.4 or 2.5 can also import simplejson
    # as working alternative to the json included.
    import simplejson as json

class imagemodel:
    interestingkeys = set([u'vmi_uid'])
    def __init__(self,
        vmi_uid=None,
        vmi_url=None,
        vmi_hash_sha512=None,
        vmi_hypervisor=None,
        vmi_description=None,
        vmi_os_version=None,
        vmi_os_architecture=None,
        vmi_disk_size=None,
        ):

        self.metadata = {u'vmi_uid' : vmi_uid,
            u'vmi_url' : vmi_url,
            u'vmi_hash_sha512' :vmi_hash_sha512,
            u'vmi_hypervisor' : vmi_hypervisor,
            u'vmi_description' : vmi_description,
            u'vmi_os_version' : vmi_os_version,
            u'vmi_os_architecture' : vmi_os_architecture,
            u'vmi_disk_size' : vmi_disk_size}

class listmodel:
    interestingkeys = set([u'vmi_uid'])
    def __init__(self,owner_real_name=None,
        owner_email=None,
        vmic_url=None,
        images=[]):
        self.metadata = {
            u'owner_real_name' : owner_real_name,
            u'owner_email' : owner_email,
            u'vmic_url' : vmic_url,
            }
        self.images = images

class VMimageListEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, imagemodel):
            return {u'metadata' : obj.metadata}
        if isinstance(obj, listmodel):
            imagelist = []
            for i in obj.images:
                tmp_data = self.default(i)
                if tmp_data == None:
                    return json.JSONEncoder.default(self, obj)
                else:
                    imagelist.append(self.default(i))
            return {u'metadata' : obj.metadata, u'images' : imagelist}
        return json.JSONEncoder.default(self, obj)

def VMimageDecoder(dct):
    if not u'metadata' in dct.keys():
        return None
    metadata = dct[u'metadata']
    if metadata == None:
        print "coding error=%s" % (dct)

        return None
    requiredmetadata = [u'vmi_uid',
        u'vmi_url',
        u'vmi_hash_sha512',
        u'vmi_hypervisor',
        u'vmi_description',
        u'vmi_os_version',
        u'vmi_os_architecture',
        u'vmi_disk_size']
    requiredmetadataset = set(requiredmetadata)
    if not requiredmetadataset.issubset(metadata.keys()):

        return None
    return imagemodel(vmi_uid=metadata[u'vmi_uid'],
        vmi_url=metadata[u'vmi_url'],
        vmi_hash_sha512=metadata[u'vmi_hash_sha512'],
        vmi_hypervisor=metadata[u'vmi_hypervisor'],
        vmi_description=metadata[u'vmi_description'],
        vmi_os_version=metadata[u'vmi_os_version'],
        vmi_os_architecture=metadata[u'vmi_os_architecture'],
        vmi_disk_size=metadata[u'vmi_disk_size']
        )

def VMimageListDecoder(dct):
    requiredkeys = [u'metadata', u'images']
    requiredkeysset = set(requiredkeys)
    if not requiredkeysset.issubset(dct.keys()):
        return False
    listmetadata = dct[u'metadata']
    requiredmetadata = [u'owner_real_name',
        u'owner_real_name',
        u'owner_email',
        u'vmic_url']
    requiredmetadataset = set(requiredmetadata)
    if not requiredmetadataset.issubset(listmetadata.keys()):
        return False
    images = dct[u'images']
    allimages = []
    for image in images:
        translatedimage = VMimageDecoder(image)
        if not translatedimage == None:
            allimages.append(translatedimage)
    # Now we generate the output
    output = listmodel(owner_real_name=listmetadata[u'owner_real_name'],
        owner_email=listmetadata[u'owner_email'],
        vmic_url=listmetadata[u'vmic_url'],
        images=allimages
        )
    return output

def file_extract_metadata(file_name):
    if file_name == None:
        return
    if not os.path.isfile(file_name):
        return None
    m = hashlib.sha512()
    filelength = 0
    for line in open(file_name,'r'):
        filelength += len(line)
        m.update(line)
    return {u'vmi_disk_size' : filelength,
            u'vmi_hash_sha512' : m.hexdigest()}

class vmlistview:
    def load_file(self,filename):
        loadedfile = None
        fp = open(filename, 'r')

        loadedfile = json.load(fp)
        decoded_image = VMimageListDecoder(loadedfile)
        if decoded_image == False:
            return False
        return decoded_image

    def save_file(self,entry,filename):
        f = open(filename, 'w')
        json.dump(entry, f, cls=VMimageListEncoder, sort_keys=True, indent=4)
        return True
    def images_list(self,entry):
        #print "frod=%s" % (entry)
        for item in entry.images:
            print item.metadata[u'vmi_uid']
    def images_list(self,entry):
        #print "frod=%s" % (entry)
        for item in entry.images:
            print item.metadata[u'vmi_uid']
    def dumps(self,entry):
        return json.dumps(entry, cls=VMimageListEncoder, sort_keys=True, indent=4)

class vmlistcontroler:
    def __init__(self):
        self.view = vmlistview()
        self.model = listmodel()
    def load(self,filename):
        try:
            candidate = self.view.load_file(filename)
        except ValueError:
            return False
        if candidate == False:
            return False
        self.model = candidate
        return True

    def save(self,filename):
        self.view.save_file(self.model,filename)

    def images_list(self):
        if self.model == None:
            return False
        return self.view.images_list(self.model)
    def image_add(self,filename):
        f = open(filename, 'r')
        json_stuff = json.load(f)
        fred= VMimageDecoder(json_stuff)
        if fred == None:
            return False
        self.model.images.append(fred)
        return True

    def image_del(self,uuid):
        matchuuid = None
        if uuid == 'null':
            matchuuid = None
        else:
            matchuuid = uuid
        todelete = []
        for i in range(len(self.model.images)):
            item = self.model.images[i]
            if item.metadata[u'vmi_uid'] == matchuuid:
                todelete.insert(0, i)
        if len(todelete) == 0:
            return False
        for i in todelete:
            del self.model.images[i]
        return True

    def verify(self):
        if 0 == len(self.model.images):
            print "No images in data no point signing"
            return False
        requiredmetadata = [u'owner_real_name',
            u'owner_email',
            u'vmic_url']
        requiredmetadataset = set(requiredmetadata)
        if not requiredmetadataset.issubset(self.model.metadata.keys()):
            print "missing metadata"
            return False

        for item in requiredmetadata:
            value = self.model.metadata[item]
            if value == None:
                print "image list metadata set to null '%s'" % (item)
                return False
        return True

    def sign(self,signer_key,signer_cert,outfile):
        content = self.view.dumps(self.model)

        self.SMIME = SMIME.SMIME()
        self.SMIME.load_key(signer_key,signer_cert)
        buf = BIO.MemoryBuffer(content)
        p7 = self.SMIME.sign(buf, SMIME.PKCS7_DETACHED)
        buf = BIO.MemoryBuffer(content)
        out = BIO.MemoryBuffer()
        self.SMIME.write(out, p7, buf)
        self.message_signed = str(out.read())
        f = open(outfile, 'w')
        f.write(self.message_signed )
        # Save the PRNG's state.
    def generate(self,filename,imagepath=None):
        output_image = None

        if imagepath!=None:

            metadata = file_extract_metadata(imagepath)
            if metadata == None:
                print "error reading file '%s'." % (imagename)
                sys.exit(1)
            output_image = imagemodel(
                vmi_hash_sha512= metadata[u'vmi_hash_sha512'],
                vmi_disk_size=metadata[u'vmi_disk_size'])
        else:
            output_image = imagemodel()
        f = open(filename, 'w')
        json.dump(output_image, f, cls=VMimageListEncoder, sort_keys=True, indent=4)

def test_things():
    view = vmlistview()
    loadedlist = view.load_file('imagelist.json')

    avmlistOutput = json.dumps(loadedlist,cls=VMimageListEncoder)
    print "vmimagelist=%s" % (avmlistOutput)
    f = imagemodel(vmi_uid='32455374',
        vmi_url='http://www.yokel.org',
        vmi_hash_sha512='bignum',
        vmi_hypervisor='kvm',
        vmi_description='easy to run vm',
        vmi_os_version='sl6.0201101',
        vmi_os_architecture='i386',
        vmi_disk_size='20e12bytes')
    imageoutput = json.dumps(f,cls=VMimageListEncoder)
    print "xdx%s" % (imageoutput)
    fred = json.loads(imageoutput,object_hook=VMimageDecoder)
    imageoutput = json.dumps(f,cls=VMimageListEncoder)
    print "xdx%s" % (imageoutput)
    avmlist = listmodel(owner_real_name="Owen Synge",
        owner_email='owen.synge@yokel.org',
        vmi_url='www.yokel.org',
        vmic_url='www.yokel.org',
        images=[f])
    avmlistOutput = json.dumps(avmlist,cls=VMimageListEncoder)
    print "vmimagelist=%s" % (avmlistOutput)
    view.save_file(avmlist,'imagelist.json')

# User interface

def pairsNnot(list_a,list_b):
    len_generate_list = len(list_a)
    len_image_list = len(list_b)
    ocupies_generate_list = set(range(len_generate_list))
    ocupies_image_list = set(range(len_image_list))
    ocupies_pairs = ocupies_image_list.intersection(ocupies_generate_list)
    diff_a = ocupies_generate_list.difference(ocupies_image_list)
    diff_b = ocupies_image_list.difference(ocupies_generate_list)
    arepairs = []
    for i in ocupies_pairs:
        arepairs.append([list_a[i],list_b[i]])
    notpairs_a = []
    for i in diff_a:
        notpairs_a.append(list_a[i])
    notpairs_b = []
    for i in diff_b:
        notpairs_b.append(list_b[i])

    return arepairs,notpairs_a,notpairs_b

def main():
    """Runs program and handles command line options"""
    actions = set([])
    listcontroler = vmlistcontroler()
    p = optparse.OptionParser()
    p.add_option('-j', '--json', action ='store', help='Path of the json output file')
    p.add_option('-t', '--template', action ='store', help='Path of the json template file')
    p.add_option('-a', '--add', action ='append', help='adds a VM image to the JSON')
    p.add_option('-d', '--delete', action ='append', help='del a VM image to the JSON')
    p.add_option('-g', '--generate', action ='append', help='generates a VM image metadata for image')
    p.add_option('-i', '--image', action ='append', help='Sets the image to generates a VM image metadata')
    p.add_option('-l', '--list', action ='store_true', help='lists VM images in the JSON')
    p.add_option('-k', '--signer_key', action ='store', help='path to signer key')
    p.add_option('-c', '--signer_certificate', action ='store', help='path to signer certificate')
    p.add_option('-s', '--sign', action ='store', help='returns verbose output')
    p.add_option('-f', '--format', action ='store', help='Set the format valid values are JSON and XML')

    options, arguments = p.parse_args()
    template = 'imagelist.json'
    json_output = 'imagelist.json'
    generate_list = []
    add_image_file = []
    del_image_metadata = []
    format = None
    signer_key = os.environ['HOME'] + '/.globus/userkey.pem'
    signer_cert = os.environ['HOME'] + '/.globus/usercert.pem'
    signed_output = None
    list_images = False
    imagelist = []
    if options.template:
        template = options.template
        actions.add('load_template')
    if options.json:
        json_output = options.json
        actions.add('save')
    if options.add:
        add_image_file = options.add
        actions.add('image_add')
    if options.delete:
        del_image_file = options.delete
        actions.add('image_del')
    if options.list:
        list_images = True
        actions.add('image_list')
    if options.signer_key:
        signer_key = options.signer_key
    if options.signer_certificate:
        signer_cert = options.signer_certificate
    if options.sign:
        actions.add('verify')
        actions.add('sign')
        signed_output = options.sign
    if options.generate:
        generate_list = options.generate
        actions.add('generate')
    if options.format:
        print "Currently only supports JSON output RDF XML output may come later"
    if options.image:
        imagelist = options.image
        actions.add('generate')

    # Now process the actions.

    if actions.__contains__('generate'):
        pairs, extra_gens ,extra_images = pairsNnot(generate_list,imagelist)
        if len(extra_images) > 0:
            print "error images and no target"
        for paired_items in pairs:
            listcontroler.generate(paired_items[0],paired_items[1])
        for gen_it in extra_gens:
            listcontroler.generate(gen_it)
    if actions.__contains__('load_template'):
        listcontroler.load(template)
    if actions.__contains__('image_add'):
        for item in add_image_file:
            listcontroler.image_add(item)
    if actions.__contains__('image_del'):
        for item in del_image_file:
            success = listcontroler.image_del(item)
            if success == False:
                print "Failed to delete image '%s'" % (item)
                sys.exit(1)
    if actions.__contains__('image_list'):
        listcontroler.images_list()
    if actions.__contains__('verify'):
        success = listcontroler.verify()
        if success == False:
            print "Failed to verify valid meta data for image."
            sys.exit(1)
    if actions.__contains__('sign'):
        listcontroler.sign(signer_key,signer_cert,signed_output)
    if actions.__contains__('save'):
        listcontroler.save(json_output)
    #test_things()
if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from mock import Mock, NonCallableMock

from twisted.internet import defer

import synapse.types
from synapse.api.errors import AuthError, SynapseError
from synapse.handlers.profile import MasterProfileHandler
from synapse.types import UserID

from tests import unittest
from tests.utils import setup_test_homeserver

class ProfileHandlers(object):
    def __init__(self, hs):
        self.profile_handler = MasterProfileHandler(hs)

class ProfileTestCase(unittest.TestCase):
    """ Tests profile management. """

    @defer.inlineCallbacks
    def setUp(self):
        self.mock_federation = Mock()
        self.mock_registry = Mock()

        self.query_handlers = {}

        def register_query_handler(query_type, handler):
            self.query_handlers[query_type] = handler

        self.mock_registry.register_query_handler = register_query_handler

        hs = yield setup_test_homeserver(
            self.addCleanup,
            http_client=None,
            handlers=None,
            resource_for_federation=Mock(),
            federation_client=self.mock_federation,
            federation_server=Mock(),
            federation_registry=self.mock_registry,
            ratelimiter=NonCallableMock(spec_set=["can_do_action"]),
        )

        self.ratelimiter = hs.get_ratelimiter()
        self.ratelimiter.can_do_action.return_value = (True, 0)

        self.store = hs.get_datastore()

        self.frank = UserID.from_string("@1234ABCD:test")
        self.bob = UserID.from_string("@4567:test")
        self.alice = UserID.from_string("@alice:remote")

        yield self.store.create_profile(self.frank.localpart)

        self.handler = hs.get_profile_handler()
        self.hs = hs

    @defer.inlineCallbacks
    def test_get_my_name(self):
        yield self.store.set_profile_displayname(self.frank.localpart, "Frank")

        displayname = yield self.handler.get_displayname(self.frank)

        self.assertEquals("Frank", displayname)

    @defer.inlineCallbacks
    def test_set_my_name(self):
        yield self.handler.set_displayname(
            self.frank, synapse.types.create_requester(self.frank), "Frank Jr."
        )

        self.assertEquals(
            (yield self.store.get_profile_displayname(self.frank.localpart)),
            "Frank Jr.",
        )

    @defer.inlineCallbacks
    def test_set_my_name_if_disabled(self):
        self.hs.config.disable_set_displayname = True

        # Set first displayname is allowed, if displayname is null
        self.store.set_profile_displayname(self.frank.localpart, "Frank")

        d = self.handler.set_displayname(
            self.frank, synapse.types.create_requester(self.frank), "Frank Jr."
        )

        yield self.assertFailure(d, SynapseError)

    @defer.inlineCallbacks
    def test_set_my_name_noauth(self):
        d = self.handler.set_displayname(
            self.frank, synapse.types.create_requester(self.bob), "Frank Jr."
        )

        yield self.assertFailure(d, AuthError)

    @defer.inlineCallbacks
    def test_get_other_name(self):
        self.mock_federation.make_query.return_value = defer.succeed(
            {"displayname": "Alice"}
        )

        displayname = yield self.handler.get_displayname(self.alice)

        self.assertEquals(displayname, "Alice")
        self.mock_federation.make_query.assert_called_with(
            destination="remote",
            query_type="profile",
            args={"user_id": "@alice:remote", "field": "displayname"},
            ignore_backoff=True,
        )

    @defer.inlineCallbacks
    def test_incoming_fed_query(self):
        yield self.store.create_profile("caroline")
        yield self.store.set_profile_displayname("caroline", "Caroline")

        response = yield self.query_handlers["profile"](
            {"user_id": "@caroline:test", "field": "displayname"}
        )

        self.assertEquals({"displayname": "Caroline"}, response)

    @defer.inlineCallbacks
    def test_get_my_avatar(self):
        yield self.store.set_profile_avatar_url(
            self.frank.localpart, "http://my.server/me.png"
        )

        avatar_url = yield self.handler.get_avatar_url(self.frank)

        self.assertEquals("http://my.server/me.png", avatar_url)

    @defer.inlineCallbacks
    def test_set_my_avatar(self):
        yield self.handler.set_avatar_url(
            self.frank,
            synapse.types.create_requester(self.frank),
            "http://my.server/pic.gif",
        )

        self.assertEquals(
            (yield self.store.get_profile_avatar_url(self.frank.localpart)),
            "http://my.server/pic.gif",
        )

    @defer.inlineCallbacks
    def test_set_my_avatar_if_disabled(self):
        self.hs.config.disable_set_avatar_url = True

        # Set first time avatar is allowed, if displayname is null
        self.store.set_profile_avatar_url(
            self.frank.localpart, "http://my.server/me.png"
        )

        d = self.handler.set_avatar_url(
            self.frank,
            synapse.types.create_requester(self.frank),
            "http://my.server/pic.gif",
        )

        yield self.assertFailure(d, SynapseError)
<commit_msg>fix tests<commit_after># -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from mock import Mock, NonCallableMock

from twisted.internet import defer

import synapse.types
from synapse.api.errors import AuthError, SynapseError
from synapse.handlers.profile import MasterProfileHandler
from synapse.types import UserID

from tests import unittest
from tests.utils import setup_test_homeserver

class ProfileHandlers(object):
    def __init__(self, hs):
        self.profile_handler = MasterProfileHandler(hs)

class ProfileTestCase(unittest.TestCase):
    """ Tests profile management. """

    @defer.inlineCallbacks
    def setUp(self):
        self.mock_federation = Mock()
        self.mock_registry = Mock()

        self.query_handlers = {}

        def register_query_handler(query_type, handler):
            self.query_handlers[query_type] = handler

        self.mock_registry.register_query_handler = register_query_handler

        hs = yield setup_test_homeserver(
            self.addCleanup,
            http_client=None,
            handlers=None,
            resource_for_federation=Mock(),
            federation_client=self.mock_federation,
            federation_server=Mock(),
            federation_registry=self.mock_registry,
            ratelimiter=NonCallableMock(spec_set=["can_do_action"]),
        )

        self.ratelimiter = hs.get_ratelimiter()
        self.ratelimiter.can_do_action.return_value = (True, 0)

        self.store = hs.get_datastore()

        self.frank = UserID.from_string("@1234ABCD:test")
        self.bob = UserID.from_string("@4567:test")
        self.alice = UserID.from_string("@alice:remote")

        yield self.store.create_profile(self.frank.localpart)

        self.handler = hs.get_profile_handler()
        self.hs = hs

    @defer.inlineCallbacks
    def test_get_my_name(self):
        yield self.store.set_profile_displayname(self.frank.localpart, "Frank")

        displayname = yield self.handler.get_displayname(self.frank)

        self.assertEquals("Frank", displayname)

    @defer.inlineCallbacks
    def test_set_my_name(self):
        yield self.handler.set_displayname(
            self.frank, synapse.types.create_requester(self.frank), "Frank Jr."
        )

        self.assertEquals(
            (yield self.store.get_profile_displayname(self.frank.localpart)),
            "Frank Jr.",
        )

    @defer.inlineCallbacks
    def test_set_my_name_if_disabled(self):
        self.hs.config.disable_set_displayname = True

        # Set first displayname is allowed, if displayname is null
        yield self.store.set_profile_displayname(self.frank.localpart, "Frank")

        d = self.handler.set_displayname(
            self.frank, synapse.types.create_requester(self.frank), "Frank Jr."
        )

        yield self.assertFailure(d, SynapseError)

    @defer.inlineCallbacks
    def test_set_my_name_noauth(self):
        d = self.handler.set_displayname(
            self.frank, synapse.types.create_requester(self.bob), "Frank Jr."
        )

        yield self.assertFailure(d, AuthError)

    @defer.inlineCallbacks
    def test_get_other_name(self):
        self.mock_federation.make_query.return_value = defer.succeed(
            {"displayname": "Alice"}
        )

        displayname = yield self.handler.get_displayname(self.alice)

        self.assertEquals(displayname, "Alice")
        self.mock_federation.make_query.assert_called_with(
            destination="remote",
            query_type="profile",
            args={"user_id": "@alice:remote", "field": "displayname"},
            ignore_backoff=True,
        )

    @defer.inlineCallbacks
    def test_incoming_fed_query(self):
        yield self.store.create_profile("caroline")
        yield self.store.set_profile_displayname("caroline", "Caroline")

        response = yield self.query_handlers["profile"](
            {"user_id": "@caroline:test", "field": "displayname"}
        )

        self.assertEquals({"displayname": "Caroline"}, response)

    @defer.inlineCallbacks
    def test_get_my_avatar(self):
        yield self.store.set_profile_avatar_url(
            self.frank.localpart, "http://my.server/me.png"
        )

        avatar_url = yield self.handler.get_avatar_url(self.frank)

        self.assertEquals("http://my.server/me.png", avatar_url)

    @defer.inlineCallbacks
    def test_set_my_avatar(self):
        yield self.handler.set_avatar_url(
            self.frank,
            synapse.types.create_requester(self.frank),
            "http://my.server/pic.gif",
        )

        self.assertEquals(
            (yield self.store.get_profile_avatar_url(self.frank.localpart)),
            "http://my.server/pic.gif",
        )

    @defer.inlineCallbacks
    def test_set_my_avatar_if_disabled(self):
        self.hs.config.disable_set_avatar_url = True

        # Set first time avatar is allowed, if displayname is null
        yield self.store.set_profile_avatar_url(
            self.frank.localpart, "http://my.server/me.png"
        )

        d = self.handler.set_avatar_url(
            self.frank,
            synapse.types.create_requester(self.frank),
            "http://my.server/pic.gif",
        )

        yield self.assertFailure(d, SynapseError)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import shutil
import subprocess
import time

def subprocess_on_vm(vm, command):
    return subprocess.Popen(['vagrant', 'ssh', vm, '--', command])

if __name__ == '__main__':
    # Cleanup CFFI-related files
    shutil.rmtree(
        os.path.abspath(os.path.join(os.path.dirname(__file__), '../../gssapi/bindings/autogenerated.cdef')),
        ignore_errors=True
    )
    shutil.rmtree('client.keytab', ignore_errors=True)
    shutil.rmtree('server.keytab', ignore_errors=True)

    subprocess.check_call(('vagrant', 'up'))

    for vm in ('server', 'client'):
        subprocess_on_vm(vm, 'sudo python2 -m pip install -r /python-gssapi/test_requirements.txt').wait()
        subprocess_on_vm(vm, 'sudo pypy -m pip install -r /python-gssapi/test_requirements.txt').wait()
        subprocess_on_vm(vm, 'sudo python3 -m pip install -r /python-gssapi/test_requirements.txt').wait()
        subprocess_on_vm(vm, 'cd /python-gssapi && sudo python2 setup.py install').wait()
        subprocess_on_vm(vm, 'cd /python-gssapi && sudo pypy setup.py install').wait()
        subprocess_on_vm(vm, 'cd /python-gssapi && sudo python3 setup.py install').wait()

    server_procs = (
        subprocess_on_vm('server', 'cd /python-gssapi && sudo python2 tests/integration/server.py'),
        subprocess_on_vm('server', 'cd /python-gssapi && sudo pypy tests/integration/server.py'),
        subprocess_on_vm('server', 'cd /python-gssapi && sudo python3 tests/integration/server.py'),
    )

    print("Wait for server procs to start...")
    time.sleep(5)

    client_procs = (
        subprocess_on_vm('client', ' && '.join((
            'echo "userpassword" | kinit -f testuser',
            'cd /python-gssapi',
            'python2 /usr/local/bin/nosetests-2.7 --with-xunit --xunit-file py2tests.xml tests.integration.test_client:ClientIntegrationTest'
        ))),
        subprocess_on_vm('client', ' && '.join((
            'echo "userpassword" | kinit -f testuser',
            'cd /python-gssapi',
            'pypy /usr/local/bin/nosetests-2.7 --with-xunit --xunit-file pypy2tests.xml tests.integration.test_client:ClientIntegrationTest'
        ))),
        subprocess_on_vm('client', ' && '.join((
            'echo "userpassword" | kinit -f testuser',
            'cd /python-gssapi',
            'python3 /usr/local/bin/nosetests-3.4 --with-xunit --xunit-file py3tests.xml tests.integration.test_client:ClientIntegrationTest'
        ))),
    )

    print("wait for client_procs")
    for index, client_proc in enumerate(client_procs):
        client_proc.wait()
        if client_proc.returncode == 0:
            print("wait for server proc {0}".format(index))
            server_procs[index].wait()
        else:
            print("client_proc {0} exited with status {1}, terminating server".format(
                index, client_proc.returncode
            ))
            server_procs[index].terminate()

    # Clean up any old server processes
    subprocess_on_vm('server', 'sudo pkill -f tests/integration/server.py')
<commit_msg>Have a way to run tests without the whole VM setup stage<commit_after>#!/usr/bin/env python
import os
import shutil
import subprocess
import sys
import time

def subprocess_on_vm(vm, command):
    return subprocess.Popen(['vagrant', 'ssh', vm, '--', command])

if __name__ == '__main__':
    if '--fast' not in sys.argv:
        # Cleanup CFFI-related files
        shutil.rmtree(
            os.path.abspath(os.path.join(os.path.dirname(__file__), '../../gssapi/bindings/autogenerated.cdef')),
            ignore_errors=True
        )
        shutil.rmtree('client.keytab', ignore_errors=True)
        shutil.rmtree('server.keytab', ignore_errors=True)

        subprocess.check_call(('vagrant', 'up'))

        for vm in ('server', 'client'):
            subprocess_on_vm(vm, 'sudo python2 -m pip install -r /python-gssapi/test_requirements.txt').wait()
            subprocess_on_vm(vm, 'sudo pypy -m pip install -r /python-gssapi/test_requirements.txt').wait()
            subprocess_on_vm(vm, 'sudo python3 -m pip install -r /python-gssapi/test_requirements.txt').wait()
            subprocess_on_vm(vm, 'cd /python-gssapi && sudo python2 setup.py install').wait()
            subprocess_on_vm(vm, 'cd /python-gssapi && sudo pypy setup.py install').wait()
            subprocess_on_vm(vm, 'cd /python-gssapi && sudo python3 setup.py install').wait()

    server_procs = [
        subprocess_on_vm('server', 'cd /python-gssapi && sudo python2 tests/integration/server.py')
    ]
    time.sleep(1)
    server_procs.append(subprocess_on_vm('server', 'cd /python-gssapi && sudo pypy tests/integration/server.py'))
    time.sleep(1)
    server_procs.append(subprocess_on_vm('server', 'cd /python-gssapi && sudo python3 tests/integration/server.py'))

    print("Wait for server procs to start...")
    time.sleep(5)

    client_procs = (
        subprocess_on_vm('client', ' && '.join((
            'echo "userpassword" | kinit -f testuser',
            'cd /python-gssapi',
            'python2 /usr/local/bin/nosetests-2.7 --with-xunit --xunit-file py2tests.xml tests.integration.test_client:ClientIntegrationTest'
        ))),
        subprocess_on_vm('client', ' && '.join((
            'echo "userpassword" | kinit -f testuser',
            'cd /python-gssapi',
            'pypy /usr/local/bin/nosetests-2.7 --with-xunit --xunit-file pypy2tests.xml tests.integration.test_client:ClientIntegrationTest'
        ))),
        subprocess_on_vm('client', ' && '.join((
            'echo "userpassword" | kinit -f testuser',
            'cd /python-gssapi',
            'python3 /usr/local/bin/nosetests-3.4 --with-xunit --xunit-file py3tests.xml tests.integration.test_client:ClientIntegrationTest'
        ))),
    )

    print("wait for client_procs")
    for index, client_proc in enumerate(client_procs):
        client_proc.wait()
        if client_proc.returncode == 0:
            print("wait for server proc {0}".format(index))
            server_procs[index].wait()
        else:
            print("client_proc {0} exited with status {1}, terminating server".format(
                index, client_proc.returncode
            ))
            server_procs[index].terminate()

    # Clean up any old server processes
    subprocess_on_vm('server', 'sudo pkill -f tests/integration/server.py')
<|endoftext|>


<commit_before>import pytest
import os

from pipenv.utils import temp_environ

from flaky import flaky

@pytest.mark.lock
@pytest.mark.requirements
def test_lock_handle_eggs(PipenvInstance, pypi):
    """Ensure locking works with packages provoding egg formats.
    """
    with PipenvInstance() as p:
        with open(p.pipfile_path, 'w') as f:
            f.write("""
[packages]
RandomWords = "*"
            """)
        c = p.pipenv('lock --verbose')
        assert c.return_code == 0
        assert 'randomwords' in p.lockfile['default']
        assert p.lockfile['default']['randomwords']['version'] == '==0.2.1'

@pytest.mark.lock
@pytest.mark.requirements
def test_lock_requirements_file(PipenvInstance, pypi):

    with PipenvInstance(pypi=pypi) as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[packages]
requests = "==2.14.0"
[dev-packages]
flask = "==0.12.2"
            """.strip()
            f.write(contents)

        req_list = ("requests==2.14.0")

        dev_req_list = ("flask==0.12.2")

        c = p.pipenv('lock -r')
        d = p.pipenv('lock -r -d')
        assert c.return_code == 0
        assert d.return_code == 0

        for req in req_list:
            assert req in c.out

        for req in dev_req_list:
            assert req in d.out

@pytest.mark.lock
def test_lock_keep_outdated(PipenvInstance, pypi):

    with PipenvInstance(pypi=pypi) as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[packages]
requests = {version = "==2.14.0"}
PyTest = "==3.1.0"
            """.strip()
            f.write(contents)

        c = p.pipenv('lock')
        assert c.return_code == 0
        lock = p.lockfile
        assert 'requests' in lock['default']
        assert lock['default']['requests']['version'] == "==2.14.0"
        assert 'pytest' in lock['default']
        assert lock['default']['pytest']['version'] == "==3.1.0"

        with open(p.pipfile_path, 'w') as f:
            updated_contents = """
[packages]
requests = {version = "==2.18.4"}
PyTest = "*"
            """.strip()
            f.write(updated_contents)

        c = p.pipenv('lock --keep-outdated')
        assert c.return_code == 0
        lock = p.lockfile
        assert 'requests' in lock['default']
        assert lock['default']['requests']['version'] == "==2.18.4"
        assert 'pytest' in lock['default']
        assert lock['default']['pytest']['version'] == "==3.1.0"

@pytest.mark.lock
@pytest.mark.complex
@pytest.mark.needs_internet
def test_complex_lock_with_vcs_deps(PipenvInstance, pip_src_dir):
    # This uses the real PyPI since we need Internet to access the Git
    # dependency anyway.
    with PipenvInstance() as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[packages]
click = "==6.7"

[dev-packages]
requests = {git = "https://github.com/requests/requests.git"}
            """.strip()
            f.write(contents)

        c = p.pipenv('install')
        assert c.return_code == 0
        lock = p.lockfile
        assert 'requests' in lock['develop']
        assert 'click' in lock['default']

        c = p.pipenv('run pip install -e git+https://github.com/dateutil/dateutil#egg=python_dateutil')
        assert c.return_code == 0

        c = p.pipenv('lock')
        assert c.return_code == 0
        lock = p.lockfile
        assert 'requests' in lock['develop']
        assert 'click' in lock['default']
        assert 'python_dateutil' not in lock['default']
        assert 'python_dateutil' not in lock['develop']

@pytest.mark.lock
@pytest.mark.requirements
def test_lock_with_prereleases(PipenvInstance, pypi):

    with PipenvInstance(pypi=pypi) as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[packages]
sqlalchemy = "==1.2.0b3"

[pipenv]
allow_prereleases = true
            """.strip()
            f.write(contents)

        c = p.pipenv('lock')
        assert c.return_code == 0
        assert p.lockfile['default']['sqlalchemy']['version'] == '==1.2.0b3'

@pytest.mark.lock
@pytest.mark.complex
@pytest.mark.maya
@pytest.mark.needs_internet
@flaky
def test_complex_deps_lock_and_install_properly(PipenvInstance, pip_src_dir, pypi):
    # This uses the real PyPI because Maya has too many dependencies...
    with PipenvInstance(chdir=True, pypi=pypi) as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[packages]
maya = "*"
            """.strip()
            f.write(contents)

        c = p.pipenv('lock --verbose')
        assert c.return_code == 0

        c = p.pipenv('install')
        assert c.return_code == 0

@pytest.mark.extras
@pytest.mark.lock
@pytest.mark.complex
@pytest.mark.skip(reason='Needs numpy to be mocked')
@pytest.mark.needs_internet
def test_complex_lock_deep_extras(PipenvInstance, pypi):
    # records[pandas] requires tablib[pandas] which requires pandas.
    # This uses the real PyPI; Pandas has too many requirements to mock.

    with PipenvInstance(pypi=pypi) as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[packages]
records = {extras = ["pandas"], version = "==0.5.2"}
            """.strip()
            f.write(contents)

        c = p.pipenv('install')
        assert c.return_code == 0
        c = p.pipenv('lock')
        assert c.return_code == 0
        assert 'tablib' in p.lockfile['default']
        assert 'pandas' in p.lockfile['default']

@pytest.mark.skip_lock
@pytest.mark.index
@pytest.mark.needs_internet
@pytest.mark.install  # private indexes need to be uncached for resolution
def test_private_index_skip_lock(PipenvInstance):
    with PipenvInstance() as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"

[[source]]
url = "https://test.pypi.org/simple"
verify_ssl = true
name = "testpypi"

[packages]
pipenv-test-private-package = {version = "*", index = "testpypi"}
requests = "*"
            """.strip()
            f.write(contents)
        c = p.pipenv('install --skip-lock')
        assert c.return_code == 0

@pytest.mark.requirements
@pytest.mark.lock
@pytest.mark.index
@pytest.mark.install  # private indexes need to be uncached for resolution
@pytest.mark.needs_internet
def test_private_index_lock_requirements(PipenvInstance):
    # Don't use the local fake pypi
    with PipenvInstance() as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"

[[source]]
url = "https://test.pypi.org/simple"
verify_ssl = true
name = "testpypi"

[packages]
pipenv-test-private-package = {version = "*", index = "testpypi"}
requests = "*"
            """.strip()
            f.write(contents)
        c = p.pipenv('install')
        assert c.return_code == 0
        c = p.pipenv('lock -r')
        assert c.return_code == 0
        assert '-i https://pypi.org/simple' in c.out.strip()
        assert '--extra-index-url https://test.pypi.org/simple' in c.out.strip()

@pytest.mark.requirements
@pytest.mark.lock
@pytest.mark.index
@pytest.mark.install  # private indexes need to be uncached for resolution
@pytest.mark.needs_internet
def test_private_index_mirror_lock_requirements(PipenvInstance):
    # Don't use the local fake pypi
    with temp_environ(), PipenvInstance(chdir=True) as p:
        # Using pypi.python.org as pipenv-test-public-package is not
        # included in the local pypi mirror
        mirror_url = os.environ.pop('PIPENV_TEST_INDEX', "https://pypi.kennethreitz.org/simple")
        # os.environ.pop('PIPENV_TEST_INDEX', None)
        with open(p.pipfile_path, 'w') as f:
            contents = """
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"

[[source]]
url = "https://test.pypi.org/simple"
verify_ssl = true
name = "testpypi"

[packages]
six = {version = "*", index = "testpypi"}
requests = "*"
            """.strip()
            f.write(contents)
        c = p.pipenv('install --pypi-mirror {0}'.format(mirror_url))
        assert c.return_code == 0
        c = p.pipenv('lock -r --pypi-mirror {0}'.format(mirror_url))
        assert c.return_code == 0
        assert '-i https://pypi.org/simple' in c.out.strip()
        assert '--extra-index-url https://test.pypi.org/simple' in c.out.strip()
        # Mirror url should not have replaced source URLs
        assert '-i {0}'.format(mirror_url) not in c.out.strip()
        assert '--extra-index-url {}'.format(mirror_url) not in c.out.strip()

@pytest.mark.install
@pytest.mark.index
def test_lock_updated_source(PipenvInstance, pypi):

    with PipenvInstance(pypi=pypi) as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[[source]]
url = "{url}/${{MY_ENV_VAR}}"

[packages]
requests = "==2.14.0"
            """.strip().format(url=pypi.url)
            f.write(contents)

        with temp_environ():
            os.environ['MY_ENV_VAR'] = 'simple'
            c = p.pipenv('lock')
            assert c.return_code == 0
            assert 'requests' in p.lockfile['default']

        with open(p.pipfile_path, 'w') as f:
            contents = """
[[source]]
url = "{url}/simple"

[packages]
requests = "==2.14.0"
            """.strip().format(url=pypi.url)
            f.write(contents)

        c = p.pipenv('lock')
        assert c.return_code == 0
        assert 'requests' in p.lockfile['default']

@pytest.mark.lock
@pytest.mark.vcs
@pytest.mark.needs_internet
def test_lock_editable_vcs_without_install(PipenvInstance, pypi):
    with PipenvInstance(pypi=pypi, chdir=True) as p:
        with open(p.pipfile_path, 'w') as f:
            f.write("""
[packages]
requests = {git = "https://github.com/requests/requests.git", ref = "master", editable = true}
            """.strip())
        c = p.pipenv('lock')
        assert c.return_code == 0
        assert 'requests' in p.lockfile['default']
        assert 'idna' in p.lockfile['default']
        assert 'chardet' in p.lockfile['default']
        c = p.pipenv('install')
        assert c.return_code == 0

@pytest.mark.extras
@pytest.mark.lock
@pytest.mark.vcs
@pytest.mark.needs_internet
def test_lock_editable_vcs_with_extras_without_install(PipenvInstance, pypi):
    with PipenvInstance(pypi=pypi, chdir=True) as p:
        with open(p.pipfile_path, 'w') as f:
            f.write("""
[packages]
requests = {git = "https://github.com/requests/requests.git", editable = true, extras = ["socks"]}
            """.strip())
        c = p.pipenv('lock')
        assert c.return_code == 0
        assert 'requests' in p.lockfile['default']
        assert 'idna' in p.lockfile['default']
        assert 'chardet' in p.lockfile['default']
        assert "socks" in p.lockfile["default"]["requests"]["extras"]
        c = p.pipenv('install')
        assert c.return_code == 0

@pytest.mark.lock
@pytest.mark.skip(reason="This doesn't work for some reason.")
def test_lock_respecting_python_version(PipenvInstance, pypi):
    with PipenvInstance(pypi=pypi, chdir=True) as p:
        with open(p.pipfile_path, 'w') as f:
            f.write("""
[packages]
django = "*"
            """.strip())
        c = p.pipenv('install ')
        assert c.return_code == 0
        c = p.pipenv('run python --version')
        assert c.return_code == 0
        py_version = c.err.splitlines()[-1].strip().split()[-1]
        django_version = '==2.0.6' if py_version.startswith('3') else '==1.11.13'
        assert py_version == '2.7.14'
        assert p.lockfile['default']['django']['version'] == django_version

@pytest.mark.lock
@pytest.mark.install
def test_lockfile_corrupted(PipenvInstance):
    with PipenvInstance() as p:
        with open(p.lockfile_path, 'w') as f:
            f.write('{corrupted}')
        c = p.pipenv('install')
        assert c.return_code == 0
        assert 'Pipfile.lock is corrupted' in c.err
        assert p.lockfile['_meta']

@pytest.mark.lock
@pytest.mark.install
def test_lockfile_with_empty_dict(PipenvInstance):
    with PipenvInstance() as p:
        with open(p.lockfile_path, 'w') as f:
            f.write('{}')
        c = p.pipenv('install')
        assert c.return_code == 0
        assert 'Pipfile.lock is corrupted' in c.err
        assert p.lockfile['_meta']
<commit_msg>Add test on locking editable package with markers<commit_after>import pytest
import os

from pipenv.utils import temp_environ

from flaky import flaky

@pytest.mark.lock
@pytest.mark.requirements
def test_lock_handle_eggs(PipenvInstance, pypi):
    """Ensure locking works with packages provoding egg formats.
    """
    with PipenvInstance() as p:
        with open(p.pipfile_path, 'w') as f:
            f.write("""
[packages]
RandomWords = "*"
            """)
        c = p.pipenv('lock --verbose')
        assert c.return_code == 0
        assert 'randomwords' in p.lockfile['default']
        assert p.lockfile['default']['randomwords']['version'] == '==0.2.1'

@pytest.mark.lock
@pytest.mark.requirements
def test_lock_requirements_file(PipenvInstance, pypi):

    with PipenvInstance(pypi=pypi) as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[packages]
requests = "==2.14.0"
[dev-packages]
flask = "==0.12.2"
            """.strip()
            f.write(contents)

        req_list = ("requests==2.14.0")

        dev_req_list = ("flask==0.12.2")

        c = p.pipenv('lock -r')
        d = p.pipenv('lock -r -d')
        assert c.return_code == 0
        assert d.return_code == 0

        for req in req_list:
            assert req in c.out

        for req in dev_req_list:
            assert req in d.out

@pytest.mark.lock
def test_lock_keep_outdated(PipenvInstance, pypi):

    with PipenvInstance(pypi=pypi) as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[packages]
requests = {version = "==2.14.0"}
PyTest = "==3.1.0"
            """.strip()
            f.write(contents)

        c = p.pipenv('lock')
        assert c.return_code == 0
        lock = p.lockfile
        assert 'requests' in lock['default']
        assert lock['default']['requests']['version'] == "==2.14.0"
        assert 'pytest' in lock['default']
        assert lock['default']['pytest']['version'] == "==3.1.0"

        with open(p.pipfile_path, 'w') as f:
            updated_contents = """
[packages]
requests = {version = "==2.18.4"}
PyTest = "*"
            """.strip()
            f.write(updated_contents)

        c = p.pipenv('lock --keep-outdated')
        assert c.return_code == 0
        lock = p.lockfile
        assert 'requests' in lock['default']
        assert lock['default']['requests']['version'] == "==2.18.4"
        assert 'pytest' in lock['default']
        assert lock['default']['pytest']['version'] == "==3.1.0"

@pytest.mark.lock
@pytest.mark.complex
@pytest.mark.needs_internet
def test_complex_lock_with_vcs_deps(PipenvInstance, pip_src_dir):
    # This uses the real PyPI since we need Internet to access the Git
    # dependency anyway.
    with PipenvInstance() as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[packages]
click = "==6.7"

[dev-packages]
requests = {git = "https://github.com/requests/requests.git"}
            """.strip()
            f.write(contents)

        c = p.pipenv('install')
        assert c.return_code == 0
        lock = p.lockfile
        assert 'requests' in lock['develop']
        assert 'click' in lock['default']

        c = p.pipenv('run pip install -e git+https://github.com/dateutil/dateutil#egg=python_dateutil')
        assert c.return_code == 0

        c = p.pipenv('lock')
        assert c.return_code == 0
        lock = p.lockfile
        assert 'requests' in lock['develop']
        assert 'click' in lock['default']
        assert 'python_dateutil' not in lock['default']
        assert 'python_dateutil' not in lock['develop']

@pytest.mark.lock
@pytest.mark.requirements
def test_lock_with_prereleases(PipenvInstance, pypi):

    with PipenvInstance(pypi=pypi) as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[packages]
sqlalchemy = "==1.2.0b3"

[pipenv]
allow_prereleases = true
            """.strip()
            f.write(contents)

        c = p.pipenv('lock')
        assert c.return_code == 0
        assert p.lockfile['default']['sqlalchemy']['version'] == '==1.2.0b3'

@pytest.mark.lock
@pytest.mark.complex
@pytest.mark.maya
@pytest.mark.needs_internet
@flaky
def test_complex_deps_lock_and_install_properly(PipenvInstance, pip_src_dir, pypi):
    # This uses the real PyPI because Maya has too many dependencies...
    with PipenvInstance(chdir=True, pypi=pypi) as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[packages]
maya = "*"
            """.strip()
            f.write(contents)

        c = p.pipenv('lock --verbose')
        assert c.return_code == 0

        c = p.pipenv('install')
        assert c.return_code == 0

@pytest.mark.extras
@pytest.mark.lock
@pytest.mark.complex
@pytest.mark.skip(reason='Needs numpy to be mocked')
@pytest.mark.needs_internet
def test_complex_lock_deep_extras(PipenvInstance, pypi):
    # records[pandas] requires tablib[pandas] which requires pandas.
    # This uses the real PyPI; Pandas has too many requirements to mock.

    with PipenvInstance(pypi=pypi) as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[packages]
records = {extras = ["pandas"], version = "==0.5.2"}
            """.strip()
            f.write(contents)

        c = p.pipenv('install')
        assert c.return_code == 0
        c = p.pipenv('lock')
        assert c.return_code == 0
        assert 'tablib' in p.lockfile['default']
        assert 'pandas' in p.lockfile['default']

@pytest.mark.skip_lock
@pytest.mark.index
@pytest.mark.needs_internet
@pytest.mark.install  # private indexes need to be uncached for resolution
def test_private_index_skip_lock(PipenvInstance):
    with PipenvInstance() as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"

[[source]]
url = "https://test.pypi.org/simple"
verify_ssl = true
name = "testpypi"

[packages]
pipenv-test-private-package = {version = "*", index = "testpypi"}
requests = "*"
            """.strip()
            f.write(contents)
        c = p.pipenv('install --skip-lock')
        assert c.return_code == 0

@pytest.mark.requirements
@pytest.mark.lock
@pytest.mark.index
@pytest.mark.install  # private indexes need to be uncached for resolution
@pytest.mark.needs_internet
def test_private_index_lock_requirements(PipenvInstance):
    # Don't use the local fake pypi
    with PipenvInstance() as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"

[[source]]
url = "https://test.pypi.org/simple"
verify_ssl = true
name = "testpypi"

[packages]
pipenv-test-private-package = {version = "*", index = "testpypi"}
requests = "*"
            """.strip()
            f.write(contents)
        c = p.pipenv('install')
        assert c.return_code == 0
        c = p.pipenv('lock -r')
        assert c.return_code == 0
        assert '-i https://pypi.org/simple' in c.out.strip()
        assert '--extra-index-url https://test.pypi.org/simple' in c.out.strip()

@pytest.mark.requirements
@pytest.mark.lock
@pytest.mark.index
@pytest.mark.install  # private indexes need to be uncached for resolution
@pytest.mark.needs_internet
def test_private_index_mirror_lock_requirements(PipenvInstance):
    # Don't use the local fake pypi
    with temp_environ(), PipenvInstance(chdir=True) as p:
        # Using pypi.python.org as pipenv-test-public-package is not
        # included in the local pypi mirror
        mirror_url = os.environ.pop('PIPENV_TEST_INDEX', "https://pypi.kennethreitz.org/simple")
        # os.environ.pop('PIPENV_TEST_INDEX', None)
        with open(p.pipfile_path, 'w') as f:
            contents = """
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"

[[source]]
url = "https://test.pypi.org/simple"
verify_ssl = true
name = "testpypi"

[packages]
six = {version = "*", index = "testpypi"}
requests = "*"
            """.strip()
            f.write(contents)
        c = p.pipenv('install --pypi-mirror {0}'.format(mirror_url))
        assert c.return_code == 0
        c = p.pipenv('lock -r --pypi-mirror {0}'.format(mirror_url))
        assert c.return_code == 0
        assert '-i https://pypi.org/simple' in c.out.strip()
        assert '--extra-index-url https://test.pypi.org/simple' in c.out.strip()
        # Mirror url should not have replaced source URLs
        assert '-i {0}'.format(mirror_url) not in c.out.strip()
        assert '--extra-index-url {}'.format(mirror_url) not in c.out.strip()

@pytest.mark.install
@pytest.mark.index
def test_lock_updated_source(PipenvInstance, pypi):

    with PipenvInstance(pypi=pypi) as p:
        with open(p.pipfile_path, 'w') as f:
            contents = """
[[source]]
url = "{url}/${{MY_ENV_VAR}}"

[packages]
requests = "==2.14.0"
            """.strip().format(url=pypi.url)
            f.write(contents)

        with temp_environ():
            os.environ['MY_ENV_VAR'] = 'simple'
            c = p.pipenv('lock')
            assert c.return_code == 0
            assert 'requests' in p.lockfile['default']

        with open(p.pipfile_path, 'w') as f:
            contents = """
[[source]]
url = "{url}/simple"

[packages]
requests = "==2.14.0"
            """.strip().format(url=pypi.url)
            f.write(contents)

        c = p.pipenv('lock')
        assert c.return_code == 0
        assert 'requests' in p.lockfile['default']

@pytest.mark.lock
@pytest.mark.vcs
@pytest.mark.needs_internet
def test_lock_editable_vcs_without_install(PipenvInstance, pypi):
    with PipenvInstance(pypi=pypi, chdir=True) as p:
        with open(p.pipfile_path, 'w') as f:
            f.write("""
[packages]
requests = {git = "https://github.com/requests/requests.git", ref = "master", editable = true}
            """.strip())
        c = p.pipenv('lock')
        assert c.return_code == 0
        assert 'requests' in p.lockfile['default']
        assert 'idna' in p.lockfile['default']
        assert 'chardet' in p.lockfile['default']
        c = p.pipenv('install')
        assert c.return_code == 0

@pytest.mark.extras
@pytest.mark.lock
@pytest.mark.vcs
@pytest.mark.needs_internet
def test_lock_editable_vcs_with_extras_without_install(PipenvInstance, pypi):
    with PipenvInstance(pypi=pypi, chdir=True) as p:
        with open(p.pipfile_path, 'w') as f:
            f.write("""
[packages]
requests = {git = "https://github.com/requests/requests.git", editable = true, extras = ["socks"]}
            """.strip())
        c = p.pipenv('lock')
        assert c.return_code == 0
        assert 'requests' in p.lockfile['default']
        assert 'idna' in p.lockfile['default']
        assert 'chardet' in p.lockfile['default']
        assert "socks" in p.lockfile["default"]["requests"]["extras"]
        c = p.pipenv('install')
        assert c.return_code == 0

@pytest.mark.lock
@pytest.mark.vcs
@pytest.mark.needs_internet
def test_lock_editable_vcs_with_markers_without_install(PipenvInstance, pypi):
    with PipenvInstance(pypi=pypi, chdir=True) as p:
        with open(p.pipfile_path, 'w') as f:
            f.write("""
[packages]
requests = {git = "https://github.com/requests/requests.git", ref = "master", editable = true, markers = "python_version >= '2.6'"}
            """.strip())
        c = p.pipenv('lock')
        assert c.return_code == 0
        assert 'requests' in p.lockfile['default']
        assert 'idna' in p.lockfile['default']
        assert 'chardet' in p.lockfile['default']
        c = p.pipenv('install')
        assert c.return_code == 0

@pytest.mark.lock
@pytest.mark.skip(reason="This doesn't work for some reason.")
def test_lock_respecting_python_version(PipenvInstance, pypi):
    with PipenvInstance(pypi=pypi, chdir=True) as p:
        with open(p.pipfile_path, 'w') as f:
            f.write("""
[packages]
django = "*"
            """.strip())
        c = p.pipenv('install ')
        assert c.return_code == 0
        c = p.pipenv('run python --version')
        assert c.return_code == 0
        py_version = c.err.splitlines()[-1].strip().split()[-1]
        django_version = '==2.0.6' if py_version.startswith('3') else '==1.11.13'
        assert py_version == '2.7.14'
        assert p.lockfile['default']['django']['version'] == django_version

@pytest.mark.lock
@pytest.mark.install
def test_lockfile_corrupted(PipenvInstance):
    with PipenvInstance() as p:
        with open(p.lockfile_path, 'w') as f:
            f.write('{corrupted}')
        c = p.pipenv('install')
        assert c.return_code == 0
        assert 'Pipfile.lock is corrupted' in c.err
        assert p.lockfile['_meta']

@pytest.mark.lock
@pytest.mark.install
def test_lockfile_with_empty_dict(PipenvInstance):
    with PipenvInstance() as p:
        with open(p.lockfile_path, 'w') as f:
            f.write('{}')
        c = p.pipenv('install')
        assert c.return_code == 0
        assert 'Pipfile.lock is corrupted' in c.err
        assert p.lockfile['_meta']
<|endoftext|>


<commit_before># encoding: utf-8
"""
Paging capabilities for IPython.core

Notes
-----

For now this uses IPython hooks, so it can't be in IPython.utils.  If we can get
rid of that dependency, we could move it there.
-----
"""

# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.

from __future__ import print_function

import os
import re
import sys
import tempfile

from io import UnsupportedOperation

from IPython import get_ipython
from IPython.core.display import display
from IPython.core.error import TryNext
from IPython.utils.data import chop
from IPython.utils.process import system
from IPython.utils.terminal import get_terminal_size
from IPython.utils import py3compat

def display_page(strng, start=0, screen_lines=25):
    """Just display, no paging. screen_lines is ignored."""
    if isinstance(strng, dict):
        data = strng
    else:
        if start:
            strng = u'\n'.join(strng.splitlines()[start:])
        data = {'text/plain': strng}
    display(data, raw=True)

def as_hook(page_func):
    """Wrap a pager func to strip the `self` arg

    so it can be called as a hook.
    """
    return lambda self, *args, **kwargs: page_func(*args, **kwargs)

esc_re = re.compile(r"(\x1b[^m]+m)")

def page_dumb(strng, start=0, screen_lines=25):
    """Very dumb 'pager' in Python, for when nothing else works.

    Only moves forward, same interface as page(), except for pager_cmd and
    mode."""

    out_ln  = strng.splitlines()[start:]
    screens = chop(out_ln,screen_lines-1)
    if len(screens) == 1:
        print(os.linesep.join(screens[0]))
    else:
        last_escape = ""
        for scr in screens[0:-1]:
            hunk = os.linesep.join(scr)
            print(last_escape + hunk)
            if not page_more():
                return
            esc_list = esc_re.findall(hunk)
            if len(esc_list) > 0:
                last_escape = esc_list[-1]
        print(last_escape + os.linesep.join(screens[-1]))

def _detect_screen_size(screen_lines_def):
    """Attempt to work out the number of lines on the screen.

    This is called by page(). It can raise an error (e.g. when run in the
    test suite), so it's separated out so it can easily be called in a try block.
    """
    TERM = os.environ.get('TERM',None)
    if not((TERM=='xterm' or TERM=='xterm-color') and sys.platform != 'sunos5'):
        # curses causes problems on many terminals other than xterm, and
        # some termios calls lock up on Sun OS5.
        return screen_lines_def

    try:
        import termios
        import curses
    except ImportError:
        return screen_lines_def

    # There is a bug in curses, where *sometimes* it fails to properly
    # initialize, and then after the endwin() call is made, the
    # terminal is left in an unusable state.  Rather than trying to
    # check everytime for this (by requesting and comparing termios
    # flags each time), we just save the initial terminal state and
    # unconditionally reset it every time.  It's cheaper than making
    # the checks.
    try:
        term_flags = termios.tcgetattr(sys.stdout)
    except termios.error as err:
        # can fail on Linux 2.6, pager_page will catch the TypeError
        raise TypeError('termios error: {0}'.format(err))

    # Curses modifies the stdout buffer size by default, which messes
    # up Python's normal stdout buffering.  This would manifest itself
    # to IPython users as delayed printing on stdout after having used
    # the pager.
    #
    # We can prevent this by manually setting the NCURSES_NO_SETBUF
    # environment variable.  For more details, see:
    # http://bugs.python.org/issue10144
    NCURSES_NO_SETBUF = os.environ.get('NCURSES_NO_SETBUF', None)
    os.environ['NCURSES_NO_SETBUF'] = ''

    # Proceed with curses initialization
    try:
        scr = curses.initscr()
    except AttributeError:
        # Curses on Solaris may not be complete, so we can't use it there
        return screen_lines_def

    screen_lines_real,screen_cols = scr.getmaxyx()
    curses.endwin()

    # Restore environment
    if NCURSES_NO_SETBUF is None:
        del os.environ['NCURSES_NO_SETBUF']
    else:
        os.environ['NCURSES_NO_SETBUF'] = NCURSES_NO_SETBUF

    # Restore terminal state in case endwin() didn't.
    termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)
    # Now we have what we needed: the screen size in rows/columns
    return screen_lines_real
    #print '***Screen size:',screen_lines_real,'lines x',\
    #screen_cols,'columns.' # dbg

def pager_page(strng, start=0, screen_lines=0, pager_cmd=None):
    """Display a string, piping through a pager after a certain length.

    strng can be a mime-bundle dict, supplying multiple representations,
    keyed by mime-type.

    The screen_lines parameter specifies the number of *usable* lines of your
    terminal screen (total lines minus lines you need to reserve to show other
    information).

    If you set screen_lines to a number <=0, page() will try to auto-determine
    your screen size and will only use up to (screen_size+screen_lines) for
    printing, paging after that. That is, if you want auto-detection but need
    to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for
    auto-detection without any lines reserved simply use screen_lines = 0.

    If a string won't fit in the allowed lines, it is sent through the
    specified pager command. If none given, look for PAGER in the environment,
    and ultimately default to less.

    If no system pager works, the string is sent through a 'dumb pager'
    written in python, very simplistic.
    """

    # for compatibility with mime-bundle form:
    if isinstance(strng, dict):
        strng = strng['text/plain']

    # Ugly kludge, but calling curses.initscr() flat out crashes in emacs
    TERM = os.environ.get('TERM','dumb')
    if TERM in ['dumb','emacs'] and os.name != 'nt':
        print(strng)
        return
    # chop off the topmost part of the string we don't want to see
    str_lines = strng.splitlines()[start:]
    str_toprint = os.linesep.join(str_lines)
    num_newlines = len(str_lines)
    len_str = len(str_toprint)

    # Dumb heuristics to guesstimate number of on-screen lines the string
    # takes.  Very basic, but good enough for docstrings in reasonable
    # terminals. If someone later feels like refining it, it's not hard.
    numlines = max(num_newlines,int(len_str/80)+1)

    screen_lines_def = get_terminal_size()[1]

    # auto-determine screen size
    if screen_lines <= 0:
        try:
            screen_lines += _detect_screen_size(screen_lines_def)
        except (TypeError, UnsupportedOperation):
            print(str_toprint)
            return

    #print 'numlines',numlines,'screenlines',screen_lines  # dbg
    if numlines <= screen_lines :
        #print '*** normal print'  # dbg
        print(str_toprint)
    else:
        # Try to open pager and default to internal one if that fails.
        # All failure modes are tagged as 'retval=1', to match the return
        # value of a failed system command.  If any intermediate attempt
        # sets retval to 1, at the end we resort to our own page_dumb() pager.
        pager_cmd = get_pager_cmd(pager_cmd)
        pager_cmd += ' ' + get_pager_start(pager_cmd,start)
        if os.name == 'nt':
            if pager_cmd.startswith('type'):
                # The default WinXP 'type' command is failing on complex strings.
                retval = 1
            else:
                fd, tmpname = tempfile.mkstemp('.txt')
                try:
                    os.close(fd)
                    with open(tmpname, 'wt') as tmpfile:
                        tmpfile.write(strng)
                        cmd = "%s < %s" % (pager_cmd, tmpname)
                    # tmpfile needs to be closed for windows
                    if os.system(cmd):
                        retval = 1
                    else:
                        retval = None
                finally:
                    os.remove(tmpname)
        else:
            try:
                retval = None
                # if I use popen4, things hang. No idea why.
                #pager,shell_out = os.popen4(pager_cmd)
                pager = os.popen(pager_cmd, 'w')
                try:
                    pager_encoding = pager.encoding or sys.stdout.encoding
                    pager.write(py3compat.cast_bytes_py2(
                        strng, encoding=pager_encoding))
                finally:
                    retval = pager.close()
            except IOError as msg:  # broken pipe when user quits
                if msg.args == (32, 'Broken pipe'):
                    retval = None
                else:
                    retval = 1
            except OSError:
                # Other strange problems, sometimes seen in Win2k/cygwin
                retval = 1
        if retval is not None:
            page_dumb(strng,screen_lines=screen_lines)

def page(data, start=0, screen_lines=0, pager_cmd=None):
    """Display content in a pager, piping through a pager after a certain length.

    data can be a mime-bundle dict, supplying multiple representations,
    keyed by mime-type, or text.

    Pager is dispatched via the `show_in_pager` IPython hook.
    If no hook is registered, `pager_page` will be used.
    """
    # Some routines may auto-compute start offsets incorrectly and pass a
    # negative value.  Offset to 0 for robustness.
    start = max(0, start)

    # first, try the hook
    ip = get_ipython()
    if ip:
        try:
            ip.hooks.show_in_pager(data, start=start, screen_lines=screen_lines)
            return
        except TryNext:
            pass

    # fallback on default pager
    return pager_page(data, start, screen_lines, pager_cmd)

def page_file(fname, start=0, pager_cmd=None):
    """Page a file, using an optional pager command and starting line.
    """

    pager_cmd = get_pager_cmd(pager_cmd)
    pager_cmd += ' ' + get_pager_start(pager_cmd,start)

    try:
        if os.environ['TERM'] in ['emacs','dumb']:
            raise EnvironmentError
        system(pager_cmd + ' ' + fname)
    except:
        try:
            if start > 0:
                start -= 1
            page(open(fname).read(),start)
        except:
            print('Unable to show file',repr(fname))

def get_pager_cmd(pager_cmd=None):
    """Return a pager command.

    Makes some attempts at finding an OS-correct one.
    """
    if os.name == 'posix':
        default_pager_cmd = 'less -R'  # -R for color control sequences
    elif os.name in ['nt','dos']:
        default_pager_cmd = 'type'

    if pager_cmd is None:
        try:
            pager_cmd = os.environ['PAGER']
        except:
            pager_cmd = default_pager_cmd

    if pager_cmd == 'less' and '-r' not in os.environ.get('LESS', '').lower():
        pager_cmd += ' -R'

    return pager_cmd

def get_pager_start(pager, start):
    """Return the string for paging files with an offset.

    This is the '+N' argument which less and more (under Unix) accept.
    """

    if pager in ['less','more']:
        if start:
            start_string = '+' + str(start)
        else:
            start_string = ''
    else:
        start_string = ''
    return start_string

# (X)emacs on win32 doesn't like to be bypassed with msvcrt.getch()
if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs':
    import msvcrt
    def page_more():
        """ Smart pausing between pages

        @return:    True if need print more lines, False if quit
        """
        sys.stdout.write('---Return to continue, q to quit--- ')
        ans = msvcrt.getwch()
        if ans in ("q", "Q"):
            result = False
        else:
            result = True
        sys.stdout.write("\b"*37 + " "*37 + "\b"*37)
        return result
else:
    def page_more():
        ans = py3compat.input('---Return to continue, q to quit--- ')
        if ans.lower().startswith('q'):
            return False
        else:
            return True

def snip_print(str,width = 75,print_full = 0,header = ''):
    """Print a string snipping the midsection to fit in width.

    print_full: mode control:

      - 0: only snip long strings
      - 1: send to page() directly.
      - 2: snip long strings and ask for full length viewing with page()

    Return 1 if snipping was necessary, 0 otherwise."""

    if print_full == 1:
        page(header+str)
        return 0

    print(header, end=' ')
    if len(str) < width:
        print(str)
        snip = 0
    else:
        whalf = int((width -5)/2)
        print(str[:whalf] + ' <...> ' + str[-whalf:])
        snip = 1
    if snip and print_full == 2:
        if py3compat.input(header+' Snipped. View (y/n)? [N]').lower() == 'y':
            page(str)
    return snip
<commit_msg>page_dumb mime bundle<commit_after># encoding: utf-8
"""
Paging capabilities for IPython.core

Notes
-----

For now this uses IPython hooks, so it can't be in IPython.utils.  If we can get
rid of that dependency, we could move it there.
-----
"""

# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.

from __future__ import print_function

import os
import re
import sys
import tempfile

from io import UnsupportedOperation

from IPython import get_ipython
from IPython.core.display import display
from IPython.core.error import TryNext
from IPython.utils.data import chop
from IPython.utils.process import system
from IPython.utils.terminal import get_terminal_size
from IPython.utils import py3compat

def display_page(strng, start=0, screen_lines=25):
    """Just display, no paging. screen_lines is ignored."""
    if isinstance(strng, dict):
        data = strng
    else:
        if start:
            strng = u'\n'.join(strng.splitlines()[start:])
        data = { 'text/plain': strng }
    display(data, raw=True)

def as_hook(page_func):
    """Wrap a pager func to strip the `self` arg

    so it can be called as a hook.
    """
    return lambda self, *args, **kwargs: page_func(*args, **kwargs)

esc_re = re.compile(r"(\x1b[^m]+m)")

def page_dumb(strng, start=0, screen_lines=25):
    """Very dumb 'pager' in Python, for when nothing else works.

    Only moves forward, same interface as page(), except for pager_cmd and
    mode.
    """
    if isinstance(strng, dict):
        strng = strng.get('text/plain', '')
    out_ln  = strng.splitlines()[start:]
    screens = chop(out_ln,screen_lines-1)
    if len(screens) == 1:
        print(os.linesep.join(screens[0]))
    else:
        last_escape = ""
        for scr in screens[0:-1]:
            hunk = os.linesep.join(scr)
            print(last_escape + hunk)
            if not page_more():
                return
            esc_list = esc_re.findall(hunk)
            if len(esc_list) > 0:
                last_escape = esc_list[-1]
        print(last_escape + os.linesep.join(screens[-1]))

def _detect_screen_size(screen_lines_def):
    """Attempt to work out the number of lines on the screen.

    This is called by page(). It can raise an error (e.g. when run in the
    test suite), so it's separated out so it can easily be called in a try block.
    """
    TERM = os.environ.get('TERM',None)
    if not((TERM=='xterm' or TERM=='xterm-color') and sys.platform != 'sunos5'):
        # curses causes problems on many terminals other than xterm, and
        # some termios calls lock up on Sun OS5.
        return screen_lines_def

    try:
        import termios
        import curses
    except ImportError:
        return screen_lines_def

    # There is a bug in curses, where *sometimes* it fails to properly
    # initialize, and then after the endwin() call is made, the
    # terminal is left in an unusable state.  Rather than trying to
    # check everytime for this (by requesting and comparing termios
    # flags each time), we just save the initial terminal state and
    # unconditionally reset it every time.  It's cheaper than making
    # the checks.
    try:
        term_flags = termios.tcgetattr(sys.stdout)
    except termios.error as err:
        # can fail on Linux 2.6, pager_page will catch the TypeError
        raise TypeError('termios error: {0}'.format(err))

    # Curses modifies the stdout buffer size by default, which messes
    # up Python's normal stdout buffering.  This would manifest itself
    # to IPython users as delayed printing on stdout after having used
    # the pager.
    #
    # We can prevent this by manually setting the NCURSES_NO_SETBUF
    # environment variable.  For more details, see:
    # http://bugs.python.org/issue10144
    NCURSES_NO_SETBUF = os.environ.get('NCURSES_NO_SETBUF', None)
    os.environ['NCURSES_NO_SETBUF'] = ''

    # Proceed with curses initialization
    try:
        scr = curses.initscr()
    except AttributeError:
        # Curses on Solaris may not be complete, so we can't use it there
        return screen_lines_def

    screen_lines_real,screen_cols = scr.getmaxyx()
    curses.endwin()

    # Restore environment
    if NCURSES_NO_SETBUF is None:
        del os.environ['NCURSES_NO_SETBUF']
    else:
        os.environ['NCURSES_NO_SETBUF'] = NCURSES_NO_SETBUF

    # Restore terminal state in case endwin() didn't.
    termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)
    # Now we have what we needed: the screen size in rows/columns
    return screen_lines_real
    #print '***Screen size:',screen_lines_real,'lines x',\
    #screen_cols,'columns.' # dbg

def pager_page(strng, start=0, screen_lines=0, pager_cmd=None):
    """Display a string, piping through a pager after a certain length.

    strng can be a mime-bundle dict, supplying multiple representations,
    keyed by mime-type.

    The screen_lines parameter specifies the number of *usable* lines of your
    terminal screen (total lines minus lines you need to reserve to show other
    information).

    If you set screen_lines to a number <=0, page() will try to auto-determine
    your screen size and will only use up to (screen_size+screen_lines) for
    printing, paging after that. That is, if you want auto-detection but need
    to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for
    auto-detection without any lines reserved simply use screen_lines = 0.

    If a string won't fit in the allowed lines, it is sent through the
    specified pager command. If none given, look for PAGER in the environment,
    and ultimately default to less.

    If no system pager works, the string is sent through a 'dumb pager'
    written in python, very simplistic.
    """

    # for compatibility with mime-bundle form:
    if isinstance(strng, dict):
        strng = strng['text/plain']

    # Ugly kludge, but calling curses.initscr() flat out crashes in emacs
    TERM = os.environ.get('TERM','dumb')
    if TERM in ['dumb','emacs'] and os.name != 'nt':
        print(strng)
        return
    # chop off the topmost part of the string we don't want to see
    str_lines = strng.splitlines()[start:]
    str_toprint = os.linesep.join(str_lines)
    num_newlines = len(str_lines)
    len_str = len(str_toprint)

    # Dumb heuristics to guesstimate number of on-screen lines the string
    # takes.  Very basic, but good enough for docstrings in reasonable
    # terminals. If someone later feels like refining it, it's not hard.
    numlines = max(num_newlines,int(len_str/80)+1)

    screen_lines_def = get_terminal_size()[1]

    # auto-determine screen size
    if screen_lines <= 0:
        try:
            screen_lines += _detect_screen_size(screen_lines_def)
        except (TypeError, UnsupportedOperation):
            print(str_toprint)
            return

    #print 'numlines',numlines,'screenlines',screen_lines  # dbg
    if numlines <= screen_lines :
        #print '*** normal print'  # dbg
        print(str_toprint)
    else:
        # Try to open pager and default to internal one if that fails.
        # All failure modes are tagged as 'retval=1', to match the return
        # value of a failed system command.  If any intermediate attempt
        # sets retval to 1, at the end we resort to our own page_dumb() pager.
        pager_cmd = get_pager_cmd(pager_cmd)
        pager_cmd += ' ' + get_pager_start(pager_cmd,start)
        if os.name == 'nt':
            if pager_cmd.startswith('type'):
                # The default WinXP 'type' command is failing on complex strings.
                retval = 1
            else:
                fd, tmpname = tempfile.mkstemp('.txt')
                try:
                    os.close(fd)
                    with open(tmpname, 'wt') as tmpfile:
                        tmpfile.write(strng)
                        cmd = "%s < %s" % (pager_cmd, tmpname)
                    # tmpfile needs to be closed for windows
                    if os.system(cmd):
                        retval = 1
                    else:
                        retval = None
                finally:
                    os.remove(tmpname)
        else:
            try:
                retval = None
                # if I use popen4, things hang. No idea why.
                #pager,shell_out = os.popen4(pager_cmd)
                pager = os.popen(pager_cmd, 'w')
                try:
                    pager_encoding = pager.encoding or sys.stdout.encoding
                    pager.write(py3compat.cast_bytes_py2(
                        strng, encoding=pager_encoding))
                finally:
                    retval = pager.close()
            except IOError as msg:  # broken pipe when user quits
                if msg.args == (32, 'Broken pipe'):
                    retval = None
                else:
                    retval = 1
            except OSError:
                # Other strange problems, sometimes seen in Win2k/cygwin
                retval = 1
        if retval is not None:
            page_dumb(strng,screen_lines=screen_lines)

def page(data, start=0, screen_lines=0, pager_cmd=None):
    """Display content in a pager, piping through a pager after a certain length.

    data can be a mime-bundle dict, supplying multiple representations,
    keyed by mime-type, or text.

    Pager is dispatched via the `show_in_pager` IPython hook.
    If no hook is registered, `pager_page` will be used.
    """
    # Some routines may auto-compute start offsets incorrectly and pass a
    # negative value.  Offset to 0 for robustness.
    start = max(0, start)

    # first, try the hook
    ip = get_ipython()
    if ip:
        try:
            ip.hooks.show_in_pager(data, start=start, screen_lines=screen_lines)
            return
        except TryNext:
            pass

    # fallback on default pager
    return pager_page(data, start, screen_lines, pager_cmd)

def page_file(fname, start=0, pager_cmd=None):
    """Page a file, using an optional pager command and starting line.
    """

    pager_cmd = get_pager_cmd(pager_cmd)
    pager_cmd += ' ' + get_pager_start(pager_cmd,start)

    try:
        if os.environ['TERM'] in ['emacs','dumb']:
            raise EnvironmentError
        system(pager_cmd + ' ' + fname)
    except:
        try:
            if start > 0:
                start -= 1
            page(open(fname).read(),start)
        except:
            print('Unable to show file',repr(fname))

def get_pager_cmd(pager_cmd=None):
    """Return a pager command.

    Makes some attempts at finding an OS-correct one.
    """
    if os.name == 'posix':
        default_pager_cmd = 'less -R'  # -R for color control sequences
    elif os.name in ['nt','dos']:
        default_pager_cmd = 'type'

    if pager_cmd is None:
        try:
            pager_cmd = os.environ['PAGER']
        except:
            pager_cmd = default_pager_cmd

    if pager_cmd == 'less' and '-r' not in os.environ.get('LESS', '').lower():
        pager_cmd += ' -R'

    return pager_cmd

def get_pager_start(pager, start):
    """Return the string for paging files with an offset.

    This is the '+N' argument which less and more (under Unix) accept.
    """

    if pager in ['less','more']:
        if start:
            start_string = '+' + str(start)
        else:
            start_string = ''
    else:
        start_string = ''
    return start_string

# (X)emacs on win32 doesn't like to be bypassed with msvcrt.getch()
if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs':
    import msvcrt
    def page_more():
        """ Smart pausing between pages

        @return:    True if need print more lines, False if quit
        """
        sys.stdout.write('---Return to continue, q to quit--- ')
        ans = msvcrt.getwch()
        if ans in ("q", "Q"):
            result = False
        else:
            result = True
        sys.stdout.write("\b"*37 + " "*37 + "\b"*37)
        return result
else:
    def page_more():
        ans = py3compat.input('---Return to continue, q to quit--- ')
        if ans.lower().startswith('q'):
            return False
        else:
            return True

def snip_print(str,width = 75,print_full = 0,header = ''):
    """Print a string snipping the midsection to fit in width.

    print_full: mode control:

      - 0: only snip long strings
      - 1: send to page() directly.
      - 2: snip long strings and ask for full length viewing with page()

    Return 1 if snipping was necessary, 0 otherwise."""

    if print_full == 1:
        page(header+str)
        return 0

    print(header, end=' ')
    if len(str) < width:
        print(str)
        snip = 0
    else:
        whalf = int((width -5)/2)
        print(str[:whalf] + ' <...> ' + str[-whalf:])
        snip = 1
    if snip and print_full == 2:
        if py3compat.input(header+' Snipped. View (y/n)? [N]').lower() == 'y':
            page(str)
    return snip
<|endoftext|>